diff --git a/.vsts-dotnet-ci.yml b/.vsts-dotnet-ci.yml index b3668f511b..3c727fc99a 100644 --- a/.vsts-dotnet-ci.yml +++ b/.vsts-dotnet-ci.yml @@ -10,11 +10,12 @@ resources: - container: UbuntuContainer image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-mlnet-207e097-20190312152303 -phases: -- template: /build/ci/phase-template.yml +jobs: +- template: /build/ci/job-template.yml parameters: name: Centos_x64_NetCoreApp30 buildScript: ./build.sh + container: CentosContainer customMatrixes: Debug_Build: _configuration: Debug-Intrinsics @@ -24,26 +25,25 @@ phases: _configuration: Release-Intrinsics _config_short: RI _includeBenchmarkData: true - queue: + pool: name: Hosted Ubuntu 1604 - container: CentosContainer -- template: /build/ci/phase-template.yml +- template: /build/ci/job-template.yml parameters: name: Ubuntu_x64_NetCoreApp21 buildScript: ./build.sh - queue: + container: UbuntuContainer + pool: name: Hosted Ubuntu 1604 - container: UbuntuContainer -- template: /build/ci/phase-template.yml +- template: /build/ci/job-template.yml parameters: name: MacOS_x64_NetCoreApp21 buildScript: ./build.sh - queue: + pool: name: Hosted macOS -- template: /build/ci/phase-template.yml +- template: /build/ci/job-template.yml parameters: name: Windows_x64_NetCoreApp30 buildScript: build.cmd @@ -56,17 +56,18 @@ phases: _configuration: Release-Intrinsics _config_short: RI _includeBenchmarkData: true - queue: - name: Hosted VS2017 + pool: + name: NetCorePublic-Pool + queue: buildpool.windows.10.amd64.vs2017.open -- template: /build/ci/phase-template.yml +- template: /build/ci/job-template.yml parameters: name: Windows_x64_NetCoreApp21 buildScript: build.cmd - queue: + pool: name: Hosted VS2017 -- template: /build/ci/phase-template.yml +- template: /build/ci/job-template.yml parameters: name: Windows_x64_NetFx461 buildScript: build.cmd @@ -79,13 +80,13 @@ phases: _configuration: Release-netfx _config_short: RFX _includeBenchmarkData: false - queue: + pool: name: Hosted VS2017 -- template: /build/ci/phase-template.yml +- template: /build/ci/job-template.yml parameters: name: Windows_x86_NetCoreApp21 architecture: x86 buildScript: build.cmd - queue: + pool: name: Hosted VS2017 diff --git a/Directory.Build.targets b/Directory.Build.targets index 5e6446add9..58448682ae 100644 --- a/Directory.Build.targets +++ b/Directory.Build.targets @@ -21,6 +21,14 @@ + + + + $(NativeOutputPath)$(LibPrefix)%(NativeAssemblyReferenceWithMajorVersion.Identity)$(LibExtension).%(NativeAssemblyReferenceWithMajorVersion.MajorVersion) + $(NativeOutputPath)$(LibPrefix)%(NativeAssemblyReferenceWithMajorVersion.Identity).%(NativeAssemblyReferenceWithMajorVersion.MajorVersion)$(LibExtension) + + + - + + + + + + diff --git a/build/BranchInfo.props b/build/BranchInfo.props index 016f3c30ac..017da9e572 100644 --- a/build/BranchInfo.props +++ b/build/BranchInfo.props @@ -15,6 +15,9 @@ Microsoft.ML.LightGbm; Microsoft.ML.Mkl.Components; Microsoft.ML.Mkl.Redist; + Microsoft.ML.TimeSeries; + Microsoft.ML.TensorFlow; + Microsoft.ML.OnnxTransformer; $(StableProjects.Contains($(MSBuildProjectName))) $(StableProjects.Contains($(MSBuildProjectName.Substring(0, $(MSBuildProjectName.IndexOf(.symbols)))))) @@ -22,13 +25,13 @@ 1 - 2 + 3 0 preview 0 - 14 + 15 0 preview diff --git a/build/Dependencies.props b/build/Dependencies.props index 2ed9835fb4..ea5a6a8736 100644 --- a/build/Dependencies.props +++ b/build/Dependencies.props @@ -15,13 +15,14 @@ 3.5.1 2.2.3 2.1.0 - 0.3.0 + 0.4.0 0.0.0.9 2.1.3 4.5.0 4.5.0 4.5.0 - 1.13.1 + 1.14.0 + 1 @@ -44,9 +45,9 @@ 0.11.3 1.0.0-beta1-63812-02 - 0.0.4-test + 0.0.5-test 0.0.11-test - 0.0.4-test + 0.0.5-test diff --git a/build/ci/phase-template.yml b/build/ci/job-template.yml similarity index 69% rename from build/ci/phase-template.yml rename to build/ci/job-template.yml index 6d70c14bda..82415d87ba 100644 --- a/build/ci/phase-template.yml +++ b/build/ci/job-template.yml @@ -2,23 +2,18 @@ parameters: name: '' architecture: x64 buildScript: '' - queue: {} + pool: {} customMatrixes: '' codeCoverage: false + container: '' -phases: - - phase: ${{ parameters.name }} - variables: - _buildScript: ${{ parameters.buildScript }} - _phaseName: ${{ parameters.name }} - _arch: ${{ parameters.architecture }} - _codeCoverage: ${{ parameters.codeCoverage }} - queue: - ${{ if eq(variables._codeCoverage, 'false') }}: - timeoutInMinutes: 30 - ${{ if eq(variables._codeCoverage, 'true') }}: - timeoutInMinutes: 60 - parallel: 99 +jobs: + - job: ${{ parameters.name }} + ${{ if eq(parameters.codeCoverage, 'false') }}: + timeoutInMinutes: 40 + ${{ if eq(parameters.codeCoverage, 'true') }}: + timeoutInMinutes: 60 + strategy: matrix: ${{ if eq(parameters.customMatrixes, '') }}: Debug_Build: @@ -31,20 +26,24 @@ phases: _includeBenchmarkData: true ${{ if ne(parameters.customMatrixes, '') }}: ${{ insert }}: ${{ parameters.customMatrixes }} - ${{ insert }}: ${{ parameters.queue }} + + pool: ${{ parameters.pool }} + ${{ if ne(parameters.container, '') }}: + container: ${{ parameters.container }} + steps: - - ${{ if eq(parameters.queue.name, 'Hosted macOS') }}: + - ${{ if eq(parameters.pool.name, 'Hosted macOS') }}: - script: brew update && brew install https://raw.githubusercontent.com/Homebrew/homebrew-core/f5b1ac99a7fba27c19cee0bc4f036775c889b359/Formula/libomp.rb && brew install mono-libgdiplus gettext && brew link gettext --force && brew link libomp --force displayName: Install build dependencies - - script: $(_buildScript) -$(_configuration) -buildArch=$(_arch) + - script: ${{ parameters.buildScript }} -$(_configuration) -buildArch=${{ parameters.architecture }} displayName: Build - - script: $(_buildScript) -- /t:DownloadExternalTestFiles /p:IncludeBenchmarkData=$(_includeBenchmarkData) + - script: ${{ parameters.buildScript }} -- /t:DownloadExternalTestFiles /p:IncludeBenchmarkData=$(_includeBenchmarkData) displayName: Download Benchmark Data - - script: $(_buildScript) -$(_configuration) -runtests -coverage=$(_codeCoverage) + - script: ${{ parameters.buildScript }} -$(_configuration) -runtests -coverage=${{ parameters.codeCoverage }} displayName: Run Tests. - script: $(Build.SourcesDirectory)/Tools/dotnetcli/dotnet msbuild build/Codecoverage.proj /p:CodeCovToken=$(CODECOV_TOKEN) displayName: Upload coverage to codecov.io - condition: and(succeeded(), eq(variables._codeCoverage, 'true')) + condition: and(succeeded(), eq(${{ parameters.codeCoverage }}, True)) - task: PublishTestResults@2 displayName: Publish Test Results condition: succeededOrFailed() @@ -52,7 +51,7 @@ phases: testRunner: 'vSTest' searchFolder: '$(System.DefaultWorkingDirectory)/bin' testResultsFiles: '**/*.trx' - testRunTitle: Machinelearning_Tests_$(_phaseName)_$(_configuration)_$(Build.BuildNumber) + testRunTitle: Machinelearning_Tests_${{ parameters.name }}_$(_configuration)_$(Build.BuildNumber) configuration: $(_configuration) mergeTestResults: true - task: CopyFiles@2 @@ -78,5 +77,5 @@ phases: pathToPublish: $(Build.ArtifactStagingDirectory) artifactName: ${{ parameters.name }} $(_config_short) artifactType: container - - script: $(_buildScript) -buildPackages + - script: ${{ parameters.buildScript }} -buildPackages displayName: Build Packages diff --git a/build/codecoverage-ci.yml b/build/codecoverage-ci.yml index 7dc3910255..3f6005f53e 100644 --- a/build/codecoverage-ci.yml +++ b/build/codecoverage-ci.yml @@ -3,7 +3,7 @@ ################################################################################ phases: -- template: /build/ci/phase-template.yml +- template: /build/ci/job-template.yml parameters: name: Windows_x64 buildScript: build.cmd diff --git a/docs/api-reference/io-columns-tree-featurization-binary-classification.md b/docs/api-reference/io-columns-tree-featurization-binary-classification.md new file mode 100644 index 0000000000..1fd9a68a89 --- /dev/null +++ b/docs/api-reference/io-columns-tree-featurization-binary-classification.md @@ -0,0 +1,14 @@ +### Input and Output Columns +The input label column data must be . +The input features column data must be a known-sized vector of . + +This estimator outputs the following columns: + +| Output Column Name | Column Type | Description| +| -- | -- | -- | +| `Trees` | Known-sized vector of | The output values of all trees. Its size is identical to the total number of trees in the tree ensemble model. | +| `Leaves` | Known-sized vector of | 0-1 vector representation to the IDs of all leaves where the input feature vector falls into. Its size is the number of total leaves in the tree ensemble model. | +| `Paths` | Known-sized vector of | 0-1 vector representation to the paths the input feature vector passed through to reach the leaves. Its size is the number of non-leaf nodes in the tree ensemble model. | + +Those output columns are all optional and user can change their names. +Please set the names of skipped columns to null so that they would not be produced. \ No newline at end of file diff --git a/docs/api-reference/io-columns-tree-featurization-ranking.md b/docs/api-reference/io-columns-tree-featurization-ranking.md new file mode 100644 index 0000000000..375ad18f9c --- /dev/null +++ b/docs/api-reference/io-columns-tree-featurization-ranking.md @@ -0,0 +1,20 @@ +### Input and Output Columns +The input label data type must be [key](xref:Microsoft.ML.Data.KeyDataViewType) +type or . The value of the label determines relevance, where +higher values indicate higher relevance. If the label is a +[key](xref:Microsoft.ML.Data.KeyDataViewType) type, then the key index is the +relevance value, where the smallest index is the least relevant. If the label is a +, larger values indicate higher relevance. The feature +column must be a known-sized vector of and input row group +column must be [key](xref:Microsoft.ML.Data.KeyDataViewType) type. + +This estimator outputs the following columns: + +| Output Column Name | Column Type | Description| +| -- | -- | -- | +| `Trees` | Known-sized vector of | The output values of all trees. Its size is identical to the total number of trees in the tree ensemble model. | +| `Leaves` | Known-sized vector of | 0-1 vector representation to the IDs of all leaves where the input feature vector falls into. Its size is the number of total leaves in the tree ensemble model. | +| `Paths` | Known-sized vector of | 0-1 vector representation to the paths the input feature vector passed through to reach the leaves. Its size is the number of non-leaf nodes in the tree ensemble model. | + +Those output columns are all optional and user can change their names. +Please set the names of skipped columns to null so that they would not be produced. \ No newline at end of file diff --git a/docs/api-reference/io-columns-tree-featurization-regression.md b/docs/api-reference/io-columns-tree-featurization-regression.md new file mode 100644 index 0000000000..d4acf06f39 --- /dev/null +++ b/docs/api-reference/io-columns-tree-featurization-regression.md @@ -0,0 +1,14 @@ +### Input and Output Columns +The input label column data must be . +The input features column data must be a known-sized vector of . + +This estimator outputs the following columns: + +| Output Column Name | Column Type | Description| +| -- | -- | -- | +| `Trees` | Known-sized vector of | The output values of all trees. Its size is identical to the total number of trees in the tree ensemble model. | +| `Leaves` | Known-sized vector of | 0-1 vector representation to the IDs of all leaves where the input feature vector falls into. Its size is the number of total leaves in the tree ensemble model. | +| `Paths` | Known-sized vector of | 0-1 vector representation to the paths the input feature vector passed through to reach the leaves. Its size is the number of non-leaf nodes in the tree ensemble model. | + +Those output columns are all optional and user can change their names. +Please set the names of skipped columns to null so that they would not be produced. \ No newline at end of file diff --git a/docs/api-reference/io-time-series-ssa-forecast.md b/docs/api-reference/io-time-series-ssa-forecast.md new file mode 100644 index 0000000000..ae510add41 --- /dev/null +++ b/docs/api-reference/io-time-series-ssa-forecast.md @@ -0,0 +1,5 @@ +### Input and Output Columns +There is only one input column. +The input column must be where a value indicates a value at a timestamp in the time series. + +It produces either just one vector of forecasted values or three vectors: a vector of forecasted values, a vector of confidence lower bounds and a vector of confidence upper bounds. diff --git a/docs/api-reference/regularization-l1-l2.md b/docs/api-reference/regularization-l1-l2.md index 758d060a32..3f6bda200c 100644 --- a/docs/api-reference/regularization-l1-l2.md +++ b/docs/api-reference/regularization-l1-l2.md @@ -1,6 +1,6 @@ -This class uses [empricial risk minimization](https://en.wikipedia.org/wiki/Empirical_risk_minimization) (i.e., ERM) +This class uses [empirical risk minimization](https://en.wikipedia.org/wiki/Empirical_risk_minimization) (i.e., ERM) to formulate the optimization problem built upon collected data. -Note that empricial risk is usually measured by applying a loss function on the model's predictions on collected data points. +Note that empirical risk is usually measured by applying a loss function on the model's predictions on collected data points. If the training data does not contain enough data points (for example, to train a linear model in $n$-dimensional space, we need at least $n$ data points), [overfitting](https://en.wikipedia.org/wiki/Overfitting) may happen so that diff --git a/docs/api-reference/tree-featurization-prediction.md b/docs/api-reference/tree-featurization-prediction.md new file mode 100644 index 0000000000..6516dfef8c --- /dev/null +++ b/docs/api-reference/tree-featurization-prediction.md @@ -0,0 +1,25 @@ +### Prediction Details +This estimator produces several output columns from a tree ensemble model. Assume that the model contains only one decision tree: + + Node 0 + / \ + / \ + / \ + / \ + Node 1 Node 2 + / \ / \ + / \ / \ + / \ Leaf -3 Node 3 + Leaf -1 Leaf -2 / \ + / \ + Leaf -4 Leaf -5 + +Assume that the input feature vector falls into `Leaf -1`. The output `Trees` may be a 1-element vector where +the only value is the decision value carried by `Leaf -1`. The output `Leaves` is a 0-1 vector. If the reached +leaf is the $i$-th (indexed by $-(i+1)$ so the first leaf is `Leaf -1`) leaf in the tree, the $i$-th value in `Leaves` +would be 1 and all other values would be 0. The output `Paths` is a 0-1 representation of the nodes passed +through before reaching the leaf. The $i$-th element in `Paths` indicates if the $i$-th node (indexed by $i$) is touched. +For example, reaching `Leaf -1` lead to $[1, 1, 0, 0]$ as the `Paths`. If there are multiple trees, this estimator +just concatenates `Trees`'s, `Leaves`'s, `Paths`'s from all trees (first tree's information comes first in the concatenated vectors). + +Check the See Also section for links to usage examples. \ No newline at end of file diff --git a/docs/release-notes/1.2.0/release-1.2.0.md b/docs/release-notes/1.2.0/release-1.2.0.md new file mode 100644 index 0000000000..c6b703e434 --- /dev/null +++ b/docs/release-notes/1.2.0/release-1.2.0.md @@ -0,0 +1,84 @@ +# [ML.NET](http://dot.net/ml) 1.2.0 +## **General Availability** +- **Microsoft.ML.TimeSeries** + - Anomaly detection algorithms (Spike and Change Point): + - Independent and identically distributed. + - Singular spectrum analysis. + - Spectral residual from Azure Anomaly Detector/Kensho team. + - Forecasting models: + - Singular spectrum analysis. + - Prediction Engine for online learning + - Enables updating time series model with new observations at scoring so that the user does not have to re-train the time series with old data each time. + + [Samples](https://github.com/dotnet/machinelearning/tree/master/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries) + +- **Microsoft.ML.OnnxTransformer** + Enables scoring of ONNX models in the learning pipeline. Uses ONNX Runtime v0.4. + + [Sample](https://github.com/dotnet/machinelearning/blob/master/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/ApplyOnnxModel.cs) + +- **Microsoft.ML.TensorFlow** + Enables scoring of TensorFlow models in the learning pipeline. Uses TensorFlow v1.13. Very useful for image and text classification. Users can featurize images or text using DNN models and feed the result into a classical machine learning model like a decision tree or logistic regression trainer. + + [Samples](https://github.com/dotnet/machinelearning/tree/master/docs/samples/Microsoft.ML.Samples/Dynamic/TensorFlow) + +## **New Features** +- **Tree-based featurization** ([#3812](https://github.com/dotnet/machinelearning/pull/3812)) + + Generating features using tree structure has been a popular technique in data mining. Useful for capturing feature interactions when creating a stacked model, dimensionality reduction, or featurizing towards an alternative label. [ML.NET](dot.net/ml)'s tree featurization trains a tree-based model and then maps input feature vector to several non-linear feature vectors. Those generated feature vectors are: + - The leaves it falls into. It's a binary vector with ones happens at the indexes of reached leaves, + - The paths that the input vector passes before hitting the leaves, and + - The reached leaves values. + + Here are two references. + - [p. 9](https://www.csie.ntu.edu.tw/~r01922136/kaggle-2014-criteo.pdf) (a Kaggle solution adopted by FB below). + - [Section 3](http://www.quinonero.net/Publications/predicting-clicks-facebook.pdf). (Facebook) + - [Section of Entity-level personalization with GLMix](https://engineering.linkedin.com/blog/2019/04/ai-behind-linkedin-recruiter-search-and-recommendation-systems). (LinkedIn) + + [Samples](https://github.com/dotnet/machinelearning/tree/master/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TreeFeaturization) + +- **Microsoft.Extensions.ML integration package.** ([#3827](https://github.com/dotnet/machinelearning/pull/3827)) + + This package makes it easier to use [ML.NET](dot.net/ml) with app models that support Microsoft.Extensions - i.e. ASP.NET and Azure Functions. + + Specifically it contains functionality for: + - Dependency Injection + - Pooling PredictionEngines + - Reloading models when the file or URI has changed + - Hooking ML.NET logging to Microsoft.Extensions.Logging + +## **Bug Fixes** +### Serious +- **Time series Sequential Transform needs to have a binding mechanism:** This bug made it impossible to use time series in NimbusML. ([#3875](https://github.com/dotnet/machinelearning/pull/3875)) + +- **Build errors resulting from upgrading to VS2019 compilers:** The default CMAKE_C_FLAG for debug configuration sets /ZI to generate a PDB capable of edit and continue. In the new compilers, this is incompatible with /guard:cf which we set for security reasons. ([#3894](https://github.com/dotnet/machinelearning/pull/3894)) + +- **LightGBM Evaluation metric parameters:** In LightGbm EvaluateMetricType where if a user specified EvaluateMetricType.Default, the metric would not get added to the options Dictionary, and LightGbmWrappedTraining would throw because of that. ([#3815](https://github.com/dotnet/machinelearning/pull/3815)) + +- **Change default EvaluationMetric for LightGbm:** In [ML.NET](dot.net/ml), the default EvaluationMetric for LightGbm is set to EvaluateMetricType.Error for multiclass, EvaluationMetricType.LogLoss for binary etc. This leads to inconsistent behavior from the user's perspective. ([#3859](https://github.com/dotnet/machinelearning/pull/3859)) +### Other +- CustomGains should allow multiple values in argument attribute. ([#3854](https://github.com/dotnet/machinelearning/pull/3854)) + +## **Breaking Changes** +None + +## **Enhancements** +- Fixes the Hardcoded Sigmoid value from -0.5 to the value specified during training. ([#3850](https://github.com/dotnet/machinelearning/pull/3850)) +- Fix TextLoader constructor and add exception message. ([#3788](https://github.com/dotnet/machinelearning/pull/3788)) +- Introduce the `FixZero` argument to the LogMeanVariance normalizer. ([#3916](https://github.com/dotnet/machinelearning/pull/3916)) +- Ensembles trainer now work with ITrainerEstimators instead of ITrainers. ([#3796](https://github.com/dotnet/machinelearning/pull/3796)) +- LightGBM Unbalanced Data Argument. ([#3925](https://github.com/dotnet/machinelearning/pull/3925)) +- Tree based trainers implement ICanGetSummaryAsIDataView. ([#3892](https://github.com/dotnet/machinelearning/pull/3892)) + +- **CLI and AutoML API** + - Internationalization fixes to generate proper [ML.NET](dot.net/ml) C# code. ([#3725](https://github.com/dotnet/machinelearning/pull/3725)) + - Automatic Cross Validation for small datasets, and CV stability fixes. ([#3794](https://github.com/dotnet/machinelearning/pull/3794)) + - Code cleanup to match .NET style. ([#3823](https://github.com/dotnet/machinelearning/pull/3823)) + + +## **Documentation and Samples** +- Samples for applying ONNX model to in-memory images. ([#3851](https://github.com/dotnet/machinelearning/pull/3851)) +- Reformatted all ~200 samples to 85 character width so the horizontal scrollbar does not appear on docs webpage. ([#3930](https://github.com/dotnet/machinelearning/pull/3930), [3941](https://github.com/dotnet/machinelearning/pull/3941), [3949](https://github.com/dotnet/machinelearning/pull/3949), [3950](https://github.com/dotnet/machinelearning/pull/3950), [3947](https://github.com/dotnet/machinelearning/pull/3947), [3943](https://github.com/dotnet/machinelearning/pull/3943), [3942](https://github.com/dotnet/machinelearning/pull/3942), [3946](https://github.com/dotnet/machinelearning/pull/3946), [3948](https://github.com/dotnet/machinelearning/pull/3948)) + +## **Remarks** +- Roughly 200 Github issues were closed, the count decreased from **~550 to 351**. Most of the issues got resolved due to the release of stable API and availability of samples. \ No newline at end of file diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/BootstrapSample.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/BootstrapSample.cs index 33413352de..be93d56fae 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/BootstrapSample.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/BootstrapSample.cs @@ -7,8 +7,9 @@ public static class BootstrapSample { public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. var mlContext = new MLContext(); // Get a small dataset as an IEnumerable. @@ -23,20 +24,27 @@ public static void Example() var data = mlContext.Data.LoadFromEnumerable(rawData); - // Now take a bootstrap sample of this dataset to create a new dataset. The bootstrap is a resampling technique that - // creates a training set of the same size by picking with replacement from the original dataset. With the bootstrap, - // we expect that the resampled dataset will have about 63% of the rows of the original dataset (i.e. 1-e^-1), with some - // rows represented more than once. - // BootstrapSample is a streaming implementation of the boostrap that enables sampling from a dataset too large to hold in memory. - // To enable streaming, BootstrapSample approximates the bootstrap by sampling each row according to a Poisson(1) distribution. - // Note that this streaming approximation treats each row independently, thus the resampled dataset is not guaranteed to be the - // same length as the input dataset. - // Let's take a look at the behavior of the BootstrapSample by examining a few draws: + // Now take a bootstrap sample of this dataset to create a new dataset. + // The bootstrap is a resampling technique that creates a training set + // of the same size by picking with replacement from the original + // dataset. With the bootstrap, we expect that the resampled dataset + // will have about 63% of the rows of the original dataset + // (i.e. 1-e^-1), with some rows represented more than once. + // BootstrapSample is a streaming implementation of the boostrap that + // enables sampling from a dataset too large to hold in memory. To + // enable streaming, BootstrapSample approximates the bootstrap by + // sampling each row according to a Poisson(1) distribution. Note that + // this streaming approximation treats each row independently, thus the + // resampled dataset is not guaranteed to be the same length as the + // input dataset. Let's take a look at the behavior of the + // BootstrapSample by examining a few draws: for (int i = 0; i < 3; i++) { var resample = mlContext.Data.BootstrapSample(data, seed: i); - var enumerable = mlContext.Data.CreateEnumerable(resample, reuseRowObject: false); + var enumerable = mlContext.Data + .CreateEnumerable(resample, reuseRowObject: false); + Console.WriteLine($"Label\tFeature"); foreach (var row in enumerable) { diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/Cache.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/Cache.cs index 4ea7c33723..09855b2d07 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/Cache.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/Cache.cs @@ -8,56 +8,78 @@ public static class Cache { public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. + // Create a new context for ML.NET operations. It can be used for except + // ion tracking and logging, as a catalog of available operations and as + // the source of randomness. var mlContext = new MLContext(); var data = DatasetUtils.LoadHousingRegressionDataset(mlContext); // Time how long it takes to page through the records if we don't cache. - (int lines, double columnAverage, double elapsedSeconds) = TimeToScanIDataView(mlContext, data); - Console.WriteLine($"Lines={lines}, averageOfColumn0={columnAverage:0.00} and took {elapsedSeconds} seconds."); + (int lines, double columnAverage, double elapsedSeconds) = + TimeToScanIDataView(mlContext, data); + + Console.WriteLine($"Lines={lines}," + + $"averageOfColumn0={columnAverage:0.00} and took {elapsedSeconds}" + + $"seconds."); // Expected output (time is approximate): // Lines=506, averageOfColumn0=564.17 and took 0.314 seconds. // Now create a cached view of the data. var cachedData = mlContext.Data.Cache(data); - // Time how long it takes to page through the records the first time they're accessed after a cache is applied. - // This iteration will be longer than subsequent calls, as the dataset is being accessed and stored for later. - // Note that this operation may be relatively quick, as the system may have cached the file. - (lines, columnAverage, elapsedSeconds) = TimeToScanIDataView(mlContext, cachedData); - Console.WriteLine($"Lines={lines}, averageOfColumn0={columnAverage:0.00} and took {elapsedSeconds} seconds."); + // Time how long it takes to page through the records the first time + // they're accessed after a cache is applied. This iteration will be + // longer than subsequent calls, as the dataset is being accessed and + // stored for later. Note that this operation may be relatively quick, + // as the system may have cached the file. + (lines, columnAverage, elapsedSeconds) = TimeToScanIDataView(mlContext, + cachedData); + + Console.WriteLine($"Lines={lines}," + + $"averageOfColumn0={columnAverage:0.00} and took {elapsedSeconds}" + + $"seconds."); // Expected output (time is approximate): // Lines=506, averageOfColumn0=564.17 and took 0.056 seconds. - // Time how long it takes to page through the records now that the data is cached. After the first iteration that caches the IDataView, - // future iterations, like this one, are faster because they are pulling from data cached in memory. - (lines, columnAverage, elapsedSeconds) = TimeToScanIDataView(mlContext, cachedData); - Console.WriteLine($"Lines={lines}, averageOfColumn0={columnAverage:0.00} and took {elapsedSeconds} seconds."); + // Time how long it takes to page through the records now that the data + // is cached. After the first iteration that caches the IDataView, + // future iterations, like this one, are faster because they are pulling + // from data cached in memory. + (lines, columnAverage, elapsedSeconds) = TimeToScanIDataView(mlContext, + cachedData); + + Console.WriteLine( + $"Lines={lines}, averageOfColumn0={columnAverage:0.00} and took " + + $"{elapsedSeconds} seconds."); // Expected output (time is approximate): // Lines=506, averageOfColumn0=564.17 and took 0.006 seconds. } - private static (int lines, double columnAverage, double elapsedSeconds) TimeToScanIDataView(MLContext mlContext, IDataView data) + private static (int lines, double columnAverage, double elapsedSeconds) + TimeToScanIDataView(MLContext mlContext, IDataView data) { int lines = 0; double columnAverage = 0.0; - var enumerable = mlContext.Data.CreateEnumerable(data, reuseRowObject: true); + var enumerable = mlContext.Data + .CreateEnumerable(data, reuseRowObject: true); + var watch = System.Diagnostics.Stopwatch.StartNew(); foreach (var row in enumerable) { lines++; - columnAverage += row.MedianHomeValue + row.CrimesPerCapita + row.PercentResidental + row.PercentNonRetail + row.CharlesRiver - + row.NitricOxides + row.RoomsPerDwelling + row.PercentPre40s + row.EmploymentDistance - + row.HighwayDistance + row.TaxRate + row.TeacherRatio; + columnAverage += row.MedianHomeValue + row.CrimesPerCapita + + row.PercentResidental + row.PercentNonRetail + row.CharlesRiver + + row.NitricOxides + row.RoomsPerDwelling + row.PercentPre40s + + row.EmploymentDistance + row.HighwayDistance + row.TaxRate + + row.TeacherRatio; } watch.Stop(); columnAverage /= lines; var elapsed = watch.Elapsed; return (lines, columnAverage, elapsed.Seconds); - } + } /// /// A class to hold the raw housing regression rows. diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/CrossValidationSplit.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/CrossValidationSplit.cs index d7f3faad94..cbe68ba117 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/CrossValidationSplit.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/CrossValidationSplit.cs @@ -17,16 +17,28 @@ public static void Example() // Generate some data points. var examples = GenerateRandomDataPoints(10); - // Convert the examples list to an IDataView object, which is consumable by ML.NET API. + // Convert the examples list to an IDataView object, which is consumable + // by ML.NET API. var dataview = mlContext.Data.LoadFromEnumerable(examples); - // Cross validation splits your data randomly into set of "folds", and creates groups of Train and Test sets, - // where for each group, one fold is the Test and the rest of the folds the Train. - // So below, we specify Group column as the column containing the sampling keys. - // If we pass that column to cross validation it would be used to break data into certain chunks. - var folds = mlContext.Data.CrossValidationSplit(dataview, numberOfFolds: 3, samplingKeyColumnName: "Group"); - var trainSet = mlContext.Data.CreateEnumerable(folds[0].TrainSet, reuseRowObject: false); - var testSet = mlContext.Data.CreateEnumerable(folds[0].TestSet, reuseRowObject: false); + // Cross validation splits your data randomly into set of "folds", and + // creates groups of Train and Test sets, where for each group, one fold + // is the Test and the rest of the folds the Train. So below, we specify + // Group column as the column containing the sampling keys. If we pass + // that column to cross validation it would be used to break data into + // certain chunks. + var folds = mlContext.Data + .CrossValidationSplit(dataview, numberOfFolds:3, + samplingKeyColumnName: "Group"); + + var trainSet = mlContext.Data + .CreateEnumerable(folds[0].TrainSet, + reuseRowObject: false); + + var testSet = mlContext.Data + .CreateEnumerable(folds[0].TestSet, + reuseRowObject: false); + PrintPreviewRows(trainSet, testSet); // The data in the Train split. @@ -43,8 +55,14 @@ public static void Example() // [Group, 0], [Features, 0.9060271] // [Group, 0], [Features, 0.2737045] - trainSet = mlContext.Data.CreateEnumerable(folds[1].TrainSet, reuseRowObject: false); - testSet = mlContext.Data.CreateEnumerable(folds[1].TestSet, reuseRowObject: false); + trainSet = mlContext.Data + .CreateEnumerable(folds[1].TrainSet, + reuseRowObject: false); + + testSet = mlContext.Data + .CreateEnumerable(folds[1].TestSet, + reuseRowObject: false); + PrintPreviewRows(trainSet, testSet); // The data in the Train split. // [Group, 0], [Features, 0.7262433] @@ -60,8 +78,14 @@ public static void Example() // [Group, 1], [Features, 0.2060332] // [Group, 1], [Features, 0.4421779] - trainSet = mlContext.Data.CreateEnumerable(folds[2].TrainSet, reuseRowObject: false); - testSet = mlContext.Data.CreateEnumerable(folds[2].TestSet, reuseRowObject: false); + trainSet = mlContext.Data + .CreateEnumerable(folds[2].TrainSet, + reuseRowObject: false); + + testSet = mlContext.Data + .CreateEnumerable(folds[2].TestSet, + reuseRowObject: false); + PrintPreviewRows(trainSet, testSet); // The data in the Train split. // [Group, 0], [Features, 0.7262433] @@ -79,8 +103,14 @@ public static void Example() // Example of a split without specifying a sampling key column. folds = mlContext.Data.CrossValidationSplit(dataview, numberOfFolds: 3); - trainSet = mlContext.Data.CreateEnumerable(folds[0].TrainSet, reuseRowObject: false); - testSet = mlContext.Data.CreateEnumerable(folds[0].TestSet, reuseRowObject: false); + trainSet = mlContext.Data + .CreateEnumerable(folds[0].TrainSet, + reuseRowObject: false); + + testSet = mlContext.Data + .CreateEnumerable(folds[0].TestSet, + reuseRowObject: false); + PrintPreviewRows(trainSet, testSet); // The data in the Train split. // [Group, 0], [Features, 0.7262433] @@ -96,8 +126,14 @@ public static void Example() // [Group, 2], [Features, 0.5588848] // [Group, 0], [Features, 0.9060271] - trainSet = mlContext.Data.CreateEnumerable(folds[1].TrainSet, reuseRowObject: false); - testSet = mlContext.Data.CreateEnumerable(folds[1].TestSet, reuseRowObject: false); + trainSet = mlContext.Data + .CreateEnumerable(folds[1].TrainSet, + reuseRowObject: false); + + testSet = mlContext.Data + .CreateEnumerable(folds[1].TestSet, + reuseRowObject: false); + PrintPreviewRows(trainSet, testSet); // The data in the Train split. // [Group, 2], [Features, 0.7680227] @@ -113,8 +149,13 @@ public static void Example() // [Group, 2], [Features, 0.9775497] // [Group, 0], [Features, 0.2737045] - trainSet = mlContext.Data.CreateEnumerable(folds[2].TrainSet, reuseRowObject: false); - testSet = mlContext.Data.CreateEnumerable(folds[2].TestSet, reuseRowObject: false); + trainSet = mlContext.Data + .CreateEnumerable(folds[2].TrainSet, + reuseRowObject: false); + + testSet = mlContext.Data.CreateEnumerable(folds[2].TestSet, + reuseRowObject: false); + PrintPreviewRows(trainSet, testSet); // The data in the Train split. // [Group, 0], [Features, 0.7262433] @@ -131,7 +172,9 @@ public static void Example() // [Group, 1], [Features, 0.4421779] } - private static IEnumerable GenerateRandomDataPoints(int count, int seed = 0) + private static IEnumerable GenerateRandomDataPoints(int count, + int seed = 0) + { var random = new Random(seed); for (int i = 0; i < count; i++) @@ -146,7 +189,8 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se } } - // Example with features and group column. A data set is a collection of such examples. + // Example with features and group column. A data set is a collection of + // such examples. private class DataPoint { public float Group { get; set; } @@ -155,7 +199,9 @@ private class DataPoint } // print helper - private static void PrintPreviewRows(IEnumerable trainSet, IEnumerable testSet) + private static void PrintPreviewRows(IEnumerable trainSet, + IEnumerable testSet) + { Console.WriteLine($"The data in the Train split."); diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/DataViewEnumerable.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/DataViewEnumerable.cs index 667bd39758..aaefc85597 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/DataViewEnumerable.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/DataViewEnumerable.cs @@ -6,31 +6,41 @@ namespace Samples.Dynamic { public static class DataViewEnumerable { - // A simple case of creating IDataView from IEnumerable. + // A simple case of creating IDataView from + //IEnumerable. public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, // as a catalog of available operations and as the source of randomness. var mlContext = new MLContext(); // Get a small dataset as an IEnumerable. - IEnumerable enumerableOfData = GetSampleTemperatureData(5); + IEnumerable enumerableOfData = + GetSampleTemperatureData(5); // Load dataset into an IDataView. IDataView data = mlContext.Data.LoadFromEnumerable(enumerableOfData); - // We can now examine the records in the IDataView. We first create an enumerable of rows in the IDataView. - var rowEnumerable = mlContext.Data.CreateEnumerable(data, reuseRowObject: true); + // We can now examine the records in the IDataView. We first create an + // enumerable of rows in the IDataView. + var rowEnumerable = mlContext.Data + .CreateEnumerable(data, + reuseRowObject: true); - // SampleTemperatureDataWithLatitude has the definition of a Latitude column of type float. - // We can use the parameter ignoreMissingColumns to true to ignore any missing columns in the IDataView. - // The produced enumerable will have the Latitude field set to the default for the data type, in this case 0. - var rowEnumerableIgnoreMissing = mlContext.Data.CreateEnumerable(data, - reuseRowObject: true, ignoreMissingColumns: true); + // SampleTemperatureDataWithLatitude has the definition of a Latitude + // column of type float. We can use the parameter ignoreMissingColumns + // to true to ignore any missing columns in the IDataView. The produced + // enumerable will have the Latitude field set to the default for the + // data type, in this case 0. + var rowEnumerableIgnoreMissing = mlContext.Data + .CreateEnumerable(data, + reuseRowObject: true, ignoreMissingColumns: true); Console.WriteLine($"Date\tTemperature"); foreach (var row in rowEnumerable) - Console.WriteLine($"{row.Date.ToString("d")}\t{row.Temperature}"); + Console.WriteLine( + $"{row.Date.ToString("d")}\t{row.Temperature}"); // Expected output: // Date Temperature @@ -42,7 +52,8 @@ public static void Example() Console.WriteLine($"Date\tTemperature\tLatitude"); foreach (var row in rowEnumerableIgnoreMissing) - Console.WriteLine($"{row.Date.ToString("d")}\t{row.Temperature}\t{row.Latitude}"); + Console.WriteLine($"{row.Date.ToString("d")}\t{row.Temperature}" + + $"\t{row.Latitude}"); // Expected output: // Date Temperature Latitude @@ -71,7 +82,9 @@ private class SampleTemperatureDataWithLatitude /// /// The number of examples to return. /// An enumerable of . - private static IEnumerable GetSampleTemperatureData(int exampleCount) + private static IEnumerable GetSampleTemperatureData( + int exampleCount) + { var rng = new Random(1234321); var date = new DateTime(2012, 1, 1); @@ -81,7 +94,9 @@ private static IEnumerable GetSampleTemperatureData(int e { date = date.AddDays(1); temperature += rng.Next(-5, 5); - yield return new SampleTemperatureData { Date = date, Temperature = temperature }; + yield return new SampleTemperatureData { Date = date, Temperature = + temperature }; + } } } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/DataViewEnumerable.tt b/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/DataViewEnumerable.tt index 680ef76b66..ebcce161ff 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/DataViewEnumerable.tt +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/DataViewEnumerable.tt @@ -3,29 +3,39 @@ string NameSpace = "Samples.Dynamic"; string ClassName="DataViewEnumerable"; string AddExtraClass = "true"; -string ExampleShortDoc = @"// A simple case of creating IDataView from IEnumerable."; -string Example = @"// Create a new context for ML.NET operations. It can be used for exception tracking and logging, +string ExampleShortDoc = @"// A simple case of creating IDataView from + //IEnumerable."; +string Example = @"// Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, // as a catalog of available operations and as the source of randomness. var mlContext = new MLContext(); // Get a small dataset as an IEnumerable. - IEnumerable enumerableOfData = GetSampleTemperatureData(5); + IEnumerable enumerableOfData = + GetSampleTemperatureData(5); // Load dataset into an IDataView. IDataView data = mlContext.Data.LoadFromEnumerable(enumerableOfData); - // We can now examine the records in the IDataView. We first create an enumerable of rows in the IDataView. - var rowEnumerable = mlContext.Data.CreateEnumerable(data, reuseRowObject: true); - - // SampleTemperatureDataWithLatitude has the definition of a Latitude column of type float. - // We can use the parameter ignoreMissingColumns to true to ignore any missing columns in the IDataView. - // The produced enumerable will have the Latitude field set to the default for the data type, in this case 0. - var rowEnumerableIgnoreMissing = mlContext.Data.CreateEnumerable(data, - reuseRowObject: true, ignoreMissingColumns: true); + // We can now examine the records in the IDataView. We first create an + // enumerable of rows in the IDataView. + var rowEnumerable = mlContext.Data + .CreateEnumerable(data, + reuseRowObject: true); + + // SampleTemperatureDataWithLatitude has the definition of a Latitude + // column of type float. We can use the parameter ignoreMissingColumns + // to true to ignore any missing columns in the IDataView. The produced + // enumerable will have the Latitude field set to the default for the + // data type, in this case 0. + var rowEnumerableIgnoreMissing = mlContext.Data + .CreateEnumerable(data, + reuseRowObject: true, ignoreMissingColumns: true); Console.WriteLine($""Date\tTemperature""); foreach (var row in rowEnumerable) - Console.WriteLine($""{row.Date.ToString(""d"")}\t{row.Temperature}""); + Console.WriteLine( + $""{row.Date.ToString(""d"")}\t{row.Temperature}""); // Expected output: // Date Temperature @@ -37,7 +47,8 @@ string Example = @"// Create a new context for ML.NET operations. It can be used Console.WriteLine($""Date\tTemperature\tLatitude""); foreach (var row in rowEnumerableIgnoreMissing) - Console.WriteLine($""{row.Date.ToString(""d"")}\t{row.Temperature}\t{row.Latitude}""); + Console.WriteLine($""{row.Date.ToString(""d"")}\t{row.Temperature}"" + + $""\t{row.Latitude}""); // Expected output: // Date Temperature Latitude diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/FilterRowsByColumn.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/FilterRowsByColumn.cs index bddc168e78..af6df27200 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/FilterRowsByColumn.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/FilterRowsByColumn.cs @@ -6,11 +6,13 @@ namespace Samples.Dynamic { public static class FilterRowsByColumn { - // // Sample class showing how to filter out some rows in IDataView. + // // Sample class showing how to filter out some rows in + // IDataView. public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available + // operations and as the source of randomness. var mlContext = new MLContext(); // Get a small dataset as an IEnumerable. @@ -21,7 +23,9 @@ public static void Example() Console.WriteLine($"Date\tTemperature"); foreach (var row in enumerableOfData) { - Console.WriteLine($"{row.Date.ToString("d")}\t{row.Temperature}"); + Console.WriteLine( + $"{row.Date.ToString("d")}\t{row.Temperature}"); + } Console.WriteLine(); // Expected output: @@ -37,15 +41,24 @@ public static void Example() // 1/10/2012 30 // 1/11/2012 29 - // Filter the data by the values of the temperature. The lower bound is inclusive, the upper exclusive. - var filteredData = mlContext.Data.FilterRowsByColumn(data, columnName: "Temperature", lowerBound: 34, upperBound: 37); + // Filter the data by the values of the temperature. The lower bound is + // inclusive, the upper exclusive. + var filteredData = mlContext.Data + .FilterRowsByColumn(data, columnName: "Temperature", + lowerBound: 34, upperBound: 37); + + // Look at the filtered data and observe that values outside [34,37) + // have been dropped. + var enumerable = mlContext.Data + .CreateEnumerable(filteredData, + reuseRowObject: true); - // Look at the filtered data and observe that values outside [34,37) have been dropped. - var enumerable = mlContext.Data.CreateEnumerable(filteredData, reuseRowObject: true); Console.WriteLine($"Date\tTemperature"); foreach (var row in enumerable) { - Console.WriteLine($"{row.Date.ToString("d")}\t{row.Temperature}"); + Console.WriteLine( + $"{row.Date.ToString("d")}\t{row.Temperature}"); + } // Expected output: @@ -69,7 +82,9 @@ private class SampleTemperatureData /// /// The number of examples to return. /// An enumerable of . - private static IEnumerable GetSampleTemperatureData(int exampleCount) + private static IEnumerable GetSampleTemperatureData( + int exampleCount) + { var rng = new Random(1234321); var date = new DateTime(2012, 1, 1); @@ -79,7 +94,9 @@ private static IEnumerable GetSampleTemperatureData(int e { date = date.AddDays(1); temperature += rng.Next(-5, 5); - yield return new SampleTemperatureData { Date = date, Temperature = temperature }; + yield return new SampleTemperatureData { Date = date, Temperature = + temperature }; + } } } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/FilterRowsByColumn.tt b/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/FilterRowsByColumn.tt index 4b8ed39faf..878cc2eb9e 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/FilterRowsByColumn.tt +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/FilterRowsByColumn.tt @@ -3,9 +3,11 @@ string NameSpace = "Samples.Dynamic"; string ClassName="FilterRowsByColumn"; string AddExtraClass = null; -string ExampleShortDoc = @"// // Sample class showing how to filter out some rows in IDataView."; -string Example = @"// Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. +string ExampleShortDoc = @"// // Sample class showing how to filter out some rows in + // IDataView."; +string Example = @"// Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available + // operations and as the source of randomness. var mlContext = new MLContext(); // Get a small dataset as an IEnumerable. @@ -17,6 +19,7 @@ string Example = @"// Create a new context for ML.NET operations. It can be used foreach (var row in enumerableOfData) { Console.WriteLine($""{row.Date.ToString(""d"")}\t{row.Temperature}""); + } Console.WriteLine(); // Expected output: @@ -32,15 +35,24 @@ string Example = @"// Create a new context for ML.NET operations. It can be used // 1/10/2012 30 // 1/11/2012 29 - // Filter the data by the values of the temperature. The lower bound is inclusive, the upper exclusive. - var filteredData = mlContext.Data.FilterRowsByColumn(data, columnName: ""Temperature"", lowerBound: 34, upperBound: 37); + // Filter the data by the values of the temperature. The lower bound is + // inclusive, the upper exclusive. + var filteredData = mlContext.Data + .FilterRowsByColumn(data, columnName: ""Temperature"", + lowerBound: 34, upperBound: 37); + + // Look at the filtered data and observe that values outside [34,37) + // have been dropped. + var enumerable = mlContext.Data + .CreateEnumerable(filteredData, + reuseRowObject: true); - // Look at the filtered data and observe that values outside [34,37) have been dropped. - var enumerable = mlContext.Data.CreateEnumerable(filteredData, reuseRowObject: true); Console.WriteLine($""Date\tTemperature""); foreach (var row in enumerable) { - Console.WriteLine($""{row.Date.ToString(""d"")}\t{row.Temperature}""); + Console.WriteLine( + $""{row.Date.ToString(""d"")}\t{row.Temperature}""); + } // Expected output: diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/FilterRowsByKeyColumnFraction.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/FilterRowsByKeyColumnFraction.cs index 4074579614..6b95bdf05e 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/FilterRowsByKeyColumnFraction.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/FilterRowsByKeyColumnFraction.cs @@ -11,7 +11,8 @@ public static class FilterRowsByKeyColumnFraction { public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, // as a catalog of available operations and as the source of randomness. var mlContext = new MLContext(); @@ -36,7 +37,9 @@ public static void Example() var transformedData = pipeline.Fit(data).Transform(data); // Before we apply a filter, examine all the records in the dataset. - var enumerable = mlContext.Data.CreateEnumerable(transformedData, reuseRowObject: true); + var enumerable = mlContext.Data + .CreateEnumerable(transformedData, reuseRowObject: true); + Console.WriteLine($"Age"); foreach (var row in enumerable) { @@ -56,12 +59,20 @@ public static void Example() // Now filter down to half the keys, choosing the lower half of values. // For the keys we have the sorted values: 1 1 2 2 2 3 4 4. - // Projected in the [0, 1[ interval as per: (key - 0.5)/(Count of Keys) the values of the keys for our data would be: + // Projected in the [0, 1[ interval as per: (key - 0.5)/(Count of Keys) + // the values of the keys for our data would be: // 0.125 0.125 0.375 0.375 0.375 0.625 0.875 0.875 - // so the keys resulting from filtering in the [0, 0.5 [ interval are the ones with normalized values 0.125 and 0.375, respectively keys + // so the keys resulting from filtering in the [0, 0.5 [ interval are + // the ones with normalized values 0.125 and 0.375, respectively keys // with values 1 and 2. - var filteredHalfData = mlContext.Data.FilterRowsByKeyColumnFraction(transformedData, columnName: "Age", lowerBound: 0, upperBound: 0.5); - var filteredHalfEnumerable = mlContext.Data.CreateEnumerable(filteredHalfData, reuseRowObject: true); + var filteredHalfData = mlContext.Data + .FilterRowsByKeyColumnFraction(transformedData, columnName: "Age", + lowerBound: 0, upperBound: 0.5); + + var filteredHalfEnumerable = mlContext.Data + .CreateEnumerable(filteredHalfData, + reuseRowObject: true); + Console.WriteLine($"Age"); foreach (var row in filteredHalfEnumerable) { @@ -76,12 +87,21 @@ public static void Example() // 2 // 1 - // As mentioned above, the normalized keys are: 0.125 0.125 0.375 0.375 0.375 0.625 0.875 0.875 - // so the keys resulting from filtering in the [0.3, 0.6 [ interval are the ones with normalized value 0.375, respectively key - // with value = 2. - var filteredMiddleData = mlContext.Data.FilterRowsByKeyColumnFraction(transformedData, columnName: "Age", lowerBound: 0.3, upperBound: 0.6); - // Look at the data and observe that values above 2 have been filtered out - var filteredMiddleEnumerable = mlContext.Data.CreateEnumerable(filteredMiddleData, reuseRowObject: true); + // As mentioned above, the normalized keys are: + // 0.125 0.125 0.375 0.375 0.375 0.625 0.875 0.875 + // so the keys resulting from filtering in the [0.3, 0.6 [ interval are + // the ones with normalized value 0.375, respectively key with + // value = 2. + var filteredMiddleData = mlContext.Data + .FilterRowsByKeyColumnFraction(transformedData, columnName: "Age", + lowerBound: 0.3, upperBound: 0.6); + + // Look at the data and observe that values above 2 have been filtered + // out + var filteredMiddleEnumerable = mlContext.Data + .CreateEnumerable(filteredMiddleData, + reuseRowObject: true); + Console.WriteLine($"Age"); foreach (var row in filteredMiddleEnumerable) { diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/FilterRowsByMissingValues.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/FilterRowsByMissingValues.cs index 306f4f5185..02f40a6d2c 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/FilterRowsByMissingValues.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/FilterRowsByMissingValues.cs @@ -12,32 +12,40 @@ public class FilterRowsByMissingValues /// public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(); // Create a small dataset as an IEnumerable. var samples = new List() { - new DataPoint(){ Feature1 = 21, Feature2 = new [] { 1, 2, float.NaN} }, + new DataPoint(){ Feature1 = 21, Feature2 = new [] { 1, 2, float.NaN} + }, + new DataPoint(){ Feature1 = 40, Feature2 = new [] { 1f, 2f, 3f} }, - new DataPoint(){ Feature1 = float.NaN, Feature2 = new [] { 1, 2, float.NaN} } + new DataPoint(){ Feature1 = float.NaN, Feature2 = new [] { 1, 2, + float.NaN} } + }; // Convert training data to IDataView. var data = mlContext.Data.LoadFromEnumerable(samples); // Filter out any row with an NaN values in either column - var filteredData = mlContext.Data.FilterRowsByMissingValues(data, new[] { "Feature1", "Feature2" }); + var filteredData = mlContext.Data + .FilterRowsByMissingValues(data, new[] { "Feature1", "Feature2" }); + + // Take a look at the resulting dataset and note that rows with NaNs are + // filtered out. Only the second data point is left + var enumerable = mlContext.Data + .CreateEnumerable(filteredData, reuseRowObject: true); - // Take a look at the resulting dataset and note that rows with NaNs are filtered out. - // Only the second data point is left - var enumerable = mlContext.Data.CreateEnumerable(filteredData, reuseRowObject: true); Console.WriteLine($"Feature1 Feature2"); foreach (var row in enumerable) { - Console.WriteLine($"{row.Feature1}\t({string.Join(", ", row.Feature2)})"); + Console.WriteLine($"{row.Feature1}" + + $"\t({string.Join(", ", row.Feature2)})"); } // Feature1 Feature2 diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/LoadFromEnumerable.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/LoadFromEnumerable.cs index 64bc929850..0bc83d8831 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/LoadFromEnumerable.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/LoadFromEnumerable.cs @@ -7,37 +7,48 @@ namespace Samples.Dynamic { public static class LoadFromEnumerable { - // Creating IDataView from IEnumerable, and setting the size of the vector at runtime. - // When the data model is defined through types, setting the size of the vector is done through the VectorType - // annotation. When the size of the data is not known at compile time, the Schema can be directly modified at runtime - // and the size of the vector set there. - // This is important, because most of the ML.NET trainers require the Features vector to be of known size. + // Creating IDataView from IEnumerable, and setting the size of the vector + // at runtime. When the data model is defined through types, setting the + // size of the vector is done through the VectorType annotation. When the + // size of the data is not known at compile time, the Schema can be directly + // modified at runtime and the size of the vector set there. This is + // important, because most of the ML.NET trainers require the Features + // vector to be of known size. public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. var mlContext = new MLContext(); // Get a small dataset as an IEnumerable. IEnumerable enumerableKnownSize = new DataPointVector[] { - new DataPointVector{ Features = new float[]{ 1.2f, 3.4f, 4.5f, 3.2f, 7,5f } }, - new DataPointVector{ Features = new float[]{ 4.2f, 3.4f, 14.65f, 3.2f, 3,5f } }, - new DataPointVector{ Features = new float[]{ 1.6f, 3.5f, 4.5f, 6.2f, 3,5f } }, + new DataPointVector{ Features = new float[]{ 1.2f, 3.4f, 4.5f, 3.2f, + 7,5f } }, + + new DataPointVector{ Features = new float[]{ 4.2f, 3.4f, 14.65f, + 3.2f, 3,5f } }, + + new DataPointVector{ Features = new float[]{ 1.6f, 3.5f, 4.5f, 6.2f, + 3,5f } }, + }; // Load dataset into an IDataView. IDataView data = mlContext.Data.LoadFromEnumerable(enumerableKnownSize); var featureColumn = data.Schema["Features"].Type as VectorDataViewType; // Inspecting the schema - Console.WriteLine($"Is the size of the Features column known: {featureColumn.IsKnownSize}.\nSize: {featureColumn.Size}"); + Console.WriteLine($"Is the size of the Features column known: " + + $"{featureColumn.IsKnownSize}.\nSize: {featureColumn.Size}"); // Preview // // Is the size of the Features column known? True. // Size: 5. - // If the size of the vector is unknown at compile time, it can be set at runtime. + // If the size of the vector is unknown at compile time, it can be set + // at runtime. IEnumerable enumerableUnknownSize = new DataPoint[] { new DataPoint{ Features = new float[]{ 1.2f, 3.4f, 4.5f } }, @@ -45,12 +56,15 @@ public static void Example() new DataPoint{ Features = new float[]{ 1.6f, 3.5f, 4.5f } }, }; - // The feature dimension (typically this will be the Count of the array of the features vector - // known at runtime). + // The feature dimension (typically this will be the Count of the array + // of the features vector known at runtime). int featureDimension = 3; var definedSchema = SchemaDefinition.Create(typeof(DataPoint)); - featureColumn = definedSchema["Features"].ColumnType as VectorDataViewType; - Console.WriteLine($"Is the size of the Features column known: {featureColumn.IsKnownSize}.\nSize: {featureColumn.Size}"); + featureColumn = definedSchema["Features"] + .ColumnType as VectorDataViewType; + + Console.WriteLine($"Is the size of the Features column known: " + + $"{featureColumn.IsKnownSize}.\nSize: {featureColumn.Size}"); // Preview // @@ -58,15 +72,19 @@ public static void Example() // Size: 0. // Set the column type to be a known-size vector. - var vectorItemType = ((VectorDataViewType)definedSchema[0].ColumnType).ItemType; - definedSchema[0].ColumnType = new VectorDataViewType(vectorItemType, featureDimension); + var vectorItemType = ((VectorDataViewType)definedSchema[0].ColumnType) + .ItemType; + definedSchema[0].ColumnType = new VectorDataViewType(vectorItemType, + featureDimension); // Read the data into an IDataView with the modified schema supplied in - IDataView data2 = mlContext.Data.LoadFromEnumerable(enumerableUnknownSize, definedSchema); + IDataView data2 = mlContext.Data + .LoadFromEnumerable(enumerableUnknownSize, definedSchema); featureColumn = data2.Schema["Features"].Type as VectorDataViewType; // Inspecting the schema - Console.WriteLine($"Is the size of the Features column known: {featureColumn.IsKnownSize}.\nSize: {featureColumn.Size}"); + Console.WriteLine($"Is the size of the Features column known: " + + $"{featureColumn.IsKnownSize}.\nSize: {featureColumn.Size}"); // Preview // diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/SaveAndLoadFromBinary.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/SaveAndLoadFromBinary.cs index b6448ec857..9fa1253a62 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/SaveAndLoadFromBinary.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/SaveAndLoadFromBinary.cs @@ -9,9 +9,10 @@ public static class SaveAndLoadFromBinary { public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. @@ -24,18 +25,22 @@ public static void Example() new DataPoint(){ Label = 1, Features = 9}, }; - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. IDataView data = mlContext.Data.LoadFromEnumerable(dataPoints); - // Create a FileStream object and write the IDataView to it as a binary IDV file. + // Create a FileStream object and write the IDataView to it as a binary + // IDV file. using (FileStream stream = new FileStream("data.idv", FileMode.Create)) mlContext.Data.SaveAsBinary(data, stream); // Create an IDataView object by loading the binary IDV file. IDataView loadedData = mlContext.Data.LoadFromBinary("data.idv"); - // Inspect the data that is loaded from the previously saved binary file. - var loadedDataEnumerable = mlContext.Data.CreateEnumerable(loadedData, reuseRowObject: false); + // Inspect the data that is loaded from the previously saved binary file + var loadedDataEnumerable = mlContext.Data + .CreateEnumerable(loadedData, reuseRowObject: false); + foreach (DataPoint row in loadedDataEnumerable) Console.WriteLine($"{row.Label}, {row.Features}"); @@ -47,7 +52,8 @@ public static void Example() // 1, 9 } - // Example with label and feature values. A data set is a collection of such examples. + // Example with label and feature values. A data set is a collection of such + // examples. private class DataPoint { public float Label { get; set; } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/SaveAndLoadFromText.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/SaveAndLoadFromText.cs index 9918b736ce..c61e9cff94 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/SaveAndLoadFromText.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/SaveAndLoadFromText.cs @@ -9,9 +9,10 @@ public static class SaveAndLoadFromText { public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. @@ -24,10 +25,12 @@ public static void Example() new DataPoint(){ Label = 1, Features = 9}, }; - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. IDataView data = mlContext.Data.LoadFromEnumerable(dataPoints); - // Create a FileStream object and write the IDataView to it as a text file. + // Create a FileStream object and write the IDataView to it as a text + // file. using (FileStream stream = new FileStream("data.tsv", FileMode.Create)) mlContext.Data.SaveAsText(data, stream); @@ -35,7 +38,9 @@ public static void Example() IDataView loadedData = mlContext.Data.LoadFromTextFile("data.tsv"); // Inspect the data that is loaded from the previously saved text file. - var loadedDataEnumerable = mlContext.Data.CreateEnumerable(loadedData, reuseRowObject: false); + var loadedDataEnumerable = mlContext.Data + .CreateEnumerable(loadedData, reuseRowObject: false); + foreach (DataPoint row in loadedDataEnumerable) Console.WriteLine($"{row.Label}, {row.Features}"); @@ -47,7 +52,8 @@ public static void Example() // 1, 9 } - // Example with label and feature values. A data set is a collection of such examples. + // Example with label and feature values. A data set is a collection of such + // examples. private class DataPoint { public float Label { get; set; } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/ShuffleRows.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/ShuffleRows.cs index 2b786a7da4..73ef6657e1 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/ShuffleRows.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/ShuffleRows.cs @@ -6,11 +6,13 @@ namespace Samples.Dynamic { public static class ShuffleRows { - // Sample class showing how to shuffle rows in IDataView. + // Sample class showing how to shuffle rows in + // IDataView. public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. var mlContext = new MLContext(); // Get a small dataset as an IEnumerable. @@ -21,7 +23,8 @@ public static void Example() Console.WriteLine($"Date\tTemperature"); foreach (var row in enumerableOfData) { - Console.WriteLine($"{row.Date.ToString("d")}\t{row.Temperature}"); + Console.WriteLine($"{row.Date.ToString("d")}" + + $"\t{row.Temperature}"); } Console.WriteLine(); // Expected output: @@ -35,12 +38,17 @@ public static void Example() // Shuffle the dataset. var shuffledData = mlContext.Data.ShuffleRows(data, seed: 123); - // Look at the shuffled data and observe that the rows are in a randomized order. - var enumerable = mlContext.Data.CreateEnumerable(shuffledData, reuseRowObject: true); + // Look at the shuffled data and observe that the rows are in a + // randomized order. + var enumerable = mlContext.Data + .CreateEnumerable(shuffledData, + reuseRowObject: true); + Console.WriteLine($"Date\tTemperature"); foreach (var row in enumerable) { - Console.WriteLine($"{row.Date.ToString("d")}\t{row.Temperature}"); + Console.WriteLine($"{row.Date.ToString("d")}" + + $"\t{row.Temperature}"); } // Expected output: // Date Temperature @@ -62,7 +70,9 @@ private class SampleTemperatureData /// /// The number of examples to return. /// An enumerable of . - private static IEnumerable GetSampleTemperatureData(int exampleCount) + private static IEnumerable GetSampleTemperatureData( + int exampleCount) + { var rng = new Random(1234321); var date = new DateTime(2012, 1, 1); @@ -72,7 +82,9 @@ private static IEnumerable GetSampleTemperatureData(int e { date = date.AddDays(1); temperature += rng.Next(-5, 5); - yield return new SampleTemperatureData { Date = date, Temperature = temperature }; + yield return new SampleTemperatureData { Date = date, Temperature = + temperature }; + } } } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/ShuffleRows.tt b/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/ShuffleRows.tt index fa2adb3e9a..4c77c521a9 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/ShuffleRows.tt +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/ShuffleRows.tt @@ -3,9 +3,11 @@ string NameSpace = "Samples.Dynamic"; string ClassName="ShuffleRows"; string AddExtraClass = null; -string ExampleShortDoc = @"// Sample class showing how to shuffle rows in IDataView."; -string Example = @"// Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. +string ExampleShortDoc = @"// Sample class showing how to shuffle rows in + // IDataView."; +string Example = @"// Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. var mlContext = new MLContext(); // Get a small dataset as an IEnumerable. @@ -16,7 +18,8 @@ string Example = @"// Create a new context for ML.NET operations. It can be used Console.WriteLine($""Date\tTemperature""); foreach (var row in enumerableOfData) { - Console.WriteLine($""{row.Date.ToString(""d"")}\t{row.Temperature}""); + Console.WriteLine($""{row.Date.ToString(""d"")}"" + + $""\t{row.Temperature}""); } Console.WriteLine(); // Expected output: @@ -30,12 +33,17 @@ string Example = @"// Create a new context for ML.NET operations. It can be used // Shuffle the dataset. var shuffledData = mlContext.Data.ShuffleRows(data, seed: 123); - // Look at the shuffled data and observe that the rows are in a randomized order. - var enumerable = mlContext.Data.CreateEnumerable(shuffledData, reuseRowObject: true); + // Look at the shuffled data and observe that the rows are in a + // randomized order. + var enumerable = mlContext.Data + .CreateEnumerable(shuffledData, + reuseRowObject: true); + Console.WriteLine($""Date\tTemperature""); foreach (var row in enumerable) { - Console.WriteLine($""{row.Date.ToString(""d"")}\t{row.Temperature}""); + Console.WriteLine($""{row.Date.ToString(""d"")}"" + + $""\t{row.Temperature}""); } // Expected output: // Date Temperature diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/SkipRows.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/SkipRows.cs index cd1744a0f6..006082f238 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/SkipRows.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/SkipRows.cs @@ -9,8 +9,9 @@ public static class SkipRows // Sample class showing how to skip rows in IDataView. public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. var mlContext = new MLContext(); // Get a small dataset as an IEnumerable. @@ -21,7 +22,8 @@ public static void Example() Console.WriteLine($"Date\tTemperature"); foreach (var row in enumerableOfData) { - Console.WriteLine($"{row.Date.ToString("d")}\t{row.Temperature}"); + Console.WriteLine($"{row.Date.ToString("d")}" + + $"\t{row.Temperature}"); } Console.WriteLine(); // Expected output: @@ -40,12 +42,17 @@ public static void Example() // Skip the first 5 rows in the dataset var filteredData = mlContext.Data.SkipRows(data, 5); - // Look at the filtered data and observe that the first 5 rows have been dropped - var enumerable = mlContext.Data.CreateEnumerable(filteredData, reuseRowObject: true); + // Look at the filtered data and observe that the first 5 rows have been + // dropped + var enumerable = mlContext.Data + .CreateEnumerable(filteredData, + reuseRowObject: true); + Console.WriteLine($"Date\tTemperature"); foreach (var row in enumerable) { - Console.WriteLine($"{row.Date.ToString("d")}\t{row.Temperature}"); + Console.WriteLine($"{row.Date.ToString("d")}" + + $"\t{row.Temperature}"); } // Expected output: // Date Temperature @@ -67,7 +74,9 @@ private class SampleTemperatureData /// /// The number of examples to return. /// An enumerable of . - private static IEnumerable GetSampleTemperatureData(int exampleCount) + private static IEnumerable GetSampleTemperatureData( + int exampleCount) + { var rng = new Random(1234321); var date = new DateTime(2012, 1, 1); @@ -77,7 +86,9 @@ private static IEnumerable GetSampleTemperatureData(int e { date = date.AddDays(1); temperature += rng.Next(-5, 5); - yield return new SampleTemperatureData { Date = date, Temperature = temperature }; + yield return new SampleTemperatureData { Date = date, Temperature = + temperature }; + } } } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/SkipRows.tt b/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/SkipRows.tt index 411b4cf85a..59bd97c443 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/SkipRows.tt +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/SkipRows.tt @@ -4,8 +4,9 @@ string NameSpace = "Samples.Dynamic"; string ClassName="SkipRows"; string AddExtraClass = null; string ExampleShortDoc = @"// Sample class showing how to skip rows in IDataView."; -string Example = @"// Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. +string Example = @"// Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. var mlContext = new MLContext(); // Get a small dataset as an IEnumerable. @@ -16,7 +17,8 @@ string Example = @"// Create a new context for ML.NET operations. It can be used Console.WriteLine($""Date\tTemperature""); foreach (var row in enumerableOfData) { - Console.WriteLine($""{row.Date.ToString(""d"")}\t{row.Temperature}""); + Console.WriteLine($""{row.Date.ToString(""d"")}"" + + $""\t{row.Temperature}""); } Console.WriteLine(); // Expected output: @@ -35,12 +37,17 @@ string Example = @"// Create a new context for ML.NET operations. It can be used // Skip the first 5 rows in the dataset var filteredData = mlContext.Data.SkipRows(data, 5); - // Look at the filtered data and observe that the first 5 rows have been dropped - var enumerable = mlContext.Data.CreateEnumerable(filteredData, reuseRowObject: true); + // Look at the filtered data and observe that the first 5 rows have been + // dropped + var enumerable = mlContext.Data + .CreateEnumerable(filteredData, + reuseRowObject: true); + Console.WriteLine($""Date\tTemperature""); foreach (var row in enumerable) { - Console.WriteLine($""{row.Date.ToString(""d"")}\t{row.Temperature}""); + Console.WriteLine($""{row.Date.ToString(""d"")}"" + + $""\t{row.Temperature}""); } // Expected output: // Date Temperature diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/TakeRows.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/TakeRows.cs index 26a489f5e8..5ad6f13f7f 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/TakeRows.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/TakeRows.cs @@ -6,10 +6,12 @@ namespace Samples.Dynamic { public static class TakeRows { - // Sample class showing how to take some rows from the beginning of IDataView. + // Sample class showing how to take some rows from the + // beginning of IDataView. public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, + // Create a new context for ML.NET operations. It can be used for + //exception tracking and logging, // as a catalog of available operations and as the source of randomness. var mlContext = new MLContext(); @@ -21,7 +23,8 @@ public static void Example() Console.WriteLine($"Date\tTemperature"); foreach (var row in enumerableOfData) { - Console.WriteLine($"{row.Date.ToString("d")}\t{row.Temperature}"); + Console.WriteLine($"{row.Date.ToString("d")}" + + $"\t{row.Temperature}"); } Console.WriteLine(); // Expected output: @@ -40,12 +43,17 @@ public static void Example() // Take the first 5 rows in the dataset var filteredData = mlContext.Data.TakeRows(data, 5); - // Look at the filtered data and observe that only the first 5 rows are in the resulting dataset. - var enumerable = mlContext.Data.CreateEnumerable(filteredData, reuseRowObject: true); + // Look at the filtered data and observe that only the first 5 rows are + // in the resulting dataset. + var enumerable = mlContext.Data + .CreateEnumerable(filteredData, + reuseRowObject: true); + Console.WriteLine($"Date\tTemperature"); foreach (var row in enumerable) { - Console.WriteLine($"{row.Date.ToString("d")}\t{row.Temperature}"); + Console.WriteLine($"{row.Date.ToString("d")}" + + $"\t{row.Temperature}"); } // Expected output: // Date Temperature @@ -67,7 +75,9 @@ private class SampleTemperatureData /// /// The number of examples to return. /// An enumerable of . - private static IEnumerable GetSampleTemperatureData(int exampleCount) + private static IEnumerable GetSampleTemperatureData( + int exampleCount) + { var rng = new Random(1234321); var date = new DateTime(2012, 1, 1); @@ -77,7 +87,9 @@ private static IEnumerable GetSampleTemperatureData(int e { date = date.AddDays(1); temperature += rng.Next(-5, 5); - yield return new SampleTemperatureData { Date = date, Temperature = temperature }; + yield return new SampleTemperatureData { Date = date, Temperature = + temperature }; + } } } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/TakeRows.tt b/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/TakeRows.tt index 76f1edeb80..1e482591db 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/TakeRows.tt +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/TakeRows.tt @@ -3,20 +3,23 @@ string NameSpace = "Samples.Dynamic"; string ClassName="TakeRows"; string AddExtraClass = null; -string ExampleShortDoc = @"// Sample class showing how to take some rows from the beginning of IDataView."; -string Example = @"// Create a new context for ML.NET operations. It can be used for exception tracking and logging, +string ExampleShortDoc = @"// Sample class showing how to take some rows from the + // beginning of IDataView."; +string Example = @"// Create a new context for ML.NET operations. It can be used for + //exception tracking and logging, // as a catalog of available operations and as the source of randomness. var mlContext = new MLContext(); // Get a small dataset as an IEnumerable. - var enumerableOfData = Microsoft.ML.SamplesUtils.DatasetUtils.GetSampleTemperatureData(10); + var enumerableOfData = GetSampleTemperatureData(10); var data = mlContext.Data.LoadFromEnumerable(enumerableOfData); // Before we apply a filter, examine all the records in the dataset. Console.WriteLine($""Date\tTemperature""); foreach (var row in enumerableOfData) { - Console.WriteLine($""{row.Date.ToString(""d"")}\t{row.Temperature}""); + Console.WriteLine($""{row.Date.ToString(""d"")}"" + + $""\t{row.Temperature}""); } Console.WriteLine(); // Expected output: @@ -35,12 +38,17 @@ string Example = @"// Create a new context for ML.NET operations. It can be used // Take the first 5 rows in the dataset var filteredData = mlContext.Data.TakeRows(data, 5); - // Look at the filtered data and observe that only the first 5 rows are in the resulting dataset. - var enumerable = mlContext.Data.CreateEnumerable(filteredData, reuseRowObject: true); + // Look at the filtered data and observe that only the first 5 rows are + // in the resulting dataset. + var enumerable = mlContext.Data + .CreateEnumerable(filteredData, + reuseRowObject: true); + Console.WriteLine($""Date\tTemperature""); foreach (var row in enumerable) { - Console.WriteLine($""{row.Date.ToString(""d"")}\t{row.Temperature}""); + Console.WriteLine($""{row.Date.ToString(""d"")}"" + + $""\t{row.Temperature}""); } // Expected output: // Date Temperature diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/TemperatureAndLatitude.ttinclude b/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/TemperatureAndLatitude.ttinclude index 448e743791..07ecca7ddf 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/TemperatureAndLatitude.ttinclude +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/TemperatureAndLatitude.ttinclude @@ -32,7 +32,9 @@ namespace <#=NameSpace#> /// /// The number of examples to return. /// An enumerable of . - private static IEnumerable GetSampleTemperatureData(int exampleCount) + private static IEnumerable GetSampleTemperatureData( + int exampleCount) + { var rng = new Random(1234321); var date = new DateTime(2012, 1, 1); @@ -42,7 +44,9 @@ namespace <#=NameSpace#> { date = date.AddDays(1); temperature += rng.Next(-5, 5); - yield return new SampleTemperatureData { Date = date, Temperature = temperature }; + yield return new SampleTemperatureData { Date = date, + Temperature = temperature }; + } } } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/TrainTestSplit.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/TrainTestSplit.cs index 47c56266bc..e4fc26296b 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/TrainTestSplit.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/TrainTestSplit.cs @@ -17,16 +17,26 @@ public static void Example() // Generate some data points. var examples = GenerateRandomDataPoints(10); - // Convert the examples list to an IDataView object, which is consumable by ML.NET API. + // Convert the examples list to an IDataView object, which is consumable + // by ML.NET API. var dataview = mlContext.Data.LoadFromEnumerable(examples); - // Leave out 10% of the dataset for testing.For some types of problems, for example for ranking or anomaly detection, - // we must ensure that the split leaves the rows with the same value in a particular column, in one of the splits. - // So below, we specify Group column as the column containing the sampling keys. - // Notice how keeping the rows with the same value in the Group column overrides the testFraction definition. - var split = mlContext.Data.TrainTestSplit(dataview, testFraction: 0.1, samplingKeyColumnName: "Group"); - var trainSet = mlContext.Data.CreateEnumerable(split.TrainSet, reuseRowObject: false); - var testSet = mlContext.Data.CreateEnumerable(split.TestSet, reuseRowObject: false); + // Leave out 10% of the dataset for testing.For some types of problems, + // for example for ranking or anomaly detection, we must ensure that the + // split leaves the rows with the same value in a particular column, in + // one of the splits. So below, we specify Group column as the column + // containing the sampling keys. Notice how keeping the rows with the + // same value in the Group column overrides the testFraction definition. + var split = mlContext.Data + .TrainTestSplit(dataview, testFraction: 0.1, + samplingKeyColumnName: "Group"); + + var trainSet = mlContext.Data + .CreateEnumerable(split.TrainSet, reuseRowObject: false); + + var testSet = mlContext.Data + .CreateEnumerable(split.TestSet,reuseRowObject: false); + PrintPreviewRows(trainSet, testSet); // The data in the Train split. @@ -45,8 +55,12 @@ public static void Example() // Example of a split without specifying a sampling key column. split = mlContext.Data.TrainTestSplit(dataview, testFraction: 0.2); - trainSet = mlContext.Data.CreateEnumerable(split.TrainSet, reuseRowObject: false); - testSet = mlContext.Data.CreateEnumerable(split.TestSet, reuseRowObject: false); + trainSet = mlContext.Data + .CreateEnumerable(split.TrainSet,reuseRowObject: false); + + testSet = mlContext.Data + .CreateEnumerable(split.TestSet,reuseRowObject: false); + PrintPreviewRows(trainSet, testSet); // The data in the Train split. @@ -65,7 +79,9 @@ public static void Example() } - private static IEnumerable GenerateRandomDataPoints(int count, int seed = 0) + private static IEnumerable GenerateRandomDataPoints(int count, + int seed = 0) + { var random = new Random(seed); for (int i = 0; i < count; i++) @@ -80,7 +96,8 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se } } - // Example with label and group column. A data set is a collection of such examples. + // Example with label and group column. A data set is a collection of such + // examples. private class DataPoint { public float Group { get; set; } @@ -89,7 +106,9 @@ private class DataPoint } // print helper - private static void PrintPreviewRows(IEnumerable trainSet, IEnumerable testSet) + private static void PrintPreviewRows(IEnumerable trainSet, + IEnumerable testSet) + { Console.WriteLine($"The data in the Train split."); diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/ModelOperations/SaveLoadModel.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/ModelOperations/SaveLoadModel.cs index a5f70802bb..67e9226ad6 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/ModelOperations/SaveLoadModel.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/ModelOperations/SaveLoadModel.cs @@ -9,8 +9,8 @@ public class SaveLoadModel { public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(); // Generate sample data. @@ -25,7 +25,8 @@ public static void Example() var outputColumnName = nameof(Transformation.Key); // Transform. - ITransformer model = mlContext.Transforms.Conversion.MapValueToKey(outputColumnName, inputColumnName).Fit(dataView); + ITransformer model = mlContext.Transforms.Conversion + .MapValueToKey(outputColumnName, inputColumnName).Fit(dataView); // Save model. mlContext.Model.Save(model, dataView.Schema, "model.zip"); @@ -35,11 +36,15 @@ public static void Example() model = mlContext.Model.Load(file, out DataViewSchema schema); // Create a prediction engine from the model for feeding new data. - var engine = mlContext.Model.CreatePredictionEngine(model); + var engine = mlContext.Model + .CreatePredictionEngine(model); + var transformation = engine.Predict(new Data() { Value = "abc" }); // Print transformation to console. - Console.WriteLine("Value: {0}\t Key:{1}", transformation.Value, transformation.Key); + Console.WriteLine("Value: {0}\t Key:{1}", transformation.Value, + transformation.Key); + // Value: abc Key:1 } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/ModelOperations/SaveLoadModelFile.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/ModelOperations/SaveLoadModelFile.cs index 9884f8c8e9..88192cb3d3 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/ModelOperations/SaveLoadModelFile.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/ModelOperations/SaveLoadModelFile.cs @@ -9,8 +9,8 @@ public class SaveLoadModelFile { public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(); // Generate sample data. @@ -25,7 +25,8 @@ public static void Example() var outputColumnName = nameof(Transformation.Key); // Transform. - ITransformer model = mlContext.Transforms.Conversion.MapValueToKey(outputColumnName, inputColumnName).Fit(dataView); + ITransformer model = mlContext.Transforms.Conversion + .MapValueToKey(outputColumnName, inputColumnName).Fit(dataView); // Save model. mlContext.Model.Save(model, dataView.Schema, "model.zip"); @@ -34,11 +35,15 @@ public static void Example() model = mlContext.Model.Load("model.zip", out DataViewSchema schema); // Create a prediction engine from the model for feeding new data. - var engine = mlContext.Model.CreatePredictionEngine(model); + var engine = mlContext.Model + .CreatePredictionEngine(model); + var transformation = engine.Predict(new Data() { Value = "abc" }); // Print transformation to console. - Console.WriteLine("Value: {0}\t Key:{1}", transformation.Value, transformation.Key); + Console.WriteLine("Value: {0}\t Key:{1}", transformation.Value, + transformation.Key); + // Value: abc Key:1 } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/NgramExtraction.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/NgramExtraction.cs index 1fe2f70325..9f607a9c9e 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/NgramExtraction.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/NgramExtraction.cs @@ -9,15 +9,21 @@ public static partial class TransformSamples { public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var ml = new MLContext(); // Get a small dataset as an IEnumerable and convert to IDataView. var data = new List() { - new SampleSentimentData { Sentiment = true, SentimentText = "Best game I've ever played." }, - new SampleSentimentData { Sentiment = false, SentimentText = "==RUDE== Dude, 2" }, - new SampleSentimentData { Sentiment = true, SentimentText = "Until the next game, this is the best Xbox game!" } }; + new SampleSentimentData { Sentiment = true, + SentimentText = "Best game I've ever played." }, + + new SampleSentimentData { Sentiment = false, + SentimentText = "==RUDE== Dude, 2" }, + + new SampleSentimentData { Sentiment = true, + SentimentText = "Until the next game," + + "this is the best Xbox game!" } }; // Convert IEnumerable to IDataView. var trainData = ml.Data.LoadFromEnumerable(data); @@ -29,23 +35,42 @@ public static void Example() // false ==RUDE== Dude, 2. // true Until the next game, this is the best Xbox game! - // A pipeline to tokenize text as characters and then combine them together into n-grams - // The pipeline uses the default settings to featurize. + // A pipeline to tokenize text as characters and then combine them + // together into n-grams. The pipeline uses the default settings to + // featurize. + + var charsPipeline = ml.Transforms.Text + .TokenizeIntoCharactersAsKeys("Chars", "SentimentText", + useMarkerCharacters: false); + + var ngramOnePipeline = ml.Transforms.Text + .ProduceNgrams("CharsUnigrams", "Chars", ngramLength: 1); - var charsPipeline = ml.Transforms.Text.TokenizeIntoCharactersAsKeys("Chars", "SentimentText", useMarkerCharacters: false); - var ngramOnePipeline = ml.Transforms.Text.ProduceNgrams("CharsUnigrams", "Chars", ngramLength: 1); - var ngramTwpPipeline = ml.Transforms.Text.ProduceNgrams("CharsTwograms", "Chars"); - var oneCharsPipeline = charsPipeline.Append(ngramOnePipeline); - var twoCharsPipeline = charsPipeline.Append(ngramTwpPipeline); + var ngramTwpPipeline = ml.Transforms.Text + .ProduceNgrams("CharsTwograms", "Chars"); + + var oneCharsPipeline = charsPipeline + .Append(ngramOnePipeline); + + var twoCharsPipeline = charsPipeline + .Append(ngramTwpPipeline); // The transformed data for pipelines. - var transformedData_onechars = oneCharsPipeline.Fit(trainData).Transform(trainData); - var transformedData_twochars = twoCharsPipeline.Fit(trainData).Transform(trainData); + var transformedData_onechars = oneCharsPipeline.Fit(trainData) + .Transform(trainData); + + var transformedData_twochars = twoCharsPipeline.Fit(trainData) + .Transform(trainData); // Small helper to print the text inside the columns, in the console. - Action>, VBuffer>> printHelper = (columnName, column, names) => + Action>, + VBuffer>> + printHelper = (columnName, column, names) => + { - Console.WriteLine($"{columnName} column obtained post-transformation."); + Console.WriteLine( + $"{columnName} column obtained post-transformation."); + var slots = names.GetValues(); foreach (var featureRow in column) { @@ -54,12 +79,19 @@ public static void Example() Console.WriteLine(""); } - Console.WriteLine("==================================================="); + Console.WriteLine( + "==================================================="); }; - // Preview of the CharsUnigrams column obtained after processing the input. + // Preview of the CharsUnigrams column obtained after processing the + // input. VBuffer> slotNames = default; - transformedData_onechars.Schema["CharsUnigrams"].GetSlotNames(ref slotNames); - var charsOneGramColumn = transformedData_onechars.GetColumn>(transformedData_onechars.Schema["CharsUnigrams"]); + transformedData_onechars.Schema["CharsUnigrams"] + .GetSlotNames(ref slotNames); + + var charsOneGramColumn = transformedData_onechars + .GetColumn>(transformedData_onechars + .Schema["CharsUnigrams"]); + printHelper("CharsUnigrams", charsOneGramColumn, slotNames); // CharsUnigrams column obtained post-transformation. @@ -67,8 +99,13 @@ public static void Example() // 'e' - 1 '' - 2 'd' - 1 '=' - 4 'R' - 1 'U' - 1 'D' - 2 'E' - 1 'u' - 1 ',' - 1 '2' - 1 // 'B' - 0 'e' - 6 's' - 3 't' - 6 '' - 9 'g' - 2 'a' - 2 'm' - 2 'I' - 0 ''' - 0 'v' - 0 ... // Preview of the CharsTwoGrams column obtained after processing the input. - var charsTwoGramColumn = transformedData_twochars.GetColumn>(transformedData_twochars.Schema["CharsTwograms"]); - transformedData_twochars.Schema["CharsTwograms"].GetSlotNames(ref slotNames); + var charsTwoGramColumn = transformedData_twochars + .GetColumn>(transformedData_twochars + .Schema["CharsTwograms"]); + + transformedData_twochars.Schema["CharsTwograms"] + .GetSlotNames(ref slotNames); + printHelper("CharsTwograms", charsTwoGramColumn, slotNames); // CharsTwograms column obtained post-transformation. @@ -78,7 +115,8 @@ public static void Example() } /// - /// A dataset that contains a tweet and the sentiment assigned to that tweet: 0 - negative and 1 - positive sentiment. + /// A dataset that contains a tweet and the sentiment assigned to that + /// tweet: 0 - negative and 1 - positive sentiment. /// public class SampleSentimentData { diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/SimpleDataViewImplementation.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/SimpleDataViewImplementation.cs index b1a134b192..f19f42348d 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/SimpleDataViewImplementation.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/SimpleDataViewImplementation.cs @@ -7,17 +7,21 @@ namespace Samples.Dynamic { /// - /// The interface is the central concept of "data" in ML.NET. While many conveniences exist - /// to create pre-baked implementations, it is also useful to know how to create one completely from scratch. We also - /// take this opportunity to illustrate and motivate the basic principles of how the IDataView system is architected, - /// since people interested in implementing need at least some knowledge of those principles. + /// The interface is the central concept of "data" in + /// ML.NET. While many conveniences exist to create pre-baked implementations, + /// it is also useful to know how to create one completely from scratch. We also + /// take this opportunity to illustrate and motivate the basic principles of how + /// the IDataView system is architected, since people interested in + /// implementing need at least some knowledge of those + /// principles. /// public static class SimpleDataViewImplementation { public static void Example() { - // First we create an array of these objects, which we "present" as this IDataView implementation so that it - // can be used in a simple ML.NET pipeline. + // First we create an array of these objects, which we "present" as this + // IDataView implementation so that it can be used in a simple ML.NET + // pipeline. var inputArray = new[] { new InputObject(false, "Hello my friend."), @@ -26,8 +30,8 @@ public static void Example() }; var dataView = new InputObjectDataView(inputArray); - // So, this is a very simple pipeline: a transformer that tokenizes Text, does nothing with the Label column - // at all. + // So, this is a very simple pipeline: a transformer that tokenizes + // Text, does nothing with the Label column at all. var mlContext = new MLContext(); var transformedDataView = mlContext.Transforms.Text.TokenizeIntoWords( "TokenizedText", "Text").Fit(dataView).Transform(dataView); @@ -35,22 +39,31 @@ public static void Example() var textColumn = transformedDataView.Schema["Text"]; var tokensColumn = transformedDataView.Schema["TokenizedText"]; - using (var cursor = transformedDataView.GetRowCursor(new[] { textColumn, tokensColumn })) + using (var cursor = transformedDataView.GetRowCursor( + new[] { textColumn, tokensColumn })) + { - // Note that it is best to get the getters and values *before* iteration, so as to faciliate buffer - // sharing (if applicable), and column-type validation once, rather than many times. + // Note that it is best to get the getters and values *before* + // iteration, so as to faciliate buffer sharing (if applicable), + // and column-type validation once, rather than many times. ReadOnlyMemory textValue = default; VBuffer> tokensValue = default; - var textGetter = cursor.GetGetter>(textColumn); - var tokensGetter = cursor.GetGetter>>(tokensColumn); + var textGetter = cursor + .GetGetter>(textColumn); + + var tokensGetter = cursor + .GetGetter>>(tokensColumn); while (cursor.MoveNext()) { textGetter(ref textValue); tokensGetter(ref tokensValue); - Console.WriteLine($"{textValue} => {string.Join(", ", tokensValue.DenseValues())}"); + Console.WriteLine( + $"{textValue} => " + + $"{string.Join(", ", tokensValue.DenseValues())}"); + } // The output to console is this: @@ -59,12 +72,15 @@ public static void Example() // Stay awhile and listen. => Stay, awhile, and, listen. // Masterfully done hero! => Masterfully, done, hero! - // Note that it may be interesting to set a breakpoint on the Console.WriteLine, and explore - // what is going on with the cursor, and the buffers. In particular, on the third iteration, - // while `tokensValue` is logically presented as a three element array, internally you will - // see that the arrays internal to that structure have (at least) four items, specifically: - // `Masterfully`, `done`, `hero!`, `listen.`. In this way we see a simple example of the details - // of how buffer sharing from one iteration to the next actually works. + // Note that it may be interesting to set a breakpoint on the + // Console.WriteLine, and explore what is going on with the cursor, + // and the buffers. In particular, on the third iteration, while + // `tokensValue` is logically presented as a three element array, + // internally you will see that the arrays internal to that + // structure have (at least) four items, specifically: + // `Masterfully`, `done`, `hero!`, `listen.`. In this way we see a + // simple example of the details of how buffer sharing from one + // iteration to the next actually works. } } @@ -81,37 +97,51 @@ public InputObject(bool label, string text) } /// - /// This is an implementation of that wraps an - /// of the above . Note that normally under these circumstances, the first - /// recommendation would be to use a convenience like - /// - /// or something like that, rather than implementing outright. However, sometimes when - /// code generation is impossible on some situations, like Unity or other similar platforms, implementing + /// This is an implementation of that wraps an + /// of the above . + /// Note that normally under these circumstances, the first recommendation + /// would be to use a convenience like + /// + /// or something like that, rather than implementing + /// outright. However, sometimes when code generation is impossible on some + /// situations, like Unity or other similar platforms, implementing /// something even closely resembling this may become necessary. /// - /// This implementation of , being didactic, is much simpler than practically - /// anything one would find in the ML.NET codebase. In this case we have a completely fixed schema (the two - /// fields of ), with fixed types. + /// This implementation of , being didactic, is much + /// simpler than practically anything one would find in the ML.NET codebase. + /// In this case we have a completely fixed schema (the two fields of + /// ), with fixed types. /// - /// For , note that we keep a very simple schema based off the members of the object. You - /// may in fact note that it is possible in this specific case, this implementation of - /// could share the same object across all instances of this - /// object, but since this is almost never the case, I do not take advantage of that. + /// For , note that we keep a very simple schema based + /// off the members of the object. You may in fact note that it is possible + /// in this specific case, this implementation of + /// could share the same object across all + /// instances of this object, but since this is almost never the case, I do + /// not take advantage of that. /// - /// We have chosen to wrap an , so in fact only a very simple implementation is - /// possible. Specifically: we cannot meaningfully shuffle (so is - /// , and even if a parameter were passed to - /// , we could not make use of it), we do - /// not know the count of the item right away without counting (so, it is most correct for - /// to return , even after we might hypothetically know after - /// the first pass, given the immutability principle of ), and the - /// method returns a single item. + /// We have chosen to wrap an , so in fact only + /// a very simple implementation is possible. Specifically: we cannot + /// meaningfully shuffle (so is + /// , and even if a + /// parameter were passed to + /// , + /// we could not make use of it), we do not know the count of the item right + /// away without counting (so, it is most correct for + /// to return , even after + /// we might hypothetically know after the first pass, given the + /// immutability principle of ), and the + /// method returns a + /// single item. /// - /// The derived class has more documentation specific to its behavior. + /// The derived class has more documentation + /// specific to its behavior. /// - /// Note that this implementation, as well as the nested derived class, does - /// almost no validation of parameters or guard against misuse than we would like from, say, implementations of - /// the same classes within the ML.NET codebase. + /// Note that this implementation, as well as the nested + /// derived class, does almost no validation + /// of parameters or guard against misuse than we would like from, say, + /// implementations of the same classes within the ML.NET codebase. /// private sealed class InputObjectDataView : IDataView { @@ -131,52 +161,76 @@ public InputObjectDataView(IEnumerable data) public long? GetRowCount() => null; - public DataViewRowCursor GetRowCursor(IEnumerable columnsNeeded, Random rand = null) - => new Cursor(this, columnsNeeded.Any(c => c.Index == 0), columnsNeeded.Any(c => c.Index == 1)); + public DataViewRowCursor GetRowCursor( + IEnumerable columnsNeeded, + Random rand = null) + + => new Cursor(this, columnsNeeded.Any(c => c.Index == 0), + columnsNeeded.Any(c => c.Index == 1)); + + public DataViewRowCursor[] GetRowCursorSet( + IEnumerable columnsNeeded, int n, + Random rand = null) - public DataViewRowCursor[] GetRowCursorSet(IEnumerable columnsNeeded, int n, Random rand = null) => new[] { GetRowCursor(columnsNeeded, rand) }; /// - /// Having this be a private sealed nested class follows the typical pattern: in most - /// implementations, the cursor instance is almost always that. The only "common" - /// exceptions to this tendency are those implementations that are such thin wrappings of existing - /// without even bothering to change the schema. + /// Having this be a private sealed nested class follows the typical + /// pattern: in most implementations, the cursor + /// instance is almost always that. The only "common" exceptions to this + /// tendency are those implementations that are such thin wrappings of + /// existing without even bothering to change + /// the schema. /// - /// On the subject of schema, note that there is an expectation that the object is - /// reference equal to the object that created this cursor, as we see here. + /// On the subject of schema, note that there is an expectation that + /// the object is reference equal to the + /// object that created this cursor, as + /// we see here. /// - /// Note that returns 0. As described in the documentation of that property, that - /// is meant to facilitate the reconciliation of the partitioning of the data in the case where multiple - /// cursors are returned from - /// , but since only one is - /// ever returned from the implementation, this behavior is appropriate. + /// Note that returns 0. As described in the + /// documentation of that property, that is meant to facilitate the + /// reconciliation of the partitioning of the data in the case where + /// multiple cursors are returned from + /// , + /// but since only one is ever returned from the implementation, this + /// behavior is appropriate. /// - /// Similarly, since it is impossible to have a shuffled cursor or a cursor set, it is sufficient for the - /// implementation to return a simple ID based on the position. If, however, this - /// had been something built on, hypothetically, an or some other such structure, and - /// shuffling and partitioning was available, an ID based on the index of whatever item was being returned - /// would be appropriate. + /// Similarly, since it is impossible to have a shuffled cursor or a + /// cursor set, it is sufficient for the + /// implementation to return a simple ID based on the position. If, + /// however, this had been something built on, hypothetically, an + /// or some other such structure, and shuffling + /// and partitioning was available, an ID based on the index of whatever + /// item was being returned would be appropriate. /// - /// Note the usage of the parameters on the - /// implementations. This is most valuable in the case of buffer sharing for , but - /// we still of course have to deal with it here. + /// Note the usage of the parameters on the + /// implementations. This is most + /// valuable in the case of buffer sharing for , + /// but we still of course have to deal with it here. /// - /// Note also that we spend a considerable amount of effort to not make the - /// and - /// methods correctly reflect what was asked for from - /// the - /// method that was used to create this method. In this particular case, the point is somewhat moot: this - /// mechanism exists to enable lazy evaluation, but since this cursor is implemented to wrap an - /// which has no concept of lazy evaluation, there is no real practical benefit - /// to doing this. However, it is best of course to illustrate the general principle for the sake of the - /// example. + /// Note also that we spend a considerable amount of effort to not make + /// the and + /// methods + /// correctly reflect what was asked for from the + /// method that was used + /// to create this method. In this particular case, the point is + /// somewhat moot: this mechanism exists to enable lazy evaluation, + /// but since this cursor is implemented to wrap an + /// which has no concept of lazy + /// evaluation, there is no real practical benefit to doing this. + /// However, it is best of course to illustrate the general principle + /// for the sake of the example. /// - /// Even in this simple form, we see the reason why - /// is beneficial: the implementations themselves are simple to the point - /// where their operation is dwarfed by the simple acts of casting and validation checking one sees in - /// . In this way we only pay the cost of validation - /// and casting once, not every time we get a value. + /// Even in this simple form, we see the reason why + /// is + /// beneficial: the implementations + /// themselves are simple to the point where their operation is dwarfed + /// by the simple acts of casting and validation checking one sees in + /// . In this way + /// we only pay the cost of validation and casting once, not every time + /// we get a value. /// private sealed class Cursor : DataViewRowCursor { @@ -189,15 +243,22 @@ private sealed class Cursor : DataViewRowCursor public override long Batch => 0; public override DataViewSchema Schema { get; } - public Cursor(InputObjectDataView parent, bool wantsLabel, bool wantsText) + public Cursor(InputObjectDataView parent, bool wantsLabel, + bool wantsText) + { Schema = parent.Schema; _position = -1; _enumerator = parent._data.GetEnumerator(); _getters = new Delegate[] { - wantsLabel ? (ValueGetter)LabelGetterImplementation : null, - wantsText ? (ValueGetter>)TextGetterImplementation : null + wantsLabel ? + (ValueGetter)LabelGetterImplementation : null, + + wantsText ? + (ValueGetter>) + TextGetterImplementation : null + }; } @@ -217,13 +278,17 @@ protected override void Dispose(bool disposing) private void LabelGetterImplementation(ref bool value) => value = _enumerator.Current.Label; - private void TextGetterImplementation(ref ReadOnlyMemory value) + private void TextGetterImplementation( + ref ReadOnlyMemory value) + => value = _enumerator.Current.Text.AsMemory(); private void IdGetterImplementation(ref DataViewRowId id) => id = new DataViewRowId((ulong)_position, 0); - public override ValueGetter GetGetter(DataViewSchema.Column column) + public override ValueGetter GetGetter( + DataViewSchema.Column column) + { if (!IsColumnActive(column)) throw new ArgumentOutOfRangeException(nameof(column)); diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/TensorFlow/ImageClassification.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/TensorFlow/ImageClassification.cs index 3fa852fb44..611eb501c1 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/TensorFlow/ImageClassification.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/TensorFlow/ImageClassification.cs @@ -23,7 +23,9 @@ public static void Example() if (!File.Exists(modelLocation)) { modelLocation = Download(@"https://storage.googleapis.com/download.tensorflow.org/models/tflite_11_05_08/resnet_v2_101.tgz", @"resnet_v2_101_299_frozen.tgz"); - Unzip(Path.Join(Directory.GetCurrentDirectory(), modelLocation), Directory.GetCurrentDirectory()); + Unzip(Path.Join(Directory.GetCurrentDirectory(), modelLocation), + Directory.GetCurrentDirectory()); + modelLocation = "resnet_v2_101_299_frozen.pb"; } @@ -32,7 +34,8 @@ public static void Example() var idv = mlContext.Data.LoadFromEnumerable(data); // Create a ML pipeline. - var pipeline = mlContext.Model.LoadTensorFlowModel(modelLocation).ScoreTensorFlowModel( + var pipeline = mlContext.Model.LoadTensorFlowModel(modelLocation) + .ScoreTensorFlowModel( new[] { nameof(OutputScores.output) }, new[] { nameof(TensorData.input) }, addBatchDimensionInput: true); @@ -41,15 +44,18 @@ public static void Example() var transformedValues = estimator.Transform(idv); // Retrieve model scores. - var outScores = mlContext.Data.CreateEnumerable(transformedValues, reuseRowObject: false); + var outScores = mlContext.Data.CreateEnumerable( + transformedValues, reuseRowObject: false); - // Display scores. (for the sake of brevity we display scores of the first 3 classes) + // Display scores. (for the sake of brevity we display scores of the + // first 3 classes) foreach (var prediction in outScores) { int numClasses = 0; foreach (var classScore in prediction.output.Take(3)) { - Console.WriteLine($"Class #{numClasses++} score = {classScore}"); + Console.WriteLine( + $"Class #{numClasses++} score = {classScore}"); } Console.WriteLine(new string('-', 10)); } @@ -72,7 +78,8 @@ public static void Example() /// /// A class to hold sample tensor data. - /// Member name should match the inputs that the model expects (in this case, input). + /// Member name should match the inputs that the model expects (in this + /// case, input). /// public class TensorData { @@ -86,9 +93,13 @@ public class TensorData public static TensorData[] GetTensorData() { // This can be any numerical data. Assume image pixel values. - var image1 = Enumerable.Range(0, inputSize).Select(x => (float)x / inputSize).ToArray(); - var image2 = Enumerable.Range(0, inputSize).Select(x => (float)(x + 10000) / inputSize).ToArray(); - return new TensorData[] { new TensorData() { input = image1 }, new TensorData() { input = image2 } }; + var image1 = Enumerable.Range(0, inputSize).Select( + x => (float)x / inputSize).ToArray(); + + var image2 = Enumerable.Range(0, inputSize).Select( + x => (float)(x + 10000) / inputSize).ToArray(); + return new TensorData[] { new TensorData() { input = image1 }, + new TensorData() { input = image2 } }; } /// @@ -110,7 +121,8 @@ private static string Download(string baseGitPath, string dataFile) } /// - /// Taken from https://github.com/icsharpcode/SharpZipLib/wiki/GZip-and-Tar-Samples. + /// Taken from + /// https://github.com/icsharpcode/SharpZipLib/wiki/GZip-and-Tar-Samples. /// private static void Unzip(string path, string targetDir) { @@ -125,4 +137,4 @@ private static void Unzip(string path, string targetDir) inStream.Close(); } } -} \ No newline at end of file +} diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/TensorFlow/TextClassification.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/TensorFlow/TextClassification.cs index 808a8eb347..4f09cf55db 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/TensorFlow/TextClassification.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/TensorFlow/TextClassification.cs @@ -13,27 +13,36 @@ public static class TextClassification /// public static void Example() { - string modelLocation = Microsoft.ML.SamplesUtils.DatasetUtils.DownloadTensorFlowSentimentModel(); + string modelLocation = Microsoft.ML.SamplesUtils.DatasetUtils + .DownloadTensorFlowSentimentModel(); var mlContext = new MLContext(); var data = new[] { new IMDBSentiment() { - Sentiment_Text = "this film was just brilliant casting location scenery story direction " + - "everyone's really suited the part they played and you could just imagine being there robert " + - "is an amazing actor and now the same being director father came from the same scottish " + - "island as myself so i loved the fact there was a real connection with this film the witty " + - "remarks throughout the film were great it was just brilliant so much that i bought the " + - "film as soon as it was released for and would recommend it to everyone to watch and the " + - "fly fishing was amazing really cried at the end it was so sad and you know what they say " + - "if you cry at a film it must have been good and this definitely was also to the two " + - "little boy's that played the of norman and paul they were just brilliant children are " + - "often left out of the list i think because the stars that play them all grown up are " + - "such a big profile for the whole film but these children are amazing and should be praised " + - "for what they have done don't you think the whole story was so lovely because it was true " + - "and was someone's life after all that was shared with us all" } }; + Sentiment_Text = "this film was just brilliant casting location " + + "scenery story direction everyone's really suited the part they " + + "played and you could just imagine being there robert is an " + + "amazing actor and now the same being director father came from " + + "the same scottish island as myself so i loved the fact there " + + "was a real connection with this film the witty remarks " + + "throughout the film were great it was just brilliant so much " + + "that i bought the film as soon as it was released for and " + + "would recommend it to everyone to watch and the fly fishing was " + + "amazing really cried at the end it was so sad and you know what " + + "they say if you cry at a film it must have been good and this " + + "definitely was also to the two little boy's that played the of " + + "norman and paul they were just brilliant children are often " + + "left out of the list i think because the stars that play them " + + "all grown up are such a big profile for the whole film but " + + "these children are amazing and should be praised for what " + + "they have done don't you think the whole story was so lovely" + + "because it was true and was someone's life after all that was" + + "shared with us all" } }; var dataView = mlContext.Data.LoadFromEnumerable(data); // This is the dictionary to convert words into the integer indexes. - var lookupMap = mlContext.Data.LoadFromTextFile(Path.Combine(modelLocation, "imdb_word_index.csv"), + var lookupMap = mlContext.Data.LoadFromTextFile(Path.Combine( + modelLocation, "imdb_word_index.csv"), + columns: new[] { new TextLoader.Column("Words", DataKind.String, 0), @@ -43,25 +52,38 @@ public static void Example() ); // Load the TensorFlow model once. - // - Use it for quering the schema for input and output in the model + // - Use it for quering the schema for input and output in the + // model // - Use it for prediction in the pipeline. - var tensorFlowModel = mlContext.Model.LoadTensorFlowModel(modelLocation); + var tensorFlowModel = mlContext.Model.LoadTensorFlowModel( + modelLocation); var schema = tensorFlowModel.GetModelSchema(); var featuresType = (VectorDataViewType)schema["Features"].Type; - Console.WriteLine("Name: {0}, Type: {1}, Shape: (-1, {2})", "Features", featuresType.ItemType.RawType, featuresType.Dimensions[0]); - var predictionType = (VectorDataViewType)schema["Prediction/Softmax"].Type; - Console.WriteLine("Name: {0}, Type: {1}, Shape: (-1, {2})", "Prediction/Softmax", predictionType.ItemType.RawType, predictionType.Dimensions[0]); - - // The model expects the input feature vector to be a fixed length vector. - // In this sample, CustomMappingEstimator is used to resize variable length vector to fixed length vector. + Console.WriteLine("Name: {0}, Type: {1}, Shape: (-1, {2})", "Features", + featuresType.ItemType.RawType, featuresType.Dimensions[0]); + + var predictionType = (VectorDataViewType)schema["Prediction/Softmax"] + .Type; + Console.WriteLine("Name: {0}, Type: {1}, Shape: (-1, {2})", + "Prediction/Softmax", predictionType.ItemType.RawType, + predictionType.Dimensions[0]); + + // The model expects the input feature vector to be a fixed length + // vector. + // In this sample, CustomMappingEstimator is used to resize variable + // length vector to fixed length vector. // The following ML.NET pipeline // 1. tokenzies the string into words, - // 2. maps each word to an integer which is an index in the dictionary ('lookupMap'), - // 3. Resizes the integer vector to a fixed length vector using CustomMappingEstimator ('ResizeFeaturesAction') + // 2. maps each word to an integer which is an index in the + // dictionary ('lookupMap'), + // 3. Resizes the integer vector to a fixed length vector using + // CustomMappingEstimator ('ResizeFeaturesAction') // 4. Passes the data to TensorFlow for scoring. - // 5. Retreives the 'Prediction' from TensorFlow and put it into ML.NET Pipeline + // 5. Retreives the 'Prediction' from TensorFlow and put it into + // ML.NET Pipeline - Action ResizeFeaturesAction = (i, j) => + Action ResizeFeaturesAction = + (i, j) => { j.Sentiment_Text = i.Sentiment_Text; var features = i.VariableLengthFeatures; @@ -69,23 +91,40 @@ public static void Example() j.Features = features; }; - var model = mlContext.Transforms.Text.TokenizeIntoWords("TokenizedWords", "Sentiment_Text") - .Append(mlContext.Transforms.Conversion.MapValue("VariableLengthFeatures", lookupMap, - lookupMap.Schema["Words"], lookupMap.Schema["Ids"], "TokenizedWords")) - .Append(mlContext.Transforms.CustomMapping(ResizeFeaturesAction, "Resize")) - .Append(tensorFlowModel.ScoreTensorFlowModel("Prediction/Softmax", "Features")) - .Append(mlContext.Transforms.CopyColumns("Prediction", "Prediction/Softmax")) + var model = + mlContext.Transforms.Text.TokenizeIntoWords( + "TokenizedWords", + "Sentiment_Text") + .Append(mlContext.Transforms.Conversion.MapValue( + "VariableLengthFeatures", + lookupMap, + lookupMap.Schema["Words"], + lookupMap.Schema["Ids"], + "TokenizedWords")) + .Append(mlContext.Transforms.CustomMapping( + ResizeFeaturesAction, + "Resize")) + .Append(tensorFlowModel.ScoreTensorFlowModel( + "Prediction/Softmax", + "Features")) + .Append(mlContext.Transforms.CopyColumns( + "Prediction", + "Prediction/Softmax")) .Fit(dataView); - var engine = mlContext.Model.CreatePredictionEngine(model); + var engine = mlContext.Model.CreatePredictionEngine(model); // Predict with TensorFlow pipeline. var prediction = engine.Predict(data[0]); - Console.WriteLine("Number of classes: {0}", prediction.Prediction.Length); - Console.WriteLine("Is sentiment/review positive? {0}", prediction.Prediction[1] > 0.5 ? "Yes." : "No."); - Console.WriteLine("Prediction Confidence: {0}", prediction.Prediction[1].ToString("0.00")); + Console.WriteLine("Number of classes: {0}", prediction.Prediction + .Length); + Console.WriteLine("Is sentiment/review positive? {0}", prediction + .Prediction[1] > 0.5 ? "Yes." : "No."); + Console.WriteLine("Prediction Confidence: {0}", prediction.Prediction[1] + .ToString("0.00")); - /////////////////////////////////// Expected output /////////////////////////////////// + ///////////////////////////// Expected output ////////////////////////// // // Name: Features, Type: System.Int32, Shape: (-1, 600) // Name: Prediction/Softmax, Type: System.Single, Shape: (-1, 2) @@ -105,8 +144,9 @@ public class IMDBSentiment /// /// This is a variable length vector designated by VectorType attribute. - /// Variable length vectors are produced by applying operations such as 'TokenizeWords' on strings - /// resulting in vectors of tokens of variable lengths. + /// Variable length vectors are produced by applying operations such as + /// 'TokenizeWords' on strings resulting in vectors of tokens of + /// variable lengths. /// [VectorType] public int[] VariableLengthFeatures { get; set; } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/TextTransform.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/TextTransform.cs index 5c49ac6bbd..db8f7c4961 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/TextTransform.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/TextTransform.cs @@ -10,16 +10,22 @@ public static class TextTransform { public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var ml = new MLContext(); // Get a small dataset as an IEnumerable and convert to IDataView. // Get a small dataset as an IEnumerable and convert to IDataView. var data = new List() { - new SampleSentimentData { Sentiment = true, SentimentText = "Best game I've ever played." }, - new SampleSentimentData { Sentiment = false, SentimentText = "==RUDE== Dude, 2" }, - new SampleSentimentData { Sentiment = true, SentimentText = "Until the next game, this is the best Xbox game!" } }; + new SampleSentimentData { Sentiment = true, + SentimentText = "Best game I've ever played." }, + + new SampleSentimentData { Sentiment = false, + SentimentText = "==RUDE== Dude, 2" }, + + new SampleSentimentData { Sentiment = true, + SentimentText = "Until the next game," + + "this is the best Xbox game!" } }; // Convert IEnumerable to IDataView. var trainData = ml.Data.LoadFromEnumerable(data); @@ -31,29 +37,47 @@ public static void Example() // false ==RUDE== Dude, 2. // true Until the next game, this is the best Xbox game! - // A pipeline for featurization of the "SentimentText" column, and placing the output in a new column named "DefaultTextFeatures" - // The pipeline uses the default settings to featurize. + // A pipeline for featurization of the "SentimentText" column, and + // placing the output in a new column named "DefaultTextFeatures". The + // pipeline uses the default settings to featurize. string defaultColumnName = "DefaultTextFeatures"; - var default_pipeline = ml.Transforms.Text.FeaturizeText(defaultColumnName , "SentimentText"); + var default_pipeline = ml.Transforms.Text + .FeaturizeText(defaultColumnName , "SentimentText"); - // Another pipeline, that customizes the advanced settings of the FeaturizeText transformer. + // Another pipeline, that customizes the advanced settings of the + // FeaturizeText transformer. string customizedColumnName = "CustomizedTextFeatures"; - var customized_pipeline = ml.Transforms.Text.FeaturizeText(customizedColumnName, new TextFeaturizingEstimator.Options + var customized_pipeline = ml.Transforms.Text + .FeaturizeText(customizedColumnName, + new TextFeaturizingEstimator.Options + { KeepPunctuations = false, KeepNumbers = false, OutputTokensColumnName = "OutputTokens", - StopWordsRemoverOptions = new StopWordsRemovingEstimator.Options() { Language = TextFeaturizingEstimator.Language.English }, // supports English, French, German, Dutch, Italian, Spanish, Japanese + StopWordsRemoverOptions = + new StopWordsRemovingEstimator.Options() { + Language = TextFeaturizingEstimator.Language.English }, + // supports English, French, German, Dutch, Italian, Spanish, + // Japanese + }, "SentimentText"); // The transformed data for both pipelines. - var transformedData_default = default_pipeline.Fit(trainData).Transform(trainData); - var transformedData_customized = customized_pipeline.Fit(trainData).Transform(trainData); + var transformedData_default = default_pipeline.Fit(trainData) + .Transform(trainData); + + var transformedData_customized = customized_pipeline.Fit(trainData) + .Transform(trainData); // Small helper to print the text inside the columns, in the console. - Action>> printHelper = (columnName, column) => + Action>> printHelper = (columnName, + column) => + { - Console.WriteLine($"{columnName} column obtained post-transformation."); + Console.WriteLine( + $"{columnName} column obtained post-transformation."); + foreach (var featureRow in column) { foreach (var value in featureRow.GetValues()) @@ -61,11 +85,17 @@ public static void Example() Console.WriteLine(""); } - Console.WriteLine("==================================================="); + Console.WriteLine( + "==================================================="); + }; - // Preview of the DefaultTextFeatures column obtained after processing the input. - var defaultColumn = transformedData_default.GetColumn>(transformedData_default.Schema[defaultColumnName]); + // Preview of the DefaultTextFeatures column obtained after processing + // the input. + var defaultColumn = transformedData_default + .GetColumn>(transformedData_default + .Schema[defaultColumnName]); + printHelper(defaultColumnName, defaultColumn); // DefaultTextFeatures column obtained post-transformation. @@ -74,8 +104,12 @@ public static void Example() // 0.2357023 0.2357023 0.2357023 0.2357023 0.4714046 0.2357023 0.2357023 0.2357023 0.2357023 0.2357023 0.2357023 0.2357023 0.2357023 0.2357023 0.2357023 0.5773503 0.5773503 0.5773503 0.1924501 0.1924501 0.1924501 0.1924501 0.1924501 0.1924501 0.1924501 0.1924501 0.1924501 0.4472136 0.4472136 0.4472136 0.4472136 0.4472136 // 0 0.1230915 0.1230915 0.1230915 0.1230915 0.246183 0.246183 0.246183 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0.1230915 0 0 0.1230915 0.1230915 0.1230915 0.1230915 0.1230915 0.1230915 0.3692745 0.246183 0.246183 0.1230915 0.1230915 0.1230915 0.1230915 0.1230915 0.1230915 0.1230915 0.1230915 0.1230915 0.246183 0.1230915 0.1230915 0.1230915 0.1230915 0.1230915 0.1230915 0.1230915 0.1230915 0.1230915 0.1230915 0.1230915 0.1230915 0.1230915 0.2886751 0 0 0 0 0 0 0 0.2886751 0.5773503 0.2886751 0.2886751 0.2886751 0.2886751 0.2886751 0.2886751 - // Preview of the CustomizedTextFeatures column obtained after processing the input. - var customizedColumn = transformedData_customized.GetColumn>(transformedData_customized.Schema[customizedColumnName]); + // Preview of the CustomizedTextFeatures column obtained after + // processing the input. + var customizedColumn = transformedData_customized + .GetColumn>(transformedData_customized + .Schema[customizedColumnName]); + printHelper(customizedColumnName, customizedColumn); // CustomizedTextFeatures column obtained post-transformation. @@ -86,7 +120,8 @@ public static void Example() } /// - /// A dataset that contains a tweet and the sentiment assigned to that tweet: 0 - negative and 1 - positive sentiment. + /// A dataset that contains a tweet and the sentiment assigned to that + /// tweet: 0 - negative and 1 - positive sentiment. /// public class SampleSentimentData { diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/AnomalyDetection/RandomizedPcaSample.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/AnomalyDetection/RandomizedPcaSample.cs index 62287c4ba1..adccee9e81 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/AnomalyDetection/RandomizedPcaSample.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/AnomalyDetection/RandomizedPcaSample.cs @@ -10,9 +10,10 @@ public static class RandomizedPcaSample { public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for except + // ion tracking and logging, as a catalog of available operations and as + // the source of randomness. Setting the seed to a fixed number in this + // example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Training data. @@ -26,11 +27,15 @@ public static void Example() new DataPoint(){ Features = new float[3] {-100, 50, -100} } }; - // Convert the List to IDataView, a consumble format to ML.NET functions. + // Convert the List to IDataView, a consumble format to + // ML.NET functions. var data = mlContext.Data.LoadFromEnumerable(samples); - // Create an anomaly detector. Its underlying algorithm is randomized PCA. - var pipeline = mlContext.AnomalyDetection.Trainers.RandomizedPca(featureColumnName: nameof(DataPoint.Features), rank: 1, ensureZeroMean: false); + // Create an anomaly detector. Its underlying algorithm is randomized + // PCA. + var pipeline = mlContext.AnomalyDetection.Trainers.RandomizedPca( + featureColumnName: nameof(DataPoint.Features), rank: 1, + ensureZeroMean: false); // Train the anomaly detector. var model = pipeline.Fit(data); @@ -39,7 +44,8 @@ public static void Example() var transformed = model.Transform(data); // Read ML.NET predictions into IEnumerable. - var results = mlContext.Data.CreateEnumerable(transformed, reuseRowObject: false).ToList(); + var results = mlContext.Data.CreateEnumerable(transformed, + reuseRowObject: false).ToList(); // Let's go through all predictions. for (int i = 0; i < samples.Count; ++i) @@ -52,12 +58,14 @@ public static void Example() if (result.PredictedLabel) // The i-th sample is predicted as an inlier. - Console.WriteLine("The {0}-th example with features [{1}] is an inlier with a score of being inlier {2}", - i, featuresInText, result.Score); + Console.WriteLine("The {0}-th example with features [{1}]" + + "is an inlier with a score of being inlier {2}", i, + featuresInText, result.Score); else // The i-th sample is predicted as an outlier. - Console.WriteLine("The {0}-th example with features [{1}] is an outlier with a score of being inlier {2}", - i, featuresInText, result.Score); + Console.WriteLine("The {0}-th example with features [{1}] is" + + "an outlier with a score of being inlier {2}", i, + featuresInText, result.Score); } // Lines printed out should be // The 0 - th example with features[1, 0, 0] is an inlier with a score of being inlier 0.7453707 @@ -68,7 +76,8 @@ public static void Example() // The 5 - th example with features[-100, 50, -100] is an outlier with a score of being inlier 0 } - // Example with 3 feature values. A training data set is a collection of such examples. + // Example with 3 feature values. A training data set is a collection of + // such examples. private class DataPoint { [VectorType(3)] diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/AnomalyDetection/RandomizedPcaSampleWithOptions.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/AnomalyDetection/RandomizedPcaSampleWithOptions.cs index 02b725f7ce..7a281880b7 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/AnomalyDetection/RandomizedPcaSampleWithOptions.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/AnomalyDetection/RandomizedPcaSampleWithOptions.cs @@ -10,9 +10,10 @@ public static class RandomizedPcaSampleWithOptions { public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Training data. @@ -26,7 +27,8 @@ public static void Example() new DataPoint(){ Features = new float[3] {-100, 50, -100} } }; - // Convert the List to IDataView, a consumble format to ML.NET functions. + // Convert the List to IDataView, a consumble format to + // ML.NET functions. var data = mlContext.Data.LoadFromEnumerable(samples); var options = new Microsoft.ML.Trainers.RandomizedPcaTrainer.Options() @@ -36,8 +38,10 @@ public static void Example() Seed = 10, }; - // Create an anomaly detector. Its underlying algorithm is randomized PCA. - var pipeline = mlContext.AnomalyDetection.Trainers.RandomizedPca(options); + // Create an anomaly detector. Its underlying algorithm is randomized + // PCA. + var pipeline = mlContext.AnomalyDetection.Trainers.RandomizedPca( + options); // Train the anomaly detector. var model = pipeline.Fit(data); @@ -46,7 +50,8 @@ public static void Example() var transformed = model.Transform(data); // Read ML.NET predictions into IEnumerable. - var results = mlContext.Data.CreateEnumerable(transformed, reuseRowObject: false).ToList(); + var results = mlContext.Data.CreateEnumerable(transformed, + reuseRowObject: false).ToList(); // Let's go through all predictions. for (int i = 0; i < samples.Count; ++i) @@ -59,11 +64,13 @@ public static void Example() if (result.PredictedLabel) // The i-th sample is predicted as an inlier. - Console.WriteLine("The {0}-th example with features [{1}] is an inlier with a score of being inlier {2}", - i, featuresInText, result.Score); + Console.WriteLine("The {0}-th example with features [{1}] is" + + "an inlier with a score of being inlier {2}", i, + featuresInText, result.Score); else // The i-th sample is predicted as an outlier. - Console.WriteLine("The {0}-th example with features [{1}] is an outlier with a score of being inlier {2}", + Console.WriteLine("The {0}-th example with features [{1}] is" + + "an outlier with a score of being inlier {2}", i, featuresInText, result.Score); } // Lines printed out should be @@ -75,7 +82,8 @@ public static void Example() // The 5 - th example with features[-100, 50, -100] is an outlier with a score of being inlier 0 } - // Example with 3 feature values. A training data set is a collection of such examples. + // Example with 3 feature values. A training data set is a collection of + // such examples. private class DataPoint { [VectorType(3)] diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/AveragedPerceptron.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/AveragedPerceptron.cs index 1506fc1abd..fd24a32b39 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/AveragedPerceptron.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/AveragedPerceptron.cs @@ -10,35 +10,43 @@ public static class AveragedPerceptron { public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); // Define the trainer. - var pipeline = mlContext.BinaryClassification.Trainers.AveragedPerceptron(); + var pipeline = mlContext.BinaryClassification.Trainers + .AveragedPerceptron(); // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(500, seed:123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data + .LoadFromEnumerable(GenerateRandomDataPoints(500, seed:123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(transformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data + .CreateEnumerable(transformedTestData, + reuseRowObject: false).ToList(); // Print 5 predictions. foreach (var p in predictions.Take(5)) - Console.WriteLine($"Label: {p.Label}, Prediction: {p.PredictedLabel}"); + Console.WriteLine($"Label: {p.Label}, " + + $"Prediction: {p.PredictedLabel}"); // Expected output: // Label: True, Prediction: True @@ -48,7 +56,9 @@ public static void Example() // Label: False, Prediction: False // Evaluate the overall metrics. - var metrics = mlContext.BinaryClassification.EvaluateNonCalibrated(transformedTestData); + var metrics = mlContext.BinaryClassification + .EvaluateNonCalibrated(transformedTestData); + PrintMetrics(metrics); // Expected output: @@ -71,7 +81,9 @@ public static void Example() // Precision || 0.7402 | 0.7061 | } - private static IEnumerable GenerateRandomDataPoints(int count, int seed=0) + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) + { var random = new Random(seed); float randomFloat() => (float)random.NextDouble(); @@ -82,13 +94,18 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se { Label = label, // Create random features that are correlated with the label. - // For data points with false label, the feature values are slightly increased by adding a constant. - Features = Enumerable.Repeat(label, 50).Select(x => x ? randomFloat() : randomFloat() + 0.1f).ToArray() + // For data points with false label, the feature values are + // slightly increased by adding a constant. + Features = Enumerable.Repeat(label, 50) + .Select(x => x ? randomFloat() : randomFloat() + + 0.1f).ToArray() + }; } } - // Example with label and 50 feature values. A data set is a collection of such examples. + // Example with label and 50 feature values. A data set is a collection of + // such examples. private class DataPoint { public bool Label { get; set; } @@ -111,11 +128,16 @@ private static void PrintMetrics(BinaryClassificationMetrics metrics) Console.WriteLine($"Accuracy: {metrics.Accuracy:F2}"); Console.WriteLine($"AUC: {metrics.AreaUnderRocCurve:F2}"); Console.WriteLine($"F1 Score: {metrics.F1Score:F2}"); - Console.WriteLine($"Negative Precision: {metrics.NegativePrecision:F2}"); + Console.WriteLine($"Negative Precision: " + + $"{metrics.NegativePrecision:F2}"); + Console.WriteLine($"Negative Recall: {metrics.NegativeRecall:F2}"); - Console.WriteLine($"Positive Precision: {metrics.PositivePrecision:F2}"); + Console.WriteLine($"Positive Precision: " + + $"{metrics.PositivePrecision:F2}"); + Console.WriteLine($"Positive Recall: {metrics.PositiveRecall:F2}\n"); Console.WriteLine(metrics.ConfusionMatrix.GetFormattedConfusionTable()); } } } + diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/AveragedPerceptronWithOptions.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/AveragedPerceptronWithOptions.cs index 748e37be8e..8f31f474e4 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/AveragedPerceptronWithOptions.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/AveragedPerceptronWithOptions.cs @@ -11,15 +11,17 @@ public static class AveragedPerceptronWithOptions { public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); // Define trainer options. @@ -33,23 +35,29 @@ public static void Example() }; // Define the trainer. - var pipeline = mlContext.BinaryClassification.Trainers.AveragedPerceptron(options); + var pipeline = mlContext.BinaryClassification.Trainers + .AveragedPerceptron(options); // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(500, seed:123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data + .LoadFromEnumerable(GenerateRandomDataPoints(500, seed:123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(transformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data + .CreateEnumerable(transformedTestData, + reuseRowObject: false).ToList(); // Print 5 predictions. foreach (var p in predictions.Take(5)) - Console.WriteLine($"Label: {p.Label}, Prediction: {p.PredictedLabel}"); + Console.WriteLine($"Label: {p.Label}, " + + $"Prediction: {p.PredictedLabel}"); // Expected output: // Label: True, Prediction: True @@ -59,7 +67,9 @@ public static void Example() // Label: False, Prediction: False // Evaluate the overall metrics. - var metrics = mlContext.BinaryClassification.EvaluateNonCalibrated(transformedTestData); + var metrics = mlContext.BinaryClassification + .EvaluateNonCalibrated(transformedTestData); + PrintMetrics(metrics); // Expected output: @@ -82,7 +92,9 @@ public static void Example() // Precision || 0.7402 | 0.7061 | } - private static IEnumerable GenerateRandomDataPoints(int count, int seed=0) + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) + { var random = new Random(seed); float randomFloat() => (float)random.NextDouble(); @@ -93,13 +105,18 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se { Label = label, // Create random features that are correlated with the label. - // For data points with false label, the feature values are slightly increased by adding a constant. - Features = Enumerable.Repeat(label, 50).Select(x => x ? randomFloat() : randomFloat() + 0.1f).ToArray() + // For data points with false label, the feature values are + // slightly increased by adding a constant. + Features = Enumerable.Repeat(label, 50) + .Select(x => x ? randomFloat() : randomFloat() + + 0.1f).ToArray() + }; } } - // Example with label and 50 feature values. A data set is a collection of such examples. + // Example with label and 50 feature values. A data set is a collection of + // such examples. private class DataPoint { public bool Label { get; set; } @@ -122,11 +139,16 @@ private static void PrintMetrics(BinaryClassificationMetrics metrics) Console.WriteLine($"Accuracy: {metrics.Accuracy:F2}"); Console.WriteLine($"AUC: {metrics.AreaUnderRocCurve:F2}"); Console.WriteLine($"F1 Score: {metrics.F1Score:F2}"); - Console.WriteLine($"Negative Precision: {metrics.NegativePrecision:F2}"); + Console.WriteLine($"Negative Precision: " + + $"{metrics.NegativePrecision:F2}"); + Console.WriteLine($"Negative Recall: {metrics.NegativeRecall:F2}"); - Console.WriteLine($"Positive Precision: {metrics.PositivePrecision:F2}"); + Console.WriteLine($"Positive Precision: " + + $"{metrics.PositivePrecision:F2}"); + Console.WriteLine($"Positive Recall: {metrics.PositiveRecall:F2}\n"); Console.WriteLine(metrics.ConfusionMatrix.GetFormattedConfusionTable()); } } } + diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/BinaryClassification.ttinclude b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/BinaryClassification.ttinclude index 6bc5660d46..72dc7cc111 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/BinaryClassification.ttinclude +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/BinaryClassification.ttinclude @@ -13,63 +13,79 @@ namespace Samples.Dynamic.Trainers.BinaryClassification {<#=Comments#> public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); <# if (CacheData) { #> - // ML.NET doesn't cache data set by default. Therefore, if one reads a data set from a file and accesses it many times, - // it can be slow due to expensive featurization and disk operations. When the considered data can fit into memory, - // a solution is to cache the data in memory. Caching is especially helpful when working with iterative algorithms + // ML.NET doesn't cache data set by default. Therefore, if one reads a + // data set from a file and accesses it many times, it can be slow due + // to expensive featurization and disk operations. When the considered + // data can fit into memory, a solution is to cache the data in memory. + // Caching is especially helpful when working with iterative algorithms // which needs many data passes. trainingData = mlContext.Data.Cache(trainingData); <# } #> <# if (TrainerOptions == null) { #> // Define the trainer. - var pipeline = mlContext.BinaryClassification.Trainers.<#=Trainer#>(); + var pipeline = mlContext.BinaryClassification.Trainers + .<#=Trainer#>(); <# } else { #> // Define trainer options. var options = new <#=TrainerOptions#>; // Define the trainer. - var pipeline = mlContext.BinaryClassification.Trainers.<#=Trainer#>(options); + var pipeline = mlContext.BinaryClassification.Trainers + .<#=Trainer#>(options); <# } #> // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(500, seed:123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data + .LoadFromEnumerable(GenerateRandomDataPoints(500, seed:123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(transformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data + .CreateEnumerable(transformedTestData, + reuseRowObject: false).ToList(); // Print 5 predictions. foreach (var p in predictions.Take(5)) - Console.WriteLine($"Label: {p.Label}, Prediction: {p.PredictedLabel}"); + Console.WriteLine($"Label: {p.Label}, " + + $"Prediction: {p.PredictedLabel}"); <#=ExpectedOutputPerInstance#> - <# string Evaluator = IsCalibrated ? "Evaluate" : "EvaluateNonCalibrated"; #> + <# string Evaluator = IsCalibrated ? "Evaluate" : + "EvaluateNonCalibrated"; #> // Evaluate the overall metrics. - var metrics = mlContext.BinaryClassification.<#=Evaluator#>(transformedTestData); + var metrics = mlContext.BinaryClassification + .<#=Evaluator#>(transformedTestData); + PrintMetrics(metrics); <#=ExpectedOutput#> } - private static IEnumerable GenerateRandomDataPoints(int count, int seed=0) + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) + { var random = new Random(seed); float randomFloat() => (float)random.NextDouble(); @@ -80,13 +96,18 @@ namespace Samples.Dynamic.Trainers.BinaryClassification { Label = label, // Create random features that are correlated with the label. - // For data points with false label, the feature values are slightly increased by adding a constant. - Features = Enumerable.Repeat(label, 50).Select(x => x ? randomFloat() : randomFloat() + <#=DataSepValue#>).ToArray() + // For data points with false label, the feature values are + // slightly increased by adding a constant. + Features = Enumerable.Repeat(label, 50) + .Select(x => x ? randomFloat() : randomFloat() + + <#=DataSepValue#>).ToArray() + }; } } - // Example with label and 50 feature values. A data set is a collection of such examples. + // Example with label and 50 feature values. A data set is a collection of + // such examples. private class DataPoint { public bool Label { get; set; } @@ -109,11 +130,15 @@ namespace Samples.Dynamic.Trainers.BinaryClassification Console.WriteLine($"Accuracy: {metrics.Accuracy:F2}"); Console.WriteLine($"AUC: {metrics.AreaUnderRocCurve:F2}"); Console.WriteLine($"F1 Score: {metrics.F1Score:F2}"); - Console.WriteLine($"Negative Precision: {metrics.NegativePrecision:F2}"); + Console.WriteLine($"Negative Precision: " + + $"{metrics.NegativePrecision:F2}"); + Console.WriteLine($"Negative Recall: {metrics.NegativeRecall:F2}"); - Console.WriteLine($"Positive Precision: {metrics.PositivePrecision:F2}"); + Console.WriteLine($"Positive Precision: " + + $"{metrics.PositivePrecision:F2}"); + Console.WriteLine($"Positive Recall: {metrics.PositiveRecall:F2}\n"); Console.WriteLine(metrics.ConfusionMatrix.GetFormattedConfusionTable()); } } -} \ No newline at end of file +} diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/Calibrators/FixedPlatt.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/Calibrators/FixedPlatt.cs index 1da43a7790..2b56ef63f0 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/Calibrators/FixedPlatt.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/Calibrators/FixedPlatt.cs @@ -9,26 +9,36 @@ public static class FixedPlatt { public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Download and featurize the dataset. - var data = Microsoft.ML.SamplesUtils.DatasetUtils.LoadFeaturizedAdultDataset(mlContext); + var data = Microsoft.ML.SamplesUtils.DatasetUtils + .LoadFeaturizedAdultDataset(mlContext); + // Leave out 10% of data for testing. - var trainTestData = mlContext.Data.TrainTestSplit(data, testFraction: 0.3); + var trainTestData = mlContext.Data + .TrainTestSplit(data, testFraction: 0.3); - // Create data training pipeline for non calibrated trainer and train Naive calibrator on top of it. - var pipeline = mlContext.BinaryClassification.Trainers.AveragedPerceptron(); + // Create data training pipeline for non calibrated trainer and train + // Naive calibrator on top of it. + var pipeline = mlContext.BinaryClassification.Trainers + .AveragedPerceptron(); - // Fit the pipeline, and get a transformer that knows how to score new data. + // Fit the pipeline, and get a transformer that knows how to score new + // data. var transformer = pipeline.Fit(trainTestData.TrainSet); // Fit this pipeline to the training data. - // Let's score the new data. The score will give us a numerical estimation of the chance that the particular sample - // bears positive sentiment. This estimate is relative to the numbers obtained. + // Let's score the new data. The score will give us a numerical + // estimation of the chance that the particular sample bears positive + // sentiment. This estimate is relative to the numbers obtained. var scoredData = transformer.Transform(trainTestData.TestSet); - var outScores = mlContext.Data.CreateEnumerable(scoredData, reuseRowObject: false); + var outScores = mlContext.Data + .CreateEnumerable(scoredData, reuseRowObject: false); + PrintScore(outScores, 5); // Preview of scoredDataPreview.RowView // Score 4.18144 @@ -37,16 +47,24 @@ public static void Example() // Score -2.554229 // Score 5.36571 - // Let's train a calibrator estimator on this scored dataset. The trained calibrator estimator produces a transformer - // that can transform the scored data by adding a new column names "Probability". - var calibratorEstimator = mlContext.BinaryClassification.Calibrators.Platt(slope: -1f, offset: -0.05f); + // Let's train a calibrator estimator on this scored dataset. The + // trained calibrator estimator produces a transformer that can + // transform the scored data by adding a new column names "Probability". + var calibratorEstimator = mlContext.BinaryClassification.Calibrators + .Platt(slope: -1f, offset: -0.05f); + var calibratorTransformer = calibratorEstimator.Fit(scoredData); - // Transform the scored data with a calibrator transfomer by adding a new column names "Probability". - // This column is a calibrated version of the "Score" column, meaning its values are a valid probability value in the [0, 1] interval - // representing the chance that the respective sample bears positive sentiment. + // Transform the scored data with a calibrator transfomer by adding a + // new column names "Probability". This column is a calibrated version + // of the "Score" column, meaning its values are a valid probability + // value in the [0, 1] interval representing the chance that the + // respective sample bears positive sentiment. var finalData = calibratorTransformer.Transform(scoredData); - var outScoresAndProbabilities = mlContext.Data.CreateEnumerable(finalData, reuseRowObject: false); + var outScoresAndProbabilities = mlContext.Data + .CreateEnumerable(finalData, + reuseRowObject: false); + PrintScoreAndProbability(outScoresAndProbabilities, 5); // Score 4.18144 Probability 0.9856767 // Score -14.10248 Probability 7.890148E-07 @@ -61,10 +79,13 @@ private static void PrintScore(IEnumerable values, int numRows) Console.WriteLine("{0, -10} {1, -10}", "Score", value.Score); } - private static void PrintScoreAndProbability(IEnumerable values, int numRows) + private static void PrintScoreAndProbability( + IEnumerable values, int numRows) + { foreach (var value in values.Take(numRows)) - Console.WriteLine("{0, -10} {1, -10} {2, -10} {3, -10}", "Score", value.Score, "Probability", value.Probability); + Console.WriteLine("{0, -10} {1, -10} {2, -10} {3, -10}", "Score", + value.Score, "Probability", value.Probability); } private class ScoreValue diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/Calibrators/Isotonic.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/Calibrators/Isotonic.cs index 15a3162d4e..1b1b63139e 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/Calibrators/Isotonic.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/Calibrators/Isotonic.cs @@ -9,26 +9,35 @@ public static class Isotonic { public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Download and featurize the dataset. - var data = Microsoft.ML.SamplesUtils.DatasetUtils.LoadFeaturizedAdultDataset(mlContext); + var data = Microsoft.ML.SamplesUtils.DatasetUtils + .LoadFeaturizedAdultDataset(mlContext); + // Leave out 10% of data for testing. - var trainTestData = mlContext.Data.TrainTestSplit(data, testFraction: 0.3); + var trainTestData = mlContext.Data + .TrainTestSplit(data, testFraction: 0.3); - // Create data training pipeline for non calibrated trainer and train Naive calibrator on top of it. - var pipeline = mlContext.BinaryClassification.Trainers.AveragedPerceptron(); + // Create data training pipeline for non calibrated trainer and train + // Naive calibrator on top of it. + var pipeline = mlContext.BinaryClassification.Trainers + .AveragedPerceptron(); - // Fit the pipeline, and get a transformer that knows how to score new data. + // Fit the pipeline, and get a transformer that knows how to score new + // data. var transformer = pipeline.Fit(trainTestData.TrainSet); // Fit this pipeline to the training data. - // Let's score the new data. The score will give us a numerical estimation of the chance that the particular sample - // bears positive sentiment. This estimate is relative to the numbers obtained. + // Let's score the new data. The score will give us a numerical + // estimation of the chance that the particular sample bears positive + // sentiment. This estimate is relative to the numbers obtained. var scoredData = transformer.Transform(trainTestData.TestSet); - var outScores = mlContext.Data.CreateEnumerable(scoredData, reuseRowObject: false); + var outScores = mlContext.Data + .CreateEnumerable(scoredData, reuseRowObject: false); PrintScore(outScores, 5); // Preview of scoredDataPreview.RowView @@ -38,16 +47,24 @@ public static void Example() // Score -2.554229 // Score 5.36571 - // Let's train a calibrator estimator on this scored dataset. The trained calibrator estimator produces a transformer - // that can transform the scored data by adding a new column names "Probability". - var calibratorEstimator = mlContext.BinaryClassification.Calibrators.Isotonic(); + // Let's train a calibrator estimator on this scored dataset. The + // trained calibrator estimator produces a transformer that can + // transform the scored data by adding a new column names "Probability". + var calibratorEstimator = mlContext.BinaryClassification.Calibrators + .Isotonic(); + var calibratorTransformer = calibratorEstimator.Fit(scoredData); - // Transform the scored data with a calibrator transfomer by adding a new column names "Probability". - // This column is a calibrated version of the "Score" column, meaning its values are a valid probability value in the [0, 1] interval - // representing the chance that the respective sample bears positive sentiment. + // Transform the scored data with a calibrator transfomer by adding a + // new column names "Probability". This column is a calibrated version + // of the "Score" column, meaning its values are a valid probability + // value in the [0, 1] interval representing the chance that the + // respective sample bears positive sentiment. var finalData = calibratorTransformer.Transform(scoredData); - var outScoresAndProbabilities = mlContext.Data.CreateEnumerable(finalData, reuseRowObject: false); + var outScoresAndProbabilities = mlContext.Data + .CreateEnumerable(finalData, + reuseRowObject: false); + PrintScoreAndProbability(outScoresAndProbabilities, 5); // Score 4.18144 Probability 0.8 // Score -14.10248 Probability 1E-15 @@ -62,10 +79,14 @@ private static void PrintScore(IEnumerable values, int numRows) Console.WriteLine("{0, -10} {1, -10}", "Score", value.Score); } - private static void PrintScoreAndProbability(IEnumerable values, int numRows) + private static void PrintScoreAndProbability( + IEnumerable values, int numRows) + { foreach (var value in values.Take(numRows)) - Console.WriteLine("{0, -10} {1, -10} {2, -10} {3, -10}", "Score", value.Score, "Probability", value.Probability); + Console.WriteLine("{0, -10} {1, -10} {2, -10} {3, -10}", "Score", + value.Score, "Probability", value.Probability); + } private class ScoreValue diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/Calibrators/Naive.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/Calibrators/Naive.cs index 84a004b1c5..054d4f9e31 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/Calibrators/Naive.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/Calibrators/Naive.cs @@ -9,26 +9,36 @@ public static class Naive { public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Download and featurize the dataset. - var data = Microsoft.ML.SamplesUtils.DatasetUtils.LoadFeaturizedAdultDataset(mlContext); + var data = Microsoft.ML.SamplesUtils.DatasetUtils + .LoadFeaturizedAdultDataset(mlContext); + // Leave out 10% of data for testing. - var trainTestData = mlContext.Data.TrainTestSplit(data, testFraction: 0.3); + var trainTestData = mlContext.Data + .TrainTestSplit(data, testFraction: 0.3); - // Create data training pipeline for non calibrated trainer and train Naive calibrator on top of it. - var pipeline = mlContext.BinaryClassification.Trainers.AveragedPerceptron(); + // Create data training pipeline for non calibrated trainer and train + // Naive calibrator on top of it. + var pipeline = mlContext.BinaryClassification.Trainers + .AveragedPerceptron(); - // Fit the pipeline, and get a transformer that knows how to score new data. + // Fit the pipeline, and get a transformer that knows how to score new + // data. var transformer = pipeline.Fit(trainTestData.TrainSet); // Fit this pipeline to the training data. - // Let's score the new data. The score will give us a numerical estimation of the chance that the particular sample - // bears positive sentiment. This estimate is relative to the numbers obtained. + // Let's score the new data. The score will give us a numerical + // estimation of the chance that the particular sample bears positive + // sentiment. This estimate is relative to the numbers obtained. var scoredData = transformer.Transform(trainTestData.TestSet); - var outScores = mlContext.Data.CreateEnumerable(scoredData, reuseRowObject: false); + var outScores = mlContext.Data + .CreateEnumerable(scoredData, reuseRowObject: false); + PrintScore(outScores, 5); // Preview of scoredDataPreview.RowView // Score 4.18144 @@ -37,16 +47,24 @@ public static void Example() // Score -2.554229 // Score 5.36571 - // Let's train a calibrator estimator on this scored dataset. The trained calibrator estimator produces a transformer - // that can transform the scored data by adding a new column names "Probability". - var calibratorEstimator = mlContext.BinaryClassification.Calibrators.Naive(); + // Let's train a calibrator estimator on this scored dataset. The + // trained calibrator estimator produces a transformer that can + // transform the scored data by adding a new column names "Probability". + var calibratorEstimator = mlContext.BinaryClassification.Calibrators + .Naive(); + var calibratorTransformer = calibratorEstimator.Fit(scoredData); - // Transform the scored data with a calibrator transfomer by adding a new column names "Probability". - // This column is a calibrated version of the "Score" column, meaning its values are a valid probability value in the [0, 1] interval - // representing the chance that the respective sample bears positive sentiment. + // Transform the scored data with a calibrator transfomer by adding a + // new column names "Probability". This column is a calibrated version + // of the "Score" column, meaning its values are a valid probability + // value in the [0, 1] interval representing the chance that the + // respective sample bears positive sentiment. var finalData = calibratorTransformer.Transform(scoredData); - var outScoresAndProbabilities = mlContext.Data.CreateEnumerable(finalData, reuseRowObject: false); + var outScoresAndProbabilities = mlContext.Data + .CreateEnumerable(finalData, + reuseRowObject: false); + PrintScoreAndProbability(outScoresAndProbabilities, 5); // Score 4.18144 Probability 0.775 // Score -14.10248 Probability 0.01923077 @@ -61,10 +79,14 @@ private static void PrintScore(IEnumerable values, int numRows) Console.WriteLine("{0, -10} {1, -10}", "Score", value.Score); } - private static void PrintScoreAndProbability(IEnumerable values, int numRows) + private static void PrintScoreAndProbability( + IEnumerable values, int numRows) + { foreach (var value in values.Take(numRows)) - Console.WriteLine("{0, -10} {1, -10} {2, -10} {3, -10}", "Score", value.Score, "Probability", value.Probability); + Console.WriteLine("{0, -10} {1, -10} {2, -10} {3, -10}", "Score", + value.Score, "Probability", value.Probability); + } private class ScoreValue diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/Calibrators/Platt.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/Calibrators/Platt.cs index aa0d7d0798..709db362e6 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/Calibrators/Platt.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/Calibrators/Platt.cs @@ -9,26 +9,36 @@ public static class Platt { public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Download and featurize the dataset. - var data = Microsoft.ML.SamplesUtils.DatasetUtils.LoadFeaturizedAdultDataset(mlContext); + var data = Microsoft.ML.SamplesUtils.DatasetUtils + .LoadFeaturizedAdultDataset(mlContext); + // Leave out 10% of data for testing. - var trainTestData = mlContext.Data.TrainTestSplit(data, testFraction: 0.3); + var trainTestData = mlContext.Data + .TrainTestSplit(data, testFraction: 0.3); - // Create data training pipeline for non calibrated trainer and train Naive calibrator on top of it. - var pipeline = mlContext.BinaryClassification.Trainers.AveragedPerceptron(); + // Create data training pipeline for non calibrated trainer and train + // Naive calibrator on top of it. + var pipeline = mlContext.BinaryClassification.Trainers + .AveragedPerceptron(); - // Fit the pipeline, and get a transformer that knows how to score new data. + // Fit the pipeline, and get a transformer that knows how to score new + // data. var transformer = pipeline.Fit(trainTestData.TrainSet); // Fit this pipeline to the training data. - // Let's score the new data. The score will give us a numerical estimation of the chance that the particular sample - // bears positive sentiment. This estimate is relative to the numbers obtained. + // Let's score the new data. The score will give us a numerical + // estimation of the chance that the particular sample bears positive + // sentiment. This estimate is relative to the numbers obtained. var scoredData = transformer.Transform(trainTestData.TestSet); - var outScores = mlContext.Data.CreateEnumerable(scoredData, reuseRowObject: false); + var outScores = mlContext.Data + .CreateEnumerable(scoredData, reuseRowObject: false); + PrintScore(outScores, 5); // Preview of scoredDataPreview.RowView // Score 4.18144 @@ -37,16 +47,24 @@ public static void Example() // Score -2.554229 // Score 5.36571 - // Let's train a calibrator estimator on this scored dataset. The trained calibrator estimator produces a transformer - // that can transform the scored data by adding a new column names "Probability". - var calibratorEstimator = mlContext.BinaryClassification.Calibrators.Platt(); + // Let's train a calibrator estimator on this scored dataset. The + // trained calibrator estimator produces a transformer that can + // transform the scored data by adding a new column names "Probability". + var calibratorEstimator = mlContext.BinaryClassification.Calibrators + .Platt(); + var calibratorTransformer = calibratorEstimator.Fit(scoredData); - // Transform the scored data with a calibrator transfomer by adding a new column names "Probability". - // This column is a calibrated version of the "Score" column, meaning its values are a valid probability value in the [0, 1] interval - // representing the chance that the respective sample bears positive sentiment. + // Transform the scored data with a calibrator transfomer by adding a + // new column names "Probability". This column is a calibrated version + // of the "Score" column, meaning its values are a valid probability + // value in the [0, 1] interval representing the chance that the + // respective sample bears positive sentiment. var finalData = calibratorTransformer.Transform(scoredData); - var outScoresAndProbabilities = mlContext.Data.CreateEnumerable(finalData, reuseRowObject: false); + var outScoresAndProbabilities = mlContext.Data + .CreateEnumerable(finalData, + reuseRowObject: false); + PrintScoreAndProbability(outScoresAndProbabilities, 5); // Score 4.18144 Probability 0.8511352 // Score -14.10248 Probability 0.001633563 @@ -61,10 +79,14 @@ private static void PrintScore(IEnumerable values, int numRows) Console.WriteLine("{0, -10} {1, -10}", "Score", value.Score); } - private static void PrintScoreAndProbability(IEnumerable values, int numRows) + private static void PrintScoreAndProbability( + IEnumerable values, int numRows) + { foreach (var value in values.Take(numRows)) - Console.WriteLine("{0, -10} {1, -10} {2, -10} {3, -10}", "Score", value.Score, "Probability", value.Probability); + Console.WriteLine("{0, -10} {1, -10} {2, -10} {3, -10}", "Score", + value.Score, "Probability", value.Probability); + } private class ScoreValue diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/FactorizationMachine.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/FactorizationMachine.cs index afc9231814..4e8bec4f4e 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/FactorizationMachine.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/FactorizationMachine.cs @@ -10,41 +10,51 @@ public static class FactorizationMachine { public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); - // ML.NET doesn't cache data set by default. Therefore, if one reads a data set from a file and accesses it many times, - // it can be slow due to expensive featurization and disk operations. When the considered data can fit into memory, - // a solution is to cache the data in memory. Caching is especially helpful when working with iterative algorithms + // ML.NET doesn't cache data set by default. Therefore, if one reads a + // data set from a file and accesses it many times, it can be slow due + // to expensive featurization and disk operations. When the considered + // data can fit into memory, a solution is to cache the data in memory. + // Caching is especially helpful when working with iterative algorithms // which needs many data passes. trainingData = mlContext.Data.Cache(trainingData); // Define the trainer. - var pipeline = mlContext.BinaryClassification.Trainers.FieldAwareFactorizationMachine(); + var pipeline = mlContext.BinaryClassification.Trainers + .FieldAwareFactorizationMachine(); // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(500, seed:123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data + .LoadFromEnumerable(GenerateRandomDataPoints(500, seed:123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(transformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data + .CreateEnumerable(transformedTestData, + reuseRowObject: false).ToList(); // Print 5 predictions. foreach (var p in predictions.Take(5)) - Console.WriteLine($"Label: {p.Label}, Prediction: {p.PredictedLabel}"); + Console.WriteLine($"Label: {p.Label}, " + + $"Prediction: {p.PredictedLabel}"); // Expected output: // Label: True, Prediction: False @@ -54,7 +64,9 @@ public static void Example() // Label: False, Prediction: False // Evaluate the overall metrics. - var metrics = mlContext.BinaryClassification.Evaluate(transformedTestData); + var metrics = mlContext.BinaryClassification + .Evaluate(transformedTestData); + PrintMetrics(metrics); // Expected output: @@ -77,7 +89,9 @@ public static void Example() // Precision || 0.9063 | 0.8732 | } - private static IEnumerable GenerateRandomDataPoints(int count, int seed=0) + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) + { var random = new Random(seed); float randomFloat() => (float)random.NextDouble(); @@ -88,13 +102,18 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se { Label = label, // Create random features that are correlated with the label. - // For data points with false label, the feature values are slightly increased by adding a constant. - Features = Enumerable.Repeat(label, 50).Select(x => x ? randomFloat() : randomFloat() + 0.1f).ToArray() + // For data points with false label, the feature values are + // slightly increased by adding a constant. + Features = Enumerable.Repeat(label, 50) + .Select(x => x ? randomFloat() : randomFloat() + + 0.1f).ToArray() + }; } } - // Example with label and 50 feature values. A data set is a collection of such examples. + // Example with label and 50 feature values. A data set is a collection of + // such examples. private class DataPoint { public bool Label { get; set; } @@ -117,11 +136,16 @@ private static void PrintMetrics(BinaryClassificationMetrics metrics) Console.WriteLine($"Accuracy: {metrics.Accuracy:F2}"); Console.WriteLine($"AUC: {metrics.AreaUnderRocCurve:F2}"); Console.WriteLine($"F1 Score: {metrics.F1Score:F2}"); - Console.WriteLine($"Negative Precision: {metrics.NegativePrecision:F2}"); + Console.WriteLine($"Negative Precision: " + + $"{metrics.NegativePrecision:F2}"); + Console.WriteLine($"Negative Recall: {metrics.NegativeRecall:F2}"); - Console.WriteLine($"Positive Precision: {metrics.PositivePrecision:F2}"); + Console.WriteLine($"Positive Precision: " + + $"{metrics.PositivePrecision:F2}"); + Console.WriteLine($"Positive Recall: {metrics.PositiveRecall:F2}\n"); Console.WriteLine(metrics.ConfusionMatrix.GetFormattedConfusionTable()); } } } + diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/FastForest.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/FastForest.cs index b415ae5dc9..f33feb7063 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/FastForest.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/FastForest.cs @@ -8,39 +8,48 @@ namespace Samples.Dynamic.Trainers.BinaryClassification { public static class FastForest { - // This example requires installation of additional NuGet package - // Microsoft.ML.FastTree. + // This example requires installation of additional NuGet package for + // Microsoft.ML.FastTree at + // https://www.nuget.org/packages/Microsoft.ML.FastTree/ public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); // Define the trainer. - var pipeline = mlContext.BinaryClassification.Trainers.FastForest(); + var pipeline = mlContext.BinaryClassification.Trainers + .FastForest(); // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(500, seed:123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data + .LoadFromEnumerable(GenerateRandomDataPoints(500, seed:123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(transformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data + .CreateEnumerable(transformedTestData, + reuseRowObject: false).ToList(); // Print 5 predictions. foreach (var p in predictions.Take(5)) - Console.WriteLine($"Label: {p.Label}, Prediction: {p.PredictedLabel}"); + Console.WriteLine($"Label: {p.Label}, " + + $"Prediction: {p.PredictedLabel}"); // Expected output: // Label: True, Prediction: True @@ -50,7 +59,9 @@ public static void Example() // Label: False, Prediction: False // Evaluate the overall metrics. - var metrics = mlContext.BinaryClassification.EvaluateNonCalibrated(transformedTestData); + var metrics = mlContext.BinaryClassification + .EvaluateNonCalibrated(transformedTestData); + PrintMetrics(metrics); // Expected output: @@ -73,7 +84,9 @@ public static void Example() // Precision || 0.6182 | 0.5416 | } - private static IEnumerable GenerateRandomDataPoints(int count, int seed=0) + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) + { var random = new Random(seed); float randomFloat() => (float)random.NextDouble(); @@ -84,13 +97,18 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se { Label = label, // Create random features that are correlated with the label. - // For data points with false label, the feature values are slightly increased by adding a constant. - Features = Enumerable.Repeat(label, 50).Select(x => x ? randomFloat() : randomFloat() + 0.03f).ToArray() + // For data points with false label, the feature values are + // slightly increased by adding a constant. + Features = Enumerable.Repeat(label, 50) + .Select(x => x ? randomFloat() : randomFloat() + + 0.03f).ToArray() + }; } } - // Example with label and 50 feature values. A data set is a collection of such examples. + // Example with label and 50 feature values. A data set is a collection of + // such examples. private class DataPoint { public bool Label { get; set; } @@ -113,12 +131,17 @@ private static void PrintMetrics(BinaryClassificationMetrics metrics) Console.WriteLine($"Accuracy: {metrics.Accuracy:F2}"); Console.WriteLine($"AUC: {metrics.AreaUnderRocCurve:F2}"); Console.WriteLine($"F1 Score: {metrics.F1Score:F2}"); - Console.WriteLine($"Negative Precision: {metrics.NegativePrecision:F2}"); + Console.WriteLine($"Negative Precision: " + + $"{metrics.NegativePrecision:F2}"); + Console.WriteLine($"Negative Recall: {metrics.NegativeRecall:F2}"); - Console.WriteLine($"Positive Precision: {metrics.PositivePrecision:F2}"); + Console.WriteLine($"Positive Precision: " + + $"{metrics.PositivePrecision:F2}"); + Console.WriteLine($"Positive Recall: {metrics.PositiveRecall:F2}\n"); Console.WriteLine(metrics.ConfusionMatrix.GetFormattedConfusionTable()); } } } + diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/FastForestWithOptions.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/FastForestWithOptions.cs index 372be0f1fa..c5bf4d5366 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/FastForestWithOptions.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/FastForestWithOptions.cs @@ -9,19 +9,22 @@ namespace Samples.Dynamic.Trainers.BinaryClassification { public static class FastForestWithOptions { - // This example requires installation of additional NuGet package - // Microsoft.ML.FastTree. + // This example requires installation of additional NuGet package for + // Microsoft.ML.FastTree at + // https://www.nuget.org/packages/Microsoft.ML.FastTree/ public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); // Define trainer options. @@ -36,23 +39,29 @@ public static void Example() }; // Define the trainer. - var pipeline = mlContext.BinaryClassification.Trainers.FastForest(options); + var pipeline = mlContext.BinaryClassification.Trainers + .FastForest(options); // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(500, seed:123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data + .LoadFromEnumerable(GenerateRandomDataPoints(500, seed:123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(transformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data + .CreateEnumerable(transformedTestData, + reuseRowObject: false).ToList(); // Print 5 predictions. foreach (var p in predictions.Take(5)) - Console.WriteLine($"Label: {p.Label}, Prediction: {p.PredictedLabel}"); + Console.WriteLine($"Label: {p.Label}, " + + $"Prediction: {p.PredictedLabel}"); // Expected output: // Label: True, Prediction: True @@ -62,7 +71,9 @@ public static void Example() // Label: False, Prediction: True // Evaluate the overall metrics. - var metrics = mlContext.BinaryClassification.EvaluateNonCalibrated(transformedTestData); + var metrics = mlContext.BinaryClassification + .EvaluateNonCalibrated(transformedTestData); + PrintMetrics(metrics); // Expected output: @@ -85,7 +96,9 @@ public static void Example() // Precision || 0.7072 | 0.7806 | } - private static IEnumerable GenerateRandomDataPoints(int count, int seed=0) + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) + { var random = new Random(seed); float randomFloat() => (float)random.NextDouble(); @@ -96,13 +109,18 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se { Label = label, // Create random features that are correlated with the label. - // For data points with false label, the feature values are slightly increased by adding a constant. - Features = Enumerable.Repeat(label, 50).Select(x => x ? randomFloat() : randomFloat() + 0.03f).ToArray() + // For data points with false label, the feature values are + // slightly increased by adding a constant. + Features = Enumerable.Repeat(label, 50) + .Select(x => x ? randomFloat() : randomFloat() + + 0.03f).ToArray() + }; } } - // Example with label and 50 feature values. A data set is a collection of such examples. + // Example with label and 50 feature values. A data set is a collection of + // such examples. private class DataPoint { public bool Label { get; set; } @@ -125,12 +143,17 @@ private static void PrintMetrics(BinaryClassificationMetrics metrics) Console.WriteLine($"Accuracy: {metrics.Accuracy:F2}"); Console.WriteLine($"AUC: {metrics.AreaUnderRocCurve:F2}"); Console.WriteLine($"F1 Score: {metrics.F1Score:F2}"); - Console.WriteLine($"Negative Precision: {metrics.NegativePrecision:F2}"); + Console.WriteLine($"Negative Precision: " + + $"{metrics.NegativePrecision:F2}"); + Console.WriteLine($"Negative Recall: {metrics.NegativeRecall:F2}"); - Console.WriteLine($"Positive Precision: {metrics.PositivePrecision:F2}"); + Console.WriteLine($"Positive Precision: " + + $"{metrics.PositivePrecision:F2}"); + Console.WriteLine($"Positive Recall: {metrics.PositiveRecall:F2}\n"); Console.WriteLine(metrics.ConfusionMatrix.GetFormattedConfusionTable()); } } } + diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/FastTree.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/FastTree.cs index d260ec8666..f50b8f9732 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/FastTree.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/FastTree.cs @@ -8,39 +8,48 @@ namespace Samples.Dynamic.Trainers.BinaryClassification { public static class FastTree { - // This example requires installation of additional NuGet package - // Microsoft.ML.FastTree. + // This example requires installation of additional NuGet package for + // Microsoft.ML.FastTree at + // https://www.nuget.org/packages/Microsoft.ML.FastTree/ public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); // Define the trainer. - var pipeline = mlContext.BinaryClassification.Trainers.FastTree(); + var pipeline = mlContext.BinaryClassification.Trainers + .FastTree(); // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(500, seed:123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data + .LoadFromEnumerable(GenerateRandomDataPoints(500, seed:123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(transformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data + .CreateEnumerable(transformedTestData, + reuseRowObject: false).ToList(); // Print 5 predictions. foreach (var p in predictions.Take(5)) - Console.WriteLine($"Label: {p.Label}, Prediction: {p.PredictedLabel}"); + Console.WriteLine($"Label: {p.Label}, " + + $"Prediction: {p.PredictedLabel}"); // Expected output: // Label: True, Prediction: True @@ -50,7 +59,9 @@ public static void Example() // Label: False, Prediction: False // Evaluate the overall metrics. - var metrics = mlContext.BinaryClassification.Evaluate(transformedTestData); + var metrics = mlContext.BinaryClassification + .Evaluate(transformedTestData); + PrintMetrics(metrics); // Expected output: @@ -76,7 +87,9 @@ public static void Example() // Precision || 0.6903 | 0.7716 | } - private static IEnumerable GenerateRandomDataPoints(int count, int seed=0) + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) + { var random = new Random(seed); float randomFloat() => (float)random.NextDouble(); @@ -87,13 +100,18 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se { Label = label, // Create random features that are correlated with the label. - // For data points with false label, the feature values are slightly increased by adding a constant. - Features = Enumerable.Repeat(label, 50).Select(x => x ? randomFloat() : randomFloat() + 0.03f).ToArray() + // For data points with false label, the feature values are + // slightly increased by adding a constant. + Features = Enumerable.Repeat(label, 50) + .Select(x => x ? randomFloat() : randomFloat() + + 0.03f).ToArray() + }; } } - // Example with label and 50 feature values. A data set is a collection of such examples. + // Example with label and 50 feature values. A data set is a collection of + // such examples. private class DataPoint { public bool Label { get; set; } @@ -116,12 +134,17 @@ private static void PrintMetrics(BinaryClassificationMetrics metrics) Console.WriteLine($"Accuracy: {metrics.Accuracy:F2}"); Console.WriteLine($"AUC: {metrics.AreaUnderRocCurve:F2}"); Console.WriteLine($"F1 Score: {metrics.F1Score:F2}"); - Console.WriteLine($"Negative Precision: {metrics.NegativePrecision:F2}"); + Console.WriteLine($"Negative Precision: " + + $"{metrics.NegativePrecision:F2}"); + Console.WriteLine($"Negative Recall: {metrics.NegativeRecall:F2}"); - Console.WriteLine($"Positive Precision: {metrics.PositivePrecision:F2}"); + Console.WriteLine($"Positive Precision: " + + $"{metrics.PositivePrecision:F2}"); + Console.WriteLine($"Positive Recall: {metrics.PositiveRecall:F2}\n"); Console.WriteLine(metrics.ConfusionMatrix.GetFormattedConfusionTable()); } } } + diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/FastTreeWithOptions.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/FastTreeWithOptions.cs index b02e9b977b..87e894d903 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/FastTreeWithOptions.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/FastTreeWithOptions.cs @@ -9,19 +9,22 @@ namespace Samples.Dynamic.Trainers.BinaryClassification { public static class FastTreeWithOptions { - // This example requires installation of additional NuGet package - // Microsoft.ML.FastTree. + // This example requires installation of additional NuGet package for + // Microsoft.ML.FastTree at + // https://www.nuget.org/packages/Microsoft.ML.FastTree/ public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); // Define trainer options. @@ -36,23 +39,29 @@ public static void Example() }; // Define the trainer. - var pipeline = mlContext.BinaryClassification.Trainers.FastTree(options); + var pipeline = mlContext.BinaryClassification.Trainers + .FastTree(options); // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(500, seed:123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data + .LoadFromEnumerable(GenerateRandomDataPoints(500, seed:123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(transformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data + .CreateEnumerable(transformedTestData, + reuseRowObject: false).ToList(); // Print 5 predictions. foreach (var p in predictions.Take(5)) - Console.WriteLine($"Label: {p.Label}, Prediction: {p.PredictedLabel}"); + Console.WriteLine($"Label: {p.Label}, " + + $"Prediction: {p.PredictedLabel}"); // Expected output: // Label: True, Prediction: True @@ -62,7 +71,9 @@ public static void Example() // Label: False, Prediction: False // Evaluate the overall metrics. - var metrics = mlContext.BinaryClassification.Evaluate(transformedTestData); + var metrics = mlContext.BinaryClassification + .Evaluate(transformedTestData); + PrintMetrics(metrics); // Expected output: @@ -88,7 +99,9 @@ public static void Example() // Precision || 0.6903 | 0.7716 | } - private static IEnumerable GenerateRandomDataPoints(int count, int seed=0) + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) + { var random = new Random(seed); float randomFloat() => (float)random.NextDouble(); @@ -99,13 +112,18 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se { Label = label, // Create random features that are correlated with the label. - // For data points with false label, the feature values are slightly increased by adding a constant. - Features = Enumerable.Repeat(label, 50).Select(x => x ? randomFloat() : randomFloat() + 0.03f).ToArray() + // For data points with false label, the feature values are + // slightly increased by adding a constant. + Features = Enumerable.Repeat(label, 50) + .Select(x => x ? randomFloat() : randomFloat() + + 0.03f).ToArray() + }; } } - // Example with label and 50 feature values. A data set is a collection of such examples. + // Example with label and 50 feature values. A data set is a collection of + // such examples. private class DataPoint { public bool Label { get; set; } @@ -128,12 +146,17 @@ private static void PrintMetrics(BinaryClassificationMetrics metrics) Console.WriteLine($"Accuracy: {metrics.Accuracy:F2}"); Console.WriteLine($"AUC: {metrics.AreaUnderRocCurve:F2}"); Console.WriteLine($"F1 Score: {metrics.F1Score:F2}"); - Console.WriteLine($"Negative Precision: {metrics.NegativePrecision:F2}"); + Console.WriteLine($"Negative Precision: " + + $"{metrics.NegativePrecision:F2}"); + Console.WriteLine($"Negative Recall: {metrics.NegativeRecall:F2}"); - Console.WriteLine($"Positive Precision: {metrics.PositivePrecision:F2}"); + Console.WriteLine($"Positive Precision: " + + $"{metrics.PositivePrecision:F2}"); + Console.WriteLine($"Positive Recall: {metrics.PositiveRecall:F2}\n"); Console.WriteLine(metrics.ConfusionMatrix.GetFormattedConfusionTable()); } } } + diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/FieldAwareFactorizationMachine.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/FieldAwareFactorizationMachine.cs index bc2eaef343..bddc926ccf 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/FieldAwareFactorizationMachine.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/FieldAwareFactorizationMachine.cs @@ -8,28 +8,36 @@ namespace Samples.Dynamic.Trainers.BinaryClassification { public static class FieldAwareFactorizationMachine { - // This example first train a field-aware factorization to binary classification, measure the trained model's quality, and finally + // This example first train a field-aware factorization to binary + // classification, measure the trained model's quality, and finally // use the trained model to make prediction. public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. IEnumerable data = GenerateRandomDataPoints(500); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(data); // Define the trainer. - // This trainer trains field-aware factorization (FFM) for binary classification. See https://www.csie.ntu.edu.tw/~cjlin/papers/ffm.pdf - // for the theory behind and https://github.com/wschin/fast-ffm/blob/master/fast-ffm.pdf for the training - // algorithm implemented in ML.NET. - var pipeline = mlContext.BinaryClassification.Trainers.FieldAwareFactorizationMachine( + // This trainer trains field-aware factorization (FFM) + // for binary classification. + // See https://www.csie.ntu.edu.tw/~cjlin/papers/ffm.pdf for the theory + // behind and + // https://github.com/wschin/fast-ffm/blob/master/fast-ffm.pdf for the + // training algorithm implemented in ML.NET. + var pipeline = mlContext.BinaryClassification.Trainers + .FieldAwareFactorizationMachine( // Specify three feature columns! - new[] {nameof(DataPoint.Field0), nameof(DataPoint.Field1), nameof(DataPoint.Field2) }, + new[] {nameof(DataPoint.Field0), nameof(DataPoint.Field1), + nameof(DataPoint.Field2) }, // Specify binary label's column name. nameof(DataPoint.Label) ); @@ -40,7 +48,8 @@ public static void Example() var transformedTrainingData = model.Transform(trainingData); // Measure the quality of the trained model. - var metrics = mlContext.BinaryClassification.Evaluate(transformedTrainingData); + var metrics = mlContext.BinaryClassification + .Evaluate(transformedTrainingData); // Show the quality metrics. PrintMetrics(metrics); @@ -68,14 +77,19 @@ public static void Example() // Precision || 0.7878 | 0.8235 | // Create prediction function from the trained model. - var engine = mlContext.Model.CreatePredictionEngine(model); + var engine = mlContext.Model + .CreatePredictionEngine(model); // Make some predictions. foreach(var dataPoint in data.Take(5)) { var result = engine.Predict(dataPoint); - Console.WriteLine($"Actual label: {dataPoint.Label}, predicted label: {result.PredictedLabel}, " + - $"score of being positive class: {result.Score}, and probability of beling positive class: {result.Probability}."); + Console.WriteLine($"Actual label: {dataPoint.Label}, " + + $"predicted label: {result.PredictedLabel}, " + + $"score of being positive class: {result.Score}, " + + $"and probability of beling positive class: " + + $"{result.Probability}."); + } // Expected output: @@ -95,7 +109,8 @@ private class DataPoint // Label. public bool Label { get; set; } - // Features from the first field. Note that different fields can have different numbers of features. + // Features from the first field. Note that different fields can have + // different numbers of features. [VectorType(featureLength)] public float[] Field0 { get; set; } @@ -108,8 +123,8 @@ private class DataPoint public float[] Field2 { get; set; } } - // This class defines objects produced by trained model. The trained model maps - // a DataPoint to a Result. + // This class defines objects produced by trained model. The trained model + // maps a DataPoint to a Result. public class Result { // Label. @@ -123,13 +138,16 @@ public class Result } // Function used to create toy data sets. - private static IEnumerable GenerateRandomDataPoints(int exampleCount, int seed = 0) + private static IEnumerable GenerateRandomDataPoints( + int exampleCount, int seed = 0) + { var rnd = new Random(seed); var data = new List(); for (int i = 0; i < exampleCount; ++i) { - // Initialize an example with a random label and an empty feature vector. + // Initialize an example with a random label and an empty feature + // vector. var sample = new DataPoint() { Label = rnd.Next() % 2 == 0, @@ -139,9 +157,10 @@ private static IEnumerable GenerateRandomDataPoints(int exampleCount, }; // Fill feature vectors according the assigned label. - // Notice that features from different fields have different biases and therefore different distributions. - // In practices such as game recommendation, one may use one field to store features from user profile and - // another field to store features from game profile. + // Notice that features from different fields have different biases + // and therefore different distributions. In practices such as game + // recommendation, one may use one field to store features from user + // profile and another field to store features from game profile. for (int j = 0; j < featureLength; ++j) { var value0 = (float)rnd.NextDouble(); @@ -169,14 +188,20 @@ private static IEnumerable GenerateRandomDataPoints(int exampleCount, } // Function used to show evaluation metrics such as accuracy of predictions. - private static void PrintMetrics(CalibratedBinaryClassificationMetrics metrics) + private static void PrintMetrics( + CalibratedBinaryClassificationMetrics metrics) + { Console.WriteLine($"Accuracy: {metrics.Accuracy:F2}"); Console.WriteLine($"AUC: {metrics.AreaUnderRocCurve:F2}"); Console.WriteLine($"F1 Score: {metrics.F1Score:F2}"); - Console.WriteLine($"Negative Precision: {metrics.NegativePrecision:F2}"); + Console.WriteLine($"Negative Precision: " + + $"{metrics.NegativePrecision:F2}"); + Console.WriteLine($"Negative Recall: {metrics.NegativeRecall:F2}"); - Console.WriteLine($"Positive Precision: {metrics.PositivePrecision:F2}"); + Console.WriteLine($"Positive Precision: " + + $"{metrics.PositivePrecision:F2}"); + Console.WriteLine($"Positive Recall: {metrics.PositiveRecall:F2}"); Console.WriteLine($"Log Loss: {metrics.LogLoss:F2}"); Console.WriteLine($"Log Loss Reduction: {metrics.LogLossReduction:F2}"); diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/FieldAwareFactorizationMachine.tt b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/FieldAwareFactorizationMachine.tt index 3f41aa671f..22cdd45721 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/FieldAwareFactorizationMachine.tt +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/FieldAwareFactorizationMachine.tt @@ -3,19 +3,24 @@ string ClassName="FieldAwareFactorizationMachine"; string Trainer = @"FieldAwareFactorizationMachine( // Specify three feature columns! - new[] {nameof(DataPoint.Field0), nameof(DataPoint.Field1), nameof(DataPoint.Field2) }, + new[] {nameof(DataPoint.Field0), nameof(DataPoint.Field1), + nameof(DataPoint.Field2) }, // Specify binary label's column name. nameof(DataPoint.Label) )"; string OptionsInclude = null; string Comments = @" - // This example first train a field-aware factorization to binary classification, measure the trained model's quality, and finally + // This example first train a field-aware factorization to binary + // classification, measure the trained model's quality, and finally // use the trained model to make prediction."; -string TrainerDescription = @"// This trainer trains field-aware factorization (FFM) for binary classification. See https://www.csie.ntu.edu.tw/~cjlin/papers/ffm.pdf - // for the theory behind and https://github.com/wschin/fast-ffm/blob/master/fast-ffm.pdf for the training - // algorithm implemented in ML.NET."; +string TrainerDescription = @"// This trainer trains field-aware factorization (FFM) + // for binary classification. + // See https://www.csie.ntu.edu.tw/~cjlin/papers/ffm.pdf for the theory + // behind and + // https://github.com/wschin/fast-ffm/blob/master/fast-ffm.pdf for the + // training algorithm implemented in ML.NET."; string TrainerOptions = null; diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/FieldAwareFactorizationMachineWithOptions.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/FieldAwareFactorizationMachineWithOptions.cs index 47e53dfdc3..08b48a9e74 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/FieldAwareFactorizationMachineWithOptions.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/FieldAwareFactorizationMachineWithOptions.cs @@ -9,26 +9,31 @@ namespace Samples.Dynamic.Trainers.BinaryClassification { public static class FieldAwareFactorizationMachineWithOptions { - // This example first train a field-aware factorization to binary classification, measure the trained model's quality, and finally + // This example first train a field-aware factorization to binary + // classification, measure the trained model's quality, and finally // use the trained model to make prediction. public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. IEnumerable data = GenerateRandomDataPoints(500); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(data); // Define trainer options. var options = new FieldAwareFactorizationMachineTrainer.Options { FeatureColumnName = nameof(DataPoint.Field0), - ExtraFeatureColumns = new[] { nameof(DataPoint.Field1), nameof(DataPoint.Field2) }, + ExtraFeatureColumns = + new[] { nameof(DataPoint.Field1), nameof(DataPoint.Field2) }, + LabelColumnName = nameof(DataPoint.Label), LambdaLatent = 0.01f, LambdaLinear = 0.001f, @@ -38,10 +43,14 @@ public static void Example() }; // Define the trainer. - // This trainer trains field-aware factorization (FFM) for binary classification. See https://www.csie.ntu.edu.tw/~cjlin/papers/ffm.pdf - // for the theory behind and https://github.com/wschin/fast-ffm/blob/master/fast-ffm.pdf for the training - // algorithm implemented in ML.NET. - var pipeline = mlContext.BinaryClassification.Trainers.FieldAwareFactorizationMachine(options); + // This trainer trains field-aware factorization (FFM) + // for binary classification. + // See https://www.csie.ntu.edu.tw/~cjlin/papers/ffm.pdf for the theory + // behind and + // https://github.com/wschin/fast-ffm/blob/master/fast-ffm.pdf for the + // training algorithm implemented in ML.NET. + var pipeline = mlContext.BinaryClassification.Trainers + .FieldAwareFactorizationMachine(options); // Train the model. var model = pipeline.Fit(trainingData); @@ -50,7 +59,8 @@ public static void Example() var transformedTrainingData = model.Transform(trainingData); // Measure the quality of the trained model. - var metrics = mlContext.BinaryClassification.Evaluate(transformedTrainingData); + var metrics = mlContext.BinaryClassification + .Evaluate(transformedTrainingData); // Show the quality metrics. PrintMetrics(metrics); @@ -78,14 +88,19 @@ public static void Example() // Precision || 0.7425 | 0.8319 | // Create prediction function from the trained model. - var engine = mlContext.Model.CreatePredictionEngine(model); + var engine = mlContext.Model + .CreatePredictionEngine(model); // Make some predictions. foreach(var dataPoint in data.Take(5)) { var result = engine.Predict(dataPoint); - Console.WriteLine($"Actual label: {dataPoint.Label}, predicted label: {result.PredictedLabel}, " + - $"score of being positive class: {result.Score}, and probability of beling positive class: {result.Probability}."); + Console.WriteLine($"Actual label: {dataPoint.Label}, " + + $"predicted label: {result.PredictedLabel}, " + + $"score of being positive class: {result.Score}, " + + $"and probability of beling positive class: " + + $"{result.Probability}."); + } // Expected output: @@ -105,7 +120,8 @@ private class DataPoint // Label. public bool Label { get; set; } - // Features from the first field. Note that different fields can have different numbers of features. + // Features from the first field. Note that different fields can have + // different numbers of features. [VectorType(featureLength)] public float[] Field0 { get; set; } @@ -118,8 +134,8 @@ private class DataPoint public float[] Field2 { get; set; } } - // This class defines objects produced by trained model. The trained model maps - // a DataPoint to a Result. + // This class defines objects produced by trained model. The trained model + // maps a DataPoint to a Result. public class Result { // Label. @@ -133,13 +149,16 @@ public class Result } // Function used to create toy data sets. - private static IEnumerable GenerateRandomDataPoints(int exampleCount, int seed = 0) + private static IEnumerable GenerateRandomDataPoints( + int exampleCount, int seed = 0) + { var rnd = new Random(seed); var data = new List(); for (int i = 0; i < exampleCount; ++i) { - // Initialize an example with a random label and an empty feature vector. + // Initialize an example with a random label and an empty feature + // vector. var sample = new DataPoint() { Label = rnd.Next() % 2 == 0, @@ -149,9 +168,10 @@ private static IEnumerable GenerateRandomDataPoints(int exampleCount, }; // Fill feature vectors according the assigned label. - // Notice that features from different fields have different biases and therefore different distributions. - // In practices such as game recommendation, one may use one field to store features from user profile and - // another field to store features from game profile. + // Notice that features from different fields have different biases + // and therefore different distributions. In practices such as game + // recommendation, one may use one field to store features from user + // profile and another field to store features from game profile. for (int j = 0; j < featureLength; ++j) { var value0 = (float)rnd.NextDouble(); @@ -179,14 +199,20 @@ private static IEnumerable GenerateRandomDataPoints(int exampleCount, } // Function used to show evaluation metrics such as accuracy of predictions. - private static void PrintMetrics(CalibratedBinaryClassificationMetrics metrics) + private static void PrintMetrics( + CalibratedBinaryClassificationMetrics metrics) + { Console.WriteLine($"Accuracy: {metrics.Accuracy:F2}"); Console.WriteLine($"AUC: {metrics.AreaUnderRocCurve:F2}"); Console.WriteLine($"F1 Score: {metrics.F1Score:F2}"); - Console.WriteLine($"Negative Precision: {metrics.NegativePrecision:F2}"); + Console.WriteLine($"Negative Precision: " + + $"{metrics.NegativePrecision:F2}"); + Console.WriteLine($"Negative Recall: {metrics.NegativeRecall:F2}"); - Console.WriteLine($"Positive Precision: {metrics.PositivePrecision:F2}"); + Console.WriteLine($"Positive Precision: " + + $"{metrics.PositivePrecision:F2}"); + Console.WriteLine($"Positive Recall: {metrics.PositiveRecall:F2}"); Console.WriteLine($"Log Loss: {metrics.LogLoss:F2}"); Console.WriteLine($"Log Loss Reduction: {metrics.LogLossReduction:F2}"); diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/FieldAwareFactorizationMachineWithOptions.tt b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/FieldAwareFactorizationMachineWithOptions.tt index c3de9d1882..0c7e32048d 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/FieldAwareFactorizationMachineWithOptions.tt +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/FieldAwareFactorizationMachineWithOptions.tt @@ -6,17 +6,23 @@ string Trainer = "FieldAwareFactorizationMachine"; string OptionsInclude = @"using Microsoft.ML.Trainers;"; string Comments = @" - // This example first train a field-aware factorization to binary classification, measure the trained model's quality, and finally + // This example first train a field-aware factorization to binary + // classification, measure the trained model's quality, and finally // use the trained model to make prediction."; -string TrainerDescription = @"// This trainer trains field-aware factorization (FFM) for binary classification. See https://www.csie.ntu.edu.tw/~cjlin/papers/ffm.pdf - // for the theory behind and https://github.com/wschin/fast-ffm/blob/master/fast-ffm.pdf for the training - // algorithm implemented in ML.NET."; +string TrainerDescription = @"// This trainer trains field-aware factorization (FFM) + // for binary classification. + // See https://www.csie.ntu.edu.tw/~cjlin/papers/ffm.pdf for the theory + // behind and + // https://github.com/wschin/fast-ffm/blob/master/fast-ffm.pdf for the + // training algorithm implemented in ML.NET."; string TrainerOptions = @"FieldAwareFactorizationMachineTrainer.Options { FeatureColumnName = nameof(DataPoint.Field0), - ExtraFeatureColumns = new[] { nameof(DataPoint.Field1), nameof(DataPoint.Field2) }, + ExtraFeatureColumns = + new[] { nameof(DataPoint.Field1), nameof(DataPoint.Field2) }, + LabelColumnName = nameof(DataPoint.Label), LambdaLatent = 0.01f, LambdaLinear = 0.001f, diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/Gam.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/Gam.cs index 3b9c36644f..d4ce855e3c 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/Gam.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/Gam.cs @@ -7,12 +7,14 @@ namespace Samples.Dynamic.Trainers.BinaryClassification { public static class Gam { - // This example requires installation of additional NuGet package - // Microsoft.ML.FastTree. + // This example requires installation of additional NuGet package for + // Microsoft.ML.FastTree at + // https://www.nuget.org/packages/Microsoft.ML.FastTree/ public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. var mlContext = new MLContext(); // Create the dataset. @@ -27,30 +29,36 @@ public static void Example() var validSet = dataSets.TestSet; // Create a GAM trainer. - // Use a small number of bins for this example. The setting below means for each feature, - // we divide its range into 16 discrete regions for the training process. Note that these - // regions are not evenly spaced, and that the final model may contain fewer bins, as - // neighboring bins with identical values will be combined. In general, we recommend using - // at least the default number of bins, as a small number of bins limits the capacity of - // the model. - var trainer = mlContext.BinaryClassification.Trainers.Gam(maximumBinCountPerFeature: 16); - - // Fit the model using both of training and validation sets. GAM can use a technique called - // pruning to tune the model to the validation set after training to improve generalization. + // Use a small number of bins for this example. The setting below means + // for each feature, we divide its range into 16 discrete regions for + // the training process. Note that these regions are not evenly spaced, + // and that the final model may contain fewer bins, as neighboring bins + // with identical values will be combined. In general, we recommend + // using at least the default number of bins, as a small number of bins + // limits the capacity of the model. + var trainer = mlContext.BinaryClassification.Trainers + .Gam(maximumBinCountPerFeature: 16); + + // Fit the model using both of training and validation sets. GAM can use + // a technique called pruning to tune the model to the validation set + // after training to improve generalization. var model = trainer.Fit(trainSet, validSet); // Extract the model parameters. var gam = model.Model.SubModel; - // Now we can inspect the parameters of the Generalized Additive Model to understand the fit - // and potentially learn about our dataset. - // First, we will look at the bias; the bias represents the average prediction for the training data. + // Now we can inspect the parameters of the Generalized Additive Model + // to understand the fit and potentially learn about our dataset. First, + // we will look at the bias; the bias represents the average prediction + // for the training data. Console.WriteLine($"Average prediction: {gam.Bias:0.00}"); - // Now look at the shape functions that the model has learned. Similar to a linear model, we have - // one response per feature, and they are independent. Unlike a linear model, this response is a - // generic function instead of a line. Because we have included a bias term, each feature response - // represents the deviation from the average prediction as a function of the feature value. + // Now look at the shape functions that the model has learned. Similar + // to a linear model, we have one response per feature, and they are + // independent. Unlike a linear model, this response is a generic + // function instead of a line. Because we have included a bias term, + // each feature response represents the deviation from the average + // prediction as a function of the feature value. for (int i = 0; i < gam.NumberOfShapeFunctions; i++) { // Break a line. @@ -62,11 +70,14 @@ public static void Example() // Get the bin effects; these are the function values for each bin. var binEffects = gam.GetBinEffects(i); - // Now, write the function to the console. The function is a set of bins, and the corresponding - // function values. You can think of GAMs as building a bar-chart or lookup table for each feature. + // Now, write the function to the console. The function is a set of + // bins, and the corresponding function values. You can think of + // GAMs as building a bar-chart or lookup table for each feature. Console.WriteLine($"Feature{i}"); for (int j = 0; j < binUpperBounds.Count; j++) - Console.WriteLine($"x < {binUpperBounds[j]:0.00} => {binEffects[j]:0.000}"); + Console.WriteLine( + $"x < {binUpperBounds[j]:0.00} => {binEffects[j]:0.000}"); + } // Expected output: @@ -91,18 +102,23 @@ public static void Example() // x < 0.31 => -0.138 // x < ∞ => -0.188 - // Let's consider this output. To score a given example, we look up the first bin where the inequality - // is satisfied for the feature value. We can look at the whole function to get a sense for how the - // model responds to the variable on a global level. - // The model can be seen to reconstruct the parabolic and step-wise function, shifted with respect to the average - // expected output over the training set. Very few bins are used to model the second feature because the GAM model - // discards unchanged bins to create smaller models. - // One last thing to notice is that these feature functions can be noisy. While we know that Feature1 should be - // symmetric, this is not captured in the model. This is due to noise in the data. Common practice is to use - // resampling methods to estimate a confidence interval at each bin. This will help to determine if the effect is - // real or just sampling noise. See for example: - // Tan, Caruana, Hooker, and Lou. "Distill-and-Compare: Auditing Black-Box Models Using Transparent Model - // Distillation." arXiv:1710.06169." + // Let's consider this output. To score a given example, we look up the + // first bin where the inequality is satisfied for the feature value. + // We can look at the whole function to get a sense for how the model + // responds to the variable on a global level.The model can be seen to + // reconstruct the parabolic and step-wise function, shifted with + // respect to the average expected output over the training set. + // Very few bins are used to model the second feature because the GAM + // model discards unchanged bins to create smaller models. One last + // thing to notice is that these feature functions can be noisy. While + // we know that Feature1 should be symmetric, this is not captured in + // the model. This is due to noise in the data. Common practice is to + // use resampling methods to estimate a confidence interval at each bin. + // This will help to determine if the effect is real or just sampling + // noise. See for example: Tan, Caruana, Hooker, and Lou. + // "Distill-and-Compare: Auditing Black-Box Models Using Transparent + // Model Distillation." + // arXiv:1710.06169." } private class Data @@ -114,13 +130,17 @@ private class Data } /// - /// Creates a dataset, an IEnumerable of Data objects, for a GAM sample. Feature1 is a parabola centered around 0, - /// while Feature2 is a simple piecewise function. + /// Creates a dataset, an IEnumerable of Data objects, for a GAM sample. + /// Feature1 is a parabola centered around 0, while Feature2 is a simple + /// piecewise function. /// /// The number of examples to generate. - /// The seed for the random number generator used to produce data. + /// The seed for the random number generator used to + /// produce data. /// - private static IEnumerable GenerateData(int numExamples = 25000, int seed = 1) + private static IEnumerable GenerateData(int numExamples = 25000, + int seed = 1) + { var rng = new Random(seed); float centeredFloat() => (float)(rng.NextDouble() - 0.5); @@ -131,7 +151,8 @@ private static IEnumerable GenerateData(int numExamples = 25000, int seed Features = new float[2] { centeredFloat(), centeredFloat() } }; // Compute the label from the shape functions and add noise. - data.Label = Sigmoid(Parabola(data.Features[0]) + SimplePiecewise(data.Features[1]) + centeredFloat()) > 0.5; + data.Label = Sigmoid(Parabola(data.Features[0]) + + SimplePiecewise(data.Features[1]) + centeredFloat()) > 0.5; yield return data; } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/GamWithOptions.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/GamWithOptions.cs index e4a408a3ae..cdacd51b93 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/GamWithOptions.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/GamWithOptions.cs @@ -8,12 +8,14 @@ namespace Samples.Dynamic.Trainers.BinaryClassification { public static class GamWithOptions { - // This example requires installation of additional NuGet package - // Microsoft.ML.FastTree. + // This example requires installation of additional NuGet package for + // Microsoft.ML.FastTree at + // https://www.nuget.org/packages/Microsoft.ML.FastTree/ public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. var mlContext = new MLContext(); // Create the dataset. @@ -28,14 +30,15 @@ public static void Example() var validSet = dataSets.TestSet; // Create a GAM trainer. - // Use a small number of bins for this example. The setting below means for each feature, - // we divide its range into 16 discrete regions for the training process. Note that these - // regions are not evenly spaced, and that the final model may contain fewer bins, as - // neighboring bins with identical values will be combined. In general, we recommend using - // at least the default number of bins, as a small number of bins limits the capacity of - // the model. - // Also, set the learning rate to half the default to slow down the gradient descent, and - // double the number of iterations to compensate. + // Use a small number of bins for this example. The setting below means + // for each feature, we divide its range into 16 discrete regions for + // the training process. Note that these regions are not evenly spaced, + // and that the final model may contain fewer bins, as neighboring bins + // with identical values will be combined. In general, we recommend + // using at least the default number of bins, as a small number of bins + // limits the capacity of the model. Also, set the learning rate to half + // the default to slow down the gradient descent, and double the number + // of iterations to compensate. var trainer = mlContext.BinaryClassification.Trainers.Gam( new GamBinaryTrainer.Options { NumberOfIterations = 19000, @@ -43,22 +46,26 @@ public static void Example() LearningRate = 0.001 }); - // Fit the model using both of training and validation sets. GAM can use a technique called - // pruning to tune the model to the validation set after training to improve generalization. + // Fit the model using both of training and validation sets. GAM can use + // a technique called pruning to tune the model to the validation set + // after training to improve generalization. var model = trainer.Fit(trainSet, validSet); // Extract the model parameters. var gam = model.Model.SubModel; - // Now we can inspect the parameters of the Generalized Additive Model to understand the fit - // and potentially learn about our dataset. - // First, we will look at the bias; the bias represents the average prediction for the training data. + // Now we can inspect the parameters of the Generalized Additive Model + // to understand the fit and potentially learn about our dataset. First, + // we will look at the bias; the bias represents the average prediction + // for the training data. Console.WriteLine($"Average prediction: {gam.Bias:0.00}"); - // Now look at the shape functions that the model has learned. Similar to a linear model, we have - // one response per feature, and they are independent. Unlike a linear model, this response is a - // generic function instead of a line. Because we have included a bias term, each feature response - // represents the deviation from the average prediction as a function of the feature value. + // Now look at the shape functions that the model has learned. Similar + // to a linear model, we have one response per feature, and they are + // independent. Unlike a linear model, this response is a generic + // function instead of a line. Because we have included a bias term, + // each feature response represents the deviation from the average + // prediction as a function of the feature value. for (int i = 0; i < gam.NumberOfShapeFunctions; i++) { // Break a line. @@ -70,11 +77,13 @@ public static void Example() // Get the bin effects; these are the function values for each bin. var binEffects = gam.GetBinEffects(i); - // Now, write the function to the console. The function is a set of bins, and the corresponding - // function values. You can think of GAMs as building a bar-chart or lookup table for each feature. + // Now, write the function to the console. The function is a set of + // bins, and the corresponding function values. You can think of + // GAMs as building a bar-chart or lookup table for each feature. Console.WriteLine($"Feature{i}"); for (int j = 0; j < binUpperBounds.Count; j++) - Console.WriteLine($"x < {binUpperBounds[j]:0.00} => {binEffects[j]:0.000}"); + Console.WriteLine( + $"x < {binUpperBounds[j]:0.00} => {binEffects[j]:0.000}"); } // Expected output: @@ -99,18 +108,23 @@ public static void Example() // x < 0.31 => -0.138 // x < ∞ => -0.188 - // Let's consider this output. To score a given example, we look up the first bin where the inequality - // is satisfied for the feature value. We can look at the whole function to get a sense for how the - // model responds to the variable on a global level. - // The model can be seen to reconstruct the parabolic and step-wise function, shifted with respect to the average - // expected output over the training set. Very few bins are used to model the second feature because the GAM model - // discards unchanged bins to create smaller models. - // One last thing to notice is that these feature functions can be noisy. While we know that Feature1 should be - // symmetric, this is not captured in the model. This is due to noise in the data. Common practice is to use - // resampling methods to estimate a confidence interval at each bin. This will help to determine if the effect is - // real or just sampling noise. See for example: - // Tan, Caruana, Hooker, and Lou. "Distill-and-Compare: Auditing Black-Box Models Using Transparent Model - // Distillation." arXiv:1710.06169." + // Let's consider this output. To score a given example, we look up the + // first bin where the inequality is satisfied for the feature value. + // We can look at the whole function to get a sense for how the model + // responds to the variable on a global level. The model can be seen to + // reconstruct the parabolic and step-wise function, shifted with + // respect to the average expected output over the training set. Very + // few bins are used to model the second feature because the GAM model + // discards unchanged bins to create smaller models.One last thing to + // notice is that these feature functions can be noisy. While we know + // that Feature1 should be symmetric, this is not captured in the model. + // This is due to noise in the data. Common practice is to use + // resampling methods to estimate a confidence interval at each bin. + // This will help to determine if the effect is real or just sampling + // noise. See for example: Tan, Caruana, Hooker, and Lou. + // "Distill-and-Compare: Auditing Black-Box Models Using Transparent + // Model Distillation." + // arXiv:1710.06169." } private class Data @@ -122,13 +136,17 @@ private class Data } /// - /// Creates a dataset, an IEnumerable of Data objects, for a GAM sample. Feature1 is a parabola centered around 0, - /// while Feature2 is a simple piecewise function. + /// Creates a dataset, an IEnumerable of Data objects, for a GAM sample. + /// Feature1 is a parabola centered around 0, while Feature2 is a simple + /// piecewise function. /// /// The number of examples to generate. - /// The seed for the random number generator used to produce data. + /// The seed for the random number generator used to + /// produce data. /// - private static IEnumerable GenerateData(int numExamples = 25000, int seed = 1) + private static IEnumerable GenerateData(int numExamples = 25000, + int seed = 1) + { var rng = new Random(seed); float centeredFloat() => (float)(rng.NextDouble() - 0.5); @@ -140,7 +158,8 @@ private static IEnumerable GenerateData(int numExamples = 25000, int seed Features = new float[2] { centeredFloat(), centeredFloat() } }; // Compute the label from the shape functions and add noise. - data.Label = Sigmoid(Parabola(data.Features[0]) + SimplePiecewise(data.Features[1]) + centeredFloat()) > 0.5; + data.Label = Sigmoid(Parabola(data.Features[0]) + + SimplePiecewise(data.Features[1]) + centeredFloat()) > 0.5; yield return data; } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/LbfgsLogisticRegression.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/LbfgsLogisticRegression.cs index e762da494b..5aece5a264 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/LbfgsLogisticRegression.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/LbfgsLogisticRegression.cs @@ -10,35 +10,43 @@ public static class LbfgsLogisticRegression { public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); // Define the trainer. - var pipeline = mlContext.BinaryClassification.Trainers.LbfgsLogisticRegression(); + var pipeline = mlContext.BinaryClassification.Trainers + .LbfgsLogisticRegression(); // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(500, seed:123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data + .LoadFromEnumerable(GenerateRandomDataPoints(500, seed:123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(transformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data + .CreateEnumerable(transformedTestData, + reuseRowObject: false).ToList(); // Print 5 predictions. foreach (var p in predictions.Take(5)) - Console.WriteLine($"Label: {p.Label}, Prediction: {p.PredictedLabel}"); + Console.WriteLine($"Label: {p.Label}, " + + $"Prediction: {p.PredictedLabel}"); // Expected output: // Label: True, Prediction: True @@ -48,7 +56,9 @@ public static void Example() // Label: False, Prediction: False // Evaluate the overall metrics. - var metrics = mlContext.BinaryClassification.Evaluate(transformedTestData); + var metrics = mlContext.BinaryClassification + .Evaluate(transformedTestData); + PrintMetrics(metrics); // Expected output: @@ -74,7 +84,9 @@ public static void Example() // Precision || 0.8583 | 0.8972 | } - private static IEnumerable GenerateRandomDataPoints(int count, int seed=0) + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) + { var random = new Random(seed); float randomFloat() => (float)random.NextDouble(); @@ -85,13 +97,18 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se { Label = label, // Create random features that are correlated with the label. - // For data points with false label, the feature values are slightly increased by adding a constant. - Features = Enumerable.Repeat(label, 50).Select(x => x ? randomFloat() : randomFloat() + 0.1f).ToArray() + // For data points with false label, the feature values are + // slightly increased by adding a constant. + Features = Enumerable.Repeat(label, 50) + .Select(x => x ? randomFloat() : randomFloat() + + 0.1f).ToArray() + }; } } - // Example with label and 50 feature values. A data set is a collection of such examples. + // Example with label and 50 feature values. A data set is a collection of + // such examples. private class DataPoint { public bool Label { get; set; } @@ -114,11 +131,16 @@ private static void PrintMetrics(BinaryClassificationMetrics metrics) Console.WriteLine($"Accuracy: {metrics.Accuracy:F2}"); Console.WriteLine($"AUC: {metrics.AreaUnderRocCurve:F2}"); Console.WriteLine($"F1 Score: {metrics.F1Score:F2}"); - Console.WriteLine($"Negative Precision: {metrics.NegativePrecision:F2}"); + Console.WriteLine($"Negative Precision: " + + $"{metrics.NegativePrecision:F2}"); + Console.WriteLine($"Negative Recall: {metrics.NegativeRecall:F2}"); - Console.WriteLine($"Positive Precision: {metrics.PositivePrecision:F2}"); + Console.WriteLine($"Positive Precision: " + + $"{metrics.PositivePrecision:F2}"); + Console.WriteLine($"Positive Recall: {metrics.PositiveRecall:F2}\n"); Console.WriteLine(metrics.ConfusionMatrix.GetFormattedConfusionTable()); } } } + diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/LbfgsLogisticRegressionWithOptions.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/LbfgsLogisticRegressionWithOptions.cs index 4204f0c4c2..fe2fcb14ab 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/LbfgsLogisticRegressionWithOptions.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/LbfgsLogisticRegressionWithOptions.cs @@ -11,15 +11,17 @@ public static class LbfgsLogisticRegressionWithOptions { public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); // Define trainer options. @@ -31,23 +33,29 @@ public static void Example() }; // Define the trainer. - var pipeline = mlContext.BinaryClassification.Trainers.LbfgsLogisticRegression(options); + var pipeline = mlContext.BinaryClassification.Trainers + .LbfgsLogisticRegression(options); // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(500, seed:123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data + .LoadFromEnumerable(GenerateRandomDataPoints(500, seed:123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(transformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data + .CreateEnumerable(transformedTestData, + reuseRowObject: false).ToList(); // Print 5 predictions. foreach (var p in predictions.Take(5)) - Console.WriteLine($"Label: {p.Label}, Prediction: {p.PredictedLabel}"); + Console.WriteLine($"Label: {p.Label}, " + + $"Prediction: {p.PredictedLabel}"); // Expected output: // Label: True, Prediction: True @@ -57,7 +65,9 @@ public static void Example() // Label: False, Prediction: False // Evaluate the overall metrics. - var metrics = mlContext.BinaryClassification.Evaluate(transformedTestData); + var metrics = mlContext.BinaryClassification + .Evaluate(transformedTestData); + PrintMetrics(metrics); // Expected output: @@ -83,7 +93,9 @@ public static void Example() // Precision || 0.8571 | 0.8902 | } - private static IEnumerable GenerateRandomDataPoints(int count, int seed=0) + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) + { var random = new Random(seed); float randomFloat() => (float)random.NextDouble(); @@ -94,13 +106,18 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se { Label = label, // Create random features that are correlated with the label. - // For data points with false label, the feature values are slightly increased by adding a constant. - Features = Enumerable.Repeat(label, 50).Select(x => x ? randomFloat() : randomFloat() + 0.1f).ToArray() + // For data points with false label, the feature values are + // slightly increased by adding a constant. + Features = Enumerable.Repeat(label, 50) + .Select(x => x ? randomFloat() : randomFloat() + + 0.1f).ToArray() + }; } } - // Example with label and 50 feature values. A data set is a collection of such examples. + // Example with label and 50 feature values. A data set is a collection of + // such examples. private class DataPoint { public bool Label { get; set; } @@ -123,11 +140,16 @@ private static void PrintMetrics(BinaryClassificationMetrics metrics) Console.WriteLine($"Accuracy: {metrics.Accuracy:F2}"); Console.WriteLine($"AUC: {metrics.AreaUnderRocCurve:F2}"); Console.WriteLine($"F1 Score: {metrics.F1Score:F2}"); - Console.WriteLine($"Negative Precision: {metrics.NegativePrecision:F2}"); + Console.WriteLine($"Negative Precision: " + + $"{metrics.NegativePrecision:F2}"); + Console.WriteLine($"Negative Recall: {metrics.NegativeRecall:F2}"); - Console.WriteLine($"Positive Precision: {metrics.PositivePrecision:F2}"); + Console.WriteLine($"Positive Precision: " + + $"{metrics.PositivePrecision:F2}"); + Console.WriteLine($"Positive Recall: {metrics.PositiveRecall:F2}\n"); Console.WriteLine(metrics.ConfusionMatrix.GetFormattedConfusionTable()); } } } + diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/LightGbm.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/LightGbm.cs index 4bf27d017a..c89a4b2b2c 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/LightGbm.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/LightGbm.cs @@ -8,39 +8,48 @@ namespace Samples.Dynamic.Trainers.BinaryClassification { public static class LightGbm { - // This example requires installation of additional nuget package - // Microsoft.ML.LightGbm. + // This example requires installation of additional NuGet package for + // Microsoft.ML.FastTree at + // https://www.nuget.org/packages/Microsoft.ML.FastTree/ public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); // Define the trainer. - var pipeline = mlContext.BinaryClassification.Trainers.LightGbm(); + var pipeline = mlContext.BinaryClassification.Trainers + .LightGbm(); // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(500, seed:123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data + .LoadFromEnumerable(GenerateRandomDataPoints(500, seed:123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(transformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data + .CreateEnumerable(transformedTestData, + reuseRowObject: false).ToList(); // Print 5 predictions. foreach (var p in predictions.Take(5)) - Console.WriteLine($"Label: {p.Label}, Prediction: {p.PredictedLabel}"); + Console.WriteLine($"Label: {p.Label}, " + + $"Prediction: {p.PredictedLabel}"); // Expected output: // Label: True, Prediction: True @@ -50,7 +59,9 @@ public static void Example() // Label: False, Prediction: False // Evaluate the overall metrics. - var metrics = mlContext.BinaryClassification.Evaluate(transformedTestData); + var metrics = mlContext.BinaryClassification + .Evaluate(transformedTestData); + PrintMetrics(metrics); // Expected output: @@ -73,7 +84,9 @@ public static void Example() // Precision || 0.7531 | 0.7860 | } - private static IEnumerable GenerateRandomDataPoints(int count, int seed=0) + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) + { var random = new Random(seed); float randomFloat() => (float)random.NextDouble(); @@ -84,13 +97,18 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se { Label = label, // Create random features that are correlated with the label. - // For data points with false label, the feature values are slightly increased by adding a constant. - Features = Enumerable.Repeat(label, 50).Select(x => x ? randomFloat() : randomFloat() + 0.03f).ToArray() + // For data points with false label, the feature values are + // slightly increased by adding a constant. + Features = Enumerable.Repeat(label, 50) + .Select(x => x ? randomFloat() : randomFloat() + + 0.03f).ToArray() + }; } } - // Example with label and 50 feature values. A data set is a collection of such examples. + // Example with label and 50 feature values. A data set is a collection of + // such examples. private class DataPoint { public bool Label { get; set; } @@ -113,11 +131,16 @@ private static void PrintMetrics(BinaryClassificationMetrics metrics) Console.WriteLine($"Accuracy: {metrics.Accuracy:F2}"); Console.WriteLine($"AUC: {metrics.AreaUnderRocCurve:F2}"); Console.WriteLine($"F1 Score: {metrics.F1Score:F2}"); - Console.WriteLine($"Negative Precision: {metrics.NegativePrecision:F2}"); + Console.WriteLine($"Negative Precision: " + + $"{metrics.NegativePrecision:F2}"); + Console.WriteLine($"Negative Recall: {metrics.NegativeRecall:F2}"); - Console.WriteLine($"Positive Precision: {metrics.PositivePrecision:F2}"); + Console.WriteLine($"Positive Precision: " + + $"{metrics.PositivePrecision:F2}"); + Console.WriteLine($"Positive Recall: {metrics.PositiveRecall:F2}\n"); Console.WriteLine(metrics.ConfusionMatrix.GetFormattedConfusionTable()); } } } + diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/LightGbm.tt b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/LightGbm.tt index 8bef899d83..2de4012354 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/LightGbm.tt +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/LightGbm.tt @@ -10,8 +10,9 @@ string LabelThreshold = "0.5f"; string DataSepValue = "0.03f"; string OptionsInclude = ""; string Comments= @" - // This example requires installation of additional nuget package - // Microsoft.ML.LightGbm."; + // This example requires installation of additional NuGet package for + // Microsoft.ML.FastTree at + // https://www.nuget.org/packages/Microsoft.ML.FastTree/"; string ExpectedOutputPerInstance = @"// Expected output: // Label: True, Prediction: True diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/LightGbmWithOptions.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/LightGbmWithOptions.cs index 82bdace764..5df0a59ee5 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/LightGbmWithOptions.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/LightGbmWithOptions.cs @@ -9,19 +9,22 @@ namespace Samples.Dynamic.Trainers.BinaryClassification { public static class LightGbmWithOptions { - // This example requires installation of additional nuget package - // Microsoft.ML.LightGbm. + // This example requires installation of additional NuGet package for + // Microsoft.ML.FastTree at + // https://www.nuget.org/packages/Microsoft.ML.FastTree/ public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); // Define trainer options. @@ -35,23 +38,29 @@ public static void Example() }; // Define the trainer. - var pipeline = mlContext.BinaryClassification.Trainers.LightGbm(options); + var pipeline = mlContext.BinaryClassification.Trainers + .LightGbm(options); // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(500, seed:123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data + .LoadFromEnumerable(GenerateRandomDataPoints(500, seed:123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(transformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data + .CreateEnumerable(transformedTestData, + reuseRowObject: false).ToList(); // Print 5 predictions. foreach (var p in predictions.Take(5)) - Console.WriteLine($"Label: {p.Label}, Prediction: {p.PredictedLabel}"); + Console.WriteLine($"Label: {p.Label}, " + + $"Prediction: {p.PredictedLabel}"); // Expected output: // Label: True, Prediction: True @@ -61,7 +70,9 @@ public static void Example() // Label: False, Prediction: False // Evaluate the overall metrics. - var metrics = mlContext.BinaryClassification.Evaluate(transformedTestData); + var metrics = mlContext.BinaryClassification + .Evaluate(transformedTestData); + PrintMetrics(metrics); // Expected output: @@ -84,7 +95,9 @@ public static void Example() // Precision || 0.6563 | 0.7131 | } - private static IEnumerable GenerateRandomDataPoints(int count, int seed=0) + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) + { var random = new Random(seed); float randomFloat() => (float)random.NextDouble(); @@ -95,13 +108,18 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se { Label = label, // Create random features that are correlated with the label. - // For data points with false label, the feature values are slightly increased by adding a constant. - Features = Enumerable.Repeat(label, 50).Select(x => x ? randomFloat() : randomFloat() + 0.03f).ToArray() + // For data points with false label, the feature values are + // slightly increased by adding a constant. + Features = Enumerable.Repeat(label, 50) + .Select(x => x ? randomFloat() : randomFloat() + + 0.03f).ToArray() + }; } } - // Example with label and 50 feature values. A data set is a collection of such examples. + // Example with label and 50 feature values. A data set is a collection of + // such examples. private class DataPoint { public bool Label { get; set; } @@ -124,11 +142,16 @@ private static void PrintMetrics(BinaryClassificationMetrics metrics) Console.WriteLine($"Accuracy: {metrics.Accuracy:F2}"); Console.WriteLine($"AUC: {metrics.AreaUnderRocCurve:F2}"); Console.WriteLine($"F1 Score: {metrics.F1Score:F2}"); - Console.WriteLine($"Negative Precision: {metrics.NegativePrecision:F2}"); + Console.WriteLine($"Negative Precision: " + + $"{metrics.NegativePrecision:F2}"); + Console.WriteLine($"Negative Recall: {metrics.NegativeRecall:F2}"); - Console.WriteLine($"Positive Precision: {metrics.PositivePrecision:F2}"); + Console.WriteLine($"Positive Precision: " + + $"{metrics.PositivePrecision:F2}"); + Console.WriteLine($"Positive Recall: {metrics.PositiveRecall:F2}\n"); Console.WriteLine(metrics.ConfusionMatrix.GetFormattedConfusionTable()); } } } + diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/LightGbmWithOptions.tt b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/LightGbmWithOptions.tt index b193e17b54..3488a6178e 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/LightGbmWithOptions.tt +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/LightGbmWithOptions.tt @@ -9,8 +9,9 @@ string LabelThreshold = "0.5f"; string DataSepValue = "0.03f"; string OptionsInclude = "using Microsoft.ML.Trainers.LightGbm;"; string Comments= @" - // This example requires installation of additional nuget package - // Microsoft.ML.LightGbm."; + // This example requires installation of additional NuGet package for + // Microsoft.ML.FastTree at + // https://www.nuget.org/packages/Microsoft.ML.FastTree/"; string TrainerOptions = @"LightGbmBinaryTrainer.Options { diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/LinearSvm.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/LinearSvm.cs index 80c58da58a..f47279c331 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/LinearSvm.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/LinearSvm.cs @@ -10,35 +10,43 @@ public static class LinearSvm { public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); // Define the trainer. - var pipeline = mlContext.BinaryClassification.Trainers.LinearSvm(); + var pipeline = mlContext.BinaryClassification.Trainers + .LinearSvm(); // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(500, seed:123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data + .LoadFromEnumerable(GenerateRandomDataPoints(500, seed:123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(transformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data + .CreateEnumerable(transformedTestData, + reuseRowObject: false).ToList(); // Print 5 predictions. foreach (var p in predictions.Take(5)) - Console.WriteLine($"Label: {p.Label}, Prediction: {p.PredictedLabel}"); + Console.WriteLine($"Label: {p.Label}, " + + $"Prediction: {p.PredictedLabel}"); // Expected output: // Label: True, Prediction: True @@ -48,7 +56,9 @@ public static void Example() // Label: False, Prediction: True // Evaluate the overall metrics. - var metrics = mlContext.BinaryClassification.EvaluateNonCalibrated(transformedTestData); + var metrics = mlContext.BinaryClassification + .EvaluateNonCalibrated(transformedTestData); + PrintMetrics(metrics); // Expected output: @@ -71,7 +81,9 @@ public static void Example() // Precision || 0.6624 | 0.8387 | } - private static IEnumerable GenerateRandomDataPoints(int count, int seed=0) + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) + { var random = new Random(seed); float randomFloat() => (float)random.NextDouble(); @@ -82,13 +94,18 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se { Label = label, // Create random features that are correlated with the label. - // For data points with false label, the feature values are slightly increased by adding a constant. - Features = Enumerable.Repeat(label, 50).Select(x => x ? randomFloat() : randomFloat() + 0.1f).ToArray() + // For data points with false label, the feature values are + // slightly increased by adding a constant. + Features = Enumerable.Repeat(label, 50) + .Select(x => x ? randomFloat() : randomFloat() + + 0.1f).ToArray() + }; } } - // Example with label and 50 feature values. A data set is a collection of such examples. + // Example with label and 50 feature values. A data set is a collection of + // such examples. private class DataPoint { public bool Label { get; set; } @@ -111,11 +128,16 @@ private static void PrintMetrics(BinaryClassificationMetrics metrics) Console.WriteLine($"Accuracy: {metrics.Accuracy:F2}"); Console.WriteLine($"AUC: {metrics.AreaUnderRocCurve:F2}"); Console.WriteLine($"F1 Score: {metrics.F1Score:F2}"); - Console.WriteLine($"Negative Precision: {metrics.NegativePrecision:F2}"); + Console.WriteLine($"Negative Precision: " + + $"{metrics.NegativePrecision:F2}"); + Console.WriteLine($"Negative Recall: {metrics.NegativeRecall:F2}"); - Console.WriteLine($"Positive Precision: {metrics.PositivePrecision:F2}"); + Console.WriteLine($"Positive Precision: " + + $"{metrics.PositivePrecision:F2}"); + Console.WriteLine($"Positive Recall: {metrics.PositiveRecall:F2}\n"); Console.WriteLine(metrics.ConfusionMatrix.GetFormattedConfusionTable()); } } } + diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/LinearSvmWithOptions.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/LinearSvmWithOptions.cs index 293769438b..857ac93439 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/LinearSvmWithOptions.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/LinearSvmWithOptions.cs @@ -11,15 +11,17 @@ public static class LinearSvmWithOptions { public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); // Define trainer options. @@ -31,23 +33,29 @@ public static void Example() }; // Define the trainer. - var pipeline = mlContext.BinaryClassification.Trainers.LinearSvm(options); + var pipeline = mlContext.BinaryClassification.Trainers + .LinearSvm(options); // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(500, seed:123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data + .LoadFromEnumerable(GenerateRandomDataPoints(500, seed:123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(transformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data + .CreateEnumerable(transformedTestData, + reuseRowObject: false).ToList(); // Print 5 predictions. foreach (var p in predictions.Take(5)) - Console.WriteLine($"Label: {p.Label}, Prediction: {p.PredictedLabel}"); + Console.WriteLine($"Label: {p.Label}, " + + $"Prediction: {p.PredictedLabel}"); // Expected output: // Label: True, Prediction: True @@ -57,7 +65,9 @@ public static void Example() // Label: False, Prediction: False // Evaluate the overall metrics. - var metrics = mlContext.BinaryClassification.EvaluateNonCalibrated(transformedTestData); + var metrics = mlContext.BinaryClassification + .EvaluateNonCalibrated(transformedTestData); + PrintMetrics(metrics); // Expected output: @@ -80,7 +90,9 @@ public static void Example() // Precision || 0.8044 | 0.9127 | } - private static IEnumerable GenerateRandomDataPoints(int count, int seed=0) + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) + { var random = new Random(seed); float randomFloat() => (float)random.NextDouble(); @@ -91,13 +103,18 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se { Label = label, // Create random features that are correlated with the label. - // For data points with false label, the feature values are slightly increased by adding a constant. - Features = Enumerable.Repeat(label, 50).Select(x => x ? randomFloat() : randomFloat() + 0.1f).ToArray() + // For data points with false label, the feature values are + // slightly increased by adding a constant. + Features = Enumerable.Repeat(label, 50) + .Select(x => x ? randomFloat() : randomFloat() + + 0.1f).ToArray() + }; } } - // Example with label and 50 feature values. A data set is a collection of such examples. + // Example with label and 50 feature values. A data set is a collection of + // such examples. private class DataPoint { public bool Label { get; set; } @@ -120,11 +137,16 @@ private static void PrintMetrics(BinaryClassificationMetrics metrics) Console.WriteLine($"Accuracy: {metrics.Accuracy:F2}"); Console.WriteLine($"AUC: {metrics.AreaUnderRocCurve:F2}"); Console.WriteLine($"F1 Score: {metrics.F1Score:F2}"); - Console.WriteLine($"Negative Precision: {metrics.NegativePrecision:F2}"); + Console.WriteLine($"Negative Precision: " + + $"{metrics.NegativePrecision:F2}"); + Console.WriteLine($"Negative Recall: {metrics.NegativeRecall:F2}"); - Console.WriteLine($"Positive Precision: {metrics.PositivePrecision:F2}"); + Console.WriteLine($"Positive Precision: " + + $"{metrics.PositivePrecision:F2}"); + Console.WriteLine($"Positive Recall: {metrics.PositiveRecall:F2}\n"); Console.WriteLine(metrics.ConfusionMatrix.GetFormattedConfusionTable()); } } } + diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/MultipleFeatureColumnsBinaryClassification.ttinclude b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/MultipleFeatureColumnsBinaryClassification.ttinclude index ace4bdec1d..d6f7e53a24 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/MultipleFeatureColumnsBinaryClassification.ttinclude +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/MultipleFeatureColumnsBinaryClassification.ttinclude @@ -13,28 +13,32 @@ namespace Samples.Dynamic.Trainers.BinaryClassification {<#=Comments#> public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. IEnumerable data = GenerateRandomDataPoints(500); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(data); <# if (TrainerOptions == null) { #> // Define the trainer. <#=TrainerDescription#> - var pipeline = mlContext.BinaryClassification.Trainers.<#=Trainer#>; + var pipeline = mlContext.BinaryClassification.Trainers + .<#=Trainer#>; <# } else { #> // Define trainer options. var options = new <#=TrainerOptions#>; // Define the trainer. <#=TrainerDescription#> - var pipeline = mlContext.BinaryClassification.Trainers.<#=Trainer#>(options); + var pipeline = mlContext.BinaryClassification.Trainers + .<#=Trainer#>(options); <# } #> // Train the model. @@ -44,7 +48,8 @@ namespace Samples.Dynamic.Trainers.BinaryClassification var transformedTrainingData = model.Transform(trainingData); // Measure the quality of the trained model. - var metrics = mlContext.BinaryClassification.Evaluate(transformedTrainingData); + var metrics = mlContext.BinaryClassification + .Evaluate(transformedTrainingData); // Show the quality metrics. PrintMetrics(metrics); @@ -52,14 +57,19 @@ namespace Samples.Dynamic.Trainers.BinaryClassification <#=ExpectedOutput#> // Create prediction function from the trained model. - var engine = mlContext.Model.CreatePredictionEngine(model); + var engine = mlContext.Model + .CreatePredictionEngine(model); // Make some predictions. foreach(var dataPoint in data.Take(5)) { var result = engine.Predict(dataPoint); - Console.WriteLine($"Actual label: {dataPoint.Label}, predicted label: {result.PredictedLabel}, " + - $"score of being positive class: {result.Score}, and probability of beling positive class: {result.Probability}."); + Console.WriteLine($"Actual label: {dataPoint.Label}, " + + $"predicted label: {result.PredictedLabel}, " + + $"score of being positive class: {result.Score}, " + + $"and probability of beling positive class: " + + $"{result.Probability}."); + } <#=ExpectedOutputPerInstance#> @@ -74,7 +84,8 @@ namespace Samples.Dynamic.Trainers.BinaryClassification // Label. public bool Label { get; set; } - // Features from the first field. Note that different fields can have different numbers of features. + // Features from the first field. Note that different fields can have + // different numbers of features. [VectorType(featureLength)] public float[] Field0 { get; set; } @@ -87,8 +98,8 @@ namespace Samples.Dynamic.Trainers.BinaryClassification public float[] Field2 { get; set; } } - // This class defines objects produced by trained model. The trained model maps - // a DataPoint to a Result. + // This class defines objects produced by trained model. The trained model + // maps a DataPoint to a Result. public class Result { // Label. @@ -102,13 +113,16 @@ namespace Samples.Dynamic.Trainers.BinaryClassification } // Function used to create toy data sets. - private static IEnumerable GenerateRandomDataPoints(int exampleCount, int seed = 0) + private static IEnumerable GenerateRandomDataPoints( + int exampleCount, int seed = 0) + { var rnd = new Random(seed); var data = new List(); for (int i = 0; i < exampleCount; ++i) { - // Initialize an example with a random label and an empty feature vector. + // Initialize an example with a random label and an empty feature + // vector. var sample = new DataPoint() { Label = rnd.Next() % 2 == 0, @@ -118,9 +132,10 @@ namespace Samples.Dynamic.Trainers.BinaryClassification }; // Fill feature vectors according the assigned label. - // Notice that features from different fields have different biases and therefore different distributions. - // In practices such as game recommendation, one may use one field to store features from user profile and - // another field to store features from game profile. + // Notice that features from different fields have different biases + // and therefore different distributions. In practices such as game + // recommendation, one may use one field to store features from user + // profile and another field to store features from game profile. for (int j = 0; j < featureLength; ++j) { var value0 = (float)rnd.NextDouble(); @@ -148,14 +163,20 @@ namespace Samples.Dynamic.Trainers.BinaryClassification } // Function used to show evaluation metrics such as accuracy of predictions. - private static void PrintMetrics(CalibratedBinaryClassificationMetrics metrics) + private static void PrintMetrics( + CalibratedBinaryClassificationMetrics metrics) + { Console.WriteLine($"Accuracy: {metrics.Accuracy:F2}"); Console.WriteLine($"AUC: {metrics.AreaUnderRocCurve:F2}"); Console.WriteLine($"F1 Score: {metrics.F1Score:F2}"); - Console.WriteLine($"Negative Precision: {metrics.NegativePrecision:F2}"); + Console.WriteLine($"Negative Precision: " + + $"{metrics.NegativePrecision:F2}"); + Console.WriteLine($"Negative Recall: {metrics.NegativeRecall:F2}"); - Console.WriteLine($"Positive Precision: {metrics.PositivePrecision:F2}"); + Console.WriteLine($"Positive Precision: " + + $"{metrics.PositivePrecision:F2}"); + Console.WriteLine($"Positive Recall: {metrics.PositiveRecall:F2}"); Console.WriteLine($"Log Loss: {metrics.LogLoss:F2}"); Console.WriteLine($"Log Loss Reduction: {metrics.LogLossReduction:F2}"); diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/PermutationFeatureImportance.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/PermutationFeatureImportance.cs index 55d6c54cc4..f8db811758 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/PermutationFeatureImportance.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/PermutationFeatureImportance.cs @@ -9,8 +9,9 @@ public static class PermutationFeatureImportance { public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. var mlContext = new MLContext(seed:1); // Create sample data. @@ -19,12 +20,15 @@ public static void Example() // Load the sample data as an IDataView. var data = mlContext.Data.LoadFromEnumerable(samples); - // Define a training pipeline that concatenates features into a vector, normalizes them, and then - // trains a linear model. - var featureColumns = new string[] { nameof(Data.Feature1), nameof(Data.Feature2) }; - var pipeline = mlContext.Transforms.Concatenate("Features", featureColumns) - .Append(mlContext.Transforms.NormalizeMinMax("Features")) - .Append(mlContext.BinaryClassification.Trainers.SdcaLogisticRegression()); + // Define a training pipeline that concatenates features into a vector, + // normalizes them, and then trains a linear model. + var featureColumns = + new string[] { nameof(Data.Feature1), nameof(Data.Feature2) }; + var pipeline = mlContext.Transforms + .Concatenate("Features", featureColumns) + .Append(mlContext.Transforms.NormalizeMinMax("Features")) + .Append(mlContext.BinaryClassification.Trainers + .SdcaLogisticRegression()); // Fit the pipeline to the data. var model = pipeline.Fit(data); @@ -35,17 +39,22 @@ public static void Example() // Extract the predictor. var linearPredictor = model.LastTransformer; - // Compute the permutation metrics for the linear model using the normalized data. - var permutationMetrics = mlContext.BinaryClassification.PermutationFeatureImportance( - linearPredictor, transformedData, permutationCount: 30); - - // Now let's look at which features are most important to the model overall. - // Get the feature indices sorted by their impact on AUC. - var sortedIndices = permutationMetrics.Select((metrics, index) => new { index, metrics.AreaUnderRocCurve}) - .OrderByDescending(feature => Math.Abs(feature.AreaUnderRocCurve.Mean)) + // Compute the permutation metrics for the linear model using the + // normalized data. + var permutationMetrics = mlContext.BinaryClassification + .PermutationFeatureImportance(linearPredictor, transformedData, + permutationCount: 30); + + // Now let's look at which features are most important to the model + // overall. Get the feature indices sorted by their impact on AUC. + var sortedIndices = permutationMetrics + .Select((metrics, index) => new { index, metrics.AreaUnderRocCurve}) + .OrderByDescending( + feature => Math.Abs(feature.AreaUnderRocCurve.Mean)) .Select(feature => feature.index); - Console.WriteLine("Feature\tModel Weight\tChange in AUC\t95% Confidence in the Mean Change in AUC"); + Console.WriteLine("Feature\tModel Weight\tChange in AUC" + + "\t95% Confidence in the Mean Change in AUC"); var auc = permutationMetrics.Select(x => x.AreaUnderRocCurve).ToArray(); foreach (int i in sortedIndices) { @@ -76,10 +85,14 @@ private class Data /// linear combination of the features. /// /// The number of examples. - /// The bias, or offset, in the calculation of the label. - /// The weight to multiply the first feature with to compute the label. - /// The weight to multiply the second feature with to compute the label. - /// The seed for generating feature values and label noise. + /// The bias, or offset, in the calculation of the label. + /// + /// The weight to multiply the first feature with to + /// compute the label. + /// The weight to multiply the second feature with to + /// compute the label. + /// The seed for generating feature values and label + /// noise. /// An enumerable of Data objects. private static IEnumerable GenerateData(int nExamples = 10000, double bias = 0, double weight1 = 1, double weight2 = 2, int seed = 1) @@ -94,7 +107,9 @@ private static IEnumerable GenerateData(int nExamples = 10000, }; // Create a noisy label. - var value = (float)(bias + weight1 * data.Feature1 + weight2 * data.Feature2 + rng.NextDouble() - 0.5); + var value = (float)(bias + weight1 * data.Feature1 + weight2 * + data.Feature2 + rng.NextDouble() - 0.5); + data.Label = Sigmoid(value) > 0.5; yield return data; } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/PriorTrainer.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/PriorTrainer.cs index ed9949a6ee..be54597f13 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/PriorTrainer.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/PriorTrainer.cs @@ -10,35 +10,43 @@ public static class Prior { public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); // Define the trainer. - var pipeline = mlContext.BinaryClassification.Trainers.Prior(); + var pipeline = mlContext.BinaryClassification.Trainers + .Prior(); // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(500, seed:123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data + .LoadFromEnumerable(GenerateRandomDataPoints(500, seed:123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(transformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data + .CreateEnumerable(transformedTestData, + reuseRowObject: false).ToList(); // Print 5 predictions. foreach (var p in predictions.Take(5)) - Console.WriteLine($"Label: {p.Label}, Prediction: {p.PredictedLabel}"); + Console.WriteLine($"Label: {p.Label}, " + + $"Prediction: {p.PredictedLabel}"); // Expected output: // Label: True, Prediction: True @@ -48,7 +56,9 @@ public static void Example() // Label: False, Prediction: True // Evaluate the overall metrics. - var metrics = mlContext.BinaryClassification.Evaluate(transformedTestData); + var metrics = mlContext.BinaryClassification + .Evaluate(transformedTestData); + PrintMetrics(metrics); // Expected output: @@ -71,7 +81,9 @@ public static void Example() // Precision || 0.6840 | 0.0000 | } - private static IEnumerable GenerateRandomDataPoints(int count, int seed=0) + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) + { var random = new Random(seed); float randomFloat() => (float)random.NextDouble(); @@ -82,13 +94,18 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se { Label = label, // Create random features that are correlated with the label. - // For data points with false label, the feature values are slightly increased by adding a constant. - Features = Enumerable.Repeat(label, 50).Select(x => x ? randomFloat() : randomFloat() + 0.3f).ToArray() + // For data points with false label, the feature values are + // slightly increased by adding a constant. + Features = Enumerable.Repeat(label, 50) + .Select(x => x ? randomFloat() : randomFloat() + + 0.3f).ToArray() + }; } } - // Example with label and 50 feature values. A data set is a collection of such examples. + // Example with label and 50 feature values. A data set is a collection of + // such examples. private class DataPoint { public bool Label { get; set; } @@ -111,11 +128,16 @@ private static void PrintMetrics(BinaryClassificationMetrics metrics) Console.WriteLine($"Accuracy: {metrics.Accuracy:F2}"); Console.WriteLine($"AUC: {metrics.AreaUnderRocCurve:F2}"); Console.WriteLine($"F1 Score: {metrics.F1Score:F2}"); - Console.WriteLine($"Negative Precision: {metrics.NegativePrecision:F2}"); + Console.WriteLine($"Negative Precision: " + + $"{metrics.NegativePrecision:F2}"); + Console.WriteLine($"Negative Recall: {metrics.NegativeRecall:F2}"); - Console.WriteLine($"Positive Precision: {metrics.PositivePrecision:F2}"); + Console.WriteLine($"Positive Precision: " + + $"{metrics.PositivePrecision:F2}"); + Console.WriteLine($"Positive Recall: {metrics.PositiveRecall:F2}\n"); Console.WriteLine(metrics.ConfusionMatrix.GetFormattedConfusionTable()); } } } + diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/SdcaLogisticRegression.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/SdcaLogisticRegression.cs index f3e56fc24f..521c6e671f 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/SdcaLogisticRegression.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/SdcaLogisticRegression.cs @@ -10,41 +10,51 @@ public static class SdcaLogisticRegression { public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); - // ML.NET doesn't cache data set by default. Therefore, if one reads a data set from a file and accesses it many times, - // it can be slow due to expensive featurization and disk operations. When the considered data can fit into memory, - // a solution is to cache the data in memory. Caching is especially helpful when working with iterative algorithms + // ML.NET doesn't cache data set by default. Therefore, if one reads a + // data set from a file and accesses it many times, it can be slow due + // to expensive featurization and disk operations. When the considered + // data can fit into memory, a solution is to cache the data in memory. + // Caching is especially helpful when working with iterative algorithms // which needs many data passes. trainingData = mlContext.Data.Cache(trainingData); // Define the trainer. - var pipeline = mlContext.BinaryClassification.Trainers.SdcaLogisticRegression(); + var pipeline = mlContext.BinaryClassification.Trainers + .SdcaLogisticRegression(); // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(500, seed:123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data + .LoadFromEnumerable(GenerateRandomDataPoints(500, seed:123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(transformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data + .CreateEnumerable(transformedTestData, + reuseRowObject: false).ToList(); // Print 5 predictions. foreach (var p in predictions.Take(5)) - Console.WriteLine($"Label: {p.Label}, Prediction: {p.PredictedLabel}"); + Console.WriteLine($"Label: {p.Label}, " + + $"Prediction: {p.PredictedLabel}"); // Expected output: // Label: True, Prediction: True @@ -54,7 +64,9 @@ public static void Example() // Label: False, Prediction: True // Evaluate the overall metrics. - var metrics = mlContext.BinaryClassification.Evaluate(transformedTestData); + var metrics = mlContext.BinaryClassification + .Evaluate(transformedTestData); + PrintMetrics(metrics); // Expected output: @@ -77,7 +89,9 @@ public static void Example() // Precision || 0.6210 | 0.6667 | } - private static IEnumerable GenerateRandomDataPoints(int count, int seed=0) + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) + { var random = new Random(seed); float randomFloat() => (float)random.NextDouble(); @@ -88,13 +102,18 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se { Label = label, // Create random features that are correlated with the label. - // For data points with false label, the feature values are slightly increased by adding a constant. - Features = Enumerable.Repeat(label, 50).Select(x => x ? randomFloat() : randomFloat() + 0.03f).ToArray() + // For data points with false label, the feature values are + // slightly increased by adding a constant. + Features = Enumerable.Repeat(label, 50) + .Select(x => x ? randomFloat() : randomFloat() + + 0.03f).ToArray() + }; } } - // Example with label and 50 feature values. A data set is a collection of such examples. + // Example with label and 50 feature values. A data set is a collection of + // such examples. private class DataPoint { public bool Label { get; set; } @@ -117,11 +136,16 @@ private static void PrintMetrics(BinaryClassificationMetrics metrics) Console.WriteLine($"Accuracy: {metrics.Accuracy:F2}"); Console.WriteLine($"AUC: {metrics.AreaUnderRocCurve:F2}"); Console.WriteLine($"F1 Score: {metrics.F1Score:F2}"); - Console.WriteLine($"Negative Precision: {metrics.NegativePrecision:F2}"); + Console.WriteLine($"Negative Precision: " + + $"{metrics.NegativePrecision:F2}"); + Console.WriteLine($"Negative Recall: {metrics.NegativeRecall:F2}"); - Console.WriteLine($"Positive Precision: {metrics.PositivePrecision:F2}"); + Console.WriteLine($"Positive Precision: " + + $"{metrics.PositivePrecision:F2}"); + Console.WriteLine($"Positive Recall: {metrics.PositiveRecall:F2}\n"); Console.WriteLine(metrics.ConfusionMatrix.GetFormattedConfusionTable()); } } } + diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/SdcaLogisticRegressionWithOptions.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/SdcaLogisticRegressionWithOptions.cs index 8e564fb8e3..9ef70a2193 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/SdcaLogisticRegressionWithOptions.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/SdcaLogisticRegressionWithOptions.cs @@ -11,20 +11,24 @@ public static class SdcaLogisticRegressionWithOptions { public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); - // ML.NET doesn't cache data set by default. Therefore, if one reads a data set from a file and accesses it many times, - // it can be slow due to expensive featurization and disk operations. When the considered data can fit into memory, - // a solution is to cache the data in memory. Caching is especially helpful when working with iterative algorithms + // ML.NET doesn't cache data set by default. Therefore, if one reads a + // data set from a file and accesses it many times, it can be slow due + // to expensive featurization and disk operations. When the considered + // data can fit into memory, a solution is to cache the data in memory. + // Caching is especially helpful when working with iterative algorithms // which needs many data passes. trainingData = mlContext.Data.Cache(trainingData); @@ -40,23 +44,29 @@ public static void Example() }; // Define the trainer. - var pipeline = mlContext.BinaryClassification.Trainers.SdcaLogisticRegression(options); + var pipeline = mlContext.BinaryClassification.Trainers + .SdcaLogisticRegression(options); // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(500, seed:123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data + .LoadFromEnumerable(GenerateRandomDataPoints(500, seed:123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(transformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data + .CreateEnumerable(transformedTestData, + reuseRowObject: false).ToList(); // Print 5 predictions. foreach (var p in predictions.Take(5)) - Console.WriteLine($"Label: {p.Label}, Prediction: {p.PredictedLabel}"); + Console.WriteLine($"Label: {p.Label}, " + + $"Prediction: {p.PredictedLabel}"); // Expected output: // Label: True, Prediction: True @@ -66,7 +76,9 @@ public static void Example() // Label: False, Prediction: True // Evaluate the overall metrics. - var metrics = mlContext.BinaryClassification.Evaluate(transformedTestData); + var metrics = mlContext.BinaryClassification + .Evaluate(transformedTestData); + PrintMetrics(metrics); // Expected output: @@ -89,7 +101,9 @@ public static void Example() // Precision || 0.5957 | 0.6726 | } - private static IEnumerable GenerateRandomDataPoints(int count, int seed=0) + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) + { var random = new Random(seed); float randomFloat() => (float)random.NextDouble(); @@ -100,13 +114,18 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se { Label = label, // Create random features that are correlated with the label. - // For data points with false label, the feature values are slightly increased by adding a constant. - Features = Enumerable.Repeat(label, 50).Select(x => x ? randomFloat() : randomFloat() + 0.03f).ToArray() + // For data points with false label, the feature values are + // slightly increased by adding a constant. + Features = Enumerable.Repeat(label, 50) + .Select(x => x ? randomFloat() : randomFloat() + + 0.03f).ToArray() + }; } } - // Example with label and 50 feature values. A data set is a collection of such examples. + // Example with label and 50 feature values. A data set is a collection of + // such examples. private class DataPoint { public bool Label { get; set; } @@ -129,11 +148,16 @@ private static void PrintMetrics(BinaryClassificationMetrics metrics) Console.WriteLine($"Accuracy: {metrics.Accuracy:F2}"); Console.WriteLine($"AUC: {metrics.AreaUnderRocCurve:F2}"); Console.WriteLine($"F1 Score: {metrics.F1Score:F2}"); - Console.WriteLine($"Negative Precision: {metrics.NegativePrecision:F2}"); + Console.WriteLine($"Negative Precision: " + + $"{metrics.NegativePrecision:F2}"); + Console.WriteLine($"Negative Recall: {metrics.NegativeRecall:F2}"); - Console.WriteLine($"Positive Precision: {metrics.PositivePrecision:F2}"); + Console.WriteLine($"Positive Precision: " + + $"{metrics.PositivePrecision:F2}"); + Console.WriteLine($"Positive Recall: {metrics.PositiveRecall:F2}\n"); Console.WriteLine(metrics.ConfusionMatrix.GetFormattedConfusionTable()); } } } + diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/SdcaNonCalibrated.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/SdcaNonCalibrated.cs index 5127c84b28..73107bb4d7 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/SdcaNonCalibrated.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/SdcaNonCalibrated.cs @@ -10,41 +10,51 @@ public static class SdcaNonCalibrated { public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); - // ML.NET doesn't cache data set by default. Therefore, if one reads a data set from a file and accesses it many times, - // it can be slow due to expensive featurization and disk operations. When the considered data can fit into memory, - // a solution is to cache the data in memory. Caching is especially helpful when working with iterative algorithms + // ML.NET doesn't cache data set by default. Therefore, if one reads a + // data set from a file and accesses it many times, it can be slow due + // to expensive featurization and disk operations. When the considered + // data can fit into memory, a solution is to cache the data in memory. + // Caching is especially helpful when working with iterative algorithms // which needs many data passes. trainingData = mlContext.Data.Cache(trainingData); // Define the trainer. - var pipeline = mlContext.BinaryClassification.Trainers.SdcaNonCalibrated(); + var pipeline = mlContext.BinaryClassification.Trainers + .SdcaNonCalibrated(); // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(500, seed:123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data + .LoadFromEnumerable(GenerateRandomDataPoints(500, seed:123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(transformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data + .CreateEnumerable(transformedTestData, + reuseRowObject: false).ToList(); // Print 5 predictions. foreach (var p in predictions.Take(5)) - Console.WriteLine($"Label: {p.Label}, Prediction: {p.PredictedLabel}"); + Console.WriteLine($"Label: {p.Label}, " + + $"Prediction: {p.PredictedLabel}"); // Expected output: // Label: True, Prediction: True @@ -54,7 +64,9 @@ public static void Example() // Label: False, Prediction: True // Evaluate the overall metrics. - var metrics = mlContext.BinaryClassification.EvaluateNonCalibrated(transformedTestData); + var metrics = mlContext.BinaryClassification + .EvaluateNonCalibrated(transformedTestData); + PrintMetrics(metrics); // Expected output: @@ -76,7 +88,9 @@ public static void Example() // Precision || 0.6185 | 0.6653 | } - private static IEnumerable GenerateRandomDataPoints(int count, int seed=0) + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) + { var random = new Random(seed); float randomFloat() => (float)random.NextDouble(); @@ -87,13 +101,18 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se { Label = label, // Create random features that are correlated with the label. - // For data points with false label, the feature values are slightly increased by adding a constant. - Features = Enumerable.Repeat(label, 50).Select(x => x ? randomFloat() : randomFloat() + 0.03f).ToArray() + // For data points with false label, the feature values are + // slightly increased by adding a constant. + Features = Enumerable.Repeat(label, 50) + .Select(x => x ? randomFloat() : randomFloat() + + 0.03f).ToArray() + }; } } - // Example with label and 50 feature values. A data set is a collection of such examples. + // Example with label and 50 feature values. A data set is a collection of + // such examples. private class DataPoint { public bool Label { get; set; } @@ -116,11 +135,16 @@ private static void PrintMetrics(BinaryClassificationMetrics metrics) Console.WriteLine($"Accuracy: {metrics.Accuracy:F2}"); Console.WriteLine($"AUC: {metrics.AreaUnderRocCurve:F2}"); Console.WriteLine($"F1 Score: {metrics.F1Score:F2}"); - Console.WriteLine($"Negative Precision: {metrics.NegativePrecision:F2}"); + Console.WriteLine($"Negative Precision: " + + $"{metrics.NegativePrecision:F2}"); + Console.WriteLine($"Negative Recall: {metrics.NegativeRecall:F2}"); - Console.WriteLine($"Positive Precision: {metrics.PositivePrecision:F2}"); + Console.WriteLine($"Positive Precision: " + + $"{metrics.PositivePrecision:F2}"); + Console.WriteLine($"Positive Recall: {metrics.PositiveRecall:F2}\n"); Console.WriteLine(metrics.ConfusionMatrix.GetFormattedConfusionTable()); } } } + diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/SdcaNonCalibratedWithOptions.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/SdcaNonCalibratedWithOptions.cs index 6d55a0286e..f08d65eed1 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/SdcaNonCalibratedWithOptions.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/SdcaNonCalibratedWithOptions.cs @@ -11,20 +11,24 @@ public static class SdcaNonCalibratedWithOptions { public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); - // ML.NET doesn't cache data set by default. Therefore, if one reads a data set from a file and accesses it many times, - // it can be slow due to expensive featurization and disk operations. When the considered data can fit into memory, - // a solution is to cache the data in memory. Caching is especially helpful when working with iterative algorithms + // ML.NET doesn't cache data set by default. Therefore, if one reads a + // data set from a file and accesses it many times, it can be slow due + // to expensive featurization and disk operations. When the considered + // data can fit into memory, a solution is to cache the data in memory. + // Caching is especially helpful when working with iterative algorithms // which needs many data passes. trainingData = mlContext.Data.Cache(trainingData); @@ -42,23 +46,29 @@ public static void Example() }; // Define the trainer. - var pipeline = mlContext.BinaryClassification.Trainers.SdcaNonCalibrated(options); + var pipeline = mlContext.BinaryClassification.Trainers + .SdcaNonCalibrated(options); // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(500, seed:123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data + .LoadFromEnumerable(GenerateRandomDataPoints(500, seed:123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(transformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data + .CreateEnumerable(transformedTestData, + reuseRowObject: false).ToList(); // Print 5 predictions. foreach (var p in predictions.Take(5)) - Console.WriteLine($"Label: {p.Label}, Prediction: {p.PredictedLabel}"); + Console.WriteLine($"Label: {p.Label}, " + + $"Prediction: {p.PredictedLabel}"); // Expected output: // Label: True, Prediction: False @@ -68,7 +78,9 @@ public static void Example() // Label: False, Prediction: True // Evaluate the overall metrics. - var metrics = mlContext.BinaryClassification.EvaluateNonCalibrated(transformedTestData); + var metrics = mlContext.BinaryClassification + .EvaluateNonCalibrated(transformedTestData); + PrintMetrics(metrics); // Expected output: @@ -91,7 +103,9 @@ public static void Example() // Precision || 0.5705 | 0.6809 | } - private static IEnumerable GenerateRandomDataPoints(int count, int seed=0) + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) + { var random = new Random(seed); float randomFloat() => (float)random.NextDouble(); @@ -102,13 +116,18 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se { Label = label, // Create random features that are correlated with the label. - // For data points with false label, the feature values are slightly increased by adding a constant. - Features = Enumerable.Repeat(label, 50).Select(x => x ? randomFloat() : randomFloat() + 0.03f).ToArray() + // For data points with false label, the feature values are + // slightly increased by adding a constant. + Features = Enumerable.Repeat(label, 50) + .Select(x => x ? randomFloat() : randomFloat() + + 0.03f).ToArray() + }; } } - // Example with label and 50 feature values. A data set is a collection of such examples. + // Example with label and 50 feature values. A data set is a collection of + // such examples. private class DataPoint { public bool Label { get; set; } @@ -131,11 +150,16 @@ private static void PrintMetrics(BinaryClassificationMetrics metrics) Console.WriteLine($"Accuracy: {metrics.Accuracy:F2}"); Console.WriteLine($"AUC: {metrics.AreaUnderRocCurve:F2}"); Console.WriteLine($"F1 Score: {metrics.F1Score:F2}"); - Console.WriteLine($"Negative Precision: {metrics.NegativePrecision:F2}"); + Console.WriteLine($"Negative Precision: " + + $"{metrics.NegativePrecision:F2}"); + Console.WriteLine($"Negative Recall: {metrics.NegativeRecall:F2}"); - Console.WriteLine($"Positive Precision: {metrics.PositivePrecision:F2}"); + Console.WriteLine($"Positive Precision: " + + $"{metrics.PositivePrecision:F2}"); + Console.WriteLine($"Positive Recall: {metrics.PositiveRecall:F2}\n"); Console.WriteLine(metrics.ConfusionMatrix.GetFormattedConfusionTable()); } } } + diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/SgdCalibrated.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/SgdCalibrated.cs index 5eb6a3c947..dcc28ad25c 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/SgdCalibrated.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/SgdCalibrated.cs @@ -10,35 +10,43 @@ public static class SgdCalibrated { public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); // Define the trainer. - var pipeline = mlContext.BinaryClassification.Trainers.SgdCalibrated(); + var pipeline = mlContext.BinaryClassification.Trainers + .SgdCalibrated(); // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(500, seed:123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data + .LoadFromEnumerable(GenerateRandomDataPoints(500, seed:123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(transformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data + .CreateEnumerable(transformedTestData, + reuseRowObject: false).ToList(); // Print 5 predictions. foreach (var p in predictions.Take(5)) - Console.WriteLine($"Label: {p.Label}, Prediction: {p.PredictedLabel}"); + Console.WriteLine($"Label: {p.Label}, " + + $"Prediction: {p.PredictedLabel}"); // Expected output: // Label: True, Prediction: False @@ -48,7 +56,9 @@ public static void Example() // Label: False, Prediction: False // Evaluate the overall metrics. - var metrics = mlContext.BinaryClassification.Evaluate(transformedTestData); + var metrics = mlContext.BinaryClassification + .Evaluate(transformedTestData); + PrintMetrics(metrics); // Expected output: @@ -71,7 +81,9 @@ public static void Example() // Precision || 0.6417 | 0.5763 | } - private static IEnumerable GenerateRandomDataPoints(int count, int seed=0) + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) + { var random = new Random(seed); float randomFloat() => (float)random.NextDouble(); @@ -82,13 +94,18 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se { Label = label, // Create random features that are correlated with the label. - // For data points with false label, the feature values are slightly increased by adding a constant. - Features = Enumerable.Repeat(label, 50).Select(x => x ? randomFloat() : randomFloat() + 0.03f).ToArray() + // For data points with false label, the feature values are + // slightly increased by adding a constant. + Features = Enumerable.Repeat(label, 50) + .Select(x => x ? randomFloat() : randomFloat() + + 0.03f).ToArray() + }; } } - // Example with label and 50 feature values. A data set is a collection of such examples. + // Example with label and 50 feature values. A data set is a collection of + // such examples. private class DataPoint { public bool Label { get; set; } @@ -111,11 +128,16 @@ private static void PrintMetrics(BinaryClassificationMetrics metrics) Console.WriteLine($"Accuracy: {metrics.Accuracy:F2}"); Console.WriteLine($"AUC: {metrics.AreaUnderRocCurve:F2}"); Console.WriteLine($"F1 Score: {metrics.F1Score:F2}"); - Console.WriteLine($"Negative Precision: {metrics.NegativePrecision:F2}"); + Console.WriteLine($"Negative Precision: " + + $"{metrics.NegativePrecision:F2}"); + Console.WriteLine($"Negative Recall: {metrics.NegativeRecall:F2}"); - Console.WriteLine($"Positive Precision: {metrics.PositivePrecision:F2}"); + Console.WriteLine($"Positive Precision: " + + $"{metrics.PositivePrecision:F2}"); + Console.WriteLine($"Positive Recall: {metrics.PositiveRecall:F2}\n"); Console.WriteLine(metrics.ConfusionMatrix.GetFormattedConfusionTable()); } } } + diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/SgdCalibratedWithOptions.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/SgdCalibratedWithOptions.cs index bbd7772ec4..eea72de60a 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/SgdCalibratedWithOptions.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/SgdCalibratedWithOptions.cs @@ -11,15 +11,17 @@ public static class SgdCalibratedWithOptions { public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); // Define trainer options. @@ -34,23 +36,29 @@ public static void Example() }; // Define the trainer. - var pipeline = mlContext.BinaryClassification.Trainers.SgdCalibrated(options); + var pipeline = mlContext.BinaryClassification.Trainers + .SgdCalibrated(options); // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(500, seed:123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data + .LoadFromEnumerable(GenerateRandomDataPoints(500, seed:123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(transformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data + .CreateEnumerable(transformedTestData, + reuseRowObject: false).ToList(); // Print 5 predictions. foreach (var p in predictions.Take(5)) - Console.WriteLine($"Label: {p.Label}, Prediction: {p.PredictedLabel}"); + Console.WriteLine($"Label: {p.Label}, " + + $"Prediction: {p.PredictedLabel}"); // Expected output: // Label: True, Prediction: False @@ -60,7 +68,9 @@ public static void Example() // Label: False, Prediction: False // Evaluate the overall metrics. - var metrics = mlContext.BinaryClassification.Evaluate(transformedTestData); + var metrics = mlContext.BinaryClassification + .Evaluate(transformedTestData); + PrintMetrics(metrics); // Expected output: @@ -83,7 +93,9 @@ public static void Example() // Precision || 0.5412 | 0.6625 | } - private static IEnumerable GenerateRandomDataPoints(int count, int seed=0) + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) + { var random = new Random(seed); float randomFloat() => (float)random.NextDouble(); @@ -94,13 +106,18 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se { Label = label, // Create random features that are correlated with the label. - // For data points with false label, the feature values are slightly increased by adding a constant. - Features = Enumerable.Repeat(label, 50).Select(x => x ? randomFloat() : randomFloat() + 0.03f).ToArray() + // For data points with false label, the feature values are + // slightly increased by adding a constant. + Features = Enumerable.Repeat(label, 50) + .Select(x => x ? randomFloat() : randomFloat() + + 0.03f).ToArray() + }; } } - // Example with label and 50 feature values. A data set is a collection of such examples. + // Example with label and 50 feature values. A data set is a collection of + // such examples. private class DataPoint { public bool Label { get; set; } @@ -123,11 +140,16 @@ private static void PrintMetrics(BinaryClassificationMetrics metrics) Console.WriteLine($"Accuracy: {metrics.Accuracy:F2}"); Console.WriteLine($"AUC: {metrics.AreaUnderRocCurve:F2}"); Console.WriteLine($"F1 Score: {metrics.F1Score:F2}"); - Console.WriteLine($"Negative Precision: {metrics.NegativePrecision:F2}"); + Console.WriteLine($"Negative Precision: " + + $"{metrics.NegativePrecision:F2}"); + Console.WriteLine($"Negative Recall: {metrics.NegativeRecall:F2}"); - Console.WriteLine($"Positive Precision: {metrics.PositivePrecision:F2}"); + Console.WriteLine($"Positive Precision: " + + $"{metrics.PositivePrecision:F2}"); + Console.WriteLine($"Positive Recall: {metrics.PositiveRecall:F2}\n"); Console.WriteLine(metrics.ConfusionMatrix.GetFormattedConfusionTable()); } } } + diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/SgdNonCalibrated.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/SgdNonCalibrated.cs index 91cee50153..e55f8d2d65 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/SgdNonCalibrated.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/SgdNonCalibrated.cs @@ -10,35 +10,43 @@ public static class SgdNonCalibrated { public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); // Define the trainer. - var pipeline = mlContext.BinaryClassification.Trainers.SgdNonCalibrated(); + var pipeline = mlContext.BinaryClassification.Trainers + .SgdNonCalibrated(); // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(500, seed:123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data + .LoadFromEnumerable(GenerateRandomDataPoints(500, seed:123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(transformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data + .CreateEnumerable(transformedTestData, + reuseRowObject: false).ToList(); // Print 5 predictions. foreach (var p in predictions.Take(5)) - Console.WriteLine($"Label: {p.Label}, Prediction: {p.PredictedLabel}"); + Console.WriteLine($"Label: {p.Label}, " + + $"Prediction: {p.PredictedLabel}"); // Expected output: // Label: True, Prediction: False @@ -48,7 +56,9 @@ public static void Example() // Label: False, Prediction: False // Evaluate the overall metrics. - var metrics = mlContext.BinaryClassification.EvaluateNonCalibrated(transformedTestData); + var metrics = mlContext.BinaryClassification + .EvaluateNonCalibrated(transformedTestData); + PrintMetrics(metrics); // Expected output: @@ -71,7 +81,9 @@ public static void Example() // Precision || 0.6441 | 0.5759 | } - private static IEnumerable GenerateRandomDataPoints(int count, int seed=0) + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) + { var random = new Random(seed); float randomFloat() => (float)random.NextDouble(); @@ -82,13 +94,18 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se { Label = label, // Create random features that are correlated with the label. - // For data points with false label, the feature values are slightly increased by adding a constant. - Features = Enumerable.Repeat(label, 50).Select(x => x ? randomFloat() : randomFloat() + 0.03f).ToArray() + // For data points with false label, the feature values are + // slightly increased by adding a constant. + Features = Enumerable.Repeat(label, 50) + .Select(x => x ? randomFloat() : randomFloat() + + 0.03f).ToArray() + }; } } - // Example with label and 50 feature values. A data set is a collection of such examples. + // Example with label and 50 feature values. A data set is a collection of + // such examples. private class DataPoint { public bool Label { get; set; } @@ -111,11 +128,16 @@ private static void PrintMetrics(BinaryClassificationMetrics metrics) Console.WriteLine($"Accuracy: {metrics.Accuracy:F2}"); Console.WriteLine($"AUC: {metrics.AreaUnderRocCurve:F2}"); Console.WriteLine($"F1 Score: {metrics.F1Score:F2}"); - Console.WriteLine($"Negative Precision: {metrics.NegativePrecision:F2}"); + Console.WriteLine($"Negative Precision: " + + $"{metrics.NegativePrecision:F2}"); + Console.WriteLine($"Negative Recall: {metrics.NegativeRecall:F2}"); - Console.WriteLine($"Positive Precision: {metrics.PositivePrecision:F2}"); + Console.WriteLine($"Positive Precision: " + + $"{metrics.PositivePrecision:F2}"); + Console.WriteLine($"Positive Recall: {metrics.PositiveRecall:F2}\n"); Console.WriteLine(metrics.ConfusionMatrix.GetFormattedConfusionTable()); } } } + diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/SgdNonCalibratedWithOptions.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/SgdNonCalibratedWithOptions.cs index 5509e6f426..1729433565 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/SgdNonCalibratedWithOptions.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/SgdNonCalibratedWithOptions.cs @@ -11,15 +11,17 @@ public static class SgdNonCalibratedWithOptions { public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); // Define trainer options. @@ -31,23 +33,29 @@ public static void Example() }; // Define the trainer. - var pipeline = mlContext.BinaryClassification.Trainers.SgdNonCalibrated(options); + var pipeline = mlContext.BinaryClassification.Trainers + .SgdNonCalibrated(options); // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(500, seed:123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data + .LoadFromEnumerable(GenerateRandomDataPoints(500, seed:123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(transformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data + .CreateEnumerable(transformedTestData, + reuseRowObject: false).ToList(); // Print 5 predictions. foreach (var p in predictions.Take(5)) - Console.WriteLine($"Label: {p.Label}, Prediction: {p.PredictedLabel}"); + Console.WriteLine($"Label: {p.Label}, " + + $"Prediction: {p.PredictedLabel}"); // Expected output: // Label: True, Prediction: False @@ -57,7 +65,9 @@ public static void Example() // Label: False, Prediction: False // Evaluate the overall metrics. - var metrics = mlContext.BinaryClassification.EvaluateNonCalibrated(transformedTestData); + var metrics = mlContext.BinaryClassification + .EvaluateNonCalibrated(transformedTestData); + PrintMetrics(metrics); // Expected output: @@ -80,7 +90,9 @@ public static void Example() // Precision || 0.5373 | 0.5878 | } - private static IEnumerable GenerateRandomDataPoints(int count, int seed=0) + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) + { var random = new Random(seed); float randomFloat() => (float)random.NextDouble(); @@ -91,13 +103,18 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se { Label = label, // Create random features that are correlated with the label. - // For data points with false label, the feature values are slightly increased by adding a constant. - Features = Enumerable.Repeat(label, 50).Select(x => x ? randomFloat() : randomFloat() + 0.03f).ToArray() + // For data points with false label, the feature values are + // slightly increased by adding a constant. + Features = Enumerable.Repeat(label, 50) + .Select(x => x ? randomFloat() : randomFloat() + + 0.03f).ToArray() + }; } } - // Example with label and 50 feature values. A data set is a collection of such examples. + // Example with label and 50 feature values. A data set is a collection of + // such examples. private class DataPoint { public bool Label { get; set; } @@ -120,11 +137,16 @@ private static void PrintMetrics(BinaryClassificationMetrics metrics) Console.WriteLine($"Accuracy: {metrics.Accuracy:F2}"); Console.WriteLine($"AUC: {metrics.AreaUnderRocCurve:F2}"); Console.WriteLine($"F1 Score: {metrics.F1Score:F2}"); - Console.WriteLine($"Negative Precision: {metrics.NegativePrecision:F2}"); + Console.WriteLine($"Negative Precision: " + + $"{metrics.NegativePrecision:F2}"); + Console.WriteLine($"Negative Recall: {metrics.NegativeRecall:F2}"); - Console.WriteLine($"Positive Precision: {metrics.PositivePrecision:F2}"); + Console.WriteLine($"Positive Precision: " + + $"{metrics.PositivePrecision:F2}"); + Console.WriteLine($"Positive Recall: {metrics.PositiveRecall:F2}\n"); Console.WriteLine(metrics.ConfusionMatrix.GetFormattedConfusionTable()); } } } + diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/SymbolicSgdLogisticRegression.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/SymbolicSgdLogisticRegression.cs index 243d99cccc..172199df0c 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/SymbolicSgdLogisticRegression.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/SymbolicSgdLogisticRegression.cs @@ -8,39 +8,48 @@ namespace Samples.Dynamic.Trainers.BinaryClassification { public static class SymbolicSgdLogisticRegression { - // This example requires installation of additional NuGet package - // Microsoft.ML.Mkl.Components. + // This example requires installation of additional NuGet package for + // Microsoft.ML.FastTree at + // https://www.nuget.org/packages/Microsoft.ML.FastTree/ public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); // Define the trainer. - var pipeline = mlContext.BinaryClassification.Trainers.SymbolicSgdLogisticRegression(); + var pipeline = mlContext.BinaryClassification.Trainers + .SymbolicSgdLogisticRegression(); // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(500, seed:123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data + .LoadFromEnumerable(GenerateRandomDataPoints(500, seed:123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(transformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data + .CreateEnumerable(transformedTestData, + reuseRowObject: false).ToList(); // Print 5 predictions. foreach (var p in predictions.Take(5)) - Console.WriteLine($"Label: {p.Label}, Prediction: {p.PredictedLabel}"); + Console.WriteLine($"Label: {p.Label}, " + + $"Prediction: {p.PredictedLabel}"); // Expected output: // Label: True, Prediction: False @@ -50,7 +59,9 @@ public static void Example() // Label: False, Prediction: True // Evaluate the overall metrics. - var metrics = mlContext.BinaryClassification.Evaluate(transformedTestData); + var metrics = mlContext.BinaryClassification + .Evaluate(transformedTestData); + PrintMetrics(metrics); // Expected output: @@ -73,7 +84,9 @@ public static void Example() // Precision || 0.8235 | 0.8397 | } - private static IEnumerable GenerateRandomDataPoints(int count, int seed=0) + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) + { var random = new Random(seed); float randomFloat() => (float)random.NextDouble(); @@ -84,13 +97,18 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se { Label = label, // Create random features that are correlated with the label. - // For data points with false label, the feature values are slightly increased by adding a constant. - Features = Enumerable.Repeat(label, 50).Select(x => x ? randomFloat() : randomFloat() + 0.1f).ToArray() + // For data points with false label, the feature values are + // slightly increased by adding a constant. + Features = Enumerable.Repeat(label, 50) + .Select(x => x ? randomFloat() : randomFloat() + + 0.1f).ToArray() + }; } } - // Example with label and 50 feature values. A data set is a collection of such examples. + // Example with label and 50 feature values. A data set is a collection of + // such examples. private class DataPoint { public bool Label { get; set; } @@ -113,11 +131,16 @@ private static void PrintMetrics(BinaryClassificationMetrics metrics) Console.WriteLine($"Accuracy: {metrics.Accuracy:F2}"); Console.WriteLine($"AUC: {metrics.AreaUnderRocCurve:F2}"); Console.WriteLine($"F1 Score: {metrics.F1Score:F2}"); - Console.WriteLine($"Negative Precision: {metrics.NegativePrecision:F2}"); + Console.WriteLine($"Negative Precision: " + + $"{metrics.NegativePrecision:F2}"); + Console.WriteLine($"Negative Recall: {metrics.NegativeRecall:F2}"); - Console.WriteLine($"Positive Precision: {metrics.PositivePrecision:F2}"); + Console.WriteLine($"Positive Precision: " + + $"{metrics.PositivePrecision:F2}"); + Console.WriteLine($"Positive Recall: {metrics.PositiveRecall:F2}\n"); Console.WriteLine(metrics.ConfusionMatrix.GetFormattedConfusionTable()); } } } + diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/SymbolicSgdLogisticRegression.tt b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/SymbolicSgdLogisticRegression.tt index 8374d8c301..71f4e6ed66 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/SymbolicSgdLogisticRegression.tt +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/SymbolicSgdLogisticRegression.tt @@ -10,8 +10,9 @@ string LabelThreshold = "0.5f"; string DataSepValue = "0.1f"; string OptionsInclude = ""; string Comments = @" - // This example requires installation of additional NuGet package - // Microsoft.ML.Mkl.Components."; + // This example requires installation of additional NuGet package for + // Microsoft.ML.FastTree at + // https://www.nuget.org/packages/Microsoft.ML.FastTree/"; string ExpectedOutputPerInstance = @"// Expected output: // Label: True, Prediction: False diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/SymbolicSgdLogisticRegressionWithOptions.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/SymbolicSgdLogisticRegressionWithOptions.cs index c8d179e617..a68fadc8c8 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/SymbolicSgdLogisticRegressionWithOptions.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/SymbolicSgdLogisticRegressionWithOptions.cs @@ -9,19 +9,22 @@ namespace Samples.Dynamic.Trainers.BinaryClassification { public static class SymbolicSgdLogisticRegressionWithOptions { - // This example requires installation of additional NuGet package - // Microsoft.ML.Mkl.Components. + // This example requires installation of additional NuGet package for + // Microsoft.ML.FastTree at + // https://www.nuget.org/packages/Microsoft.ML.FastTree/ public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); // Define trainer options. @@ -33,23 +36,29 @@ public static void Example() }; // Define the trainer. - var pipeline = mlContext.BinaryClassification.Trainers.SymbolicSgdLogisticRegression(options); + var pipeline = mlContext.BinaryClassification.Trainers + .SymbolicSgdLogisticRegression(options); // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(500, seed:123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data + .LoadFromEnumerable(GenerateRandomDataPoints(500, seed:123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(transformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data + .CreateEnumerable(transformedTestData, + reuseRowObject: false).ToList(); // Print 5 predictions. foreach (var p in predictions.Take(5)) - Console.WriteLine($"Label: {p.Label}, Prediction: {p.PredictedLabel}"); + Console.WriteLine($"Label: {p.Label}, " + + $"Prediction: {p.PredictedLabel}"); // Expected output: // Label: True, Prediction: False @@ -59,7 +68,9 @@ public static void Example() // Label: False, Prediction: False // Evaluate the overall metrics. - var metrics = mlContext.BinaryClassification.Evaluate(transformedTestData); + var metrics = mlContext.BinaryClassification + .Evaluate(transformedTestData); + PrintMetrics(metrics); // Expected output: @@ -82,7 +93,9 @@ public static void Example() // Precision || 0.7964 | 0.6847 | } - private static IEnumerable GenerateRandomDataPoints(int count, int seed=0) + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) + { var random = new Random(seed); float randomFloat() => (float)random.NextDouble(); @@ -93,13 +106,18 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se { Label = label, // Create random features that are correlated with the label. - // For data points with false label, the feature values are slightly increased by adding a constant. - Features = Enumerable.Repeat(label, 50).Select(x => x ? randomFloat() : randomFloat() + 0.1f).ToArray() + // For data points with false label, the feature values are + // slightly increased by adding a constant. + Features = Enumerable.Repeat(label, 50) + .Select(x => x ? randomFloat() : randomFloat() + + 0.1f).ToArray() + }; } } - // Example with label and 50 feature values. A data set is a collection of such examples. + // Example with label and 50 feature values. A data set is a collection of + // such examples. private class DataPoint { public bool Label { get; set; } @@ -122,11 +140,16 @@ private static void PrintMetrics(BinaryClassificationMetrics metrics) Console.WriteLine($"Accuracy: {metrics.Accuracy:F2}"); Console.WriteLine($"AUC: {metrics.AreaUnderRocCurve:F2}"); Console.WriteLine($"F1 Score: {metrics.F1Score:F2}"); - Console.WriteLine($"Negative Precision: {metrics.NegativePrecision:F2}"); + Console.WriteLine($"Negative Precision: " + + $"{metrics.NegativePrecision:F2}"); + Console.WriteLine($"Negative Recall: {metrics.NegativeRecall:F2}"); - Console.WriteLine($"Positive Precision: {metrics.PositivePrecision:F2}"); + Console.WriteLine($"Positive Precision: " + + $"{metrics.PositivePrecision:F2}"); + Console.WriteLine($"Positive Recall: {metrics.PositiveRecall:F2}\n"); Console.WriteLine(metrics.ConfusionMatrix.GetFormattedConfusionTable()); } } } + diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/SymbolicSgdLogisticRegressionWithOptions.tt b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/SymbolicSgdLogisticRegressionWithOptions.tt index c1b3ccd0e4..51bb6d0515 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/SymbolicSgdLogisticRegressionWithOptions.tt +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/SymbolicSgdLogisticRegressionWithOptions.tt @@ -9,8 +9,9 @@ string LabelThreshold = "0.5f"; string DataSepValue = "0.1f"; string OptionsInclude = "using Microsoft.ML.Trainers;"; string Comments = @" - // This example requires installation of additional NuGet package - // Microsoft.ML.Mkl.Components."; + // This example requires installation of additional NuGet package for + // Microsoft.ML.FastTree at + // https://www.nuget.org/packages/Microsoft.ML.FastTree/"; string TrainerOptions = @"SymbolicSgdLogisticRegressionBinaryTrainer.Options() { diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/TreeSamplesTemplate.ttinclude b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/TreeSamplesTemplate.ttinclude index fa37323b89..4cda1fc808 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/TreeSamplesTemplate.ttinclude +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/TreeSamplesTemplate.ttinclude @@ -4,6 +4,7 @@ string LabelThreshold = "0.5f"; string DataSepValue = "0.03f"; string OptionsInclude = "using Microsoft.ML.Trainers.FastTree;"; string Comments= @" - // This example requires installation of additional NuGet package - // Microsoft.ML.FastTree."; + // This example requires installation of additional NuGet package for + // Microsoft.ML.FastTree at + // https://www.nuget.org/packages/Microsoft.ML.FastTree/"; #> \ No newline at end of file diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/LbfgsMaximumEntropy.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/LbfgsMaximumEntropy.cs index ab45d4ba53..7130a7c9ea 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/LbfgsMaximumEntropy.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/LbfgsMaximumEntropy.cs @@ -10,39 +10,48 @@ public static class LbfgsMaximumEntropy { public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); // Define the trainer. var pipeline = - // Convert the string labels into key types. - mlContext.Transforms.Conversion.MapValueToKey(nameof(DataPoint.Label)) - // Apply LbfgsMaximumEntropy multiclass trainer. - .Append(mlContext.MulticlassClassification.Trainers.LbfgsMaximumEntropy()); + // Convert the string labels into key types. + mlContext.Transforms.Conversion + .MapValueToKey(nameof(DataPoint.Label)) + // Apply LbfgsMaximumEntropy multiclass trainer. + .Append(mlContext.MulticlassClassification.Trainers + .LbfgsMaximumEntropy()); // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(500, seed: 123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data + .LoadFromEnumerable(GenerateRandomDataPoints(500, seed: 123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(transformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data + .CreateEnumerable(transformedTestData, + reuseRowObject: false).ToList(); // Look at 5 predictions foreach (var p in predictions.Take(5)) - Console.WriteLine($"Label: {p.Label}, Prediction: {p.PredictedLabel}"); + Console.WriteLine($"Label: {p.Label}, " + + $"Prediction: {p.PredictedLabel}"); // Expected output: // Label: 1, Prediction: 1 @@ -52,7 +61,9 @@ public static void Example() // Label: 3, Prediction: 3 // Evaluate the overall metrics - var metrics = mlContext.MulticlassClassification.Evaluate(transformedTestData); + var metrics = mlContext.MulticlassClassification + .Evaluate(transformedTestData); + PrintMetrics(metrics); // Expected output: @@ -72,8 +83,11 @@ public static void Example() // Precision ||0.9308 |0.9593 |0.8580 | } - // Generates random uniform doubles in [-0.5, 0.5) range with labels 1, 2 or 3. - private static IEnumerable GenerateRandomDataPoints(int count, int seed=0) + // Generates random uniform doubles in [-0.5, 0.5) + // range with labels 1, 2 or 3. + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) + { var random = new Random(seed); float randomFloat() => (float)(random.NextDouble() - 0.5); @@ -85,13 +99,17 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se { Label = (uint)label, // Create random features that are correlated with the label. - // The feature values are slightly increased by adding a constant multiple of label. - Features = Enumerable.Repeat(label, 20).Select(x => randomFloat() + label * 0.2f).ToArray() + // The feature values are slightly increased by adding a + // constant multiple of label. + Features = Enumerable.Repeat(label, 20) + .Select(x => randomFloat() + label * 0.2f).ToArray() + }; } } - // Example with label and 20 feature values. A data set is a collection of such examples. + // Example with label and 20 feature values. A data set is a collection of + // such examples. private class DataPoint { public uint Label { get; set; } @@ -114,8 +132,11 @@ public static void PrintMetrics(MulticlassClassificationMetrics metrics) Console.WriteLine($"Micro Accuracy: {metrics.MicroAccuracy:F2}"); Console.WriteLine($"Macro Accuracy: {metrics.MacroAccuracy:F2}"); Console.WriteLine($"Log Loss: {metrics.LogLoss:F2}"); - Console.WriteLine($"Log Loss Reduction: {metrics.LogLossReduction:F2}\n"); + Console.WriteLine( + $"Log Loss Reduction: {metrics.LogLossReduction:F2}\n"); + Console.WriteLine(metrics.ConfusionMatrix.GetFormattedConfusionTable()); } } } + diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/LbfgsMaximumEntropy.tt b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/LbfgsMaximumEntropy.tt index a58f4f0917..519c549bd9 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/LbfgsMaximumEntropy.tt +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/LbfgsMaximumEntropy.tt @@ -8,7 +8,8 @@ string TrainerOptions = null; string OptionsInclude = ""; string Comments = ""; bool CacheData = false; -string DataGenerationComments= "// Generates random uniform doubles in [-0.5, 0.5) range with labels 1, 2 or 3."; +string DataGenerationComments= "// Generates random uniform doubles in [-0.5, 0.5)" + + "\n // range with labels 1, 2 or 3."; string ExpectedOutputPerInstance = @"// Expected output: // Label: 1, Prediction: 1 diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/LbfgsMaximumEntropyWithOptions.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/LbfgsMaximumEntropyWithOptions.cs index a818057677..597f34af06 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/LbfgsMaximumEntropyWithOptions.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/LbfgsMaximumEntropyWithOptions.cs @@ -11,15 +11,17 @@ public static class LbfgsMaximumEntropyWithOptions { public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); // Define trainer options. @@ -32,27 +34,32 @@ public static void Example() // Define the trainer. var pipeline = - // Convert the string labels into key types. - mlContext.Transforms.Conversion.MapValueToKey("Label") - // Apply LbfgsMaximumEntropy multiclass trainer. - .Append(mlContext.MulticlassClassification.Trainers.LbfgsMaximumEntropy(options)); - + // Convert the string labels into key types. + mlContext.Transforms.Conversion.MapValueToKey("Label") + // Apply LbfgsMaximumEntropy multiclass trainer. + .Append(mlContext.MulticlassClassification.Trainers + .LbfgsMaximumEntropy(options)); // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(500, seed: 123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data + .LoadFromEnumerable(GenerateRandomDataPoints(500, seed: 123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(transformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data + .CreateEnumerable(transformedTestData, + reuseRowObject: false).ToList(); // Look at 5 predictions foreach (var p in predictions.Take(5)) - Console.WriteLine($"Label: {p.Label}, Prediction: {p.PredictedLabel}"); + Console.WriteLine($"Label: {p.Label}, " + + $"Prediction: {p.PredictedLabel}"); // Expected output: // Label: 1, Prediction: 1 @@ -62,7 +69,9 @@ public static void Example() // Label: 3, Prediction: 3 // Evaluate the overall metrics - var metrics = mlContext.MulticlassClassification.Evaluate(transformedTestData); + var metrics = mlContext.MulticlassClassification + .Evaluate(transformedTestData); + PrintMetrics(metrics); // Expected output: @@ -82,8 +91,11 @@ public static void Example() // Precision ||0.9304 |0.9593 |0.8529 | } - // Generates random uniform doubles in [-0.5, 0.5) range with labels 1, 2 or 3. - private static IEnumerable GenerateRandomDataPoints(int count, int seed=0) + // Generates random uniform doubles in [-0.5, 0.5) + // range with labels 1, 2 or 3. + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) + { var random = new Random(seed); float randomFloat() => (float)(random.NextDouble() - 0.5); @@ -95,13 +107,17 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se { Label = (uint)label, // Create random features that are correlated with the label. - // The feature values are slightly increased by adding a constant multiple of label. - Features = Enumerable.Repeat(label, 20).Select(x => randomFloat() + label * 0.2f).ToArray() + // The feature values are slightly increased by adding a + // constant multiple of label. + Features = Enumerable.Repeat(label, 20) + .Select(x => randomFloat() + label * 0.2f).ToArray() + }; } } - // Example with label and 20 feature values. A data set is a collection of such examples. + // Example with label and 20 feature values. A data set is a collection of + // such examples. private class DataPoint { public uint Label { get; set; } @@ -124,8 +140,11 @@ public static void PrintMetrics(MulticlassClassificationMetrics metrics) Console.WriteLine($"Micro Accuracy: {metrics.MicroAccuracy:F2}"); Console.WriteLine($"Macro Accuracy: {metrics.MacroAccuracy:F2}"); Console.WriteLine($"Log Loss: {metrics.LogLoss:F2}"); - Console.WriteLine($"Log Loss Reduction: {metrics.LogLossReduction:F2}\n"); + Console.WriteLine( + $"Log Loss Reduction: {metrics.LogLossReduction:F2}\n"); + Console.WriteLine(metrics.ConfusionMatrix.GetFormattedConfusionTable()); } } } + diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/LbfgsMaximumEntropyWithOptions.tt b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/LbfgsMaximumEntropyWithOptions.tt index 12caedbd3b..1e96c1eab9 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/LbfgsMaximumEntropyWithOptions.tt +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/LbfgsMaximumEntropyWithOptions.tt @@ -12,7 +12,9 @@ string TrainerOptions = @"LbfgsMaximumEntropyMulticlassTrainer.Options string OptionsInclude = "using Microsoft.ML.Trainers;"; string Comments = ""; -string DataGenerationComments= "// Generates random uniform doubles in [-0.5, 0.5) range with labels 1, 2 or 3."; +string DataGenerationComments= "// Generates random uniform doubles in [-0.5, 0.5)" + + "\n // range with labels 1, 2 or 3."; + bool CacheData = false; string ExpectedOutputPerInstance = @"// Expected output: diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/LightGbm.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/LightGbm.cs index 84ac92ecef..7ab6d5f80e 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/LightGbm.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/LightGbm.cs @@ -8,43 +8,53 @@ namespace Samples.Dynamic.Trainers.MulticlassClassification { public static class LightGbm { - // This example requires installation of additional NuGet package - // Microsoft.ML.FastTree. + // This example requires installation of additional NuGet package for + // Microsoft.ML.FastTree at + // https://www.nuget.org/packages/Microsoft.ML.FastTree/ public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); // Define the trainer. var pipeline = - // Convert the string labels into key types. - mlContext.Transforms.Conversion.MapValueToKey(nameof(DataPoint.Label)) - // Apply LightGbm multiclass trainer. - .Append(mlContext.MulticlassClassification.Trainers.LightGbm()); + // Convert the string labels into key types. + mlContext.Transforms.Conversion + .MapValueToKey(nameof(DataPoint.Label)) + // Apply LightGbm multiclass trainer. + .Append(mlContext.MulticlassClassification.Trainers + .LightGbm()); // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(500, seed: 123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data + .LoadFromEnumerable(GenerateRandomDataPoints(500, seed: 123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(transformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data + .CreateEnumerable(transformedTestData, + reuseRowObject: false).ToList(); // Look at 5 predictions foreach (var p in predictions.Take(5)) - Console.WriteLine($"Label: {p.Label}, Prediction: {p.PredictedLabel}"); + Console.WriteLine($"Label: {p.Label}, " + + $"Prediction: {p.PredictedLabel}"); // Expected output: // Label: 1, Prediction: 1 @@ -54,7 +64,9 @@ public static void Example() // Label: 3, Prediction: 3 // Evaluate the overall metrics - var metrics = mlContext.MulticlassClassification.Evaluate(transformedTestData); + var metrics = mlContext.MulticlassClassification + .Evaluate(transformedTestData); + PrintMetrics(metrics); // Expected output: @@ -74,8 +86,11 @@ public static void Example() // Precision ||0.9936 |1.0000 |0.9701 | } - // Generates random uniform doubles in [-0.5, 0.5) range with labels 1, 2 or 3. - private static IEnumerable GenerateRandomDataPoints(int count, int seed=0) + // Generates random uniform doubles in [-0.5, 0.5) + // range with labels 1, 2 or 3. + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) + { var random = new Random(seed); float randomFloat() => (float)(random.NextDouble() - 0.5); @@ -87,13 +102,17 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se { Label = (uint)label, // Create random features that are correlated with the label. - // The feature values are slightly increased by adding a constant multiple of label. - Features = Enumerable.Repeat(label, 20).Select(x => randomFloat() + label * 0.2f).ToArray() + // The feature values are slightly increased by adding a + // constant multiple of label. + Features = Enumerable.Repeat(label, 20) + .Select(x => randomFloat() + label * 0.2f).ToArray() + }; } } - // Example with label and 20 feature values. A data set is a collection of such examples. + // Example with label and 20 feature values. A data set is a collection of + // such examples. private class DataPoint { public uint Label { get; set; } @@ -116,8 +135,11 @@ public static void PrintMetrics(MulticlassClassificationMetrics metrics) Console.WriteLine($"Micro Accuracy: {metrics.MicroAccuracy:F2}"); Console.WriteLine($"Macro Accuracy: {metrics.MacroAccuracy:F2}"); Console.WriteLine($"Log Loss: {metrics.LogLoss:F2}"); - Console.WriteLine($"Log Loss Reduction: {metrics.LogLossReduction:F2}\n"); + Console.WriteLine( + $"Log Loss Reduction: {metrics.LogLossReduction:F2}\n"); + Console.WriteLine(metrics.ConfusionMatrix.GetFormattedConfusionTable()); } } } + diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/LightGbm.tt b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/LightGbm.tt index 216ca2a0b6..47944467ba 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/LightGbm.tt +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/LightGbm.tt @@ -7,10 +7,13 @@ string TrainerOptions = null; string OptionsInclude = ""; string Comments = @" - // This example requires installation of additional NuGet package - // Microsoft.ML.FastTree."; + // This example requires installation of additional NuGet package for + // Microsoft.ML.FastTree at + // https://www.nuget.org/packages/Microsoft.ML.FastTree/"; + bool CacheData = false; -string DataGenerationComments= "// Generates random uniform doubles in [-0.5, 0.5) range with labels 1, 2 or 3."; +string DataGenerationComments= "// Generates random uniform doubles in [-0.5, 0.5)" + + "\n // range with labels 1, 2 or 3."; string ExpectedOutputPerInstance = @"// Expected output: // Label: 1, Prediction: 1 diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/LightGbmWithOptions.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/LightGbmWithOptions.cs index 3b4444081c..76337146dc 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/LightGbmWithOptions.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/LightGbmWithOptions.cs @@ -9,19 +9,22 @@ namespace Samples.Dynamic.Trainers.MulticlassClassification { public static class LightGbmWithOptions { - // This example requires installation of additional NuGet package - // Microsoft.ML.FastTree. + // This example requires installation of additional NuGet package for + // Microsoft.ML.FastTree at + // https://www.nuget.org/packages/Microsoft.ML.FastTree/ public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); // Define trainer options. @@ -36,27 +39,32 @@ public static void Example() // Define the trainer. var pipeline = - // Convert the string labels into key types. - mlContext.Transforms.Conversion.MapValueToKey("Label") - // Apply LightGbm multiclass trainer. - .Append(mlContext.MulticlassClassification.Trainers.LightGbm(options)); - + // Convert the string labels into key types. + mlContext.Transforms.Conversion.MapValueToKey("Label") + // Apply LightGbm multiclass trainer. + .Append(mlContext.MulticlassClassification.Trainers + .LightGbm(options)); // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(500, seed: 123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data + .LoadFromEnumerable(GenerateRandomDataPoints(500, seed: 123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(transformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data + .CreateEnumerable(transformedTestData, + reuseRowObject: false).ToList(); // Look at 5 predictions foreach (var p in predictions.Take(5)) - Console.WriteLine($"Label: {p.Label}, Prediction: {p.PredictedLabel}"); + Console.WriteLine($"Label: {p.Label}, " + + $"Prediction: {p.PredictedLabel}"); // Expected output: // Label: 1, Prediction: 1 @@ -66,7 +74,9 @@ public static void Example() // Label: 3, Prediction: 3 // Evaluate the overall metrics - var metrics = mlContext.MulticlassClassification.Evaluate(transformedTestData); + var metrics = mlContext.MulticlassClassification + .Evaluate(transformedTestData); + PrintMetrics(metrics); // Expected output: @@ -86,8 +96,11 @@ public static void Example() // Precision ||0.9936 |1.0000 |0.9419 | } - // Generates random uniform doubles in [-0.5, 0.5) range with labels 1, 2 or 3. - private static IEnumerable GenerateRandomDataPoints(int count, int seed=0) + // Generates random uniform doubles in [-0.5, 0.5) + // range with labels 1, 2 or 3. + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) + { var random = new Random(seed); float randomFloat() => (float)(random.NextDouble() - 0.5); @@ -99,13 +112,17 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se { Label = (uint)label, // Create random features that are correlated with the label. - // The feature values are slightly increased by adding a constant multiple of label. - Features = Enumerable.Repeat(label, 20).Select(x => randomFloat() + label * 0.2f).ToArray() + // The feature values are slightly increased by adding a + // constant multiple of label. + Features = Enumerable.Repeat(label, 20) + .Select(x => randomFloat() + label * 0.2f).ToArray() + }; } } - // Example with label and 20 feature values. A data set is a collection of such examples. + // Example with label and 20 feature values. A data set is a collection of + // such examples. private class DataPoint { public uint Label { get; set; } @@ -128,8 +145,11 @@ public static void PrintMetrics(MulticlassClassificationMetrics metrics) Console.WriteLine($"Micro Accuracy: {metrics.MicroAccuracy:F2}"); Console.WriteLine($"Macro Accuracy: {metrics.MacroAccuracy:F2}"); Console.WriteLine($"Log Loss: {metrics.LogLoss:F2}"); - Console.WriteLine($"Log Loss Reduction: {metrics.LogLossReduction:F2}\n"); + Console.WriteLine( + $"Log Loss Reduction: {metrics.LogLossReduction:F2}\n"); + Console.WriteLine(metrics.ConfusionMatrix.GetFormattedConfusionTable()); } } } + diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/LightGbmWithOptions.tt b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/LightGbmWithOptions.tt index 7ec6706227..b267db33f0 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/LightGbmWithOptions.tt +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/LightGbmWithOptions.tt @@ -14,11 +14,13 @@ string TrainerOptions = @"LightGbmMulticlassTrainer.Options string OptionsInclude = "using Microsoft.ML.Trainers.LightGbm;"; string Comments = @" - // This example requires installation of additional NuGet package - // Microsoft.ML.FastTree."; + // This example requires installation of additional NuGet package for + // Microsoft.ML.FastTree at + // https://www.nuget.org/packages/Microsoft.ML.FastTree/"; bool CacheData = false; -string DataGenerationComments= "// Generates random uniform doubles in [-0.5, 0.5) range with labels 1, 2 or 3."; +string DataGenerationComments= "// Generates random uniform doubles in [-0.5, 0.5)" + + "\n // range with labels 1, 2 or 3."; string ExpectedOutputPerInstance = @"// Expected output: // Label: 1, Prediction: 1 diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/LogLossPerClass.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/LogLossPerClass.cs index 86ab646083..8b1076f830 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/LogLossPerClass.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/LogLossPerClass.cs @@ -10,42 +10,50 @@ public static class LogLossPerClass { public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); // Define the trainer. var pipeline = - // Convert the string labels into key types. - mlContext.Transforms.Conversion.MapValueToKey(nameof(DataPoint.Label)) - // Apply a multiclass trainer. - .Append(mlContext.MulticlassClassification.Trainers.LightGbm()); + // Convert the string labels into key types. + mlContext.Transforms.Conversion + .MapValueToKey(nameof(DataPoint.Label)) + // Apply a multiclass trainer. + .Append(mlContext.MulticlassClassification.Trainers + .LightGbm()); // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(500, seed: 123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data + .LoadFromEnumerable(GenerateRandomDataPoints(500, seed: 123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Evaluate the overall metrics - var metrics = mlContext.MulticlassClassification.Evaluate(transformedTestData); + var metrics = mlContext.MulticlassClassification + .Evaluate(transformedTestData); // Find the original label values. VBuffer keys = default; transformedTestData.Schema["PredictedLabel"].GetKeyValues(ref keys); var originalLabels = keys.DenseValues().ToArray(); for (var i = 0; i < originalLabels.Length; i++) - Console.WriteLine($"LogLoss for label {originalLabels[i]}: {metrics.PerClassLogLoss[i]:F4}"); + Console.WriteLine($"LogLoss for label " + + $"{originalLabels[i]}: {metrics.PerClassLogLoss[i]:F4}"); // Expected output: // LogLoss for label 7: 0.2578 @@ -60,7 +68,9 @@ public static void Example() } // Generates data points with random features and labels 1 to 9. - private static IEnumerable GenerateRandomDataPoints(int count, int seed = 0) + private static IEnumerable GenerateRandomDataPoints(int count, + int seed = 0) + { var random = new Random(seed); float randomFloat() => (float)(random.NextDouble() - 0.5); @@ -72,13 +82,17 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se { Label = (uint)label, // Create random features that are correlated with the label. - // The feature values are slightly increased by adding a constant multiple of label. - Features = Enumerable.Repeat(label, 20).Select(x => randomFloat() + label * 0.2f).ToArray() + // The feature values are slightly increased by adding a + // constant multiple of label. + Features = Enumerable.Repeat(label, 20) + .Select(x => randomFloat() + label * 0.2f).ToArray() + }; } } - // Example with label and 20 feature values. A data set is a collection of such examples. + // Example with label and 20 feature values. A data set is a collection of + // such examples. private class DataPoint { public uint Label { get; set; } @@ -95,4 +109,4 @@ private class Prediction public uint PredictedLabel { get; set; } } } -} +} \ No newline at end of file diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/MulticlassClassification.ttinclude b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/MulticlassClassification.ttinclude index 87201f12a3..ee3f245566 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/MulticlassClassification.ttinclude +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/MulticlassClassification.ttinclude @@ -13,79 +13,97 @@ namespace Samples.Dynamic.Trainers.MulticlassClassification {<#=Comments#> public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); <# if (CacheData) { #> - // ML.NET doesn't cache data set by default. Therefore, if one reads a data set from a file and accesses it many times, - // it can be slow due to expensive featurization and disk operations. When the considered data can fit into memory, - // a solution is to cache the data in memory. Caching is especially helpful when working with iterative algorithms - // which needs many data passes. - trainingData = mlContext.Data.Cache(trainingData); + // ML.NET doesn't cache data set by default. Therefore, if one reads a + // data set from a file and accesses it many times, it can be slow due + // to expensive featurization and disk operations. When the considered + // data can fit into memory, a solution is to cache the data in memory. + // Caching is especially helpful when working with iterative algorithms + // which needs many data passes. + trainingData = mlContext.Data.Cache(trainingData); <# } #> <# if (MetaTrainer != null) { #> // Define the trainer. var pipeline = - // Convert the string labels into key types. - mlContext.Transforms.Conversion.MapValueToKey("Label") - // Apply <#=MetaTrainer#> multiclass meta trainer on top of binary trainer. - .Append(mlContext.MulticlassClassification.Trainers.<#=MetaTrainer#>(<#=Trainer#>())); + // Convert the string labels into key types. + mlContext.Transforms.Conversion.MapValueToKey("Label") + // Apply <#=MetaTrainer#> multiclass meta trainer on top of + // binary trainer. + .Append(mlContext.MulticlassClassification.Trainers + .<#=MetaTrainer#>( + <#=Trainer#>())); <# } else if (TrainerOptions == null) { #> // Define the trainer. var pipeline = - // Convert the string labels into key types. - mlContext.Transforms.Conversion.MapValueToKey(nameof(DataPoint.Label)) - // Apply <#=Trainer#> multiclass trainer. - .Append(mlContext.MulticlassClassification.Trainers.<#=Trainer#>()); + // Convert the string labels into key types. + mlContext.Transforms.Conversion + .MapValueToKey(nameof(DataPoint.Label)) + // Apply <#=Trainer#> multiclass trainer. + .Append(mlContext.MulticlassClassification.Trainers + .<#=Trainer#>()); <# } else { #> // Define trainer options. var options = new <#=TrainerOptions#>; // Define the trainer. var pipeline = - // Convert the string labels into key types. - mlContext.Transforms.Conversion.MapValueToKey("Label") - // Apply <#=Trainer#> multiclass trainer. - .Append(mlContext.MulticlassClassification.Trainers.<#=Trainer#>(options)); - + // Convert the string labels into key types. + mlContext.Transforms.Conversion.MapValueToKey("Label") + // Apply <#=Trainer#> multiclass trainer. + .Append(mlContext.MulticlassClassification.Trainers + .<#=Trainer#>(options)); <# } #> // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(500, seed: 123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data + .LoadFromEnumerable(GenerateRandomDataPoints(500, seed: 123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(transformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data + .CreateEnumerable(transformedTestData, + reuseRowObject: false).ToList(); // Look at 5 predictions foreach (var p in predictions.Take(5)) - Console.WriteLine($"Label: {p.Label}, Prediction: {p.PredictedLabel}"); + Console.WriteLine($"Label: {p.Label}, " + + $"Prediction: {p.PredictedLabel}"); <#=ExpectedOutputPerInstance#> // Evaluate the overall metrics - var metrics = mlContext.MulticlassClassification.Evaluate(transformedTestData); + var metrics = mlContext.MulticlassClassification + .Evaluate(transformedTestData); + PrintMetrics(metrics); <#=ExpectedOutput#> } <#=DataGenerationComments#> - private static IEnumerable GenerateRandomDataPoints(int count, int seed=0) + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) + { var random = new Random(seed); float randomFloat() => (float)(random.NextDouble() - 0.5); @@ -97,13 +115,17 @@ namespace Samples.Dynamic.Trainers.MulticlassClassification { Label = (uint)label, // Create random features that are correlated with the label. - // The feature values are slightly increased by adding a constant multiple of label. - Features = Enumerable.Repeat(label, 20).Select(x => randomFloat() + label * 0.2f).ToArray() + // The feature values are slightly increased by adding a + // constant multiple of label. + Features = Enumerable.Repeat(label, 20) + .Select(x => randomFloat() + label * 0.2f).ToArray() + }; } } - // Example with label and 20 feature values. A data set is a collection of such examples. + // Example with label and 20 feature values. A data set is a collection of + // such examples. private class DataPoint { public uint Label { get; set; } @@ -126,8 +148,10 @@ namespace Samples.Dynamic.Trainers.MulticlassClassification Console.WriteLine($"Micro Accuracy: {metrics.MicroAccuracy:F2}"); Console.WriteLine($"Macro Accuracy: {metrics.MacroAccuracy:F2}"); Console.WriteLine($"Log Loss: {metrics.LogLoss:F2}"); - Console.WriteLine($"Log Loss Reduction: {metrics.LogLossReduction:F2}\n"); + Console.WriteLine( + $"Log Loss Reduction: {metrics.LogLossReduction:F2}\n"); + Console.WriteLine(metrics.ConfusionMatrix.GetFormattedConfusionTable()); } } -} \ No newline at end of file +} diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/NaiveBayes.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/NaiveBayes.cs index d76006bf2c..4dc7f44f9e 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/NaiveBayes.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/NaiveBayes.cs @@ -9,44 +9,55 @@ namespace Samples.Dynamic.Trainers.MulticlassClassification public static class NaiveBayes { // Naive Bayes classifier is based on Bayes' theorem. - // It assumes independence among the presence of features in a class even though they may be dependent on each other. - // It is a multi-class trainer that accepts binary feature values of type float, i.e., feature values are either true or false. - // Specifically a feature value greater than zero is treated as true, zero or less is treated as false. + // It assumes independence among the presence of features in a class even + // though they may be dependent on each other. It is a multi-class trainer + // that accepts binary feature values of type float, i.e., feature values + // are either true or false. Specifically a feature value greater than zero + // is treated as true, zero or less is treated as false. public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); // Define the trainer. var pipeline = - // Convert the string labels into key types. - mlContext.Transforms.Conversion.MapValueToKey(nameof(DataPoint.Label)) - // Apply NaiveBayes multiclass trainer. - .Append(mlContext.MulticlassClassification.Trainers.NaiveBayes()); + // Convert the string labels into key types. + mlContext.Transforms.Conversion + .MapValueToKey(nameof(DataPoint.Label)) + // Apply NaiveBayes multiclass trainer. + .Append(mlContext.MulticlassClassification.Trainers + .NaiveBayes()); // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(500, seed: 123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data + .LoadFromEnumerable(GenerateRandomDataPoints(500, seed: 123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(transformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data + .CreateEnumerable(transformedTestData, + reuseRowObject: false).ToList(); // Look at 5 predictions foreach (var p in predictions.Take(5)) - Console.WriteLine($"Label: {p.Label}, Prediction: {p.PredictedLabel}"); + Console.WriteLine($"Label: {p.Label}, " + + $"Prediction: {p.PredictedLabel}"); // Expected output: // Label: 1, Prediction: 1 @@ -56,7 +67,9 @@ public static void Example() // Label: 3, Prediction: 3 // Evaluate the overall metrics - var metrics = mlContext.MulticlassClassification.Evaluate(transformedTestData); + var metrics = mlContext.MulticlassClassification + .Evaluate(transformedTestData); + PrintMetrics(metrics); // Expected output: @@ -77,9 +90,12 @@ public static void Example() } - // Generates random uniform doubles in [-0.5, 0.5) range with labels 1, 2 or 3. - // For NaiveBayes values greater than zero are treated as true, zero or less are treated as false. - private static IEnumerable GenerateRandomDataPoints(int count, int seed=0) + // Generates random uniform doubles in [-0.5, 0.5) range with labels + // 1, 2 or 3. For NaiveBayes values greater than zero are treated as true, + // zero or less are treated as false. + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) + { var random = new Random(seed); float randomFloat() => (float)(random.NextDouble() - 0.5); @@ -91,13 +107,17 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se { Label = (uint)label, // Create random features that are correlated with the label. - // The feature values are slightly increased by adding a constant multiple of label. - Features = Enumerable.Repeat(label, 20).Select(x => randomFloat() + label * 0.2f).ToArray() + // The feature values are slightly increased by adding a + // constant multiple of label. + Features = Enumerable.Repeat(label, 20) + .Select(x => randomFloat() + label * 0.2f).ToArray() + }; } } - // Example with label and 20 feature values. A data set is a collection of such examples. + // Example with label and 20 feature values. A data set is a collection of + // such examples. private class DataPoint { public uint Label { get; set; } @@ -120,8 +140,11 @@ public static void PrintMetrics(MulticlassClassificationMetrics metrics) Console.WriteLine($"Micro Accuracy: {metrics.MicroAccuracy:F2}"); Console.WriteLine($"Macro Accuracy: {metrics.MacroAccuracy:F2}"); Console.WriteLine($"Log Loss: {metrics.LogLoss:F2}"); - Console.WriteLine($"Log Loss Reduction: {metrics.LogLossReduction:F2}\n"); + Console.WriteLine( + $"Log Loss Reduction: {metrics.LogLossReduction:F2}\n"); + Console.WriteLine(metrics.ConfusionMatrix.GetFormattedConfusionTable()); } } } + diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/NaiveBayes.tt b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/NaiveBayes.tt index a74ad64a80..26e0162b12 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/NaiveBayes.tt +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/NaiveBayes.tt @@ -9,13 +9,16 @@ bool CacheData = false; string OptionsInclude = ""; string Comments= @" // Naive Bayes classifier is based on Bayes' theorem. - // It assumes independence among the presence of features in a class even though they may be dependent on each other. - // It is a multi-class trainer that accepts binary feature values of type float, i.e., feature values are either true or false. - // Specifically a feature value greater than zero is treated as true, zero or less is treated as false."; + // It assumes independence among the presence of features in a class even + // though they may be dependent on each other. It is a multi-class trainer + // that accepts binary feature values of type float, i.e., feature values + // are either true or false. Specifically a feature value greater than zero + // is treated as true, zero or less is treated as false."; string DataGenerationComments= @" - // Generates random uniform doubles in [-0.5, 0.5) range with labels 1, 2 or 3. - // For NaiveBayes values greater than zero are treated as true, zero or less are treated as false."; + // Generates random uniform doubles in [-0.5, 0.5) range with labels + // 1, 2 or 3. For NaiveBayes values greater than zero are treated as true, + // zero or less are treated as false."; string ExpectedOutputPerInstance= @"// Expected output: // Label: 1, Prediction: 1 @@ -39,4 +42,4 @@ string ExpectedOutput = @"// Expected output: // 2 || 9 | 21 | 133 | 0.8160 // ||======================== // Precision ||0.9467 |0.8735 |0.8061 |"; -#> \ No newline at end of file +#> diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/OneVersusAll.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/OneVersusAll.cs index 9b93fd3cd0..40a040325c 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/OneVersusAll.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/OneVersusAll.cs @@ -10,39 +10,49 @@ public static class OneVersusAll { public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); // Define the trainer. var pipeline = - // Convert the string labels into key types. - mlContext.Transforms.Conversion.MapValueToKey("Label") - // Apply OneVersusAll multiclass meta trainer on top of binary trainer. - .Append(mlContext.MulticlassClassification.Trainers.OneVersusAll(mlContext.BinaryClassification.Trainers.SdcaLogisticRegression())); + // Convert the string labels into key types. + mlContext.Transforms.Conversion.MapValueToKey("Label") + // Apply OneVersusAll multiclass meta trainer on top of + // binary trainer. + .Append(mlContext.MulticlassClassification.Trainers + .OneVersusAll( + mlContext.BinaryClassification.Trainers.SdcaLogisticRegression())); // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(500, seed: 123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data + .LoadFromEnumerable(GenerateRandomDataPoints(500, seed: 123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(transformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data + .CreateEnumerable(transformedTestData, + reuseRowObject: false).ToList(); // Look at 5 predictions foreach (var p in predictions.Take(5)) - Console.WriteLine($"Label: {p.Label}, Prediction: {p.PredictedLabel}"); + Console.WriteLine($"Label: {p.Label}, " + + $"Prediction: {p.PredictedLabel}"); // Expected output: // Label: 1, Prediction: 1 @@ -52,7 +62,9 @@ public static void Example() // Label: 3, Prediction: 2 // Evaluate the overall metrics - var metrics = mlContext.MulticlassClassification.Evaluate(transformedTestData); + var metrics = mlContext.MulticlassClassification + .Evaluate(transformedTestData); + PrintMetrics(metrics); // Expected output: @@ -72,8 +84,11 @@ public static void Example() // Precision ||0.8994 |0.9180 |0.8851 | } - // Generates random uniform doubles in [-0.5, 0.5) range with labels 1, 2 or 3. - private static IEnumerable GenerateRandomDataPoints(int count, int seed=0) + // Generates random uniform doubles in [-0.5, 0.5) + // range with labels 1, 2 or 3. + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) + { var random = new Random(seed); float randomFloat() => (float)(random.NextDouble() - 0.5); @@ -85,13 +100,17 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se { Label = (uint)label, // Create random features that are correlated with the label. - // The feature values are slightly increased by adding a constant multiple of label. - Features = Enumerable.Repeat(label, 20).Select(x => randomFloat() + label * 0.2f).ToArray() + // The feature values are slightly increased by adding a + // constant multiple of label. + Features = Enumerable.Repeat(label, 20) + .Select(x => randomFloat() + label * 0.2f).ToArray() + }; } } - // Example with label and 20 feature values. A data set is a collection of such examples. + // Example with label and 20 feature values. A data set is a collection of + // such examples. private class DataPoint { public uint Label { get; set; } @@ -114,8 +133,11 @@ public static void PrintMetrics(MulticlassClassificationMetrics metrics) Console.WriteLine($"Micro Accuracy: {metrics.MicroAccuracy:F2}"); Console.WriteLine($"Macro Accuracy: {metrics.MacroAccuracy:F2}"); Console.WriteLine($"Log Loss: {metrics.LogLoss:F2}"); - Console.WriteLine($"Log Loss Reduction: {metrics.LogLossReduction:F2}\n"); + Console.WriteLine( + $"Log Loss Reduction: {metrics.LogLossReduction:F2}\n"); + Console.WriteLine(metrics.ConfusionMatrix.GetFormattedConfusionTable()); } } } + diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/OneVersusAll.tt b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/OneVersusAll.tt index 0233d9948e..ce355c56b0 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/OneVersusAll.tt +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/OneVersusAll.tt @@ -7,7 +7,8 @@ string TrainerOptions = null; string OptionsInclude = ""; string Comments= ""; -string DataGenerationComments= "// Generates random uniform doubles in [-0.5, 0.5) range with labels 1, 2 or 3."; +string DataGenerationComments= "// Generates random uniform doubles in [-0.5, 0.5)" + + "\n // range with labels 1, 2 or 3."; bool CacheData = false; diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/PairwiseCoupling.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/PairwiseCoupling.cs index de0dab8a9c..be42ec3bbd 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/PairwiseCoupling.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/PairwiseCoupling.cs @@ -10,39 +10,49 @@ public static class PairwiseCoupling { public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); // Define the trainer. var pipeline = - // Convert the string labels into key types. - mlContext.Transforms.Conversion.MapValueToKey("Label") - // Apply PairwiseCoupling multiclass meta trainer on top of binary trainer. - .Append(mlContext.MulticlassClassification.Trainers.PairwiseCoupling(mlContext.BinaryClassification.Trainers.SdcaLogisticRegression())); + // Convert the string labels into key types. + mlContext.Transforms.Conversion.MapValueToKey("Label") + // Apply PairwiseCoupling multiclass meta trainer on top of + // binary trainer. + .Append(mlContext.MulticlassClassification.Trainers + .PairwiseCoupling( + mlContext.BinaryClassification.Trainers.SdcaLogisticRegression())); // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(500, seed: 123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data + .LoadFromEnumerable(GenerateRandomDataPoints(500, seed: 123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(transformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data + .CreateEnumerable(transformedTestData, + reuseRowObject: false).ToList(); // Look at 5 predictions foreach (var p in predictions.Take(5)) - Console.WriteLine($"Label: {p.Label}, Prediction: {p.PredictedLabel}"); + Console.WriteLine($"Label: {p.Label}, " + + $"Prediction: {p.PredictedLabel}"); // Expected output: // Label: 1, Prediction: 1 @@ -52,7 +62,9 @@ public static void Example() // Label: 3, Prediction: 2 // Evaluate the overall metrics - var metrics = mlContext.MulticlassClassification.Evaluate(transformedTestData); + var metrics = mlContext.MulticlassClassification + .Evaluate(transformedTestData); + PrintMetrics(metrics); // Expected output: @@ -72,8 +84,11 @@ public static void Example() // Precision ||0.9091 |0.9171 |0.8636 | } - // Generates random uniform doubles in [-0.5, 0.5) range with labels 1, 2 or 3. - private static IEnumerable GenerateRandomDataPoints(int count, int seed=0) + // Generates random uniform doubles in [-0.5, 0.5) + // range with labels 1, 2 or 3. + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) + { var random = new Random(seed); float randomFloat() => (float)(random.NextDouble() - 0.5); @@ -85,13 +100,17 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se { Label = (uint)label, // Create random features that are correlated with the label. - // The feature values are slightly increased by adding a constant multiple of label. - Features = Enumerable.Repeat(label, 20).Select(x => randomFloat() + label * 0.2f).ToArray() + // The feature values are slightly increased by adding a + // constant multiple of label. + Features = Enumerable.Repeat(label, 20) + .Select(x => randomFloat() + label * 0.2f).ToArray() + }; } } - // Example with label and 20 feature values. A data set is a collection of such examples. + // Example with label and 20 feature values. A data set is a collection of + // such examples. private class DataPoint { public uint Label { get; set; } @@ -114,8 +133,11 @@ public static void PrintMetrics(MulticlassClassificationMetrics metrics) Console.WriteLine($"Micro Accuracy: {metrics.MicroAccuracy:F2}"); Console.WriteLine($"Macro Accuracy: {metrics.MacroAccuracy:F2}"); Console.WriteLine($"Log Loss: {metrics.LogLoss:F2}"); - Console.WriteLine($"Log Loss Reduction: {metrics.LogLossReduction:F2}\n"); + Console.WriteLine( + $"Log Loss Reduction: {metrics.LogLossReduction:F2}\n"); + Console.WriteLine(metrics.ConfusionMatrix.GetFormattedConfusionTable()); } } } + diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/PairwiseCoupling.tt b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/PairwiseCoupling.tt index 4f3617e693..a568fd9736 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/PairwiseCoupling.tt +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/PairwiseCoupling.tt @@ -7,7 +7,9 @@ string TrainerOptions = null; string OptionsInclude = ""; string Comments= ""; -string DataGenerationComments= "// Generates random uniform doubles in [-0.5, 0.5) range with labels 1, 2 or 3."; +string DataGenerationComments= "// Generates random uniform doubles in [-0.5, 0.5)" + + "\n // range with labels 1, 2 or 3."; + bool CacheData = false; string ExpectedOutputPerInstance= @"// Expected output: diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/PermutationFeatureImportance.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/PermutationFeatureImportance.cs index 963fd238ca..71d1c02106 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/PermutationFeatureImportance.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/PermutationFeatureImportance.cs @@ -9,9 +9,10 @@ public static class PermutationFeatureImportance { public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - var mlContext = new MLContext(seed:1); + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. + var mlContext = new MLContext(seed: 1); // Create sample data. var samples = GenerateData(); @@ -19,13 +20,17 @@ public static void Example() // Load the sample data as an IDataView. var data = mlContext.Data.LoadFromEnumerable(samples); - // Define a training pipeline that concatenates features into a vector, normalizes them, and then - // trains a linear model. - var featureColumns = new string[] { nameof(Data.Feature1), nameof(Data.Feature2) }; - var pipeline = mlContext.Transforms.Concatenate("Features", featureColumns) - .Append(mlContext.Transforms.Conversion.MapValueToKey("Label")) - .Append(mlContext.Transforms.NormalizeMinMax("Features")) - .Append(mlContext.MulticlassClassification.Trainers.SdcaMaximumEntropy()); + // Define a training pipeline that concatenates features into a vector, + // normalizes them, and then trains a linear model. + var featureColumns = + new string[] { nameof(Data.Feature1), nameof(Data.Feature2) }; + + var pipeline = mlContext.Transforms + .Concatenate("Features", featureColumns) + .Append(mlContext.Transforms.Conversion.MapValueToKey("Label")) + .Append(mlContext.Transforms.NormalizeMinMax("Features")) + .Append(mlContext.MulticlassClassification.Trainers + .SdcaMaximumEntropy()); // Fit the pipeline to the data. var model = pipeline.Fit(data); @@ -36,18 +41,26 @@ public static void Example() // Extract the predictor. var linearPredictor = model.LastTransformer; - // Compute the permutation metrics for the linear model using the normalized data. - var permutationMetrics = mlContext.MulticlassClassification.PermutationFeatureImportance( - linearPredictor, transformedData, permutationCount: 30); - - // Now let's look at which features are most important to the model overall. - // Get the feature indices sorted by their impact on microaccuracy. - var sortedIndices = permutationMetrics.Select((metrics, index) => new { index, metrics.MicroAccuracy}) + // Compute the permutation metrics for the linear model using the + // normalized data. + var permutationMetrics = mlContext.MulticlassClassification + .PermutationFeatureImportance(linearPredictor, transformedData, + permutationCount: 30); + + // Now let's look at which features are most important to the model + // overall. Get the feature indices sorted by their impact on + // microaccuracy. + var sortedIndices = permutationMetrics + .Select((metrics, index) => new { index, metrics.MicroAccuracy }) .OrderByDescending(feature => Math.Abs(feature.MicroAccuracy.Mean)) .Select(feature => feature.index); - Console.WriteLine("Feature\tChange in MicroAccuracy\t95% Confidence in the Mean Change in MicroAccuracy"); - var microAccuracy = permutationMetrics.Select(x => x.MicroAccuracy).ToArray(); + Console.WriteLine("Feature\tChange in MicroAccuracy\t95% Confidence in " + + "the Mean Change in MicroAccuracy"); + + var microAccuracy = permutationMetrics.Select(x => x.MicroAccuracy) + .ToArray(); + foreach (int i in sortedIndices) { Console.WriteLine("{0}\t{1:G4}\t{2:G4}", @@ -76,10 +89,14 @@ private class Data /// linear combination of the features. /// /// The number of examples. - /// The bias, or offset, in the calculation of the label. - /// The weight to multiply the first feature with to compute the label. - /// The weight to multiply the second feature with to compute the label. - /// The seed for generating feature values and label noise. + /// The bias, or offset, in the calculation of the + /// label. + /// The weight to multiply the first feature with to + /// compute the label. + /// The weight to multiply the second feature with to + /// compute the label. + /// The seed for generating feature values and label + /// noise. /// An enumerable of Data objects. private static IEnumerable GenerateData(int nExamples = 10000, double bias = 0, double weight1 = 1, double weight2 = 2, int seed = 1) @@ -95,7 +112,10 @@ private static IEnumerable GenerateData(int nExamples = 10000, }; // Create a noisy label. - var value = (float)(bias + weight1 * data.Feature1 + weight2 * data.Feature2 + rng.NextDouble() - 0.5); + var value = (float) + (bias + weight1 * data.Feature1 + weight2 * data.Feature2 + + rng.NextDouble() - 0.5); + if (value < max / 3) data.Label = 0; else if (value < 2 * max / 3) @@ -106,4 +126,4 @@ private static IEnumerable GenerateData(int nExamples = 10000, } } } -} +} \ No newline at end of file diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/SdcaMaximumEntropy.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/SdcaMaximumEntropy.cs index 1cc66a33cc..2a83a84f3a 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/SdcaMaximumEntropy.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/SdcaMaximumEntropy.cs @@ -10,45 +10,56 @@ public static class SdcaMaximumEntropy { public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); - // ML.NET doesn't cache data set by default. Therefore, if one reads a data set from a file and accesses it many times, - // it can be slow due to expensive featurization and disk operations. When the considered data can fit into memory, - // a solution is to cache the data in memory. Caching is especially helpful when working with iterative algorithms - // which needs many data passes. - trainingData = mlContext.Data.Cache(trainingData); + // ML.NET doesn't cache data set by default. Therefore, if one reads a + // data set from a file and accesses it many times, it can be slow due + // to expensive featurization and disk operations. When the considered + // data can fit into memory, a solution is to cache the data in memory. + // Caching is especially helpful when working with iterative algorithms + // which needs many data passes. + trainingData = mlContext.Data.Cache(trainingData); // Define the trainer. var pipeline = - // Convert the string labels into key types. - mlContext.Transforms.Conversion.MapValueToKey(nameof(DataPoint.Label)) - // Apply SdcaMaximumEntropy multiclass trainer. - .Append(mlContext.MulticlassClassification.Trainers.SdcaMaximumEntropy()); + // Convert the string labels into key types. + mlContext.Transforms.Conversion + .MapValueToKey(nameof(DataPoint.Label)) + // Apply SdcaMaximumEntropy multiclass trainer. + .Append(mlContext.MulticlassClassification.Trainers + .SdcaMaximumEntropy()); // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(500, seed: 123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data + .LoadFromEnumerable(GenerateRandomDataPoints(500, seed: 123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(transformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data + .CreateEnumerable(transformedTestData, + reuseRowObject: false).ToList(); // Look at 5 predictions foreach (var p in predictions.Take(5)) - Console.WriteLine($"Label: {p.Label}, Prediction: {p.PredictedLabel}"); + Console.WriteLine($"Label: {p.Label}, " + + $"Prediction: {p.PredictedLabel}"); // Expected output: // Label: 1, Prediction: 1 @@ -58,7 +69,9 @@ public static void Example() // Label: 3, Prediction: 3 // Evaluate the overall metrics - var metrics = mlContext.MulticlassClassification.Evaluate(transformedTestData); + var metrics = mlContext.MulticlassClassification + .Evaluate(transformedTestData); + PrintMetrics(metrics); // Expected output: @@ -77,8 +90,11 @@ public static void Example() // Precision ||0.9130 |0.9538 |0.8494 | } - // Generates random uniform doubles in [-0.5, 0.5) range with labels 1, 2 or 3. - private static IEnumerable GenerateRandomDataPoints(int count, int seed=0) + // Generates random uniform doubles in [-0.5, 0.5) + // range with labels 1, 2 or 3. + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) + { var random = new Random(seed); float randomFloat() => (float)(random.NextDouble() - 0.5); @@ -90,13 +106,17 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se { Label = (uint)label, // Create random features that are correlated with the label. - // The feature values are slightly increased by adding a constant multiple of label. - Features = Enumerable.Repeat(label, 20).Select(x => randomFloat() + label * 0.2f).ToArray() + // The feature values are slightly increased by adding a + // constant multiple of label. + Features = Enumerable.Repeat(label, 20) + .Select(x => randomFloat() + label * 0.2f).ToArray() + }; } } - // Example with label and 20 feature values. A data set is a collection of such examples. + // Example with label and 20 feature values. A data set is a collection of + // such examples. private class DataPoint { public uint Label { get; set; } @@ -119,8 +139,11 @@ public static void PrintMetrics(MulticlassClassificationMetrics metrics) Console.WriteLine($"Micro Accuracy: {metrics.MicroAccuracy:F2}"); Console.WriteLine($"Macro Accuracy: {metrics.MacroAccuracy:F2}"); Console.WriteLine($"Log Loss: {metrics.LogLoss:F2}"); - Console.WriteLine($"Log Loss Reduction: {metrics.LogLossReduction:F2}\n"); + Console.WriteLine( + $"Log Loss Reduction: {metrics.LogLossReduction:F2}\n"); + Console.WriteLine(metrics.ConfusionMatrix.GetFormattedConfusionTable()); } } } + diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/SdcaMaximumEntropy.tt b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/SdcaMaximumEntropy.tt index 4eccf95005..f83d618268 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/SdcaMaximumEntropy.tt +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/SdcaMaximumEntropy.tt @@ -8,7 +8,8 @@ string TrainerOptions = null; string OptionsInclude = ""; string Comments = ""; bool CacheData = true; -string DataGenerationComments= "// Generates random uniform doubles in [-0.5, 0.5) range with labels 1, 2 or 3."; +string DataGenerationComments= "// Generates random uniform doubles in [-0.5, 0.5)" + + "\n // range with labels 1, 2 or 3."; string ExpectedOutputPerInstance = @"// Expected output: // Label: 1, Prediction: 1 diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/SdcaMaximumEntropyWithOptions.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/SdcaMaximumEntropyWithOptions.cs index a3cfcf4cbc..62ed191c0f 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/SdcaMaximumEntropyWithOptions.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/SdcaMaximumEntropyWithOptions.cs @@ -11,22 +11,26 @@ public static class SdcaMaximumEntropyWithOptions { public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); - // ML.NET doesn't cache data set by default. Therefore, if one reads a data set from a file and accesses it many times, - // it can be slow due to expensive featurization and disk operations. When the considered data can fit into memory, - // a solution is to cache the data in memory. Caching is especially helpful when working with iterative algorithms - // which needs many data passes. - trainingData = mlContext.Data.Cache(trainingData); + // ML.NET doesn't cache data set by default. Therefore, if one reads a + // data set from a file and accesses it many times, it can be slow due + // to expensive featurization and disk operations. When the considered + // data can fit into memory, a solution is to cache the data in memory. + // Caching is especially helpful when working with iterative algorithms + // which needs many data passes. + trainingData = mlContext.Data.Cache(trainingData); // Define trainer options. var options = new SdcaMaximumEntropyMulticlassTrainer.Options @@ -39,27 +43,32 @@ public static void Example() // Define the trainer. var pipeline = - // Convert the string labels into key types. - mlContext.Transforms.Conversion.MapValueToKey("Label") - // Apply SdcaMaximumEntropy multiclass trainer. - .Append(mlContext.MulticlassClassification.Trainers.SdcaMaximumEntropy(options)); - + // Convert the string labels into key types. + mlContext.Transforms.Conversion.MapValueToKey("Label") + // Apply SdcaMaximumEntropy multiclass trainer. + .Append(mlContext.MulticlassClassification.Trainers + .SdcaMaximumEntropy(options)); // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(500, seed: 123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data + .LoadFromEnumerable(GenerateRandomDataPoints(500, seed: 123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(transformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data + .CreateEnumerable(transformedTestData, + reuseRowObject: false).ToList(); // Look at 5 predictions foreach (var p in predictions.Take(5)) - Console.WriteLine($"Label: {p.Label}, Prediction: {p.PredictedLabel}"); + Console.WriteLine($"Label: {p.Label}, " + + $"Prediction: {p.PredictedLabel}"); // Expected output: // Label: 1, Prediction: 1 @@ -69,7 +78,9 @@ public static void Example() // Label: 3, Prediction: 3 // Evaluate the overall metrics - var metrics = mlContext.MulticlassClassification.Evaluate(transformedTestData); + var metrics = mlContext.MulticlassClassification + .Evaluate(transformedTestData); + PrintMetrics(metrics); // Expected output: @@ -89,8 +100,11 @@ public static void Example() // Precision ||0.9363 |0.9647 |0.8497 | } - // Generates random uniform doubles in [-0.5, 0.5) range with labels 1, 2 or 3. - private static IEnumerable GenerateRandomDataPoints(int count, int seed=0) + // Generates random uniform doubles in [-0.5, 0.5) + // range with labels 1, 2 or 3. + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) + { var random = new Random(seed); float randomFloat() => (float)(random.NextDouble() - 0.5); @@ -102,13 +116,17 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se { Label = (uint)label, // Create random features that are correlated with the label. - // The feature values are slightly increased by adding a constant multiple of label. - Features = Enumerable.Repeat(label, 20).Select(x => randomFloat() + label * 0.2f).ToArray() + // The feature values are slightly increased by adding a + // constant multiple of label. + Features = Enumerable.Repeat(label, 20) + .Select(x => randomFloat() + label * 0.2f).ToArray() + }; } } - // Example with label and 20 feature values. A data set is a collection of such examples. + // Example with label and 20 feature values. A data set is a collection of + // such examples. private class DataPoint { public uint Label { get; set; } @@ -131,8 +149,11 @@ public static void PrintMetrics(MulticlassClassificationMetrics metrics) Console.WriteLine($"Micro Accuracy: {metrics.MicroAccuracy:F2}"); Console.WriteLine($"Macro Accuracy: {metrics.MacroAccuracy:F2}"); Console.WriteLine($"Log Loss: {metrics.LogLoss:F2}"); - Console.WriteLine($"Log Loss Reduction: {metrics.LogLossReduction:F2}\n"); + Console.WriteLine( + $"Log Loss Reduction: {metrics.LogLossReduction:F2}\n"); + Console.WriteLine(metrics.ConfusionMatrix.GetFormattedConfusionTable()); } } } + diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/SdcaMaximumEntropyWithOptions.tt b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/SdcaMaximumEntropyWithOptions.tt index 88eead7e0d..f6bb41ac40 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/SdcaMaximumEntropyWithOptions.tt +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/SdcaMaximumEntropyWithOptions.tt @@ -14,7 +14,8 @@ string TrainerOptions = @"SdcaMaximumEntropyMulticlassTrainer.Options string OptionsInclude = "using Microsoft.ML.Trainers;"; string Comments = ""; bool CacheData = true; -string DataGenerationComments= "// Generates random uniform doubles in [-0.5, 0.5) range with labels 1, 2 or 3."; +string DataGenerationComments= "// Generates random uniform doubles in [-0.5, 0.5)" + + "\n // range with labels 1, 2 or 3."; string ExpectedOutputPerInstance = @"// Expected output: diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/SdcaNonCalibrated.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/SdcaNonCalibrated.cs index d9f59fe3e7..6a899f9432 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/SdcaNonCalibrated.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/SdcaNonCalibrated.cs @@ -10,45 +10,56 @@ public static class SdcaNonCalibrated { public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); - // ML.NET doesn't cache data set by default. Therefore, if one reads a data set from a file and accesses it many times, - // it can be slow due to expensive featurization and disk operations. When the considered data can fit into memory, - // a solution is to cache the data in memory. Caching is especially helpful when working with iterative algorithms - // which needs many data passes. - trainingData = mlContext.Data.Cache(trainingData); + // ML.NET doesn't cache data set by default. Therefore, if one reads a + // data set from a file and accesses it many times, it can be slow due + // to expensive featurization and disk operations. When the considered + // data can fit into memory, a solution is to cache the data in memory. + // Caching is especially helpful when working with iterative algorithms + // which needs many data passes. + trainingData = mlContext.Data.Cache(trainingData); // Define the trainer. var pipeline = - // Convert the string labels into key types. - mlContext.Transforms.Conversion.MapValueToKey(nameof(DataPoint.Label)) - // Apply SdcaNonCalibrated multiclass trainer. - .Append(mlContext.MulticlassClassification.Trainers.SdcaNonCalibrated()); + // Convert the string labels into key types. + mlContext.Transforms.Conversion + .MapValueToKey(nameof(DataPoint.Label)) + // Apply SdcaNonCalibrated multiclass trainer. + .Append(mlContext.MulticlassClassification.Trainers + .SdcaNonCalibrated()); // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(500, seed: 123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data + .LoadFromEnumerable(GenerateRandomDataPoints(500, seed: 123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(transformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data + .CreateEnumerable(transformedTestData, + reuseRowObject: false).ToList(); // Look at 5 predictions foreach (var p in predictions.Take(5)) - Console.WriteLine($"Label: {p.Label}, Prediction: {p.PredictedLabel}"); + Console.WriteLine($"Label: {p.Label}, " + + $"Prediction: {p.PredictedLabel}"); // Expected output: // Label: 1, Prediction: 1 @@ -58,7 +69,9 @@ public static void Example() // Label: 3, Prediction: 3 // Evaluate the overall metrics - var metrics = mlContext.MulticlassClassification.Evaluate(transformedTestData); + var metrics = mlContext.MulticlassClassification + .Evaluate(transformedTestData); + PrintMetrics(metrics); // Expected output: @@ -78,8 +91,11 @@ public static void Example() // Precision ||0.9304 |0.9538 |0.8521 | } - // Generates random uniform doubles in [-0.5, 0.5) range with labels 1, 2 or 3. - private static IEnumerable GenerateRandomDataPoints(int count, int seed=0) + // Generates random uniform doubles in [-0.5, 0.5) + // range with labels 1, 2 or 3. + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) + { var random = new Random(seed); float randomFloat() => (float)(random.NextDouble() - 0.5); @@ -91,13 +107,17 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se { Label = (uint)label, // Create random features that are correlated with the label. - // The feature values are slightly increased by adding a constant multiple of label. - Features = Enumerable.Repeat(label, 20).Select(x => randomFloat() + label * 0.2f).ToArray() + // The feature values are slightly increased by adding a + // constant multiple of label. + Features = Enumerable.Repeat(label, 20) + .Select(x => randomFloat() + label * 0.2f).ToArray() + }; } } - // Example with label and 20 feature values. A data set is a collection of such examples. + // Example with label and 20 feature values. A data set is a collection of + // such examples. private class DataPoint { public uint Label { get; set; } @@ -120,8 +140,11 @@ public static void PrintMetrics(MulticlassClassificationMetrics metrics) Console.WriteLine($"Micro Accuracy: {metrics.MicroAccuracy:F2}"); Console.WriteLine($"Macro Accuracy: {metrics.MacroAccuracy:F2}"); Console.WriteLine($"Log Loss: {metrics.LogLoss:F2}"); - Console.WriteLine($"Log Loss Reduction: {metrics.LogLossReduction:F2}\n"); + Console.WriteLine( + $"Log Loss Reduction: {metrics.LogLossReduction:F2}\n"); + Console.WriteLine(metrics.ConfusionMatrix.GetFormattedConfusionTable()); } } } + diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/SdcaNonCalibrated.tt b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/SdcaNonCalibrated.tt index 5437228267..cd403b1a44 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/SdcaNonCalibrated.tt +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/SdcaNonCalibrated.tt @@ -8,7 +8,8 @@ string TrainerOptions = null; string OptionsInclude = ""; string Comments = ""; bool CacheData = true; -string DataGenerationComments= "// Generates random uniform doubles in [-0.5, 0.5) range with labels 1, 2 or 3."; +string DataGenerationComments= "// Generates random uniform doubles in [-0.5, 0.5)" + + "\n // range with labels 1, 2 or 3."; string ExpectedOutputPerInstance = @"// Expected output: // Label: 1, Prediction: 1 diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/SdcaNonCalibratedWithOptions.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/SdcaNonCalibratedWithOptions.cs index 2577756d9b..6367e9fe21 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/SdcaNonCalibratedWithOptions.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/SdcaNonCalibratedWithOptions.cs @@ -11,22 +11,26 @@ public static class SdcaNonCalibratedWithOptions { public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); - // ML.NET doesn't cache data set by default. Therefore, if one reads a data set from a file and accesses it many times, - // it can be slow due to expensive featurization and disk operations. When the considered data can fit into memory, - // a solution is to cache the data in memory. Caching is especially helpful when working with iterative algorithms - // which needs many data passes. - trainingData = mlContext.Data.Cache(trainingData); + // ML.NET doesn't cache data set by default. Therefore, if one reads a + // data set from a file and accesses it many times, it can be slow due + // to expensive featurization and disk operations. When the considered + // data can fit into memory, a solution is to cache the data in memory. + // Caching is especially helpful when working with iterative algorithms + // which needs many data passes. + trainingData = mlContext.Data.Cache(trainingData); // Define trainer options. var options = new SdcaNonCalibratedMulticlassTrainer.Options @@ -39,27 +43,32 @@ public static void Example() // Define the trainer. var pipeline = - // Convert the string labels into key types. - mlContext.Transforms.Conversion.MapValueToKey("Label") - // Apply SdcaNonCalibrated multiclass trainer. - .Append(mlContext.MulticlassClassification.Trainers.SdcaNonCalibrated(options)); - + // Convert the string labels into key types. + mlContext.Transforms.Conversion.MapValueToKey("Label") + // Apply SdcaNonCalibrated multiclass trainer. + .Append(mlContext.MulticlassClassification.Trainers + .SdcaNonCalibrated(options)); // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(500, seed: 123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data + .LoadFromEnumerable(GenerateRandomDataPoints(500, seed: 123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(transformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data + .CreateEnumerable(transformedTestData, + reuseRowObject: false).ToList(); // Look at 5 predictions foreach (var p in predictions.Take(5)) - Console.WriteLine($"Label: {p.Label}, Prediction: {p.PredictedLabel}"); + Console.WriteLine($"Label: {p.Label}, " + + $"Prediction: {p.PredictedLabel}"); // Expected output: // Label: 1, Prediction: 1 @@ -69,7 +78,9 @@ public static void Example() // Label: 3, Prediction: 3 // Evaluate the overall metrics - var metrics = mlContext.MulticlassClassification.Evaluate(transformedTestData); + var metrics = mlContext.MulticlassClassification + .Evaluate(transformedTestData); + PrintMetrics(metrics); // Expected output: @@ -89,8 +100,11 @@ public static void Example() // Precision ||0.9236 |0.9591 |0.8372 | } - // Generates random uniform doubles in [-0.5, 0.5) range with labels 1, 2 or 3. - private static IEnumerable GenerateRandomDataPoints(int count, int seed=0) + // Generates random uniform doubles in [-0.5, 0.5) + // range with labels 1, 2 or 3. + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) + { var random = new Random(seed); float randomFloat() => (float)(random.NextDouble() - 0.5); @@ -102,13 +116,17 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se { Label = (uint)label, // Create random features that are correlated with the label. - // The feature values are slightly increased by adding a constant multiple of label. - Features = Enumerable.Repeat(label, 20).Select(x => randomFloat() + label * 0.2f).ToArray() + // The feature values are slightly increased by adding a + // constant multiple of label. + Features = Enumerable.Repeat(label, 20) + .Select(x => randomFloat() + label * 0.2f).ToArray() + }; } } - // Example with label and 20 feature values. A data set is a collection of such examples. + // Example with label and 20 feature values. A data set is a collection of + // such examples. private class DataPoint { public uint Label { get; set; } @@ -131,8 +149,11 @@ public static void PrintMetrics(MulticlassClassificationMetrics metrics) Console.WriteLine($"Micro Accuracy: {metrics.MicroAccuracy:F2}"); Console.WriteLine($"Macro Accuracy: {metrics.MacroAccuracy:F2}"); Console.WriteLine($"Log Loss: {metrics.LogLoss:F2}"); - Console.WriteLine($"Log Loss Reduction: {metrics.LogLossReduction:F2}\n"); + Console.WriteLine( + $"Log Loss Reduction: {metrics.LogLossReduction:F2}\n"); + Console.WriteLine(metrics.ConfusionMatrix.GetFormattedConfusionTable()); } } } + diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/SdcaNonCalibratedWithOptions.tt b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/SdcaNonCalibratedWithOptions.tt index 19f7ee9dd9..5f0f2a4775 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/SdcaNonCalibratedWithOptions.tt +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/SdcaNonCalibratedWithOptions.tt @@ -14,7 +14,8 @@ string TrainerOptions = @"SdcaNonCalibratedMulticlassTrainer.Options string OptionsInclude = "using Microsoft.ML.Trainers;"; string Comments = ""; bool CacheData = true; -string DataGenerationComments= "// Generates random uniform doubles in [-0.5, 0.5) range with labels 1, 2 or 3."; +string DataGenerationComments= "// Generates random uniform doubles in [-0.5, 0.5)" + + "\n // range with labels 1, 2 or 3."; string ExpectedOutputPerInstance = @"// Expected output: // Label: 1, Prediction: 1 diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Ranking/FastTree.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Ranking/FastTree.cs index aa0a99e245..9952909c64 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Ranking/FastTree.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Ranking/FastTree.cs @@ -8,19 +8,22 @@ namespace Samples.Dynamic.Trainers.Ranking { public static class FastTree { - // This example requires installation of additional NuGet package - // Microsoft.ML.FastTree. + // This example requires installation of additional NuGet package for + // Microsoft.ML.FastTree at + // https://www.nuget.org/packages/Microsoft.ML.FastTree/ public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); // Define the trainer. @@ -29,17 +32,21 @@ public static void Example() // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(500, seed:123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data.LoadFromEnumerable( + GenerateRandomDataPoints(500, seed:123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Take the top 5 rows. - var topTransformedTestData = mlContext.Data.TakeRows(transformedTestData, 5); + var topTransformedTestData = mlContext.Data.TakeRows( + transformedTestData, 5); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(topTransformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data.CreateEnumerable( + topTransformedTestData, reuseRowObject: false).ToList(); // Print 5 predictions. foreach (var p in predictions) @@ -61,7 +68,8 @@ public static void Example() // NDCG: @1:0.99, @2:0.98, @3:0.99 } - private static IEnumerable GenerateRandomDataPoints(int count, int seed = 0, int groupSize = 10) + private static IEnumerable GenerateRandomDataPoints(int count, + int seed = 0, int groupSize = 10) { var random = new Random(seed); float randomFloat() => (float)random.NextDouble(); @@ -73,13 +81,16 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se Label = (uint)label, GroupId = (uint)(i / groupSize), // Create random features that are correlated with the label. - // For data points with larger labels, the feature values are slightly increased by adding a constant. - Features = Enumerable.Repeat(label, 50).Select(x => randomFloat() + x * 0.1f).ToArray() + // For data points with larger labels, the feature values are + // slightly increased by adding a constant. + Features = Enumerable.Repeat(label, 50).Select( + x => randomFloat() + x * 0.1f).ToArray() }; } } - // Example with label, groupId, and 50 feature values. A data set is a collection of such examples. + // Example with label, groupId, and 50 feature values. A data set is a + // collection of such examples. private class DataPoint { [KeyType(5)] @@ -102,8 +113,13 @@ private class Prediction // Pretty-print RankerMetrics objects. public static void PrintMetrics(RankingMetrics metrics) { - Console.WriteLine($"DCG: {string.Join(", ", metrics.DiscountedCumulativeGains.Select((d, i) => $"@{i + 1}:{d:F2}").ToArray())}"); - Console.WriteLine($"NDCG: {string.Join(", ", metrics.NormalizedDiscountedCumulativeGains.Select((d, i) => $"@{i + 1}:{d:F2}").ToArray())}"); + Console.WriteLine("DCG: " + string.Join(", ", + metrics.DiscountedCumulativeGains.Select( + (d, i) => (i + 1) + ":" + d + ":F2").ToArray())); + + Console.WriteLine("NDCG: " + string.Join(", ", + metrics.NormalizedDiscountedCumulativeGains.Select( + (d, i) => (i + 1) + ":" + d + ":F2").ToArray())); } } } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Ranking/FastTree.tt b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Ranking/FastTree.tt index ac6ee1e854..dfe2abd966 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Ranking/FastTree.tt +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Ranking/FastTree.tt @@ -6,8 +6,9 @@ string TrainerOptions = null; string OptionsInclude = ""; string Comments= @" - // This example requires installation of additional NuGet package - // Microsoft.ML.FastTree."; + // This example requires installation of additional NuGet package for + // Microsoft.ML.FastTree at + // https://www.nuget.org/packages/Microsoft.ML.FastTree/"; string ExpectedOutputPerInstance = @"// Expected output: // Label: 5, Score: 13.0154 @@ -19,4 +20,4 @@ string ExpectedOutputPerInstance = @"// Expected output: string ExpectedOutput = @"// Expected output: // DCG: @1:41.95, @2:63.33, @3:75.65 // NDCG: @1:0.99, @2:0.98, @3:0.99"; -#> \ No newline at end of file +#> diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Ranking/FastTreeWithOptions.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Ranking/FastTreeWithOptions.cs index 5683153940..5d575ec160 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Ranking/FastTreeWithOptions.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Ranking/FastTreeWithOptions.cs @@ -9,19 +9,22 @@ namespace Samples.Dynamic.Trainers.Ranking { public static class FastTreeWithOptions { - // This example requires installation of additional NuGet package - // Microsoft.ML.FastTree. + // This example requires installation of additional NuGet package for + // Microsoft.ML.FastTree at + // https://www.nuget.org/packages/Microsoft.ML.FastTree/ public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); // Define trainer options. @@ -43,17 +46,21 @@ public static void Example() // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(500, seed:123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data.LoadFromEnumerable( + GenerateRandomDataPoints(500, seed:123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Take the top 5 rows. - var topTransformedTestData = mlContext.Data.TakeRows(transformedTestData, 5); + var topTransformedTestData = mlContext.Data.TakeRows( + transformedTestData, 5); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(topTransformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data.CreateEnumerable( + topTransformedTestData, reuseRowObject: false).ToList(); // Print 5 predictions. foreach (var p in predictions) @@ -75,7 +82,8 @@ public static void Example() // NDCG: @1:0.96, @2:0.95, @3:0.97 } - private static IEnumerable GenerateRandomDataPoints(int count, int seed = 0, int groupSize = 10) + private static IEnumerable GenerateRandomDataPoints(int count, + int seed = 0, int groupSize = 10) { var random = new Random(seed); float randomFloat() => (float)random.NextDouble(); @@ -87,13 +95,16 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se Label = (uint)label, GroupId = (uint)(i / groupSize), // Create random features that are correlated with the label. - // For data points with larger labels, the feature values are slightly increased by adding a constant. - Features = Enumerable.Repeat(label, 50).Select(x => randomFloat() + x * 0.1f).ToArray() + // For data points with larger labels, the feature values are + // slightly increased by adding a constant. + Features = Enumerable.Repeat(label, 50).Select( + x => randomFloat() + x * 0.1f).ToArray() }; } } - // Example with label, groupId, and 50 feature values. A data set is a collection of such examples. + // Example with label, groupId, and 50 feature values. A data set is a + // collection of such examples. private class DataPoint { [KeyType(5)] @@ -116,8 +127,12 @@ private class Prediction // Pretty-print RankerMetrics objects. public static void PrintMetrics(RankingMetrics metrics) { - Console.WriteLine($"DCG: {string.Join(", ", metrics.DiscountedCumulativeGains.Select((d, i) => $"@{i + 1}:{d:F2}").ToArray())}"); - Console.WriteLine($"NDCG: {string.Join(", ", metrics.NormalizedDiscountedCumulativeGains.Select((d, i) => $"@{i + 1}:{d:F2}").ToArray())}"); + Console.WriteLine("DCG: " + string.Join(", ", + metrics.DiscountedCumulativeGains.Select( + (d, i) => (i + 1) + ":" + d + ":F2").ToArray())); + Console.WriteLine("NDCG: " + string.Join(", ", + metrics.NormalizedDiscountedCumulativeGains.Select( + (d, i) => (i + 1) + ":" + d + ":F2").ToArray())); } } } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Ranking/FastTreeWithOptions.tt b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Ranking/FastTreeWithOptions.tt index b4a65602aa..ce1140edf6 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Ranking/FastTreeWithOptions.tt +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Ranking/FastTreeWithOptions.tt @@ -16,8 +16,9 @@ string TrainerOptions = @"FastTreeRankingTrainer.Options string OptionsInclude = "using Microsoft.ML.Trainers.FastTree;"; string Comments= @" - // This example requires installation of additional NuGet package - // Microsoft.ML.FastTree."; + // This example requires installation of additional NuGet package for + // Microsoft.ML.FastTree at + // https://www.nuget.org/packages/Microsoft.ML.FastTree/"; string ExpectedOutputPerInstance = @"// Expected output: // Label: 5, Score: 8.807633 diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Ranking/LightGbm.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Ranking/LightGbm.cs index 3b6e542ef7..d6b3945d2b 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Ranking/LightGbm.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Ranking/LightGbm.cs @@ -8,17 +8,22 @@ namespace Samples.Dynamic.Trainers.Ranking { public static class LightGbm { + // This example requires installation of additional NuGet package for + // Microsoft.ML.FastTree at + // https://www.nuget.org/packages/Microsoft.ML.FastTree/ public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); // Define the trainer. @@ -27,39 +32,44 @@ public static void Example() // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(500, seed:123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data.LoadFromEnumerable( + GenerateRandomDataPoints(500, seed:123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Take the top 5 rows. - var topTransformedTestData = mlContext.Data.TakeRows(transformedTestData, 5); + var topTransformedTestData = mlContext.Data.TakeRows( + transformedTestData, 5); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(topTransformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data.CreateEnumerable( + topTransformedTestData, reuseRowObject: false).ToList(); // Print 5 predictions. foreach (var p in predictions) Console.WriteLine($"Label: {p.Label}, Score: {p.Score}"); // Expected output: - // Label: 5, Score: 2.195333 - // Label: 4, Score: 0.2596574 - // Label: 4, Score: -2.168355 - // Label: 1, Score: -3.074823 - // Label: 1, Score: -1.523607 + // Label: 5, Score: 2.493263 + // Label: 1, Score: -4.528436 + // Label: 3, Score: -3.002865 + // Label: 3, Score: -2.151812 + // Label: 1, Score: -4.089102 // Evaluate the overall metrics. var metrics = mlContext.Ranking.Evaluate(transformedTestData); PrintMetrics(metrics); // Expected output: - // DCG: @1:26.03, @2:37.57, @3:45.83 - // NDCG: @1:0.61, @2:0.57, @3:0.59 + // DCG: @1:41.95, @2:63.76, @3:75.97 + // NDCG: @1:0.99, @2:0.99, @3:0.99 } - private static IEnumerable GenerateRandomDataPoints(int count, int seed = 0, int groupSize = 10) + private static IEnumerable GenerateRandomDataPoints(int count, + int seed = 0, int groupSize = 10) { var random = new Random(seed); float randomFloat() => (float)random.NextDouble(); @@ -71,13 +81,16 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se Label = (uint)label, GroupId = (uint)(i / groupSize), // Create random features that are correlated with the label. - // For data points with larger labels, the feature values are slightly increased by adding a constant. - Features = Enumerable.Repeat(label, 50).Select(x => randomFloat() + x * 0.1f).ToArray() + // For data points with larger labels, the feature values are + // slightly increased by adding a constant. + Features = Enumerable.Repeat(label, 50).Select( + x => randomFloat() + x * 0.1f).ToArray() }; } } - // Example with label, groupId, and 50 feature values. A data set is a collection of such examples. + // Example with label, groupId, and 50 feature values. A data set is a + // collection of such examples. private class DataPoint { [KeyType(5)] @@ -100,8 +113,12 @@ private class Prediction // Pretty-print RankerMetrics objects. public static void PrintMetrics(RankingMetrics metrics) { - Console.WriteLine($"DCG: {string.Join(", ", metrics.DiscountedCumulativeGains.Select((d, i) => $"@{i + 1}:{d:F2}").ToArray())}"); - Console.WriteLine($"NDCG: {string.Join(", ", metrics.NormalizedDiscountedCumulativeGains.Select((d, i) => $"@{i + 1}:{d:F2}").ToArray())}"); + Console.WriteLine("DCG: " + string.Join(", ", + metrics.DiscountedCumulativeGains.Select( + (d, i) => (i + 1) + ":" + d + ":F2").ToArray())); + Console.WriteLine("NDCG: " + string.Join(", ", + metrics.NormalizedDiscountedCumulativeGains.Select( + (d, i) => (i + 1) + ":" + d + ":F2").ToArray())); } } } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Ranking/LightGbm.tt b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Ranking/LightGbm.tt index 136cad7843..f8c628c26d 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Ranking/LightGbm.tt +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Ranking/LightGbm.tt @@ -6,8 +6,9 @@ string TrainerOptions = null; string OptionsInclude = ""; string Comments= @" - // This example requires installation of additional NuGet package - // Microsoft.ML.LightGbm."; + // This example requires installation of additional NuGet package for + // Microsoft.ML.FastTree at + // https://www.nuget.org/packages/Microsoft.ML.FastTree/"; string ExpectedOutputPerInstance = @"// Expected output: // Label: 5, Score: 2.493263 diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Ranking/LightGbmWithOptions.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Ranking/LightGbmWithOptions.cs index 1c21c7704a..4859cd7b81 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Ranking/LightGbmWithOptions.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Ranking/LightGbmWithOptions.cs @@ -9,19 +9,22 @@ namespace Samples.Dynamic.Trainers.Ranking { public static class LightGbmWithOptions { - // This example requires installation of additional NuGet package - // Microsoft.ML.LightGbm. + // This example requires installation of additional NuGet package for + // Microsoft.ML.FastTree at + // https://www.nuget.org/packages/Microsoft.ML.FastTree/ public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); // Define trainer options. @@ -44,17 +47,21 @@ public static void Example() // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(500, seed:123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data.LoadFromEnumerable( + GenerateRandomDataPoints(500, seed:123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Take the top 5 rows. - var topTransformedTestData = mlContext.Data.TakeRows(transformedTestData, 5); + var topTransformedTestData = mlContext.Data.TakeRows( + transformedTestData, 5); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(topTransformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data.CreateEnumerable( + topTransformedTestData, reuseRowObject: false).ToList(); // Print 5 predictions. foreach (var p in predictions) @@ -76,7 +83,8 @@ public static void Example() // NDCG: @1:0.69, @2:0.72, @3:0.74 } - private static IEnumerable GenerateRandomDataPoints(int count, int seed = 0, int groupSize = 10) + private static IEnumerable GenerateRandomDataPoints(int count, + int seed = 0, int groupSize = 10) { var random = new Random(seed); float randomFloat() => (float)random.NextDouble(); @@ -88,13 +96,16 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se Label = (uint)label, GroupId = (uint)(i / groupSize), // Create random features that are correlated with the label. - // For data points with larger labels, the feature values are slightly increased by adding a constant. - Features = Enumerable.Repeat(label, 50).Select(x => randomFloat() + x * 0.1f).ToArray() + // For data points with larger labels, the feature values are + // slightly increased by adding a constant. + Features = Enumerable.Repeat(label, 50).Select( + x => randomFloat() + x * 0.1f).ToArray() }; } } - // Example with label, groupId, and 50 feature values. A data set is a collection of such examples. + // Example with label, groupId, and 50 feature values. A data set is a + // collection of such examples. private class DataPoint { [KeyType(5)] @@ -117,8 +128,12 @@ private class Prediction // Pretty-print RankerMetrics objects. public static void PrintMetrics(RankingMetrics metrics) { - Console.WriteLine($"DCG: {string.Join(", ", metrics.DiscountedCumulativeGains.Select((d, i) => $"@{i + 1}:{d:F2}").ToArray())}"); - Console.WriteLine($"NDCG: {string.Join(", ", metrics.NormalizedDiscountedCumulativeGains.Select((d, i) => $"@{i + 1}:{d:F2}").ToArray())}"); + Console.WriteLine("DCG: " + string.Join(", ", + metrics.DiscountedCumulativeGains.Select( + (d, i) => (i + 1) + ":" + d + ":F2").ToArray())); + Console.WriteLine("NDCG: " + string.Join(", ", + metrics.NormalizedDiscountedCumulativeGains.Select( + (d, i) => (i + 1) + ":" + d + ":F2").ToArray())); } } } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Ranking/LightGbmWithOptions.tt b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Ranking/LightGbmWithOptions.tt index 0de6d61d6d..939260bd94 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Ranking/LightGbmWithOptions.tt +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Ranking/LightGbmWithOptions.tt @@ -17,8 +17,9 @@ string TrainerOptions = @"LightGbmRankingTrainer.Options string OptionsInclude = "using Microsoft.ML.Trainers.LightGbm;"; string Comments= @" - // This example requires installation of additional NuGet package - // Microsoft.ML.LightGbm."; + // This example requires installation of additional NuGet package for + // Microsoft.ML.FastTree at + // https://www.nuget.org/packages/Microsoft.ML.FastTree/"; string ExpectedOutputPerInstance = @"// Expected output: // Label: 5, Score: 0.05836755 diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Ranking/PermutationFeatureImportance.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Ranking/PermutationFeatureImportance.cs index 41928a70ee..04b82151ac 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Ranking/PermutationFeatureImportance.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Ranking/PermutationFeatureImportance.cs @@ -9,8 +9,9 @@ public static class PermutationFeatureImportance { public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. var mlContext = new MLContext(seed:1); // Create sample data. @@ -19,12 +20,15 @@ public static void Example() // Load the sample data as an IDataView. var data = mlContext.Data.LoadFromEnumerable(samples); - // Define a training pipeline that concatenates features into a vector, normalizes them, and then - // trains a linear model. - var featureColumns = new string[] { nameof(Data.Feature1), nameof(Data.Feature2) }; - var pipeline = mlContext.Transforms.Concatenate("Features", featureColumns) + // Define a training pipeline that concatenates features into a vector, + // normalizes them, and then trains a linear model. + var featureColumns = new string[] { nameof(Data.Feature1), nameof( + Data.Feature2) }; + var pipeline = mlContext.Transforms.Concatenate("Features", + featureColumns) .Append(mlContext.Transforms.Conversion.MapValueToKey("Label")) - .Append(mlContext.Transforms.Conversion.MapValueToKey("GroupId")) + .Append(mlContext.Transforms.Conversion.MapValueToKey( + "GroupId")) .Append(mlContext.Transforms.NormalizeMinMax("Features")) .Append(mlContext.Ranking.Trainers.FastTree()); @@ -37,18 +41,24 @@ public static void Example() // Extract the predictor. var linearPredictor = model.LastTransformer; - // Compute the permutation metrics for the linear model using the normalized data. + // Compute the permutation metrics for the linear model using the + // normalized data. var permutationMetrics = mlContext.Ranking.PermutationFeatureImportance( linearPredictor, transformedData, permutationCount: 30); - // Now let's look at which features are most important to the model overall. - // Get the feature indices sorted by their impact on NDCG@1. - var sortedIndices = permutationMetrics.Select((metrics, index) => new { index, metrics.NormalizedDiscountedCumulativeGains}) - .OrderByDescending(feature => Math.Abs(feature.NormalizedDiscountedCumulativeGains[0].Mean)) + // Now let's look at which features are most important to the model + // overall. Get the feature indices sorted by their impact on NDCG@1. + var sortedIndices = permutationMetrics.Select((metrics, index) => new { + index, metrics.NormalizedDiscountedCumulativeGains}) + .OrderByDescending(feature => Math.Abs( + feature.NormalizedDiscountedCumulativeGains[0].Mean)) + .Select(feature => feature.index); - Console.WriteLine("Feature\tChange in NDCG@1\t95% Confidence in the Mean Change in NDCG@1"); - var ndcg = permutationMetrics.Select(x => x.NormalizedDiscountedCumulativeGains).ToArray(); + Console.WriteLine("Feature\tChange in NDCG@1\t95% Confidence in the" + + "Mean Change in NDCG@1"); + var ndcg = permutationMetrics.Select( + x => x.NormalizedDiscountedCumulativeGains).ToArray(); foreach (int i in sortedIndices) { Console.WriteLine("{0}\t{1:G4}\t{2:G4}", @@ -78,14 +88,25 @@ private class Data /// Generate an enumerable of Data objects, creating the label as a simple /// linear combination of the features. /// + /// /// The number of examples. - /// The bias, or offset, in the calculation of the label. - /// The weight to multiply the first feature with to compute the label. - /// The weight to multiply the second feature with to compute the label. - /// The seed for generating feature values and label noise. + /// + /// The bias, or offset, in the calculation of the label. + /// + /// + /// The weight to multiply the first feature with to + /// compute the label. + /// + /// The weight to multiply the second feature with to + /// compute the label. + /// + /// The seed for generating feature values and label + /// noise. + /// /// An enumerable of Data objects. private static IEnumerable GenerateData(int nExamples = 10000, - double bias = 0, double weight1 = 1, double weight2 = 2, int seed = 1, int groupSize = 5) + double bias = 0, double weight1 = 1, double weight2 = 2, int seed = 1, + int groupSize = 5) { var rng = new Random(seed); var max = bias + 4.5 * weight1 + 4.5 * weight2 + 0.5; @@ -99,7 +120,8 @@ private static IEnumerable GenerateData(int nExamples = 10000, }; // Create a noisy label. - var value = (float)(bias + weight1 * data.Feature1 + weight2 * data.Feature2 + rng.NextDouble() - 0.5); + var value = (float)(bias + weight1 * data.Feature1 + weight2 * + data.Feature2 + rng.NextDouble() - 0.5); if (value < max / 3) data.Label = 0; else if (value < 2 * max / 3) diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Ranking/Ranking.ttinclude b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Ranking/Ranking.ttinclude index cc90a58187..05ed4a68b4 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Ranking/Ranking.ttinclude +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Ranking/Ranking.ttinclude @@ -13,15 +13,17 @@ namespace Samples.Dynamic.Trainers.Ranking {<#=Comments#> public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); <# if (TrainerOptions == null) { #> @@ -38,17 +40,21 @@ namespace Samples.Dynamic.Trainers.Ranking // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(500, seed:123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data.LoadFromEnumerable( + GenerateRandomDataPoints(500, seed:123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Take the top 5 rows. - var topTransformedTestData = mlContext.Data.TakeRows(transformedTestData, 5); + var topTransformedTestData = mlContext.Data.TakeRows( + transformedTestData, 5); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(topTransformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data.CreateEnumerable( + topTransformedTestData, reuseRowObject: false).ToList(); // Print 5 predictions. foreach (var p in predictions) @@ -63,7 +69,8 @@ namespace Samples.Dynamic.Trainers.Ranking <#=ExpectedOutput#> } - private static IEnumerable GenerateRandomDataPoints(int count, int seed = 0, int groupSize = 10) + private static IEnumerable GenerateRandomDataPoints(int count, + int seed = 0, int groupSize = 10) { var random = new Random(seed); float randomFloat() => (float)random.NextDouble(); @@ -75,13 +82,16 @@ namespace Samples.Dynamic.Trainers.Ranking Label = (uint)label, GroupId = (uint)(i / groupSize), // Create random features that are correlated with the label. - // For data points with larger labels, the feature values are slightly increased by adding a constant. - Features = Enumerable.Repeat(label, 50).Select(x => randomFloat() + x * 0.1f).ToArray() + // For data points with larger labels, the feature values are + // slightly increased by adding a constant. + Features = Enumerable.Repeat(label, 50).Select( + x => randomFloat() + x * 0.1f).ToArray() }; } } - // Example with label, groupId, and 50 feature values. A data set is a collection of such examples. + // Example with label, groupId, and 50 feature values. A data set is a + // collection of such examples. private class DataPoint { [KeyType(5)] @@ -104,8 +114,12 @@ namespace Samples.Dynamic.Trainers.Ranking // Pretty-print RankerMetrics objects. public static void PrintMetrics(RankingMetrics metrics) { - Console.WriteLine($"DCG: {string.Join(", ", metrics.DiscountedCumulativeGains.Select((d, i) => $"@{i + 1}:{d:F2}").ToArray())}"); - Console.WriteLine($"NDCG: {string.Join(", ", metrics.NormalizedDiscountedCumulativeGains.Select((d, i) => $"@{i + 1}:{d:F2}").ToArray())}"); + Console.WriteLine("DCG: " + string.Join(", ", + metrics.DiscountedCumulativeGains.Select( + (d, i) => (i + 1) + ":" + d + ":F2").ToArray())); + Console.WriteLine("NDCG: " + string.Join(", ", + metrics.NormalizedDiscountedCumulativeGains.Select( + (d, i) => (i + 1) + ":" + d + ":F2").ToArray())); } } -} \ No newline at end of file +} diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Recommendation/MatrixFactorization.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Recommendation/MatrixFactorization.cs index f77cc7ee8d..8c2dd2509d 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Recommendation/MatrixFactorization.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Recommendation/MatrixFactorization.cs @@ -9,25 +9,32 @@ namespace Samples.Dynamic.Trainers.Recommendation public static class MatrixFactorization { - // This example requires installation of additional nuget package Microsoft.ML.Recommender. - // In this example we will create in-memory data and then use it to train - // a matrix factorization model with default parameters. Afterward, quality metrics are reported. + // This example requires installation of additional nuget package at + // for Microsoft.ML.Recommender at + // https://www.nuget.org/packages/Microsoft.ML.Recommender/ + // In this example we will create in-memory data and then use it to train + // a matrix factorization model with default parameters. Afterward, quality + // metrics are reported. public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateMatrix(); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); // Define the trainer. - var pipeline = mlContext.Recommendation().Trainers.MatrixFactorization(nameof(MatrixElement.Value), nameof(MatrixElement.MatrixColumnIndex), - nameof(MatrixElement.MatrixRowIndex), 10, 0.2, 1); + var pipeline = mlContext.Recommendation().Trainers. + MatrixFactorization(nameof(MatrixElement.Value), + nameof(MatrixElement.MatrixColumnIndex), + nameof(MatrixElement.MatrixRowIndex), 10, 0.2, 1); // Train the model. var model = pipeline.Fit(trainingData); @@ -36,11 +43,15 @@ public static void Example() var transformedData = model.Transform(trainingData); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(transformedData, reuseRowObject: false).Take(5).ToList(); + var predictions = mlContext.Data + .CreateEnumerable(transformedData, + reuseRowObject: false).Take(5).ToList(); - // Look at 5 predictions for the Label, side by side with the actual Label for comparison. + // Look at 5 predictions for the Label, side by side with the actual + // Label for comparison. foreach (var p in predictions) - Console.WriteLine($"Actual value: {p.Value:F3}, Predicted score: {p.Score:F3}"); + Console.WriteLine($"Actual value: {p.Value:F3}," + + $"Predicted score: {p.Score:F3}"); // Expected output: // Actual value: 0.000, Predicted score: 1.234 @@ -50,7 +61,10 @@ public static void Example() // Actual value: 4.000, Predicted score: 2.362 // Evaluate the overall metrics - var metrics = mlContext.Regression.Evaluate(transformedData, labelColumnName: nameof(MatrixElement.Value), scoreColumnName: nameof(MatrixElement.Score)); + var metrics = mlContext.Regression.Evaluate(transformedData, + labelColumnName: nameof(MatrixElement.Value), + scoreColumnName: nameof(MatrixElement.Score)); + PrintMetrics(metrics); // Expected output: @@ -60,11 +74,15 @@ public static void Example() // RSquared: 0.61 (closer to 1 is better. The worest case is 0) } - // The following variables are used to define the shape of the example matrix. Its shape is MatrixRowCount-by-MatrixColumnCount. - // Because in ML.NET key type's minimal value is zero, the first row index is always zero in C# data structure (e.g., MatrixColumnIndex=0 - // and MatrixRowIndex=0 in MatrixElement below specifies the value at the upper-left corner in the training matrix). If user's row index - // starts with 1, their row index 1 would be mapped to the 2nd row in matrix factorization module and their first row may contain no values. - // This behavior is also true to column index. + // The following variables are used to define the shape of the example + // matrix. Its shape is MatrixRowCount-by-MatrixColumnCount. Because in + // ML.NET key type's minimal value is zero, the first row index is always + // zero in C# data structure (e.g., MatrixColumnIndex=0 and MatrixRowIndex=0 + // in MatrixElement below specifies the value at the upper-left corner in + // the training matrix). If user's row index starts with 1, their row index + // 1 would be mapped to the 2nd row in matrix factorization module and their + // first row may contain no values. This behavior is also true to column + // index. private const uint MatrixColumnCount = 60; private const uint MatrixRowCount = 100; @@ -74,32 +92,40 @@ private static List GenerateMatrix() var dataMatrix = new List(); for (uint i = 0; i < MatrixColumnCount; ++i) for (uint j = 0; j < MatrixRowCount; ++j) - dataMatrix.Add(new MatrixElement() { MatrixColumnIndex = i, MatrixRowIndex = j, Value = (i + j) % 5 }); + dataMatrix.Add(new MatrixElement() { MatrixColumnIndex = i, + MatrixRowIndex = j, Value = (i + j) % 5 }); + return dataMatrix; } - // A class used to define a matrix element and capture its prediction result. + // A class used to define a matrix element and capture its prediction + // result. private class MatrixElement { - // Matrix column index. Its allowed range is from 0 to MatrixColumnCount - 1. + // Matrix column index. Its allowed range is from 0 to + // MatrixColumnCount - 1. [KeyType(MatrixColumnCount)] public uint MatrixColumnIndex { get; set; } // Matrix row index. Its allowed range is from 0 to MatrixRowCount - 1. [KeyType(MatrixRowCount)] public uint MatrixRowIndex { get; set; } - // The actual value at the MatrixColumnIndex-th column and the MatrixRowIndex-th row. + // The actual value at the MatrixColumnIndex-th column and the + // MatrixRowIndex-th row. public float Value { get; set; } - // The predicted value at the MatrixColumnIndex-th column and the MatrixRowIndex-th row. + // The predicted value at the MatrixColumnIndex-th column and the + // MatrixRowIndex-th row. public float Score { get; set; } } // Print some evaluation metrics to regression problems. private static void PrintMetrics(RegressionMetrics metrics) { - Console.WriteLine($"Mean Absolute Error: {metrics.MeanAbsoluteError:F2}"); - Console.WriteLine($"Mean Squared Error: {metrics.MeanSquaredError:F2}"); - Console.WriteLine($"Root Mean Squared Error: {metrics.RootMeanSquaredError:F2}"); - Console.WriteLine($"RSquared: {metrics.RSquared:F2}"); + Console.WriteLine("Mean Absolute Error: " + metrics.MeanAbsoluteError); + Console.WriteLine("Mean Squared Error: " + metrics.MeanSquaredError); + Console.WriteLine("Root Mean Squared Error: " + + metrics.RootMeanSquaredError); + + Console.WriteLine("RSquared: " + metrics.RSquared); } } } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Recommendation/MatrixFactorization.tt b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Recommendation/MatrixFactorization.tt index 496330722e..0c0092d12c 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Recommendation/MatrixFactorization.tt +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Recommendation/MatrixFactorization.tt @@ -2,13 +2,18 @@ <#+ string ClassHeader = @" - // This example requires installation of additional nuget package Microsoft.ML.Recommender. - // In this example we will create in-memory data and then use it to train - // a matrix factorization model with default parameters. Afterward, quality metrics are reported."; + // This example requires installation of additional nuget package at + // for Microsoft.ML.Recommender at + // https://www.nuget.org/packages/Microsoft.ML.Recommender/ + // In this example we will create in-memory data and then use it to train + // a matrix factorization model with default parameters. Afterward, quality + // metrics are reported."; string ClassName="MatrixFactorization"; string ExtraUsing = null; -string Trainer = @"MatrixFactorization(nameof(MatrixElement.Value), nameof(MatrixElement.MatrixColumnIndex), - nameof(MatrixElement.MatrixRowIndex), 10, 0.2, 1)"; +string Trainer = @" + MatrixFactorization(nameof(MatrixElement.Value), + nameof(MatrixElement.MatrixColumnIndex), + nameof(MatrixElement.MatrixRowIndex), 10, 0.2, 1)"; string TrainerOptions = null; string ExpectedOutputPerInstance= @"// Expected output: @@ -23,4 +28,4 @@ string ExpectedOutput = @"// Expected output: // Mean Squared Error: 0.79 // Root Mean Squared Error: 0.89 // RSquared: 0.61 (closer to 1 is better. The worest case is 0)"; -#> +#> \ No newline at end of file diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Recommendation/MatrixFactorizationTemplate.ttinclude b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Recommendation/MatrixFactorizationTemplate.ttinclude index c2ce0fe08d..1a6f593c8a 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Recommendation/MatrixFactorizationTemplate.ttinclude +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Recommendation/MatrixFactorizationTemplate.ttinclude @@ -16,15 +16,17 @@ namespace Samples.Dynamic.Trainers.Recommendation <# } #> public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateMatrix(); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); <# if (TrainerOptions == null) { #> @@ -35,7 +37,8 @@ namespace Samples.Dynamic.Trainers.Recommendation var options = new <#=TrainerOptions#>; // Define the trainer. - var pipeline = mlContext.Recommendation().Trainers.<#=Trainer#>(options); + var pipeline = mlContext.Recommendation().Trainers.<#=Trainer#>( + options); <# } #> // Train the model. @@ -45,26 +48,37 @@ namespace Samples.Dynamic.Trainers.Recommendation var transformedData = model.Transform(trainingData); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(transformedData, reuseRowObject: false).Take(5).ToList(); + var predictions = mlContext.Data + .CreateEnumerable(transformedData, + reuseRowObject: false).Take(5).ToList(); - // Look at 5 predictions for the Label, side by side with the actual Label for comparison. + // Look at 5 predictions for the Label, side by side with the actual + // Label for comparison. foreach (var p in predictions) - Console.WriteLine($"Actual value: {p.Value:F3}, Predicted score: {p.Score:F3}"); + Console.WriteLine($"Actual value: {p.Value:F3}," + + $"Predicted score: {p.Score:F3}"); <#=ExpectedOutputPerInstance#> // Evaluate the overall metrics - var metrics = mlContext.Regression.Evaluate(transformedData, labelColumnName: nameof(MatrixElement.Value), scoreColumnName: nameof(MatrixElement.Score)); + var metrics = mlContext.Regression.Evaluate(transformedData, + labelColumnName: nameof(MatrixElement.Value), + scoreColumnName: nameof(MatrixElement.Score)); + PrintMetrics(metrics); <#=ExpectedOutput#> } - // The following variables are used to define the shape of the example matrix. Its shape is MatrixRowCount-by-MatrixColumnCount. - // Because in ML.NET key type's minimal value is zero, the first row index is always zero in C# data structure (e.g., MatrixColumnIndex=0 - // and MatrixRowIndex=0 in MatrixElement below specifies the value at the upper-left corner in the training matrix). If user's row index - // starts with 1, their row index 1 would be mapped to the 2nd row in matrix factorization module and their first row may contain no values. - // This behavior is also true to column index. + // The following variables are used to define the shape of the example + // matrix. Its shape is MatrixRowCount-by-MatrixColumnCount. Because in + // ML.NET key type's minimal value is zero, the first row index is always + // zero in C# data structure (e.g., MatrixColumnIndex=0 and MatrixRowIndex=0 + // in MatrixElement below specifies the value at the upper-left corner in + // the training matrix). If user's row index starts with 1, their row index + // 1 would be mapped to the 2nd row in matrix factorization module and their + // first row may contain no values. This behavior is also true to column + // index. private const uint MatrixColumnCount = 60; private const uint MatrixRowCount = 100; @@ -74,32 +88,40 @@ namespace Samples.Dynamic.Trainers.Recommendation var dataMatrix = new List(); for (uint i = 0; i < MatrixColumnCount; ++i) for (uint j = 0; j < MatrixRowCount; ++j) - dataMatrix.Add(new MatrixElement() { MatrixColumnIndex = i, MatrixRowIndex = j, Value = (i + j) % 5 }); + dataMatrix.Add(new MatrixElement() { MatrixColumnIndex = i, + MatrixRowIndex = j, Value = (i + j) % 5 }); + return dataMatrix; } - // A class used to define a matrix element and capture its prediction result. + // A class used to define a matrix element and capture its prediction + // result. private class MatrixElement { - // Matrix column index. Its allowed range is from 0 to MatrixColumnCount - 1. + // Matrix column index. Its allowed range is from 0 to + // MatrixColumnCount - 1. [KeyType(MatrixColumnCount)] public uint MatrixColumnIndex { get; set; } // Matrix row index. Its allowed range is from 0 to MatrixRowCount - 1. [KeyType(MatrixRowCount)] public uint MatrixRowIndex { get; set; } - // The actual value at the MatrixColumnIndex-th column and the MatrixRowIndex-th row. + // The actual value at the MatrixColumnIndex-th column and the + // MatrixRowIndex-th row. public float Value { get; set; } - // The predicted value at the MatrixColumnIndex-th column and the MatrixRowIndex-th row. + // The predicted value at the MatrixColumnIndex-th column and the + // MatrixRowIndex-th row. public float Score { get; set; } } // Print some evaluation metrics to regression problems. private static void PrintMetrics(RegressionMetrics metrics) { - Console.WriteLine($"Mean Absolute Error: {metrics.MeanAbsoluteError:F2}"); - Console.WriteLine($"Mean Squared Error: {metrics.MeanSquaredError:F2}"); - Console.WriteLine($"Root Mean Squared Error: {metrics.RootMeanSquaredError:F2}"); - Console.WriteLine($"RSquared: {metrics.RSquared:F2}"); + Console.WriteLine("Mean Absolute Error: " + metrics.MeanAbsoluteError); + Console.WriteLine("Mean Squared Error: " + metrics.MeanSquaredError); + Console.WriteLine("Root Mean Squared Error: " + + metrics.RootMeanSquaredError); + + Console.WriteLine("RSquared: " + metrics.RSquared); } } } \ No newline at end of file diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Recommendation/MatrixFactorizationWithOptions.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Recommendation/MatrixFactorizationWithOptions.cs index 45ea6813af..4d27300dfb 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Recommendation/MatrixFactorizationWithOptions.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Recommendation/MatrixFactorizationWithOptions.cs @@ -10,44 +10,55 @@ namespace Samples.Dynamic.Trainers.Recommendation public static class MatrixFactorizationWithOptions { - // This example requires installation of additional nuget package Microsoft.ML.Recommender. - // In this example we will create in-memory data and then use it to train - // a matrix factorization model with non-default parameters. Afterward, quality metrics are reported. + // This example requires installation of additional nuget package at + // for Microsoft.ML.Recommender at + // https://www.nuget.org/packages/Microsoft.ML.Recommender/ + // In this example we will create in-memory data and then use it to train + // a matrix factorization model with default parameters. Afterward, quality + // metrics are reported. public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateMatrix(); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); // Define trainer options. var options = new MatrixFactorizationTrainer.Options { - // Specify IDataView colum which stores matrix column indexes. - MatrixColumnIndexColumnName = nameof(MatrixElement.MatrixColumnIndex), - // Specify IDataView colum which stores matrix row indexes. + // Specify IDataView colum which stores matrix column indexes. + MatrixColumnIndexColumnName = nameof(MatrixElement.MatrixColumnIndex + ), + + // Specify IDataView colum which stores matrix row indexes. MatrixRowIndexColumnName = nameof(MatrixElement.MatrixRowIndex), - // Specify IDataView colum which stores matrix elements' values. + // Specify IDataView colum which stores matrix elements' values. LabelColumnName = nameof(MatrixElement.Value), - // Time of going through the entire data set once. + // Time of going through the entire data set once. NumberOfIterations = 10, - // Number of threads used to run this trainers. + // Number of threads used to run this trainers. NumberOfThreads = 1, - // The rank of factor matrices. Note that the product of the two factor matrices approximates the training matrix. + // The rank of factor matrices. Note that the product of the two + // factor matrices approximates the training matrix. ApproximationRank = 32, - // Step length when moving toward stochastic gradient. Training algorithm may adjust it for faster convergence. - // Note that faster convergence means we can use less iterations to achieve similar test scores. + // Step length when moving toward stochastic gradient. Training + // algorithm may adjust it for faster convergence. Note that faster + // convergence means we can use less iterations to achieve similar + // test scores. LearningRate = 0.3 }; // Define the trainer. - var pipeline = mlContext.Recommendation().Trainers.MatrixFactorization(options); + var pipeline = mlContext.Recommendation().Trainers.MatrixFactorization( + options); // Train the model. var model = pipeline.Fit(trainingData); @@ -56,11 +67,15 @@ public static void Example() var transformedData = model.Transform(trainingData); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(transformedData, reuseRowObject: false).Take(5).ToList(); + var predictions = mlContext.Data + .CreateEnumerable(transformedData, + reuseRowObject: false).Take(5).ToList(); - // Look at 5 predictions for the Label, side by side with the actual Label for comparison. + // Look at 5 predictions for the Label, side by side with the actual + // Label for comparison. foreach (var p in predictions) - Console.WriteLine($"Actual value: {p.Value:F3}, Predicted score: {p.Score:F3}"); + Console.WriteLine($"Actual value: {p.Value:F3}," + + $"Predicted score: {p.Score:F3}"); // Expected output: // Actual value: 0.000, Predicted score: 0.031 @@ -70,7 +85,10 @@ public static void Example() // Actual value: 4.000, Predicted score: 3.176 // Evaluate the overall metrics - var metrics = mlContext.Regression.Evaluate(transformedData, labelColumnName: nameof(MatrixElement.Value), scoreColumnName: nameof(MatrixElement.Score)); + var metrics = mlContext.Regression.Evaluate(transformedData, + labelColumnName: nameof(MatrixElement.Value), + scoreColumnName: nameof(MatrixElement.Score)); + PrintMetrics(metrics); // Expected output: @@ -80,11 +98,15 @@ public static void Example() // RSquared: 0.97 (closer to 1 is better. The worest case is 0) } - // The following variables are used to define the shape of the example matrix. Its shape is MatrixRowCount-by-MatrixColumnCount. - // Because in ML.NET key type's minimal value is zero, the first row index is always zero in C# data structure (e.g., MatrixColumnIndex=0 - // and MatrixRowIndex=0 in MatrixElement below specifies the value at the upper-left corner in the training matrix). If user's row index - // starts with 1, their row index 1 would be mapped to the 2nd row in matrix factorization module and their first row may contain no values. - // This behavior is also true to column index. + // The following variables are used to define the shape of the example + // matrix. Its shape is MatrixRowCount-by-MatrixColumnCount. Because in + // ML.NET key type's minimal value is zero, the first row index is always + // zero in C# data structure (e.g., MatrixColumnIndex=0 and MatrixRowIndex=0 + // in MatrixElement below specifies the value at the upper-left corner in + // the training matrix). If user's row index starts with 1, their row index + // 1 would be mapped to the 2nd row in matrix factorization module and their + // first row may contain no values. This behavior is also true to column + // index. private const uint MatrixColumnCount = 60; private const uint MatrixRowCount = 100; @@ -94,32 +116,40 @@ private static List GenerateMatrix() var dataMatrix = new List(); for (uint i = 0; i < MatrixColumnCount; ++i) for (uint j = 0; j < MatrixRowCount; ++j) - dataMatrix.Add(new MatrixElement() { MatrixColumnIndex = i, MatrixRowIndex = j, Value = (i + j) % 5 }); + dataMatrix.Add(new MatrixElement() { MatrixColumnIndex = i, + MatrixRowIndex = j, Value = (i + j) % 5 }); + return dataMatrix; } - // A class used to define a matrix element and capture its prediction result. + // A class used to define a matrix element and capture its prediction + // result. private class MatrixElement { - // Matrix column index. Its allowed range is from 0 to MatrixColumnCount - 1. + // Matrix column index. Its allowed range is from 0 to + // MatrixColumnCount - 1. [KeyType(MatrixColumnCount)] public uint MatrixColumnIndex { get; set; } // Matrix row index. Its allowed range is from 0 to MatrixRowCount - 1. [KeyType(MatrixRowCount)] public uint MatrixRowIndex { get; set; } - // The actual value at the MatrixColumnIndex-th column and the MatrixRowIndex-th row. + // The actual value at the MatrixColumnIndex-th column and the + // MatrixRowIndex-th row. public float Value { get; set; } - // The predicted value at the MatrixColumnIndex-th column and the MatrixRowIndex-th row. + // The predicted value at the MatrixColumnIndex-th column and the + // MatrixRowIndex-th row. public float Score { get; set; } } // Print some evaluation metrics to regression problems. private static void PrintMetrics(RegressionMetrics metrics) { - Console.WriteLine($"Mean Absolute Error: {metrics.MeanAbsoluteError:F2}"); - Console.WriteLine($"Mean Squared Error: {metrics.MeanSquaredError:F2}"); - Console.WriteLine($"Root Mean Squared Error: {metrics.RootMeanSquaredError:F2}"); - Console.WriteLine($"RSquared: {metrics.RSquared:F2}"); + Console.WriteLine("Mean Absolute Error: " + metrics.MeanAbsoluteError); + Console.WriteLine("Mean Squared Error: " + metrics.MeanSquaredError); + Console.WriteLine("Root Mean Squared Error: " + + metrics.RootMeanSquaredError); + + Console.WriteLine("RSquared: " + metrics.RSquared); } } } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Recommendation/MatrixFactorizationWithOptions.tt b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Recommendation/MatrixFactorizationWithOptions.tt index e4d483fcbe..63aadfaf35 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Recommendation/MatrixFactorizationWithOptions.tt +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Recommendation/MatrixFactorizationWithOptions.tt @@ -2,28 +2,36 @@ <#+ string ClassHeader = @" - // This example requires installation of additional nuget package Microsoft.ML.Recommender. - // In this example we will create in-memory data and then use it to train - // a matrix factorization model with non-default parameters. Afterward, quality metrics are reported."; + // This example requires installation of additional nuget package at + // for Microsoft.ML.Recommender at + // https://www.nuget.org/packages/Microsoft.ML.Recommender/ + // In this example we will create in-memory data and then use it to train + // a matrix factorization model with default parameters. Afterward, quality + // metrics are reported."; string ClassName="MatrixFactorizationWithOptions"; string ExtraUsing = "using Microsoft.ML.Trainers;"; string Trainer = "MatrixFactorization"; string TrainerOptions = @"MatrixFactorizationTrainer.Options { - // Specify IDataView colum which stores matrix column indexes. - MatrixColumnIndexColumnName = nameof(MatrixElement.MatrixColumnIndex), - // Specify IDataView colum which stores matrix row indexes. + // Specify IDataView colum which stores matrix column indexes. + MatrixColumnIndexColumnName = nameof(MatrixElement.MatrixColumnIndex + ), + + // Specify IDataView colum which stores matrix row indexes. MatrixRowIndexColumnName = nameof(MatrixElement.MatrixRowIndex), - // Specify IDataView colum which stores matrix elements' values. + // Specify IDataView colum which stores matrix elements' values. LabelColumnName = nameof(MatrixElement.Value), - // Time of going through the entire data set once. + // Time of going through the entire data set once. NumberOfIterations = 10, - // Number of threads used to run this trainers. + // Number of threads used to run this trainers. NumberOfThreads = 1, - // The rank of factor matrices. Note that the product of the two factor matrices approximates the training matrix. + // The rank of factor matrices. Note that the product of the two + // factor matrices approximates the training matrix. ApproximationRank = 32, - // Step length when moving toward stochastic gradient. Training algorithm may adjust it for faster convergence. - // Note that faster convergence means we can use less iterations to achieve similar test scores. + // Step length when moving toward stochastic gradient. Training + // algorithm may adjust it for faster convergence. Note that faster + // convergence means we can use less iterations to achieve similar + // test scores. LearningRate = 0.3 }"; diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Recommendation/OneClassMatrixFactorizationWithOptions.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Recommendation/OneClassMatrixFactorizationWithOptions.cs index 57cb7f2e6b..2553efe5b0 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Recommendation/OneClassMatrixFactorizationWithOptions.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Recommendation/OneClassMatrixFactorizationWithOptions.cs @@ -9,60 +9,80 @@ namespace Samples.Dynamic.Trainers.Recommendation { public static class OneClassMatrixFactorizationWithOptions { - // This example shows the use of ML.NET's one-class matrix factorization module which implements a coordinate descent method - // described in Algorithm 1 in a paper. - // See page 28 in of slides for a brief introduction to - // one-class matrix factorization. - // In this example we will create in-memory data and then use it to train a one-class matrix factorization model. - // Afterward, prediction values are reported. - // To run this example, it requires installation of additional nuget package - // Microsoft.ML.Recommender. + // This example shows the use of ML.NET's one-class matrix factorization + // module which implements a coordinate descent method described in + // Algorithm 1 in the paper found at + // https://www.csie.ntu.edu.tw/~cjlin/papers/one-class-mf/biased-mf-sdm-with-supp.pdf + // See page 28 in of the slides + // at https://www.csie.ntu.edu.tw/~cjlin/talks/facebook.pdf for a brief + // introduction to one-class matrix factorization. + // In this example we will create in-memory data and then use it to train a + // one-class matrix factorization model. Afterward, prediction values are + // reported. To run this example, it requires installation of additional + // nuget package Microsoft.ML.Recommender found at + // https://www.nuget.org/packages/Microsoft.ML.Recommender/ public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. var mlContext = new MLContext(seed: 0); // Get a small in-memory dataset. - GetOneClassMatrix(out List data, out List testData); + GetOneClassMatrix(out List data, + out List testData); - // Convert the in-memory matrix into an IDataView so that ML.NET components can consume it. + // Convert the in-memory matrix into an IDataView so that ML.NET + // components can consume it. var dataView = mlContext.Data.LoadFromEnumerable(data); - // Create a matrix factorization trainer which takes "Value" as the training label, "MatrixColumnIndex" as the - // matrix's column index, and "MatrixRowIndex" as the matrix's row index. Here nameof(...) is used to extract field + // Create a matrix factorization trainer which takes "Value" as the + // training label, "MatrixColumnIndex" as the matrix's column index, and + // "MatrixRowIndex" as the matrix's row index. Here nameof(...) is used + // to extract field // names' in MatrixElement class. var options = new MatrixFactorizationTrainer.Options { - MatrixColumnIndexColumnName = nameof(MatrixElement.MatrixColumnIndex), - MatrixRowIndexColumnName = nameof(MatrixElement.MatrixRowIndex), - LabelColumnName = nameof(MatrixElement.Value), - NumberOfIterations = 20, - NumberOfThreads = 8, - ApproximationRank = 32, - Alpha = 1, - // The desired values of matrix elements not specified in the training set. - // If the training set doesn't tell the value at the u-th row and v-th column, - // its desired value would be set 0.15. In other words, this parameter determines - // the value of all missing matrix elements. + MatrixColumnIndexColumnName = nameof( + MatrixElement.MatrixColumnIndex), + MatrixRowIndexColumnName = nameof(MatrixElement.MatrixRowIndex), + LabelColumnName = nameof(MatrixElement.Value), + NumberOfIterations = 20, + NumberOfThreads = 8, + ApproximationRank = 32, + Alpha = 1, + + // The desired values of matrix elements not specified in the + // training set. If the training set doesn't tell the value at the + // u -th row and v-th column, its desired value would be set 0.15. + // In other words, this parameter determines the value of all + // missing matrix elements. C = 0.15, // This argument enables one-class matrix factorization. - LossFunction = MatrixFactorizationTrainer.LossFunctionType.SquareLossOneClass + LossFunction = MatrixFactorizationTrainer.LossFunctionType + .SquareLossOneClass }; - var pipeline = mlContext.Recommendation().Trainers.MatrixFactorization(options); + var pipeline = mlContext.Recommendation().Trainers.MatrixFactorization( + options); // Train a matrix factorization model. var model = pipeline.Fit(dataView); - // Apply the trained model to the test set. Notice that training is a partial - var prediction = model.Transform(mlContext.Data.LoadFromEnumerable(testData)); + // Apply the trained model to the test set. Notice that training is a + // partial + var prediction = model.Transform(mlContext.Data.LoadFromEnumerable( + testData)); - var results = mlContext.Data.CreateEnumerable(prediction, false).ToList(); - // Feed the test data into the model and then iterate through a few predictions. + var results = mlContext.Data.CreateEnumerable(prediction, + false).ToList(); + // Feed the test data into the model and then iterate through a few + // predictions. foreach (var pred in results.Take(15)) - Console.WriteLine($"Predicted value at row {pred.MatrixRowIndex - 1} and column {pred.MatrixColumnIndex - 1} is " + - $"{pred.Score} and its expected value is {pred.Value}."); + Console.WriteLine($"Predicted value at row " + + $"{pred.MatrixRowIndex - 1} and column " + + $"{pred.MatrixColumnIndex - 1} is {pred.Score} and its " + + $"expected value is {pred.Value}."); // Expected output similar to: // Predicted value at row 0 and column 0 is 0.9873335 and its expected value is 1. @@ -81,17 +101,24 @@ public static void Example() // Predicted value at row 13 and column 0 is 0.1499254 and its expected value is 0.15. // Predicted value at row 14 and column 0 is 0.1499074 and its expected value is 0.15. // - // Note: use the advanced options constructor to set the number of threads to 1 for a deterministic behavior. + // Note: use the advanced options constructor to set the number of + // threads to 1 for a deterministic behavior. - // Assume that row index is user ID and column index game ID, the following list contains the games recommended by the trained model. - // Note that sometime, you may want to exclude training data from your predicted results because those would represent games that - // were already purchased. - // The variable topColumns stores two matrix elements with the highest predicted scores on the 1st row. - var topColumns = results.Where(element => element.MatrixRowIndex == 1).OrderByDescending(element => element.Score).Take(2); + // Assume that row index is user ID and column index game ID, the + // following list contains the games recommended by the trained model. + // Note that sometime, you may want to exclude training data from your + // predicted results because those would represent games that were + // already purchased. The variable topColumns stores two matrix elements + // with the highest predicted scores on the 1st row. + var topColumns = results.Where(element => element.MatrixRowIndex == 1) + .OrderByDescending(element => element.Score).Take(2); Console.WriteLine("Top 2 predictions on the 1st row:"); foreach (var top in topColumns) - Console.WriteLine($"Predicted value at row {top.MatrixRowIndex - 1} and column {top.MatrixColumnIndex - 1} is {top.Score} and its expected value is {top.Value}."); + Console.WriteLine($"Predicted value at row " + + $"{top.MatrixRowIndex - 1} and column " + + $"{top.MatrixColumnIndex - 1} is {top.Score} and its " + + $"expected value is {top.Value}."); // Expected output similar to: // Top 2 predictions at the 2nd row: @@ -99,10 +126,14 @@ public static void Example() // Predicted value at row 0 and column 10 is 0.9871138 and its expected value is 1. } - // The following variables defines the shape of a matrix. Its shape is _synthesizedMatrixRowCount-by-_synthesizedMatrixColumnCount. - // Because in ML.NET key type's minimal value is zero, the first row index is always zero in C# data structure (e.g., MatrixColumnIndex=0 - // and MatrixRowIndex=0 in MatrixElement below specifies the value at the upper-left corner in the training matrix). If user's row index - // starts with 1, their row index 1 would be mapped to the 2nd row in matrix factorization module and their first row may contain no values. + // The following variables defines the shape of a matrix. Its shape is + // _synthesizedMatrixRowCount-by-_synthesizedMatrixColumnCount. + // Because in ML.NET key type's minimal value is zero, the first row index + // is always zero in C# data structure (e.g., MatrixColumnIndex=0 and + // MatrixRowIndex=0 in MatrixElement below specifies the value at the + // upper-left corner in the training matrix). If user's row index + // starts with 1, their row index 1 would be mapped to the 2nd row in matrix + // factorization module and their first row may contain no values. // This behavior is also true to column index. private const uint _synthesizedMatrixColumnCount = 60; private const uint _synthesizedMatrixRowCount = 100; @@ -110,25 +141,33 @@ public static void Example() // A data structure used to encode a single value in matrix private class MatrixElement { - // Matrix column index. Its allowed range is from 0 to _synthesizedMatrixColumnCount - 1. + // Matrix column index. Its allowed range is from 0 to + // _synthesizedMatrixColumnCount - 1. [KeyType(_synthesizedMatrixColumnCount)] public uint MatrixColumnIndex { get; set; } - // Matrix row index. Its allowed range is from 0 to _synthesizedMatrixRowCount - 1. + // Matrix row index. Its allowed range is from 0 to + // _synthesizedMatrixRowCount - 1. [KeyType(_synthesizedMatrixRowCount)] public uint MatrixRowIndex { get; set; } - // The value at the MatrixColumnIndex-th column and the MatrixRowIndex-th row. + // The value at the MatrixColumnIndex-th column and the + // MatrixRowIndex-th row. public float Value { get; set; } - // The predicted value at the MatrixColumnIndex-th column and the MatrixRowIndex-th row. + // The predicted value at the MatrixColumnIndex-th column and the + // MatrixRowIndex-th row. public float Score { get; set; } } - // Create an in-memory matrix as a list of tuples (column index, row index, value). Notice that one-class matrix - // factorization handle scenerios where only positive signals (e.g., on Facebook, only likes are recorded and no dislike before) - // can be observed so that all values are set to 1. - private static void GetOneClassMatrix(out List observedMatrix, out List fullMatrix) + // Create an in-memory matrix as a list of tuples (column index, row index, + // value). Notice that one-class matrix factorization handle scenerios where + // only positive signals (e.g., on Facebook, only likes are recorded and no + // dislike before) can be observed so that all values are set to 1. + private static void GetOneClassMatrix( + out List observedMatrix, + out List fullMatrix) { - // The matrix factorization model will be trained only using observedMatrix but we will see it can learn all information - // carried in fullMatrix. + // The matrix factorization model will be trained only using + // observedMatrix but we will see it can learn all information carried + // sin fullMatrix. observedMatrix = new List(); fullMatrix = new List(); for (uint i = 0; i < _synthesizedMatrixColumnCount; ++i) @@ -137,13 +176,32 @@ private static void GetOneClassMatrix(out List observedMatrix, ou if ((i + j) % 10 == 0) { // Set observed elements' values to 1 (means like). - observedMatrix.Add(new MatrixElement() { MatrixColumnIndex = i, MatrixRowIndex = j, Value = 1, Score = 0 }); - fullMatrix.Add(new MatrixElement() { MatrixColumnIndex = i, MatrixRowIndex = j, Value = 1, Score = 0 }); + observedMatrix.Add(new MatrixElement() + { + MatrixColumnIndex = i, + MatrixRowIndex = j, + Value = 1, + Score = 0 + }); + fullMatrix.Add(new MatrixElement() + { + MatrixColumnIndex = i, + MatrixRowIndex = j, + Value = 1, + Score = 0 + }); } else - // Set unobserved elements' values to 0.15, a value smaller than observed values (means dislike). - fullMatrix.Add(new MatrixElement() { MatrixColumnIndex = i, MatrixRowIndex = j, Value = 0.15f, Score = 0 }); + // Set unobserved elements' values to 0.15, a value smaller + // than observed values (means dislike). + fullMatrix.Add(new MatrixElement() + { + MatrixColumnIndex = i, + MatrixRowIndex = j, + Value = 0.15f, + Score = 0 + }); } } } -} +} \ No newline at end of file diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/FastForest.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/FastForest.cs index bcb132a815..1cb2c7c466 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/FastForest.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/FastForest.cs @@ -8,37 +8,46 @@ namespace Samples.Dynamic.Trainers.Regression { public static class FastForestRegression { - // This example requires installation of additional NuGet package - // Microsoft.ML.FastTree. + // This example requires installation of additional NuGet + // package for Microsoft.ML.FastTree found at + // https://www.nuget.org/packages/Microsoft.ML.FastTree/ public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); // Define the trainer. - var pipeline = mlContext.Regression.Trainers.FastForest(labelColumnName: nameof(DataPoint.Label), featureColumnName: nameof(DataPoint.Features)); + var pipeline = mlContext.Regression.Trainers.FastForest( + labelColumnName: nameof(DataPoint.Label), + featureColumnName: nameof(DataPoint.Features)); // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(5, seed: 123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data.LoadFromEnumerable( + GenerateRandomDataPoints(5, seed: 123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(transformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data.CreateEnumerable( + transformedTestData, reuseRowObject: false).ToList(); - // Look at 5 predictions for the Label, side by side with the actual Label for comparison. + // Look at 5 predictions for the Label, side by side with the actual + // Label for comparison. foreach (var p in predictions) Console.WriteLine($"Label: {p.Label:F3}, Prediction: {p.Score:F3}"); @@ -60,7 +69,8 @@ public static void Example() // RSquared: 0.96 (closer to 1 is better. The worest case is 0) } - private static IEnumerable GenerateRandomDataPoints(int count, int seed=0) + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) { var random = new Random(seed); for (int i = 0; i < count; i++) @@ -70,12 +80,14 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se { Label = label, // Create random features that are correlated with the label. - Features = Enumerable.Repeat(label, 50).Select(x => x + (float)random.NextDouble()).ToArray() + Features = Enumerable.Repeat(label, 50).Select( + x => x + (float)random.NextDouble()).ToArray() }; } } - // Example with label and 50 feature values. A data set is a collection of such examples. + // Example with label and 50 feature values. A data set is a collection of + // such examples. private class DataPoint { public float Label { get; set; } @@ -95,10 +107,12 @@ private class Prediction // Print some evaluation metrics to regression problems. private static void PrintMetrics(RegressionMetrics metrics) { - Console.WriteLine($"Mean Absolute Error: {metrics.MeanAbsoluteError:F2}"); - Console.WriteLine($"Mean Squared Error: {metrics.MeanSquaredError:F2}"); - Console.WriteLine($"Root Mean Squared Error: {metrics.RootMeanSquaredError:F2}"); - Console.WriteLine($"RSquared: {metrics.RSquared:F2}"); + Console.WriteLine("Mean Absolute Error: " + metrics.MeanAbsoluteError); + Console.WriteLine("Mean Squared Error: " + metrics.MeanSquaredError); + Console.WriteLine( + "Root Mean Squared Error: " + metrics.RootMeanSquaredError); + + Console.WriteLine("RSquared: " + metrics.RSquared); } } } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/FastForest.tt b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/FastForest.tt index fbc1d6253d..92b968f921 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/FastForest.tt +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/FastForest.tt @@ -1,12 +1,16 @@ <#@ include file="RegressionSamplesTemplate.ttinclude"#> <#+ -string ClassHeader = @"// This example requires installation of additional NuGet package - // Microsoft.ML.FastTree. "; +string ClassHeader = @"// This example requires installation of additional NuGet + // package for Microsoft.ML.FastTree found at + // https://www.nuget.org/packages/Microsoft.ML.FastTree/"; string ClassName="FastForestRegression"; string ExtraUsing = null; -string Trainer = @"FastForest(labelColumnName: nameof(DataPoint.Label), featureColumnName: nameof(DataPoint.Features))"; +string Trainer = @"FastForest( + labelColumnName: nameof(DataPoint.Label), + featureColumnName: nameof(DataPoint.Features))"; + string TrainerOptions = null; string ExpectedOutputPerInstance= @"// Expected output: diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/FastForestWithOptions.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/FastForestWithOptions.cs index 780545f68a..5482db46be 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/FastForestWithOptions.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/FastForestWithOptions.cs @@ -9,19 +9,22 @@ namespace Samples.Dynamic.Trainers.Regression { public static class FastForestWithOptionsRegression { - // This example requires installation of additional NuGet package - // Microsoft.ML.FastTree. + // This example requires installation of additional NuGet + // package for Microsoft.ML.FastTree found at + // https://www.nuget.org/packages/Microsoft.ML.FastTree/ public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); // Define trainer options. @@ -38,21 +41,26 @@ public static void Example() }; // Define the trainer. - var pipeline = mlContext.Regression.Trainers.FastForest(options); + var pipeline = + mlContext.Regression.Trainers.FastForest(options); // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(5, seed: 123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data.LoadFromEnumerable( + GenerateRandomDataPoints(5, seed: 123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(transformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data.CreateEnumerable( + transformedTestData, reuseRowObject: false).ToList(); - // Look at 5 predictions for the Label, side by side with the actual Label for comparison. + // Look at 5 predictions for the Label, side by side with the actual + // Label for comparison. foreach (var p in predictions) Console.WriteLine($"Label: {p.Label:F3}, Prediction: {p.Score:F3}"); @@ -74,7 +82,8 @@ public static void Example() // RSquared: 0.95 (closer to 1 is better. The worest case is 0) } - private static IEnumerable GenerateRandomDataPoints(int count, int seed=0) + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) { var random = new Random(seed); for (int i = 0; i < count; i++) @@ -84,12 +93,14 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se { Label = label, // Create random features that are correlated with the label. - Features = Enumerable.Repeat(label, 50).Select(x => x + (float)random.NextDouble()).ToArray() + Features = Enumerable.Repeat(label, 50).Select( + x => x + (float)random.NextDouble()).ToArray() }; } } - // Example with label and 50 feature values. A data set is a collection of such examples. + // Example with label and 50 feature values. A data set is a collection of + // such examples. private class DataPoint { public float Label { get; set; } @@ -109,10 +120,12 @@ private class Prediction // Print some evaluation metrics to regression problems. private static void PrintMetrics(RegressionMetrics metrics) { - Console.WriteLine($"Mean Absolute Error: {metrics.MeanAbsoluteError:F2}"); - Console.WriteLine($"Mean Squared Error: {metrics.MeanSquaredError:F2}"); - Console.WriteLine($"Root Mean Squared Error: {metrics.RootMeanSquaredError:F2}"); - Console.WriteLine($"RSquared: {metrics.RSquared:F2}"); + Console.WriteLine("Mean Absolute Error: " + metrics.MeanAbsoluteError); + Console.WriteLine("Mean Squared Error: " + metrics.MeanSquaredError); + Console.WriteLine( + "Root Mean Squared Error: " + metrics.RootMeanSquaredError); + + Console.WriteLine("RSquared: " + metrics.RSquared); } } } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/FastForestWithOptions.tt b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/FastForestWithOptions.tt index 00b3740ddb..b32a5920b0 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/FastForestWithOptions.tt +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/FastForestWithOptions.tt @@ -1,8 +1,9 @@ <#@ include file="RegressionSamplesTemplate.ttinclude"#> <#+ -string ClassHeader = @"// This example requires installation of additional NuGet package - // Microsoft.ML.FastTree. "; +string ClassHeader = @"// This example requires installation of additional NuGet + // package for Microsoft.ML.FastTree found at + // https://www.nuget.org/packages/Microsoft.ML.FastTree/"; string ClassName="FastForestWithOptionsRegression"; string ExtraUsing = "using Microsoft.ML.Trainers.FastTree;"; diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/FastTree.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/FastTree.cs index 7a46027bc6..24f4626727 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/FastTree.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/FastTree.cs @@ -8,37 +8,46 @@ namespace Samples.Dynamic.Trainers.Regression { public static class FastTreeRegression { - // This example requires installation of additional NuGet package - // Microsoft.ML.FastTree. + // This example requires installation of additional NuGet + // package for Microsoft.ML.FastTree found at + // https://www.nuget.org/packages/Microsoft.ML.FastTree/ public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); // Define the trainer. - var pipeline = mlContext.Regression.Trainers.FastTree(labelColumnName: nameof(DataPoint.Label), featureColumnName: nameof(DataPoint.Features)); + var pipeline = mlContext.Regression.Trainers.FastForest( + labelColumnName: nameof(DataPoint.Label), + featureColumnName: nameof(DataPoint.Features)); // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(5, seed: 123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data.LoadFromEnumerable( + GenerateRandomDataPoints(5, seed: 123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(transformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data.CreateEnumerable( + transformedTestData, reuseRowObject: false).ToList(); - // Look at 5 predictions for the Label, side by side with the actual Label for comparison. + // Look at 5 predictions for the Label, side by side with the actual + // Label for comparison. foreach (var p in predictions) Console.WriteLine($"Label: {p.Label:F3}, Prediction: {p.Score:F3}"); @@ -60,7 +69,8 @@ public static void Example() // RSquared: 0.99 (closer to 1 is better. The worest case is 0) } - private static IEnumerable GenerateRandomDataPoints(int count, int seed=0) + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) { var random = new Random(seed); for (int i = 0; i < count; i++) @@ -70,12 +80,14 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se { Label = label, // Create random features that are correlated with the label. - Features = Enumerable.Repeat(label, 50).Select(x => x + (float)random.NextDouble()).ToArray() + Features = Enumerable.Repeat(label, 50).Select( + x => x + (float)random.NextDouble()).ToArray() }; } } - // Example with label and 50 feature values. A data set is a collection of such examples. + // Example with label and 50 feature values. A data set is a collection of + // such examples. private class DataPoint { public float Label { get; set; } @@ -95,10 +107,12 @@ private class Prediction // Print some evaluation metrics to regression problems. private static void PrintMetrics(RegressionMetrics metrics) { - Console.WriteLine($"Mean Absolute Error: {metrics.MeanAbsoluteError:F2}"); - Console.WriteLine($"Mean Squared Error: {metrics.MeanSquaredError:F2}"); - Console.WriteLine($"Root Mean Squared Error: {metrics.RootMeanSquaredError:F2}"); - Console.WriteLine($"RSquared: {metrics.RSquared:F2}"); + Console.WriteLine("Mean Absolute Error: " + metrics.MeanAbsoluteError); + Console.WriteLine("Mean Squared Error: " + metrics.MeanSquaredError); + Console.WriteLine( + "Root Mean Squared Error: " + metrics.RootMeanSquaredError); + + Console.WriteLine("RSquared: " + metrics.RSquared); } } } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/FastTree.tt b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/FastTree.tt index 5c06288114..b23398c9fd 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/FastTree.tt +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/FastTree.tt @@ -1,12 +1,15 @@ <#@ include file="RegressionSamplesTemplate.ttinclude"#> <#+ -string ClassHeader = @"// This example requires installation of additional NuGet package - // Microsoft.ML.FastTree. "; +string ClassHeader = @"// This example requires installation of additional NuGet + // package for Microsoft.ML.FastTree found at + // https://www.nuget.org/packages/Microsoft.ML.FastTree/"; string ClassName="FastTreeRegression"; string ExtraUsing = null; -string Trainer = @"FastTree(labelColumnName: nameof(DataPoint.Label), featureColumnName: nameof(DataPoint.Features))"; +string Trainer = @"FastForest( + labelColumnName: nameof(DataPoint.Label), + featureColumnName: nameof(DataPoint.Features))"; string TrainerOptions = null; string ExpectedOutputPerInstance= @"// Expected output: diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/FastTreeTweedie.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/FastTreeTweedie.cs index 22e6ff2e01..497919cdcb 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/FastTreeTweedie.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/FastTreeTweedie.cs @@ -8,37 +8,46 @@ namespace Samples.Dynamic.Trainers.Regression { public static class FastTreeTweedieRegression { - // This example requires installation of additional NuGet package - // Microsoft.ML.FastTree. + // This example requires installation of additional NuGet + // package for Microsoft.ML.FastTree found at + // https://www.nuget.org/packages/Microsoft.ML.FastTree/ public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); // Define the trainer. - var pipeline = mlContext.Regression.Trainers.FastTreeTweedie(labelColumnName: nameof(DataPoint.Label), featureColumnName: nameof(DataPoint.Features)); + var pipeline = mlContext.Regression.Trainers.FastForest( + labelColumnName: nameof(DataPoint.Label), + featureColumnName: nameof(DataPoint.Features)); // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(5, seed: 123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data.LoadFromEnumerable( + GenerateRandomDataPoints(5, seed: 123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(transformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data.CreateEnumerable( + transformedTestData, reuseRowObject: false).ToList(); - // Look at 5 predictions for the Label, side by side with the actual Label for comparison. + // Look at 5 predictions for the Label, side by side with the actual + // Label for comparison. foreach (var p in predictions) Console.WriteLine($"Label: {p.Label:F3}, Prediction: {p.Score:F3}"); @@ -60,7 +69,8 @@ public static void Example() // RSquared: 0.96 (closer to 1 is better. The worest case is 0) } - private static IEnumerable GenerateRandomDataPoints(int count, int seed=0) + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) { var random = new Random(seed); for (int i = 0; i < count; i++) @@ -70,12 +80,14 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se { Label = label, // Create random features that are correlated with the label. - Features = Enumerable.Repeat(label, 50).Select(x => x + (float)random.NextDouble()).ToArray() + Features = Enumerable.Repeat(label, 50).Select( + x => x + (float)random.NextDouble()).ToArray() }; } } - // Example with label and 50 feature values. A data set is a collection of such examples. + // Example with label and 50 feature values. A data set is a collection of + // such examples. private class DataPoint { public float Label { get; set; } @@ -95,10 +107,12 @@ private class Prediction // Print some evaluation metrics to regression problems. private static void PrintMetrics(RegressionMetrics metrics) { - Console.WriteLine($"Mean Absolute Error: {metrics.MeanAbsoluteError:F2}"); - Console.WriteLine($"Mean Squared Error: {metrics.MeanSquaredError:F2}"); - Console.WriteLine($"Root Mean Squared Error: {metrics.RootMeanSquaredError:F2}"); - Console.WriteLine($"RSquared: {metrics.RSquared:F2}"); + Console.WriteLine("Mean Absolute Error: " + metrics.MeanAbsoluteError); + Console.WriteLine("Mean Squared Error: " + metrics.MeanSquaredError); + Console.WriteLine( + "Root Mean Squared Error: " + metrics.RootMeanSquaredError); + + Console.WriteLine("RSquared: " + metrics.RSquared); } } } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/FastTreeTweedie.tt b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/FastTreeTweedie.tt index 4ecbbf1134..9befcb25e1 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/FastTreeTweedie.tt +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/FastTreeTweedie.tt @@ -1,12 +1,15 @@ <#@ include file="RegressionSamplesTemplate.ttinclude"#> <#+ -string ClassHeader = @"// This example requires installation of additional NuGet package - // Microsoft.ML.FastTree. "; +string ClassHeader = @"// This example requires installation of additional NuGet + // package for Microsoft.ML.FastTree found at + // https://www.nuget.org/packages/Microsoft.ML.FastTree/"; string ClassName="FastTreeTweedieRegression"; string ExtraUsing = null; -string Trainer = @"FastTreeTweedie(labelColumnName: nameof(DataPoint.Label), featureColumnName: nameof(DataPoint.Features))"; +string Trainer = @"FastForest( + labelColumnName: nameof(DataPoint.Label), + featureColumnName: nameof(DataPoint.Features))"; string TrainerOptions = null; string ExpectedOutputPerInstance= @"// Expected output: diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/FastTreeTweedieWithOptions.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/FastTreeTweedieWithOptions.cs index a0f8230954..eb4de40868 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/FastTreeTweedieWithOptions.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/FastTreeTweedieWithOptions.cs @@ -9,19 +9,22 @@ namespace Samples.Dynamic.Trainers.Regression { public static class FastTreeTweedieWithOptionsRegression { - // This example requires installation of additional NuGet package - // Microsoft.ML.FastTree. + // This example requires installation of additional NuGet + // package for Microsoft.ML.FastTree found at + // https://www.nuget.org/packages/Microsoft.ML.FastTree/ public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); // Define trainer options. @@ -30,7 +33,9 @@ public static void Example() LabelColumnName = nameof(DataPoint.Label), FeatureColumnName = nameof(DataPoint.Features), // Use L2Norm for early stopping. - EarlyStoppingMetric = Microsoft.ML.Trainers.FastTree.EarlyStoppingMetric.L2Norm, + EarlyStoppingMetric = + Microsoft.ML.Trainers.FastTree.EarlyStoppingMetric.L2Norm, + // Create a simpler model by penalizing usage of new features. FeatureFirstUsePenalty = 0.1, // Reduce the number of trees to 50. @@ -38,21 +43,26 @@ public static void Example() }; // Define the trainer. - var pipeline = mlContext.Regression.Trainers.FastTreeTweedie(options); + var pipeline = + mlContext.Regression.Trainers.FastTreeTweedie(options); // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(5, seed: 123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data.LoadFromEnumerable( + GenerateRandomDataPoints(5, seed: 123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(transformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data.CreateEnumerable( + transformedTestData, reuseRowObject: false).ToList(); - // Look at 5 predictions for the Label, side by side with the actual Label for comparison. + // Look at 5 predictions for the Label, side by side with the actual + // Label for comparison. foreach (var p in predictions) Console.WriteLine($"Label: {p.Label:F3}, Prediction: {p.Score:F3}"); @@ -74,7 +84,8 @@ public static void Example() // RSquared: 0.98 (closer to 1 is better. The worest case is 0) } - private static IEnumerable GenerateRandomDataPoints(int count, int seed=0) + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) { var random = new Random(seed); for (int i = 0; i < count; i++) @@ -84,12 +95,14 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se { Label = label, // Create random features that are correlated with the label. - Features = Enumerable.Repeat(label, 50).Select(x => x + (float)random.NextDouble()).ToArray() + Features = Enumerable.Repeat(label, 50).Select( + x => x + (float)random.NextDouble()).ToArray() }; } } - // Example with label and 50 feature values. A data set is a collection of such examples. + // Example with label and 50 feature values. A data set is a collection of + // such examples. private class DataPoint { public float Label { get; set; } @@ -109,10 +122,12 @@ private class Prediction // Print some evaluation metrics to regression problems. private static void PrintMetrics(RegressionMetrics metrics) { - Console.WriteLine($"Mean Absolute Error: {metrics.MeanAbsoluteError:F2}"); - Console.WriteLine($"Mean Squared Error: {metrics.MeanSquaredError:F2}"); - Console.WriteLine($"Root Mean Squared Error: {metrics.RootMeanSquaredError:F2}"); - Console.WriteLine($"RSquared: {metrics.RSquared:F2}"); + Console.WriteLine("Mean Absolute Error: " + metrics.MeanAbsoluteError); + Console.WriteLine("Mean Squared Error: " + metrics.MeanSquaredError); + Console.WriteLine( + "Root Mean Squared Error: " + metrics.RootMeanSquaredError); + + Console.WriteLine("RSquared: " + metrics.RSquared); } } } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/FastTreeTweedieWithOptions.tt b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/FastTreeTweedieWithOptions.tt index 98d4d5e6a5..1c37bbe506 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/FastTreeTweedieWithOptions.tt +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/FastTreeTweedieWithOptions.tt @@ -1,8 +1,9 @@ <#@ include file="RegressionSamplesTemplate.ttinclude"#> <#+ -string ClassHeader = @"// This example requires installation of additional NuGet package - // Microsoft.ML.FastTree. "; +string ClassHeader = @"// This example requires installation of additional NuGet + // package for Microsoft.ML.FastTree found at + // https://www.nuget.org/packages/Microsoft.ML.FastTree/"; string ClassName="FastTreeTweedieWithOptionsRegression"; string ExtraUsing = "using Microsoft.ML.Trainers.FastTree;"; @@ -12,7 +13,9 @@ string TrainerOptions = @"FastTreeTweedieTrainer.Options LabelColumnName = nameof(DataPoint.Label), FeatureColumnName = nameof(DataPoint.Features), // Use L2Norm for early stopping. - EarlyStoppingMetric = Microsoft.ML.Trainers.FastTree.EarlyStoppingMetric.L2Norm, + EarlyStoppingMetric = + Microsoft.ML.Trainers.FastTree.EarlyStoppingMetric.L2Norm, + // Create a simpler model by penalizing usage of new features. FeatureFirstUsePenalty = 0.1, // Reduce the number of trees to 50. diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/FastTreeWithOptions.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/FastTreeWithOptions.cs index 1ed25cff1b..235a298d82 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/FastTreeWithOptions.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/FastTreeWithOptions.cs @@ -9,19 +9,22 @@ namespace Samples.Dynamic.Trainers.Regression { public static class FastTreeWithOptionsRegression { - // This example requires installation of additional NuGet package - // Microsoft.ML.FastTree. + // This example requires installation of additional NuGet + // package for Microsoft.ML.FastTree found at + // https://www.nuget.org/packages/Microsoft.ML.FastTree/ public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); // Define trainer options. @@ -29,9 +32,11 @@ public static void Example() { LabelColumnName = nameof(DataPoint.Label), FeatureColumnName = nameof(DataPoint.Features), - // Use L2-norm for early stopping. If the gradient's L2-norm is smaller than - // an auto-computed value, training process will stop. - EarlyStoppingMetric = Microsoft.ML.Trainers.FastTree.EarlyStoppingMetric.L2Norm, + // Use L2-norm for early stopping. If the gradient's L2-norm is + // smaller than an auto-computed value, training process will stop. + EarlyStoppingMetric = + Microsoft.ML.Trainers.FastTree.EarlyStoppingMetric.L2Norm, + // Create a simpler model by penalizing usage of new features. FeatureFirstUsePenalty = 0.1, // Reduce the number of trees to 50. @@ -39,21 +44,26 @@ public static void Example() }; // Define the trainer. - var pipeline = mlContext.Regression.Trainers.FastTree(options); + var pipeline = + mlContext.Regression.Trainers.FastTree(options); // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(5, seed: 123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data.LoadFromEnumerable( + GenerateRandomDataPoints(5, seed: 123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(transformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data.CreateEnumerable( + transformedTestData, reuseRowObject: false).ToList(); - // Look at 5 predictions for the Label, side by side with the actual Label for comparison. + // Look at 5 predictions for the Label, side by side with the actual + // Label for comparison. foreach (var p in predictions) Console.WriteLine($"Label: {p.Label:F3}, Prediction: {p.Score:F3}"); @@ -75,7 +85,8 @@ public static void Example() // RSquared: 0.99 (closer to 1 is better. The worest case is 0) } - private static IEnumerable GenerateRandomDataPoints(int count, int seed=0) + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) { var random = new Random(seed); for (int i = 0; i < count; i++) @@ -85,12 +96,14 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se { Label = label, // Create random features that are correlated with the label. - Features = Enumerable.Repeat(label, 50).Select(x => x + (float)random.NextDouble()).ToArray() + Features = Enumerable.Repeat(label, 50).Select( + x => x + (float)random.NextDouble()).ToArray() }; } } - // Example with label and 50 feature values. A data set is a collection of such examples. + // Example with label and 50 feature values. A data set is a collection of + // such examples. private class DataPoint { public float Label { get; set; } @@ -110,10 +123,12 @@ private class Prediction // Print some evaluation metrics to regression problems. private static void PrintMetrics(RegressionMetrics metrics) { - Console.WriteLine($"Mean Absolute Error: {metrics.MeanAbsoluteError:F2}"); - Console.WriteLine($"Mean Squared Error: {metrics.MeanSquaredError:F2}"); - Console.WriteLine($"Root Mean Squared Error: {metrics.RootMeanSquaredError:F2}"); - Console.WriteLine($"RSquared: {metrics.RSquared:F2}"); + Console.WriteLine("Mean Absolute Error: " + metrics.MeanAbsoluteError); + Console.WriteLine("Mean Squared Error: " + metrics.MeanSquaredError); + Console.WriteLine( + "Root Mean Squared Error: " + metrics.RootMeanSquaredError); + + Console.WriteLine("RSquared: " + metrics.RSquared); } } } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/FastTreeWithOptions.tt b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/FastTreeWithOptions.tt index df768cf53d..cb80ea4566 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/FastTreeWithOptions.tt +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/FastTreeWithOptions.tt @@ -1,8 +1,9 @@ <#@ include file="RegressionSamplesTemplate.ttinclude"#> <#+ -string ClassHeader = @"// This example requires installation of additional NuGet package - // Microsoft.ML.FastTree. "; +string ClassHeader = @"// This example requires installation of additional NuGet + // package for Microsoft.ML.FastTree found at + // https://www.nuget.org/packages/Microsoft.ML.FastTree/"; string ClassName="FastTreeWithOptionsRegression"; string ExtraUsing = "using Microsoft.ML.Trainers.FastTree;"; @@ -11,9 +12,11 @@ string TrainerOptions = @"FastTreeRegressionTrainer.Options { LabelColumnName = nameof(DataPoint.Label), FeatureColumnName = nameof(DataPoint.Features), - // Use L2-norm for early stopping. If the gradient's L2-norm is smaller than - // an auto-computed value, training process will stop. - EarlyStoppingMetric = Microsoft.ML.Trainers.FastTree.EarlyStoppingMetric.L2Norm, + // Use L2-norm for early stopping. If the gradient's L2-norm is + // smaller than an auto-computed value, training process will stop. + EarlyStoppingMetric = + Microsoft.ML.Trainers.FastTree.EarlyStoppingMetric.L2Norm, + // Create a simpler model by penalizing usage of new features. FeatureFirstUsePenalty = 0.1, // Reduce the number of trees to 50. diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/Gam.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/Gam.cs index f58450d9f8..f1c568ce9c 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/Gam.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/Gam.cs @@ -8,37 +8,46 @@ namespace Samples.Dynamic.Trainers.Regression { public static class Gam { - // This example requires installation of additional NuGet package - // Microsoft.ML.FastTree. + // This example requires installation of additional NuGet + // package for Microsoft.ML.FastTree found at + // https://www.nuget.org/packages/Microsoft.ML.FastTree/ public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); // Define the trainer. - var pipeline = mlContext.Regression.Trainers.Gam(labelColumnName: nameof(DataPoint.Label), featureColumnName: nameof(DataPoint.Features)); + var pipeline = mlContext.Regression.Trainers.Gam( + labelColumnName: nameof(DataPoint.Label), + featureColumnName: nameof(DataPoint.Features)); // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(5, seed: 123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data.LoadFromEnumerable( + GenerateRandomDataPoints(5, seed: 123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(transformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data.CreateEnumerable( + transformedTestData, reuseRowObject: false).ToList(); - // Look at 5 predictions for the Label, side by side with the actual Label for comparison. + // Look at 5 predictions for the Label, side by side with the actual + // Label for comparison. foreach (var p in predictions) Console.WriteLine($"Label: {p.Label:F3}, Prediction: {p.Score:F3}"); @@ -60,7 +69,8 @@ public static void Example() // RSquared: 0.99 (closer to 1 is better. The worest case is 0) } - private static IEnumerable GenerateRandomDataPoints(int count, int seed=0) + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) { var random = new Random(seed); for (int i = 0; i < count; i++) @@ -70,12 +80,14 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se { Label = label, // Create random features that are correlated with the label. - Features = Enumerable.Repeat(label, 50).Select(x => x + (float)random.NextDouble()).ToArray() + Features = Enumerable.Repeat(label, 50).Select( + x => x + (float)random.NextDouble()).ToArray() }; } } - // Example with label and 50 feature values. A data set is a collection of such examples. + // Example with label and 50 feature values. A data set is a collection of + // such examples. private class DataPoint { public float Label { get; set; } @@ -95,10 +107,12 @@ private class Prediction // Print some evaluation metrics to regression problems. private static void PrintMetrics(RegressionMetrics metrics) { - Console.WriteLine($"Mean Absolute Error: {metrics.MeanAbsoluteError:F2}"); - Console.WriteLine($"Mean Squared Error: {metrics.MeanSquaredError:F2}"); - Console.WriteLine($"Root Mean Squared Error: {metrics.RootMeanSquaredError:F2}"); - Console.WriteLine($"RSquared: {metrics.RSquared:F2}"); + Console.WriteLine("Mean Absolute Error: " + metrics.MeanAbsoluteError); + Console.WriteLine("Mean Squared Error: " + metrics.MeanSquaredError); + Console.WriteLine( + "Root Mean Squared Error: " + metrics.RootMeanSquaredError); + + Console.WriteLine("RSquared: " + metrics.RSquared); } } } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/Gam.tt b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/Gam.tt index e928c0bcc2..06a0d9bf0b 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/Gam.tt +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/Gam.tt @@ -1,12 +1,16 @@ <#@ include file="RegressionSamplesTemplate.ttinclude"#> <#+ -string ClassHeader = @"// This example requires installation of additional NuGet package - // Microsoft.ML.FastTree. "; +string ClassHeader = @"// This example requires installation of additional NuGet + // package for Microsoft.ML.FastTree found at + // https://www.nuget.org/packages/Microsoft.ML.FastTree/"; string ClassName="Gam"; string ExtraUsing = null; -string Trainer = @"Gam(labelColumnName: nameof(DataPoint.Label), featureColumnName: nameof(DataPoint.Features))"; +string Trainer = @"Gam( + labelColumnName: nameof(DataPoint.Label), + featureColumnName: nameof(DataPoint.Features))"; + string TrainerOptions = null; string ExpectedOutputPerInstance= @"// Expected output: diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/GamAdvanced.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/GamAdvanced.cs index 419ee531ee..98485d0697 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/GamAdvanced.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/GamAdvanced.cs @@ -7,11 +7,13 @@ namespace Samples.Dynamic.Trainers.BinaryClassification { public static class GamAdvanced { - // This example requires installation of additional NuGet package - // Microsoft.ML.FastTree. + // This example requires installation of additional NuGet package for + // Microsoft.ML.FastTree found at + // https://www.nuget.org/packages/Microsoft.ML.FastTree/ public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, // as a catalog of available operations and as the source of randomness. var mlContext = new MLContext(); @@ -27,30 +29,36 @@ public static void Example() var validSet = dataSets.TestSet; // Create a GAM trainer. - // Use a small number of bins for this example. The setting below means for each feature, - // we divide its range into 16 discrete regions for the training process. Note that these - // regions are not evenly spaced, and that the final model may contain fewer bins, as - // neighboring bins with identical values will be combined. In general, we recommend using - // at least the default number of bins, as a small number of bins limits the capacity of - // the model. - var trainer = mlContext.BinaryClassification.Trainers.Gam(maximumBinCountPerFeature: 16); - - // Fit the model using both of training and validation sets. GAM can use a technique called - // pruning to tune the model to the validation set after training to improve generalization. + // Use a small number of bins for this example. The setting below means + // for each feature, we divide its range into 16 discrete regions for + // the training process. Note that these regions are not evenly spaced, + // and that the final model may contain fewer bins, as neighboring bins + // with identical values will be combined. In general, we recommend + // using at least the default number of bins, as a small number of bins + // limits the capacity of the model. + var trainer = mlContext.BinaryClassification.Trainers.Gam( + maximumBinCountPerFeature: 16); + + // Fit the model using both of training and validation sets. GAM can use + // a technique called pruning to tune the model to the validation set + // after training to improve generalization. var model = trainer.Fit(trainSet, validSet); // Extract the model parameters. var gam = model.Model.SubModel; - // Now we can inspect the parameters of the Generalized Additive Model to understand the fit - // and potentially learn about our dataset. - // First, we will look at the bias; the bias represents the average prediction for the training data. + // Now we can inspect the parameters of the Generalized Additive Model + // to understand the fit and potentially learn about our dataset. + // First, we will look at the bias; the bias represents the average + // prediction for the training data. Console.WriteLine($"Average prediction: {gam.Bias:0.00}"); - // Now look at the shape functions that the model has learned. Similar to a linear model, we have - // one response per feature, and they are independent. Unlike a linear model, this response is a - // generic function instead of a line. Because we have included a bias term, each feature response - // represents the deviation from the average prediction as a function of the feature value. + // Now look at the shape functions that the model has learned. Similar + // to a linear model, we have one response per feature, and they are + // independent. Unlike a linear model, this response is a generic + // function instead of a line. Because we have included a bias term, + // each feature response represents the deviation from the average + // prediction as a function of the feature value. for (int i = 0; i < gam.NumberOfShapeFunctions; i++) { // Break a line. @@ -62,11 +70,13 @@ public static void Example() // Get the bin effects; these are the function values for each bin. var binEffects = gam.GetBinEffects(i); - // Now, write the function to the console. The function is a set of bins, and the corresponding - // function values. You can think of GAMs as building a bar-chart or lookup table for each feature. + // Now, write the function to the console. The function is a set of + // bins, and the corresponding function values. You can think of + // GAMs as building a bar-chart or lookup table for each feature. Console.WriteLine($"Feature{i}"); for (int j = 0; j < binUpperBounds.Count; j++) - Console.WriteLine($"x < {binUpperBounds[j]:0.00} => {binEffects[j]:0.000}"); + Console.WriteLine( + $"x < {binUpperBounds[j]:0.00} => {binEffects[j]:0.000}"); } // Expected output: @@ -91,18 +101,23 @@ public static void Example() // x < 0.31 => -0.138 // x < ∞ => -0.188 - // Let's consider this output. To score a given example, we look up the first bin where the inequality - // is satisfied for the feature value. We can look at the whole function to get a sense for how the - // model responds to the variable on a global level. - // The model can be seen to reconstruct the parabolic and step-wise function, shifted with respect to the average - // expected output over the training set. Very few bins are used to model the second feature because the GAM model - // discards unchanged bins to create smaller models. - // One last thing to notice is that these feature functions can be noisy. While we know that Feature1 should be - // symmetric, this is not captured in the model. This is due to noise in the data. Common practice is to use - // resampling methods to estimate a confidence interval at each bin. This will help to determine if the effect is - // real or just sampling noise. See for example: - // Tan, Caruana, Hooker, and Lou. "Distill-and-Compare: Auditing Black-Box Models Using Transparent Model - // Distillation." arXiv:1710.06169." + // Let's consider this output. To score a given example, we look up the + // first bin where the inequality is satisfied for the feature value. + // We can look at the whole function to get a sense for how the model + // responds to the variable on a global level. The model can be seen to + // reconstruct the parabolic and step-wise function, shifted with + // respect to the average expected output over the training set. Very + // few bins are used to model the second feature because the GAM model + // discards unchanged bins to create smaller models. One last thing to + // notice is that these feature functions can be noisy. While we know + // that Feature1 should be symmetric, this is not captured in the model. + // This is due to noise in the data. Common practice is to use + // resampling methods to estimate a confidence interval at each bin. + // This will help to determine if the effect is real or just sampling + // noise. See for example: Tan, Caruana, Hooker, and Lou. + // "Distill-and-Compare: Auditing Black-Box Models Using Transparent + // Model Distillation." + // arXiv:1710.06169." } private class Data @@ -114,13 +129,16 @@ private class Data } /// - /// Creates a dataset, an IEnumerable of Data objects, for a GAM sample. Feature1 is a parabola centered around 0, - /// while Feature2 is a simple piecewise function. + /// Creates a dataset, an IEnumerable of Data objects, for a GAM sample. + /// Feature1 is a parabola centered around 0, while Feature2 is a simple + /// piecewise function. /// /// The number of examples to generate. - /// The seed for the random number generator used to produce data. + /// The seed for the random number generator used to + /// produce data. /// - private static IEnumerable GenerateData(int numExamples = 25000, int seed = 1) + private static IEnumerable GenerateData(int numExamples = 25000, + int seed = 1) { var rng = new Random(seed); float centeredFloat() => (float)(rng.NextDouble() - 0.5); @@ -132,7 +150,8 @@ private static IEnumerable GenerateData(int numExamples = 25000, int seed Features = new float[2] { centeredFloat(), centeredFloat() } }; // Compute the label from the shape functions and add noise. - data.Label = Sigmoid(Parabola(data.Features[0]) + SimplePiecewise(data.Features[1]) + centeredFloat()) > 0.5; + data.Label = Sigmoid(Parabola(data.Features[0]) + SimplePiecewise( + data.Features[1]) + centeredFloat()) > 0.5; yield return data; } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/GamWithOptions.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/GamWithOptions.cs index 004f90fb20..1b69ae8764 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/GamWithOptions.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/GamWithOptions.cs @@ -9,19 +9,22 @@ namespace Samples.Dynamic.Trainers.Regression { public static class GamWithOptions { - // This example requires installation of additional NuGet package - // Microsoft.ML.FastTree. + // This example requires installation of additional NuGet + // package for Microsoft.ML.FastTree found at + // https://www.nuget.org/packages/Microsoft.ML.FastTree/ public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); // Define trainer options. @@ -36,21 +39,26 @@ public static void Example() }; // Define the trainer. - var pipeline = mlContext.Regression.Trainers.Gam(options); + var pipeline = + mlContext.Regression.Trainers.Gam(options); // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(5, seed: 123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data.LoadFromEnumerable( + GenerateRandomDataPoints(5, seed: 123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(transformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data.CreateEnumerable( + transformedTestData, reuseRowObject: false).ToList(); - // Look at 5 predictions for the Label, side by side with the actual Label for comparison. + // Look at 5 predictions for the Label, side by side with the actual + // Label for comparison. foreach (var p in predictions) Console.WriteLine($"Label: {p.Label:F3}, Prediction: {p.Score:F3}"); @@ -72,7 +80,8 @@ public static void Example() // RSquared: 0.98 (closer to 1 is better. The worest case is 0) } - private static IEnumerable GenerateRandomDataPoints(int count, int seed=0) + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) { var random = new Random(seed); for (int i = 0; i < count; i++) @@ -82,12 +91,14 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se { Label = label, // Create random features that are correlated with the label. - Features = Enumerable.Repeat(label, 50).Select(x => x + (float)random.NextDouble()).ToArray() + Features = Enumerable.Repeat(label, 50).Select( + x => x + (float)random.NextDouble()).ToArray() }; } } - // Example with label and 50 feature values. A data set is a collection of such examples. + // Example with label and 50 feature values. A data set is a collection of + // such examples. private class DataPoint { public float Label { get; set; } @@ -107,10 +118,12 @@ private class Prediction // Print some evaluation metrics to regression problems. private static void PrintMetrics(RegressionMetrics metrics) { - Console.WriteLine($"Mean Absolute Error: {metrics.MeanAbsoluteError:F2}"); - Console.WriteLine($"Mean Squared Error: {metrics.MeanSquaredError:F2}"); - Console.WriteLine($"Root Mean Squared Error: {metrics.RootMeanSquaredError:F2}"); - Console.WriteLine($"RSquared: {metrics.RSquared:F2}"); + Console.WriteLine("Mean Absolute Error: " + metrics.MeanAbsoluteError); + Console.WriteLine("Mean Squared Error: " + metrics.MeanSquaredError); + Console.WriteLine( + "Root Mean Squared Error: " + metrics.RootMeanSquaredError); + + Console.WriteLine("RSquared: " + metrics.RSquared); } } } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/GamWithOptions.tt b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/GamWithOptions.tt index cd45b0442b..c15a436f13 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/GamWithOptions.tt +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/GamWithOptions.tt @@ -1,8 +1,9 @@ <#@ include file="RegressionSamplesTemplate.ttinclude"#> <#+ -string ClassHeader = @"// This example requires installation of additional NuGet package - // Microsoft.ML.FastTree. "; +string ClassHeader = @"// This example requires installation of additional NuGet + // package for Microsoft.ML.FastTree found at + // https://www.nuget.org/packages/Microsoft.ML.FastTree/"; string ClassName="GamWithOptions"; string ExtraUsing = "using Microsoft.ML.Trainers.FastTree;"; diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/GamWithOptionsAdvanced.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/GamWithOptionsAdvanced.cs index 167f7aec2e..1f5b546239 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/GamWithOptionsAdvanced.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/GamWithOptionsAdvanced.cs @@ -8,12 +8,14 @@ namespace Samples.Dynamic.Trainers.BinaryClassification { public static class GamWithOptionsAdvanced { - // This example requires installation of additional NuGet package - // Microsoft.ML.FastTree. + // This example requires installation of additional NuGet package for + // Microsoft.ML.FastTree found at + // https://www.nuget.org/packages/Microsoft.ML.FastTree/ public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. var mlContext = new MLContext(); // Create the dataset. @@ -28,14 +30,15 @@ public static void Example() var validSet = dataSets.TestSet; // Create a GAM trainer. - // Use a small number of bins for this example. The setting below means for each feature, - // we divide its range into 16 discrete regions for the training process. Note that these - // regions are not evenly spaced, and that the final model may contain fewer bins, as - // neighboring bins with identical values will be combined. In general, we recommend using - // at least the default number of bins, as a small number of bins limits the capacity of - // the model. - // Also, set the learning rate to half the default to slow down the gradient descent, and - // double the number of iterations to compensate. + // Use a small number of bins for this example. The setting below means + // for each feature, we divide its range into 16 discrete regions for + // the training process. Note that these regions are not evenly spaced, + // and that the final model may contain fewer bins, as neighboring bins + // with identical values will be combined. In general, we recommend + // using at least the default number of bins, as a small number of bins + // limits the capacity of the model. Also, set the learning rate to half + // the default to slow down the gradient descent, and double the number + // of iterations to compensate. var trainer = mlContext.BinaryClassification.Trainers.Gam( new GamBinaryTrainer.Options { @@ -44,22 +47,26 @@ public static void Example() LearningRate = 0.001 }); - // Fit the model using both of training and validation sets. GAM can use a technique called - // pruning to tune the model to the validation set after training to improve generalization. + // Fit the model using both of training and validation sets. GAM can use + // a technique called pruning to tune the model to the validation set + // after training to improve generalization. var model = trainer.Fit(trainSet, validSet); // Extract the model parameters. var gam = model.Model.SubModel; - // Now we can inspect the parameters of the Generalized Additive Model to understand the fit - // and potentially learn about our dataset. - // First, we will look at the bias; the bias represents the average prediction for the training data. + // Now we can inspect the parameters of the Generalized Additive Model + // to understand the fit and potentially learn about our dataset. + // First, we will look at the bias; the bias represents the average + // prediction for the training data. Console.WriteLine($"Average prediction: {gam.Bias:0.00}"); - // Now look at the shape functions that the model has learned. Similar to a linear model, we have - // one response per feature, and they are independent. Unlike a linear model, this response is a - // generic function instead of a line. Because we have included a bias term, each feature response - // represents the deviation from the average prediction as a function of the feature value. + // Now look at the shape functions that the model has learned. Similar + // to a linear model, we have one response per feature, and they are + // independent. Unlike a linear model, this response is a generic + // function instead of a line. Because we have included a bias term, + // each feature response represents the deviation from the average + // prediction as a function of the feature value. for (int i = 0; i < gam.NumberOfShapeFunctions; i++) { // Break a line. @@ -71,11 +78,13 @@ public static void Example() // Get the bin effects; these are the function values for each bin. var binEffects = gam.GetBinEffects(i); - // Now, write the function to the console. The function is a set of bins, and the corresponding - // function values. You can think of GAMs as building a bar-chart or lookup table for each feature. + // Now, write the function to the console. The function is a set of + // bins, and the corresponding function values. You can think of + // GAMs as building a bar-chart or lookup table for each feature. Console.WriteLine($"Feature{i}"); for (int j = 0; j < binUpperBounds.Count; j++) - Console.WriteLine($"x < {binUpperBounds[j]:0.00} => {binEffects[j]:0.000}"); + Console.WriteLine( + $"x < {binUpperBounds[j]:0.00} => {binEffects[j]:0.000}"); } // Expected output: @@ -100,18 +109,23 @@ public static void Example() // x < 0.31 => -0.138 // x < ∞ => -0.188 - // Let's consider this output. To score a given example, we look up the first bin where the inequality - // is satisfied for the feature value. We can look at the whole function to get a sense for how the - // model responds to the variable on a global level. - // The model can be seen to reconstruct the parabolic and step-wise function, shifted with respect to the average - // expected output over the training set. Very few bins are used to model the second feature because the GAM model - // discards unchanged bins to create smaller models. - // One last thing to notice is that these feature functions can be noisy. While we know that Feature1 should be - // symmetric, this is not captured in the model. This is due to noise in the data. Common practice is to use - // resampling methods to estimate a confidence interval at each bin. This will help to determine if the effect is - // real or just sampling noise. See for example: - // Tan, Caruana, Hooker, and Lou. "Distill-and-Compare: Auditing Black-Box Models Using Transparent Model - // Distillation." arXiv:1710.06169." + // Let's consider this output. To score a given example, we look up the + // first bin where the inequality is satisfied for the feature value. + // We can look at the whole function to get a sense for how the model + // responds to the variable on a global level. The model can be seen to + // reconstruct the parabolic and step-wise function, shifted with + // respect to the average expected output over the training set. Very + // few bins are used to model the second feature because the GAM model + // discards unchanged bins to create smaller models. One last thing to + // notice is that these feature functions can be noisy. While we know + // that Feature1 should be symmetric, this is not captured in the model. + // This is due to noise in the data. Common practice is to use + // resampling methods to estimate a confidence interval at each bin. + // This will help to determine if the effect is real or just sampling + // noise. See for example: Tan, Caruana, Hooker, and Lou. + // "Distill-and-Compare: Auditing Black-Box Models Using Transparent + // Model Distillation." + // arXiv:1710.06169." } private class Data @@ -123,13 +137,16 @@ private class Data } /// - /// Creates a dataset, an IEnumerable of Data objects, for a GAM sample. Feature1 is a parabola centered around 0, - /// while Feature2 is a simple piecewise function. + /// Creates a dataset, an IEnumerable of Data objects, for a GAM sample. + /// Feature1 is a parabola centered around 0, while Feature2 is a simple + /// piecewise function. /// /// The number of examples to generate. - /// The seed for the random number generator used to produce data. + /// The seed for the random number generator used to + /// produce data. /// - private static IEnumerable GenerateData(int numExamples = 25000, int seed = 1) + private static IEnumerable GenerateData(int numExamples = 25000, + int seed = 1) { var rng = new Random(seed); float centeredFloat() => (float)(rng.NextDouble() - 0.5); @@ -141,7 +158,8 @@ private static IEnumerable GenerateData(int numExamples = 25000, int seed Features = new float[2] { centeredFloat(), centeredFloat() } }; // Compute the label from the shape functions and add noise. - data.Label = Sigmoid(Parabola(data.Features[0]) + SimplePiecewise(data.Features[1]) + centeredFloat()) > 0.5; + data.Label = Sigmoid(Parabola(data.Features[0]) + SimplePiecewise( + data.Features[1]) + centeredFloat()) > 0.5; yield return data; } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/LbfgsPoissonRegression.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/LbfgsPoissonRegression.cs index 854cddc537..bf6978f300 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/LbfgsPoissonRegression.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/LbfgsPoissonRegression.cs @@ -10,33 +10,42 @@ public static class LbfgsPoissonRegression { public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); // Define the trainer. - var pipeline = mlContext.Regression.Trainers.LbfgsPoissonRegression(labelColumnName: nameof(DataPoint.Label), featureColumnName: nameof(DataPoint.Features)); + var pipeline = mlContext.Regression.Trainers. + LbfgsPoissonRegression( + labelColumnName: nameof(DataPoint.Label), + featureColumnName: nameof(DataPoint.Features)); // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(5, seed: 123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data.LoadFromEnumerable( + GenerateRandomDataPoints(5, seed: 123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(transformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data.CreateEnumerable( + transformedTestData, reuseRowObject: false).ToList(); - // Look at 5 predictions for the Label, side by side with the actual Label for comparison. + // Look at 5 predictions for the Label, side by side with the actual + // Label for comparison. foreach (var p in predictions) Console.WriteLine($"Label: {p.Label:F3}, Prediction: {p.Score:F3}"); @@ -58,7 +67,8 @@ public static void Example() // RSquared: 0.93 (closer to 1 is better. The worest case is 0) } - private static IEnumerable GenerateRandomDataPoints(int count, int seed=0) + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) { var random = new Random(seed); for (int i = 0; i < count; i++) @@ -68,12 +78,14 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se { Label = label, // Create random features that are correlated with the label. - Features = Enumerable.Repeat(label, 50).Select(x => x + (float)random.NextDouble()).ToArray() + Features = Enumerable.Repeat(label, 50).Select( + x => x + (float)random.NextDouble()).ToArray() }; } } - // Example with label and 50 feature values. A data set is a collection of such examples. + // Example with label and 50 feature values. A data set is a collection of + // such examples. private class DataPoint { public float Label { get; set; } @@ -93,10 +105,12 @@ private class Prediction // Print some evaluation metrics to regression problems. private static void PrintMetrics(RegressionMetrics metrics) { - Console.WriteLine($"Mean Absolute Error: {metrics.MeanAbsoluteError:F2}"); - Console.WriteLine($"Mean Squared Error: {metrics.MeanSquaredError:F2}"); - Console.WriteLine($"Root Mean Squared Error: {metrics.RootMeanSquaredError:F2}"); - Console.WriteLine($"RSquared: {metrics.RSquared:F2}"); + Console.WriteLine("Mean Absolute Error: " + metrics.MeanAbsoluteError); + Console.WriteLine("Mean Squared Error: " + metrics.MeanSquaredError); + Console.WriteLine( + "Root Mean Squared Error: " + metrics.RootMeanSquaredError); + + Console.WriteLine("RSquared: " + metrics.RSquared); } } } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/LbfgsPoissonRegression.tt b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/LbfgsPoissonRegression.tt index 08d6d22969..2c410cf954 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/LbfgsPoissonRegression.tt +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/LbfgsPoissonRegression.tt @@ -4,7 +4,10 @@ string ClassHeader = null; string ClassName="LbfgsPoissonRegression"; string ExtraUsing = null; -string Trainer = "LbfgsPoissonRegression(labelColumnName: nameof(DataPoint.Label), featureColumnName: nameof(DataPoint.Features))"; +string Trainer = @" + LbfgsPoissonRegression( + labelColumnName: nameof(DataPoint.Label), + featureColumnName: nameof(DataPoint.Features))"; string TrainerOptions = null; string ExpectedOutputPerInstance= @"// Expected output: diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/LbfgsPoissonRegressionWithOptions.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/LbfgsPoissonRegressionWithOptions.cs index b649699f9c..3c5a7da604 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/LbfgsPoissonRegressionWithOptions.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/LbfgsPoissonRegressionWithOptions.cs @@ -11,15 +11,17 @@ public static class LbfgsPoissonRegressionWithOptions { public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); // Define trainer options. @@ -27,30 +29,37 @@ public static void Example() { LabelColumnName = nameof(DataPoint.Label), FeatureColumnName = nameof(DataPoint.Features), - // Reduce optimization tolerance to speed up training at the cost of accuracy. + // Reduce optimization tolerance to speed up training at the cost of + // accuracy. OptimizationTolerance = 1e-4f, - // Decrease history size to speed up training at the cost of accuracy. + // Decrease history size to speed up training at the cost of + // accuracy. HistorySize = 30, // Specify scale for initial weights. InitialWeightsDiameter = 0.2f }; // Define the trainer. - var pipeline = mlContext.Regression.Trainers.LbfgsPoissonRegression(options); + var pipeline = + mlContext.Regression.Trainers.LbfgsPoissonRegression(options); // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(5, seed: 123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data.LoadFromEnumerable( + GenerateRandomDataPoints(5, seed: 123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(transformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data.CreateEnumerable( + transformedTestData, reuseRowObject: false).ToList(); - // Look at 5 predictions for the Label, side by side with the actual Label for comparison. + // Look at 5 predictions for the Label, side by side with the actual + // Label for comparison. foreach (var p in predictions) Console.WriteLine($"Label: {p.Label:F3}, Prediction: {p.Score:F3}"); @@ -72,7 +81,8 @@ public static void Example() // RSquared: 0.89 (closer to 1 is better. The worest case is 0) } - private static IEnumerable GenerateRandomDataPoints(int count, int seed=0) + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) { var random = new Random(seed); for (int i = 0; i < count; i++) @@ -82,12 +92,14 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se { Label = label, // Create random features that are correlated with the label. - Features = Enumerable.Repeat(label, 50).Select(x => x + (float)random.NextDouble()).ToArray() + Features = Enumerable.Repeat(label, 50).Select( + x => x + (float)random.NextDouble()).ToArray() }; } } - // Example with label and 50 feature values. A data set is a collection of such examples. + // Example with label and 50 feature values. A data set is a collection of + // such examples. private class DataPoint { public float Label { get; set; } @@ -107,10 +119,12 @@ private class Prediction // Print some evaluation metrics to regression problems. private static void PrintMetrics(RegressionMetrics metrics) { - Console.WriteLine($"Mean Absolute Error: {metrics.MeanAbsoluteError:F2}"); - Console.WriteLine($"Mean Squared Error: {metrics.MeanSquaredError:F2}"); - Console.WriteLine($"Root Mean Squared Error: {metrics.RootMeanSquaredError:F2}"); - Console.WriteLine($"RSquared: {metrics.RSquared:F2}"); + Console.WriteLine("Mean Absolute Error: " + metrics.MeanAbsoluteError); + Console.WriteLine("Mean Squared Error: " + metrics.MeanSquaredError); + Console.WriteLine( + "Root Mean Squared Error: " + metrics.RootMeanSquaredError); + + Console.WriteLine("RSquared: " + metrics.RSquared); } } } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/LbfgsPoissonRegressionWithOptions.tt b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/LbfgsPoissonRegressionWithOptions.tt index f1513d9868..bbe4dbaefa 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/LbfgsPoissonRegressionWithOptions.tt +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/LbfgsPoissonRegressionWithOptions.tt @@ -9,9 +9,11 @@ string TrainerOptions = @"LbfgsPoissonRegressionTrainer.Options { LabelColumnName = nameof(DataPoint.Label), FeatureColumnName = nameof(DataPoint.Features), - // Reduce optimization tolerance to speed up training at the cost of accuracy. + // Reduce optimization tolerance to speed up training at the cost of + // accuracy. OptimizationTolerance = 1e-4f, - // Decrease history size to speed up training at the cost of accuracy. + // Decrease history size to speed up training at the cost of + // accuracy. HistorySize = 30, // Specify scale for initial weights. InitialWeightsDiameter = 0.2f diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/LightGbm.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/LightGbm.cs index 4579280109..4502682f26 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/LightGbm.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/LightGbm.cs @@ -8,37 +8,47 @@ namespace Samples.Dynamic.Trainers.Regression { public static class LightGbm { - // This example requires installation of additional NuGet package - // Microsoft.ML.LightGBM. + // This example requires installation of additional NuGet + // package for Microsoft.ML.LightGBM + // at https://www.nuget.org/packages/Microsoft.ML.LightGbm/ public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); // Define the trainer. - var pipeline = mlContext.Regression.Trainers.LightGbm(labelColumnName: nameof(DataPoint.Label), featureColumnName: nameof(DataPoint.Features)); + var pipeline = mlContext.Regression.Trainers. + LightGbm( + labelColumnName: nameof(DataPoint.Label), + featureColumnName: nameof(DataPoint.Features)); // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(5, seed: 123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data.LoadFromEnumerable( + GenerateRandomDataPoints(5, seed: 123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(transformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data.CreateEnumerable( + transformedTestData, reuseRowObject: false).ToList(); - // Look at 5 predictions for the Label, side by side with the actual Label for comparison. + // Look at 5 predictions for the Label, side by side with the actual + // Label for comparison. foreach (var p in predictions) Console.WriteLine($"Label: {p.Label:F3}, Prediction: {p.Score:F3}"); @@ -60,7 +70,8 @@ public static void Example() // RSquared: 0.89 (closer to 1 is better. The worest case is 0) } - private static IEnumerable GenerateRandomDataPoints(int count, int seed=0) + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) { var random = new Random(seed); for (int i = 0; i < count; i++) @@ -70,12 +81,14 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se { Label = label, // Create random features that are correlated with the label. - Features = Enumerable.Repeat(label, 50).Select(x => x + (float)random.NextDouble()).ToArray() + Features = Enumerable.Repeat(label, 50).Select( + x => x + (float)random.NextDouble()).ToArray() }; } } - // Example with label and 50 feature values. A data set is a collection of such examples. + // Example with label and 50 feature values. A data set is a collection of + // such examples. private class DataPoint { public float Label { get; set; } @@ -95,10 +108,12 @@ private class Prediction // Print some evaluation metrics to regression problems. private static void PrintMetrics(RegressionMetrics metrics) { - Console.WriteLine($"Mean Absolute Error: {metrics.MeanAbsoluteError:F2}"); - Console.WriteLine($"Mean Squared Error: {metrics.MeanSquaredError:F2}"); - Console.WriteLine($"Root Mean Squared Error: {metrics.RootMeanSquaredError:F2}"); - Console.WriteLine($"RSquared: {metrics.RSquared:F2}"); + Console.WriteLine("Mean Absolute Error: " + metrics.MeanAbsoluteError); + Console.WriteLine("Mean Squared Error: " + metrics.MeanSquaredError); + Console.WriteLine( + "Root Mean Squared Error: " + metrics.RootMeanSquaredError); + + Console.WriteLine("RSquared: " + metrics.RSquared); } } } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/LightGbm.tt b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/LightGbm.tt index cb7f481342..c3af40c187 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/LightGbm.tt +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/LightGbm.tt @@ -1,12 +1,16 @@ <#@ include file="RegressionSamplesTemplate.ttinclude"#> <#+ -string ClassHeader = @"// This example requires installation of additional NuGet package - // Microsoft.ML.LightGBM. "; +string ClassHeader = @"// This example requires installation of additional NuGet + // package for Microsoft.ML.LightGBM + // at https://www.nuget.org/packages/Microsoft.ML.LightGbm/"; string ClassName="LightGbm"; string ExtraUsing = null; -string Trainer = @"LightGbm(labelColumnName: nameof(DataPoint.Label), featureColumnName: nameof(DataPoint.Features))"; +string Trainer = @" + LightGbm( + labelColumnName: nameof(DataPoint.Label), + featureColumnName: nameof(DataPoint.Features))"; string TrainerOptions = null; string ExpectedOutputPerInstance= @"// Expected output: diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/LightGbmAdvanced.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/LightGbmAdvanced.cs index 363d746305..feba7ca9d3 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/LightGbmAdvanced.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/LightGbmAdvanced.cs @@ -7,15 +7,18 @@ namespace Samples.Dynamic.Trainers.Regression { class LightGbmAdvanced { - // This example requires installation of additional nuget package Microsoft.ML.LightGBM. + // This example requires installation of additional NuGet package + // for Microsoft.ML.LightGBM + // at https://www.nuget.org/packages/Microsoft.ML.LightGbm/ public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(); // Download and load the housing dataset into an IDataView. - var dataView = Microsoft.ML.SamplesUtils.DatasetUtils.LoadHousingRegressionDataset(mlContext); + var dataView = Microsoft.ML.SamplesUtils.DatasetUtils + .LoadHousingRegressionDataset(mlContext); //////////////////// Data Preview //////////////////// /// Only 6 columns are displayed here. @@ -33,26 +36,32 @@ public static void Example() .Select(column => column.Name) // Get the column names .Where(name => name != labelName) // Drop the Label .ToArray(); - var pipeline = mlContext.Transforms.Concatenate("Features", featureNames) - .Append(mlContext.Regression.Trainers.LightGbm( - labelColumnName: labelName, - numberOfLeaves: 4, - minimumExampleCountPerLeaf: 6, - learningRate: 0.001)); + var pipeline = mlContext.Transforms.Concatenate("Features", + featureNames) + .Append(mlContext.Regression.Trainers.LightGbm( + labelColumnName: labelName, + numberOfLeaves: 4, + minimumExampleCountPerLeaf: 6, + learningRate: 0.001)); // Fit this pipeline to the training data. var model = pipeline.Fit(split.TrainSet); - // Get the feature importance based on the information gain used during training. + // Get the feature importance based on the information gain used during + // training. VBuffer weights = default; model.LastTransformer.Model.GetFeatureWeights(ref weights); var weightsValues = weights.DenseValues().ToArray(); - Console.WriteLine($"weight 0 - {weightsValues[0]}"); // CrimesPerCapita (weight 0) = 0.1898361 - Console.WriteLine($"weight 5 - {weightsValues[5]}"); // RoomsPerDwelling (weight 5) = 1 + Console.WriteLine($"weight 0 - {weightsValues[0]}"); + // CrimesPerCapita (weight 0) = 0.1898361 + Console.WriteLine($"weight 5 - {weightsValues[5]}"); + // RoomsPerDwelling (weight 5) = 1 // Evaluate how the model is doing on the test data. var dataWithPredictions = model.Transform(split.TestSet); - var metrics = mlContext.Regression.Evaluate(dataWithPredictions, labelColumnName: labelName); + var metrics = mlContext.Regression.Evaluate( + dataWithPredictions, + labelColumnName: labelName); PrintMetrics(metrics); // Expected output @@ -63,12 +72,14 @@ public static void Example() // RSquared: 0.08 } - public static void PrintMetrics(RegressionMetrics metrics) + private static void PrintMetrics(RegressionMetrics metrics) { - Console.WriteLine($"Mean Absolute Error: {metrics.MeanAbsoluteError:F2}"); - Console.WriteLine($"Mean Squared Error: {metrics.MeanSquaredError:F2}"); - Console.WriteLine($"Root Mean Squared Error: {metrics.RootMeanSquaredError:F2}"); - Console.WriteLine($"RSquared: {metrics.RSquared:F2}"); + Console.WriteLine("Mean Absolute Error: " + metrics.MeanAbsoluteError); + Console.WriteLine("Mean Squared Error: " + metrics.MeanSquaredError); + Console.WriteLine( + "Root Mean Squared Error: " + metrics.RootMeanSquaredError); + + Console.WriteLine("RSquared: " + metrics.RSquared); } } } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/LightGbmWithOptions.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/LightGbmWithOptions.cs index 24caa23360..fb9b783631 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/LightGbmWithOptions.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/LightGbmWithOptions.cs @@ -9,19 +9,22 @@ namespace Samples.Dynamic.Trainers.Regression { public static class LightGbmWithOptions { - // This example requires installation of additional NuGet package - // Microsoft.ML.LightGBM. + // This example requires installation of additional NuGet + // package for Microsoft.ML.LightGBM + // at https://www.nuget.org/packages/Microsoft.ML.LightGbm/ public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); // Define trainer options. @@ -34,7 +37,8 @@ public static void Example() // Each leaf contains at least this number of training data points. MinimumExampleCountPerLeaf = 6, // The step size per update. Using a large value might reduce the - // training time but also increase the algorithm's numerical stability. + // training time but also increase the algorithm's numerical + // stability. LearningRate = 0.001, Booster = new Microsoft.ML.Trainers.LightGbm.GossBooster.Options() { @@ -44,21 +48,26 @@ public static void Example() }; // Define the trainer. - var pipeline = mlContext.Regression.Trainers.LightGbm(options); + var pipeline = + mlContext.Regression.Trainers.LightGbm(options); // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(5, seed: 123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data.LoadFromEnumerable( + GenerateRandomDataPoints(5, seed: 123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(transformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data.CreateEnumerable( + transformedTestData, reuseRowObject: false).ToList(); - // Look at 5 predictions for the Label, side by side with the actual Label for comparison. + // Look at 5 predictions for the Label, side by side with the actual + // Label for comparison. foreach (var p in predictions) Console.WriteLine($"Label: {p.Label:F3}, Prediction: {p.Score:F3}"); @@ -80,7 +89,8 @@ public static void Example() // RSquared: 0.97 (closer to 1 is better. The worest case is 0) } - private static IEnumerable GenerateRandomDataPoints(int count, int seed=0) + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) { var random = new Random(seed); for (int i = 0; i < count; i++) @@ -90,12 +100,14 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se { Label = label, // Create random features that are correlated with the label. - Features = Enumerable.Repeat(label, 50).Select(x => x + (float)random.NextDouble()).ToArray() + Features = Enumerable.Repeat(label, 50).Select( + x => x + (float)random.NextDouble()).ToArray() }; } } - // Example with label and 50 feature values. A data set is a collection of such examples. + // Example with label and 50 feature values. A data set is a collection of + // such examples. private class DataPoint { public float Label { get; set; } @@ -115,10 +127,12 @@ private class Prediction // Print some evaluation metrics to regression problems. private static void PrintMetrics(RegressionMetrics metrics) { - Console.WriteLine($"Mean Absolute Error: {metrics.MeanAbsoluteError:F2}"); - Console.WriteLine($"Mean Squared Error: {metrics.MeanSquaredError:F2}"); - Console.WriteLine($"Root Mean Squared Error: {metrics.RootMeanSquaredError:F2}"); - Console.WriteLine($"RSquared: {metrics.RSquared:F2}"); + Console.WriteLine("Mean Absolute Error: " + metrics.MeanAbsoluteError); + Console.WriteLine("Mean Squared Error: " + metrics.MeanSquaredError); + Console.WriteLine( + "Root Mean Squared Error: " + metrics.RootMeanSquaredError); + + Console.WriteLine("RSquared: " + metrics.RSquared); } } } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/LightGbmWithOptions.tt b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/LightGbmWithOptions.tt index 22b6522baf..b1105186db 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/LightGbmWithOptions.tt +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/LightGbmWithOptions.tt @@ -1,8 +1,9 @@ <#@ include file="RegressionSamplesTemplate.ttinclude"#> <#+ -string ClassHeader = @"// This example requires installation of additional NuGet package - // Microsoft.ML.LightGBM. "; +string ClassHeader = @"// This example requires installation of additional NuGet + // package for Microsoft.ML.LightGBM + // at https://www.nuget.org/packages/Microsoft.ML.LightGbm/"; string ClassName="LightGbmWithOptions"; string ExtraUsing = "using Microsoft.ML.Trainers.LightGbm;"; @@ -16,7 +17,8 @@ string TrainerOptions = @"LightGbmRegressionTrainer.Options // Each leaf contains at least this number of training data points. MinimumExampleCountPerLeaf = 6, // The step size per update. Using a large value might reduce the - // training time but also increase the algorithm's numerical stability. + // training time but also increase the algorithm's numerical + // stability. LearningRate = 0.001, Booster = new Microsoft.ML.Trainers.LightGbm.GossBooster.Options() { diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/LightGbmWithOptionsAdvanced.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/LightGbmWithOptionsAdvanced.cs index acc48bbdf7..ea98c44670 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/LightGbmWithOptionsAdvanced.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/LightGbmWithOptionsAdvanced.cs @@ -8,11 +8,13 @@ namespace Samples.Dynamic.Trainers.Regression { class LightGbmWithOptionsAdvanced { - // This example requires installation of additional nuget package Microsoft.ML.LightGBM. + // This example requires installation of additional NuGet package + // for Microsoft.ML.LightGBM + // at https://www.nuget.org/packages/Microsoft.ML.LightGbm/ public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(); // Download and load the housing dataset into an IDataView. @@ -35,33 +37,40 @@ public static void Example() .Select(column => column.Name) // Get the column names .Where(name => name != labelName) // Drop the Label .ToArray(); - var pipeline = mlContext.Transforms.Concatenate("Features", featureNames) - .Append(mlContext.Regression.Trainers.LightGbm(new LightGbmRegressionTrainer.Options - { - LabelColumnName = labelName, - NumberOfLeaves = 4, - MinimumExampleCountPerLeaf = 6, - LearningRate = 0.001, - Booster = new GossBooster.Options() - { - TopRate = 0.3, - OtherRate = 0.2 - } - })); + var pipeline = mlContext.Transforms.Concatenate( + "Features", featureNames) + .Append(mlContext.Regression.Trainers.LightGbm( + new LightGbmRegressionTrainer.Options + { + LabelColumnName = labelName, + NumberOfLeaves = 4, + MinimumExampleCountPerLeaf = 6, + LearningRate = 0.001, + Booster = new GossBooster.Options() + { + TopRate = 0.3, + OtherRate = 0.2 + } + })); // Fit this pipeline to the training data. var model = pipeline.Fit(split.TrainSet); - // Get the feature importance based on the information gain used during training. + // Get the feature importance based on the information gain used during + // training. VBuffer weights = default; model.LastTransformer.Model.GetFeatureWeights(ref weights); var weightsValues = weights.DenseValues().ToArray(); - Console.WriteLine($"weight 0 - {weightsValues[0]}"); // CrimesPerCapita (weight 0) = 0.1898361 - Console.WriteLine($"weight 5 - {weightsValues[5]}"); // RoomsPerDwelling (weight 5) = 1 + Console.WriteLine($"weight 0 - {weightsValues[0]}"); + // CrimesPerCapita (weight 0) = 0.1898361 + Console.WriteLine($"weight 5 - {weightsValues[5]}"); + // RoomsPerDwelling (weight 5) = 1 // Evaluate how the model is doing on the test data. var dataWithPredictions = model.Transform(split.TestSet); - var metrics = mlContext.Regression.Evaluate(dataWithPredictions, labelColumnName: labelName); + var metrics = mlContext.Regression.Evaluate( + dataWithPredictions, + labelColumnName: labelName); PrintMetrics(metrics); // Expected output @@ -74,10 +83,12 @@ public static void Example() public static void PrintMetrics(RegressionMetrics metrics) { - Console.WriteLine($"Mean Absolute Error: {metrics.MeanAbsoluteError:F2}"); - Console.WriteLine($"Mean Squared Error: {metrics.MeanSquaredError:F2}"); - Console.WriteLine($"Root Mean Squared Error: {metrics.RootMeanSquaredError:F2}"); - Console.WriteLine($"RSquared: {metrics.RSquared:F2}"); + Console.WriteLine("Mean Absolute Error: " + metrics.MeanAbsoluteError); + Console.WriteLine("Mean Squared Error: " + metrics.MeanSquaredError); + Console.WriteLine( + "Root Mean Squared Error: " + metrics.RootMeanSquaredError); + + Console.WriteLine("RSquared: " + metrics.RSquared); } } } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/OnlineGradientDescent.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/OnlineGradientDescent.cs index 3914d43c03..e02d4c626f 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/OnlineGradientDescent.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/OnlineGradientDescent.cs @@ -10,37 +10,46 @@ public static class OnlineGradientDescent { public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); // Define the trainer. - var pipeline = mlContext.Regression.Trainers.OnlineGradientDescent(labelColumnName: nameof(DataPoint.Label), featureColumnName: nameof(DataPoint.Features)); + var pipeline = mlContext.Regression.Trainers.OnlineGradientDescent( + labelColumnName: nameof(DataPoint.Label), + featureColumnName: nameof(DataPoint.Features)); // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(5, seed: 123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data.LoadFromEnumerable( + GenerateRandomDataPoints(5, seed: 123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(transformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data.CreateEnumerable( + transformedTestData, reuseRowObject: false).ToList(); - // Look at 5 predictions for the Label, side by side with the actual Label for comparison. + // Look at 5 predictions for the Label, side by side with the actual + // Label for comparison. foreach (var p in predictions) Console.WriteLine($"Label: {p.Label:F3}, Prediction: {p.Score:F3}"); - // This trainer is not numerically stable. Please see issue #2425. + // This trainer is not numerically stable. + // Please see issue #2425. // Evaluate the overall metrics var metrics = mlContext.Regression.Evaluate(transformedTestData); @@ -49,7 +58,8 @@ public static void Example() } - private static IEnumerable GenerateRandomDataPoints(int count, int seed=0) + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) { var random = new Random(seed); for (int i = 0; i < count; i++) @@ -59,12 +69,14 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se { Label = label, // Create random features that are correlated with the label. - Features = Enumerable.Repeat(label, 50).Select(x => x + (float)random.NextDouble()).ToArray() + Features = Enumerable.Repeat(label, 50).Select( + x => x + (float)random.NextDouble()).ToArray() }; } } - // Example with label and 50 feature values. A data set is a collection of such examples. + // Example with label and 50 feature values. A data set is a collection of + // such examples. private class DataPoint { public float Label { get; set; } @@ -84,10 +96,12 @@ private class Prediction // Print some evaluation metrics to regression problems. private static void PrintMetrics(RegressionMetrics metrics) { - Console.WriteLine($"Mean Absolute Error: {metrics.MeanAbsoluteError:F2}"); - Console.WriteLine($"Mean Squared Error: {metrics.MeanSquaredError:F2}"); - Console.WriteLine($"Root Mean Squared Error: {metrics.RootMeanSquaredError:F2}"); - Console.WriteLine($"RSquared: {metrics.RSquared:F2}"); + Console.WriteLine("Mean Absolute Error: " + metrics.MeanAbsoluteError); + Console.WriteLine("Mean Squared Error: " + metrics.MeanSquaredError); + Console.WriteLine( + "Root Mean Squared Error: " + metrics.RootMeanSquaredError); + + Console.WriteLine("RSquared: " + metrics.RSquared); } } } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/OnlineGradientDescent.tt b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/OnlineGradientDescent.tt index 783f535261..e9579e9121 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/OnlineGradientDescent.tt +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/OnlineGradientDescent.tt @@ -4,9 +4,13 @@ string ClassHeader = null; string ClassName="OnlineGradientDescent"; string ExtraUsing = null; -string Trainer = "OnlineGradientDescent(labelColumnName: nameof(DataPoint.Label), featureColumnName: nameof(DataPoint.Features))"; +string Trainer = @"OnlineGradientDescent( + labelColumnName: nameof(DataPoint.Label), + featureColumnName: nameof(DataPoint.Features))"; string TrainerOptions = null; -string ExpectedOutputPerInstance= @"// This trainer is not numerically stable. Please see issue #2425."; +string ExpectedOutputPerInstance= @"// This trainer is not numerically stable. + // Please see issue #2425."; + string ExpectedOutput = @""; #> \ No newline at end of file diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/OnlineGradientDescentWithOptions.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/OnlineGradientDescentWithOptions.cs index 36e3d187ca..59a7d1fdbc 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/OnlineGradientDescentWithOptions.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/OnlineGradientDescentWithOptions.cs @@ -11,15 +11,17 @@ public static class OnlineGradientDescentWithOptions { public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); // Define trainer options. @@ -38,34 +40,42 @@ public static void Example() }; // Define the trainer. - var pipeline = mlContext.Regression.Trainers.OnlineGradientDescent(options); + var pipeline = + mlContext.Regression.Trainers.OnlineGradientDescent(options); // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(5, seed: 123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data.LoadFromEnumerable( + GenerateRandomDataPoints(5, seed: 123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(transformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data.CreateEnumerable( + transformedTestData, reuseRowObject: false).ToList(); - // Look at 5 predictions for the Label, side by side with the actual Label for comparison. + // Look at 5 predictions for the Label, side by side with the actual + // Label for comparison. foreach (var p in predictions) Console.WriteLine($"Label: {p.Label:F3}, Prediction: {p.Score:F3}"); - // This trainer is not numerically stable. Please see issue #2425. + // This trainer is not numerically stable. + // Please see issue #2425. // Evaluate the overall metrics var metrics = mlContext.Regression.Evaluate(transformedTestData); PrintMetrics(metrics); - // This trainer is not numerically stable. Please see issue #2425. + // This trainer is not numerically stable. Please see + // issue #2425. } - private static IEnumerable GenerateRandomDataPoints(int count, int seed=0) + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) { var random = new Random(seed); for (int i = 0; i < count; i++) @@ -75,12 +85,14 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se { Label = label, // Create random features that are correlated with the label. - Features = Enumerable.Repeat(label, 50).Select(x => x + (float)random.NextDouble()).ToArray() + Features = Enumerable.Repeat(label, 50).Select( + x => x + (float)random.NextDouble()).ToArray() }; } } - // Example with label and 50 feature values. A data set is a collection of such examples. + // Example with label and 50 feature values. A data set is a collection of + // such examples. private class DataPoint { public float Label { get; set; } @@ -100,10 +112,12 @@ private class Prediction // Print some evaluation metrics to regression problems. private static void PrintMetrics(RegressionMetrics metrics) { - Console.WriteLine($"Mean Absolute Error: {metrics.MeanAbsoluteError:F2}"); - Console.WriteLine($"Mean Squared Error: {metrics.MeanSquaredError:F2}"); - Console.WriteLine($"Root Mean Squared Error: {metrics.RootMeanSquaredError:F2}"); - Console.WriteLine($"RSquared: {metrics.RSquared:F2}"); + Console.WriteLine("Mean Absolute Error: " + metrics.MeanAbsoluteError); + Console.WriteLine("Mean Squared Error: " + metrics.MeanSquaredError); + Console.WriteLine( + "Root Mean Squared Error: " + metrics.RootMeanSquaredError); + + Console.WriteLine("RSquared: " + metrics.RSquared); } } } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/OnlineGradientDescentWithOptions.tt b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/OnlineGradientDescentWithOptions.tt index 1616bf8fcb..c794a418a2 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/OnlineGradientDescentWithOptions.tt +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/OnlineGradientDescentWithOptions.tt @@ -19,6 +19,8 @@ string TrainerOptions = @" OnlineGradientDescentTrainer.Options InitialWeightsDiameter = 0.2f }"; -string ExpectedOutputPerInstance= @"// This trainer is not numerically stable. Please see issue #2425."; -string ExpectedOutput = @"// This trainer is not numerically stable. Please see issue #2425."; +string ExpectedOutputPerInstance= @"// This trainer is not numerically stable. + // Please see issue #2425."; +string ExpectedOutput = @"// This trainer is not numerically stable. Please see + // issue #2425."; #> \ No newline at end of file diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/OrdinaryLeastSquares.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/OrdinaryLeastSquares.cs index 31ca5cfd9d..1b50d80443 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/OrdinaryLeastSquares.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/OrdinaryLeastSquares.cs @@ -10,33 +10,41 @@ public static class Ols { public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); // Define the trainer. - var pipeline = mlContext.Regression.Trainers.Ols(labelColumnName: nameof(DataPoint.Label), featureColumnName: nameof(DataPoint.Features)); + var pipeline = mlContext.Regression.Trainers.Ols( + labelColumnName: nameof(DataPoint.Label), + featureColumnName: nameof(DataPoint.Features)); // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(5, seed: 123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data.LoadFromEnumerable( + GenerateRandomDataPoints(5, seed: 123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(transformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data.CreateEnumerable( + transformedTestData, reuseRowObject: false).ToList(); - // Look at 5 predictions for the Label, side by side with the actual Label for comparison. + // Look at 5 predictions for the Label, side by side with the actual + // Label for comparison. foreach (var p in predictions) Console.WriteLine($"Label: {p.Label:F3}, Prediction: {p.Score:F3}"); @@ -58,7 +66,8 @@ public static void Example() // RSquared: 0.97 (closer to 1 is better. The worest case is 0) } - private static IEnumerable GenerateRandomDataPoints(int count, int seed=0) + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) { var random = new Random(seed); for (int i = 0; i < count; i++) @@ -68,12 +77,14 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se { Label = label, // Create random features that are correlated with the label. - Features = Enumerable.Repeat(label, 50).Select(x => x + (float)random.NextDouble()).ToArray() + Features = Enumerable.Repeat(label, 50).Select( + x => x + (float)random.NextDouble()).ToArray() }; } } - // Example with label and 50 feature values. A data set is a collection of such examples. + // Example with label and 50 feature values. A data set is a collection of + // such examples. private class DataPoint { public float Label { get; set; } @@ -93,10 +104,12 @@ private class Prediction // Print some evaluation metrics to regression problems. private static void PrintMetrics(RegressionMetrics metrics) { - Console.WriteLine($"Mean Absolute Error: {metrics.MeanAbsoluteError:F2}"); - Console.WriteLine($"Mean Squared Error: {metrics.MeanSquaredError:F2}"); - Console.WriteLine($"Root Mean Squared Error: {metrics.RootMeanSquaredError:F2}"); - Console.WriteLine($"RSquared: {metrics.RSquared:F2}"); + Console.WriteLine("Mean Absolute Error: " + metrics.MeanAbsoluteError); + Console.WriteLine("Mean Squared Error: " + metrics.MeanSquaredError); + Console.WriteLine( + "Root Mean Squared Error: " + metrics.RootMeanSquaredError); + + Console.WriteLine("RSquared: " + metrics.RSquared); } } } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/OrdinaryLeastSquares.tt b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/OrdinaryLeastSquares.tt index b2aed0bae7..909ef50ba0 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/OrdinaryLeastSquares.tt +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/OrdinaryLeastSquares.tt @@ -4,7 +4,9 @@ string ClassHeader = null; string ClassName="Ols"; string ExtraUsing = null; -string Trainer = "Ols(labelColumnName: nameof(DataPoint.Label), featureColumnName: nameof(DataPoint.Features))"; +string Trainer = @"Ols( + labelColumnName: nameof(DataPoint.Label), + featureColumnName: nameof(DataPoint.Features))"; string TrainerOptions = null; string ExpectedOutputPerInstance= @"// Expected output: diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/OrdinaryLeastSquaresAdvanced.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/OrdinaryLeastSquaresAdvanced.cs index c5c3b2c097..dd09f21fef 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/OrdinaryLeastSquaresAdvanced.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/OrdinaryLeastSquaresAdvanced.cs @@ -7,23 +7,29 @@ namespace Samples.Dynamic.Trainers.Regression { public static class OrdinaryLeastSquaresAdvanced { - // This example requires installation of additional nuget package Microsoft.ML.Mkl.Components. - // In this examples we will use the housing price dataset. The goal is to predict median home value. - // For more details about this dataset, please see https://archive.ics.uci.edu/ml/machine-learning-databases/housing/ + // This example requires installation of additional nuget package + // for Microsoft.ML.Mkl.Components at + // "https://www.nuget.org/packages/Microsoft.ML.Mkl.Components/" + // In this examples we will use the housing price dataset. The goal is to + // predict median home value. For more details about this dataset, please + // see https://archive.ics.uci.edu/ml/machine-learning-databases/housing/ public static void Example() { - // Downloading a regression dataset from github.com/dotnet/machinelearning - string dataFile = Microsoft.ML.SamplesUtils.DatasetUtils.DownloadHousingRegressionDataset(); + // Downloading a regression dataset from + // github.com/dotnet/machinelearning + string dataFile = Microsoft.ML.SamplesUtils.DatasetUtils + .DownloadHousingRegressionDataset(); - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(seed: 3); // Creating a data loader, based on the format of the data // The data is tab separated with all numeric columns. // The first column being the label and rest are numeric features // Here only seven numeric columns are used as features - var dataView = mlContext.Data.LoadFromTextFile(dataFile, new TextLoader.Options + var dataView = mlContext.Data.LoadFromTextFile(dataFile, + new TextLoader.Options { Separators = new[] { '\t' }, HasHeader = true, @@ -50,8 +56,10 @@ public static void Example() // Check the weights that the model learned var weightsValues = model.Model.Weights; - Console.WriteLine($"weight 0 - {weightsValues[0]}"); // CrimesPerCapita (weight 0) = -0.1682112 - Console.WriteLine($"weight 3 - {weightsValues[3]}"); // CharlesRiver (weight 1) = 3.663493 + Console.WriteLine($"weight 0 - {weightsValues[0]}"); + // CrimesPerCapita (weight 0) = -0.1682112 + Console.WriteLine($"weight 3 - {weightsValues[3]}"); + // CharlesRiver (weight 1) = 3.663493 var dataWithPredictions = model.Transform(split.TestSet); var metrics = mlContext.Regression.Evaluate(dataWithPredictions); @@ -67,10 +75,12 @@ public static void Example() public static void PrintMetrics(RegressionMetrics metrics) { - Console.WriteLine($"Mean Absolute Error: {metrics.MeanAbsoluteError:F2}"); - Console.WriteLine($"Mean Squared Error: {metrics.MeanSquaredError:F2}"); - Console.WriteLine($"Root Mean Squared Error: {metrics.RootMeanSquaredError:F2}"); - Console.WriteLine($"RSquared: {metrics.RSquared:F2}"); + Console.WriteLine("Mean Absolute Error: " + metrics.MeanAbsoluteError); + Console.WriteLine("Mean Squared Error: " + metrics.MeanSquaredError); + Console.WriteLine( + "Root Mean Squared Error: " + metrics.RootMeanSquaredError); + + Console.WriteLine("RSquared: " + metrics.RSquared); } } } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/OrdinaryLeastSquaresWithOptions.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/OrdinaryLeastSquaresWithOptions.cs index c56b71cd33..04b55ecfbe 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/OrdinaryLeastSquaresWithOptions.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/OrdinaryLeastSquaresWithOptions.cs @@ -11,15 +11,17 @@ public static class OlsWithOptions { public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); // Define trainer options. @@ -29,26 +31,32 @@ public static void Example() FeatureColumnName = nameof(DataPoint.Features), // Larger values leads to smaller (closer to zero) model parameters. L2Regularization = 0.1f, - // Whether to computate standard error and other statistics of model parameters. + // Whether to computate standard error and other statistics of model + // parameters. CalculateStatistics = false }; // Define the trainer. - var pipeline = mlContext.Regression.Trainers.Ols(options); + var pipeline = + mlContext.Regression.Trainers.Ols(options); // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(5, seed: 123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data.LoadFromEnumerable( + GenerateRandomDataPoints(5, seed: 123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(transformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data.CreateEnumerable( + transformedTestData, reuseRowObject: false).ToList(); - // Look at 5 predictions for the Label, side by side with the actual Label for comparison. + // Look at 5 predictions for the Label, side by side with the actual + // Label for comparison. foreach (var p in predictions) Console.WriteLine($"Label: {p.Label:F3}, Prediction: {p.Score:F3}"); @@ -70,7 +78,8 @@ public static void Example() // RSquared: 0.97 (closer to 1 is better. The worest case is 0) } - private static IEnumerable GenerateRandomDataPoints(int count, int seed=0) + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) { var random = new Random(seed); for (int i = 0; i < count; i++) @@ -80,12 +89,14 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se { Label = label, // Create random features that are correlated with the label. - Features = Enumerable.Repeat(label, 50).Select(x => x + (float)random.NextDouble()).ToArray() + Features = Enumerable.Repeat(label, 50).Select( + x => x + (float)random.NextDouble()).ToArray() }; } } - // Example with label and 50 feature values. A data set is a collection of such examples. + // Example with label and 50 feature values. A data set is a collection of + // such examples. private class DataPoint { public float Label { get; set; } @@ -105,10 +116,12 @@ private class Prediction // Print some evaluation metrics to regression problems. private static void PrintMetrics(RegressionMetrics metrics) { - Console.WriteLine($"Mean Absolute Error: {metrics.MeanAbsoluteError:F2}"); - Console.WriteLine($"Mean Squared Error: {metrics.MeanSquaredError:F2}"); - Console.WriteLine($"Root Mean Squared Error: {metrics.RootMeanSquaredError:F2}"); - Console.WriteLine($"RSquared: {metrics.RSquared:F2}"); + Console.WriteLine("Mean Absolute Error: " + metrics.MeanAbsoluteError); + Console.WriteLine("Mean Squared Error: " + metrics.MeanSquaredError); + Console.WriteLine( + "Root Mean Squared Error: " + metrics.RootMeanSquaredError); + + Console.WriteLine("RSquared: " + metrics.RSquared); } } } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/OrdinaryLeastSquaresWithOptions.tt b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/OrdinaryLeastSquaresWithOptions.tt index d37761637e..03c4fd961e 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/OrdinaryLeastSquaresWithOptions.tt +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/OrdinaryLeastSquaresWithOptions.tt @@ -11,7 +11,8 @@ string TrainerOptions = @"OlsTrainer.Options FeatureColumnName = nameof(DataPoint.Features), // Larger values leads to smaller (closer to zero) model parameters. L2Regularization = 0.1f, - // Whether to computate standard error and other statistics of model parameters. + // Whether to computate standard error and other statistics of model + // parameters. CalculateStatistics = false }"; diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/OrdinaryLeastSquaresWithOptionsAdvanced.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/OrdinaryLeastSquaresWithOptionsAdvanced.cs index 69728ac5b9..e53b14cce7 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/OrdinaryLeastSquaresWithOptionsAdvanced.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/OrdinaryLeastSquaresWithOptionsAdvanced.cs @@ -8,23 +8,28 @@ namespace Samples.Dynamic.Trainers.Regression { public static class OrdinaryLeastSquaresWithOptionsAdvanced { - // This example requires installation of additional nuget package Microsoft.ML.Mkl.Components. - // In this examples we will use the housing price dataset. The goal is to predict median home value. - // For more details about this dataset, please see https://archive.ics.uci.edu/ml/machine-learning-databases/housing/ + // This example requires installation of additional nuget package + // for Microsoft.ML.Mkl.Components at + // "https://www.nuget.org/packages/Microsoft.ML.Mkl.Components/" + // In this examples we will use the housing price dataset. The goal is to + // predict median home value. For more details about this dataset, please + // see https://archive.ics.uci.edu/ml/machine-learning-databases/housing/ public static void Example() { - // Downloading a regression dataset from github.com/dotnet/machinelearning + // Downloading a regression dataset from + // github.com/dotnet/machinelearning string dataFile = DatasetUtils.DownloadHousingRegressionDataset(); - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(seed: 3); // Creating a data loader, based on the format of the data // The data is tab separated with all numeric columns. // The first column being the label and rest are numeric features // Here only seven numeric columns are used as features - var dataView = mlContext.Data.LoadFromTextFile(dataFile, new TextLoader.Options + var dataView = mlContext.Data.LoadFromTextFile(dataFile, + new TextLoader.Options { Separators = new[] { '\t' }, HasHeader = true, @@ -45,7 +50,8 @@ public static void Example() // Create the estimator, here we only need OrdinaryLeastSquares trainer // as data is already processed in a form consumable by the trainer - var pipeline = mlContext.Regression.Trainers.Ols(new OlsTrainer.Options() + var pipeline = mlContext.Regression.Trainers.Ols( + new OlsTrainer.Options() { L2Regularization = 0.1f, CalculateStatistics = false @@ -54,8 +60,10 @@ public static void Example() // Check the weights that the model learned var weightsValues = model.Model.Weights; - Console.WriteLine($"weight 0 - {weightsValues[0]}"); // CrimesPerCapita (weight 0) = -0.1783206 - Console.WriteLine($"weight 3 - {weightsValues[3]}"); // CharlesRiver (weight 1) = 3.118422 + Console.WriteLine($"weight 0 - {weightsValues[0]}"); + // CrimesPerCapita (weight 0) = -0.1783206 + Console.WriteLine($"weight 3 - {weightsValues[3]}"); + // CharlesRiver (weight 1) = 3.118422 var dataWithPredictions = model.Transform(split.TestSet); var metrics = mlContext.Regression.Evaluate(dataWithPredictions); @@ -71,10 +79,12 @@ public static void Example() public static void PrintMetrics(RegressionMetrics metrics) { - Console.WriteLine($"Mean Absolute Error: {metrics.MeanAbsoluteError:F2}"); - Console.WriteLine($"Mean Squared Error: {metrics.MeanSquaredError:F2}"); - Console.WriteLine($"Root Mean Squared Error: {metrics.RootMeanSquaredError:F2}"); - Console.WriteLine($"RSquared: {metrics.RSquared:F2}"); + Console.WriteLine("Mean Absolute Error: " + metrics.MeanAbsoluteError); + Console.WriteLine("Mean Squared Error: " + metrics.MeanSquaredError); + Console.WriteLine( + "Root Mean Squared Error: " + metrics.RootMeanSquaredError); + + Console.WriteLine("RSquared: " + metrics.RSquared); } } } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/PermutationFeatureImportance.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/PermutationFeatureImportance.cs index 90cf94db2a..dfa04d1b76 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/PermutationFeatureImportance.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/PermutationFeatureImportance.cs @@ -9,8 +9,9 @@ public static class PermutationFeatureImportance { public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. var mlContext = new MLContext(seed:1); // Create sample data. @@ -19,10 +20,14 @@ public static void Example() // Load the sample data as an IDataView. var data = mlContext.Data.LoadFromEnumerable(samples); - // Define a training pipeline that concatenates features into a vector, normalizes them, and then - // trains a linear model. - var featureColumns = new string[] { nameof(Data.Feature1), nameof(Data.Feature2) }; - var pipeline = mlContext.Transforms.Concatenate("Features", featureColumns) + // Define a training pipeline that concatenates features into a vector, + // normalizes them, and then trains a linear model. + var featureColumns = new string[] { nameof(Data.Feature1), + nameof(Data.Feature2) }; + + var pipeline = mlContext.Transforms.Concatenate( + "Features", + featureColumns) .Append(mlContext.Transforms.NormalizeMinMax("Features")) .Append(mlContext.Regression.Trainers.Ols()); @@ -35,18 +40,29 @@ public static void Example() // Extract the predictor. var linearPredictor = model.LastTransformer; - // Compute the permutation metrics for the linear model using the normalized data. - var permutationMetrics = mlContext.Regression.PermutationFeatureImportance( + // Compute the permutation metrics for the linear model using the + // normalized data. + var permutationMetrics = mlContext.Regression + .PermutationFeatureImportance( linearPredictor, transformedData, permutationCount: 30); - // Now let's look at which features are most important to the model overall. - // Get the feature indices sorted by their impact on RMSE. - var sortedIndices = permutationMetrics.Select((metrics, index) => new { index, metrics.RootMeanSquaredError}) - .OrderByDescending(feature => Math.Abs(feature.RootMeanSquaredError.Mean)) + // Now let's look at which features are most important to the model + // overall. Get the feature indices sorted by their impact on RMSE. + var sortedIndices = permutationMetrics + .Select((metrics, index) => new { index, + metrics.RootMeanSquaredError}) + + .OrderByDescending(feature => Math.Abs( + feature.RootMeanSquaredError.Mean)) + .Select(feature => feature.index); - Console.WriteLine("Feature\tModel Weight\tChange in RMSE\t95% Confidence in the Mean Change in RMSE"); - var rmse = permutationMetrics.Select(x => x.RootMeanSquaredError).ToArray(); + Console.WriteLine("Feature\tModel Weight\tChange in RMSE\t95%" + + "Confidence in the Mean Change in RMSE"); + + var rmse = permutationMetrics.Select(x => x.RootMeanSquaredError) + .ToArray(); + foreach (int i in sortedIndices) { Console.WriteLine("{0}\t{1:0.00}\t{2:G4}\t{3:G4}", @@ -76,10 +92,14 @@ private class Data /// linear combination of the features. /// /// The number of examples. - /// The bias, or offset, in the calculation of the label. - /// The weight to multiply the first feature with to compute the label. - /// The weight to multiply the second feature with to compute the label. - /// The seed for generating feature values and label noise. + /// The bias, or offset, in the calculation of the label. + /// + /// The weight to multiply the first feature with to + /// compute the label. + /// The weight to multiply the second feature with to + /// compute the label. + /// The seed for generating feature values and label + /// noise. /// An enumerable of Data objects. private static IEnumerable GenerateData(int nExamples = 10000, double bias = 0, double weight1 = 1, double weight2 = 2, int seed = 1) @@ -94,7 +114,8 @@ private static IEnumerable GenerateData(int nExamples = 10000, }; // Create a noisy label. - data.Label = (float)(bias + weight1 * data.Feature1 + weight2 * data.Feature2 + rng.NextDouble() - 0.5); + data.Label = (float)(bias + weight1 * data.Feature1 + weight2 * + data.Feature2 + rng.NextDouble() - 0.5); yield return data; } } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/RegressionSamplesTemplate.ttinclude b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/RegressionSamplesTemplate.ttinclude index f4193eefd4..6b8d3ff2b4 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/RegressionSamplesTemplate.ttinclude +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/RegressionSamplesTemplate.ttinclude @@ -16,15 +16,17 @@ namespace Samples.Dynamic.Trainers.Regression <# } #> public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); <# if (TrainerOptions == null) { #> @@ -35,22 +37,27 @@ namespace Samples.Dynamic.Trainers.Regression var options = new <#=TrainerOptions#>; // Define the trainer. - var pipeline = mlContext.Regression.Trainers.<#=Trainer#>(options); + var pipeline = + mlContext.Regression.Trainers.<#=Trainer#>(options); <# } #> // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(5, seed: 123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data.LoadFromEnumerable( + GenerateRandomDataPoints(5, seed: 123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(transformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data.CreateEnumerable( + transformedTestData, reuseRowObject: false).ToList(); - // Look at 5 predictions for the Label, side by side with the actual Label for comparison. + // Look at 5 predictions for the Label, side by side with the actual + // Label for comparison. foreach (var p in predictions) Console.WriteLine($"Label: {p.Label:F3}, Prediction: {p.Score:F3}"); @@ -63,7 +70,8 @@ namespace Samples.Dynamic.Trainers.Regression <#=ExpectedOutput#> } - private static IEnumerable GenerateRandomDataPoints(int count, int seed=0) + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) { var random = new Random(seed); for (int i = 0; i < count; i++) @@ -73,12 +81,14 @@ namespace Samples.Dynamic.Trainers.Regression { Label = label, // Create random features that are correlated with the label. - Features = Enumerable.Repeat(label, 50).Select(x => x + (float)random.NextDouble()).ToArray() + Features = Enumerable.Repeat(label, 50).Select( + x => x + (float)random.NextDouble()).ToArray() }; } } - // Example with label and 50 feature values. A data set is a collection of such examples. + // Example with label and 50 feature values. A data set is a collection of + // such examples. private class DataPoint { public float Label { get; set; } @@ -98,10 +108,12 @@ namespace Samples.Dynamic.Trainers.Regression // Print some evaluation metrics to regression problems. private static void PrintMetrics(RegressionMetrics metrics) { - Console.WriteLine($"Mean Absolute Error: {metrics.MeanAbsoluteError:F2}"); - Console.WriteLine($"Mean Squared Error: {metrics.MeanSquaredError:F2}"); - Console.WriteLine($"Root Mean Squared Error: {metrics.RootMeanSquaredError:F2}"); - Console.WriteLine($"RSquared: {metrics.RSquared:F2}"); + Console.WriteLine("Mean Absolute Error: " + metrics.MeanAbsoluteError); + Console.WriteLine("Mean Squared Error: " + metrics.MeanSquaredError); + Console.WriteLine( + "Root Mean Squared Error: " + metrics.RootMeanSquaredError); + + Console.WriteLine("RSquared: " + metrics.RSquared); } } } \ No newline at end of file diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/Sdca.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/Sdca.cs index 460334f283..4247c1447e 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/Sdca.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/Sdca.cs @@ -10,33 +10,41 @@ public static class Sdca { public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); // Define the trainer. - var pipeline = mlContext.Regression.Trainers.Sdca(labelColumnName: nameof(DataPoint.Label), featureColumnName: nameof(DataPoint.Features)); + var pipeline = mlContext.Regression.Trainers.Sdca( + labelColumnName: nameof(DataPoint.Label), + featureColumnName: nameof(DataPoint.Features)); // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(5, seed: 123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data.LoadFromEnumerable( + GenerateRandomDataPoints(5, seed: 123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(transformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data.CreateEnumerable( + transformedTestData, reuseRowObject: false).ToList(); - // Look at 5 predictions for the Label, side by side with the actual Label for comparison. + // Look at 5 predictions for the Label, side by side with the actual + // Label for comparison. foreach (var p in predictions) Console.WriteLine($"Label: {p.Label:F3}, Prediction: {p.Score:F3}"); @@ -58,7 +66,8 @@ public static void Example() // RSquared: 0.97 (closer to 1 is better. The worest case is 0) } - private static IEnumerable GenerateRandomDataPoints(int count, int seed=0) + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) { var random = new Random(seed); for (int i = 0; i < count; i++) @@ -68,12 +77,14 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se { Label = label, // Create random features that are correlated with the label. - Features = Enumerable.Repeat(label, 50).Select(x => x + (float)random.NextDouble()).ToArray() + Features = Enumerable.Repeat(label, 50).Select( + x => x + (float)random.NextDouble()).ToArray() }; } } - // Example with label and 50 feature values. A data set is a collection of such examples. + // Example with label and 50 feature values. A data set is a collection of + // such examples. private class DataPoint { public float Label { get; set; } @@ -93,10 +104,12 @@ private class Prediction // Print some evaluation metrics to regression problems. private static void PrintMetrics(RegressionMetrics metrics) { - Console.WriteLine($"Mean Absolute Error: {metrics.MeanAbsoluteError:F2}"); - Console.WriteLine($"Mean Squared Error: {metrics.MeanSquaredError:F2}"); - Console.WriteLine($"Root Mean Squared Error: {metrics.RootMeanSquaredError:F2}"); - Console.WriteLine($"RSquared: {metrics.RSquared:F2}"); + Console.WriteLine("Mean Absolute Error: " + metrics.MeanAbsoluteError); + Console.WriteLine("Mean Squared Error: " + metrics.MeanSquaredError); + Console.WriteLine( + "Root Mean Squared Error: " + metrics.RootMeanSquaredError); + + Console.WriteLine("RSquared: " + metrics.RSquared); } } } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/Sdca.tt b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/Sdca.tt index 2a74a40116..bd1581da6b 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/Sdca.tt +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/Sdca.tt @@ -4,7 +4,9 @@ string ClassHeader = null; string ClassName="Sdca"; string ExtraUsing = null; -string Trainer = "Sdca(labelColumnName: nameof(DataPoint.Label), featureColumnName: nameof(DataPoint.Features))"; +string Trainer = @"Sdca( + labelColumnName: nameof(DataPoint.Label), + featureColumnName: nameof(DataPoint.Features))"; string TrainerOptions = null; string ExpectedOutputPerInstance= @"// Expected output: diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/SdcaWithOptions.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/SdcaWithOptions.cs index c5c06350e7..55e05ab3a1 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/SdcaWithOptions.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/SdcaWithOptions.cs @@ -11,15 +11,17 @@ public static class SdcaWithOptions { public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(1000); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var trainingData = mlContext.Data.LoadFromEnumerable(dataPoints); // Define trainer options. @@ -27,31 +29,38 @@ public static void Example() { LabelColumnName = nameof(DataPoint.Label), FeatureColumnName = nameof(DataPoint.Features), - // Make the convergence tolerance tighter. It effectively leads to more training iterations. + // Make the convergence tolerance tighter. It effectively leads to + // more training iterations. ConvergenceTolerance = 0.02f, - // Increase the maximum number of passes over training data. Similar to ConvergenceTolerance, - // this value specifics the hard iteration limit on the training algorithm. + // Increase the maximum number of passes over training data. Similar + // to ConvergenceTolerance, this value specifics the hard iteration + // limit on the training algorithm. MaximumNumberOfIterations = 30, // Increase learning rate for bias. BiasLearningRate = 0.1f }; // Define the trainer. - var pipeline = mlContext.Regression.Trainers.Sdca(options); + var pipeline = + mlContext.Regression.Trainers.Sdca(options); // Train the model. var model = pipeline.Fit(trainingData); - // Create testing data. Use different random seed to make it different from training data. - var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(5, seed: 123)); + // Create testing data. Use different random seed to make it different + // from training data. + var testData = mlContext.Data.LoadFromEnumerable( + GenerateRandomDataPoints(5, seed: 123)); // Run the model on test data set. var transformedTestData = model.Transform(testData); // Convert IDataView object to a list. - var predictions = mlContext.Data.CreateEnumerable(transformedTestData, reuseRowObject: false).ToList(); + var predictions = mlContext.Data.CreateEnumerable( + transformedTestData, reuseRowObject: false).ToList(); - // Look at 5 predictions for the Label, side by side with the actual Label for comparison. + // Look at 5 predictions for the Label, side by side with the actual + // Label for comparison. foreach (var p in predictions) Console.WriteLine($"Label: {p.Label:F3}, Prediction: {p.Score:F3}"); @@ -73,7 +82,8 @@ public static void Example() // RSquared: 0.97 (closer to 1 is better. The worest case is 0) } - private static IEnumerable GenerateRandomDataPoints(int count, int seed=0) + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) { var random = new Random(seed); for (int i = 0; i < count; i++) @@ -83,12 +93,14 @@ private static IEnumerable GenerateRandomDataPoints(int count, int se { Label = label, // Create random features that are correlated with the label. - Features = Enumerable.Repeat(label, 50).Select(x => x + (float)random.NextDouble()).ToArray() + Features = Enumerable.Repeat(label, 50).Select( + x => x + (float)random.NextDouble()).ToArray() }; } } - // Example with label and 50 feature values. A data set is a collection of such examples. + // Example with label and 50 feature values. A data set is a collection of + // such examples. private class DataPoint { public float Label { get; set; } @@ -108,10 +120,12 @@ private class Prediction // Print some evaluation metrics to regression problems. private static void PrintMetrics(RegressionMetrics metrics) { - Console.WriteLine($"Mean Absolute Error: {metrics.MeanAbsoluteError:F2}"); - Console.WriteLine($"Mean Squared Error: {metrics.MeanSquaredError:F2}"); - Console.WriteLine($"Root Mean Squared Error: {metrics.RootMeanSquaredError:F2}"); - Console.WriteLine($"RSquared: {metrics.RSquared:F2}"); + Console.WriteLine("Mean Absolute Error: " + metrics.MeanAbsoluteError); + Console.WriteLine("Mean Squared Error: " + metrics.MeanSquaredError); + Console.WriteLine( + "Root Mean Squared Error: " + metrics.RootMeanSquaredError); + + Console.WriteLine("RSquared: " + metrics.RSquared); } } } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/SdcaWithOptions.tt b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/SdcaWithOptions.tt index 083efe8cf6..5c350caf66 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/SdcaWithOptions.tt +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/SdcaWithOptions.tt @@ -9,10 +9,12 @@ string TrainerOptions = @"SdcaRegressionTrainer.Options { LabelColumnName = nameof(DataPoint.Label), FeatureColumnName = nameof(DataPoint.Features), - // Make the convergence tolerance tighter. It effectively leads to more training iterations. + // Make the convergence tolerance tighter. It effectively leads to + // more training iterations. ConvergenceTolerance = 0.02f, - // Increase the maximum number of passes over training data. Similar to ConvergenceTolerance, - // this value specifics the hard iteration limit on the training algorithm. + // Increase the maximum number of passes over training data. Similar + // to ConvergenceTolerance, this value specifics the hard iteration + // limit on the training algorithm. MaximumNumberOfIterations = 30, // Increase learning rate for bias. BiasLearningRate = 0.1f diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/ApplyONNXModelWithInMemoryImages.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/ApplyONNXModelWithInMemoryImages.cs new file mode 100644 index 0000000000..4c1c3a3d4b --- /dev/null +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/ApplyONNXModelWithInMemoryImages.cs @@ -0,0 +1,116 @@ +using System; +using System.Drawing; +using System.Linq; +using Microsoft.ML; +using Microsoft.ML.Data; +using Microsoft.ML.Transforms.Image; + +namespace Samples.Dynamic +{ + public static class ApplyOnnxModelWithInMemoryImages + { + // Example of applying ONNX transform on in-memory images. + public static void Example() + { + // Download the squeeznet image model from ONNX model zoo, version 1.2 + // https://github.com/onnx/models/tree/master/squeezenet or use + // Microsoft.ML.Onnx.TestModels nuget. + // It's a multiclass classifier. It consumes an input "data_0" and + // produces an output "softmaxout_1". + var modelPath = @"squeezenet\00000001\model.onnx"; + + // Create ML pipeline to score the data using OnnxScoringEstimator + var mlContext = new MLContext(); + + // Create in-memory data points. Its Image/Scores field is the + // input /output of the used ONNX model. + var dataPoints = new ImageDataPoint[] + { + new ImageDataPoint(Color.Red), + new ImageDataPoint(Color.Green) + }; + + // Convert training data to IDataView, the general data type used in + // ML.NET. + var dataView = mlContext.Data.LoadFromEnumerable(dataPoints); + + // Create a ML.NET pipeline which contains two steps. First, + // ExtractPixle is used to convert the 224x224 image to a 3x224x224 + // float tensor. Then the float tensor is fed into a ONNX model with an + // input called "data_0" and an output called "softmaxout_1". Note that + // "data_0" and "softmaxout_1" are model input and output names stored + // in the used ONNX model file. Users may need to inspect their own + // models to get the right input and output column names. + // Map column "Image" to column "data_0" + // Map column "data_0" to column "softmaxout_1" + var pipeline = mlContext.Transforms.ExtractPixels("data_0", "Image") + .Append(mlContext.Transforms.ApplyOnnxModel("softmaxout_1", + "data_0", modelPath)); + + var model = pipeline.Fit(dataView); + var onnx = model.Transform(dataView); + + // Convert IDataView back to IEnumerable so that user + // can inspect the output, column "softmaxout_1", of the ONNX transform. + // Note that Column "softmaxout_1" would be stored in ImageDataPont + //.Scores because the added attributed [ColumnName("softmaxout_1")] + // tells that ImageDataPont.Scores is equivalent to column + // "softmaxout_1". + var transformedDataPoints = mlContext.Data.CreateEnumerable< + ImageDataPoint>(onnx, false).ToList(); + + // The scores are probabilities of all possible classes, so they should + // all be positive. + foreach (var dataPoint in transformedDataPoints) + { + var firstClassProb = dataPoint.Scores.First(); + var lastClassProb = dataPoint.Scores.Last(); + Console.WriteLine("The probability of being the first class is " + + (firstClassProb * 100) + "%."); + + Console.WriteLine($"The probability of being the last class is " + + (lastClassProb * 100) + "%."); + } + + // Expected output: + // The probability of being the first class is 0.002542659%. + // The probability of being the last class is 0.0292684%. + // The probability of being the first class is 0.02258059%. + // The probability of being the last class is 0.394428%. + } + + // This class is used in Example() to describe data points which will be + // consumed by ML.NET pipeline. + private class ImageDataPoint + { + // Height of Image. + private const int height = 224; + + // Width of Image. + private const int width = 224; + + // Image will be consumed by ONNX image multiclass classification model. + [ImageType(height, width)] + public Bitmap Image { get; set; } + + // Expected output of ONNX model. It contains probabilities of all + // classes. Note that the ColumnName below should match the output name + // in the used ONNX model file. + [ColumnName("softmaxout_1")] + public float[] Scores { get; set; } + + public ImageDataPoint() + { + Image = null; + } + + public ImageDataPoint(Color color) + { + Image = new Bitmap(width, height); + for (int i = 0; i < width; ++i) + for (int j = 0; j < height; ++j) + Image.SetPixel(i, j, color); + } + } + } +} diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/ApplyOnnxModel.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/ApplyOnnxModel.cs index abff3a339d..adb2268060 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/ApplyOnnxModel.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/ApplyOnnxModel.cs @@ -19,14 +19,16 @@ public static void Example() // Generate sample test data. var samples = GetTensorData(); - // Convert training data to IDataView, the general data type used in ML.NET. + // Convert training data to IDataView, the general data type used in + // ML.NET. var data = mlContext.Data.LoadFromEnumerable(samples); // Create the pipeline to score using provided onnx model. var pipeline = mlContext.Transforms.ApplyOnnxModel(modelPath); // Fit the pipeline and get the transformed values var transformedValues = pipeline.Fit(data).Transform(data); // Retrieve model scores into Prediction class - var predictions = mlContext.Data.CreateEnumerable(transformedValues, reuseRowObject: false); + var predictions = mlContext.Data.CreateEnumerable( + transformedValues, reuseRowObject: false); // Iterate rows foreach (var prediction in predictions) @@ -34,7 +36,8 @@ public static void Example() int numClasses = 0; foreach (var classScore in prediction.softmaxout_1.Take(3)) { - Console.WriteLine($"Class #{numClasses++} score = {classScore}"); + Console.WriteLine("Class #" + numClasses++ + " score = " + + classScore); } Console.WriteLine(new string('-', 10)); } @@ -65,9 +68,14 @@ public class TensorData public static TensorData[] GetTensorData() { // This can be any numerical data. Assume image pixel values. - var image1 = Enumerable.Range(0, inputSize).Select(x => (float)x / inputSize).ToArray(); - var image2 = Enumerable.Range(0, inputSize).Select(x => (float)(x + 10000) / inputSize).ToArray(); - return new TensorData[] { new TensorData() { data_0 = image1 }, new TensorData() { data_0 = image2 } }; + var image1 = Enumerable.Range(0, inputSize).Select(x => (float)x / + inputSize).ToArray(); + + var image2 = Enumerable.Range(0, inputSize).Select(x => (float)(x + + 10000) / inputSize).ToArray(); + + return new TensorData[] { new TensorData() { data_0 = image1 }, new + TensorData() { data_0 = image2 } }; } // Class to contain the output values from the transformation. diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/ApproximatedKernelMap.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/ApproximatedKernelMap.cs index ce7152b50f..7b21512883 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/ApproximatedKernelMap.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/ApproximatedKernelMap.cs @@ -9,11 +9,12 @@ namespace Samples.Dynamic { public static class ApproximatedKernelMap { - // Transform feature vector to another non-linear space. See https://people.eecs.berkeley.edu/~brecht/papers/07.rah.rec.nips.pdf. + // Transform feature vector to another non-linear space. See + // https://people.eecs.berkeley.edu/~brecht/papers/07.rah.rec.nips.pdf. public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(); var samples = new List() { @@ -22,19 +23,26 @@ public static void Example() new DataPoint(){ Features = new float[7] {-1, 1, 0,-1,-1, 0,-1} }, new DataPoint(){ Features = new float[7] { 0,-1, 0, 1, 0,-1,-1} } }; - // Convert training data to IDataView, the general data type used in ML.NET. + // Convert training data to IDataView, the general data type used in + // ML.NET. var data = mlContext.Data.LoadFromEnumerable(samples); - // ApproximatedKernel map takes data and maps it's to a random low-dimensional space. - var approximation = mlContext.Transforms.ApproximatedKernelMap("Features", rank: 4, generator: new GaussianKernel(gamma: 0.7f), seed: 1); + // ApproximatedKernel map takes data and maps it's to a random + // low -dimensional space. + var approximation = mlContext.Transforms.ApproximatedKernelMap( + "Features", rank: 4, generator: new GaussianKernel(gamma: 0.7f), + seed: 1); - // Now we can transform the data and look at the output to confirm the behavior of the estimator. - // This operation doesn't actually evaluate data until we read the data below. + // Now we can transform the data and look at the output to confirm the + // behavior of the estimator. This operation doesn't actually evaluate + // data until we read the data below. var tansformer = approximation.Fit(data); var transformedData = tansformer.Transform(data); var column = transformedData.GetColumn("Features").ToArray(); foreach (var row in column) - Console.WriteLine(string.Join(", ", row.Select(x => x.ToString("f4")))); + Console.WriteLine(string.Join(", ", row.Select(x => x.ToString( + "f4")))); + // Expected output: // -0.0119, 0.5867, 0.4942, 0.7041 // 0.4720, 0.5639, 0.4346, 0.2671 diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/CalculateFeatureContribution.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/CalculateFeatureContribution.cs index 3a342f8809..2f0f5b464e 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/CalculateFeatureContribution.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/CalculateFeatureContribution.cs @@ -9,7 +9,8 @@ public static class CalculateFeatureContribution { public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, // as a catalog of available operations and as the source of randomness. var mlContext = new MLContext(seed: 1); @@ -19,7 +20,8 @@ public static void Example() // Convert training data to IDataView. var data = mlContext.Data.LoadFromEnumerable(samples); - // Create a pipeline to concatenate the features into a feature vector and normalize it. + // Create a pipeline to concatenate the features into a feature vector + // and normalize it. var transformPipeline = mlContext.Transforms.Concatenate("Features", new string[] { nameof(Data.Feature1), nameof(Data.Feature2) }) .Append(mlContext.Transforms.NormalizeMeanVariance("Features")); @@ -37,28 +39,45 @@ public static void Example() var linearModel = linearTrainer.Fit(transformedData); // Print the model parameters. Console.WriteLine($"Linear Model Parameters"); - Console.WriteLine($"Bias: {linearModel.Model.Bias} Feature1: {linearModel.Model.Weights[0]} Feature2: {linearModel.Model.Weights[1]}"); + Console.WriteLine("Bias: " + linearModel.Model.Bias+ " Feature1: " + + linearModel.Model.Weights[0] + " Feature2: " +linearModel.Model + .Weights[1]); - // Define a feature contribution calculator for all the features, and don't normalize the contributions. - // These are "trivial estimators" and they don't need to fit to the data, so we can feed a subset. - var simpleScoredDataset = linearModel.Transform(mlContext.Data.TakeRows(transformedData, 1)); - var linearFeatureContributionCalculator = mlContext.Transforms.CalculateFeatureContribution(linearModel, normalize: false).Fit(simpleScoredDataset); + // Define a feature contribution calculator for all the features, and + // don't normalize the contributions.These are "trivial estimators" and + // they don't need to fit to the data, so we can feed a subset. + var simpleScoredDataset = linearModel.Transform(mlContext.Data + .TakeRows(transformedData, 1)); + + var linearFeatureContributionCalculator = mlContext.Transforms + .CalculateFeatureContribution(linearModel, normalize: false).Fit( + simpleScoredDataset); // Create a transformer chain to describe the entire pipeline. - var scoringPipeline = transformer.Append(linearModel).Append(linearFeatureContributionCalculator); + var scoringPipeline = transformer.Append(linearModel).Append( + linearFeatureContributionCalculator); - // Create the prediction engine to get the features extracted from the text. - var predictionEngine = mlContext.Model.CreatePredictionEngine(scoringPipeline); + // Create the prediction engine to get the features extracted from the + // text. + var predictionEngine = mlContext.Model.CreatePredictionEngine(scoringPipeline); // Convert the text into numeric features. var prediction = predictionEngine.Predict(samples.First()); // Write out the prediction, with contributions. - // Note that for the linear model, the feature contributions for a feature in an example is the feature-weight*feature-value. + // Note that for the linear model, the feature contributions for a + // feature in an example is the feature-weight*feature-value. // The total prediction is thus the bias plus the feature contributions. - Console.WriteLine($"Label: {prediction.Label} Prediction: {prediction.Score}"); - Console.WriteLine($"Feature1: {prediction.Features[0]} Feature2: {prediction.Features[1]}"); - Console.WriteLine($"Feature Contributions: {prediction.FeatureContributions[0]} {prediction.FeatureContributions[1]}"); + Console.WriteLine("Label: " + prediction.Label + " Prediction: " + + prediction.Score); + + Console.WriteLine("Feature1: " + prediction.Features[0] + + " Feature2: " + prediction.Features[1]); + + Console.WriteLine("Feature Contributions: " + prediction + .FeatureContributions[0] + " " + prediction + .FeatureContributions[1]); // Expected output: // Linear Model Parameters @@ -107,7 +126,9 @@ private static IEnumerable GenerateData(int nExamples = 10000, }; // Create a noisy label. - data.Label = (float)(bias + weight1 * data.Feature1 + weight2 * data.Feature2 + rng.NextDouble() - 0.5); + data.Label = (float)(bias + weight1 * data.Feature1 + weight2 * + data.Feature2 + rng.NextDouble() - 0.5); + yield return data; } } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/CalculateFeatureContributionCalibrated.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/CalculateFeatureContributionCalibrated.cs index c05e7e5468..adaf66389c 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/CalculateFeatureContributionCalibrated.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/CalculateFeatureContributionCalibrated.cs @@ -9,8 +9,9 @@ public static class CalculateFeatureContributionCalibrated { public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. var mlContext = new MLContext(); // Create a small dataset. @@ -19,7 +20,8 @@ public static void Example() // Convert training data to IDataView. var data = mlContext.Data.LoadFromEnumerable(samples); - // Create a pipeline to concatenate the features into a feature vector and normalize it. + // Create a pipeline to concatenate the features into a feature vector + // and normalize it. var transformPipeline = mlContext.Transforms.Concatenate("Features", new string[] { nameof(Data.Feature1), nameof(Data.Feature2) }) .Append(mlContext.Transforms.NormalizeMeanVariance("Features")); @@ -31,7 +33,8 @@ public static void Example() var transformedData = transformer.Transform(data); // Define a linear trainer. - var linearTrainer = mlContext.BinaryClassification.Trainers.SdcaLogisticRegression(); + var linearTrainer = mlContext.BinaryClassification.Trainers + .SdcaLogisticRegression(); // Now we train the model and score it on the transformed data. var linearModel = linearTrainer.Fit(transformedData); @@ -42,26 +45,42 @@ public static void Example() linearModel.Model.SubModel.Weights[0], linearModel.Model.SubModel.Weights[1]); - // Define a feature contribution calculator for all the features, and don't normalize the contributions. - // These are "trivial estimators" and they don't need to fit to the data, so we can feed a subset. - var simpleScoredDataset = linearModel.Transform(mlContext.Data.TakeRows(transformedData, 1)); - var linearFeatureContributionCalculator = mlContext.Transforms.CalculateFeatureContribution(linearModel, normalize: false).Fit(simpleScoredDataset); + // Define a feature contribution calculator for all the features, and + // don't normalize the contributions. These are "trivial estimators" and + // they don't need to fit to the data, so we can feed a subset. + var simpleScoredDataset = linearModel.Transform(mlContext.Data + .TakeRows(transformedData, 1)); + + var linearFeatureContributionCalculator = mlContext.Transforms + .CalculateFeatureContribution(linearModel, normalize: false) + .Fit(simpleScoredDataset); // Create a transformer chain to describe the entire pipeline. - var scoringPipeline = transformer.Append(linearModel).Append(linearFeatureContributionCalculator); + var scoringPipeline = transformer.Append(linearModel) + .Append(linearFeatureContributionCalculator); - // Create the prediction engine to get the features extracted from the text. - var predictionEngine = mlContext.Model.CreatePredictionEngine(scoringPipeline); + // Create the prediction engine to get the features extracted from the + // text. + var predictionEngine = mlContext.Model.CreatePredictionEngine(scoringPipeline); // Convert the text into numeric features. var prediction = predictionEngine.Predict(samples.First()); // Write out the prediction, with contributions. - // Note that for the linear model, the feature contributions for a feature in an example is the feature-weight*feature-value. - // The total prediction is thus the bias plus the feature contributions. - Console.WriteLine($"Label: {prediction.Label} Prediction-Score: {prediction.Score} Prediction-Probability: {prediction.Probability}"); - Console.WriteLine($"Feature1: {prediction.Features[0]} Feature2: {prediction.Features[1]}"); - Console.WriteLine($"Feature Contributions: {prediction.FeatureContributions[0]} {prediction.FeatureContributions[1]}"); + // Note that for the linear model, the feature contributions for a + // feature in an example is the feature-weight*feature-value. The total + // prediction is thus the bias plus the feature contributions. + Console.WriteLine("Label: " + prediction.Label + " Prediction-Score: " + + prediction.Score + " Prediction-Probability: " + prediction + .Probability); + + Console.WriteLine("Feature1: " + prediction.Features[0] + " Feature2: " + + prediction.Features[1]); + + Console.WriteLine("Feature Contributions: " + prediction + .FeatureContributions[0] + " " + prediction + .FeatureContributions[1]); // Expected output: // Linear Model Parameters @@ -114,7 +133,9 @@ private static IEnumerable GenerateData(int nExamples = 10000, }; // Create a Boolean label with noise. - var value = bias + weight1 * data.Feature1 + weight2 * data.Feature2 + rng.NextDouble() - 0.5; + var value = bias + weight1 * data.Feature1 + weight2 * data.Feature2 + + rng.NextDouble() - 0.5; + data.Label = Sigmoid(value) > 0.5; yield return data; } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Concatenate.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Concatenate.cs index 9b350227be..c62cc6b743 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Concatenate.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Concatenate.cs @@ -9,38 +9,56 @@ public static class Concatenate { public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(); // Create a small dataset as an IEnumerable. var samples = new List() { - new InputData(){ Feature1 = 0.1f, Feature2 = new[]{ 1.1f, 2.1f, 3.1f}, Feature3 = 1 }, - new InputData(){ Feature1 = 0.2f, Feature2 = new[]{ 1.2f, 2.2f, 3.2f}, Feature3 = 2 }, - new InputData(){ Feature1 = 0.3f, Feature2 = new[]{ 1.3f, 2.3f, 3.3f}, Feature3 = 3 }, - new InputData(){ Feature1 = 0.4f, Feature2 = new[]{ 1.4f, 2.4f, 3.4f}, Feature3 = 4 }, - new InputData(){ Feature1 = 0.5f, Feature2 = new[]{ 1.5f, 2.5f, 3.5f}, Feature3 = 5 }, - new InputData(){ Feature1 = 0.6f, Feature2 = new[]{ 1.6f, 2.6f, 3.6f}, Feature3 = 6 }, + new InputData(){ Feature1 = 0.1f, Feature2 = new[]{ 1.1f, 2.1f, + 3.1f }, Feature3 = 1 }, + + new InputData(){ Feature1 = 0.2f, Feature2 = new[]{ 1.2f, 2.2f, + 3.2f }, Feature3 = 2 }, + + new InputData(){ Feature1 = 0.3f, Feature2 = new[]{ 1.3f, 2.3f, + 3.3f }, Feature3 = 3 }, + + new InputData(){ Feature1 = 0.4f, Feature2 = new[]{ 1.4f, 2.4f, + 3.4f }, Feature3 = 4 }, + + new InputData(){ Feature1 = 0.5f, Feature2 = new[]{ 1.5f, 2.5f, + 3.5f }, Feature3 = 5 }, + + new InputData(){ Feature1 = 0.6f, Feature2 = new[]{ 1.6f, 2.6f, + 3.6f }, Feature3 = 6 }, }; // Convert training data to IDataView. var dataview = mlContext.Data.LoadFromEnumerable(samples); - // A pipeline for concatenating the "Feature1", "Feature2" and "Feature3" columns together into a vector that will be the Features column. - // Concatenation is necessary because trainers take feature vectors as inputs. + // A pipeline for concatenating the "Feature1", "Feature2" and + // "Feature3" columns together into a vector that will be the Features + // column. Concatenation is necessary because trainers take feature + // vectors as inputs. // - // Please note that the "Feature3" column is converted from int32 to float using the ConvertType. - // The Concatenate requires all columns to be of same type. - var pipeline = mlContext.Transforms.Conversion.ConvertType("Feature3", outputKind: DataKind.Single) - .Append(mlContext.Transforms.Concatenate("Features", new[] { "Feature1", "Feature2", "Feature3" })); + // Please note that the "Feature3" column is converted from int32 to + // float using the ConvertType. The Concatenate requires all columns to + // be of same type. + var pipeline = mlContext.Transforms.Conversion.ConvertType("Feature3", + outputKind: DataKind.Single) + .Append(mlContext.Transforms.Concatenate("Features", new[] + { "Feature1", "Feature2", "Feature3" })); // The transformed data. var transformedData = pipeline.Fit(dataview).Transform(dataview); // Now let's take a look at what this concatenation did. - // We can extract the newly created column as an IEnumerable of TransformedData. - var featuresColumn = mlContext.Data.CreateEnumerable(transformedData, reuseRowObject: false); + // We can extract the newly created column as an IEnumerable of + // TransformedData. + var featuresColumn = mlContext.Data.CreateEnumerable( + transformedData, reuseRowObject: false); // And we can write out a few rows Console.WriteLine($"Features column obtained post-transformation."); diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/ConvertType.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/ConvertType.cs index 8349922768..610a77983a 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/ConvertType.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/ConvertType.cs @@ -20,17 +20,22 @@ public static void Example() var data = mlContext.Data.LoadFromEnumerable(rawData); // Construct the pipeline. - var pipeline = mlContext.Transforms.Conversion.ConvertType("SurvivedInt32", "Survived", DataKind.Int32); + var pipeline = mlContext.Transforms.Conversion.ConvertType( + "SurvivedInt32", "Survived", DataKind.Int32); // Let's train our pipeline, and then apply it to the same data. var transformer = pipeline.Fit(data); var transformedData = transformer.Transform(data); - // Display original column 'Survived' (boolean) and converted column 'SurvivedInt32' (Int32) - var convertedData = mlContext.Data.CreateEnumerable(transformedData, true); + // Display original column 'Survived' (boolean) and converted column + // SurvivedInt32' (Int32) + var convertedData = mlContext.Data.CreateEnumerable( + transformedData, true); + foreach (var item in convertedData) { - Console.WriteLine("A:{0,-10} Aconv:{1}", item.Survived, item.SurvivedInt32); + Console.WriteLine("A:{0,-10} Aconv:{1}", item.Survived, + item.SurvivedInt32); } // Output diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/ConvertTypeMultiColumn.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/ConvertTypeMultiColumn.cs index b7790d9f11..bae0bfea02 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/ConvertTypeMultiColumn.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/ConvertTypeMultiColumn.cs @@ -4,22 +4,33 @@ namespace Samples.Dynamic { - // This example illustrates how to convert multiple columns of different types to one type, in this case System.Single. - // This is often a useful data transformation before concatenating the features together and passing them to a particular estimator. + // This example illustrates how to convert multiple columns of different types + // to one type, in this case System.Single. + // This is often a useful data transformation before concatenating the features + // together and passing them to a particular estimator. public static class ConvertTypeMultiColumn { public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(seed: 1); var rawData = new[] { - new InputData() { Feature1 = true, Feature2 = "0.4", Feature3 = DateTime.Now, Feature4 = 0.145}, - new InputData() { Feature1 = false, Feature2 = "0.5", Feature3 = DateTime.Today, Feature4 = 3.14}, - new InputData() { Feature1 = false, Feature2 = "14", Feature3 = DateTime.Today, Feature4 = 0.2046}, - new InputData() { Feature1 = false, Feature2 = "23", Feature3 = DateTime.Now, Feature4 = 0.1206}, - new InputData() { Feature1 = true, Feature2 = "8904", Feature3 = DateTime.UtcNow, Feature4 = 8.09}, + new InputData() { Feature1 = true, Feature2 = "0.4", + Feature3 = DateTime.Now, Feature4 = 0.145}, + + new InputData() { Feature1 = false, Feature2 = "0.5", + Feature3 = DateTime.Today, Feature4 = 3.14}, + + new InputData() { Feature1 = false, Feature2 = "14", + Feature3 = DateTime.Today, Feature4 = 0.2046}, + + new InputData() { Feature1 = false, Feature2 = "23", + Feature3 = DateTime.Now, Feature4 = 0.1206}, + + new InputData() { Feature1 = true, Feature2 = "8904", + Feature3 = DateTime.UtcNow, Feature4 = 8.09}, }; // Convert the data to an IDataView. @@ -37,17 +48,20 @@ public static void Example() // Let's fit our pipeline to the data. var transformer = pipeline.Fit(data); - // Transforming the same data. This will add the 4 columns defined in the pipeline, containing the converted + // Transforming the same data. This will add the 4 columns defined in + // the pipeline, containing the converted // values of the initial columns. var transformedData = transformer.Transform(data); // Shape the transformed data as a strongly typed IEnumerable. - var convertedData = mlContext.Data.CreateEnumerable(transformedData, true); + var convertedData = mlContext.Data.CreateEnumerable( + transformedData, true); // Printing the results. Console.WriteLine("Converted1\t Converted2\t Converted3\t Converted4"); foreach (var item in convertedData) - Console.WriteLine($"\t{item.Converted1}\t {item.Converted2}\t\t {item.Converted3}\t {item.Converted4}"); + Console.WriteLine($"\t{item.Converted1}\t {item.Converted2}\t\t " + + $"{item.Converted3}\t {item.Converted4}"); // Transformed data. // diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/Hash.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/Hash.cs index daee047cff..7617840f6d 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/Hash.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/Hash.cs @@ -9,8 +9,8 @@ public static class Hash { public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(seed: 1); // Get a small dataset as an IEnumerable. @@ -24,30 +24,40 @@ public static void Example() var data = mlContext.Data.LoadFromEnumerable(rawData); - // Construct the pipeline that would hash the two columns and store the results in new columns. - // The first transform hashes the string column and the second transform hashes the integer column. + // Construct the pipeline that would hash the two columns and store the + // results in new columns. The first transform hashes the string column + // and the second transform hashes the integer column. // - // Hashing is not a reversible operation, so there is no way to retrive the original value from the hashed value. - // Sometimes, for debugging, or model explainability, users will need to know what values in the original columns generated - // the values in the hashed columns, since the algorithms will mostly use the hashed values for further computations. - // The Hash method will preserve the mapping from the original values to the hashed values in the Annotations of the - // newly created column (column populated with the hashed values). - // - // Setting the maximumNumberOfInverts parameters to -1 will preserve the full map. - // If that parameter is left to the default 0 value, the mapping is not preserved. - var pipeline = mlContext.Transforms.Conversion.Hash("CategoryHashed", "Category", numberOfBits: 16, maximumNumberOfInverts: -1) - .Append(mlContext.Transforms.Conversion.Hash("AgeHashed", "Age", numberOfBits: 8)); + // Hashing is not a reversible operation, so there is no way to retrive + // the original value from the hashed value. Sometimes, for debugging, + // or model explainability, users will need to know what values in the + // original columns generated the values in the hashed columns, since + // the algorithms will mostly use the hashed values for further + // computations. The Hash method will preserve the mapping from the + // original values to the hashed values in the Annotations of the newly + // created column (column populated with the hashed values). + // + // Setting the maximumNumberOfInverts parameters to -1 will preserve the + // full map. If that parameter is left to the default 0 value, the + // mapping is not preserved. + var pipeline = mlContext.Transforms.Conversion.Hash("CategoryHashed", + "Category", numberOfBits: 16, maximumNumberOfInverts: -1) + .Append(mlContext.Transforms.Conversion.Hash("AgeHashed", "Age", + numberOfBits: 8)); // Let's fit our pipeline, and then apply it to the same data. var transformer = pipeline.Fit(data); var transformedData = transformer.Transform(data); - // Convert the post transformation from the IDataView format to an IEnumerable for easy consumption. - var convertedData = mlContext.Data.CreateEnumerable(transformedData, true); + // Convert the post transformation from the IDataView format to an + // IEnumerable for easy consumption. + var convertedData = mlContext.Data.CreateEnumerable< + TransformedDataPoint>(transformedData, true); Console.WriteLine("Category CategoryHashed\t Age\t AgeHashed"); foreach (var item in convertedData) - Console.WriteLine($"{item.Category}\t {item.CategoryHashed}\t\t {item.Age}\t {item.AgeHashed}"); + Console.WriteLine($"{item.Category}\t {item.CategoryHashed}\t\t " + + $"{item.Age}\t {item.AgeHashed}"); // Expected data after the transformation. // @@ -58,20 +68,24 @@ public static void Example() // MLB 36206 18 127 // MLS 6013 14 62 - // For the Category column, where we set the maximumNumberOfInverts parameter, the names of the original categories, - // and their correspondance with the generated hash values is preserved in the Annotations in the format of indices and values. - // the indices array will have the hashed values, and the corresponding element, position-wise, in the values array will - // contain the original value. + // For the Category column, where we set the maximumNumberOfInverts + // parameter, the names of the original categories, and their + // correspondance with the generated hash values is preserved in the + // Annotations in the format of indices and values.the indices array + // will have the hashed values, and the corresponding element, + // position -wise, in the values array will contain the original value. // // See below for an example on how to retrieve the mapping. var slotNames = new VBuffer>(); - transformedData.Schema["CategoryHashed"].Annotations.GetValue("KeyValues", ref slotNames); + transformedData.Schema["CategoryHashed"].Annotations.GetValue( + "KeyValues", ref slotNames); var indices = slotNames.GetIndices(); var categoryNames = slotNames.GetValues(); for (int i = 0; i < indices.Length; i++) - Console.WriteLine($"The original value of the {indices[i]} category is {categoryNames[i]}"); + Console.WriteLine($"The original value of the {indices[i]} " + + $"category is {categoryNames[i]}"); // Output Data // diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/KeyToValueToKey.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/KeyToValueToKey.cs index 4d967647d0..82b2dea250 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/KeyToValueToKey.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/KeyToValueToKey.cs @@ -11,8 +11,8 @@ public class KeyToValueToKey { public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(); // Get a small dataset as an IEnumerable. @@ -27,25 +27,40 @@ public static void Example() // A pipeline to convert the terms of the 'Review' column in // making use of default settings. - var defaultPipeline = mlContext.Transforms.Text.TokenizeIntoWords("TokenizedText", nameof(DataPoint.Review)) - .Append(mlContext.Transforms.Conversion.MapValueToKey(nameof(TransformedData.Keys), "TokenizedText")); + var defaultPipeline = mlContext.Transforms.Text.TokenizeIntoWords( + "TokenizedText", nameof(DataPoint.Review)).Append(mlContext + .Transforms.Conversion.MapValueToKey(nameof(TransformedData.Keys), + "TokenizedText")); - // Another pipeline, that customizes the advanced settings of the ValueToKeyMappingEstimator. - // We can change the maximumNumberOfKeys to limit how many keys will get generated out of the set of words, - // and condition the order in which they get evaluated by changing keyOrdinality from the default ByOccurence (order in which they get encountered) - // to value/alphabetically. - var customizedPipeline = mlContext.Transforms.Text.TokenizeIntoWords("TokenizedText", nameof(DataPoint.Review)) - .Append(mlContext.Transforms.Conversion.MapValueToKey(nameof(TransformedData.Keys), "TokenizedText", maximumNumberOfKeys: 10, - keyOrdinality: ValueToKeyMappingEstimator.KeyOrdinality.ByValue)); + // Another pipeline, that customizes the advanced settings of the + // ValueToKeyMappingEstimator. We can change the maximumNumberOfKeys to + // limit how many keys will get generated out of the set of words, and + // condition the order in which they get evaluated by changing + // keyOrdinality from the default ByOccurence (order in which they get + // encountered) to value/alphabetically. + var customizedPipeline = mlContext.Transforms.Text.TokenizeIntoWords( + "TokenizedText", nameof(DataPoint.Review)).Append(mlContext + .Transforms.Conversion.MapValueToKey(nameof(TransformedData.Keys), + "TokenizedText", maximumNumberOfKeys: 10, keyOrdinality: + ValueToKeyMappingEstimator.KeyOrdinality.ByValue)); // The transformed data. - var transformedDataDefault = defaultPipeline.Fit(trainData).Transform(trainData); - var transformedDataCustomized = customizedPipeline.Fit(trainData).Transform(trainData); + var transformedDataDefault = defaultPipeline.Fit(trainData).Transform( + trainData); + + var transformedDataCustomized = customizedPipeline.Fit(trainData) + .Transform(trainData); // Getting the resulting data as an IEnumerable. // This will contain the newly created columns. - IEnumerable defaultData = mlContext.Data.CreateEnumerable(transformedDataDefault, reuseRowObject: false); - IEnumerable customizedData = mlContext.Data.CreateEnumerable(transformedDataCustomized, reuseRowObject: false); + IEnumerable defaultData = mlContext.Data. + CreateEnumerable(transformedDataDefault, + reuseRowObject: false); + + IEnumerable customizedData = mlContext.Data. + CreateEnumerable(transformedDataCustomized, + reuseRowObject: false); + Console.WriteLine($"Keys"); foreach (var dataRow in defaultData) Console.WriteLine($"{string.Join(',', dataRow.Keys)}"); @@ -65,13 +80,17 @@ public static void Example() // 8,2,9,7,6,4 // 3,10,0,0,0 // 3,10,0,0,0,8 - // Retrieve the original values, by appending the KeyToValue etimator to the existing pipelines - // to convert the keys back to the strings. - var pipeline = defaultPipeline.Append(mlContext.Transforms.Conversion.MapKeyToValue(nameof(TransformedData.Keys))); + // Retrieve the original values, by appending the KeyToValue etimator to + // the existing pipelines to convert the keys back to the strings. + var pipeline = defaultPipeline.Append(mlContext.Transforms.Conversion + .MapKeyToValue(nameof(TransformedData.Keys))); + transformedDataDefault = pipeline.Fit(trainData).Transform(trainData); // Preview of the DefaultColumnName column obtained. - var originalColumnBack = transformedDataDefault.GetColumn>>(transformedDataDefault.Schema[nameof(TransformedData.Keys)]); + var originalColumnBack = transformedDataDefault.GetColumn>>(transformedDataDefault.Schema[nameof( + TransformedData.Keys)]); foreach (var row in originalColumnBack) { diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/MapKeyToBinaryVector.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/MapKeyToBinaryVector.cs index e3b652f0af..57ae091124 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/MapKeyToBinaryVector.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/MapKeyToBinaryVector.cs @@ -7,14 +7,16 @@ namespace Samples.Dynamic { class MapKeyToBinaryVector { - /// This example demonstrates the use of MapKeyToVector by mapping keys to floats[] of 0 and 1, representing the number in binary format. - /// Because the ML.NET KeyType maps the missing value to zero, counting starts at 1, so the uint values - /// converted to KeyTypes will appear skewed by one. + /// This example demonstrates the use of MapKeyToVector by mapping keys to + /// floats[] of 0 and 1, representing the number in binary format. + /// Because the ML.NET KeyType maps the missing value to zero, counting + /// starts at 1, so the uint values converted to KeyTypes will appear + /// skewed by one. /// See https://github.com/dotnet/machinelearning/blob/master/docs/code/IDataViewTypeSystem.md#key-types public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(); // Get a small dataset as an IEnumerable. @@ -30,18 +32,21 @@ public static void Example() var data = mlContext.Data.LoadFromEnumerable(rawData); // Constructs the ML.net pipeline - var pipeline = mlContext.Transforms.Conversion.MapKeyToBinaryVector("TimeframeVector", "Timeframe"); + var pipeline = mlContext.Transforms.Conversion.MapKeyToBinaryVector( + "TimeframeVector", "Timeframe"); // Fits the pipeline to the data. IDataView transformedData = pipeline.Fit(data).Transform(data); // Getting the resulting data as an IEnumerable. // This will contain the newly created columns. - IEnumerable features = mlContext.Data.CreateEnumerable(transformedData, reuseRowObject: false); + IEnumerable features = mlContext.Data.CreateEnumerable< + TransformedData>(transformedData, reuseRowObject: false); Console.WriteLine($" Timeframe TimeframeVector"); foreach (var featureRow in features) - Console.WriteLine($"{featureRow.Timeframe}\t\t\t{string.Join(',', featureRow.TimeframeVector)}"); + Console.WriteLine($"{featureRow.Timeframe}\t\t\t" + + $"{string.Join(',', featureRow.TimeframeVector)}"); // Timeframe TimeframeVector // 10 0,1,0,0,1 //binary representation of 9, the original value diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/MapKeyToValueMultiColumn.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/MapKeyToValueMultiColumn.cs index 6c801c14e5..5784faaa99 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/MapKeyToValueMultiColumn.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/MapKeyToValueMultiColumn.cs @@ -5,22 +5,25 @@ namespace Samples.Dynamic { - /// This example demonstrates the use of the ValueToKeyMappingEstimator, by mapping KeyType values to the original strings. - /// For more on ML.NET KeyTypes see: https://github.com/dotnet/machinelearning/blob/master/docs/code/IDataViewTypeSystem.md#key-types + /// This example demonstrates the use of the ValueToKeyMappingEstimator, by + /// mapping KeyType values to the original strings. For more on ML.NET KeyTypes + /// see: https://github.com/dotnet/machinelearning/blob/master/docs/code/IDataViewTypeSystem.md#key-types public class MapKeyToValueMultiColumn { public static void Example() { - // Create a new context for ML.NET operations. It can be used for exception tracking and logging, - // as a catalog of available operations and as the source of randomness. - // Setting the seed to a fixed number in this example to make outputs deterministic. + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. var mlContext = new MLContext(seed: 0); // Get a small dataset as an IEnumerable. // Create a list of data examples. var examples = GenerateRandomDataPoints(1000, 10); - // Convert the examples list to an IDataView object, which is consumable by ML.NET API. + // Convert the examples list to an IDataView object, which is consumable + // by ML.NET API. var dataView = mlContext.Data.LoadFromEnumerable(examples); // Create a pipeline. @@ -28,31 +31,41 @@ public static void Example() // Convert the string labels into key types. mlContext.Transforms.Conversion.MapValueToKey("Label") // Apply StochasticDualCoordinateAscent multiclass trainer. - .Append(mlContext.MulticlassClassification.Trainers.SdcaMaximumEntropy()); + .Append(mlContext.MulticlassClassification.Trainers. + SdcaMaximumEntropy()); // Train the model and do predictions on same data set. // Typically predictions would be in a different, validation set. var dataWithPredictions = pipeline.Fit(dataView).Transform(dataView); - // At this point, the Label colum is tranformed from strings, to DataViewKeyType and - // the transformation has added the PredictedLabel column, with same DataViewKeyType as - // transformed Label column. - // MapKeyToValue would take columns with DataViewKeyType and convert them back to thier original values. + // At this point, the Label colum is tranformed from strings, to + // DataViewKeyType and the transformation has added the PredictedLabel + // column, with same DataViewKeyType as transformed Label column. + // MapKeyToValue would take columns with DataViewKeyType and convert + // them back to thier original values. var newPipeline = mlContext.Transforms.Conversion.MapKeyToValue(new[] { new InputOutputColumnPair("LabelOriginalValue","Label"), - new InputOutputColumnPair("PredictedLabelOriginalValue","PredictedLabel") + new InputOutputColumnPair("PredictedLabelOriginalValue", + "PredictedLabel") + }); - var transformedData = newPipeline.Fit(dataWithPredictions).Transform(dataWithPredictions); + var transformedData = newPipeline.Fit(dataWithPredictions).Transform( + dataWithPredictions); + // Let's iterate over first 5 items. transformedData = mlContext.Data.TakeRows(transformedData, 5); - var values = mlContext.Data.CreateEnumerable(transformedData, reuseRowObject: false); + var values = mlContext.Data.CreateEnumerable( + transformedData, reuseRowObject: false); // Printing the column names of the transformed data. - Console.WriteLine($"Label LabelOriginalValue PredictedLabel PredictedLabelOriginalValue"); + Console.WriteLine($"Label LabelOriginalValue PredictedLabel " + + $"PredictedLabelOriginalValue"); + foreach (var row in values) - Console.WriteLine($"{row.Label}\t\t{row.LabelOriginalValue}\t\t\t{row.PredictedLabel}\t\t\t{row.PredictedLabelOriginalValue}"); + Console.WriteLine($"{row.Label}\t\t{row.LabelOriginalValue}\t\t\t" + + $"{row.PredictedLabel}\t\t\t{row.PredictedLabelOriginalValue}"); // Expected output: // Label LabelOriginalValue PredictedLabel PredictedLabelOriginalValue @@ -71,7 +84,8 @@ private class DataPoint public float[] Features { get; set; } } - private static List GenerateRandomDataPoints(int count, int featureVectorLenght) + private static List GenerateRandomDataPoints(int count, + int featureVectorLenght) { var examples = new List(); var rnd = new Random(0); diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/MapKeyToVector.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/MapKeyToVector.cs index 12f5adc7fb..221ff8d2d4 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/MapKeyToVector.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/MapKeyToVector.cs @@ -8,14 +8,14 @@ namespace Samples.Dynamic { class MapKeyToVector { - /// This example demonstrates the use of MapKeyToVector by mapping keys to floats[]. - /// Because the ML.NET KeyType maps the missing value to zero, counting starts at 1, so the uint values - /// converted to KeyTypes will appear skewed by one. - /// See https://github.com/dotnet/machinelearning/blob/master/docs/code/IDataViewTypeSystem.md#key-types + /// This example demonstrates the use of MapKeyToVector by mapping keys to + /// floats[]. Because the ML.NET KeyType maps the missing value to zero, + /// counting starts at 1, so the uint values converted to KeyTypes will + /// appear skewed by one. See https://github.com/dotnet/machinelearning/blob/master/docs/code/IDataViewTypeSystem.md#key-types public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(); // Get a small dataset as an IEnumerable. @@ -28,31 +28,43 @@ public static void Example() var data = mlContext.Data.LoadFromEnumerable(rawData); - // First transform just maps key type to indicator vector. i.e. it's produces vector filled with - // zeros with size of key cardinality and set 1 to corresponding key's value index in that array. - // After that we concatenate two columns with single int values into vector of ints. - // Third transform will create vector of keys, where key type is shared across whole vector. - // Forth transfrom output data as count vector and that vector would have size equal to shared key type - // cardinality and put key counts to corresponding indexes in array. - // Fifth transform output indicator vector for each key and concatenate them together. - // Result vector would be size of key cardinality multiplied by size of original vector. - var pipeline = mlContext.Transforms.Conversion.MapKeyToVector("TimeframeVector", "Timeframe") - .Append(mlContext.Transforms.Concatenate("Parts", "PartA", "PartB")) - .Append(mlContext.Transforms.Conversion.MapValueToKey("Parts")) - .Append(mlContext.Transforms.Conversion.MapKeyToVector("PartsCount", "Parts", outputCountVector: true)) - .Append(mlContext.Transforms.Conversion.MapKeyToVector("PartsNoCount", "Parts")); + // First transform just maps key type to indicator vector. i.e. it's + // produces vector filled with zeros with size of key cardinality and + // set 1 to corresponding key's value index in that array. After that we + // concatenate two columns with single int values into vector of ints. + // Third transform will create vector of keys, where key type is shared + // across whole vector. Forth transfrom output data as count vector and + // that vector would have size equal to shared key type cardinality and + // put key counts to corresponding indexes in array. Fifth transform + // output indicator vector for each key and concatenate them together. + // Result vector would be size of key cardinality multiplied by size of + // original vector. + var pipeline = mlContext.Transforms.Conversion.MapKeyToVector( + "TimeframeVector", "Timeframe") + .Append(mlContext.Transforms.Concatenate("Parts", "PartA", "PartB")) + .Append(mlContext.Transforms.Conversion.MapValueToKey("Parts")) + .Append(mlContext.Transforms.Conversion.MapKeyToVector( + "PartsCount", "Parts", outputCountVector:true)) + .Append(mlContext.Transforms.Conversion.MapKeyToVector( + "PartsNoCount", "Parts")); // Fits the pipeline to the data. IDataView transformedData = pipeline.Fit(data).Transform(data); // Getting the resulting data as an IEnumerable. // This will contain the newly created columns. - IEnumerable features = mlContext.Data.CreateEnumerable(transformedData, reuseRowObject: false); + IEnumerable features = mlContext.Data.CreateEnumerable< + TransformedData>(transformedData, reuseRowObject: false); + + Console.WriteLine("Timeframe TimeframeVector PartsCount " + + "PartsNoCount"); - Console.WriteLine($"Timeframe TimeframeVector PartsCount PartsNoCount"); foreach (var featureRow in features) - Console.WriteLine($"{featureRow.Timeframe} {string.Join(',', featureRow.TimeframeVector.Select(x=>x))} " + - $"{string.Join(',', featureRow.PartsCount.Select(x => x))} {string.Join(',', featureRow.PartsNoCount.Select(x => x))}"); + Console.WriteLine(featureRow.Timeframe + " " + + string.Join(',', featureRow.TimeframeVector.Select(x=>x)) + " " + + string.Join(',', featureRow.PartsCount.Select(x => x)) + + " " + string.Join(',', featureRow.PartsNoCount.Select( + x => x))); // Expected output: // Timeframe TimeframeVector PartsCount PartsNoCount diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/MapKeyToVectorMultiColumn.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/MapKeyToVectorMultiColumn.cs index 7407ee67cf..606c371497 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/MapKeyToVectorMultiColumn.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/MapKeyToVectorMultiColumn.cs @@ -7,14 +7,15 @@ namespace Samples.Dynamic { public class MapKeyToVectorMultiColumn { - /// This example demonstrates the use of MapKeyToVector by mapping keys to floats[] for multiple columns at once. - /// Because the ML.NET KeyType maps the missing value to zero, counting starts at 1, so the uint values + /// This example demonstrates the use of MapKeyToVector by mapping keys to + /// floats[] for multiple columns at once. Because the ML.NET KeyType maps + /// the missing value to zero, counting starts at 1, so the uint values /// converted to KeyTypes will appear skewed by one. /// See https://github.com/dotnet/machinelearning/blob/master/docs/code/IDataViewTypeSystem.md#key-types public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(); // Get a small dataset as an IEnumerable. @@ -40,11 +41,17 @@ public static void Example() // Getting the resulting data as an IEnumerable. // This will contain the newly created columns. - IEnumerable features = mlContext.Data.CreateEnumerable(transformedData, reuseRowObject: false); + IEnumerable features = mlContext.Data.CreateEnumerable< + TransformedData>(transformedData, reuseRowObject: false); + + Console.WriteLine($" Timeframe TimeframeVector " + + $"Category CategoryVector"); - Console.WriteLine($" Timeframe TimeframeVector Category CategoryVector"); foreach (var featureRow in features) - Console.WriteLine($"{featureRow.Timeframe} {string.Join(',', featureRow.TimeframeVector)} {featureRow.Category} {string.Join(',', featureRow.CategoryVector)}"); + Console.WriteLine(featureRow.Timeframe + " " + + string.Join(',', featureRow.TimeframeVector) + " " + + featureRow.Category + " " + + string.Join(',', featureRow.CategoryVector)); // TransformedData obtained post-transformation. // @@ -59,8 +66,8 @@ public static void Example() private class DataPoint { - // The maximal value used is 9; but since 0 is reserved for missing value, - // we set the count to 10. + // The maximal value used is 9; but since 0 is reserved for missing + // value, we set the count to 10. [KeyType(10)] public uint Timeframe { get; set; } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/MapValue.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/MapValue.cs index 0b7b5ab4dd..60bea2b951 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/MapValue.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/MapValue.cs @@ -6,12 +6,13 @@ namespace Samples.Dynamic { public static class MapValue { - /// This example demonstrates the use of the ValueMappingEstimator by mapping strings to other string values, or floats to strings. - /// This is useful to map types to a category. + /// This example demonstrates the use of the ValueMappingEstimator by + /// mapping strings to other string values, or floats to strings. This is + /// useful to map types to a category. public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(); // Get a small dataset as an IEnumerable. @@ -34,7 +35,8 @@ public static void Example() timeframeMap["12-25yrs"] = "Long"; timeframeMap["25+yrs"] = "Long"; - // Construct the mapping of strings to keys(uints) for the Timeframe column. + // Construct the mapping of strings to keys(uints) for the Timeframe + // column. var timeframeKeyMap = new Dictionary(); timeframeKeyMap["0-4yrs"] = 1; timeframeKeyMap["0-5yrs"] = 1; @@ -51,22 +53,31 @@ public static void Example() scoreMap[5] = "High"; // Constructs the ML.net pipeline - var pipeline = mlContext.Transforms.Conversion.MapValue("TimeframeCategory", timeframeMap, "Timeframe") - .Append(mlContext.Transforms.Conversion.MapValue("ScoreCategory", scoreMap, "Score")) - // on the MapValue below, the treatValuesAsKeyType is set to true. The type of the Label column will be a KeyDataViewType type, - // and it can be used as input for trainers performing multiclass classification. - .Append(mlContext.Transforms.Conversion.MapValue("Label", timeframeKeyMap, "Timeframe", treatValuesAsKeyType: true)); + var pipeline = mlContext.Transforms.Conversion.MapValue( + "TimeframeCategory", timeframeMap, "Timeframe").Append(mlContext. + Transforms.Conversion.MapValue("ScoreCategory", scoreMap, "Score")) + // on the MapValue below, the treatValuesAsKeyType is set to true. + // The type of the Label column will be a KeyDataViewType type, + // and it can be used as input for trainers performing multiclass + // classification. + .Append(mlContext.Transforms.Conversion.MapValue("Label", + timeframeKeyMap, "Timeframe", treatValuesAsKeyType: true)); // Fits the pipeline to the data. IDataView transformedData = pipeline.Fit(data).Transform(data); // Getting the resulting data as an IEnumerable. // This will contain the newly created columns. - IEnumerable features = mlContext.Data.CreateEnumerable(transformedData, reuseRowObject: false); + IEnumerable features = mlContext.Data.CreateEnumerable< + TransformedData>(transformedData, reuseRowObject: false); + + Console.WriteLine(" Timeframe TimeframeCategory Label Score " + + "ScoreCategory"); - Console.WriteLine($" Timeframe TimeframeCategory Label Score ScoreCategory"); foreach (var featureRow in features) - Console.WriteLine($"{featureRow.Timeframe}\t\t{featureRow.TimeframeCategory}\t\t\t{featureRow.Label}\t\t{featureRow.Score}\t{featureRow.ScoreCategory}"); + Console.WriteLine($"{featureRow.Timeframe}\t\t" + + $"{featureRow.TimeframeCategory}\t\t\t{featureRow.Label}\t\t" + + $"{featureRow.Score}\t{featureRow.ScoreCategory}"); // TransformedData obtained post-transformation. // diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/MapValueIdvLookup.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/MapValueIdvLookup.cs index 9e1cbf2a99..5c57b4078a 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/MapValueIdvLookup.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/MapValueIdvLookup.cs @@ -6,12 +6,13 @@ namespace Samples.Dynamic { public static class MapValueIdvLookup { - /// This example demonstrates the use of MapValue by mapping floats to strings, looking up the mapping in an IDataView. - /// This is useful to map types to a grouping. + /// This example demonstrates the use of MapValue by mapping floats to + /// strings, looking up the mapping in an IDataView. This is useful to map + /// types to a grouping. public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(); // Get a small dataset as an IEnumerable. @@ -41,17 +42,22 @@ public static void Example() var lookupIdvMap = mlContext.Data.LoadFromEnumerable(lookupData); // Constructs the ValueMappingEstimator making the ML.NET pipeline - var pipeline = mlContext.Transforms.Conversion.MapValue("PriceCategory", lookupIdvMap, lookupIdvMap.Schema["Value"], lookupIdvMap.Schema["Category"], "Price"); + var pipeline = mlContext.Transforms.Conversion.MapValue("PriceCategory", + lookupIdvMap, lookupIdvMap.Schema["Value"], lookupIdvMap.Schema[ + "Category"], "Price"); - // Fits the ValueMappingEstimator and transforms the data converting the Price to PriceCategory. + // Fits the ValueMappingEstimator and transforms the data converting the + // Price to PriceCategory. IDataView transformedData = pipeline.Fit(data).Transform(data); // Getting the resulting data as an IEnumerable. - IEnumerable features = mlContext.Data.CreateEnumerable(transformedData, reuseRowObject: false); + IEnumerable features = mlContext.Data.CreateEnumerable< + TransformedData>(transformedData, reuseRowObject: false); Console.WriteLine($" Price PriceCategory"); foreach (var featureRow in features) - Console.WriteLine($"{featureRow.Price}\t\t{featureRow.PriceCategory}"); + Console.WriteLine($"{featureRow.Price}\t\t" + + $"{featureRow.PriceCategory}"); // TransformedData obtained post-transformation. // diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/MapValueToArray.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/MapValueToArray.cs index 18362ec9a0..57bff5a3b8 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/MapValueToArray.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/MapValueToArray.cs @@ -6,13 +6,16 @@ namespace Samples.Dynamic { public static class MapValueToArray { - /// This example demonstrates the use of MapValue by mapping strings to array values, which allows for mapping data to numeric arrays. - /// This functionality is useful when the generated column will serve as the Features column for a trainer. Most of the trainers take a numeric vector, as the Features column. - /// In this example, we are mapping the Timeframe data to arbitrary integer arrays. + /// This example demonstrates the use of MapValue by mapping strings to + /// array values, which allows for mapping data to numeric arrays. This + /// functionality is useful when the generated column will serve as the + /// Features column for a trainer. Most of the trainers take a numeric + /// vector, as the Features column. In this example, we are mapping the + /// Timeframe data to arbitrary integer arrays. public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(); // Get a small dataset as an IEnumerable. @@ -27,8 +30,8 @@ public static void Example() var data = mlContext.Data.LoadFromEnumerable(rawData); - // Creating a list of key-value pairs to indicate the mapping between the - // DataPoint values, and the arrays they should map to. + // Creating a list of key-value pairs to indicate the mapping between + // the DataPoint values, and the arrays they should map to. var timeframeMap = new Dictionary(); timeframeMap["0-4yrs"] = new int[] { 0, 5, 300 }; timeframeMap["0-5yrs"] = new int[] { 0, 5, 300 }; @@ -37,17 +40,22 @@ public static void Example() timeframeMap["25+yrs"] = new int[] { 12, 50, 300 }; // Constructs the ValueMappingEstimator making the ML.NET pipeline. - var pipeline = mlContext.Transforms.Conversion.MapValue("Features", timeframeMap, "Timeframe"); + var pipeline = mlContext.Transforms.Conversion.MapValue("Features", + timeframeMap, "Timeframe"); - // Fits the ValueMappingEstimator and transforms the data adding the Features column. + // Fits the ValueMappingEstimator and transforms the data adding the + // Features column. IDataView transformedData = pipeline.Fit(data).Transform(data); // Getting the resulting data as an IEnumerable. - IEnumerable featuresColumn = mlContext.Data.CreateEnumerable(transformedData, reuseRowObject: false); + IEnumerable featuresColumn = mlContext.Data + .CreateEnumerable(transformedData, reuseRowObject: + false); Console.WriteLine($"Timeframe Features"); foreach (var featureRow in featuresColumn) - Console.WriteLine($"{featureRow.Timeframe}\t\t {string.Join(",", featureRow.Features)}"); + Console.WriteLine($"{featureRow.Timeframe}\t\t " + + $"{string.Join(",", featureRow.Features)}"); // Timeframe Features // 0-4yrs 0, 5, 300 diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/MapValueToKeyMultiColumn.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/MapValueToKeyMultiColumn.cs index ccd79f4fae..e54578cb60 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/MapValueToKeyMultiColumn.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Conversion/MapValueToKeyMultiColumn.cs @@ -6,13 +6,14 @@ namespace Samples.Dynamic { public static class MapValueToKeyMultiColumn { - /// This example demonstrates the use of the ValueToKeyMappingEstimator, by mapping strings to KeyType values. - /// For more on ML.NET KeyTypes see: https://github.com/dotnet/machinelearning/blob/master/docs/code/IDataViewTypeSystem.md#key-types + /// This example demonstrates the use of the ValueToKeyMappingEstimator, by + /// mapping strings to KeyType values. For more on ML.NET KeyTypes see: + /// https://github.com/dotnet/machinelearning/blob/master/docs/code/IDataViewTypeSystem.md#key-types /// It is possible to have multiple values map to the same category. public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(); // Get a small dataset as an IEnumerable. @@ -30,19 +31,24 @@ public static void Example() new InputOutputColumnPair("StudyTimeCategory", "StudyTime"), new InputOutputColumnPair("CourseCategory", "Course") }, - keyOrdinality: Microsoft.ML.Transforms.ValueToKeyMappingEstimator.KeyOrdinality.ByValue, - addKeyValueAnnotationsAsText: true); + keyOrdinality: Microsoft.ML.Transforms.ValueToKeyMappingEstimator + .KeyOrdinality.ByValue, addKeyValueAnnotationsAsText: true); // Fits the pipeline to the data. IDataView transformedData = pipeline.Fit(data).Transform(data); // Getting the resulting data as an IEnumerable. // This will contain the newly created columns. - IEnumerable features = mlContext.Data.CreateEnumerable(transformedData, reuseRowObject: false); + IEnumerable features = mlContext.Data.CreateEnumerable< + TransformedData>(transformedData, reuseRowObject: false); + + Console.WriteLine($" StudyTime StudyTimeCategory Course " + + $"CourseCategory"); - Console.WriteLine($" StudyTime StudyTimeCategory Course CourseCategory"); foreach (var featureRow in features) - Console.WriteLine($"{featureRow.StudyTime}\t\t{featureRow.StudyTimeCategory}\t\t\t{featureRow.Course}\t\t{featureRow.CourseCategory}"); + Console.WriteLine($"{featureRow.StudyTime}\t\t" + + $"{featureRow.StudyTimeCategory}\t\t\t{featureRow.Course}\t\t" + + $"{featureRow.CourseCategory}"); // TransformedData obtained post-transformation. // @@ -52,11 +58,13 @@ public static void Example() // 12-25yrs 3 LA 3 // 0-5yrs 2 DS 2 - // If we wanted to provide the mapping, rather than letting the transform create it, - // we could do so by creating an IDataView one column containing the values to map to. - // If the values in the dataset are not found in the lookup IDataView they will get mapped to the mising value, 0. - // The keyData are shared among the columns, therefore the keys are not contiguous for the column. - // Create the lookup map data IEnumerable. + // If we wanted to provide the mapping, rather than letting the + // transform create it, we could do so by creating an IDataView one + // column containing the values to map to. If the values in the dataset + // are not found in the lookup IDataView they will get mapped to the + // mising value, 0. The keyData are shared among the columns, therefore + // the keys are not contiguous for the column. Create the lookup map + // data IEnumerable. var lookupData = new[] { new LookupMap { Key = "0-4yrs" }, new LookupMap { Key = "6-11yrs" }, @@ -70,22 +78,28 @@ public static void Example() var lookupIdvMap = mlContext.Data.LoadFromEnumerable(lookupData); // Constructs the ML.net pipeline - var pipelineWithLookupMap = mlContext.Transforms.Conversion.MapValueToKey(new[] { - new InputOutputColumnPair("StudyTimeCategory", "StudyTime"), - new InputOutputColumnPair("CourseCategory", "Course") - }, - keyData: lookupIdvMap); + var pipelineWithLookupMap = mlContext.Transforms.Conversion + .MapValueToKey(new[] { + new InputOutputColumnPair("StudyTimeCategory", "StudyTime"), + new InputOutputColumnPair("CourseCategory", "Course") + }, + keyData: lookupIdvMap); // Fits the pipeline to the data. transformedData = pipelineWithLookupMap.Fit(data).Transform(data); // Getting the resulting data as an IEnumerable. // This will contain the newly created columns. - features = mlContext.Data.CreateEnumerable(transformedData, reuseRowObject: false); + features = mlContext.Data.CreateEnumerable( + transformedData, reuseRowObject: false); + + Console.WriteLine($" StudyTime StudyTimeCategory " + + $"Course CourseCategory"); - Console.WriteLine($" StudyTime StudyTimeCategory Course CourseCategory"); foreach (var featureRow in features) - Console.WriteLine($"{featureRow.StudyTime}\t\t{featureRow.StudyTimeCategory}\t\t\t{featureRow.Course}\t\t{featureRow.CourseCategory}"); + Console.WriteLine($"{featureRow.StudyTime}\t\t" + + $"{featureRow.StudyTimeCategory}\t\t\t{featureRow.Course}\t\t" + + $"{featureRow.CourseCategory}"); // StudyTime StudyTimeCategory Course CourseCategory // 0 - 4yrs 1 CS 4 diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/CopyColumns.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/CopyColumns.cs index 212292f087..a4a7498e0c 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/CopyColumns.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/CopyColumns.cs @@ -8,40 +8,59 @@ public static class CopyColumns { public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(); // Create a small dataset as an IEnumerable. var samples = new List() { - new InputData(){ ImageId = 1, Features = new [] { 1.0f, 1.0f, 1.0f} }, - new InputData(){ ImageId = 2, Features = new [] { 2.0f, 2.0f, 2.0f} }, - new InputData(){ ImageId = 3, Features = new [] { 3.0f, 3.0f, 3.0f} }, - new InputData(){ ImageId = 4, Features = new [] { 4.0f, 4.0f, 4.0f} }, - new InputData(){ ImageId = 5, Features = new [] { 5.0f, 5.0f, 5.0f} }, - new InputData(){ ImageId = 6, Features = new [] { 6.0f, 6.0f, 6.0f} }, + new InputData(){ ImageId = 1, Features = new [] { 1.0f, 1.0f, + 1.0f } }, + + new InputData(){ ImageId = 2, Features = new [] { 2.0f, 2.0f, + 2.0f } }, + + new InputData(){ ImageId = 3, Features = new [] { 3.0f, 3.0f, + 3.0f } }, + + new InputData(){ ImageId = 4, Features = new [] { 4.0f, 4.0f, + 4.0f } }, + + new InputData(){ ImageId = 5, Features = new [] { 5.0f, 5.0f, + 5.0f } }, + + new InputData(){ ImageId = 6, Features = new [] { 6.0f, 6.0f, + 6.0f } }, }; // Convert training data to IDataView. var dataview = mlContext.Data.LoadFromEnumerable(samples); // CopyColumns is commonly used to rename columns. - // For example, if you want to train towards ImageId, and your trainer expects a "Label" column, you can - // use CopyColumns to rename ImageId to Label. Technically, the ImageId column still exists, but it won't be - // materialized unless you actually need it somewhere (e.g. if you were to save the transformed data - // without explicitly dropping the column). This is a general property of IDataView's lazy evaluation. + // For example, if you want to train towards ImageId, and your trainer + // expects a "Label" column, you can use CopyColumns to rename ImageId + // to Label. Technically, the ImageId column still exists, but it won't + // be materialized unless you actually need it somewhere (e.g. if you + // were to save the transformed data without explicitly dropping the + // column). This is a general property of IDataView's lazy evaluation. var pipeline = mlContext.Transforms.CopyColumns("Label", "ImageId"); - // Now we can transform the data and look at the output to confirm the behavior of CopyColumns. - // Don't forget that this operation doesn't actually evaluate data until we read the data below. + // Now we can transform the data and look at the output to confirm the + // behavior of CopyColumns. Don't forget that this operation doesn't + // actually evaluate data until we read the data below. var transformedData = pipeline.Fit(dataview).Transform(dataview); - // We can extract the newly created column as an IEnumerable of SampleInfertDataTransformed, the class we define below. - var rowEnumerable = mlContext.Data.CreateEnumerable(transformedData, reuseRowObject: false); + // We can extract the newly created column as an IEnumerable of + // SampleInfertDataTransformed, the class we define below. + var rowEnumerable = mlContext.Data.CreateEnumerable( + transformedData, reuseRowObject: false); + + // And finally, we can write out the rows of the dataset, looking at the + // columns of interest. + Console.WriteLine($"Label and ImageId columns obtained " + + $"post-transformation."); - // And finally, we can write out the rows of the dataset, looking at the columns of interest. - Console.WriteLine($"Label and ImageId columns obtained post-transformation."); foreach (var row in rowEnumerable) Console.WriteLine($"Label: {row.Label} ImageId: {row.ImageId}"); diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/CustomMapping.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/CustomMapping.cs index 9a0e7aabf3..c3ad19caaf 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/CustomMapping.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/CustomMapping.cs @@ -8,8 +8,8 @@ public static class CustomMapping { public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(); // Get a small dataset as an IEnumerable and convert it to an IDataView. @@ -22,21 +22,27 @@ public static void Example() }; var data = mlContext.Data.LoadFromEnumerable(samples); - // We define the custom mapping between input and output rows that will be applied by the transformation. + // We define the custom mapping between input and output rows that will + // be applied by the transformation. Action mapping = (input, output) => output.IsUnderThirty = input.Age < 30; - // Custom transformations can be used to transform data directly, or as part of a pipeline of estimators. - // Note: If contractName is null in the CustomMapping estimator, any pipeline of estimators containing it, - // cannot be saved and loaded back. - var pipeline = mlContext.Transforms.CustomMapping(mapping, contractName: null); + // Custom transformations can be used to transform data directly, or as + // part of a pipeline of estimators. Note: If contractName is null in + // the CustomMapping estimator, any pipeline of estimators containing + // it, cannot be saved and loaded back. + var pipeline = mlContext.Transforms.CustomMapping(mapping, contractName: + null); - // Now we can transform the data and look at the output to confirm the behavior of the estimator. - // This operation doesn't actually evaluate data until we read the data below. + // Now we can transform the data and look at the output to confirm the + // behavior of the estimator. This operation doesn't actually evaluate + // data until we read the data below. var transformer = pipeline.Fit(data); var transformedData = transformer.Transform(data); - var dataEnumerable = mlContext.Data.CreateEnumerable(transformedData, reuseRowObject: true); + var dataEnumerable = mlContext.Data.CreateEnumerable( + transformedData, reuseRowObject: true); + Console.WriteLine("Age\t IsUnderThirty"); foreach (var row in dataEnumerable) Console.WriteLine($"{row.Age}\t {row.IsUnderThirty}"); @@ -49,7 +55,8 @@ public static void Example() // 28 True } - // Defines only the column to be generated by the custom mapping transformation in addition to the columns already present. + // Defines only the column to be generated by the custom mapping + // transformation in addition to the columns already present. private class CustomMappingOutput { public bool IsUnderThirty { get; set; } @@ -61,7 +68,8 @@ private class InputData public float Age { get; set; } } - // Defines the schema of the transformed data, which includes the new column IsUnderThirty. + // Defines the schema of the transformed data, which includes the new column + // IsUnderThirty. private class TransformedData : InputData { public bool IsUnderThirty { get; set; } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/CustomMappingSaveAndLoad.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/CustomMappingSaveAndLoad.cs index 8bc4439190..745169ef4e 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/CustomMappingSaveAndLoad.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/CustomMappingSaveAndLoad.cs @@ -9,8 +9,8 @@ public static class CustomMappingSaveAndLoad { public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(); // Get a small dataset as an IEnumerable and convert it to an IDataView. @@ -23,23 +23,35 @@ public static void Example() }; var data = mlContext.Data.LoadFromEnumerable(samples); - // Custom transformations can be used to transform data directly, or as part of a pipeline of estimators. - var pipeline = mlContext.Transforms.CustomMapping(new IsUnderThirtyCustomAction().GetMapping(), contractName: "IsUnderThirty"); + // Custom transformations can be used to transform data directly, or as + // part of a pipeline of estimators. + var pipeline = mlContext.Transforms.CustomMapping(new + IsUnderThirtyCustomAction().GetMapping(), contractName: + "IsUnderThirty"); + var transformer = pipeline.Fit(data); - // To save and load the CustomMapping estimator, the assembly in which the custom action is defined needs to be registered in the - // environment. The following registers the assembly where IsUnderThirtyCustomAction is defined. - mlContext.ComponentCatalog.RegisterAssembly(typeof(IsUnderThirtyCustomAction).Assembly); + // To save and load the CustomMapping estimator, the assembly in which + // the custom action is defined needs to be registered in the + // environment. The following registers the assembly where + // IsUnderThirtyCustomAction is defined. + mlContext.ComponentCatalog.RegisterAssembly(typeof( + IsUnderThirtyCustomAction).Assembly); - // Now the transform pipeline can be saved and loaded through the usual MLContext method. + // Now the transform pipeline can be saved and loaded through the usual + // MLContext method. mlContext.Model.Save(transformer, data.Schema, "customTransform.zip"); - var loadedTransform = mlContext.Model.Load("customTransform.zip", out var inputSchema); + var loadedTransform = mlContext.Model.Load("customTransform.zip", out + var inputSchema); - // Now we can transform the data and look at the output to confirm the behavior of the estimator. - // This operation doesn't actually evaluate data until we read the data below. + // Now we can transform the data and look at the output to confirm the + // behavior of the estimator. This operation doesn't actually evaluate + // data until we read the data below. var transformedData = loadedTransform.Transform(data); - var dataEnumerable = mlContext.Data.CreateEnumerable(transformedData, reuseRowObject: true); + var dataEnumerable = mlContext.Data.CreateEnumerable( + transformedData, reuseRowObject: true); + Console.WriteLine("Age\tIsUnderThirty"); foreach (var row in dataEnumerable) Console.WriteLine($"{row.Age}\t {row.IsUnderThirty}"); @@ -52,21 +64,25 @@ public static void Example() // 28 True } - // The custom action needs to implement the abstract class CustomMappingFactory, and needs to have attribute - // CustomMappingFactoryAttribute with argument equal to the contractName used to define the CustomMapping estimator - // which uses the action. + // The custom action needs to implement the abstract class + // CustomMappingFactory, and needs to have attribute + // CustomMappingFactoryAttribute with argument equal to the contractName + // used to define the CustomMapping estimator which uses the action. [CustomMappingFactoryAttribute("IsUnderThirty")] - private class IsUnderThirtyCustomAction : CustomMappingFactory + private class IsUnderThirtyCustomAction : CustomMappingFactory { - // We define the custom mapping between input and output rows that will be applied by the transformation. - public static void CustomAction(InputData input, CustomMappingOutput output) - => output.IsUnderThirty = input.Age < 30; + // We define the custom mapping between input and output rows that will + // be applied by the transformation. + public static void CustomAction(InputData input, CustomMappingOutput + output) => output.IsUnderThirty = input.Age < 30; public override Action GetMapping() => CustomAction; } - // Defines only the column to be generated by the custom mapping transformation in addition to the columns already present. + // Defines only the column to be generated by the custom mapping + // transformation in addition to the columns already present. private class CustomMappingOutput { public bool IsUnderThirty { get; set; } @@ -78,7 +94,8 @@ private class InputData public float Age { get; set; } } - // Defines the schema of the transformed data, which includes the new column IsUnderThirty. + // Defines the schema of the transformed data, which includes the new column + // IsUnderThirty. private class TransformedData : InputData { public bool IsUnderThirty { get; set; } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/CustomMappingWithInMemoryCustomType.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/CustomMappingWithInMemoryCustomType.cs index 688c5d1fe5..37fd961258 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/CustomMappingWithInMemoryCustomType.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/CustomMappingWithInMemoryCustomType.cs @@ -12,34 +12,46 @@ static public void Example() { var mlContext = new MLContext(); // Build in-memory data. - var tribe = new List() { new AlienHero("ML.NET", 2, 1000, 2000, 3000, 4000, 5000, 6000, 7000) }; + var tribe = new List() { new AlienHero("ML.NET", 2, 1000, + 2000, 3000, 4000, 5000, 6000, 7000) }; // Build a ML.NET pipeline and make prediction. var tribeDataView = mlContext.Data.LoadFromEnumerable(tribe); - var pipeline = mlContext.Transforms.CustomMapping(AlienFusionProcess.GetMapping(), contractName: null); + var pipeline = mlContext.Transforms.CustomMapping(AlienFusionProcess + .GetMapping(), contractName: null); + var model = pipeline.Fit(tribeDataView); var tribeTransformed = model.Transform(tribeDataView); // Print out prediction produced by the model. - var firstAlien = mlContext.Data.CreateEnumerable(tribeTransformed, false).First(); - Console.WriteLine($"We got a super alien with name {firstAlien.Name}, age {firstAlien.Merged.Age}, " + - $"height {firstAlien.Merged.Height}, weight {firstAlien.Merged.Weight}, and {firstAlien.Merged.HandCount} hands."); + var firstAlien = mlContext.Data.CreateEnumerable( + tribeTransformed, false).First(); + + Console.WriteLine("We got a super alien with name " + firstAlien.Name + + ", age " + firstAlien.Merged.Age + ", " + "height " + firstAlien + .Merged.Height + ", weight " + firstAlien.Merged.Weight + ", and " + + firstAlien.Merged.HandCount + " hands."); // Expected output: // We got a super alien with name Super Unknown, age 4002, height 6000, weight 8000, and 10000 hands. // Create a prediction engine and print out its prediction. - var engine = mlContext.Model.CreatePredictionEngine(model); + var engine = mlContext.Model.CreatePredictionEngine(model); + var alien = new AlienHero("TEN.LM", 1, 2, 3, 4, 5, 6, 7, 8); var superAlien = engine.Predict(alien); - Console.Write($"We got a super alien with name {superAlien.Name}, age {superAlien.Merged.Age}, " + - $"height {superAlien.Merged.Height}, weight {superAlien.Merged.Weight}, and {superAlien.Merged.HandCount} hands."); + Console.Write("We got a super alien with name " + superAlien.Name + + ", age " + superAlien.Merged.Age + ", height " + + superAlien.Merged.Height + ", weight " + superAlien.Merged.Weight + + ", and " + superAlien.Merged.HandCount + " hands."); // Expected output: // We got a super alien with name Super Unknown, age 6, height 8, weight 10, and 12 hands. } - // A custom type which ML.NET doesn't know yet. Its value will be loaded as a DataView column in this test. + // A custom type which ML.NET doesn't know yet. Its value will be loaded as + // a DataView column in this test. private class AlienBody { public int Age { get; set; } @@ -67,11 +79,12 @@ public AlienTypeAttributeAttribute(int raceId) RaceId = raceId; } - // A function implicitly invoked by ML.NET when processing a custom type. - // It binds a DataViewType to a custom type plus its attributes. + // A function implicitly invoked by ML.NET when processing a custom + // type. It binds a DataViewType to a custom type plus its attributes. public override void Register() { - DataViewTypeManager.Register(new DataViewAlienBodyType(RaceId), typeof(AlienBody), new[] { this }); + DataViewTypeManager.Register(new DataViewAlienBodyType(RaceId), + typeof(AlienBody), new[] { this }); } public override bool Equals(DataViewTypeAttribute other) @@ -84,11 +97,14 @@ public override bool Equals(DataViewTypeAttribute other) public override int GetHashCode() => RaceId.GetHashCode(); } - // A custom class with a type which ML.NET doesn't know yet. Its value will be loaded as a DataView row in this test. - // It will be the input of AlienFusionProcess.MergeBody(AlienHero, SuperAlienHero). + // A custom class with a type which ML.NET doesn't know yet. Its value will + // be loaded as a DataView row in this test. It will be the input of + // AlienFusionProcess.MergeBody(AlienHero, SuperAlienHero). // - // The members One> and Two" would be mapped to different types inside ML.NET type system because they - // have different AlienTypeAttributeAttribute's. For example, the column type of One would be DataViewAlienBodyType + // The members One> and Two" would be mapped to different types inside + // ML.NET type system because they have different + // AlienTypeAttributeAttribute's. For example, the column type of One would + // be DataViewAlienBodyType // with RaceId=100. // private class AlienHero @@ -110,11 +126,13 @@ public AlienHero() public AlienHero(string name, int age, float height, float weight, int handCount, - int anotherAge, float anotherHeight, float anotherWeight, int anotherHandCount) + int anotherAge, float anotherHeight, float anotherWeight, int + anotherHandCount) { Name = "Unknown"; One = new AlienBody(age, height, weight, handCount); - Two = new AlienBody(anotherAge, anotherHeight, anotherWeight, anotherHandCount); + Two = new AlienBody(anotherAge, anotherHeight, anotherWeight, + anotherHandCount); } } @@ -142,7 +160,8 @@ public override int GetHashCode() } } - // The output type of processing AlienHero using AlienFusionProcess.MergeBody(AlienHero, SuperAlienHero). + // The output type of processing AlienHero using AlienFusionProcess + // .MergeBody(AlienHero, SuperAlienHero). private class SuperAlienHero { public string Name { get; set; } @@ -157,7 +176,8 @@ public SuperAlienHero() } } - // The implementation of custom mapping is MergeBody. It accepts AlienHero and produces SuperAlienHero. + // The implementation of custom mapping is MergeBody. It accepts AlienHero + // and produces SuperAlienHero. private class AlienFusionProcess { public static void MergeBody(AlienHero input, SuperAlienHero output) diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/DropColumns.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/DropColumns.cs index 17188bd928..110aa3b2ef 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/DropColumns.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/DropColumns.cs @@ -8,19 +8,30 @@ public static class DropColumns { public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(); // Create a small dataset as an IEnumerable. var samples = new List() { - new InputData(){ Age = 21, Gender = "Male", Education = "BS", ExtraColumn = 1 }, - new InputData(){ Age = 23, Gender = "Female", Education = "MBA", ExtraColumn = 2 }, - new InputData(){ Age = 28, Gender = "Male", Education = "PhD", ExtraColumn = 3 }, - new InputData(){ Age = 22, Gender = "Male", Education = "BS", ExtraColumn = 4 }, - new InputData(){ Age = 23, Gender = "Female", Education = "MS", ExtraColumn = 5 }, - new InputData(){ Age = 27, Gender = "Female", Education = "PhD", ExtraColumn = 6 }, + new InputData(){ Age = 21, Gender = "Male", Education = "BS", + ExtraColumn = 1 }, + + new InputData(){ Age = 23, Gender = "Female", Education = "MBA", + ExtraColumn = 2 }, + + new InputData(){ Age = 28, Gender = "Male", Education = "PhD", + ExtraColumn = 3 }, + + new InputData(){ Age = 22, Gender = "Male", Education = "BS", + ExtraColumn = 4 }, + + new InputData(){ Age = 23, Gender = "Female", Education = "MS", + ExtraColumn = 5 }, + + new InputData(){ Age = 27, Gender = "Female", Education = "PhD", + ExtraColumn = 6 }, }; // Convert training data to IDataView. @@ -30,31 +41,40 @@ public static void Example() var pipeline = mlContext.Transforms.DropColumns("ExtraColumn"); // Now we can transform the data and look at the output. - // Don't forget that this operation doesn't actually operate on data until we perform an action that requires + // Don't forget that this operation doesn't actually operate on data + // until we perform an action that requires // the data to be materialized. var transformedData = pipeline.Fit(dataview).Transform(dataview); // Now let's take a look at what the DropColumns operations did. - // We can extract the transformed data as an IEnumerable of InputData, the class we define below. - // When we try to pull out the Age, Gender, Education and ExtraColumn columns, ML.NET will raise an exception on the ExtraColumn + // We can extract the transformed data as an IEnumerable of InputData, + // the class we define below. When we try to pull out the Age, Gender, + // Education and ExtraColumn columns, ML.NET will raise an exception on + // the ExtraColumn try { - var failingRowEnumerable = mlContext.Data.CreateEnumerable(transformedData, reuseRowObject: false); + var failingRowEnumerable = mlContext.Data.CreateEnumerable< + InputData>(transformedData, reuseRowObject: false); } catch (ArgumentOutOfRangeException exception) { - Console.WriteLine($"ExtraColumn is not available, so an exception is thrown: {exception.Message}."); + Console.WriteLine($"ExtraColumn is not available, so an exception" + + $" is thrown: {exception.Message}."); } // Expected output: // ExtraColumn is not available, so an exception is thrown: Could not find column 'ExtraColumn'. // Parameter name: Schema - // And we can write a few columns out to see that the rest of the data is still available. - var rowEnumerable = mlContext.Data.CreateEnumerable(transformedData, reuseRowObject: false); + // And we can write a few columns out to see that the rest of the data + // is still available. + var rowEnumerable = mlContext.Data.CreateEnumerable( + transformedData, reuseRowObject: false); + Console.WriteLine($"The columns we didn't drop are still available."); foreach (var row in rowEnumerable) - Console.WriteLine($"Age: {row.Age} Gender: {row.Gender} Education: {row.Education}"); + Console.WriteLine($"Age: {row.Age} Gender: {row.Gender} " + + $"Education: {row.Education}"); // Expected output: // The columns we didn't drop are still available. diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/FeatureSelection/SelectFeaturesBasedOnCount.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/FeatureSelection/SelectFeaturesBasedOnCount.cs index b4d313644a..2b5ec15af4 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/FeatureSelection/SelectFeaturesBasedOnCount.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/FeatureSelection/SelectFeaturesBasedOnCount.cs @@ -9,8 +9,8 @@ public static class SelectFeaturesBasedOnCount { public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(); // Get a small dataset as an IEnumerable and convert it to an IDataView. @@ -19,7 +19,8 @@ public static void Example() // Printing the columns of the input data. Console.WriteLine($"NumericVector StringVector"); foreach (var item in rawData) - Console.WriteLine("{0,-25} {1,-25}", string.Join(",", item.NumericVector), string.Join(",", item.StringVector)); + Console.WriteLine("{0,-25} {1,-25}", string.Join(",", item + .NumericVector), string.Join(",", item.StringVector)); // NumericVector StringVector // 4,NaN,6 A,WA,Male @@ -29,19 +30,27 @@ public static void Example() var data = mlContext.Data.LoadFromEnumerable(rawData); - // We will use the SelectFeaturesBasedOnCount to retain only those slots which have at least 'count' non-default and non-missing values per slot. + // We will use the SelectFeaturesBasedOnCount to retain only those slots + // which have at least 'count' non-default and non-missing values per + // slot. var pipeline = - mlContext.Transforms.FeatureSelection.SelectFeaturesBasedOnCount(outputColumnName: "NumericVector", count: 3) // Usage on numeric column. - .Append(mlContext.Transforms.FeatureSelection.SelectFeaturesBasedOnCount(outputColumnName: "StringVector", count: 3)); // Usage on text column. + mlContext.Transforms.FeatureSelection.SelectFeaturesBasedOnCount( + outputColumnName: "NumericVector", count: 3) // Usage on numeric + // column. + .Append(mlContext.Transforms.FeatureSelection + .SelectFeaturesBasedOnCount(outputColumnName: "StringVector", + count: 3)); // Usage on text column. var transformedData = pipeline.Fit(data).Transform(data); - var convertedData = mlContext.Data.CreateEnumerable(transformedData, true); + var convertedData = mlContext.Data.CreateEnumerable( + transformedData, true); // Printing the columns of the transformed data. Console.WriteLine($"NumericVector StringVector"); foreach (var item in convertedData) - Console.WriteLine("{0,-25} {1,-25}", string.Join(",", item.NumericVector), string.Join(",", item.StringVector)); + Console.WriteLine("{0,-25} {1,-25}", string.Join(",", item. + NumericVector), string.Join(",", item.StringVector)); // NumericVector StringVector // 4,6 A,Male diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/FeatureSelection/SelectFeaturesBasedOnCountMultiColumn.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/FeatureSelection/SelectFeaturesBasedOnCountMultiColumn.cs index ae45fa375e..ff195da136 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/FeatureSelection/SelectFeaturesBasedOnCountMultiColumn.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/FeatureSelection/SelectFeaturesBasedOnCountMultiColumn.cs @@ -9,8 +9,8 @@ public static class SelectFeaturesBasedOnCountMultiColumn { public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(); // Get a small dataset as an IEnumerable and convert it to an IDataView. @@ -19,7 +19,8 @@ public static void Example() // Printing the columns of the input data. Console.WriteLine($"NumericVector StringVector"); foreach (var item in rawData) - Console.WriteLine("{0,-25} {1,-25}", string.Join(",", item.NumericVector), string.Join(",", item.StringVector)); + Console.WriteLine("{0,-25} {1,-25}", string.Join(",", item. + NumericVector), string.Join(",", item.StringVector)); // NumericVector StringVector // 4,NaN,6 A,WA,Male @@ -29,22 +30,27 @@ public static void Example() var data = mlContext.Data.LoadFromEnumerable(rawData); - // We will use the SelectFeaturesBasedOnCount transform estimator, to retain only those slots which have - // at least 'count' non-default values per slot. + // We will use the SelectFeaturesBasedOnCount transform estimator, to + // retain only those slots which have at least 'count' non-default + // values per slot. - // Multi column example. This pipeline transform two columns using the provided parameters. - var pipeline = mlContext.Transforms.FeatureSelection.SelectFeaturesBasedOnCount( - new InputOutputColumnPair[] { new InputOutputColumnPair("NumericVector"), new InputOutputColumnPair("StringVector") }, - count: 3); + // Multi column example. This pipeline transform two columns using the + // provided parameters. + var pipeline = mlContext.Transforms.FeatureSelection + .SelectFeaturesBasedOnCount(new InputOutputColumnPair[] { new + InputOutputColumnPair("NumericVector"), new InputOutputColumnPair( + "StringVector") }, count: 3); var transformedData = pipeline.Fit(data).Transform(data); - var convertedData = mlContext.Data.CreateEnumerable(transformedData, true); + var convertedData = mlContext.Data.CreateEnumerable( + transformedData, true); // Printing the columns of the transformed data. Console.WriteLine($"NumericVector StringVector"); foreach (var item in convertedData) - Console.WriteLine("{0,-25} {1,-25}", string.Join(",", item.NumericVector), string.Join(",", item.StringVector)); + Console.WriteLine("{0,-25} {1,-25}", string.Join(",", item + .NumericVector), string.Join(",", item.StringVector)); // NumericVector StringVector // 4,6 A,Male diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/FeatureSelection/SelectFeaturesBasedOnMutualInformation.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/FeatureSelection/SelectFeaturesBasedOnMutualInformation.cs index d7aa805ceb..79d1a06021 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/FeatureSelection/SelectFeaturesBasedOnMutualInformation.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/FeatureSelection/SelectFeaturesBasedOnMutualInformation.cs @@ -9,8 +9,8 @@ public static class SelectFeaturesBasedOnMutualInformation { public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(); // Get a small dataset as an IEnumerable and convert it to an IDataView. @@ -19,7 +19,8 @@ public static void Example() // Printing the columns of the input data. Console.WriteLine($"Label NumericVector"); foreach (var item in rawData) - Console.WriteLine("{0,-25} {1,-25}", item.Label, string.Join(",", item.NumericVector)); + Console.WriteLine("{0,-25} {1,-25}", item.Label, string.Join(",", + item.NumericVector)); // Label NumericVector // True 4,0,6 @@ -29,16 +30,19 @@ public static void Example() var data = mlContext.Data.LoadFromEnumerable(rawData); - // We define a MutualInformationFeatureSelectingEstimator that selects the top k slots in a feature - // vector based on highest mutual information between that slot and a specified label. - var pipeline = mlContext.Transforms.FeatureSelection.SelectFeaturesBasedOnMutualInformation( - outputColumnName: "NumericVector", labelColumnName: "Label", - slotsInOutput:2); + // We define a MutualInformationFeatureSelectingEstimator that selects + // the top k slots in a feature vector based on highest mutual + // information between that slot and a specified label. + var pipeline = mlContext.Transforms.FeatureSelection + .SelectFeaturesBasedOnMutualInformation(outputColumnName: + "NumericVector", labelColumnName: "Label", slotsInOutput:2); - // The pipeline can then be trained, using .Fit(), and the resulting transformer can be used to transform data. + // The pipeline can then be trained, using .Fit(), and the resulting + // transformer can be used to transform data. var transformedData = pipeline.Fit(data).Transform(data); - var convertedData = mlContext.Data.CreateEnumerable(transformedData, true); + var convertedData = mlContext.Data.CreateEnumerable( + transformedData, true); // Printing the columns of the transformed data. Console.WriteLine($"NumericVector"); diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/FeatureSelection/SelectFeaturesBasedOnMutualInformationMultiColumn.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/FeatureSelection/SelectFeaturesBasedOnMutualInformationMultiColumn.cs index d9543cee7e..138abcbd60 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/FeatureSelection/SelectFeaturesBasedOnMutualInformationMultiColumn.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/FeatureSelection/SelectFeaturesBasedOnMutualInformationMultiColumn.cs @@ -9,8 +9,8 @@ public static class SelectFeaturesBasedOnMutualInformationMultiColumn { public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(); // Get a small dataset as an IEnumerable and convert it to an IDataView. @@ -19,7 +19,8 @@ public static void Example() // Printing the columns of the input data. Console.WriteLine($"NumericVectorA NumericVectorB"); foreach (var item in rawData) - Console.WriteLine("{0,-25} {1,-25}", string.Join(",", item.NumericVectorA), string.Join(",", item.NumericVectorB)); + Console.WriteLine("{0,-25} {1,-25}", string.Join(",", item + .NumericVectorA), string.Join(",", item.NumericVectorB)); // NumericVectorA NumericVectorB // 4,0,6 7,8,9 @@ -29,23 +30,28 @@ public static void Example() var data = mlContext.Data.LoadFromEnumerable(rawData); - // We define a MutualInformationFeatureSelectingEstimator that selects the top k slots in a feature - // vector based on highest mutual information between that slot and a specified label. + // We define a MutualInformationFeatureSelectingEstimator that selects + // the top k slots in a feature vector based on highest mutual + // information between that slot and a specified label. - // Multi column example : This pipeline transform two columns using the provided parameters. - var pipeline = mlContext.Transforms.FeatureSelection.SelectFeaturesBasedOnMutualInformation( - new InputOutputColumnPair[] { new InputOutputColumnPair("NumericVectorA"), new InputOutputColumnPair("NumericVectorB") }, - labelColumnName: "Label", + // Multi column example : This pipeline transform two columns using the + // provided parameters. + var pipeline = mlContext.Transforms.FeatureSelection + .SelectFeaturesBasedOnMutualInformation(new InputOutputColumnPair[] + { new InputOutputColumnPair("NumericVectorA"), new + InputOutputColumnPair("NumericVectorB") }, labelColumnName: "Label", slotsInOutput: 4); var transformedData = pipeline.Fit(data).Transform(data); - var convertedData = mlContext.Data.CreateEnumerable(transformedData, true); + var convertedData = mlContext.Data.CreateEnumerable( + transformedData, true); // Printing the columns of the transformed data. Console.WriteLine($"NumericVectorA NumericVectorB"); foreach (var item in convertedData) - Console.WriteLine("{0,-25} {1,-25}", string.Join(",", item.NumericVectorA), string.Join(",", item.NumericVectorB)); + Console.WriteLine("{0,-25} {1,-25}", string.Join(",", item + .NumericVectorA), string.Join(",", item.NumericVectorB)); // NumericVectorA NumericVectorB // 4,0,6 9 diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/ImageAnalytics/ConvertToGrayScale.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/ImageAnalytics/ConvertToGrayScale.cs index 02f8fce98b..052f949eb4 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/ImageAnalytics/ConvertToGrayScale.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/ImageAnalytics/ConvertToGrayScale.cs @@ -8,17 +8,21 @@ namespace Samples.Dynamic { public static class ConvertToGrayscale { - // Sample that loads images from the file system, and converts them to grayscale. + // Sample that loads images from the file system, and converts them to + // grayscale. public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(); - // Downloading a few images, and an images.tsv file, which contains a list of the files from the dotnet/machinelearning/test/data/images/. - // If you inspect the fileSystem, after running this line, an "images" folder will be created, containing 4 images, and a .tsv file + // Downloading a few images, and an images.tsv file, which contains a + // list of the files from the dotnet/machinelearning/test/data/images/. + // If you inspect the fileSystem, after running this line, an "images" + // folder will be created, containing 4 images, and a .tsv file // enumerating the images. - var imagesDataFile = Microsoft.ML.SamplesUtils.DatasetUtils.DownloadImages(); + var imagesDataFile = Microsoft.ML.SamplesUtils.DatasetUtils + .DownloadImages(); // Preview of the content of the images.tsv file // @@ -39,8 +43,10 @@ public static void Example() var imagesFolder = Path.GetDirectoryName(imagesDataFile); // Image loading pipeline. - var pipeline = mlContext.Transforms.LoadImages("ImageObject", imagesFolder, "ImagePath") - .Append(mlContext.Transforms.ConvertToGrayscale("Grayscale", "ImageObject")); + var pipeline = mlContext.Transforms.LoadImages("ImageObject", + imagesFolder, "ImagePath") + .Append(mlContext.Transforms.ConvertToGrayscale("Grayscale", + "ImageObject")); var transformedData = pipeline.Fit(data).Transform(data); @@ -54,20 +60,32 @@ public static void Example() private static void PrintColumns(IDataView transformedData) { - Console.WriteLine("{0, -25} {1, -25} {2, -25} {3, -25}", "ImagePath", "Name", "ImageObject", "Grayscale"); - using (var cursor = transformedData.GetRowCursor(transformedData.Schema)) + Console.WriteLine("{0, -25} {1, -25} {2, -25} {3, -25}", "ImagePath", + "Name", "ImageObject", "Grayscale"); + + using (var cursor = transformedData.GetRowCursor(transformedData + .Schema)) { - // Note that it is best to get the getters and values *before* iteration, so as to faciliate buffer - // sharing (if applicable), and column-type validation once, rather than many times. + // Note that it is best to get the getters and values *before* + // iteration, so as to faciliate buffer sharing (if applicable), and + // column -type validation once, rather than many times. ReadOnlyMemory imagePath = default; ReadOnlyMemory name = default; Bitmap imageObject = null; Bitmap grayscaleImageObject = null; - var imagePathGetter = cursor.GetGetter>(cursor.Schema["ImagePath"]); - var nameGetter = cursor.GetGetter>(cursor.Schema["Name"]); - var imageObjectGetter = cursor.GetGetter(cursor.Schema["ImageObject"]); - var grayscaleGetter = cursor.GetGetter(cursor.Schema["Grayscale"]); + var imagePathGetter = cursor.GetGetter>(cursor + .Schema["ImagePath"]); + + var nameGetter = cursor.GetGetter>(cursor + .Schema["Name"]); + + var imageObjectGetter = cursor.GetGetter(cursor.Schema[ + "ImageObject"]); + + var grayscaleGetter = cursor.GetGetter(cursor.Schema[ + "Grayscale"]); + while (cursor.MoveNext()) { imagePathGetter(ref imagePath); @@ -75,7 +93,9 @@ private static void PrintColumns(IDataView transformedData) imageObjectGetter(ref imageObject); grayscaleGetter(ref grayscaleImageObject); - Console.WriteLine("{0, -25} {1, -25} {2, -25} {3, -25}", imagePath, name, imageObject.PhysicalDimension, grayscaleImageObject.PhysicalDimension); + Console.WriteLine("{0, -25} {1, -25} {2, -25} {3, -25}", + imagePath, name, imageObject.PhysicalDimension, + grayscaleImageObject.PhysicalDimension); } // Dispose the image. diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/ImageAnalytics/ConvertToGrayScaleInMemory.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/ImageAnalytics/ConvertToGrayScaleInMemory.cs index 883dfa5dc1..3bb0586cc0 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/ImageAnalytics/ConvertToGrayScaleInMemory.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/ImageAnalytics/ConvertToGrayScaleInMemory.cs @@ -11,13 +11,16 @@ static public void Example() { var mlContext = new MLContext(); // Create an image list. - var images = new[] { new ImageDataPoint(2, 3, Color.Blue), new ImageDataPoint(2, 3, Color.Red) }; + var images = new[] { new ImageDataPoint(2, 3, Color.Blue), new + ImageDataPoint(2, 3, Color.Red) }; - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var data = mlContext.Data.LoadFromEnumerable(images); // Convert image to gray scale. - var pipeline = mlContext.Transforms.ConvertToGrayscale("GrayImage", "Image"); + var pipeline = mlContext.Transforms.ConvertToGrayscale("GrayImage", + "Image"); // Fit the model. var model = pipeline.Fit(data); @@ -26,7 +29,8 @@ static public void Example() var transformedData = model.Transform(data); // Load images in DataView back to Enumerable. - var transformedDataPoints = mlContext.Data.CreateEnumerable(transformedData, false); + var transformedDataPoints = mlContext.Data.CreateEnumerable< + ImageDataPoint>(transformedData, false); // Print out input and output pixels. foreach (var dataPoint in transformedDataPoints) @@ -39,7 +43,8 @@ static public void Example() { var pixel = image.GetPixel(x, y); var grayPixel = grayImage.GetPixel(x, y); - Console.WriteLine($"The original pixel is {pixel} and its pixel in gray is {grayPixel}"); + Console.WriteLine($"The original pixel is {pixel} and its" + + $"pixel in gray is {grayPixel}"); } } } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/ImageAnalytics/ConvertToImage.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/ImageAnalytics/ConvertToImage.cs index 04ccb71093..f0f58c7e45 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/ImageAnalytics/ConvertToImage.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/ImageAnalytics/ConvertToImage.cs @@ -14,22 +14,25 @@ public static class ConvertToImage private const int numberOfChannels = 3; private const int inputSize = imageHeight * imageWidth * numberOfChannels; - // Sample that shows how an input array (of doubles) can be used to interop with image related estimators in ML.NET. + // Sample that shows how an input array (of doubles) can be used to interop + // with image related estimators in ML.NET. public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(); // Create a list of training data points. var dataPoints = GenerateRandomDataPoints(4); - // Convert the list of data points to an IDataView object, which is consumable by ML.NET API. + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. var data = mlContext.Data.LoadFromEnumerable(dataPoints); // Image loading pipeline. - var pipeline = mlContext.Transforms.ConvertToImage(imageHeight, imageWidth, "Image", "Features") - .Append(mlContext.Transforms.ExtractPixels("Pixels", "Image")); + var pipeline = mlContext.Transforms.ConvertToImage(imageHeight, + imageWidth, "Image", "Features") + .Append(mlContext.Transforms.ExtractPixels("Pixels", "Image")); var transformedData = pipeline.Fit(data).Transform(data); @@ -45,18 +48,25 @@ public static void Example() private static void PrintColumns(IDataView transformedData) { - Console.WriteLine("{0, -25} {1, -25} {2, -25}", "Features", "Image", "Pixels"); + Console.WriteLine("{0, -25} {1, -25} {2, -25}", "Features", "Image", + "Pixels"); - using (var cursor = transformedData.GetRowCursor(transformedData.Schema)) + using (var cursor = transformedData.GetRowCursor(transformedData + .Schema)) { - // Note that it is best to get the getters and values *before* iteration, so as to faciliate buffer - // sharing (if applicable), and column-type validation once, rather than many times. + // Note that it is best to get the getters and values *before* + // iteration, so as to faciliate buffer sharing (if applicable), and + // column -type validation once, rather than many times. VBuffer features = default; VBuffer pixels = default; Bitmap imageObject = null; - var featuresGetter = cursor.GetGetter>(cursor.Schema["Features"]); - var pixelsGetter = cursor.GetGetter>(cursor.Schema["Pixels"]); + var featuresGetter = cursor.GetGetter>(cursor.Schema[ + "Features"]); + + var pixelsGetter = cursor.GetGetter>(cursor.Schema[ + "Pixels"]); + var imageGetter = cursor.GetGetter(cursor.Schema["Image"]); while (cursor.MoveNext()) { @@ -65,8 +75,10 @@ private static void PrintColumns(IDataView transformedData) pixelsGetter(ref pixels); imageGetter(ref imageObject); - Console.WriteLine("{0, -25} {1, -25} {2, -25}", string.Join(",", features.DenseValues().Take(5)) + "...", - imageObject.PhysicalDimension, string.Join(",", pixels.DenseValues().Take(5)) + "..."); + Console.WriteLine("{0, -25} {1, -25} {2, -25}", string.Join(",", + features.DenseValues().Take(5)) + "...", imageObject + .PhysicalDimension, string.Join(",", pixels.DenseValues() + .Take(5)) + "..."); } // Dispose the image. @@ -80,12 +92,14 @@ private class DataPoint public float[] Features { get; set; } } - private static IEnumerable GenerateRandomDataPoints(int count, int seed = 0) + private static IEnumerable GenerateRandomDataPoints(int count, + int seed = 0) { var random = new Random(seed); for (int i = 0; i < count; i++) - yield return new DataPoint { Features = Enumerable.Repeat(0, inputSize).Select(x => (float)random.Next(0, 256)).ToArray() }; + yield return new DataPoint { Features = Enumerable.Repeat(0, + inputSize).Select(x => (float)random.Next(0, 256)).ToArray() }; } } } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/ImageAnalytics/DnnFeaturizeImage.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/ImageAnalytics/DnnFeaturizeImage.cs index 97929a38cc..99d00b900c 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/ImageAnalytics/DnnFeaturizeImage.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/ImageAnalytics/DnnFeaturizeImage.cs @@ -9,16 +9,20 @@ public static class DnnFeaturizeImage { public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(); - // Downloading a few images, and an images.tsv file, which contains a list of the files from the dotnet/machinelearning/test/data/images/. - // If you inspect the fileSystem, after running this line, an "images" folder will be created, containing 4 images, and a .tsv file + // Downloading a few images, and an images.tsv file, which contains a + // list of the files from the dotnet/machinelearning/test/data/images/. + // If you inspect the fileSystem, after running this line, an "images" + // folder will be created, containing 4 images, and a .tsv file // enumerating the images. - var imagesDataFile = Microsoft.ML.SamplesUtils.DatasetUtils.DownloadImages(); + var imagesDataFile = Microsoft.ML.SamplesUtils.DatasetUtils + .DownloadImages(); - // Preview of the content of the images.tsv file, which lists the images to operate on + // Preview of the content of the images.tsv file, which lists the images + // to operate on // // imagePath imageType // tomato.bmp tomato @@ -40,16 +44,22 @@ public static void Example() // Installing the Microsoft.ML.DNNImageFeaturizer packages copies the models in the // `DnnImageModels` folder. // Image loading pipeline. - var pipeline = mlContext.Transforms.LoadImages("ImageObject", imagesFolder, "ImagePath") - .Append(mlContext.Transforms.ResizeImages("ImageObject", imageWidth: 224, imageHeight: 224)) - .Append(mlContext.Transforms.ExtractPixels("Pixels", "ImageObject")) - .Append(mlContext.Transforms.DnnFeaturizeImage("FeaturizedImage", m => m.ModelSelector.ResNet18(mlContext, m.OutputColumn, m.InputColumn), "Pixels")); + var pipeline = mlContext.Transforms.LoadImages("ImageObject", + imagesFolder, "ImagePath") + .Append(mlContext.Transforms.ResizeImages("ImageObject", imageWidth: + 224, imageHeight: 224)) + .Append(mlContext.Transforms.ExtractPixels("Pixels", "ImageObject")) + .Append(mlContext.Transforms.DnnFeaturizeImage("FeaturizedImage", + m => m.ModelSelector.ResNet18(mlContext, m.OutputColumn, m + .InputColumn), "Pixels")); var transformedData = pipeline.Fit(data).Transform(data); - var FeaturizedImageColumnsPerRow = transformedData.GetColumn("FeaturizedImage").ToArray(); + var FeaturizedImageColumnsPerRow = transformedData.GetColumn( + "FeaturizedImage").ToArray(); - // Preview of FeaturizedImageColumnsPerRow for the first row, FeaturizedImageColumnsPerRow[0] + // Preview of FeaturizedImageColumnsPerRow for the first row, + // FeaturizedImageColumnsPerRow[0] // // 0.696136236 // 0.2661711 diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/ImageAnalytics/ExtractPixels.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/ImageAnalytics/ExtractPixels.cs index 4f8793fd7a..e608e484df 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/ImageAnalytics/ExtractPixels.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/ImageAnalytics/ExtractPixels.cs @@ -9,18 +9,22 @@ namespace Samples.Dynamic { public static class ExtractPixels { - // Sample that loads the images from the file system, resizes them (ExtractPixels requires a resizing operation), and extracts the - // values of the pixels as a vector. + // Sample that loads the images from the file system, resizes them ( + // ExtractPixels requires a resizing operation), and extracts the values of + // the pixels as a vector. public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(); - // Downloading a few images, and an images.tsv file, which contains a list of the files from the dotnet/machinelearning/test/data/images/. - // If you inspect the fileSystem, after running this line, an "images" folder will be created, containing 4 images, and a .tsv file + // Downloading a few images, and an images.tsv file, which contains a + // list of the files from the dotnet/machinelearning/test/data/images/. + // If you inspect the fileSystem, after running this line, an "images" + // folder will be created, containing 4 images, and a .tsv file // enumerating the images. - var imagesDataFile = Microsoft.ML.SamplesUtils.DatasetUtils.DownloadImages(); + var imagesDataFile = Microsoft.ML.SamplesUtils.DatasetUtils + .DownloadImages(); // Preview of the content of the images.tsv file // @@ -41,9 +45,13 @@ public static void Example() var imagesFolder = Path.GetDirectoryName(imagesDataFile); // Image loading pipeline. - var pipeline = mlContext.Transforms.LoadImages("ImageObject", imagesFolder, "ImagePath") - .Append(mlContext.Transforms.ResizeImages("ImageObjectResized", inputColumnName: "ImageObject", imageWidth: 100, imageHeight: 100)) - .Append(mlContext.Transforms.ExtractPixels("Pixels", "ImageObjectResized")); + var pipeline = mlContext.Transforms.LoadImages("ImageObject", + imagesFolder, "ImagePath") + .Append(mlContext.Transforms.ResizeImages("ImageObjectResized", + inputColumnName: "ImageObject", imageWidth: 100, imageHeight: + 100)) + .Append(mlContext.Transforms.ExtractPixels("Pixels", + "ImageObjectResized")); var transformedData = pipeline.Fit(data).Transform(data); @@ -59,12 +67,15 @@ public static void Example() private static void PrintColumns(IDataView transformedData) { - Console.WriteLine("{0, -25} {1, -25} {2, -25} {3, -25} {4, -25}", "ImagePath", "Name", "ImageObject", "ImageObjectResized", "Pixels"); + Console.WriteLine("{0, -25} {1, -25} {2, -25} {3, -25} {4, -25}", + "ImagePath", "Name", "ImageObject", "ImageObjectResized", "Pixels"); - using (var cursor = transformedData.GetRowCursor(transformedData.Schema)) + using (var cursor = transformedData.GetRowCursor(transformedData + .Schema)) { - // Note that it is best to get the getters and values *before* iteration, so as to faciliate buffer - // sharing (if applicable), and column-type validation once, rather than many times. + // Note that it is best to get the getters and values *before* + // iteration, so as to faciliate buffer sharing (if applicable), and + // column -type validation once, rather than many times. ReadOnlyMemory imagePath = default; ReadOnlyMemory name = default; @@ -72,11 +83,21 @@ private static void PrintColumns(IDataView transformedData) Bitmap resizedImageObject = null; VBuffer pixels = default; - var imagePathGetter = cursor.GetGetter>(cursor.Schema["ImagePath"]); - var nameGetter = cursor.GetGetter>(cursor.Schema["Name"]); - var imageObjectGetter = cursor.GetGetter(cursor.Schema["ImageObject"]); - var resizedImageGetter = cursor.GetGetter(cursor.Schema["ImageObjectResized"]); - var pixelsGetter = cursor.GetGetter>(cursor.Schema["Pixels"]); + var imagePathGetter = cursor.GetGetter>(cursor + .Schema["ImagePath"]); + + var nameGetter = cursor.GetGetter>(cursor + .Schema["Name"]); + + var imageObjectGetter = cursor.GetGetter(cursor.Schema[ + "ImageObject"]); + + var resizedImageGetter = cursor.GetGetter(cursor.Schema[ + "ImageObjectResized"]); + + var pixelsGetter = cursor.GetGetter>(cursor.Schema[ + "Pixels"]); + while (cursor.MoveNext()) { @@ -86,8 +107,10 @@ private static void PrintColumns(IDataView transformedData) resizedImageGetter(ref resizedImageObject); pixelsGetter(ref pixels); - Console.WriteLine("{0, -25} {1, -25} {2, -25} {3, -25} {4, -25}", imagePath, name, - imageObject.PhysicalDimension, resizedImageObject.PhysicalDimension, string.Join(",", pixels.DenseValues().Take(5)) + "..."); + Console.WriteLine("{0, -25} {1, -25} {2, -25} {3, -25} " + + "{4, -25}", imagePath, name, imageObject.PhysicalDimension, + resizedImageObject.PhysicalDimension, string.Join(",", + pixels.DenseValues().Take(5)) + "..."); } // Dispose the image. diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/ImageAnalytics/LoadImages.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/ImageAnalytics/LoadImages.cs index 6dba6729e2..6d38dc6e7c 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/ImageAnalytics/LoadImages.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/ImageAnalytics/LoadImages.cs @@ -11,14 +11,17 @@ public static class LoadImages // Loads the images of the imagesFolder into an IDataView. public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(); - // Downloading a few images, and an images.tsv file, which contains a list of the files from the dotnet/machinelearning/test/data/images/. - // If you inspect the fileSystem, after running this line, an "images" folder will be created, containing 4 images, and a .tsv file + // Downloading a few images, and an images.tsv file, which contains a + // list of the files from the dotnet/machinelearning/test/data/images/. + // If you inspect the fileSystem, after running this line, an "images" + // folder will be created, containing 4 images, and a .tsv file // enumerating the images. - var imagesDataFile = Microsoft.ML.SamplesUtils.DatasetUtils.DownloadImages(); + var imagesDataFile = Microsoft.ML.SamplesUtils.DatasetUtils + .DownloadImages(); // Preview of the content of the images.tsv file // @@ -39,7 +42,8 @@ public static void Example() var imagesFolder = Path.GetDirectoryName(imagesDataFile); // Image loading pipeline. - var pipeline = mlContext.Transforms.LoadImages("ImageObject", imagesFolder, "ImagePath"); + var pipeline = mlContext.Transforms.LoadImages("ImageObject", + imagesFolder, "ImagePath"); var transformedData = pipeline.Fit(data).Transform(data); @@ -55,18 +59,28 @@ public static void Example() private static void PrintColumns(IDataView transformedData) { // The transformedData IDataView contains the loaded images now. - Console.WriteLine("{0, -25} {1, -25} {2, -25}", "ImagePath", "Name", "ImageObject"); - using (var cursor = transformedData.GetRowCursor(transformedData.Schema)) + Console.WriteLine("{0, -25} {1, -25} {2, -25}", "ImagePath", "Name", + "ImageObject"); + + using (var cursor = transformedData.GetRowCursor(transformedData + .Schema)) { - // Note that it is best to get the getters and values *before* iteration, so as to faciliate buffer - // sharing (if applicable), and column-type validation once, rather than many times. + // Note that it is best to get the getters and values *before* + // iteration, so as to faciliate buffer sharing (if applicable), + // and column-type validation once, rather than many times. ReadOnlyMemory imagePath = default; ReadOnlyMemory name = default; Bitmap imageObject = null; - var imagePathGetter = cursor.GetGetter>(cursor.Schema["ImagePath"]); - var nameGetter = cursor.GetGetter>(cursor.Schema["Name"]); - var imageObjectGetter = cursor.GetGetter(cursor.Schema["ImageObject"]); + var imagePathGetter = cursor.GetGetter>(cursor + .Schema["ImagePath"]); + + var nameGetter = cursor.GetGetter>(cursor + .Schema["Name"]); + + var imageObjectGetter = cursor.GetGetter(cursor.Schema[ + "ImageObject"]); + while (cursor.MoveNext()) { @@ -74,7 +88,8 @@ private static void PrintColumns(IDataView transformedData) nameGetter(ref name); imageObjectGetter(ref imageObject); - Console.WriteLine("{0, -25} {1, -25} {2, -25}", imagePath, name, imageObject.PhysicalDimension); + Console.WriteLine("{0, -25} {1, -25} {2, -25}", imagePath, name, + imageObject.PhysicalDimension); } // Dispose the image. diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/ImageAnalytics/ResizeImages.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/ImageAnalytics/ResizeImages.cs index 11b8905c90..696895993f 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/ImageAnalytics/ResizeImages.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/ImageAnalytics/ResizeImages.cs @@ -11,14 +11,17 @@ public static class ResizeImages // Example on how to load the images from the file system, and resize them. public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(); - // Downloading a few images, and an images.tsv file, which contains a list of the files from the dotnet/machinelearning/test/data/images/. - // If you inspect the fileSystem, after running this line, an "images" folder will be created, containing 4 images, and a .tsv file + // Downloading a few images, and an images.tsv file, which contains a + // list of the files from the dotnet/machinelearning/test/data/images/. + // If you inspect the fileSystem, after running this line, an "images" + // folder will be created, containing 4 images, and a .tsv file // enumerating the images. - var imagesDataFile = Microsoft.ML.SamplesUtils.DatasetUtils.DownloadImages(); + var imagesDataFile = Microsoft.ML.SamplesUtils.DatasetUtils + .DownloadImages(); // Preview of the content of the images.tsv file // @@ -39,8 +42,10 @@ public static void Example() var imagesFolder = Path.GetDirectoryName(imagesDataFile); // Image loading pipeline. - var pipeline = mlContext.Transforms.LoadImages("ImageObject", imagesFolder, "ImagePath") - .Append(mlContext.Transforms.ResizeImages("ImageObjectResized", inputColumnName: "ImageObject", imageWidth: 100, imageHeight: 100)); + var pipeline = mlContext.Transforms.LoadImages("ImageObject", + imagesFolder, "ImagePath") + .Append(mlContext.Transforms.ResizeImages("ImageObjectResized", + inputColumnName: "ImageObject", imageWidth: 100, imageHeight: 100)); var transformedData = pipeline.Fit(data).Transform(data); // The transformedData IDataView contains the resized images now. @@ -57,20 +62,32 @@ public static void Example() private static void PrintColumns(IDataView transformedData) { - Console.WriteLine("{0, -25} {1, -25} {2, -25} {3, -25}", "ImagePath", "Name", "ImageObject", "ImageObjectResized"); - using (var cursor = transformedData.GetRowCursor(transformedData.Schema)) + Console.WriteLine("{0, -25} {1, -25} {2, -25} {3, -25}", "ImagePath", + "Name", "ImageObject", "ImageObjectResized"); + + using (var cursor = transformedData.GetRowCursor(transformedData + .Schema)) { - // Note that it is best to get the getters and values *before* iteration, so as to faciliate buffer - // sharing (if applicable), and column-type validation once, rather than many times. + // Note that it is best to get the getters and values *before* + // iteration, so as to faciliate buffer sharing (if applicable), and + // column -type validation once, rather than many times. ReadOnlyMemory imagePath = default; ReadOnlyMemory name = default; Bitmap imageObject = null; Bitmap resizedImageObject = null; - var imagePathGetter = cursor.GetGetter>(cursor.Schema["ImagePath"]); - var nameGetter = cursor.GetGetter>(cursor.Schema["Name"]); - var imageObjectGetter = cursor.GetGetter(cursor.Schema["ImageObject"]); - var resizedImageGetter = cursor.GetGetter(cursor.Schema["ImageObjectResized"]); + var imagePathGetter = cursor.GetGetter>(cursor + .Schema["ImagePath"]); + + var nameGetter = cursor.GetGetter>(cursor + .Schema["Name"]); + + var imageObjectGetter = cursor.GetGetter(cursor.Schema[ + "ImageObject"]); + + var resizedImageGetter = cursor.GetGetter(cursor.Schema[ + "ImageObjectResized"]); + while (cursor.MoveNext()) { imagePathGetter(ref imagePath); @@ -78,8 +95,9 @@ private static void PrintColumns(IDataView transformedData) imageObjectGetter(ref imageObject); resizedImageGetter(ref resizedImageObject); - Console.WriteLine("{0, -25} {1, -25} {2, -25} {3, -25}", imagePath, name, - imageObject.PhysicalDimension, resizedImageObject.PhysicalDimension); + Console.WriteLine("{0, -25} {1, -25} {2, -25} {3, -25}", + imagePath, name, imageObject.PhysicalDimension, + resizedImageObject.PhysicalDimension); } // Dispose the image. diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/IndicateMissingValues.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/IndicateMissingValues.cs index 7e880b90d5..b4eb2bf21e 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/IndicateMissingValues.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/IndicateMissingValues.cs @@ -10,8 +10,8 @@ public static class IndicateMissingValues { public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(); // Get a small dataset as an IEnumerable and convert it to an IDataView. @@ -23,21 +23,29 @@ public static void Example() }; var data = mlContext.Data.LoadFromEnumerable(samples); - // IndicateMissingValues is used to create a boolean containing 'true' where the value in the - // input column is missing. For floats and doubles, missing values are represented as NaN. - var pipeline = mlContext.Transforms.IndicateMissingValues("MissingIndicator", "Features"); + // IndicateMissingValues is used to create a boolean containing 'true' + // where the value in the input column is missing. For floats and + // doubles, missing values are represented as NaN. + var pipeline = mlContext.Transforms.IndicateMissingValues( + "MissingIndicator", "Features"); - // Now we can transform the data and look at the output to confirm the behavior of the estimator. - // This operation doesn't actually evaluate data until we read the data below. + // Now we can transform the data and look at the output to confirm the + // behavior of the estimator. This operation doesn't actually evaluate + // data until we read the data below. var tansformer = pipeline.Fit(data); var transformedData = tansformer.Transform(data); - // We can extract the newly created column as an IEnumerable of SampleDataTransformed, the class we define below. - var rowEnumerable = mlContext.Data.CreateEnumerable(transformedData, reuseRowObject: false); + // We can extract the newly created column as an IEnumerable of + // SampleDataTransformed, the class we define below. + var rowEnumerable = mlContext.Data.CreateEnumerable< + SampleDataTransformed>(transformedData, reuseRowObject: false); - // And finally, we can write out the rows of the dataset, looking at the columns of interest. + // And finally, we can write out the rows of the dataset, looking at the + // columns of interest. foreach (var row in rowEnumerable) - Console.WriteLine($"Features: [{string.Join(", ", row.Features)}]\t MissingIndicator: [{string.Join(", ", row.MissingIndicator)}]"); + Console.WriteLine("Features: [" + string.Join(", ", row.Features) + + "]\t MissingIndicator: [" + string.Join(", ", row + .MissingIndicator) + "]"); // Expected output: // Features: [1, 1, 0] MissingIndicator: [False, False, False] diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/IndicateMissingValuesMultiColumn.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/IndicateMissingValuesMultiColumn.cs index 830fb9d047..38750ed0a6 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/IndicateMissingValuesMultiColumn.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/IndicateMissingValuesMultiColumn.cs @@ -9,40 +9,53 @@ public static class IndicateMissingValuesMultiColumn { public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(); // Get a small dataset as an IEnumerable and convert it to an IDataView. var samples = new List() { - new DataPoint(){ Features1 = new float[3] {1, 1, 0}, Features2 = new float[2] {1, 1} }, - new DataPoint(){ Features1 = new float[3] {0, float.NaN, 1}, Features2 = new float[2] {float.NaN, 1} }, - new DataPoint(){ Features1 = new float[3] {-1, float.NaN, -3}, Features2 = new float[2] {1, float.PositiveInfinity} }, + new DataPoint(){ Features1 = new float[3] {1, 1, 0}, Features2 = + new float[2] {1, 1} }, + + new DataPoint(){ Features1 = new float[3] {0, float.NaN, 1}, + Features2 = new float[2] {float.NaN, 1} }, + + new DataPoint(){ Features1 = new float[3] {-1, float.NaN, -3}, + Features2 = new float[2] {1, float.PositiveInfinity} }, }; var data = mlContext.Data.LoadFromEnumerable(samples); - // IndicateMissingValues is used to create a boolean containing 'true' where the value in the - // input column is missing. For floats and doubles, missing values are NaN. - // We can use an array of InputOutputColumnPair to apply the MissingValueIndicatorEstimator + // IndicateMissingValues is used to create a boolean containing 'true' + // where the value in the input column is missing. For floats and + // doubles, missing values are NaN. We can use an array of + // InputOutputColumnPair to apply the MissingValueIndicatorEstimator // to multiple columns in one pass over the data. var pipeline = mlContext.Transforms.IndicateMissingValues(new[] { new InputOutputColumnPair("MissingIndicator1", "Features1"), new InputOutputColumnPair("MissingIndicator2", "Features2") }); - // Now we can transform the data and look at the output to confirm the behavior of the estimator. - // This operation doesn't actually evaluate data until we read the data below. + // Now we can transform the data and look at the output to confirm the + // behavior of the estimator. This operation doesn't actually evaluate + // data until we read the data below. var tansformer = pipeline.Fit(data); var transformedData = tansformer.Transform(data); - // We can extract the newly created column as an IEnumerable of SampleDataTransformed, the class we define below. - var rowEnumerable = mlContext.Data.CreateEnumerable(transformedData, reuseRowObject: false); + // We can extract the newly created column as an IEnumerable of + // SampleDataTransformed, the class we define below. + var rowEnumerable = mlContext.Data.CreateEnumerable< + SampleDataTransformed>(transformedData, reuseRowObject: false); - // And finally, we can write out the rows of the dataset, looking at the columns of interest. + // And finally, we can write out the rows of the dataset, looking at the + // columns of interest. foreach (var row in rowEnumerable) - Console.WriteLine($"Features1: [{string.Join(", ", row.Features1)}]\t MissingIndicator1: [{string.Join(", ", row.MissingIndicator1)}]\t " + - $"Features2: [{string.Join(", ", row.Features2)}]\t MissingIndicator2: [{string.Join(", ", row.MissingIndicator2)}]"); + Console.WriteLine("Features1: [" + string.Join(", ", row + .Features1) + "]\t MissingIndicator1: [" + string.Join(", ", + row.MissingIndicator1) + "]\t Features2: [" + string.Join(", ", + row.Features2) + "]\t MissingIndicator2: [" + string.Join(", ", + row.MissingIndicator2) + "]"); // Expected output: // Features1: [1, 1, 0] MissingIndicator1: [False, False, False] Features2: [1, 1] MissingIndicator2: [False, False] diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/NormalizeBinning.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/NormalizeBinning.cs index ba23d87e2f..27315443c7 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/NormalizeBinning.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/NormalizeBinning.cs @@ -12,7 +12,8 @@ public class NormalizeBinning { public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, // as well as the source of randomness. var mlContext = new MLContext(); var samples = new List() @@ -22,60 +23,91 @@ public static void Example() new DataPoint(){ Features = new float[4] { 4, 0, 1, 0} }, new DataPoint(){ Features = new float[4] { 2,-1,-1, 1} } }; - // Convert training data to IDataView, the general data type used in ML.NET. + // Convert training data to IDataView, the general data type used in + // ML.NET. var data = mlContext.Data.LoadFromEnumerable(samples); - // NormalizeBinning normalizes the data by constructing equidensity bins and produce output based on + // NormalizeBinning normalizes the data by constructing equidensity bins + // and produce output based on // to which bin the original value belongs. - var normalize = mlContext.Transforms.NormalizeBinning("Features", maximumBinCount: 4, fixZero: false); + var normalize = mlContext.Transforms.NormalizeBinning("Features", + maximumBinCount: 4, fixZero: false); - // NormalizeBinning normalizes the data by constructing equidensity bins and produce output based on - // to which bin original value belong but make sure zero values would remain zero after normalization. - // Helps preserve sparsity. - var normalizeFixZero = mlContext.Transforms.NormalizeBinning("Features", maximumBinCount: 4, fixZero: true); + // NormalizeBinning normalizes the data by constructing equidensity bins + // and produce output based on to which bin original value belong but + // make sure zero values would remain zero after normalization. Helps + // preserve sparsity. + var normalizeFixZero = mlContext.Transforms.NormalizeBinning("Features", + maximumBinCount: 4, fixZero: true); - // Now we can transform the data and look at the output to confirm the behavior of the estimator. - // This operation doesn't actually evaluate data until we read the data below. + // Now we can transform the data and look at the output to confirm the + // behavior of the estimator. This operation doesn't actually evaluate + // data until we read the data below. var normalizeTransform = normalize.Fit(data); var transformedData = normalizeTransform.Transform(data); var normalizeFixZeroTransform = normalizeFixZero.Fit(data); var fixZeroData = normalizeFixZeroTransform.Transform(data); var column = transformedData.GetColumn("Features").ToArray(); foreach (var row in column) - Console.WriteLine(string.Join(", ", row.Select(x => x.ToString("f4")))); + Console.WriteLine(string.Join(", ", row.Select(x => x.ToString( + "f4")))); // Expected output: // 1.0000, 0.6667, 1.0000, 0.0000 // 0.6667, 1.0000, 0.6667, 0.0000 // 0.3333, 0.3333, 0.3333, 0.0000 // 0.0000, 0.0000, 0.0000, 1.0000 - var columnFixZero = fixZeroData.GetColumn("Features").ToArray(); + var columnFixZero = fixZeroData.GetColumn("Features") + .ToArray(); + foreach (var row in columnFixZero) - Console.WriteLine(string.Join(", ", row.Select(x => x.ToString("f4")))); + Console.WriteLine(string.Join(", ", row.Select(x => x.ToString( + "f4")))); // Expected output: // 1.0000, 0.3333, 1.0000, 0.0000 // 0.6667, 0.6667, 0.6667, 0.0000 // 0.3333, 0.0000, 0.3333, 0.0000 // 0.0000, -0.3333, 0.0000, 1.0000 - // Let's get transformation parameters. Since we work with only one column we need to pass 0 as parameter for GetNormalizerModelParameters. - // If we have multiple columns transformations we need to pass index of InputOutputColumnPair. - var transformParams = normalizeTransform.GetNormalizerModelParameters(0) as BinNormalizerModelParameters>; + // Let's get transformation parameters. Since we work with only one + // column we need to pass 0 as parameter for + // GetNormalizerModelParameters. If we have multiple columns + // transformations we need to pass index of InputOutputColumnPair. + var transformParams = normalizeTransform.GetNormalizerModelParameters(0) + as BinNormalizerModelParameters>; + var density = transformParams.Density[0]; - var offset = (transformParams.Offset.Length == 0 ? 0 : transformParams.Offset[0]); - Console.WriteLine($"The 0-index value in resulting array would be produce by: y = (Index(x) / {density}) - {offset}"); - Console.WriteLine("Where Index(x) is the index of the bin to which x belongs"); - Console.WriteLine($"Bins upper bounds are: {string.Join(" ", transformParams.UpperBounds[0])}"); + var offset = (transformParams.Offset.Length == 0 ? 0 : transformParams + .Offset[0]); + + Console.WriteLine($"The 0-index value in resulting array would be " + + $"produce by: y = (Index(x) / {density}) - {offset}"); + + Console.WriteLine("Where Index(x) is the index of the bin to which " + + "x belongs"); + + Console.WriteLine("Bins upper bounds are: " + string.Join(" ", + transformParams.UpperBounds[0])); // Expected output: // The 0-index value in resulting array would be produce by: y = (Index(x) / 3) - 0 // Where Index(x) is the index of the bin to which x belongs // Bins upper bounds are: 3 5 7 ∞ - var fixZeroParams = (normalizeFixZeroTransform.GetNormalizerModelParameters(0) as BinNormalizerModelParameters>); + var fixZeroParams = (normalizeFixZeroTransform + .GetNormalizerModelParameters(0) as BinNormalizerModelParameters< + ImmutableArray>); + density = fixZeroParams.Density[1]; - offset = (fixZeroParams.Offset.Length == 0 ? 0 : fixZeroParams.Offset[1]); - Console.WriteLine($"The 0-index value in resulting array would be produce by: y = (Index(x) / {density}) - {offset}"); - Console.WriteLine("Where Index(x) is the index of the bin to which x belongs"); - Console.WriteLine($"Bins upper bounds are: {string.Join(" ", fixZeroParams.UpperBounds[1])}"); + offset = (fixZeroParams.Offset.Length == 0 ? 0 : fixZeroParams + .Offset[1]); + + Console.WriteLine($"The 0-index value in resulting array would be " + + $"produce by: y = (Index(x) / {density}) - {offset}"); + + Console.WriteLine("Where Index(x) is the index of the bin to which x " + + "belongs"); + + Console.WriteLine("Bins upper bounds are: " + string.Join(" ", + fixZeroParams.UpperBounds[1])); // Expected output: // The 0-index value in resulting array would be produce by: y = (Index(x) / 3) - 0.3333333 // Where Index(x) is the index of the bin to which x belongs diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/NormalizeBinningMulticolumn.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/NormalizeBinningMulticolumn.cs index 90b9360aa4..9d6ef29ed7 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/NormalizeBinningMulticolumn.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/NormalizeBinningMulticolumn.cs @@ -12,35 +12,45 @@ public class NormalizeBinningMulticolumn { public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(); var samples = new List() { - new DataPoint(){ Features = new float[4] { 8, 1, 3, 0}, Features2 = 1 }, - new DataPoint(){ Features = new float[4] { 6, 2, 2, 0}, Features2 = 4 }, - new DataPoint(){ Features = new float[4] { 4, 0, 1, 0}, Features2 = 1 }, - new DataPoint(){ Features = new float[4] { 2,-1,-1, 1}, Features2 = 2 } + new DataPoint(){ Features = new float[4] { 8, 1, 3, 0}, + Features2 = 1 }, + + new DataPoint(){ Features = new float[4] { 6, 2, 2, 0}, + Features2 = 4 }, + + new DataPoint(){ Features = new float[4] { 4, 0, 1, 0}, + Features2 = 1 }, + + new DataPoint(){ Features = new float[4] { 2,-1,-1, 1}, + Features2 = 2 } }; - // Convert training data to IDataView, the general data type used in ML.NET. + // Convert training data to IDataView, the general data type used in + // ML.NET. var data = mlContext.Data.LoadFromEnumerable(samples); - // NormalizeBinning normalizes the data by constructing equidensity bins and produce output based on - // to which bin the original value belongs. + // NormalizeBinning normalizes the data by constructing equidensity bins + // and produce output based on to which bin the original value belongs. var normalize = mlContext.Transforms.NormalizeBinning(new[]{ new InputOutputColumnPair("Features"), new InputOutputColumnPair("Features2"), }, maximumBinCount: 4, fixZero: false); - // Now we can transform the data and look at the output to confirm the behavior of the estimator. - // This operation doesn't actually evaluate data until we read the data below. + // Now we can transform the data and look at the output to confirm the + // behavior of the estimator. This operation doesn't actually evaluate + // data until we read the data below. var normalizeTransform = normalize.Fit(data); var transformedData = normalizeTransform.Transform(data); var column = transformedData.GetColumn("Features").ToArray(); var column2 = transformedData.GetColumn("Features2").ToArray(); for(int i=0; i< column.Length; i++) - Console.WriteLine(string.Join(", ", column[i].Select(x => x.ToString("f4")))+"\t\t"+column2[i]); + Console.WriteLine(string.Join(", ", column[i].Select(x => x + .ToString("f4")))+"\t\t"+column2[i]); // Expected output: // // Features Feature2 diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/NormalizeGlobalContrast.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/NormalizeGlobalContrast.cs index 322997ca3b..8d4795a1b6 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/NormalizeGlobalContrast.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/NormalizeGlobalContrast.cs @@ -10,8 +10,8 @@ class NormalizeGlobalContrast { public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(); var samples = new List() { @@ -20,18 +20,23 @@ public static void Example() new DataPoint(){ Features = new float[4] { 1, 0, 1, 0} }, new DataPoint(){ Features = new float[4] { 0, 1, 0, 1} } }; - // Convert training data to IDataView, the general data type used in ML.NET. + // Convert training data to IDataView, the general data type used in + // ML.NET. var data = mlContext.Data.LoadFromEnumerable(samples); - var approximation = mlContext.Transforms.NormalizeGlobalContrast("Features", ensureZeroMean: false, scale:2, ensureUnitStandardDeviation:true); + var approximation = mlContext.Transforms.NormalizeGlobalContrast( + "Features", ensureZeroMean: false, scale:2, + ensureUnitStandardDeviation:true); - // Now we can transform the data and look at the output to confirm the behavior of the estimator. - // This operation doesn't actually evaluate data until we read the data below. + // Now we can transform the data and look at the output to confirm the + // behavior of the estimator. This operation doesn't actually evaluate + // data until we read the data below. var tansformer = approximation.Fit(data); var transformedData = tansformer.Transform(data); var column = transformedData.GetColumn("Features").ToArray(); foreach (var row in column) - Console.WriteLine(string.Join(", ", row.Select(x => x.ToString("f4")))); + Console.WriteLine(string.Join(", ", row.Select(x => x.ToString( + "f4")))); // Expected output: // 2.0000, 2.0000,-2.0000,-2.0000 // 2.0000, 2.0000,-2.0000,-2.0000 diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/NormalizeLogMeanVariance.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/NormalizeLogMeanVariance.cs index b577270622..3ef43d9f85 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/NormalizeLogMeanVariance.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/NormalizeLogMeanVariance.cs @@ -12,70 +12,89 @@ public class NormalizeLogMeanVariance { public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(); var samples = new List() { - new DataPoint(){ Features = new float[4] { 1, 1, 3, 0} }, - new DataPoint(){ Features = new float[4] { 2, 2, 2, 0} }, - new DataPoint(){ Features = new float[4] { 0, 0, 1, 0} }, - new DataPoint(){ Features = new float[4] {-1,-1,-1, 1} } + new DataPoint(){ Features = new float[5] { 1, 1, 3, 0, float.MaxValue } }, + new DataPoint(){ Features = new float[5] { 2, 2, 2, 0, float.MinValue } }, + new DataPoint(){ Features = new float[5] { 0, 0, 1, 0, 0} }, + new DataPoint(){ Features = new float[5] {-1,-1,-1, 1, 1} } }; - // Convert training data to IDataView, the general data type used in ML.NET. + // Convert training data to IDataView, the general data type used in + // ML.NET. var data = mlContext.Data.LoadFromEnumerable(samples); - // NormalizeLogMeanVariance normalizes the data based on the computed mean and variance of the logarithm of the data. + // NormalizeLogMeanVariance normalizes the data based on the computed + // mean and variance of the logarithm of the data. // Uses Cumulative distribution function as output. - var normalize = mlContext.Transforms.NormalizeLogMeanVariance("Features", useCdf: true); + var normalize = mlContext.Transforms.NormalizeLogMeanVariance( + "Features", useCdf: true); - // NormalizeLogMeanVariance normalizes the data based on the computed mean and variance of the logarithm of the data. - var normalizeNoCdf = mlContext.Transforms.NormalizeLogMeanVariance("Features", useCdf: false); + // NormalizeLogMeanVariance normalizes the data based on the computed + // mean and variance of the logarithm of the data. + var normalizeNoCdf = mlContext.Transforms.NormalizeLogMeanVariance( + "Features", useCdf: false); - // Now we can transform the data and look at the output to confirm the behavior of the estimator. - // This operation doesn't actually evaluate data until we read the data below. + // Now we can transform the data and look at the output to confirm the + // behavior of the estimator. + // This operation doesn't actually evaluate data until we read the data + // below. var normalizeTransform = normalize.Fit(data); var transformedData = normalizeTransform.Transform(data); var normalizeNoCdfTransform = normalizeNoCdf.Fit(data); var noCdfData = normalizeNoCdfTransform.Transform(data); var column = transformedData.GetColumn("Features").ToArray(); foreach (var row in column) - Console.WriteLine(string.Join(", ", row.Select(x => x.ToString("f4")))); + Console.WriteLine(string.Join(", ", row.Select(x => x.ToString( + "f4")))); // Expected output: - // 0.1587, 0.1587, 0.8654, 0.0000 - // 0.8413, 0.8413, 0.5837, 0.0000 - // 0.0000, 0.0000, 0.0940, 0.0000 - // 0.0000, 0.0000, 0.0000, 0.0000 + // 0.1587, 0.1587, 0.8654, 0.0000, 0.8413 + // 0.8413, 0.8413, 0.5837, 0.0000, 0.0000 + // 0.0000, 0.0000, 0.0940, 0.0000, 0.0000 + // 0.0000, 0.0000, 0.0000, 0.0000, 0.1587 var columnFixZero = noCdfData.GetColumn("Features").ToArray(); foreach (var row in columnFixZero) - Console.WriteLine(string.Join(", ", row.Select(x => x.ToString("f4")))); + Console.WriteLine(string.Join(", ", row.Select(x => x.ToString( + "f4")))); // Expected output: - // 1.8854, 1.8854, 5.2970, 0.0000 - // 4.7708, 4.7708, 3.0925, 0.0000 - // -1.0000,-1.0000, 0.8879, 0.0000 - // 3.8854,-3.8854,-3.5213, 0.0000 + // 1.8854, 1.8854, 5.2970, 0.0000, 7670682000000000000000000000000000000.0000 + // 4.7708, 4.7708, 3.0925, 0.0000, -7670682000000000000000000000000000000.0000 + // -1.0000,-1.0000, 0.8879, 0.0000, -1.0000 + // -3.8854,-3.8854,-3.5213, 0.0000, -0.9775 - // Let's get transformation parameters. Since we work with only one column we need to pass 0 as parameter for GetNormalizerModelParameters. - // If we have multiple columns transformations we need to pass index of InputOutputColumnPair. - var transformParams = normalizeTransform.GetNormalizerModelParameters(0) as CdfNormalizerModelParameters>; - Console.WriteLine("The 1-index value in resulting array would be produce by:"); - Console.WriteLine($"y = 0.5* (1 + ERF((Math.Log(x)- {transformParams.Mean[1]}) / ({transformParams.StandardDeviation[1]} * sqrt(2)))"); + // Let's get transformation parameters. Since we work with only one + // column we need to pass 0 as parameter for + // GetNormalizerModelParameters. If we have multiple columns + // transformations we need to pass index of InputOutputColumnPair. + var transformParams = normalizeTransform.GetNormalizerModelParameters(0) + as CdfNormalizerModelParameters>; + + Console.WriteLine("The 1-index value in resulting array would be " + + "produce by:"); + + Console.WriteLine("y = 0.5* (1 + ERF((Math.Log(x)- " + transformParams + .Mean[1] + ") / (" + transformParams.StandardDeviation[1] + + " * sqrt(2)))"); // ERF is https://en.wikipedia.org/wiki/Error_function. // Expected output: // The 1-index value in resulting array would be produce by: // y = 0.5* (1 + ERF((Math.Log(x)- 0.3465736) / (0.3465736 * sqrt(2))) - var noCdfParams = normalizeNoCdfTransform.GetNormalizerModelParameters(0) as AffineNormalizerModelParameters>; + var noCdfParams = normalizeNoCdfTransform.GetNormalizerModelParameters( + 0) as AffineNormalizerModelParameters>; var offset = noCdfParams.Offset.Length == 0 ? 0 : noCdfParams.Offset[1]; var scale = noCdfParams.Scale[1]; - Console.WriteLine($"The 1-index value in resulting array would be produce by: y = (x - ({offset})) * {scale}"); + Console.WriteLine($"The 1-index value in resulting array would be " + + $"produce by: y = (x - ({offset})) * {scale}"); // Expected output: - // The 1-index value in resulting array would be produce by: y = (x - (2.88539)) * 0.3465736 + // The 1-index value in resulting array would be produce by: y = (x - (0.3465736)) * 2.88539 } private class DataPoint { - [VectorType(4)] + [VectorType(5)] public float[] Features { get; set; } } } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/NormalizeLogMeanVarianceFixZero.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/NormalizeLogMeanVarianceFixZero.cs new file mode 100644 index 0000000000..87adbc5ea0 --- /dev/null +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/NormalizeLogMeanVarianceFixZero.cs @@ -0,0 +1,82 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using Microsoft.ML; +using Microsoft.ML.Data; +using static Microsoft.ML.Transforms.NormalizingTransformer; + +namespace Samples.Dynamic +{ + public class NormalizeLogMeanVarianceFixZero + { + public static void Example() + { + // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, + // as well as the source of randomness. + var mlContext = new MLContext(); + var samples = new List() + { + new DataPoint(){ Features = new float[5] { 1, 1, 3, 0, float.MaxValue } }, + new DataPoint(){ Features = new float[5] { 2, 2, 2, 0, float.MinValue } }, + new DataPoint(){ Features = new float[5] { 0, 0, 1, 0, 0} }, + new DataPoint(){ Features = new float[5] {-1,-1,-1, 1, 1} } + }; + // Convert training data to IDataView, the general data type used in ML.NET. + var data = mlContext.Data.LoadFromEnumerable(samples); + // NormalizeLogMeanVariance normalizes the data based on the computed mean and variance of the logarithm of the data. + // Uses Cumulative distribution function as output. + var normalize = mlContext.Transforms.NormalizeLogMeanVariance("Features", true, useCdf: true); + + // NormalizeLogMeanVariance normalizes the data based on the computed mean and variance of the logarithm of the data. + var normalizeNoCdf = mlContext.Transforms.NormalizeLogMeanVariance("Features", true, useCdf: false); + + // Now we can transform the data and look at the output to confirm the behavior of the estimator. + // This operation doesn't actually evaluate data until we read the data below. + var normalizeTransform = normalize.Fit(data); + var transformedData = normalizeTransform.Transform(data); + var normalizeNoCdfTransform = normalizeNoCdf.Fit(data); + var noCdfData = normalizeNoCdfTransform.Transform(data); + var column = transformedData.GetColumn("Features").ToArray(); + foreach (var row in column) + Console.WriteLine(string.Join(", ", row.Select(x => x.ToString("f4")))); + // Expected output: + // 0.1587, 0.1587, 0.8654, 0.0000, 0.8413 + // 0.8413, 0.8413, 0.5837, 0.0000, 0.0000 + // 0.0000, 0.0000, 0.0940, 0.0000, 0.0000 + // 0.0000, 0.0000, 0.0000, 0.0000, 0.1587 + + var columnFixZero = noCdfData.GetColumn("Features").ToArray(); + foreach (var row in columnFixZero) + Console.WriteLine(string.Join(", ", row.Select(x => x.ToString("f4")))); + // Expected output: + // 2.0403, 2.0403, 4.0001, 0.0000, 5423991000000000000000000000000000000.0000 + // 4.0806, 4.0806, 2.6667, 0.0000,-5423991000000000000000000000000000000.0000 + // 0.0000, 0.0000, 1.3334, 0.0000, 0.0000 + // -2.0403,-2.0403,-1.3334, 0.0000, 0.0159 + + // Let's get transformation parameters. Since we work with only one column we need to pass 0 as parameter for GetNormalizerModelParameters. + // If we have multiple columns transformations we need to pass index of InputOutputColumnPair. + var transformParams = normalizeTransform.GetNormalizerModelParameters(0) as CdfNormalizerModelParameters>; + Console.WriteLine("The values in the column with index 1 in the resulting array would be produced by:"); + Console.WriteLine($"y = 0.5* (1 + ERF((Math.Log(x)- {transformParams.Mean[1]}) / ({transformParams.StandardDeviation[1]} * sqrt(2)))"); + + // ERF is https://en.wikipedia.org/wiki/Error_function. + // Expected output: + // The values in the column with index 1 in the resulting array would be produced by: + // y = 0.5 * (1 + ERF((Math.Log(x) - 0.3465736) / (0.3465736 * sqrt(2))) + var noCdfParams = normalizeNoCdfTransform.GetNormalizerModelParameters(0) as AffineNormalizerModelParameters>; + var offset = noCdfParams.Offset.Length == 0 ? 0 : noCdfParams.Offset[1]; + var scale = noCdfParams.Scale[1]; + Console.WriteLine($"The values in the column with index 1 in the resulting array would be produced by: y = (x - ({offset})) * {scale}"); + // Expected output: + // The values in the column with index 1 in the resulting array would be produced by: y = (x - (0)) * 2.040279 + } + + private class DataPoint + { + [VectorType(5)] + public float[] Features { get; set; } + } + } +} diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/NormalizeLpNorm.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/NormalizeLpNorm.cs index 83883ae49e..c6a020b850 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/NormalizeLpNorm.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/NormalizeLpNorm.cs @@ -11,8 +11,8 @@ class NormalizeLpNorm { public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(); var samples = new List() { @@ -21,18 +21,23 @@ public static void Example() new DataPoint(){ Features = new float[4] { 1, 0, 1, 0} }, new DataPoint(){ Features = new float[4] { 0, 1, 0, 1} } }; - // Convert training data to IDataView, the general data type used in ML.NET. + // Convert training data to IDataView, the general data type used in + // ML.NET. var data = mlContext.Data.LoadFromEnumerable(samples); - var approximation = mlContext.Transforms.NormalizeLpNorm("Features", norm: LpNormNormalizingEstimatorBase.NormFunction.L1, ensureZeroMean: true); + var approximation = mlContext.Transforms.NormalizeLpNorm("Features", + norm: LpNormNormalizingEstimatorBase.NormFunction.L1, + ensureZeroMean: true); - // Now we can transform the data and look at the output to confirm the behavior of the estimator. - // This operation doesn't actually evaluate data until we read the data below. + // Now we can transform the data and look at the output to confirm the + // behavior of the estimator. This operation doesn't actually evaluate + // data until we read the data below. var tansformer = approximation.Fit(data); var transformedData = tansformer.Transform(data); var column = transformedData.GetColumn("Features").ToArray(); foreach (var row in column) - Console.WriteLine(string.Join(", ", row.Select(x => x.ToString("f4")))); + Console.WriteLine(string.Join(", ", row.Select(x => x.ToString( + "f4")))); // Expected output: // 0.2500, 0.2500, -0.2500, -0.2500 // 0.2500, 0.2500, -0.2500, -0.2500 diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/NormalizeMeanVariance.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/NormalizeMeanVariance.cs index ad35d43e6f..cccbb58f1b 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/NormalizeMeanVariance.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/NormalizeMeanVariance.cs @@ -12,8 +12,8 @@ public class NormalizeMeanVariance { public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(); var samples = new List() { @@ -22,24 +22,31 @@ public static void Example() new DataPoint(){ Features = new float[4] { 0, 0, 1, 0} }, new DataPoint(){ Features = new float[4] {-1,-1,-1, 1} } }; - // Convert training data to IDataView, the general data type used in ML.NET. + // Convert training data to IDataView, the general data type used in + // ML.NET. var data = mlContext.Data.LoadFromEnumerable(samples); - // NormalizeMeanVariance normalizes the data based on the computed mean and variance of the data. - // Uses Cumulative distribution function as output. - var normalize = mlContext.Transforms.NormalizeMeanVariance("Features", useCdf: true); + // NormalizeMeanVariance normalizes the data based on the computed mean + // and variance of the data. Uses Cumulative distribution function as + // output. + var normalize = mlContext.Transforms.NormalizeMeanVariance("Features", + useCdf: true); - // NormalizeMeanVariance normalizes the data based on the computed mean and variance of the data. - var normalizeNoCdf = mlContext.Transforms.NormalizeMeanVariance("Features", useCdf: false); + // NormalizeMeanVariance normalizes the data based on the computed mean + // and variance of the data. + var normalizeNoCdf = mlContext.Transforms.NormalizeMeanVariance( + "Features", useCdf: false); - // Now we can transform the data and look at the output to confirm the behavior of the estimator. - // This operation doesn't actually evaluate data until we read the data below. + // Now we can transform the data and look at the output to confirm the + // behavior of the estimator. This operation doesn't actually evaluate + // data until we read the data below. var normalizeTransform = normalize.Fit(data); var transformedData = normalizeTransform.Transform(data); var normalizeNoCdfTransform = normalizeNoCdf.Fit(data); var noCdfData = normalizeNoCdfTransform.Transform(data); var column = transformedData.GetColumn("Features").ToArray(); foreach (var row in column) - Console.WriteLine(string.Join(", ", row.Select(x => x.ToString("f4")))); + Console.WriteLine(string.Join(", ", row.Select(x => x.ToString( + "f4")))); // Expected output: // 0.6726, 0.6726, 0.8816, 0.2819 // 0.9101, 0.9101, 0.6939, 0.2819 @@ -49,27 +56,40 @@ public static void Example() var columnFixZero = noCdfData.GetColumn("Features").ToArray(); foreach (var row in columnFixZero) - Console.WriteLine(string.Join(", ", row.Select(x => x.ToString("f4")))); + Console.WriteLine(string.Join(", ", row.Select(x => x.ToString( + "f4")))); // Expected output: // 0.8165, 0.8165, 1.5492, 0.0000 // 1.6330, 1.6330, 1.0328, 0.0000 // 0.0000, 0.0000, 0.5164, 0.0000 // -0.8165,-0.8165,-0.5164, 2.0000 - // Let's get transformation parameters. Since we work with only one column we need to pass 0 as parameter for GetNormalizerModelParameters. - // If we have multiple columns transformations we need to pass index of InputOutputColumnPair. - var transformParams = normalizeTransform.GetNormalizerModelParameters(0) as CdfNormalizerModelParameters>; - Console.WriteLine($"The 1-index value in resulting array would be produce by:"); - Console.WriteLine($" y = 0.5* (1 + ERF((x- {transformParams.Mean[1]}) / ({transformParams.StandardDeviation[1]} * sqrt(2)))"); + // Let's get transformation parameters. Since we work with only one + // column we need to pass 0 as parameter for + // GetNormalizerModelParameters. If we have multiple columns + // transformations we need to pass index of InputOutputColumnPair. + var transformParams = normalizeTransform + .GetNormalizerModelParameters(0) as CdfNormalizerModelParameters< + ImmutableArray>; + + Console.WriteLine($"The 1-index value in resulting array would " + + $"be produce by:"); + + Console.WriteLine(" y = 0.5* (1 + ERF((x- " + transformParams.Mean[1] + + ") / (" + transformParams.StandardDeviation[1] + " * sqrt(2)))"); // ERF is https://en.wikipedia.org/wiki/Error_function. // Expected output: // The 1-index value in resulting array would be produce by: // y = 0.5 * (1 + ERF((x - 0.5) / (1.118034 * sqrt(2))) - var noCdfParams = normalizeNoCdfTransform.GetNormalizerModelParameters(0) as AffineNormalizerModelParameters>; + var noCdfParams = normalizeNoCdfTransform + .GetNormalizerModelParameters(0) as + AffineNormalizerModelParameters>; + var offset = noCdfParams.Offset.Length == 0 ? 0 : noCdfParams.Offset[1]; var scale = noCdfParams.Scale[1]; - Console.WriteLine($"Values for slot 1 would be transfromed by applying y = (x - ({offset})) * {scale}"); + Console.WriteLine($"Values for slot 1 would be transfromed by " + + $"applying y = (x - ({offset})) * {scale}"); // Expected output: // The 1-index value in resulting array would be produce by: y = (x - (0)) * 0.8164966 } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/NormalizeMinMax.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/NormalizeMinMax.cs index 7b7a60d74e..f78056f9e6 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/NormalizeMinMax.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/NormalizeMinMax.cs @@ -12,8 +12,8 @@ public class NormalizeMinMax { public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(); var samples = new List() { @@ -22,46 +22,63 @@ public static void Example() new DataPoint(){ Features = new float[4] { 0, 0, 1, 0} }, new DataPoint(){ Features = new float[4] {-1,-1,-1, 1} } }; - // Convert training data to IDataView, the general data type used in ML.NET. + // Convert training data to IDataView, the general data type used in + // ML.NET. var data = mlContext.Data.LoadFromEnumerable(samples); - // NormalizeMinMax normalize rows by finding min and max values in each row slot - // and setting projection of min value to 0 and max to 1 and everything else to - // values in between. - var normalize = mlContext.Transforms.NormalizeMinMax("Features", fixZero: false); + // NormalizeMinMax normalize rows by finding min and max values in each + // row slot and setting projection of min value to 0 and max to 1 and + // everything else to values in between. + var normalize = mlContext.Transforms.NormalizeMinMax("Features", + fixZero: false); - // Normalize rows by finding min and max values in each row slot, but make sure - // zero values would remain zero after normalization. Helps preserve sparsity. - var normalizeFixZero = mlContext.Transforms.NormalizeMinMax("Features", fixZero: true); + // Normalize rows by finding min and max values in each row slot, but + // make sure zero values would remain zero after normalization. Helps + // preserve sparsity. + var normalizeFixZero = mlContext.Transforms.NormalizeMinMax("Features", + fixZero: true); - // Now we can transform the data and look at the output to confirm the behavior of the estimator. - // This operation doesn't actually evaluate data until we read the data below. + // Now we can transform the data and look at the output to confirm the + // behavior of the estimator. This operation doesn't actually evaluate + // data until we read the data below. var normalizeTransform = normalize.Fit(data); var transformedData = normalizeTransform.Transform(data); var normalizeFixZeroTransform = normalizeFixZero.Fit(data); var fixZeroData = normalizeFixZeroTransform.Transform(data); var column = transformedData.GetColumn("Features").ToArray(); foreach (var row in column) - Console.WriteLine(string.Join(", ", row.Select(x => x.ToString("f4")))); + Console.WriteLine(string.Join(", ", row.Select(x => x.ToString( + "f4")))); // Expected output: // 0.6667, 0.6667, 1.0000, 0.0000 // 1.0000, 1.0000, 0.7500, 0.0000 // 0.3333, 0.3333, 0.5000, 0.0000 // 0.0000, 0.0000, 0.0000, 1.0000 - var columnFixZero = fixZeroData.GetColumn("Features").ToArray(); + var columnFixZero = fixZeroData.GetColumn("Features") + .ToArray(); + foreach (var row in columnFixZero) - Console.WriteLine(string.Join(", ", row.Select(x => x.ToString("f4")))); + Console.WriteLine(string.Join(", ", row.Select(x => x.ToString( + "f4")))); // Expected output: // 0.5000, 0.5000, 1.0000, 0.0000 // 1.0000, 1.0000, 0.6667, 0.0000 // 0.0000, 0.0000, 0.3333, 0.0000 // -0.5000,-0.5000,-0.3333, 1.0000 - // Let's get transformation parameters. Since we work with only one column we need to pass 0 as parameter for GetNormalizerModelParameters. - // If we have multiple columns transformations we need to pass index of InputOutputColumnPair. - var transformParams = normalizeTransform.GetNormalizerModelParameters(0) as AffineNormalizerModelParameters>; - Console.WriteLine($"The 1-index value in resulting array would be produce by:"); - Console.WriteLine($" y = (x - ({(transformParams.Offset.Length == 0 ? 0 : transformParams.Offset[1])})) * {transformParams.Scale[1]}"); + // Let's get transformation parameters. Since we work with only one + // column we need to pass 0 as parameter for + // GetNormalizerModelParameters. If we have multiple columns + // transformations we need to pass index of InputOutputColumnPair. + var transformParams = normalizeTransform.GetNormalizerModelParameters(0) + as AffineNormalizerModelParameters>; + + Console.WriteLine($"The 1-index value in resulting array would be " + + $"produce by:"); + + Console.WriteLine(" y = (x - (" + (transformParams.Offset.Length == 0 ? + 0 : transformParams.Offset[1]) + ")) * " + transformParams + .Scale[1]); // Expected output: // The 1-index value in resulting array would be produce by: // y = (x - (-1)) * 0.3333333 diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/NormalizeSupervisedBinning.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/NormalizeSupervisedBinning.cs index 63fde50a9e..9441324639 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/NormalizeSupervisedBinning.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/NormalizeSupervisedBinning.cs @@ -12,39 +12,58 @@ public class NormalizeSupervisedBinning { public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(); var samples = new List() { - new DataPoint(){ Features = new float[4] { 8, 1, 3, 0}, Bin="Bin1" }, - new DataPoint(){ Features = new float[4] { 6, 2, 2, 1}, Bin="Bin2" }, - new DataPoint(){ Features = new float[4] { 5, 3, 0, 2}, Bin="Bin2" }, - new DataPoint(){ Features = new float[4] { 4,-8, 1, 3}, Bin="Bin3" }, - new DataPoint(){ Features = new float[4] { 2,-5,-1, 4}, Bin="Bin3" } + new DataPoint(){ Features = new float[4] { 8, 1, 3, 0}, + Bin ="Bin1" }, + + new DataPoint(){ Features = new float[4] { 6, 2, 2, 1}, + Bin ="Bin2" }, + + new DataPoint(){ Features = new float[4] { 5, 3, 0, 2}, + Bin ="Bin2" }, + + new DataPoint(){ Features = new float[4] { 4,-8, 1, 3}, + Bin ="Bin3" }, + + new DataPoint(){ Features = new float[4] { 2,-5,-1, 4}, + Bin ="Bin3" } }; - // Convert training data to IDataView, the general data type used in ML.NET. + // Convert training data to IDataView, the general data type used in + // ML.NET. var data = mlContext.Data.LoadFromEnumerable(samples); // Let's transform "Bin" column from string to key. - data = mlContext.Transforms.Conversion.MapValueToKey("Bin").Fit(data).Transform(data); - // NormalizeSupervisedBinning normalizes the data by constructing bins based on correlation with the label column and produce output based on - // to which bin original value belong. - var normalize = mlContext.Transforms.NormalizeSupervisedBinning("Features", labelColumnName: "Bin", mininimumExamplesPerBin: 1, fixZero: false); - - // NormalizeSupervisedBinning normalizes the data by constructing bins based on correlation with the label column and produce output based on - // to which bin original value belong but make sure zero values would remain zero after normalization. - // Helps preserve sparsity. - var normalizeFixZero = mlContext.Transforms.NormalizeSupervisedBinning("Features", labelColumnName: "Bin", mininimumExamplesPerBin: 1, fixZero: true); - - // Now we can transform the data and look at the output to confirm the behavior of the estimator. - // This operation doesn't actually evaluate data until we read the data below. + data = mlContext.Transforms.Conversion.MapValueToKey("Bin").Fit(data) + .Transform(data); + // NormalizeSupervisedBinning normalizes the data by constructing bins + // based on correlation with the label column and produce output based + // on to which bin original value belong. + var normalize = mlContext.Transforms.NormalizeSupervisedBinning( + "Features", labelColumnName: "Bin", mininimumExamplesPerBin: 1, + fixZero: false); + + // NormalizeSupervisedBinning normalizes the data by constructing bins + // based on correlation with the label column and produce output based + // on to which bin original value belong but make sure zero values would + // remain zero after normalization. Helps preserve sparsity. + var normalizeFixZero = mlContext.Transforms.NormalizeSupervisedBinning( + "Features", labelColumnName: "Bin", mininimumExamplesPerBin: 1, + fixZero: true); + + // Now we can transform the data and look at the output to confirm the + // behavior of the estimator. This operation doesn't actually evaluate + // data until we read the data below. var normalizeTransform = normalize.Fit(data); var transformedData = normalizeTransform.Transform(data); var normalizeFixZeroTransform = normalizeFixZero.Fit(data); var fixZeroData = normalizeFixZeroTransform.Transform(data); var column = transformedData.GetColumn("Features").ToArray(); foreach (var row in column) - Console.WriteLine(string.Join(", ", row.Select(x => x.ToString("f4")))); + Console.WriteLine(string.Join(", ", row.Select(x => x.ToString( + "f4")))); // Expected output: // 1.0000, 0.5000, 1.0000, 0.0000 // 0.5000, 1.0000, 0.0000, 0.5000 @@ -52,9 +71,12 @@ public static void Example() // 0.0000, 0.0000, 0.0000, 1.0000 // 0.0000, 0.0000, 0.0000, 1.0000 - var columnFixZero = fixZeroData.GetColumn("Features").ToArray(); + var columnFixZero = fixZeroData.GetColumn("Features") + .ToArray(); + foreach (var row in columnFixZero) - Console.WriteLine(string.Join(", ", row.Select(x => x.ToString("f4")))); + Console.WriteLine(string.Join(", ", row.Select(x => x.ToString( + "f4")))); // Expected output: // 1.0000, 0.0000, 1.0000, 0.0000 // 0.5000, 0.5000, 0.0000, 0.5000 @@ -62,24 +84,48 @@ public static void Example() // 0.0000,-0.5000, 0.0000, 1.0000 // 0.0000,-0.5000, 0.0000, 1.0000 - // Let's get transformation parameters. Since we work with only one column we need to pass 0 as parameter for GetNormalizerModelParameters. - // If we have multiple columns transformations we need to pass index of InputOutputColumnPair. - var transformParams = normalizeTransform.GetNormalizerModelParameters(0) as BinNormalizerModelParameters>; - Console.WriteLine($"The 1-index value in resulting array would be produce by:"); - Console.WriteLine($"y = (Index(x) / {transformParams.Density[0]}) - {(transformParams.Offset.Length == 0 ? 0 : transformParams.Offset[0])}"); - Console.WriteLine("Where Index(x) is the index of the bin to which x belongs"); - Console.WriteLine($"Bins upper borders are: {string.Join(" ", transformParams.UpperBounds[0])}"); + // Let's get transformation parameters. Since we work with only one + // column we need to pass 0 as parameter for + // GetNormalizerModelParameters. + // If we have multiple columns transformations we need to pass index of + // InputOutputColumnPair. + var transformParams = normalizeTransform.GetNormalizerModelParameters(0) + as BinNormalizerModelParameters>; + + Console.WriteLine($"The 1-index value in resulting array would be " + + $"produce by:"); + + Console.WriteLine("y = (Index(x) / " + transformParams.Density[0] + + ") - " + (transformParams.Offset.Length == 0 ? 0 : transformParams + .Offset[0])); + + Console.WriteLine("Where Index(x) is the index of the bin to which " + + "x belongs"); + + Console.WriteLine("Bins upper borders are: " + string.Join(" ", + transformParams.UpperBounds[0])); // Expected output: // The 1-index value in resulting array would be produce by: // y = (Index(x) / 2) - 0 // Where Index(x) is the index of the bin to which x belongs // Bins upper bounds are: 4.5 7 ∞ - var fixZeroParams = normalizeFixZeroTransform.GetNormalizerModelParameters(0) as BinNormalizerModelParameters>; - Console.WriteLine($"The 1-index value in resulting array would be produce by:"); - Console.WriteLine($" y = (Index(x) / {fixZeroParams.Density[1]}) - {(fixZeroParams.Offset.Length == 0 ? 0 : fixZeroParams.Offset[1])}"); - Console.WriteLine("Where Index(x) is the index of the bin to which x belongs"); - Console.WriteLine($"Bins upper borders are: {string.Join(" ", fixZeroParams.UpperBounds[1])}"); + var fixZeroParams = normalizeFixZeroTransform + .GetNormalizerModelParameters(0) as BinNormalizerModelParameters< + ImmutableArray>; + + Console.WriteLine($"The 1-index value in resulting array would be " + + $"produce by:"); + + Console.WriteLine(" y = (Index(x) / " + fixZeroParams.Density[1] + + ") - " + (fixZeroParams.Offset.Length == 0 ? 0 : fixZeroParams + .Offset[1])); + + Console.WriteLine("Where Index(x) is the index of the bin to which x " + + "belongs"); + + Console.WriteLine("Bins upper borders are: " + string.Join(" ", + fixZeroParams.UpperBounds[1])); // Expected output: // The 1-index value in resulting array would be produce by: // y = (Index(x) / 2) - 0.5 diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Projection/VectorWhiten.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Projection/VectorWhiten.cs index 9de60d5130..5e5548227b 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Projection/VectorWhiten.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Projection/VectorWhiten.cs @@ -9,11 +9,12 @@ namespace Samples.Dynamic public sealed class VectorWhiten { - /// This example requires installation of additional nuget package Microsoft.ML.Mkl.Components. + /// This example requires installation of additional nuget package + /// Microsoft.ML.Mkl.Components. public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var ml = new MLContext(); // Get a small dataset as an IEnumerable and convert it to an IDataView. @@ -32,20 +33,29 @@ public static void Example() // 6 7 8 9 0 1 2 3 4 5 // A small printing utility. - Action>> printHelper = (colName, column) => + Action>> printHelper = (colName, + column) => { - Console.WriteLine($"{colName} column obtained post-transformation."); + Console.WriteLine($"{colName} column obtained " + + $"post-transformation."); + foreach (var row in column) - Console.WriteLine($"{string.Join(" ", row.DenseValues().Select(x => x.ToString("f3")))} "); + Console.WriteLine(string.Join(" ", row.DenseValues().Select(x => + x.ToString("f3")))+" "); }; // A pipeline to project Features column into white noise vector. - var whiteningPipeline = ml.Transforms.VectorWhiten(nameof(SampleVectorOfNumbersData.Features), - kind: Microsoft.ML.Transforms.WhiteningKind.ZeroPhaseComponentAnalysis); + var whiteningPipeline = ml.Transforms.VectorWhiten(nameof( + SampleVectorOfNumbersData.Features), kind: Microsoft.ML.Transforms + .WhiteningKind.ZeroPhaseComponentAnalysis); + // The transformed (projected) data. - var transformedData = whiteningPipeline.Fit(trainData).Transform(trainData); + var transformedData = whiteningPipeline.Fit(trainData).Transform( + trainData); + // Getting the data of the newly created column, so we can preview it. - var whitening = transformedData.GetColumn>(transformedData.Schema[nameof(SampleVectorOfNumbersData.Features)]); + var whitening = transformedData.GetColumn>( + transformedData.Schema[nameof(SampleVectorOfNumbersData.Features)]); printHelper(nameof(SampleVectorOfNumbersData.Features), whitening); @@ -68,11 +78,16 @@ private class SampleVectorOfNumbersData /// /// Returns a few rows of the infertility dataset. /// - private static IEnumerable GetVectorOfNumbersData() + private static IEnumerable + GetVectorOfNumbersData() { var data = new List(); - data.Add(new SampleVectorOfNumbersData { Features = new float[10] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 } }); - data.Add(new SampleVectorOfNumbersData { Features = new float[10] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 0 } }); + data.Add(new SampleVectorOfNumbersData { Features = new float[10] { 0, + 1, 2, 3, 4, 5, 6, 7, 8, 9 } }); + + data.Add(new SampleVectorOfNumbersData { Features = new float[10] { 1, + 2, 3, 4, 5, 6, 7, 8, 9, 0 } }); + data.Add(new SampleVectorOfNumbersData { Features = new float[10] { 2, 3, 4, 5, 6, 7, 8, 9, 0, 1 } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Projection/VectorWhitenWithOptions.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Projection/VectorWhitenWithOptions.cs index 2e4e7fe46a..b1f852fc2c 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Projection/VectorWhitenWithOptions.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Projection/VectorWhitenWithOptions.cs @@ -8,11 +8,12 @@ namespace Samples.Dynamic { public sealed class VectorWhitenWithOptions { - /// This example requires installation of additional nuget package Microsoft.ML.Mkl.Components. + /// This example requires installation of additional nuget package + /// Microsoft.ML.Mkl.Components. public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var ml = new MLContext(); // Get a small dataset as an IEnumerable and convert it to an IDataView. @@ -31,20 +32,30 @@ public static void Example() // 6 7 8 9 0 1 2 3 4 5 // A small printing utility. - Action>> printHelper = (colName, column) => + Action>> printHelper = (colName, + column) => { - Console.WriteLine($"{colName} column obtained post-transformation."); + Console.WriteLine($"{colName} column obtained" + + $"post-transformation."); + foreach (var row in column) - Console.WriteLine($"{string.Join(" ", row.DenseValues().Select(x => x.ToString("f3")))} "); + Console.WriteLine(string.Join(" ", row.DenseValues().Select(x => + x.ToString("f3")))+" "); }; // A pipeline to project Features column into white noise vector. - var whiteningPipeline = ml.Transforms.VectorWhiten(nameof(SampleVectorOfNumbersData.Features), kind: Microsoft.ML.Transforms.WhiteningKind.PrincipalComponentAnalysis, rank: 4); + var whiteningPipeline = ml.Transforms.VectorWhiten(nameof( + SampleVectorOfNumbersData.Features), kind: Microsoft.ML.Transforms + .WhiteningKind.PrincipalComponentAnalysis, rank: 4); + // The transformed (projected) data. - var transformedData = whiteningPipeline.Fit(trainData).Transform(trainData); + var transformedData = whiteningPipeline.Fit(trainData).Transform( + trainData); + // Getting the data of the newly created column, so we can preview it. - var whitening = transformedData.GetColumn>(transformedData.Schema[nameof(SampleVectorOfNumbersData.Features)]); + var whitening = transformedData.GetColumn>( + transformedData.Schema[nameof(SampleVectorOfNumbersData.Features)]); printHelper(nameof(SampleVectorOfNumbersData.Features), whitening); @@ -66,11 +77,16 @@ private class SampleVectorOfNumbersData /// /// Returns a few rows of the infertility dataset. /// - private static IEnumerable GetVectorOfNumbersData() + private static IEnumerable + GetVectorOfNumbersData() { var data = new List(); - data.Add(new SampleVectorOfNumbersData { Features = new float[10] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 } }); - data.Add(new SampleVectorOfNumbersData { Features = new float[10] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 0 } }); + data.Add(new SampleVectorOfNumbersData { Features = new float[10] { 0, + 1, 2, 3, 4, 5, 6, 7, 8, 9 } }); + + data.Add(new SampleVectorOfNumbersData { Features = new float[10] { 1, + 2, 3, 4, 5, 6, 7, 8, 9, 0 } }); + data.Add(new SampleVectorOfNumbersData { Features = new float[10] { 2, 3, 4, 5, 6, 7, 8, 9, 0, 1 } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/ReplaceMissingValues.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/ReplaceMissingValues.cs index d30c6812ce..83ba740a0d 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/ReplaceMissingValues.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/ReplaceMissingValues.cs @@ -11,35 +11,46 @@ class ReplaceMissingValues { public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(); // Get a small dataset as an IEnumerable and convert it to an IDataView. var samples = new List() { - new DataPoint(){ Features = new float[3] {float.PositiveInfinity, 1, 0} }, + new DataPoint(){ Features = new float[3] {float.PositiveInfinity, 1, + 0 } }, + new DataPoint(){ Features = new float[3] {0, float.NaN, 1} }, new DataPoint(){ Features = new float[3] {-1, 2, -3} }, new DataPoint(){ Features = new float[3] {-1, float.NaN, -3} }, }; var data = mlContext.Data.LoadFromEnumerable(samples); - // Here we use the default replacement mode, which replaces the value with the default value for its type. - var defaultPipeline = mlContext.Transforms.ReplaceMissingValues("MissingReplaced", "Features", - MissingValueReplacingEstimator.ReplacementMode.DefaultValue); + // Here we use the default replacement mode, which replaces the value + // with the default value for its type. + var defaultPipeline = mlContext.Transforms.ReplaceMissingValues( + "MissingReplaced", "Features", MissingValueReplacingEstimator + .ReplacementMode.DefaultValue); - // Now we can transform the data and look at the output to confirm the behavior of the estimator. - // This operation doesn't actually evaluate data until we read the data below. + // Now we can transform the data and look at the output to confirm the + // behavior of the estimator. This operation doesn't actually evaluate + // data until we read the data below. var defaultTransformer = defaultPipeline.Fit(data); var defaultTransformedData = defaultTransformer.Transform(data); - // We can extract the newly created column as an IEnumerable of SampleDataTransformed, the class we define below. - var defaultRowEnumerable = mlContext.Data.CreateEnumerable(defaultTransformedData, reuseRowObject: false); + // We can extract the newly created column as an IEnumerable of + // SampleDataTransformed, the class we define below. + var defaultRowEnumerable = mlContext.Data.CreateEnumerable< + SampleDataTransformed>(defaultTransformedData, reuseRowObject: + false); - // And finally, we can write out the rows of the dataset, looking at the columns of interest. + // And finally, we can write out the rows of the dataset, looking at the + // columns of interest. foreach (var row in defaultRowEnumerable) - Console.WriteLine($"Features: [{string.Join(", ", row.Features)}]\t MissingReplaced: [{string.Join(", ", row.MissingReplaced)}]"); + Console.WriteLine("Features: [" + string.Join(", ", row.Features) + + "]\t MissingReplaced: [" + string.Join(", ", row + .MissingReplaced) + "]"); // Expected output: // Features: [∞, 1, 0] MissingReplaced: [∞, 1, 0] @@ -47,21 +58,29 @@ public static void Example() // Features: [-1, 2, -3] MissingReplaced: [-1, 2, -3] // Features: [-1, NaN, -3] MissingReplaced: [-1, 0, -3] - // Here we use the mean replacement mode, which replaces the value with the mean of the non values that were not missing. - var meanPipeline = mlContext.Transforms.ReplaceMissingValues("MissingReplaced", "Features", - MissingValueReplacingEstimator.ReplacementMode.Mean); + // Here we use the mean replacement mode, which replaces the value with + // the mean of the non values that were not missing. + var meanPipeline = mlContext.Transforms.ReplaceMissingValues( + "MissingReplaced", "Features", MissingValueReplacingEstimator + .ReplacementMode.Mean); - // Now we can transform the data and look at the output to confirm the behavior of the estimator. - // This operation doesn't actually evaluate data until we read the data below. + // Now we can transform the data and look at the output to confirm the + // behavior of the estimator. This operation doesn't actually evaluate + // data until we read the data below. var meanTransformer = meanPipeline.Fit(data); var meanTransformedData = meanTransformer.Transform(data); - // We can extract the newly created column as an IEnumerable of SampleDataTransformed, the class we define below. - var meanRowEnumerable = mlContext.Data.CreateEnumerable(meanTransformedData, reuseRowObject: false); + // We can extract the newly created column as an IEnumerable of + // SampleDataTransformed, the class we define below. + var meanRowEnumerable = mlContext.Data.CreateEnumerable< + SampleDataTransformed>(meanTransformedData, reuseRowObject: false); - // And finally, we can write out the rows of the dataset, looking at the columns of interest. + // And finally, we can write out the rows of the dataset, looking at the + // columns of interest. foreach (var row in meanRowEnumerable) - Console.WriteLine($"Features: [{string.Join(", ", row.Features)}]\t MissingReplaced: [{string.Join(", ", row.MissingReplaced)}]"); + Console.WriteLine("Features: [" + string.Join(", ", row.Features) + + "]\t MissingReplaced: [" + string.Join(", ", row + .MissingReplaced) + "]"); // Expected output: // Features: [∞, 1, 0] MissingReplaced: [∞, 1, 0] diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/ReplaceMissingValuesMultiColumn.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/ReplaceMissingValuesMultiColumn.cs index aa5d1acf5b..2f17e5334b 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/ReplaceMissingValuesMultiColumn.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/ReplaceMissingValuesMultiColumn.cs @@ -10,39 +10,55 @@ class ReplaceMissingValuesMultiColumn { public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(); // Get a small dataset as an IEnumerable and convert it to an IDataView. var samples = new List() { - new DataPoint(){ Features1 = new float[3] {1, 1, 0}, Features2 = new float[2] {1, 1} }, - new DataPoint(){ Features1 = new float[3] {0, float.NaN, 1}, Features2 = new float[2] {0, 1} }, - new DataPoint(){ Features1 = new float[3] {-1, float.NaN, -3}, Features2 = new float[2] {-1, float.NaN} }, - new DataPoint(){ Features1 = new float[3] {-1, 6, -3}, Features2 = new float[2] {0, float.PositiveInfinity} }, + new DataPoint(){ Features1 = new float[3] {1, 1, 0}, Features2 = + new float[2] {1, 1} }, + + new DataPoint(){ Features1 = new float[3] {0, float.NaN, 1}, + Features2 = new float[2] {0, 1} }, + + new DataPoint(){ Features1 = new float[3] {-1, float.NaN, -3}, + Features2 = new float[2] {-1, float.NaN} }, + + new DataPoint(){ Features1 = new float[3] {-1, 6, -3}, Features2 = + new float[2] {0, float.PositiveInfinity} }, }; var data = mlContext.Data.LoadFromEnumerable(samples); - // Here we use the default replacement mode, which replaces the value with the default value for its type. + // Here we use the default replacement mode, which replaces the value + // with the default value for its type. var defaultPipeline = mlContext.Transforms.ReplaceMissingValues(new[] { new InputOutputColumnPair("MissingReplaced1", "Features1"), new InputOutputColumnPair("MissingReplaced2", "Features2") }, MissingValueReplacingEstimator.ReplacementMode.DefaultValue); - // Now we can transform the data and look at the output to confirm the behavior of the estimator. - // This operation doesn't actually evaluate data until we read the data below. + // Now we can transform the data and look at the output to confirm the + // behavior of the estimator. This operation doesn't actually evaluate + // data until we read the data below. var defaultTransformer = defaultPipeline.Fit(data); var defaultTransformedData = defaultTransformer.Transform(data); - // We can extract the newly created column as an IEnumerable of SampleDataTransformed, the class we define below. - var defaultRowEnumerable = mlContext.Data.CreateEnumerable(defaultTransformedData, reuseRowObject: false); + // We can extract the newly created column as an IEnumerable of + // SampleDataTransformed, the class we define below. + var defaultRowEnumerable = mlContext.Data.CreateEnumerable< + SampleDataTransformed>(defaultTransformedData, reuseRowObject: + false); - // And finally, we can write out the rows of the dataset, looking at the columns of interest. + // And finally, we can write out the rows of the dataset, looking at the + // columns of interest. foreach (var row in defaultRowEnumerable) - Console.WriteLine($"Features1: [{string.Join(", ", row.Features1)}]\t MissingReplaced1: [{string.Join(", ", row.MissingReplaced1)}]\t " + - $"Features2: [{ string.Join(", ", row.Features2)}]\t MissingReplaced2: [{string.Join(", ", row.MissingReplaced2)}]"); + Console.WriteLine("Features1: [" + string.Join(", ", row + .Features1) + "]\t MissingReplaced1: [" + string.Join(", ", row + .MissingReplaced1) + "]\t Features2: [" + string.Join(", ", row + .Features2) + "]\t MissingReplaced2: [" + string.Join(", ", row + .MissingReplaced2) + "]"); // Expected output: // Features1: [1, 1, 0] MissingReplaced1: [1, 1, 0] Features2: [1, 1] MissingReplaced2: [1, 1] @@ -50,25 +66,34 @@ public static void Example() // Features1: [-1, NaN, -3] MissingReplaced1: [-1, 0, -3] Features2: [-1, NaN] MissingReplaced2: [-1, 0] // Features1: [-1, 6, -3] MissingReplaced1: [-1, 6, -3] Features2: [0, ∞] MissingReplaced2: [0, ∞] - // Here we use the mean replacement mode, which replaces the value with the mean of the non values that were not missing. + // Here we use the mean replacement mode, which replaces the value with + // the mean of the non values that were not missing. var meanPipeline = mlContext.Transforms.ReplaceMissingValues(new[] { new InputOutputColumnPair("MissingReplaced1", "Features1"), new InputOutputColumnPair("MissingReplaced2", "Features2") }, MissingValueReplacingEstimator.ReplacementMode.Mean); - // Now we can transform the data and look at the output to confirm the behavior of the estimator. - // This operation doesn't actually evaluate data until we read the data below. + // Now we can transform the data and look at the output to confirm the + // behavior of the estimator. + // This operation doesn't actually evaluate data until we read the data + // below. var meanTransformer = meanPipeline.Fit(data); var meanTransformedData = meanTransformer.Transform(data); - // We can extract the newly created column as an IEnumerable of SampleDataTransformed, the class we define below. - var meanRowEnumerable = mlContext.Data.CreateEnumerable(meanTransformedData, reuseRowObject: false); + // We can extract the newly created column as an IEnumerable of + // SampleDataTransformed, the class we define below. + var meanRowEnumerable = mlContext.Data.CreateEnumerable< + SampleDataTransformed>(meanTransformedData, reuseRowObject: false); - // And finally, we can write out the rows of the dataset, looking at the columns of interest. + // And finally, we can write out the rows of the dataset, looking at the + // columns of interest. foreach (var row in meanRowEnumerable) - Console.WriteLine($"Features1: [{string.Join(", ", row.Features1)}]\t MissingReplaced1: [{string.Join(", ", row.MissingReplaced1)}]\t " + - $"Features2: [{ string.Join(", ", row.Features2)}]\t MissingReplaced2: [{string.Join(", ", row.MissingReplaced2)}]"); + Console.WriteLine("Features1: [" + string.Join(", ", row + .Features1) + "]\t MissingReplaced1: [" + string.Join(", ", row + .MissingReplaced1) + "]\t Features2: [" + string.Join(", ", row + .Features2) + "]\t MissingReplaced2: [" + string.Join(", ", row + .MissingReplaced2) + "]"); // Expected output: // Features1: [1, 1, 0] MissingReplaced1: [1, 1, 0] Features2: [1, 1] MissingReplaced2: [1, 1] diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/SelectColumns.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/SelectColumns.cs index 796617d58f..22efcdc085 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/SelectColumns.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/SelectColumns.cs @@ -8,19 +8,30 @@ public static class SelectColumns { public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(); // Create a small dataset as an IEnumerable. var samples = new List() { - new InputData(){ Age = 21, Gender = "Male", Education = "BS", ExtraColumn = 1 }, - new InputData(){ Age = 23, Gender = "Female", Education = "MBA", ExtraColumn = 2 }, - new InputData(){ Age = 28, Gender = "Male", Education = "PhD", ExtraColumn = 3 }, - new InputData(){ Age = 22, Gender = "Male", Education = "BS", ExtraColumn = 4 }, - new InputData(){ Age = 23, Gender = "Female", Education = "MS", ExtraColumn = 5 }, - new InputData(){ Age = 27, Gender = "Female", Education = "PhD", ExtraColumn = 6 }, + new InputData(){ Age = 21, Gender = "Male", Education = "BS", + ExtraColumn = 1 }, + + new InputData(){ Age = 23, Gender = "Female", Education = "MBA", + ExtraColumn = 2 }, + + new InputData(){ Age = 28, Gender = "Male", Education = "PhD", + ExtraColumn = 3 }, + + new InputData(){ Age = 22, Gender = "Male", Education = "BS", + ExtraColumn = 4 }, + + new InputData(){ Age = 23, Gender = "Female", Education = "MS", + ExtraColumn = 5 }, + + new InputData(){ Age = 27, Gender = "Female", Education = "PhD", + ExtraColumn = 6 }, }; // Convert training data to IDataView. @@ -29,22 +40,29 @@ public static void Example() // Select a subset of columns to keep. var pipeline = mlContext.Transforms.SelectColumns("Age", "Education"); - // Now we can transform the data and look at the output to confirm the behavior of SelectColumns. - // Don't forget that this operation doesn't actually evaluate data until we read the data below, - // as transformations are lazy in ML.NET. + // Now we can transform the data and look at the output to confirm the + // behavior of SelectColumns. Don't forget that this operation doesn't + // actually evaluate data until we read the data below, as + // transformations are lazy in ML.NET. var transformedData = pipeline.Fit(dataview).Transform(dataview); // Print the number of columns in the schema - Console.WriteLine($"There are {transformedData.Schema.Count} columns in the dataset."); + Console.WriteLine($"There are {transformedData.Schema.Count} columns" + + $" in the dataset."); // Expected output: // There are 2 columns in the dataset. - // We can extract the newly created column as an IEnumerable of TransformedData, the class we define below. - var rowEnumerable = mlContext.Data.CreateEnumerable(transformedData, reuseRowObject: false); + // We can extract the newly created column as an IEnumerable of + // TransformedData, the class we define below. + var rowEnumerable = mlContext.Data.CreateEnumerable( + transformedData, reuseRowObject: false); + + // And finally, we can write out the rows of the dataset, looking at the + // columns of interest. + Console.WriteLine($"Age and Educations columns obtained " + + $"post-transformation."); - // And finally, we can write out the rows of the dataset, looking at the columns of interest. - Console.WriteLine($"Age and Educations columns obtained post-transformation."); foreach (var row in rowEnumerable) Console.WriteLine($"Age: {row.Age} Education: {row.Education}"); diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/ApplyCustomWordEmbedding.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/ApplyCustomWordEmbedding.cs index 149b662ee5..e1a275f763 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/ApplyCustomWordEmbedding.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/ApplyCustomWordEmbedding.cs @@ -9,12 +9,13 @@ public static class ApplyCustomWordEmbedding { public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(); - // Create an empty list as the dataset. The 'ApplyWordEmbedding' does not require training data as - // the estimator ('WordEmbeddingEstimator') created by 'ApplyWordEmbedding' API is not a trainable estimator. + // Create an empty list as the dataset. The 'ApplyWordEmbedding' does + // not require training data as the estimator ('WordEmbeddingEstimator') + // created by 'ApplyWordEmbedding' API is not a trainable estimator. // The empty list is only needed to pass input schema to the pipeline. var emptySamples = new List(); @@ -33,25 +34,33 @@ public static void Example() file.WriteLine("buy 0 0 20"); } - // A pipeline for converting text into a 9-dimension word embedding vector using the custom word embedding model. - // The 'ApplyWordEmbedding' computes the minimum, average and maximum values for each token's embedding vector. - // Tokens in 'custommodel.txt' model are represented as 3-dimension vector. - // Therefore, the output is of 9-dimension [min, avg, max]. + // A pipeline for converting text into a 9-dimension word embedding + // vector using the custom word embedding model. The + // 'ApplyWordEmbedding' computes the minimum, average and maximum values + // for each token's embedding vector. Tokens in 'custommodel.txt' model + // are represented as 3-dimension vector. Therefore, the output is of + // 9 -dimension [min, avg, max]. // // The 'ApplyWordEmbedding' API requires vector of text as input. - // The pipeline first normalizes and tokenizes text then applies word embedding transformation. + // The pipeline first normalizes and tokenizes text then applies word + // embedding transformation. var textPipeline = mlContext.Transforms.Text.NormalizeText("Text") - .Append(mlContext.Transforms.Text.TokenizeIntoWords("Tokens", "Text")) - .Append(mlContext.Transforms.Text.ApplyWordEmbedding("Features", pathToCustomModel, "Tokens")); + .Append(mlContext.Transforms.Text.TokenizeIntoWords("Tokens", + "Text")) + .Append(mlContext.Transforms.Text.ApplyWordEmbedding("Features", + pathToCustomModel, "Tokens")); // Fit to data. var textTransformer = textPipeline.Fit(emptyDataView); - // Create the prediction engine to get the embedding vector from the input text/string. - var predictionEngine = mlContext.Model.CreatePredictionEngine(textTransformer); + // Create the prediction engine to get the embedding vector from the + // input text/string. + var predictionEngine = mlContext.Model.CreatePredictionEngine(textTransformer); // Call the prediction API to convert the text into embedding vector. - var data = new TextData() { Text = "This is a great product. I would like to buy it again." }; + var data = new TextData() { Text = "This is a great product. I would " + + "like to buy it again." }; var prediction = predictionEngine.Predict(data); // Print the length of the embedding vector. diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/ApplyWordEmbedding.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/ApplyWordEmbedding.cs index d77b0a0a99..cfd1077f8f 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/ApplyWordEmbedding.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/ApplyWordEmbedding.cs @@ -9,38 +9,48 @@ public static class ApplyWordEmbedding { public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(); - // Create an empty list as the dataset. The 'ApplyWordEmbedding' does not require training data as - // the estimator ('WordEmbeddingEstimator') created by 'ApplyWordEmbedding' API is not a trainable estimator. + // Create an empty list as the dataset. The 'ApplyWordEmbedding' does + // not require training data as the estimator ('WordEmbeddingEstimator') + // created by 'ApplyWordEmbedding' API is not a trainable estimator. // The empty list is only needed to pass input schema to the pipeline. var emptySamples = new List(); // Convert sample list to an empty IDataView. var emptyDataView = mlContext.Data.LoadFromEnumerable(emptySamples); - // A pipeline for converting text into a 150-dimension embedding vector using pretrained 'SentimentSpecificWordEmbedding' model. - // The 'ApplyWordEmbedding' computes the minimum, average and maximum values for each token's embedding vector. - // Tokens in 'SentimentSpecificWordEmbedding' model are represented as 50-dimension vector. - // Therefore, the output is of 150-dimension [min, avg, max]. + // A pipeline for converting text into a 150-dimension embedding vector + // using pretrained 'SentimentSpecificWordEmbedding' model. The + // 'ApplyWordEmbedding' computes the minimum, average and maximum values + // for each token's embedding vector. Tokens in + // 'SentimentSpecificWordEmbedding' model are represented as + // 50 -dimension vector. Therefore, the output is of 150-dimension [min, + // avg, max]. // // The 'ApplyWordEmbedding' API requires vector of text as input. - // The pipeline first normalizes and tokenizes text then applies word embedding transformation. + // The pipeline first normalizes and tokenizes text then applies word + // embedding transformation. var textPipeline = mlContext.Transforms.Text.NormalizeText("Text") - .Append(mlContext.Transforms.Text.TokenizeIntoWords("Tokens", "Text")) - .Append(mlContext.Transforms.Text.ApplyWordEmbedding("Features", "Tokens", - WordEmbeddingEstimator.PretrainedModelKind.SentimentSpecificWordEmbedding)); + .Append(mlContext.Transforms.Text.TokenizeIntoWords("Tokens", + "Text")) + .Append(mlContext.Transforms.Text.ApplyWordEmbedding("Features", + "Tokens", WordEmbeddingEstimator.PretrainedModelKind + .SentimentSpecificWordEmbedding)); // Fit to data. var textTransformer = textPipeline.Fit(emptyDataView); - // Create the prediction engine to get the embedding vector from the input text/string. - var predictionEngine = mlContext.Model.CreatePredictionEngine(textTransformer); + // Create the prediction engine to get the embedding vector from the + // input text/string. + var predictionEngine = mlContext.Model.CreatePredictionEngine(textTransformer); // Call the prediction API to convert the text into embedding vector. - var data = new TextData() { Text = "This is a great product. I would like to buy it again." }; + var data = new TextData() { Text = "This is a great product. I would " + + "like to buy it again." }; var prediction = predictionEngine.Predict(data); // Print the length of the embedding vector. diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/FeaturizeText.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/FeaturizeText.cs index 334f70fe76..5b62d0639e 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/FeaturizeText.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/FeaturizeText.cs @@ -8,41 +8,58 @@ public static class FeaturizeText { public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(); // Create a small dataset as an IEnumerable. var samples = new List() { - new TextData(){ Text = "ML.NET's FeaturizeText API uses a composition of several basic transforms to convert text into numeric features." }, - new TextData(){ Text = "This API can be used as a featurizer to perform text classification." }, - new TextData(){ Text = "There are a number of approaches to text classification." }, - new TextData(){ Text = "One of the simplest and most common approaches is called “Bag of Words”." }, - new TextData(){ Text = "Text classification can be used for a wide variety of tasks" }, - new TextData(){ Text = "such as sentiment analysis, topic detection, intent identification etc." }, + new TextData(){ Text = "ML.NET's FeaturizeText API uses a " + + "composition of several basic transforms to convert text " + + "into numeric features." }, + + new TextData(){ Text = "This API can be used as a featurizer to " + + "perform text classification." }, + + new TextData(){ Text = "There are a number of approaches to text " + + "classification." }, + + new TextData(){ Text = "One of the simplest and most common " + + "approaches is called “Bag of Words”." }, + + new TextData(){ Text = "Text classification can be used for a " + + "wide variety of tasks" }, + + new TextData(){ Text = "such as sentiment analysis, topic " + + "detection, intent identification etc." }, }; // Convert training data to IDataView. var dataview = mlContext.Data.LoadFromEnumerable(samples); // A pipeline for converting text into numeric features. - // The following call to 'FeaturizeText' instantiates 'TextFeaturizingEstimator' with default parameters. + // The following call to 'FeaturizeText' instantiates + // 'TextFeaturizingEstimator' with default parameters. // The default settings for the TextFeaturizingEstimator are // * StopWordsRemover: None // * CaseMode: Lowercase // * OutputTokensColumnName: None - // * KeepDiacritics: false, KeepPunctuations: true, KeepNumbers: true + // * KeepDiacritics: false, KeepPunctuations: true, KeepNumbers: + // true // * WordFeatureExtractor: NgramLength = 1 // * CharFeatureExtractor: NgramLength = 3, UseAllLengths = false // The length of the output feature vector depends on these settings. - var textPipeline = mlContext.Transforms.Text.FeaturizeText("Features", "Text"); + var textPipeline = mlContext.Transforms.Text.FeaturizeText("Features", + "Text"); // Fit to data. var textTransformer = textPipeline.Fit(dataview); - // Create the prediction engine to get the features extracted from the text. - var predictionEngine = mlContext.Model.CreatePredictionEngine(textTransformer); + // Create the prediction engine to get the features extracted from the + // text. + var predictionEngine = mlContext.Model.CreatePredictionEngine(textTransformer); // Convert the text into numeric features. var prediction = predictionEngine.Predict(samples[0]); diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/FeaturizeTextWithOptions.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/FeaturizeTextWithOptions.cs index 97a00b6c0f..3f405176bb 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/FeaturizeTextWithOptions.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/FeaturizeTextWithOptions.cs @@ -9,44 +9,65 @@ public static class FeaturizeTextWithOptions { public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(); // Create a small dataset as an IEnumerable. var samples = new List() { - new TextData(){ Text = "ML.NET's FeaturizeText API uses a composition of several basic transforms to convert text into numeric features." }, - new TextData(){ Text = "This API can be used as a featurizer to perform text classification." }, - new TextData(){ Text = "There are a number of approaches to text classification." }, - new TextData(){ Text = "One of the simplest and most common approaches is called “Bag of Words”." }, - new TextData(){ Text = "Text classification can be used for a wide variety of tasks" }, - new TextData(){ Text = "such as sentiment analysis, topic detection, intent identification etc." }, + new TextData(){ Text = "ML.NET's FeaturizeText API uses a " + + "composition of several basic transforms to convert text into " + + "numeric features." }, + + new TextData(){ Text = "This API can be used as a featurizer to " + + "perform text classification." }, + + new TextData(){ Text = "There are a number of approaches to text " + + "classification." }, + + new TextData(){ Text = "One of the simplest and most common " + + "approaches is called “Bag of Words”." }, + + new TextData(){ Text = "Text classification can be used for a " + + "wide variety of tasks" }, + + new TextData(){ Text = "such as sentiment analysis, topic " + + "detection, intent identification etc." }, }; // Convert training data to IDataView. var dataview = mlContext.Data.LoadFromEnumerable(samples); // A pipeline for converting text into numeric features. - // The following call to 'FeaturizeText' instantiates 'TextFeaturizingEstimator' with given parameters. - // The length of the output feature vector depends on these settings. + // The following call to 'FeaturizeText' instantiates + // 'TextFeaturizingEstimator' with given parameters. The length of the + // output feature vector depends on these settings. var options = new TextFeaturizingEstimator.Options() { // Also output tokenized words OutputTokensColumnName = "OutputTokens", CaseMode = TextNormalizingEstimator.CaseMode.Lower, // Use ML.NET's built-in stop word remover - StopWordsRemoverOptions = new StopWordsRemovingEstimator.Options() { Language = TextFeaturizingEstimator.Language.English }, - WordFeatureExtractor = new WordBagEstimator.Options() { NgramLength = 2, UseAllLengths = true }, - CharFeatureExtractor = new WordBagEstimator.Options() { NgramLength = 3, UseAllLengths= false }, + StopWordsRemoverOptions = new StopWordsRemovingEstimator.Options() { + Language = TextFeaturizingEstimator.Language.English }, + + WordFeatureExtractor = new WordBagEstimator.Options() { NgramLength + = 2, UseAllLengths = true }, + + CharFeatureExtractor = new WordBagEstimator.Options() { NgramLength + = 3, UseAllLengths= false }, }; - var textPipeline = mlContext.Transforms.Text.FeaturizeText("Features", options, "Text"); + var textPipeline = mlContext.Transforms.Text.FeaturizeText("Features", + options, "Text"); // Fit to data. var textTransformer = textPipeline.Fit(dataview); - // Create the prediction engine to get the features extracted from the text. - var predictionEngine = mlContext.Model.CreatePredictionEngine(textTransformer); + // Create the prediction engine to get the features extracted from the + // text. + var predictionEngine = mlContext.Model.CreatePredictionEngine(textTransformer); // Convert the text into numeric features. var prediction = predictionEngine.Predict(samples[0]); @@ -59,7 +80,8 @@ public static void Example() for (int i = 0; i < 10; i++) Console.Write($"{prediction.Features[i]:F4} "); - Console.WriteLine($"\nTokens: {string.Join(",", prediction.OutputTokens)}"); + Console.WriteLine("\nTokens: " + string.Join(",", prediction + .OutputTokens)); // Expected output: // Number of Features: 282 diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/LatentDirichletAllocation.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/LatentDirichletAllocation.cs index 1e296111e0..15ce82e5b2 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/LatentDirichletAllocation.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/LatentDirichletAllocation.cs @@ -8,46 +8,61 @@ public static class LatentDirichletAllocation { public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(); // Create a small dataset as an IEnumerable. var samples = new List() { - new TextData(){ Text = "ML.NET's LatentDirichletAllocation API computes topic models." }, - new TextData(){ Text = "ML.NET's LatentDirichletAllocation API is the best for topic models." }, + new TextData(){ Text = "ML.NET's LatentDirichletAllocation API " + + "computes topic models." }, + + new TextData(){ Text = "ML.NET's LatentDirichletAllocation API " + + "is the best for topic models." }, + new TextData(){ Text = "I like to eat broccoli and bananas." }, new TextData(){ Text = "I eat bananas for breakfast." }, - new TextData(){ Text = "This car is expensive compared to last week's price." }, + new TextData(){ Text = "This car is expensive compared to last " + + "week's price." }, + new TextData(){ Text = "This car was $X last week." }, }; // Convert training data to IDataView. var dataview = mlContext.Data.LoadFromEnumerable(samples); - // A pipeline for featurizing the text/string using LatentDirichletAllocation API. - // To be more accurate in computing the LDA features, the pipeline first normalizes text and removes stop words - // before passing tokens (the individual words, lower cased, with common words removed) to LatentDirichletAllocation. - var pipeline = mlContext.Transforms.Text.NormalizeText("NormalizedText", "Text") - .Append(mlContext.Transforms.Text.TokenizeIntoWords("Tokens", "NormalizedText")) + // A pipeline for featurizing the text/string using + // LatentDirichletAllocation API. o be more accurate in computing the + // LDA features, the pipeline first normalizes text and removes stop + // words before passing tokens (the individual words, lower cased, with + // common words removed) to LatentDirichletAllocation. + var pipeline = mlContext.Transforms.Text.NormalizeText("NormalizedText", + "Text") + .Append(mlContext.Transforms.Text.TokenizeIntoWords("Tokens", + "NormalizedText")) .Append(mlContext.Transforms.Text.RemoveDefaultStopWords("Tokens")) .Append(mlContext.Transforms.Conversion.MapValueToKey("Tokens")) .Append(mlContext.Transforms.Text.ProduceNgrams("Tokens")) - .Append(mlContext.Transforms.Text.LatentDirichletAllocation("Features", "Tokens", numberOfTopics: 3)); + .Append(mlContext.Transforms.Text.LatentDirichletAllocation( + "Features", "Tokens", numberOfTopics: 3)); // Fit to data. var transformer = pipeline.Fit(dataview); - // Create the prediction engine to get the LDA features extracted from the text. - var predictionEngine = mlContext.Model.CreatePredictionEngine(transformer); + // Create the prediction engine to get the LDA features extracted from + // the text. + var predictionEngine = mlContext.Model.CreatePredictionEngine(transformer); // Convert the sample text into LDA features and print it. PrintLdaFeatures(predictionEngine.Predict(samples[0])); PrintLdaFeatures(predictionEngine.Predict(samples[1])); // Features obtained post-transformation. - // For LatentDirichletAllocation, we had specified numTopic:3. Hence each prediction has been featurized as a vector of floats with length 3. + // For LatentDirichletAllocation, we had specified numTopic:3. Hence + // each prediction has been featurized as a vector of floats with length + // 3. // Topic1 Topic2 Topic3 // 0.6364 0.2727 0.0909 diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/NormalizeText.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/NormalizeText.cs index e198979eda..2100c13371 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/NormalizeText.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/NormalizeText.cs @@ -9,21 +9,22 @@ public static class NormalizeText { public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(); - // Create an empty list as the dataset. The 'NormalizeText' API does not require training data as - // the estimator ('TextNormalizingEstimator') created by 'NormalizeText' API is not a trainable estimator. - // The empty list is only needed to pass input schema to the pipeline. + // Create an empty list as the dataset. The 'NormalizeText' API does not + // require training data as the estimator ('TextNormalizingEstimator') + // created by 'NormalizeText' API is not a trainable estimator. The + // empty list is only needed to pass input schema to the pipeline. var emptySamples = new List(); // Convert sample list to an empty IDataView. var emptyDataView = mlContext.Data.LoadFromEnumerable(emptySamples); // A pipeline for normalizing text. - var normTextPipeline = mlContext.Transforms.Text.NormalizeText("NormalizedText", "Text", - TextNormalizingEstimator.CaseMode.Lower, + var normTextPipeline = mlContext.Transforms.Text.NormalizeText( + "NormalizedText", "Text", TextNormalizingEstimator.CaseMode.Lower, keepDiacritics: false, keepPunctuations: false, keepNumbers: false); @@ -31,11 +32,16 @@ public static void Example() // Fit to data. var normTextTransformer = normTextPipeline.Fit(emptyDataView); - // Create the prediction engine to get the normalized text from the input text/string. - var predictionEngine = mlContext.Model.CreatePredictionEngine(normTextTransformer); + // Create the prediction engine to get the normalized text from the + // input text/string. + var predictionEngine = mlContext.Model.CreatePredictionEngine(normTextTransformer); // Call the prediction API. - var data = new TextData() { Text = "ML.NET's NormalizeText API changes the case of the TEXT and removes/keeps diâcrîtîcs, punctuations, and/or numbers (123)." }; + var data = new TextData() { Text = "ML.NET's NormalizeText API " + + "changes the case of the TEXT and removes/keeps diâcrîtîcs, " + + "punctuations, and/or numbers (123)." }; + var prediction = predictionEngine.Predict(data); // Print the normalized text. diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/ProduceHashedNgrams.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/ProduceHashedNgrams.cs index ce6f7816be..4d990d0a42 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/ProduceHashedNgrams.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/ProduceHashedNgrams.cs @@ -9,54 +9,74 @@ public static class ProduceHashedNgrams { public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(); // Create a small dataset as an IEnumerable. var samples = new List() { - new TextData(){ Text = "This is an example to compute n-grams using hashing." }, - new TextData(){ Text = "N-gram is a sequence of 'N' consecutive words/tokens." }, - new TextData(){ Text = "ML.NET's ProduceHashedNgrams API produces count of n-grams and hashes it as an index into a vector of given bit length." }, - new TextData(){ Text = "The hashing reduces the size of the output feature vector" }, - new TextData(){ Text = "which is useful in case when number of n-grams is very large." }, + new TextData(){ Text = "This is an example to compute n-grams " + + "using hashing." }, + + new TextData(){ Text = "N-gram is a sequence of 'N' consecutive" + + " words/tokens." }, + + new TextData(){ Text = "ML.NET's ProduceHashedNgrams API " + + "produces count of n-grams and hashes it as an index into a " + + "vector of given bit length." }, + + new TextData(){ Text = "The hashing reduces the size of the " + + "output feature vector" }, + + new TextData(){ Text = "which is useful in case when number of " + + "n-grams is very large." }, }; // Convert training data to IDataView. var dataview = mlContext.Data.LoadFromEnumerable(samples); // A pipeline for converting text into numeric hashed n-gram features. - // The following call to 'ProduceHashedNgrams' requires the tokenized text/string as input. - // This is acheived by calling 'TokenizeIntoWords' first followed by 'ProduceHashedNgrams'. - // Please note that the length of the output feature vector depends on the 'numberOfBits' settings. - var textPipeline = mlContext.Transforms.Text.TokenizeIntoWords("Tokens", "Text") + // The following call to 'ProduceHashedNgrams' requires the tokenized + // text /string as input. This is acheived by calling + // 'TokenizeIntoWords' first followed by 'ProduceHashedNgrams'. + // Please note that the length of the output feature vector depends on + // the 'numberOfBits' settings. + var textPipeline = mlContext.Transforms.Text.TokenizeIntoWords("Tokens", + "Text") .Append(mlContext.Transforms.Conversion.MapValueToKey("Tokens")) - .Append(mlContext.Transforms.Text.ProduceHashedNgrams("NgramFeatures", "Tokens", - numberOfBits: 5, - ngramLength: 3, - useAllLengths: false, - maximumNumberOfInverts: 1)); + .Append(mlContext.Transforms.Text.ProduceHashedNgrams( + "NgramFeatures", "Tokens", + numberOfBits: 5, + ngramLength: 3, + useAllLengths: false, + maximumNumberOfInverts: 1)); // Fit to data. var textTransformer = textPipeline.Fit(dataview); var transformedDataView = textTransformer.Transform(dataview); - // Create the prediction engine to get the features extracted from the text. - var predictionEngine = mlContext.Model.CreatePredictionEngine(textTransformer); + // Create the prediction engine to get the features extracted from the + // text. + var predictionEngine = mlContext.Model.CreatePredictionEngine(textTransformer); // Convert the text into numeric features. var prediction = predictionEngine.Predict(samples[0]); // Print the length of the feature vector. - Console.WriteLine($"Number of Features: {prediction.NgramFeatures.Length}"); + Console.WriteLine("Number of Features: " + prediction.NgramFeatures + .Length); // Preview of the produced n-grams. // Get the slot names from the column's metadata. - // The slot names for a vector column corresponds to the names associated with each position in the vector. + // The slot names for a vector column corresponds to the names + // associated with each position in the vector. VBuffer> slotNames = default; transformedDataView.Schema["NgramFeatures"].GetSlotNames(ref slotNames); - var NgramFeaturesColumn = transformedDataView.GetColumn>(transformedDataView.Schema["NgramFeatures"]); + var NgramFeaturesColumn = transformedDataView.GetColumn>( + transformedDataView.Schema["NgramFeatures"]); + var slots = slotNames.GetValues(); Console.Write("N-grams: "); foreach (var featureRow in NgramFeaturesColumn) diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/ProduceHashedWordBags.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/ProduceHashedWordBags.cs index 48a5971b2d..05d88951bc 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/ProduceHashedWordBags.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/ProduceHashedWordBags.cs @@ -9,28 +9,43 @@ public static class ProduceHashedWordBags { public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(); // Create a small dataset as an IEnumerable. var samples = new List() { - new TextData(){ Text = "This is an example to compute bag-of-word features using hashing." }, - new TextData(){ Text = "ML.NET's ProduceHashedWordBags API produces count of n-grams and hashes it as an index into a vector of given bit length." }, - new TextData(){ Text = "It does so by first tokenizing text/string into words/tokens then " }, - new TextData(){ Text = "computing n-grams and hash them to the index given by hash value." }, - new TextData(){ Text = "The hashing reduces the size of the output feature vector" }, - new TextData(){ Text = "which is useful in case when number of n-grams is very large." }, + new TextData(){ Text = "This is an example to compute " + + "bag-of-word features using hashing." }, + + new TextData(){ Text = "ML.NET's ProduceHashedWordBags API " + + "produces count of n-grams and hashes it as an index into " + + "a vector of given bit length." }, + + new TextData(){ Text = "It does so by first tokenizing " + + "text/string into words/tokens then " }, + + new TextData(){ Text = "computing n-grams and hash them to the " + + "index given by hash value." }, + + new TextData(){ Text = "The hashing reduces the size of the " + + "output feature vector" }, + + new TextData(){ Text = "which is useful in case when number of" + + " n-grams is very large." }, }; // Convert training data to IDataView. var dataview = mlContext.Data.LoadFromEnumerable(samples); - // A pipeline for converting text into numeric bag-of-word features using hashing. - // The following call to 'ProduceHashedWordBags' implicitly tokenizes the text/string into words/tokens. - // Please note that the length of the output feature vector depends on the 'numberOfBits' settings. - var textPipeline = mlContext.Transforms.Text.ProduceHashedWordBags("BagOfWordFeatures", "Text", + // A pipeline for converting text into numeric bag-of-word features + // using hashing. The following call to 'ProduceHashedWordBags' + // implicitly tokenizes the text/string into words/tokens. Please note + // that the length of the output feature vector depends on the + // 'numberOfBits' settings. + var textPipeline = mlContext.Transforms.Text.ProduceHashedWordBags( + "BagOfWordFeatures", "Text", numberOfBits: 5, ngramLength: 3, useAllLengths: false, @@ -40,21 +55,29 @@ public static void Example() var textTransformer = textPipeline.Fit(dataview); var transformedDataView = textTransformer.Transform(dataview); - // Create the prediction engine to get the bag-of-word features extracted from the text. - var predictionEngine = mlContext.Model.CreatePredictionEngine(textTransformer); + // Create the prediction engine to get the bag-of-word features + // extracted from the text. + var predictionEngine = mlContext.Model.CreatePredictionEngine(textTransformer); // Convert the text into numeric features. var prediction = predictionEngine.Predict(samples[0]); // Print the length of the feature vector. - Console.WriteLine($"Number of Features: {prediction.BagOfWordFeatures.Length}"); + Console.WriteLine("Number of Features: " + prediction.BagOfWordFeatures + .Length); // Preview of the produced n-grams. // Get the slot names from the column's metadata. - // The slot names for a vector column corresponds to the names associated with each position in the vector. + // The slot names for a vector column corresponds to the names + // associated with each position in the vector. VBuffer> slotNames = default; - transformedDataView.Schema["BagOfWordFeatures"].GetSlotNames(ref slotNames); - var BagOfWordFeaturesColumn = transformedDataView.GetColumn>(transformedDataView.Schema["BagOfWordFeatures"]); + transformedDataView.Schema["BagOfWordFeatures"].GetSlotNames(ref + slotNames); + + var BagOfWordFeaturesColumn = transformedDataView.GetColumn>(transformedDataView.Schema["BagOfWordFeatures"]); + var slots = slotNames.GetValues(); Console.Write("N-grams: "); foreach (var featureRow in BagOfWordFeaturesColumn) diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/ProduceNgrams.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/ProduceNgrams.cs index 2a41cde412..9b124daf5b 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/ProduceNgrams.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/ProduceNgrams.cs @@ -10,34 +10,52 @@ public static class ProduceNgrams { public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(); // Create a small dataset as an IEnumerable. var samples = new List() { new TextData(){ Text = "This is an example to compute n-grams." }, - new TextData(){ Text = "N-gram is a sequence of 'N' consecutive words/tokens." }, - new TextData(){ Text = "ML.NET's ProduceNgrams API produces vector of n-grams." }, - new TextData(){ Text = "Each position in the vector corresponds to a particular n-gram." }, - new TextData(){ Text = "The value at each position corresponds to," }, - new TextData(){ Text = "the number of times n-gram occured in the data (Tf), or" }, - new TextData(){ Text = "the inverse of the number of documents that contain the n-gram (Idf)," }, - new TextData(){ Text = "or compute both and multiply together (Tf-Idf)." }, + new TextData(){ Text = "N-gram is a sequence of 'N' consecutive " + + "words/tokens." }, + + new TextData(){ Text = "ML.NET's ProduceNgrams API produces " + + "vector of n-grams." }, + + new TextData(){ Text = "Each position in the vector corresponds " + + "to a particular n-gram." }, + + new TextData(){ Text = "The value at each position corresponds " + + "to," }, + + new TextData(){ Text = "the number of times n-gram occured in " + + "the data (Tf), or" }, + + new TextData(){ Text = "the inverse of the number of documents " + + "that contain the n-gram (Idf)," }, + + new TextData(){ Text = "or compute both and multiply together " + + "(Tf-Idf)." }, }; // Convert training data to IDataView. var dataview = mlContext.Data.LoadFromEnumerable(samples); // A pipeline for converting text into numeric n-gram features. - // The following call to 'ProduceNgrams' requires the tokenized text/string as input. - // This is acheived by calling 'TokenizeIntoWords' first followed by 'ProduceNgrams'. - // Please note that the length of the output feature vector depends on the n-gram settings. - var textPipeline = mlContext.Transforms.Text.TokenizeIntoWords("Tokens", "Text") - // 'ProduceNgrams' takes key type as input. Converting the tokens into key type using 'MapValueToKey'. + // The following call to 'ProduceNgrams' requires the tokenized + // text /string as input. This is acheived by calling + // 'TokenizeIntoWords' first followed by 'ProduceNgrams'. Please note + // that the length of the output feature vector depends on the n-gram + // settings. + var textPipeline = mlContext.Transforms.Text.TokenizeIntoWords("Tokens", + "Text") + // 'ProduceNgrams' takes key type as input. Converting the tokens + // into key type using 'MapValueToKey'. .Append(mlContext.Transforms.Conversion.MapValueToKey("Tokens")) - .Append(mlContext.Transforms.Text.ProduceNgrams("NgramFeatures", "Tokens", + .Append(mlContext.Transforms.Text.ProduceNgrams("NgramFeatures", + "Tokens", ngramLength: 3, useAllLengths: false, weighting: NgramExtractingEstimator.WeightingCriteria.Tf)); @@ -46,21 +64,26 @@ public static void Example() var textTransformer = textPipeline.Fit(dataview); var transformedDataView = textTransformer.Transform(dataview); - // Create the prediction engine to get the n-gram features extracted from the text. - var predictionEngine = mlContext.Model.CreatePredictionEngine(textTransformer); + // Create the prediction engine to get the n-gram features extracted + // from the text. + var predictionEngine = mlContext.Model.CreatePredictionEngine(textTransformer); // Convert the text into numeric features. var prediction = predictionEngine.Predict(samples[0]); // Print the length of the feature vector. - Console.WriteLine($"Number of Features: {prediction.NgramFeatures.Length}"); + Console.WriteLine("Number of Features: " + prediction.NgramFeatures + .Length); // Preview of the produced n-grams. // Get the slot names from the column's metadata. - // The slot names for a vector column corresponds to the names associated with each position in the vector. + // The slot names for a vector column corresponds to the names + // associated with each position in the vector. VBuffer> slotNames = default; transformedDataView.Schema["NgramFeatures"].GetSlotNames(ref slotNames); - var NgramFeaturesColumn = transformedDataView.GetColumn>(transformedDataView.Schema["NgramFeatures"]); + var NgramFeaturesColumn = transformedDataView.GetColumn>(transformedDataView.Schema["NgramFeatures"]); var slots = slotNames.GetValues(); Console.Write("N-grams: "); foreach (var featureRow in NgramFeaturesColumn) diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/ProduceWordBags.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/ProduceWordBags.cs index 7745ca3fdd..a6feae3582 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/ProduceWordBags.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/ProduceWordBags.cs @@ -10,52 +10,80 @@ public static class ProduceWordBags { public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(); // Create a small dataset as an IEnumerable. var samples = new List() { - new TextData(){ Text = "This is an example to compute bag-of-word features." }, - new TextData(){ Text = "ML.NET's ProduceWordBags API produces bag-of-word features from input text." }, - new TextData(){ Text = "It does so by first tokenizing text/string into words/tokens then " }, - new TextData(){ Text = "computing n-grams and their neumeric values." }, - new TextData(){ Text = "Each position in the output vector corresponds to a particular n-gram." }, - new TextData(){ Text = "The value at each position corresponds to," }, - new TextData(){ Text = "the number of times n-gram occured in the data (Tf), or" }, - new TextData(){ Text = "the inverse of the number of documents contain the n-gram (Idf)," }, - new TextData(){ Text = "or compute both and multipy together (Tf-Idf)." }, + new TextData(){ Text = "This is an example to compute " + + "bag-of-word features." }, + + new TextData(){ Text = "ML.NET's ProduceWordBags API produces " + + "bag-of-word features from input text." }, + + new TextData(){ Text = "It does so by first tokenizing " + + "text/string into words/tokens then " }, + + new TextData(){ Text = "computing n-grams and their neumeric " + + "values." }, + + new TextData(){ Text = "Each position in the output vector " + + "corresponds to a particular n-gram." }, + + new TextData(){ Text = "The value at each position corresponds " + + "to," }, + + new TextData(){ Text = "the number of times n-gram occured in " + + "the data (Tf), or" }, + + new TextData(){ Text = "the inverse of the number of documents " + + "contain the n-gram (Idf)," }, + + new TextData(){ Text = "or compute both and multipy together " + + "(Tf-Idf)." }, }; // Convert training data to IDataView. var dataview = mlContext.Data.LoadFromEnumerable(samples); // A pipeline for converting text into numeric bag-of-word features. - // The following call to 'ProduceWordBags' implicitly tokenizes the text/string into words/tokens. - // Please note that the length of the output feature vector depends on the n-gram settings. - var textPipeline = mlContext.Transforms.Text.ProduceWordBags("BagOfWordFeatures", "Text", - ngramLength: 3, useAllLengths: false, weighting: NgramExtractingEstimator.WeightingCriteria.Tf); + // The following call to 'ProduceWordBags' implicitly tokenizes the + // text /string into words/tokens. Please note that the length of the + // output feature vector depends on the n-gram settings. + var textPipeline = mlContext.Transforms.Text.ProduceWordBags( + "BagOfWordFeatures", "Text", + ngramLength: 3, useAllLengths: false, + weighting: NgramExtractingEstimator.WeightingCriteria.Tf); // Fit to data. var textTransformer = textPipeline.Fit(dataview); var transformedDataView = textTransformer.Transform(dataview); - // Create the prediction engine to get the bag-of-word features extracted from the text. - var predictionEngine = mlContext.Model.CreatePredictionEngine(textTransformer); + // Create the prediction engine to get the bag-of-word features + // extracted from the text. + var predictionEngine = mlContext.Model.CreatePredictionEngine(textTransformer); // Convert the text into numeric features. var prediction = predictionEngine.Predict(samples[0]); // Print the length of the feature vector. - Console.WriteLine($"Number of Features: {prediction.BagOfWordFeatures.Length}"); + Console.WriteLine("Number of Features: " + prediction.BagOfWordFeatures + .Length); // Preview of the produced n-grams. // Get the slot names from the column's metadata. - // The slot names for a vector column corresponds to the names associated with each position in the vector. + // The slot names for a vector column corresponds to the names + // associated with each position in the vector. VBuffer> slotNames = default; - transformedDataView.Schema["BagOfWordFeatures"].GetSlotNames(ref slotNames); - var BagOfWordFeaturesColumn = transformedDataView.GetColumn>(transformedDataView.Schema["BagOfWordFeatures"]); + transformedDataView.Schema["BagOfWordFeatures"].GetSlotNames(ref + slotNames); + + var BagOfWordFeaturesColumn = transformedDataView.GetColumn>(transformedDataView.Schema["BagOfWordFeatures"]); + var slots = slotNames.GetValues(); Console.Write("N-grams: "); foreach (var featureRow in BagOfWordFeaturesColumn) diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/RemoveDefaultStopWords.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/RemoveDefaultStopWords.cs index 8c9e61cf5f..6147bf155e 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/RemoveDefaultStopWords.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/RemoveDefaultStopWords.cs @@ -9,13 +9,15 @@ public static class RemoveDefaultStopWords { public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(); - // Create an empty list as the dataset. The 'RemoveDefaultStopWords' does not require training data as - // the estimator ('StopWordsRemovingEstimator') created by 'RemoveDefaultStopWords' API is not a trainable estimator. - // The empty list is only needed to pass input schema to the pipeline. + // Create an empty list as the dataset. The 'RemoveDefaultStopWords' + // does not require training data as the estimator + // ('StopWordsRemovingEstimator') created by 'RemoveDefaultStopWords' + // API is not a trainable estimator. The empty list is only needed to + // pass input schema to the pipeline. var emptySamples = new List(); // Convert sample list to an empty IDataView. @@ -23,25 +25,36 @@ public static void Example() // A pipeline for removing stop words from input text/string. // The pipeline first tokenizes text into words then removes stop words. - // The 'RemoveDefaultStopWords' API ignores casing of the text/string e.g. 'tHe' and 'the' are considered the same stop words. - var textPipeline = mlContext.Transforms.Text.TokenizeIntoWords("Words", "Text") - .Append(mlContext.Transforms.Text.RemoveDefaultStopWords("WordsWithoutStopWords", "Words", language: StopWordsRemovingEstimator.Language.English)); + // The 'RemoveDefaultStopWords' API ignores casing of the text/string + // e.g. 'tHe' and 'the' are considered the same stop words. + var textPipeline = mlContext.Transforms.Text.TokenizeIntoWords("Words", + "Text") + .Append(mlContext.Transforms.Text.RemoveDefaultStopWords( + "WordsWithoutStopWords", "Words", language: + StopWordsRemovingEstimator.Language.English)); // Fit to data. var textTransformer = textPipeline.Fit(emptyDataView); - // Create the prediction engine to remove the stop words from the input text/string. - var predictionEngine = mlContext.Model.CreatePredictionEngine(textTransformer); + // Create the prediction engine to remove the stop words from the input + // text /string. + var predictionEngine = mlContext.Model.CreatePredictionEngine(textTransformer); // Call the prediction API to remove stop words. - var data = new TextData() { Text = "ML.NET's RemoveDefaultStopWords API removes stop words from tHe text/string. It requires the text/string to be tokenized beforehand." }; + var data = new TextData() { Text = "ML.NET's RemoveDefaultStopWords " + + "API removes stop words from tHe text/string. It requires the " + + "text/string to be tokenized beforehand." }; + var prediction = predictionEngine.Predict(data); // Print the length of the word vector after the stop words removed. - Console.WriteLine($"Number of words: {prediction.WordsWithoutStopWords.Length}"); + Console.WriteLine("Number of words: " + prediction.WordsWithoutStopWords + .Length); // Print the word vector without stop words. - Console.WriteLine($"\nWords without stop words: {string.Join(",", prediction.WordsWithoutStopWords)}"); + Console.WriteLine("\nWords without stop words: " + string.Join(",", + prediction.WordsWithoutStopWords)); // Expected output: // Number of words: 11 diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/RemoveStopWords.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/RemoveStopWords.cs index 2596688a50..55a0f0c955 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/RemoveStopWords.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/RemoveStopWords.cs @@ -8,13 +8,15 @@ public static class RemoveStopWords { public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(); - // Create an empty list as the dataset. The 'RemoveStopWords' does not require training data as - // the estimator ('CustomStopWordsRemovingEstimator') created by 'RemoveStopWords' API is not a trainable estimator. - // The empty list is only needed to pass input schema to the pipeline. + // Create an empty list as the dataset. The 'RemoveStopWords' does not + // require training data as the estimator + // ('CustomStopWordsRemovingEstimator') created by 'RemoveStopWords' API + // is not a trainable estimator. The empty list is only needed to pass + // input schema to the pipeline. var emptySamples = new List(); // Convert sample list to an empty IDataView. @@ -22,25 +24,36 @@ public static void Example() // A pipeline for removing stop words from input text/string. // The pipeline first tokenizes text into words then removes stop words. - // The 'RemoveStopWords' API ignores casing of the text/string e.g. 'tHe' and 'the' are considered the same stop words. - var textPipeline = mlContext.Transforms.Text.TokenizeIntoWords("Words", "Text") - .Append(mlContext.Transforms.Text.RemoveStopWords("WordsWithoutStopWords", "Words", stopwords: new[] { "a", "the", "from", "by" })); + // The 'RemoveStopWords' API ignores casing of the text/string e.g. + // 'tHe' and 'the' are considered the same stop words. + var textPipeline = mlContext.Transforms.Text.TokenizeIntoWords("Words", + "Text") + .Append(mlContext.Transforms.Text.RemoveStopWords( + "WordsWithoutStopWords", "Words", stopwords: + new[] { "a", "the","from", "by" })); // Fit to data. var textTransformer = textPipeline.Fit(emptyDataView); - // Create the prediction engine to remove the stop words from the input text/string. - var predictionEngine = mlContext.Model.CreatePredictionEngine(textTransformer); + // Create the prediction engine to remove the stop words from the input + // text /string. + var predictionEngine = mlContext.Model.CreatePredictionEngine(textTransformer); // Call the prediction API to remove stop words. - var data = new TextData() { Text = "ML.NET's RemoveStopWords API removes stop words from tHe text/string using a list of stop words provided by the user." }; + var data = new TextData() { Text = "ML.NET's RemoveStopWords API " + + "removes stop words from tHe text/string using a list of stop " + + "words provided by the user." }; + var prediction = predictionEngine.Predict(data); // Print the length of the word vector after the stop words removed. - Console.WriteLine($"Number of words: {prediction.WordsWithoutStopWords.Length}"); + Console.WriteLine("Number of words: " + prediction.WordsWithoutStopWords + .Length); // Print the word vector without stop words. - Console.WriteLine($"\nWords without stop words: {string.Join(",", prediction.WordsWithoutStopWords)}"); + Console.WriteLine("\nWords without stop words: " + string.Join(",", + prediction.WordsWithoutStopWords)); // Expected output: // Number of words: 14 diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/TokenizeIntoCharactersAsKeys.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/TokenizeIntoCharactersAsKeys.cs index 7815eec0d3..ff984b4d46 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/TokenizeIntoCharactersAsKeys.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/TokenizeIntoCharactersAsKeys.cs @@ -8,12 +8,14 @@ public static class TokenizeIntoCharactersAsKeys { public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(); - // Create an empty list as the dataset. The 'TokenizeIntoCharactersAsKeys' does not require training data as - // the estimator ('TokenizingByCharactersEstimator') created by 'TokenizeIntoCharactersAsKeys' API is not a trainable estimator. + // Create an empty list as the dataset. The + // 'TokenizeIntoCharactersAsKeys' does not require training data as + // the estimator ('TokenizingByCharactersEstimator') created by + // 'TokenizeIntoCharactersAsKeys' API is not a trainable estimator. // The empty list is only needed to pass input schema to the pipeline. var emptySamples = new List(); @@ -23,24 +25,33 @@ public static void Example() // A pipeline for converting text into vector of characters. // The 'TokenizeIntoCharactersAsKeys' produces result as key type. // 'MapKeyToValue' is need to map keys back to their original values. - var textPipeline = mlContext.Transforms.Text.TokenizeIntoCharactersAsKeys("CharTokens", "Text", useMarkerCharacters: false) - .Append(mlContext.Transforms.Conversion.MapKeyToValue("CharTokens")); + var textPipeline = mlContext.Transforms.Text + .TokenizeIntoCharactersAsKeys("CharTokens", "Text", + useMarkerCharacters: false) + .Append(mlContext.Transforms.Conversion.MapKeyToValue( + "CharTokens")); // Fit to data. var textTransformer = textPipeline.Fit(emptyDataView); - // Create the prediction engine to get the character vector from the input text/string. - var predictionEngine = mlContext.Model.CreatePredictionEngine(textTransformer); + // Create the prediction engine to get the character vector from the + // input text/string. + var predictionEngine = mlContext.Model.CreatePredictionEngine(textTransformer); // Call the prediction API to convert the text into characters. - var data = new TextData() { Text = "ML.NET's TokenizeIntoCharactersAsKeys API splits text/string into characters." }; + var data = new TextData() { Text = "ML.NET's " + + "TokenizeIntoCharactersAsKeys API splits text/string into " + + "characters." }; + var prediction = predictionEngine.Predict(data); // Print the length of the character vector. Console.WriteLine($"Number of tokens: {prediction.CharTokens.Length}"); // Print the character vector. - Console.WriteLine($"\nCharacter Tokens: {string.Join(",", prediction.CharTokens)}"); + Console.WriteLine("\nCharacter Tokens: " + string.Join(",", prediction + .CharTokens)); // Expected output: // Number of tokens: 77 diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/TokenizeIntoWords.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/TokenizeIntoWords.cs index bfc61b8a96..ad0826ad19 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/TokenizeIntoWords.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Text/TokenizeIntoWords.cs @@ -8,31 +8,40 @@ public static class TokenizeIntoWords { public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(); - // Create an empty list as the dataset. The 'TokenizeIntoWords' does not require training data as - // the estimator ('WordTokenizingEstimator') created by 'TokenizeIntoWords' API is not a trainable estimator. - // The empty list is only needed to pass input schema to the pipeline. + // Create an empty list as the dataset. The 'TokenizeIntoWords' does + // not require training data as the estimator + // ('WordTokenizingEstimator') created by 'TokenizeIntoWords' API is not + // a trainable estimator. The empty list is only needed to pass input + // schema to the pipeline. var emptySamples = new List(); // Convert sample list to an empty IDataView. var emptyDataView = mlContext.Data.LoadFromEnumerable(emptySamples); // A pipeline for converting text into vector of words. - // The following call to 'TokenizeIntoWords' tokenizes text/string into words using space as a separator. - // Space is also a default value for the 'separators' argument if it is not specified. - var textPipeline = mlContext.Transforms.Text.TokenizeIntoWords("Words", "Text", separators: new[] { ' ' }); + // The following call to 'TokenizeIntoWords' tokenizes text/string into + // words using space as a separator. Space is also a default value for + // the 'separators' argument if it is not specified. + var textPipeline = mlContext.Transforms.Text.TokenizeIntoWords("Words", + "Text", separators: new[] { ' ' }); // Fit to data. var textTransformer = textPipeline.Fit(emptyDataView); - // Create the prediction engine to get the word vector from the input text/string. - var predictionEngine = mlContext.Model.CreatePredictionEngine(textTransformer); + // Create the prediction engine to get the word vector from the input + // text /string. + var predictionEngine = mlContext.Model.CreatePredictionEngine(textTransformer); // Call the prediction API to convert the text into words. - var data = new TextData() { Text = "ML.NET's TokenizeIntoWords API splits text/string into words using the list of characters provided as separators." }; + var data = new TextData() { Text = "ML.NET's TokenizeIntoWords API " + + "splits text/string into words using the list of characters " + + "provided as separators." }; + var prediction = predictionEngine.Predict(data); // Print the length of the word vector. diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectAnomalyBySrCnn.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectAnomalyBySrCnn.cs index 9b17e86244..db29b3ef82 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectAnomalyBySrCnn.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectAnomalyBySrCnn.cs @@ -9,12 +9,13 @@ namespace Samples.Dynamic { public static class DetectAnomalyBySrCnn { - // This example creates a time series (list of Data with the i-th element corresponding to the i-th time slot). - // The estimator is applied then to identify spiking points in the series. + // This example creates a time series (list of Data with the i-th element + // corresponding to the i-th time slot). The estimator is applied then to + // identify spiking points in the series. public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var ml = new MLContext(); // Generate sample series data with an anomaly @@ -37,12 +38,17 @@ public static void Example() string inputColumnName = nameof(TimeSeriesData.Value); // The transformed model. - ITransformer model = ml.Transforms.DetectAnomalyBySrCnn(outputColumnName, inputColumnName, 16, 5, 5, 3, 8, 0.35).Fit(dataView); + ITransformer model = ml.Transforms.DetectAnomalyBySrCnn( + outputColumnName, inputColumnName, 16, 5, 5, 3, 8, 0.35).Fit( + dataView); // Create a time series prediction engine from the model. - var engine = model.CreateTimeSeriesPredictionFunction(ml); + var engine = model.CreateTimeSeriesEngine(ml); + + Console.WriteLine($"{outputColumnName} column obtained post-" + + $"transformation."); - Console.WriteLine($"{outputColumnName} column obtained post-transformation."); Console.WriteLine("Data\tAlert\tScore\tMag"); // Prediction column obtained post-transformation. @@ -102,9 +108,10 @@ public static void Example() //5 0 0.01 0.25 } - private static void PrintPrediction(float value, SrCnnAnomalyDetection prediction) => - Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}", value, prediction.Prediction[0], - prediction.Prediction[1], prediction.Prediction[2]); + private static void PrintPrediction(float value, SrCnnAnomalyDetection + prediction) => + Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}", value, prediction + .Prediction[0], prediction.Prediction[1], prediction.Prediction[2]); private class TimeSeriesData { diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectAnomalyBySrCnnBatchPrediction.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectAnomalyBySrCnnBatchPrediction.cs index ef1d2a0de9..5ef334bc76 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectAnomalyBySrCnnBatchPrediction.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectAnomalyBySrCnnBatchPrediction.cs @@ -9,7 +9,8 @@ public static class DetectAnomalyBySrCnnBatchPrediction { public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, // as well as the source of randomness. var ml = new MLContext(); @@ -33,12 +34,18 @@ public static void Example() string inputColumnName = nameof(TimeSeriesData.Value); // The transformed data. - var transformedData = ml.Transforms.DetectAnomalyBySrCnn(outputColumnName, inputColumnName, 16, 5, 5, 3, 8, 0.35).Fit(dataView).Transform(dataView); + var transformedData = ml.Transforms.DetectAnomalyBySrCnn( + outputColumnName, inputColumnName, 16, 5, 5, 3, 8, 0.35).Fit( + dataView).Transform(dataView); - // Getting the data of the newly created column as an IEnumerable of SrCnnAnomalyDetection. - var predictionColumn = ml.Data.CreateEnumerable(transformedData, reuseRowObject: false); + // Getting the data of the newly created column as an IEnumerable of + // SrCnnAnomalyDetection. + var predictionColumn = ml.Data.CreateEnumerable( + transformedData, reuseRowObject: false); + + Console.WriteLine($"{outputColumnName} column obtained post-" + + $"transformation."); - Console.WriteLine($"{outputColumnName} column obtained post-transformation."); Console.WriteLine("Data\tAlert\tScore\tMag"); int k = 0; @@ -75,9 +82,10 @@ public static void Example() //5 0 0.01 0.25 } - private static void PrintPrediction(float value, SrCnnAnomalyDetection prediction) => - Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}", value, prediction.Prediction[0], - prediction.Prediction[1], prediction.Prediction[2]); + private static void PrintPrediction(float value, SrCnnAnomalyDetection + prediction) => + Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}", value, prediction + .Prediction[0], prediction.Prediction[1], prediction.Prediction[2]); private class TimeSeriesData { diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectChangePointBySsa.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectChangePointBySsa.cs index 8404ca2f63..7d545770f9 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectChangePointBySsa.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectChangePointBySsa.cs @@ -9,14 +9,16 @@ namespace Samples.Dynamic { public static class DetectChangePointBySsa { - // This example creates a time series (list of Data with the i-th element corresponding to the i-th time slot). - // It demonstrates stateful prediction engine that updates the state of the model and allows for saving/reloading. - // The estimator is applied then to identify points where data distribution changed. - // This estimator can account for temporal seasonality in the data. + // This example creates a time series (list of Data with the i-th element + // corresponding to the i-th time slot). It demonstrates stateful prediction + // engine that updates the state of the model and allows for + // saving/reloading. The estimator is applied then to identify points where + // data distribution changed. This estimator can account for temporal + // seasonality in the data. public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var ml = new MLContext(); // Generate sample series data with a recurring pattern @@ -54,12 +56,16 @@ public static void Example() int changeHistoryLength = 8; // Train the change point detector. - ITransformer model = ml.Transforms.DetectChangePointBySsa(outputColumnName, inputColumnName, confidence, changeHistoryLength, TrainingSize, SeasonalitySize + 1).Fit(dataView); + ITransformer model = ml.Transforms.DetectChangePointBySsa( + outputColumnName, inputColumnName, confidence, changeHistoryLength, + TrainingSize, SeasonalitySize + 1).Fit(dataView); // Create a prediction engine from the model for feeding new data. - var engine = model.CreateTimeSeriesPredictionFunction(ml); + var engine = model.CreateTimeSeriesEngine(ml); - // Start streaming new data points with no change point to the prediction engine. + // Start streaming new data points with no change point to the + // prediction engine. Console.WriteLine($"Output from ChangePoint predictions on new data:"); Console.WriteLine("Data\tAlert\tScore\tP-Value\tMartingale value"); @@ -99,7 +105,8 @@ public static void Example() model = ml.Model.Load(file, out DataViewSchema schema); // We must create a new prediction engine from the persisted model. - engine = model.CreateTimeSeriesPredictionFunction(ml); + engine = model.CreateTimeSeriesEngine(ml); // Run predictions on the loaded model. for (int i = 0; i < 5; i++) @@ -116,9 +123,11 @@ public static void Example() } - private static void PrintPrediction(float value, ChangePointPrediction prediction) => - Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}\t{4:0.00}", value, prediction.Prediction[0], - prediction.Prediction[1], prediction.Prediction[2], prediction.Prediction[3]); + private static void PrintPrediction(float value, ChangePointPrediction + prediction) => + Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}\t{4:0.00}", value, + prediction.Prediction[0], prediction.Prediction[1], + prediction.Prediction[2], prediction.Prediction[3]); class ChangePointPrediction { diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectChangePointBySsaBatchPrediction.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectChangePointBySsaBatchPrediction.cs index 2e0df19007..25819052d8 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectChangePointBySsaBatchPrediction.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectChangePointBySsaBatchPrediction.cs @@ -7,16 +7,18 @@ namespace Samples.Dynamic { public static class DetectChangePointBySsaBatchPrediction { - // This example creates a time series (list of Data with the i-th element corresponding to the i-th time slot). - // The estimator is applied then to identify points where data distribution changed. - // This estimator can account for temporal seasonality in the data. + // This example creates a time series (list of Data with the i-th element + // corresponding to the i-th time slot). The estimator is applied then to + // identify points where data distribution changed. This estimator can + // account for temporal seasonality in the data. public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var ml = new MLContext(); - // Generate sample series data with a recurring pattern and then a change in trend + // Generate sample series data with a recurring pattern and then a + // change in trend const int SeasonalitySize = 5; const int TrainingSeasons = 3; const int TrainingSize = SeasonalitySize * TrainingSeasons; @@ -56,12 +58,18 @@ public static void Example() var outputColumnName = nameof(ChangePointPrediction.Prediction); // The transformed data. - var transformedData = ml.Transforms.DetectChangePointBySsa(outputColumnName, inputColumnName, 95, 8, TrainingSize, SeasonalitySize + 1).Fit(dataView).Transform(dataView); + var transformedData = ml.Transforms.DetectChangePointBySsa( + outputColumnName, inputColumnName, 95, 8, TrainingSize, + SeasonalitySize + 1).Fit(dataView).Transform(dataView); - // Getting the data of the newly created column as an IEnumerable of ChangePointPrediction. - var predictionColumn = ml.Data.CreateEnumerable(transformedData, reuseRowObject: false); + // Getting the data of the newly created column as an IEnumerable of + // ChangePointPrediction. + var predictionColumn = ml.Data.CreateEnumerable( + transformedData, reuseRowObject: false); + + Console.WriteLine(outputColumnName + " column obtained " + + "post-transformation."); - Console.WriteLine($"{outputColumnName} column obtained post-transformation."); Console.WriteLine("Data\tAlert\tScore\tP-Value\tMartingale value"); int k = 0; foreach (var prediction in predictionColumn) @@ -91,9 +99,11 @@ public static void Example() // 400 0 357.11 0.03 45298370.86 } - private static void PrintPrediction(float value, ChangePointPrediction prediction) => - Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}\t{4:0.00}", value, prediction.Prediction[0], - prediction.Prediction[1], prediction.Prediction[2], prediction.Prediction[3]); + private static void PrintPrediction(float value, ChangePointPrediction + prediction) => + Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}\t{4:0.00}", value, + prediction.Prediction[0], prediction.Prediction[1], + prediction.Prediction[2], prediction.Prediction[3]); class ChangePointPrediction { diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectChangePointBySsaStream.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectChangePointBySsaStream.cs index 0cd9b0fa77..c65d3af987 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectChangePointBySsaStream.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectChangePointBySsaStream.cs @@ -9,14 +9,16 @@ namespace Samples.Dynamic { public static class DetectChangePointBySsaStream { - // This example creates a time series (list of Data with the i-th element corresponding to the i-th time slot). - // It demonstrates stateful prediction engine that updates the state of the model and allows for saving/reloading. - // The estimator is applied then to identify points where data distribution changed. - // This estimator can account for temporal seasonality in the data. + // This example creates a time series (list of Data with the i-th element + // corresponding to the i-th time slot). It demonstrates stateful prediction + // engine that updates the state of the model and allows for + // saving/reloading. The estimator is applied then to identify points where + // data distribution changed. This estimator can account for temporal + // seasonality in the data. public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var ml = new MLContext(); // Generate sample series data with a recurring pattern @@ -54,12 +56,16 @@ public static void Example() int changeHistoryLength = 8; // Train the change point detector. - ITransformer model = ml.Transforms.DetectChangePointBySsa(outputColumnName, inputColumnName, confidence, changeHistoryLength, TrainingSize, SeasonalitySize + 1).Fit(dataView); + ITransformer model = ml.Transforms.DetectChangePointBySsa( + outputColumnName, inputColumnName, confidence, changeHistoryLength, + TrainingSize, SeasonalitySize + 1).Fit(dataView); // Create a prediction engine from the model for feeding new data. - var engine = model.CreateTimeSeriesPredictionFunction(ml); + var engine = model.CreateTimeSeriesEngine(ml); - // Start streaming new data points with no change point to the prediction engine. + // Start streaming new data points with no change point to the + // prediction engine. Console.WriteLine($"Output from ChangePoint predictions on new data:"); Console.WriteLine("Data\tAlert\tScore\tP-Value\tMartingale value"); @@ -103,7 +109,8 @@ public static void Example() model = ml.Model.Load(stream, out DataViewSchema schema); // We must create a new prediction engine from the persisted model. - engine = model.CreateTimeSeriesPredictionFunction(ml); + engine = model.CreateTimeSeriesEngine(ml); // Run predictions on the loaded model. for (int i = 0; i < 5; i++) @@ -120,9 +127,11 @@ public static void Example() } - private static void PrintPrediction(float value, ChangePointPrediction prediction) => - Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}\t{4:0.00}", value, prediction.Prediction[0], - prediction.Prediction[1], prediction.Prediction[2], prediction.Prediction[3]); + private static void PrintPrediction(float value, ChangePointPrediction + prediction) => + Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}\t{4:0.00}", value, + prediction.Prediction[0], prediction.Prediction[1], + prediction.Prediction[2], prediction.Prediction[3]); class ChangePointPrediction { diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidChangePoint.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidChangePoint.cs index 2890c4ec57..4e44a73607 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidChangePoint.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidChangePoint.cs @@ -13,12 +13,13 @@ namespace Samples.Dynamic { public static class DetectIidChangePoint { - // This example creates a time series (list of Data with the i-th element corresponding to the i-th time slot). - // The estimator is applied then to identify points where data distribution changed. + // This example creates a time series (list of Data with the i-th element + // corresponding to the i-th time slot). The estimator is applied then to + // identify points where data distribution changed. public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var ml = new MLContext(); // Generate sample series data with a change @@ -53,12 +54,16 @@ public static void Example() string inputColumnName = nameof(TimeSeriesData.Value); // Time Series model. - ITransformer model = ml.Transforms.DetectIidChangePoint(outputColumnName, inputColumnName, 95, Size / 4).Fit(dataView); + ITransformer model = ml.Transforms.DetectIidChangePoint( + outputColumnName, inputColumnName, 95, Size / 4).Fit(dataView); // Create a time series prediction engine from the model. - var engine = model.CreateTimeSeriesPredictionFunction(ml); + var engine = model.CreateTimeSeriesEngine(ml); + + Console.WriteLine($"{outputColumnName} column obtained " + + $"post-transformation."); - Console.WriteLine($"{outputColumnName} column obtained post-transformation."); Console.WriteLine("Data\tAlert\tScore\tP-Value\tMartingale value"); // Data Alert Score P-Value Martingale value @@ -88,7 +93,8 @@ public static void Example() var modelPath = "temp.zip"; engine.CheckPoint(ml, modelPath); - // Reference to current time series engine because in the next step "engine" will point to the + // Reference to current time series engine because in the next step + // "engine" will point to the // checkpointed model being loaded from disk. var timeseries1 = engine; @@ -97,7 +103,9 @@ public static void Example() model = ml.Model.Load(file, out DataViewSchema schema); // Create a time series prediction engine from the checkpointed model. - engine = model.CreateTimeSeriesPredictionFunction(ml); + engine = model.CreateTimeSeriesEngine(ml); + for (int index = 0; index < 8; index++) { // Anomaly change point detection. @@ -112,8 +120,8 @@ public static void Example() // 7 0 7.00 0.50 0.00 // 7 0 7.00 0.50 0.00 - // Prediction from the original time series engine should match the prediction from - // check pointed model. + // Prediction from the original time series engine should match the + // prediction from check pointed model. engine = timeseries1; for (int index = 0; index < 8; index++) { @@ -130,9 +138,11 @@ public static void Example() // 7 0 7.00 0.50 0.00 } - private static void PrintPrediction(float value, ChangePointPrediction prediction) => - Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}\t{4:0.00}", value, prediction.Prediction[0], - prediction.Prediction[1], prediction.Prediction[2], prediction.Prediction[3]); + private static void PrintPrediction(float value, ChangePointPrediction + prediction) => + Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}\t{4:0.00}", value, + prediction.Prediction[0], prediction.Prediction[1], + prediction.Prediction[2], prediction.Prediction[3]); class ChangePointPrediction { diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidChangePointBatchPrediction.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidChangePointBatchPrediction.cs index 760305df33..35066b79e1 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidChangePointBatchPrediction.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidChangePointBatchPrediction.cs @@ -11,12 +11,13 @@ namespace Samples.Dynamic { public static class DetectIidChangePointBatchPrediction { - // This example creates a time series (list of Data with the i-th element corresponding to the i-th time slot). - // The estimator is applied then to identify points where data distribution changed. + // This example creates a time series (list of Data with the i-th element + // corresponding to the i-th time slot). The estimator is applied then to + // identify points where data distribution changed. public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var ml = new MLContext(); // Generate sample series data with a change @@ -51,12 +52,18 @@ public static void Example() string inputColumnName = nameof(TimeSeriesData.Value); // The transformed data. - var transformedData = ml.Transforms.DetectIidChangePoint(outputColumnName, inputColumnName, 95, Size / 4).Fit(dataView).Transform(dataView); + var transformedData = ml.Transforms.DetectIidChangePoint( + outputColumnName, inputColumnName, 95, Size / 4).Fit(dataView) + .Transform(dataView); - // Getting the data of the newly created column as an IEnumerable of ChangePointPrediction. - var predictionColumn = ml.Data.CreateEnumerable(transformedData, reuseRowObject: false); + // Getting the data of the newly created column as an IEnumerable of + // ChangePointPrediction. + var predictionColumn = ml.Data.CreateEnumerable( + transformedData, reuseRowObject: false); + + Console.WriteLine($"{outputColumnName} column obtained " + + $"post-transformation."); - Console.WriteLine($"{outputColumnName} column obtained post-transformation."); Console.WriteLine("Data\tAlert\tScore\tP-Value\tMartingale value"); int k = 0; foreach (var prediction in predictionColumn) @@ -82,9 +89,11 @@ public static void Example() // 7 0 7.00 0.50 0.00 } - private static void PrintPrediction(float value, ChangePointPrediction prediction) => - Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}\t{4:0.00}", value, prediction.Prediction[0], - prediction.Prediction[1], prediction.Prediction[2], prediction.Prediction[3]); + private static void PrintPrediction(float value, ChangePointPrediction + prediction) => + Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}\t{4:0.00}", value, + prediction.Prediction[0], prediction.Prediction[1], + prediction.Prediction[2], prediction.Prediction[3]); class ChangePointPrediction { diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidSpike.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidSpike.cs index 439006a7ec..637e5ccbd8 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidSpike.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidSpike.cs @@ -9,12 +9,13 @@ namespace Samples.Dynamic { public static class DetectIidSpike { - // This example creates a time series (list of Data with the i-th element corresponding to the i-th time slot). - // The estimator is applied then to identify spiking points in the series. + // This example creates a time series (list of Data with the i-th element + // corresponding to the i-th time slot). The estimator is applied then to + // identify spiking points in the series. public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var ml = new MLContext(); // Generate sample series data with a spike @@ -45,12 +46,16 @@ public static void Example() string inputColumnName = nameof(TimeSeriesData.Value); // The transformed model. - ITransformer model = ml.Transforms.DetectIidSpike(outputColumnName, inputColumnName, 95, Size).Fit(dataView); + ITransformer model = ml.Transforms.DetectIidSpike(outputColumnName, + inputColumnName, 95, Size).Fit(dataView); // Create a time series prediction engine from the model. - var engine = model.CreateTimeSeriesPredictionFunction(ml); + var engine = model.CreateTimeSeriesEngine(ml); + + Console.WriteLine($"{outputColumnName} column obtained " + + $"post-transformation."); - Console.WriteLine($"{outputColumnName} column obtained post-transformation."); Console.WriteLine("Data\tAlert\tScore\tP-Value"); // Prediction column obtained post-transformation. @@ -96,9 +101,11 @@ public static void Example() } - private static void PrintPrediction(float value, IidSpikePrediction prediction) => - Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}", value, prediction.Prediction[0], - prediction.Prediction[1], prediction.Prediction[2]); + private static void PrintPrediction(float value, IidSpikePrediction + prediction) => + Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}", value, + prediction.Prediction[0], prediction.Prediction[1], + prediction.Prediction[2]); class TimeSeriesData { diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidSpikeBatchPrediction.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidSpikeBatchPrediction.cs index 4145214918..e1c14e1e7d 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidSpikeBatchPrediction.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectIidSpikeBatchPrediction.cs @@ -7,12 +7,13 @@ namespace Samples.Dynamic { public static class DetectIidSpikeBatchPrediction { - // This example creates a time series (list of Data with the i-th element corresponding to the i-th time slot). - // The estimator is applied then to identify spiking points in the series. + // This example creates a time series (list of Data with the i-th element + // corresponding to the i-th time slot). The estimator is applied then to + // identify spiking points in the series. public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var ml = new MLContext(); // Generate sample series data with a spike @@ -43,12 +44,17 @@ public static void Example() string inputColumnName = nameof(TimeSeriesData.Value); // The transformed data. - var transformedData = ml.Transforms.DetectIidSpike(outputColumnName, inputColumnName, 95, Size / 4).Fit(dataView).Transform(dataView); + var transformedData = ml.Transforms.DetectIidSpike(outputColumnName, + inputColumnName, 95, Size / 4).Fit(dataView).Transform(dataView); - // Getting the data of the newly created column as an IEnumerable of IidSpikePrediction. - var predictionColumn = ml.Data.CreateEnumerable(transformedData, reuseRowObject: false); + // Getting the data of the newly created column as an IEnumerable of + // IidSpikePrediction. + var predictionColumn = ml.Data.CreateEnumerable( + transformedData, reuseRowObject: false); + + Console.WriteLine($"{outputColumnName} column obtained " + + $"post-transformation."); - Console.WriteLine($"{outputColumnName} column obtained post-transformation."); Console.WriteLine("Data\tAlert\tScore\tP-Value"); int k = 0; @@ -70,9 +76,11 @@ public static void Example() // 5 0 5.00 0.50 } - private static void PrintPrediction(float value, IidSpikePrediction prediction) => - Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}", value, prediction.Prediction[0], - prediction.Prediction[1], prediction.Prediction[2]); + private static void PrintPrediction(float value, IidSpikePrediction + prediction) => + Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}", value, + prediction.Prediction[0], prediction.Prediction[1], + prediction.Prediction[2]); class TimeSeriesData { diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectSpikeBySsa.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectSpikeBySsa.cs index 38094bb672..79a22272fa 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectSpikeBySsa.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectSpikeBySsa.cs @@ -9,13 +9,14 @@ namespace Samples.Dynamic { public static class DetectSpikeBySsa { - // This example creates a time series (list of Data with the i-th element corresponding to the i-th time slot). - // The estimator is applied then to identify spiking points in the series. - // This estimator can account for temporal seasonality in the data. + // This example creates a time series (list of Data with the i-th element + // corresponding to the i-th time slot). The estimator is applied then to + // identify spiking points in the series. This estimator can account for + // temporal seasonality in the data. public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var ml = new MLContext(); // Generate sample series data with a recurring pattern @@ -51,12 +52,16 @@ public static void Example() var outputColumnName = nameof(SsaSpikePrediction.Prediction); // Train the change point detector. - ITransformer model = ml.Transforms.DetectSpikeBySsa(outputColumnName, inputColumnName, 95, 8, TrainingSize, SeasonalitySize + 1).Fit(dataView); + ITransformer model = ml.Transforms.DetectSpikeBySsa(outputColumnName, + inputColumnName, 95, 8, TrainingSize, SeasonalitySize + 1).Fit( + dataView); // Create a prediction engine from the model for feeding new data. - var engine = model.CreateTimeSeriesPredictionFunction(ml); + var engine = model.CreateTimeSeriesEngine(ml); - // Start streaming new data points with no change point to the prediction engine. + // Start streaming new data points with no change point to the + // prediction engine. Console.WriteLine($"Output from spike predictions on new data:"); Console.WriteLine("Data\tAlert\tScore\tP-Value"); @@ -94,7 +99,8 @@ public static void Example() model = ml.Model.Load(file, out DataViewSchema schema); // We must create a new prediction engine from the persisted model. - engine = model.CreateTimeSeriesPredictionFunction(ml); + engine = model.CreateTimeSeriesEngine(ml); // Run predictions on the loaded model. for (int i = 0; i < 5; i++) @@ -107,9 +113,11 @@ public static void Example() // 4 0 -23.24 0.28 } - private static void PrintPrediction(float value, SsaSpikePrediction prediction) => - Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}", value, prediction.Prediction[0], - prediction.Prediction[1], prediction.Prediction[2]); + private static void PrintPrediction(float value, SsaSpikePrediction + prediction) => + Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}", value, + prediction.Prediction[0], prediction.Prediction[1], + prediction.Prediction[2]); class TimeSeriesData { diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectSpikeBySsaBatchPrediction.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectSpikeBySsaBatchPrediction.cs index cc6a798dcf..9104f3af26 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectSpikeBySsaBatchPrediction.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/DetectSpikeBySsaBatchPrediction.cs @@ -7,16 +7,18 @@ namespace Samples.Dynamic { public static class DetectSpikeBySsaBatchPrediction { - // This example creates a time series (list of Data with the i-th element corresponding to the i-th time slot). - // The estimator is applied then to identify spiking points in the series. - // This estimator can account for temporal seasonality in the data. + // This example creates a time series (list of Data with the i-th element + // corresponding to the i-th time slot). The estimator is applied then to + // identify spiking points in the series. This estimator can account for + // temporal seasonality in the data. public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var ml = new MLContext(); - // Generate sample series data with a recurring pattern and a spike within the pattern + // Generate sample series data with a recurring pattern and a spike + // within the pattern const int SeasonalitySize = 5; const int TrainingSeasons = 3; const int TrainingSize = SeasonalitySize * TrainingSeasons; @@ -58,12 +60,18 @@ public static void Example() var outputColumnName = nameof(SsaSpikePrediction.Prediction); // The transformed data. - var transformedData = ml.Transforms.DetectSpikeBySsa(outputColumnName, inputColumnName, 95, 8, TrainingSize, SeasonalitySize + 1).Fit(dataView).Transform(dataView); + var transformedData = ml.Transforms.DetectSpikeBySsa(outputColumnName, + inputColumnName, 95, 8, TrainingSize, SeasonalitySize + 1).Fit( + dataView).Transform(dataView); - // Getting the data of the newly created column as an IEnumerable of SsaSpikePrediction. - var predictionColumn = ml.Data.CreateEnumerable(transformedData, reuseRowObject: false); + // Getting the data of the newly created column as an IEnumerable of + // SsaSpikePrediction. + var predictionColumn = ml.Data.CreateEnumerable( + transformedData, reuseRowObject: false); + + Console.WriteLine($"{outputColumnName} column obtained " + + $"post-transformation."); - Console.WriteLine($"{outputColumnName} column obtained post-transformation."); Console.WriteLine("Data\tAlert\tScore\tP-Value"); int k = 0; foreach (var prediction in predictionColumn) @@ -94,9 +102,11 @@ public static void Example() // 4 0 -29.82 0.21 } - private static void PrintPrediction(float value, SsaSpikePrediction prediction) => - Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}", value, prediction.Prediction[0], - prediction.Prediction[1], prediction.Prediction[2]); + private static void PrintPrediction(float value, SsaSpikePrediction + prediction) => + Console.WriteLine("{0}\t{1}\t{2:0.00}\t{3:0.00}", value, + prediction.Prediction[0], prediction.Prediction[1], + prediction.Prediction[2]); class TimeSeriesData { diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/Forecasting.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/Forecasting.cs index 8ea98c8d99..07f0eef23d 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/Forecasting.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/Forecasting.cs @@ -1,23 +1,22 @@ using System; using System.Collections.Generic; +using System.IO; using Microsoft.ML; using Microsoft.ML.Transforms.TimeSeries; -using Microsoft.ML.TimeSeries; namespace Samples.Dynamic { public static class Forecasting { - // This example creates a time series (list of Data with the i-th element corresponding to the i-th time slot) and then - // does forecasting. + // This example creates a time series (list of Data with the i-th element + // corresponding to the i-th time slot) and then does forecasting. public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var ml = new MLContext(); - // Generate sample series data with a recurring pattern - const int SeasonalitySize = 5; + // Generate sample series data with a recurring pattern. var data = new List() { new TimeSeriesData(0), @@ -44,43 +43,62 @@ public static void Example() // Setup arguments. var inputColumnName = nameof(TimeSeriesData.Value); + var outputColumnName = nameof(ForecastResult.Forecast); // Instantiate the forecasting model. - var model = ml.Forecasting.AdaptiveSingularSpectrumSequenceModeler(inputColumnName, data.Count, SeasonalitySize + 1, SeasonalitySize, - 1, AdaptiveSingularSpectrumSequenceModeler.RankSelectionMethod.Exact, null, SeasonalitySize / 2, false, false); + var model = ml.Forecasting.ForecastBySsa(outputColumnName, + inputColumnName, 5, 11, data.Count, 5); // Train. - model.Train(dataView); + var transformer = model.Fit(dataView); // Forecast next five values. - var forecast = model.Forecast(5); + var forecastEngine = transformer.CreateTimeSeriesEngine(ml); + + var forecast = forecastEngine.Predict(); + Console.WriteLine($"Forecasted values:"); - Console.WriteLine("[{0}]", string.Join(", ", forecast)); + Console.WriteLine("[{0}]", string.Join(", ", forecast.Forecast)); // Forecasted values: - // [2.452744, 2.589339, 2.729183, 2.873005, 3.028931] + // [1.977226, 1.020494, 1.760543, 3.437509, 4.266461] // Update with new observations. - dataView = ml.Data.LoadFromEnumerable(new List() { new TimeSeriesData(0), new TimeSeriesData(0), new TimeSeriesData(0), new TimeSeriesData(0) }); - model.Update(dataView); + forecastEngine.Predict(new TimeSeriesData(0)); + forecastEngine.Predict(new TimeSeriesData(0)); + forecastEngine.Predict(new TimeSeriesData(0)); + forecastEngine.Predict(new TimeSeriesData(0)); // Checkpoint. - ml.Model.SaveForecastingModel(model, "model.zip"); + forecastEngine.CheckPoint(ml, "model.zip"); // Load the checkpointed model from disk. - var modelCopy = ml.Model.LoadForecastingModel("model.zip"); + // Load the model. + ITransformer modelCopy; + using (var file = File.OpenRead("model.zip")) + modelCopy = ml.Model.Load(file, out DataViewSchema schema); + + // We must create a new prediction engine from the persisted model. + var forecastEngineCopy = modelCopy.CreateTimeSeriesEngine< + TimeSeriesData, ForecastResult>(ml); // Forecast with the checkpointed model loaded from disk. - forecast = modelCopy.Forecast(5); - Console.WriteLine("[{0}]", string.Join(", ", forecast)); - // [0.8681176, 0.8185108, 0.8069275, 0.84405, 0.9455081] + forecast = forecastEngineCopy.Predict(); + Console.WriteLine("[{0}]", string.Join(", ", forecast.Forecast)); + // [1.791331, 1.255525, 0.3060154, -0.200446, 0.5657795] // Forecast with the original model(that was checkpointed to disk). - forecast = model.Forecast(5); - Console.WriteLine("[{0}]", string.Join(", ", forecast)); - // [0.8681176, 0.8185108, 0.8069275, 0.84405, 0.9455081] + forecast = forecastEngine.Predict(); + Console.WriteLine("[{0}]", string.Join(", ", forecast.Forecast)); + // [1.791331, 1.255525, 0.3060154, -0.200446, 0.5657795] } + class ForecastResult + { + public float[] Forecast { get; set; } + } + class TimeSeriesData { public float Value; diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/ForecastingWithConfidenceInterval.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/ForecastingWithConfidenceInterval.cs index b5c3c5c9c2..4903b147b8 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/ForecastingWithConfidenceInterval.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/ForecastingWithConfidenceInterval.cs @@ -2,22 +2,21 @@ using System.Collections.Generic; using Microsoft.ML; using Microsoft.ML.Transforms.TimeSeries; -using Microsoft.ML.TimeSeries; +using System.IO; namespace Samples.Dynamic { public static class ForecastingWithConfidenceInternal { - // This example creates a time series (list of Data with the i-th element corresponding to the i-th time slot) and then - // does forecasting. + // This example creates a time series (list of Data with the i-th element + // corresponding to the i-th time slot) and then does forecasting. public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var ml = new MLContext(); - // Generate sample series data with a recurring pattern - const int SeasonalitySize = 5; + // Generate sample series data with a recurring pattern. var data = new List() { new TimeSeriesData(0), @@ -44,62 +43,88 @@ public static void Example() // Setup arguments. var inputColumnName = nameof(TimeSeriesData.Value); + var outputColumnName = nameof(ForecastResult.Forecast); - // Instantiate forecasting model. - var model = ml.Forecasting.AdaptiveSingularSpectrumSequenceModeler(inputColumnName, data.Count, SeasonalitySize + 1, SeasonalitySize, - 1, AdaptiveSingularSpectrumSequenceModeler.RankSelectionMethod.Exact, null, SeasonalitySize / 2, shouldComputeForecastIntervals: true, false); + // Instantiate the forecasting model. + var model = ml.Forecasting.ForecastBySsa(outputColumnName, + inputColumnName, 5, 11, data.Count, 5, + confidenceLevel: 0.95f, + confidenceLowerBoundColumn: "ConfidenceLowerBound", + confidenceUpperBoundColumn: "ConfidenceUpperBound"); // Train. - model.Train(dataView); - - // Forecast next five values with confidence internal. - float[] forecast; - float[] confidenceIntervalLowerBounds; - float[] confidenceIntervalUpperBounds; - model.ForecastWithConfidenceIntervals(5, out forecast, out confidenceIntervalLowerBounds, out confidenceIntervalUpperBounds); - PrintForecastValuesAndIntervals(forecast, confidenceIntervalLowerBounds, confidenceIntervalUpperBounds); + var transformer = model.Fit(dataView); + + // Forecast next five values. + var forecastEngine = transformer.CreateTimeSeriesEngine(ml); + + var forecast = forecastEngine.Predict(); + + PrintForecastValuesAndIntervals(forecast.Forecast, forecast + .ConfidenceLowerBound, forecast.ConfidenceUpperBound); // Forecasted values: - // [2.452744, 2.589339, 2.729183, 2.873005, 3.028931] + // [1.977226, 1.020494, 1.760543, 3.437509, 4.266461] // Confidence intervals: - // [-0.2235315 - 5.12902] [-0.08777174 - 5.266451] [0.05076938 - 5.407597] [0.1925406 - 5.553469] [0.3469928 - 5.71087] + // [0.3451088 - 3.609343] [-0.7967533 - 2.83774] [-0.058467 - 3.579552] [1.61505 - 5.259968] [2.349299 - 6.183623] // Update with new observations. - dataView = ml.Data.LoadFromEnumerable(new List() { new TimeSeriesData(0), new TimeSeriesData(0), new TimeSeriesData(0), new TimeSeriesData(0) }); - model.Update(dataView); + forecastEngine.Predict(new TimeSeriesData(0)); + forecastEngine.Predict(new TimeSeriesData(0)); + forecastEngine.Predict(new TimeSeriesData(0)); + forecastEngine.Predict(new TimeSeriesData(0)); // Checkpoint. - ml.Model.SaveForecastingModel(model, "model.zip"); + forecastEngine.CheckPoint(ml, "model.zip"); // Load the checkpointed model from disk. - var modelCopy = ml.Model.LoadForecastingModel("model.zip"); + // Load the model. + ITransformer modelCopy; + using (var file = File.OpenRead("model.zip")) + modelCopy = ml.Model.Load(file, out DataViewSchema schema); + + // We must create a new prediction engine from the persisted model. + var forecastEngineCopy = modelCopy.CreateTimeSeriesEngine< + TimeSeriesData, ForecastResult>(ml); // Forecast with the checkpointed model loaded from disk. - modelCopy.ForecastWithConfidenceIntervals(5, out forecast, out confidenceIntervalLowerBounds, out confidenceIntervalUpperBounds); - PrintForecastValuesAndIntervals(forecast, confidenceIntervalLowerBounds, confidenceIntervalUpperBounds); - // Forecasted values: - // [0.8681176, 0.8185108, 0.8069275, 0.84405, 0.9455081] + forecast = forecastEngineCopy.Predict(); + PrintForecastValuesAndIntervals(forecast.Forecast, forecast + .ConfidenceLowerBound, forecast.ConfidenceUpperBound); + + // [1.791331, 1.255525, 0.3060154, -0.200446, 0.5657795] // Confidence intervals: - // [-1.808158 - 3.544394] [-1.8586 - 3.495622] [-1.871486 - 3.485341] [-1.836414 - 3.524514] [-1.736431 - 3.627447] + // [0.1592142 - 3.423448] [-0.5617217 - 3.072772] [-1.512994 - 2.125025] [-2.022905 - 1.622013] [-1.351382 - 2.482941] // Forecast with the original model(that was checkpointed to disk). - model.ForecastWithConfidenceIntervals(5, out forecast, out confidenceIntervalLowerBounds, out confidenceIntervalUpperBounds); - PrintForecastValuesAndIntervals(forecast, confidenceIntervalLowerBounds, confidenceIntervalUpperBounds); - // Forecasted values: - // [0.8681176, 0.8185108, 0.8069275, 0.84405, 0.9455081] + forecast = forecastEngine.Predict(); + PrintForecastValuesAndIntervals(forecast.Forecast, + forecast.ConfidenceLowerBound, forecast.ConfidenceUpperBound); + + // [1.791331, 1.255525, 0.3060154, -0.200446, 0.5657795] // Confidence intervals: - // [-1.808158 - 3.544394] [-1.8586 - 3.495622] [-1.871486 - 3.485341] [-1.836414 - 3.524514] [-1.736431 - 3.627447] + // [0.1592142 - 3.423448] [-0.5617217 - 3.072772] [-1.512994 - 2.125025] [-2.022905 - 1.622013] [-1.351382 - 2.482941] } - static void PrintForecastValuesAndIntervals(float[] forecast, float[] confidenceIntervalLowerBounds, float[] confidenceIntervalUpperBounds) + static void PrintForecastValuesAndIntervals(float[] forecast, float[] + confidenceIntervalLowerBounds, float[] confidenceIntervalUpperBounds) { Console.WriteLine($"Forecasted values:"); Console.WriteLine("[{0}]", string.Join(", ", forecast)); Console.WriteLine($"Confidence intervals:"); for (int index = 0; index < forecast.Length; index++) - Console.Write($"[{confidenceIntervalLowerBounds[index]} - {confidenceIntervalUpperBounds[index]}] "); + Console.Write($"[{confidenceIntervalLowerBounds[index]} -" + + $" {confidenceIntervalUpperBounds[index]}] "); Console.WriteLine(); } + class ForecastResult + { + public float[] Forecast { get; set; } + public float[] ConfidenceLowerBound { get; set; } + public float[] ConfidenceUpperBound { get; set; } + } + class TimeSeriesData { public float Value; diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TreeFeaturization/BinaryClassificationFeaturization.ttinclude b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TreeFeaturization/BinaryClassificationFeaturization.ttinclude new file mode 100644 index 0000000000..b223b29df4 --- /dev/null +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TreeFeaturization/BinaryClassificationFeaturization.ttinclude @@ -0,0 +1,131 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.ML; +using Microsoft.ML.Data; +<# if (TrainerOptions != null) { #> +<#=OptionsInclude#> +<# } #> + +namespace Samples.Dynamic.Transforms.TreeFeaturization +{ + public static class <#=ClassName#> + {<#=Comments#> + public static void Example() + { + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. + var mlContext = new MLContext(seed: 0); + + // Create a list of data points to be transformed. + var dataPoints = GenerateRandomDataPoints(100).ToList(); + + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. + var dataView = mlContext.Data.LoadFromEnumerable(dataPoints); +<# if (CacheData) { #> + + // ML.NET doesn't cache data set by default. Therefore, if one reads a + // data set from a file and accesses it many times, it can be slow due + // to expensive featurization and disk operations. When the considered + // data can fit into memory, a solution is to cache the data in memory. + // Caching is especially helpful when working with iterative algorithms + // which needs many data passes. + dataView = mlContext.Data.Cache(dataView); +<# } #> + + // Define input and output columns of tree-based featurizer. + string labelColumnName = nameof(DataPoint.Label); + string featureColumnName = nameof(DataPoint.Features); + string treesColumnName = nameof(TransformedDataPoint.Trees); + string leavesColumnName = nameof(TransformedDataPoint.Leaves); + string pathsColumnName = nameof(TransformedDataPoint.Paths); + + // Define the configuration of the trainer used to train a tree-based + // model. + var trainerOptions = new <#=TrainerOptions#>; + + // Define the tree-based featurizer's configuration. + var options = new <#=Options#>; + + // Define the featurizer. + var pipeline = mlContext.Transforms.<#=Trainer#>( + options); + + // Train the model. + var model = pipeline.Fit(dataView); + + // Apply the trained transformer to the considered data set. + var transformed = model.Transform(dataView); + + // Convert IDataView object to a list. Each element in the resulted list + // corresponds to a row in the IDataView. + var transformedDataPoints = mlContext.Data.CreateEnumerable< + TransformedDataPoint>(transformed, false).ToList(); + + // Print out the transformation of the first 3 data points. + for (int i = 0; i < 3; ++i) + { + var dataPoint = dataPoints[i]; + var transformedDataPoint = transformedDataPoints[i]; + Console.WriteLine("The original feature vector [" + String.Join( + ",", dataPoint.Features) + "] is transformed to three " + + "different tree-based feature vectors:"); + + Console.WriteLine(" Trees' output values: [" + String.Join(",", + transformedDataPoint.Trees) + "]."); + + Console.WriteLine(" Leave IDs' 0-1 representation: [" + String + .Join(",", transformedDataPoint.Leaves) + "]."); + + Console.WriteLine(" Paths IDs' 0-1 representation: [" + String + .Join(",", transformedDataPoint.Paths) + "]."); + } + + <#=ExpectedOutput#> + } + + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) + { + var random = new Random(seed); + float randomFloat() => (float)random.NextDouble(); + for (int i = 0; i < count; i++) + { + var label = randomFloat() > <#=LabelThreshold#>; + yield return new DataPoint + { + Label = label, + // Create random features that are correlated with the label. + // For data points with false label, the feature values are + // slightly increased by adding a constant. + Features = Enumerable.Repeat(label, 3).Select(x => x ? + randomFloat() : randomFloat() + <#=DataSepValue#>).ToArray() + }; + } + } + + // Example with label and 3 feature values. A data set is a collection of + // such examples. + private class DataPoint + { + public bool Label { get; set; } + [VectorType(3)] + public float[] Features { get; set; } + } + + // Class used to capture the output of tree-base featurization. + private class TransformedDataPoint : DataPoint + { + // The i-th value is the output value of the i-th decision tree. + public float[] Trees { get; set; } + // The 0-1 encoding of leaves the input feature vector falls into. + public float[] Leaves { get; set; } + // The 0-1 encoding of paths the input feature vector reaches the + // leaves. + public float[] Paths { get; set; } + } + } +} \ No newline at end of file diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TreeFeaturization/FastForestBinaryFeaturizationWithOptions.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TreeFeaturization/FastForestBinaryFeaturizationWithOptions.cs new file mode 100644 index 0000000000..9ca0b87a85 --- /dev/null +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TreeFeaturization/FastForestBinaryFeaturizationWithOptions.cs @@ -0,0 +1,163 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.ML; +using Microsoft.ML.Data; +using Microsoft.ML.Trainers.FastTree; + +namespace Samples.Dynamic.Transforms.TreeFeaturization +{ + public static class FastForestBinaryFeaturizationWithOptions + { + // This example requires installation of additional NuGet package + // Microsoft.ML.FastTree. + public static void Example() + { + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. + var mlContext = new MLContext(seed: 0); + + // Create a list of data points to be transformed. + var dataPoints = GenerateRandomDataPoints(100).ToList(); + + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. + var dataView = mlContext.Data.LoadFromEnumerable(dataPoints); + + // ML.NET doesn't cache data set by default. Therefore, if one reads a + // data set from a file and accesses it many times, it can be slow due + // to expensive featurization and disk operations. When the considered + // data can fit into memory, a solution is to cache the data in memory. + // Caching is especially helpful when working with iterative algorithms + // which needs many data passes. + dataView = mlContext.Data.Cache(dataView); + + // Define input and output columns of tree-based featurizer. + string labelColumnName = nameof(DataPoint.Label); + string featureColumnName = nameof(DataPoint.Features); + string treesColumnName = nameof(TransformedDataPoint.Trees); + string leavesColumnName = nameof(TransformedDataPoint.Leaves); + string pathsColumnName = nameof(TransformedDataPoint.Paths); + + // Define the configuration of the trainer used to train a tree-based + // model. + var trainerOptions = new FastForestBinaryTrainer.Options + { + // Create a simpler model by penalizing usage of new features. + FeatureFirstUsePenalty = 0.1, + // Reduce the number of trees to 3. + NumberOfTrees = 3, + // Number of leaves per tree. + NumberOfLeaves = 6, + // Feature column name. + FeatureColumnName = featureColumnName, + // Label column name. + LabelColumnName = labelColumnName + }; + + // Define the tree-based featurizer's configuration. + var options = new FastForestBinaryFeaturizationEstimator.Options + { + InputColumnName = featureColumnName, + TreesColumnName = treesColumnName, + LeavesColumnName = leavesColumnName, + PathsColumnName = pathsColumnName, + TrainerOptions = trainerOptions + }; + + // Define the featurizer. + var pipeline = mlContext.Transforms.FeaturizeByFastForestBinary( + options); + + // Train the model. + var model = pipeline.Fit(dataView); + + // Apply the trained transformer to the considered data set. + var transformed = model.Transform(dataView); + + // Convert IDataView object to a list. Each element in the resulted list + // corresponds to a row in the IDataView. + var transformedDataPoints = mlContext.Data.CreateEnumerable< + TransformedDataPoint>(transformed, false).ToList(); + + // Print out the transformation of the first 3 data points. + for (int i = 0; i < 3; ++i) + { + var dataPoint = dataPoints[i]; + var transformedDataPoint = transformedDataPoints[i]; + Console.WriteLine("The original feature vector [" + String.Join( + ",", dataPoint.Features) + "] is transformed to three " + + "different tree-based feature vectors:"); + + Console.WriteLine(" Trees' output values: [" + String.Join(",", + transformedDataPoint.Trees) + "]."); + + Console.WriteLine(" Leave IDs' 0-1 representation: [" + String + .Join(",", transformedDataPoint.Leaves) + "]."); + + Console.WriteLine(" Paths IDs' 0-1 representation: [" + String + .Join(",", transformedDataPoint.Paths) + "]."); + } + + // Expected output: + // The original feature vector [0.8173254,0.7680227,0.5581612] is + // transformed to three different tree-based feature vectors: + // Trees' output values: [0.1111111,0.8823529]. + // Leave IDs' 0-1 representation: [0,0,0,0,1,0,0,0,0,1,0]. + // Paths IDs' 0-1 representation: [1,1,1,1,1,1,0,1,0]. + // The original feature vector [0.5888848,0.9360271,0.4721779] is + // transformed to three different tree-based feature vectors: + // Trees' output values: [0.4545455,0.8]. + // Leave IDs' 0-1 representation: [0,0,0,1,0,0,0,0,0,0,1]. + // Paths IDs' 0-1 representation: [1,1,1,1,0,1,0,1,1]. + // The original feature vector [0.2737045,0.2919063,0.4673147] is + // transformed to three different tree-based feature vectors: + // Trees' output values: [0.4545455,0.1111111]. + // Leave IDs' 0-1 representation: [0,0,0,1,0,0,1,0,0,0,0]. + // Paths IDs' 0-1 representation: [1,1,1,1,0,1,0,1,1]. + } + + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) + { + var random = new Random(seed); + float randomFloat() => (float)random.NextDouble(); + for (int i = 0; i < count; i++) + { + var label = randomFloat() > 0.5f; + yield return new DataPoint + { + Label = label, + // Create random features that are correlated with the label. + // For data points with false label, the feature values are + // slightly increased by adding a constant. + Features = Enumerable.Repeat(label, 3).Select(x => x ? + randomFloat() : randomFloat() + 0.03f).ToArray() + }; + } + } + + // Example with label and 3 feature values. A data set is a collection of + // such examples. + private class DataPoint + { + public bool Label { get; set; } + [VectorType(3)] + public float[] Features { get; set; } + } + + // Class used to capture the output of tree-base featurization. + private class TransformedDataPoint : DataPoint + { + // The i-th value is the output value of the i-th decision tree. + public float[] Trees { get; set; } + // The 0-1 encoding of leaves the input feature vector falls into. + public float[] Leaves { get; set; } + // The 0-1 encoding of paths the input feature vector reaches the + // leaves. + public float[] Paths { get; set; } + } + } +} diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TreeFeaturization/FastForestBinaryFeaturizationWithOptions.tt b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TreeFeaturization/FastForestBinaryFeaturizationWithOptions.tt new file mode 100644 index 0000000000..e88e911a26 --- /dev/null +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TreeFeaturization/FastForestBinaryFeaturizationWithOptions.tt @@ -0,0 +1,53 @@ +<#@ include file="BinaryClassificationFeaturization.ttinclude"#> +<#+ +string ClassName="FastForestBinaryFeaturizationWithOptions"; +string Trainer = "FeaturizeByFastForestBinary"; +bool CacheData = true; +string LabelThreshold = "0.5f"; +string DataSepValue = "0.03f"; +string OptionsInclude = "using Microsoft.ML.Trainers.FastTree;"; +string Comments= @" + // This example requires installation of additional NuGet package + // Microsoft.ML.FastTree."; + +string TrainerOptions = @"FastForestBinaryTrainer.Options + { + // Create a simpler model by penalizing usage of new features. + FeatureFirstUsePenalty = 0.1, + // Reduce the number of trees to 3. + NumberOfTrees = 3, + // Number of leaves per tree. + NumberOfLeaves = 6, + // Feature column name. + FeatureColumnName = featureColumnName, + // Label column name. + LabelColumnName = labelColumnName + }"; + +string Options = @"FastForestBinaryFeaturizationEstimator + .Options + { + InputColumnName = featureColumnName, + TreesColumnName = treesColumnName, + LeavesColumnName = leavesColumnName, + PathsColumnName = pathsColumnName, + TrainerOptions = trainerOptions + }"; + +string ExpectedOutput = @"// Expected output: + // The original feature vector [0.8173254,0.7680227,0.5581612] is + // transformed to three different tree-based feature vectors: + // Trees' output values: [0.1111111,0.8823529]. + // Leave IDs' 0-1 representation: [0,0,0,0,1,0,0,0,0,1,0]. + // Paths IDs' 0-1 representation: [1,1,1,1,1,1,0,1,0]. + // The original feature vector [0.5888848,0.9360271,0.4721779] is + // transformed to three different tree-based feature vectors: + // Trees' output values: [0.4545455,0.8]. + // Leave IDs' 0-1 representation: [0,0,0,1,0,0,0,0,0,0,1]. + // Paths IDs' 0-1 representation: [1,1,1,1,0,1,0,1,1]. + // The original feature vector [0.2737045,0.2919063,0.4673147] is + // transformed to three different tree-based feature vectors: + // Trees' output values: [0.4545455,0.1111111]. + // Leave IDs' 0-1 representation: [0,0,0,1,0,0,1,0,0,0,0]. + // Paths IDs' 0-1 representation: [1,1,1,1,0,1,0,1,1]."; +#> \ No newline at end of file diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TreeFeaturization/FastForestRegressionFeaturizationWithOptions.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TreeFeaturization/FastForestRegressionFeaturizationWithOptions.cs new file mode 100644 index 0000000000..cc107d35fd --- /dev/null +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TreeFeaturization/FastForestRegressionFeaturizationWithOptions.cs @@ -0,0 +1,162 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.ML; +using Microsoft.ML.Data; +using Microsoft.ML.Trainers.FastTree; + +namespace Samples.Dynamic.Transforms.TreeFeaturization +{ + public static class FastForestRegressionFeaturizationWithOptions + { + // This example requires installation of additional NuGet package + // Microsoft.ML.FastTree. + public static void Example() + { + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. + var mlContext = new MLContext(seed: 0); + + // Create a list of training data points. + var dataPoints = GenerateRandomDataPoints(100).ToList(); + + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. + var dataView = mlContext.Data.LoadFromEnumerable(dataPoints); + + // ML.NET doesn't cache data set by default. Therefore, if one reads a + // data set from a file and accesses it many times, it can be slow due + // to expensive featurization and disk operations. When the considered + // data can fit into memory, a solution is to cache the data in memory. + // Caching is especially helpful when working with iterative algorithms + // which needs many data passes. + dataView = mlContext.Data.Cache(dataView); + + // Define input and output columns of tree-based featurizer. + string labelColumnName = nameof(DataPoint.Label); + string featureColumnName = nameof(DataPoint.Features); + string treesColumnName = nameof(TransformedDataPoint.Trees); + string leavesColumnName = nameof(TransformedDataPoint.Leaves); + string pathsColumnName = nameof(TransformedDataPoint.Paths); + + // Define the configuration of the trainer used to train a tree-based + // model. + var trainerOptions = new FastForestRegressionTrainer.Options + { + // Only use 80% of features to reduce over-fitting. + FeatureFraction = 0.8, + // Create a simpler model by penalizing usage of new features. + FeatureFirstUsePenalty = 0.1, + // Reduce the number of trees to 3. + NumberOfTrees = 3, + // Number of leaves per tree. + NumberOfLeaves = 6, + LabelColumnName = labelColumnName, + FeatureColumnName = featureColumnName + }; + + // Define the tree-based featurizer's configuration. + var options = new FastForestRegressionFeaturizationEstimator.Options + { + InputColumnName = featureColumnName, + TreesColumnName = treesColumnName, + LeavesColumnName = leavesColumnName, + PathsColumnName = pathsColumnName, + TrainerOptions = trainerOptions + }; + + // Define the featurizer. + var pipeline = mlContext.Transforms.FeaturizeByFastForestRegression( + options); + + // Train the model. + var model = pipeline.Fit(dataView); + + // Create testing data. Use different random seed to make it different + // from training data. + var transformed = model.Transform(dataView); + + // Convert IDataView object to a list. Each element in the resulted list + // corresponds to a row in the IDataView. + var transformedDataPoints = mlContext.Data.CreateEnumerable< + TransformedDataPoint>(transformed, false).ToList(); + + // Print out the transformation of the first 3 data points. + for (int i = 0; i < 3; ++i) + { + var dataPoint = dataPoints[i]; + var transformedDataPoint = transformedDataPoints[i]; + Console.WriteLine("The original feature vector [" + String.Join(",", + dataPoint.Features) + "] is transformed to three different " + + "tree-based feature vectors:"); + + Console.WriteLine(" Trees' output values: [" + String.Join(",", + transformedDataPoint.Trees) + "]."); + + Console.WriteLine(" Leave IDs' 0-1 representation: [" + String + .Join(",", transformedDataPoint.Leaves) + "]."); + + Console.WriteLine(" Paths IDs' 0-1 representation: [" + String + .Join(",", transformedDataPoint.Paths) + "]."); + } + + // Expected output: + // The original feature vector[1.543569, 1.494266, 1.284405] is + // transformed to three different tree - based feature vectors: + // Trees' output values: [0.7291142,0.7825329,0.8764582]. + // Leave IDs' 0-1 representation: [0,1,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0]. + // Paths IDs' 0-1 representation: [1,0,0,0,1,1,1,0,0,1,1,1,0,1]. + // The original feature vector[0.764918, 1.11206, 0.648211] is + // transformed to three different tree - based feature vectors: + // Trees' output values: [0.3802337,0.584159,0.5648927]. + // Leave IDs' 0-1 representation: [0,0,1,0,0,0,0,1,0,0,0,0,0,1,0,0,0]. + // Paths IDs' 0-1 representation: [1,1,1,0,0,1,1,0,0,0,1,1,0,0]. + // The original feature vector[1.251254, 1.269456, 1.444864] is + // transformed to three different tree - based feature vectors: + // Trees' output values: [0.7591804,0.7825329,0.7443035]. + // Leave IDs' 0-1 representation: [0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,1]. + // Paths IDs' 0-1 representation: [1,0,0,0,1,1,1,0,0,1,1,1,0,1]. + } + + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) + { + var random = new Random(seed); + for (int i = 0; i < count; i++) + { + float label = (float)random.NextDouble(); + yield return new DataPoint + { + Label = label, + // Create random features that are correlated with the label. + Features = Enumerable.Repeat(label, 3).Select(x => x + + (float)random.NextDouble()).ToArray() + }; + } + } + + // Example with label and 50 feature values. A data set is a collection of + // such examples. + private class DataPoint + { + public float Label { get; set; } + [VectorType(3)] + public float[] Features { get; set; } + } + + // Class used to capture the output of tree-base featurization. + private class TransformedDataPoint : DataPoint + { + // The i-th value is the output value of the i-th decision tree. + public float[] Trees { get; set; } + // The 0-1 encoding of leaves the input feature vector falls into. + public float[] Leaves { get; set; } + // The 0-1 encoding of paths the input feature vector reaches the + // leaves. + public float[] Paths { get; set; } + } + } +} + diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TreeFeaturization/FastForestRegressionFeaturizationWithOptions.tt b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TreeFeaturization/FastForestRegressionFeaturizationWithOptions.tt new file mode 100644 index 0000000000..a73542dacb --- /dev/null +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TreeFeaturization/FastForestRegressionFeaturizationWithOptions.tt @@ -0,0 +1,50 @@ +<#@ include file="RegressionFeaturization.ttinclude"#> + +<#+ +string ClassHeader = @"// This example requires installation of additional NuGet package + // Microsoft.ML.FastTree. "; + +string ClassName="FastForestRegressionFeaturizationWithOptions"; +bool CacheData = true; +string ExtraUsing = "using Microsoft.ML.Trainers.FastTree;"; +string Trainer = @"FeaturizeByFastForestRegression"; +string TrainerOptions = @"FastForestRegressionTrainer.Options + { + // Only use 80% of features to reduce over-fitting. + FeatureFraction = 0.8, + // Create a simpler model by penalizing usage of new features. + FeatureFirstUsePenalty = 0.1, + // Reduce the number of trees to 3. + NumberOfTrees = 3, + // Number of leaves per tree. + NumberOfLeaves = 6, + LabelColumnName = labelColumnName, + FeatureColumnName = featureColumnName + }"; + +string Options = @"FastForestRegressionFeaturizationEstimator.Options + { + InputColumnName = featureColumnName, + TreesColumnName = treesColumnName, + LeavesColumnName = leavesColumnName, + PathsColumnName = pathsColumnName, + TrainerOptions = trainerOptions + }"; + +string ExpectedOutput = @"// Expected output: + // The original feature vector[1.543569, 1.494266, 1.284405] is + // transformed to three different tree - based feature vectors: + // Trees' output values: [0.7291142,0.7825329,0.8764582]. + // Leave IDs' 0-1 representation: [0,1,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0]. + // Paths IDs' 0-1 representation: [1,0,0,0,1,1,1,0,0,1,1,1,0,1]. + // The original feature vector[0.764918, 1.11206, 0.648211] is + // transformed to three different tree - based feature vectors: + // Trees' output values: [0.3802337,0.584159,0.5648927]. + // Leave IDs' 0-1 representation: [0,0,1,0,0,0,0,1,0,0,0,0,0,1,0,0,0]. + // Paths IDs' 0-1 representation: [1,1,1,0,0,1,1,0,0,0,1,1,0,0]. + // The original feature vector[1.251254, 1.269456, 1.444864] is + // transformed to three different tree - based feature vectors: + // Trees' output values: [0.7591804,0.7825329,0.7443035]. + // Leave IDs' 0-1 representation: [0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,1]. + // Paths IDs' 0-1 representation: [1,0,0,0,1,1,1,0,0,1,1,1,0,1]."; +#> \ No newline at end of file diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TreeFeaturization/FastTreeBinaryFeaturizationWithOptions.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TreeFeaturization/FastTreeBinaryFeaturizationWithOptions.cs new file mode 100644 index 0000000000..521e04e23b --- /dev/null +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TreeFeaturization/FastTreeBinaryFeaturizationWithOptions.cs @@ -0,0 +1,165 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.ML; +using Microsoft.ML.Data; +using Microsoft.ML.Trainers.FastTree; + +namespace Samples.Dynamic.Transforms.TreeFeaturization +{ + public static class FastTreeBinaryFeaturizationWithOptions + { + // This example requires installation of additional NuGet package + // Microsoft.ML.FastTree. + public static void Example() + { + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. + var mlContext = new MLContext(seed: 0); + + // Create a list of data points to be transformed. + var dataPoints = GenerateRandomDataPoints(100).ToList(); + + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. + var dataView = mlContext.Data.LoadFromEnumerable(dataPoints); + + // ML.NET doesn't cache data set by default. Therefore, if one reads a + // data set from a file and accesses it many times, it can be slow due + // to expensive featurization and disk operations. When the considered + // data can fit into memory, a solution is to cache the data in memory. + // Caching is especially helpful when working with iterative algorithms + // which needs many data passes. + dataView = mlContext.Data.Cache(dataView); + + // Define input and output columns of tree-based featurizer. + string labelColumnName = nameof(DataPoint.Label); + string featureColumnName = nameof(DataPoint.Features); + string treesColumnName = nameof(TransformedDataPoint.Trees); + string leavesColumnName = nameof(TransformedDataPoint.Leaves); + string pathsColumnName = nameof(TransformedDataPoint.Paths); + + // Define the configuration of the trainer used to train a tree-based + // model. + var trainerOptions = new FastTreeBinaryTrainer.Options + { + // Use L2Norm for early stopping. + EarlyStoppingMetric = EarlyStoppingMetric.L2Norm, + // Create a simpler model by penalizing usage of new features. + FeatureFirstUsePenalty = 0.1, + // Reduce the number of trees to 3. + NumberOfTrees = 3, + // Number of leaves per tree. + NumberOfLeaves = 6, + // Feature column name. + FeatureColumnName = featureColumnName, + // Label column name. + LabelColumnName = labelColumnName + }; + + // Define the tree-based featurizer's configuration. + var options = new FastTreeBinaryFeaturizationEstimator.Options + { + InputColumnName = featureColumnName, + TreesColumnName = treesColumnName, + LeavesColumnName = leavesColumnName, + PathsColumnName = pathsColumnName, + TrainerOptions = trainerOptions + }; + + // Define the featurizer. + var pipeline = mlContext.Transforms.FeaturizeByFastTreeBinary( + options); + + // Train the model. + var model = pipeline.Fit(dataView); + + // Apply the trained transformer to the considered data set. + var transformed = model.Transform(dataView); + + // Convert IDataView object to a list. Each element in the resulted list + // corresponds to a row in the IDataView. + var transformedDataPoints = mlContext.Data.CreateEnumerable< + TransformedDataPoint>(transformed, false).ToList(); + + // Print out the transformation of the first 3 data points. + for (int i = 0; i < 3; ++i) + { + var dataPoint = dataPoints[i]; + var transformedDataPoint = transformedDataPoints[i]; + Console.WriteLine("The original feature vector [" + String.Join( + ",", dataPoint.Features) + "] is transformed to three " + + "different tree-based feature vectors:"); + + Console.WriteLine(" Trees' output values: [" + String.Join(",", + transformedDataPoint.Trees) + "]."); + + Console.WriteLine(" Leave IDs' 0-1 representation: [" + String + .Join(",", transformedDataPoint.Leaves) + "]."); + + Console.WriteLine(" Paths IDs' 0-1 representation: [" + String + .Join(",", transformedDataPoint.Paths) + "]."); + } + + // Expected output: + // The original feature vector [0.8173254,0.7680227,0.5581612] is + // transformed to three different tree-based feature vectors: + // Trees' output values: [0.5714286,0.4636412,0.535588]. + // Leave IDs' 0-1 representation: [0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,1]. + // Paths IDs' 0-1 representation: [1,0,0,1,1,1,0,1,0,1,1,1,1,1,1]. + // The original feature vector [0.5888848,0.9360271,0.4721779] is + // transformed to three different tree-based feature vectors: + // Trees' output values: [0.2352941,-0.1382389,0.535588]. + // Leave IDs' 0-1 representation: [0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,1]. + // Paths IDs' 0-1 representation: [1,0,0,1,1,1,0,1,0,1,1,1,1,1,1]. + // The original feature vector [0.2737045,0.2919063,0.4673147] is + // transformed to three different tree-based feature vectors: + // Trees' output values: [0.2352941,-0.1382389,-0.2184284]. + // Leave IDs' 0-1 representation: [0,0,0,0,1,0,0,0,0,1,0,0,0,1,0,0,0,0]. + // Paths IDs' 0-1 representation: [1,0,0,1,1,1,0,1,0,1,1,1,0,0,0]. + } + + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) + { + var random = new Random(seed); + float randomFloat() => (float)random.NextDouble(); + for (int i = 0; i < count; i++) + { + var label = randomFloat() > 0.5f; + yield return new DataPoint + { + Label = label, + // Create random features that are correlated with the label. + // For data points with false label, the feature values are + // slightly increased by adding a constant. + Features = Enumerable.Repeat(label, 3).Select(x => x ? + randomFloat() : randomFloat() + 0.03f).ToArray() + }; + } + } + + // Example with label and 3 feature values. A data set is a collection of + // such examples. + private class DataPoint + { + public bool Label { get; set; } + [VectorType(3)] + public float[] Features { get; set; } + } + + // Class used to capture the output of tree-base featurization. + private class TransformedDataPoint : DataPoint + { + // The i-th value is the output value of the i-th decision tree. + public float[] Trees { get; set; } + // The 0-1 encoding of leaves the input feature vector falls into. + public float[] Leaves { get; set; } + // The 0-1 encoding of paths the input feature vector reaches the + // leaves. + public float[] Paths { get; set; } + } + } +} diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TreeFeaturization/FastTreeBinaryFeaturizationWithOptions.tt b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TreeFeaturization/FastTreeBinaryFeaturizationWithOptions.tt new file mode 100644 index 0000000000..857365347f --- /dev/null +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TreeFeaturization/FastTreeBinaryFeaturizationWithOptions.tt @@ -0,0 +1,60 @@ +<#@ include file="BinaryClassificationFeaturization.ttinclude"#> +<#+ +string ClassName="FastTreeBinaryFeaturizationWithOptions"; +string Trainer = "FeaturizeByFastTreeBinary"; +bool CacheData = true; +string LabelThreshold = "0.5f"; +string DataSepValue = "0.03f"; +string OptionsInclude = "using Microsoft.ML.Trainers.FastTree;"; +string Comments= @" + // This example requires installation of additional NuGet package + // Microsoft.ML.FastTree."; +string TrainerOptions = @"FastTreeBinaryTrainer.Options + { + // Use L2Norm for early stopping. + EarlyStoppingMetric = EarlyStoppingMetric.L2Norm, + // Create a simpler model by penalizing usage of new features. + FeatureFirstUsePenalty = 0.1, + // Reduce the number of trees to 3. + NumberOfTrees = 3, + // Number of leaves per tree. + NumberOfLeaves = 6, + // Feature column name. + FeatureColumnName = featureColumnName, + // Label column name. + LabelColumnName = labelColumnName + }"; + +string Options = @"FastTreeBinaryFeaturizationEstimator.Options + { + InputColumnName = featureColumnName, + TreesColumnName = treesColumnName, + LeavesColumnName = leavesColumnName, + PathsColumnName = pathsColumnName, + TrainerOptions = trainerOptions + }"; + +string ExpectedOutputPerInstance= @"// Expected output: + // Label: True, Prediction: True + // Label: False, Prediction: False + // Label: True, Prediction: True + // Label: True, Prediction: True + // Label: False, Prediction: False"; + +string ExpectedOutput = @"// Expected output: + // The original feature vector [0.8173254,0.7680227,0.5581612] is + // transformed to three different tree-based feature vectors: + // Trees' output values: [0.5714286,0.4636412,0.535588]. + // Leave IDs' 0-1 representation: [0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,1]. + // Paths IDs' 0-1 representation: [1,0,0,1,1,1,0,1,0,1,1,1,1,1,1]. + // The original feature vector [0.5888848,0.9360271,0.4721779] is + // transformed to three different tree-based feature vectors: + // Trees' output values: [0.2352941,-0.1382389,0.535588]. + // Leave IDs' 0-1 representation: [0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,1]. + // Paths IDs' 0-1 representation: [1,0,0,1,1,1,0,1,0,1,1,1,1,1,1]. + // The original feature vector [0.2737045,0.2919063,0.4673147] is + // transformed to three different tree-based feature vectors: + // Trees' output values: [0.2352941,-0.1382389,-0.2184284]. + // Leave IDs' 0-1 representation: [0,0,0,0,1,0,0,0,0,1,0,0,0,1,0,0,0,0]. + // Paths IDs' 0-1 representation: [1,0,0,1,1,1,0,1,0,1,1,1,0,0,0]."; +#> \ No newline at end of file diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TreeFeaturization/FastTreeRankingFeaturizationWithOptions.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TreeFeaturization/FastTreeRankingFeaturizationWithOptions.cs new file mode 100644 index 0000000000..e11df0eb70 --- /dev/null +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TreeFeaturization/FastTreeRankingFeaturizationWithOptions.cs @@ -0,0 +1,165 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.ML; +using Microsoft.ML.Data; +using Microsoft.ML.Trainers.FastTree; + +namespace Samples.Dynamic.Transforms.TreeFeaturization +{ + public static class FastTreeRankingFeaturizationWithOptions + { + // This example requires installation of additional NuGet package + // Microsoft.ML.FastTree. + public static void Example() + { + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. + var mlContext = new MLContext(seed: 0); + + // Create a list of training data points. + var dataPoints = GenerateRandomDataPoints(100).ToList(); + + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. + var dataView = mlContext.Data.LoadFromEnumerable(dataPoints); + + // ML.NET doesn't cache data set by default. Therefore, if one reads a + // data set from a file and accesses it many times, it can be slow due + // to expensive featurization and disk operations. When the considered + // data can fit into memory, a solution is to cache the data in memory. + // Caching is especially helpful when working with iterative algorithms + // which needs many data passes. + dataView = mlContext.Data.Cache(dataView); + + // Define input and output columns of tree-based featurizer. + string labelColumnName = nameof(DataPoint.Label); + string featureColumnName = nameof(DataPoint.Features); + string treesColumnName = nameof(TransformedDataPoint.Trees); + string leavesColumnName = nameof(TransformedDataPoint.Leaves); + string pathsColumnName = nameof(TransformedDataPoint.Paths); + + // Define the configuration of the trainer used to train a tree-based + // model. + var trainerOptions = new FastTreeRankingTrainer.Options + { + // Reduce the number of trees to 3. + NumberOfTrees = 3, + // Number of leaves per tree. + NumberOfLeaves = 6, + // Feature column name. + FeatureColumnName = featureColumnName, + // Label column name. + LabelColumnName = labelColumnName + }; + + // Define the tree-based featurizer's configuration. + var options = new FastTreeRankingFeaturizationEstimator.Options + { + InputColumnName = featureColumnName, + TreesColumnName = treesColumnName, + LeavesColumnName = leavesColumnName, + PathsColumnName = pathsColumnName, + TrainerOptions = trainerOptions + }; + + // Define the featurizer. + var pipeline = mlContext.Transforms.FeaturizeByFastTreeRanking( + options); + + // Train the model. + var model = pipeline.Fit(dataView); + + // Apply the trained transformer to the considered data set. + var transformed = model.Transform(dataView); + + // Convert IDataView object to a list. Each element in the resulted list + // corresponds to a row in the IDataView. + var transformedDataPoints = mlContext.Data.CreateEnumerable< + TransformedDataPoint>(transformed, false).ToList(); + + // Print out the transformation of the first 3 data points. + for (int i = 0; i < 3; ++i) + { + var dataPoint = dataPoints[i]; + var transformedDataPoint = transformedDataPoints[i]; + Console.WriteLine("The original feature vector [" + String.Join(",", + dataPoint.Features) + "] is transformed to three different " + + "tree-based feature vectors:"); + + Console.WriteLine(" Trees' output values: [" + String.Join(",", + transformedDataPoint.Trees) + "]."); + + Console.WriteLine(" Leave IDs' 0-1 representation: [" + String + .Join(",", transformedDataPoint.Leaves) + "]."); + + Console.WriteLine(" Paths IDs' 0-1 representation: [" + String + .Join(",", transformedDataPoint.Paths) + "]."); + } + + // Expected output: + // The original feature vector [1.117325,1.068023,0.8581612] is + // transformed to three different tree-based feature vectors: + // Trees' output values: [0.4095458,0.2061437,0.2364294]. + // Leave IDs' 0-1 representation: [0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,1]. + // Paths IDs' 0-1 representation: [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]. + // The original feature vector [0.6588848,1.006027,0.5421779] is + // transformed to three different tree-based feature vectors: + // Trees' output values: [0.2543825,-0.06570309,-0.1456212]. + // Leave IDs' 0-1 representation: [0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,1,0,0]. + // Paths IDs' 0-1 representation: [1,1,1,1,1,1,1,1,1,1,1,1,1,1,0]. + // The original feature vector [0.6737045,0.6919063,0.8673147] is + // transformed to three different tree-based feature vectors: + // Trees' output values: [0.2543825,-0.06570309,0.01300209]. + // Leave IDs' 0-1 representation: [0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,1,0]. + // Paths IDs' 0-1 representation: [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]. + } + + private static IEnumerable GenerateRandomDataPoints(int count, + int seed = 0, int groupSize = 10) + { + var random = new Random(seed); + float randomFloat() => (float)random.NextDouble(); + for (int i = 0; i < count; i++) + { + var label = random.Next(0, 5); + yield return new DataPoint + { + Label = (uint)label, + GroupId = (uint)(i / groupSize), + // Create random features that are correlated with the label. + // For data points with larger labels, the feature values are + // slightly increased by adding a constant. + Features = Enumerable.Repeat(label, 3).Select(x => randomFloat() + + x * 0.1f).ToArray() + }; + } + } + + // Example with label, groupId, and 3 feature values. A data set is a + // collection of such examples. + private class DataPoint + { + [KeyType(5)] + public uint Label { get; set; } + [KeyType(100)] + public uint GroupId { get; set; } + [VectorType(3)] + public float[] Features { get; set; } + } + + // Class used to capture the output of tree-base featurization. + private class TransformedDataPoint : DataPoint + { + // The i-th value is the output value of the i-th decision tree. + public float[] Trees { get; set; } + // The 0-1 encoding of leaves the input feature vector falls into. + public float[] Leaves { get; set; } + // The 0-1 encoding of paths the input feature vector reaches the + // leaves. + public float[] Paths { get; set; } + } + } +} diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TreeFeaturization/FastTreeRankingFeaturizationWithOptions.tt b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TreeFeaturization/FastTreeRankingFeaturizationWithOptions.tt new file mode 100644 index 0000000000..5c1fc30d7d --- /dev/null +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TreeFeaturization/FastTreeRankingFeaturizationWithOptions.tt @@ -0,0 +1,50 @@ +<#@ include file="RankingFeaturization.ttinclude"#> +<#+ +string ClassName = "FastTreeRankingFeaturizationWithOptions"; +string Trainer = "FeaturizeByFastTreeRanking"; +bool CacheData = true; + +string TrainerOptions = @"FastTreeRankingTrainer.Options + { + // Reduce the number of trees to 3. + NumberOfTrees = 3, + // Number of leaves per tree. + NumberOfLeaves = 6, + // Feature column name. + FeatureColumnName = featureColumnName, + // Label column name. + LabelColumnName = labelColumnName + }"; + +string Options = @"FastTreeRankingFeaturizationEstimator.Options + { + InputColumnName = featureColumnName, + TreesColumnName = treesColumnName, + LeavesColumnName = leavesColumnName, + PathsColumnName = pathsColumnName, + TrainerOptions = trainerOptions + }"; + +string OptionsInclude = "using Microsoft.ML.Trainers.FastTree;"; + +string Comments= @" + // This example requires installation of additional NuGet package + // Microsoft.ML.FastTree."; + +string ExpectedOutput = @"// Expected output: + // The original feature vector [1.117325,1.068023,0.8581612] is + // transformed to three different tree-based feature vectors: + // Trees' output values: [0.4095458,0.2061437,0.2364294]. + // Leave IDs' 0-1 representation: [0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,1]. + // Paths IDs' 0-1 representation: [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]. + // The original feature vector [0.6588848,1.006027,0.5421779] is + // transformed to three different tree-based feature vectors: + // Trees' output values: [0.2543825,-0.06570309,-0.1456212]. + // Leave IDs' 0-1 representation: [0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,1,0,0]. + // Paths IDs' 0-1 representation: [1,1,1,1,1,1,1,1,1,1,1,1,1,1,0]. + // The original feature vector [0.6737045,0.6919063,0.8673147] is + // transformed to three different tree-based feature vectors: + // Trees' output values: [0.2543825,-0.06570309,0.01300209]. + // Leave IDs' 0-1 representation: [0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,1,0]. + // Paths IDs' 0-1 representation: [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]."; +#> \ No newline at end of file diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TreeFeaturization/FastTreeRegressionFeaturizationWithOptions.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TreeFeaturization/FastTreeRegressionFeaturizationWithOptions.cs new file mode 100644 index 0000000000..407bb2e22f --- /dev/null +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TreeFeaturization/FastTreeRegressionFeaturizationWithOptions.cs @@ -0,0 +1,162 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.ML; +using Microsoft.ML.Data; +using Microsoft.ML.Trainers.FastTree; + +namespace Samples.Dynamic.Transforms.TreeFeaturization +{ + public static class FastTreeRegressionFeaturizationWithOptions + { + // This example requires installation of additional NuGet package + // Microsoft.ML.FastTree. + public static void Example() + { + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. + var mlContext = new MLContext(seed: 0); + + // Create a list of training data points. + var dataPoints = GenerateRandomDataPoints(100).ToList(); + + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. + var dataView = mlContext.Data.LoadFromEnumerable(dataPoints); + + // ML.NET doesn't cache data set by default. Therefore, if one reads a + // data set from a file and accesses it many times, it can be slow due + // to expensive featurization and disk operations. When the considered + // data can fit into memory, a solution is to cache the data in memory. + // Caching is especially helpful when working with iterative algorithms + // which needs many data passes. + dataView = mlContext.Data.Cache(dataView); + + // Define input and output columns of tree-based featurizer. + string labelColumnName = nameof(DataPoint.Label); + string featureColumnName = nameof(DataPoint.Features); + string treesColumnName = nameof(TransformedDataPoint.Trees); + string leavesColumnName = nameof(TransformedDataPoint.Leaves); + string pathsColumnName = nameof(TransformedDataPoint.Paths); + + // Define the configuration of the trainer used to train a tree-based + // model. + var trainerOptions = new FastTreeRegressionTrainer.Options + { + // Only use 80% of features to reduce over-fitting. + FeatureFraction = 0.8, + // Create a simpler model by penalizing usage of new features. + FeatureFirstUsePenalty = 0.1, + // Reduce the number of trees to 3. + NumberOfTrees = 3, + // Number of leaves per tree. + NumberOfLeaves = 6, + LabelColumnName = labelColumnName, + FeatureColumnName = featureColumnName + }; + + // Define the tree-based featurizer's configuration. + var options = new FastTreeRegressionFeaturizationEstimator.Options + { + InputColumnName = featureColumnName, + TreesColumnName = treesColumnName, + LeavesColumnName = leavesColumnName, + PathsColumnName = pathsColumnName, + TrainerOptions = trainerOptions + }; + + // Define the featurizer. + var pipeline = mlContext.Transforms.FeaturizeByFastTreeRegression( + options); + + // Train the model. + var model = pipeline.Fit(dataView); + + // Create testing data. Use different random seed to make it different + // from training data. + var transformed = model.Transform(dataView); + + // Convert IDataView object to a list. Each element in the resulted list + // corresponds to a row in the IDataView. + var transformedDataPoints = mlContext.Data.CreateEnumerable< + TransformedDataPoint>(transformed, false).ToList(); + + // Print out the transformation of the first 3 data points. + for (int i = 0; i < 3; ++i) + { + var dataPoint = dataPoints[i]; + var transformedDataPoint = transformedDataPoints[i]; + Console.WriteLine("The original feature vector [" + String.Join(",", + dataPoint.Features) + "] is transformed to three different " + + "tree-based feature vectors:"); + + Console.WriteLine(" Trees' output values: [" + String.Join(",", + transformedDataPoint.Trees) + "]."); + + Console.WriteLine(" Leave IDs' 0-1 representation: [" + String + .Join(",", transformedDataPoint.Leaves) + "]."); + + Console.WriteLine(" Paths IDs' 0-1 representation: [" + String + .Join(",", transformedDataPoint.Paths) + "]."); + } + + // Expected output: + // The original feature vector [1.543569,1.494266,1.284405] is + // transformed to three different tree-based feature vectors: + // Trees' output values: [0.1507567,0.1372715,0.1019326]. + // Leave IDs' 0-1 representation: [0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0]. + // Paths IDs' 0-1 representation: [1,0,0,0,0,1,1,1,0,0,1,1,1,1,0]. + // The original feature vector [0.764918,1.11206,0.648211] is + // transformed to three different tree-based feature vectors: + // Trees' output values: [0.07604675,0.08244576,0.03080027]. + // Leave IDs' 0-1 representation: [0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1]. + // Paths IDs' 0-1 representation: [1,1,1,0,0,1,1,0,0,0,1,0,0,0,1]. + // The original feature vector [1.251254,1.269456,1.444864] is + // transformed to three different tree-based feature vectors: + // Trees' output values: [0.1507567,0.1090626,0.0731837]. + // Leave IDs' 0-1 representation: [0,1,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0]. + // Paths IDs' 0-1 representation: [1,0,0,0,0,1,1,1,0,0,1,1,1,1,0]. + } + + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) + { + var random = new Random(seed); + for (int i = 0; i < count; i++) + { + float label = (float)random.NextDouble(); + yield return new DataPoint + { + Label = label, + // Create random features that are correlated with the label. + Features = Enumerable.Repeat(label, 3).Select(x => x + + (float)random.NextDouble()).ToArray() + }; + } + } + + // Example with label and 50 feature values. A data set is a collection of + // such examples. + private class DataPoint + { + public float Label { get; set; } + [VectorType(3)] + public float[] Features { get; set; } + } + + // Class used to capture the output of tree-base featurization. + private class TransformedDataPoint : DataPoint + { + // The i-th value is the output value of the i-th decision tree. + public float[] Trees { get; set; } + // The 0-1 encoding of leaves the input feature vector falls into. + public float[] Leaves { get; set; } + // The 0-1 encoding of paths the input feature vector reaches the + // leaves. + public float[] Paths { get; set; } + } + } +} + diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TreeFeaturization/FastTreeRegressionFeaturizationWithOptions.tt b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TreeFeaturization/FastTreeRegressionFeaturizationWithOptions.tt new file mode 100644 index 0000000000..c03bf4e197 --- /dev/null +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TreeFeaturization/FastTreeRegressionFeaturizationWithOptions.tt @@ -0,0 +1,50 @@ +<#@ include file="RegressionFeaturization.ttinclude"#> + +<#+ +string ClassHeader = @"// This example requires installation of additional NuGet package + // Microsoft.ML.FastTree. "; + +string ClassName="FastTreeRegressionFeaturizationWithOptions"; +bool CacheData = true; +string ExtraUsing = "using Microsoft.ML.Trainers.FastTree;"; +string Trainer = @"FeaturizeByFastTreeRegression"; +string TrainerOptions = @"FastTreeRegressionTrainer.Options + { + // Only use 80% of features to reduce over-fitting. + FeatureFraction = 0.8, + // Create a simpler model by penalizing usage of new features. + FeatureFirstUsePenalty = 0.1, + // Reduce the number of trees to 3. + NumberOfTrees = 3, + // Number of leaves per tree. + NumberOfLeaves = 6, + LabelColumnName = labelColumnName, + FeatureColumnName = featureColumnName + }"; + +string Options = @"FastTreeRegressionFeaturizationEstimator.Options + { + InputColumnName = featureColumnName, + TreesColumnName = treesColumnName, + LeavesColumnName = leavesColumnName, + PathsColumnName = pathsColumnName, + TrainerOptions = trainerOptions + }"; + +string ExpectedOutput = @"// Expected output: + // The original feature vector [1.543569,1.494266,1.284405] is + // transformed to three different tree-based feature vectors: + // Trees' output values: [0.1507567,0.1372715,0.1019326]. + // Leave IDs' 0-1 representation: [0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0]. + // Paths IDs' 0-1 representation: [1,0,0,0,0,1,1,1,0,0,1,1,1,1,0]. + // The original feature vector [0.764918,1.11206,0.648211] is + // transformed to three different tree-based feature vectors: + // Trees' output values: [0.07604675,0.08244576,0.03080027]. + // Leave IDs' 0-1 representation: [0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1]. + // Paths IDs' 0-1 representation: [1,1,1,0,0,1,1,0,0,0,1,0,0,0,1]. + // The original feature vector [1.251254,1.269456,1.444864] is + // transformed to three different tree-based feature vectors: + // Trees' output values: [0.1507567,0.1090626,0.0731837]. + // Leave IDs' 0-1 representation: [0,1,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0]. + // Paths IDs' 0-1 representation: [1,0,0,0,0,1,1,1,0,0,1,1,1,1,0]."; +#> \ No newline at end of file diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TreeFeaturization/FastTreeTweedieFeaturizationWithOptions.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TreeFeaturization/FastTreeTweedieFeaturizationWithOptions.cs new file mode 100644 index 0000000000..085db4eeb7 --- /dev/null +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TreeFeaturization/FastTreeTweedieFeaturizationWithOptions.cs @@ -0,0 +1,162 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.ML; +using Microsoft.ML.Data; +using Microsoft.ML.Trainers.FastTree; + +namespace Samples.Dynamic.Transforms.TreeFeaturization +{ + public static class FastTreeTweedieFeaturizationWithOptions + { + // This example requires installation of additional NuGet package + // Microsoft.ML.FastTree. + public static void Example() + { + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. + var mlContext = new MLContext(seed: 0); + + // Create a list of training data points. + var dataPoints = GenerateRandomDataPoints(100).ToList(); + + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. + var dataView = mlContext.Data.LoadFromEnumerable(dataPoints); + + // ML.NET doesn't cache data set by default. Therefore, if one reads a + // data set from a file and accesses it many times, it can be slow due + // to expensive featurization and disk operations. When the considered + // data can fit into memory, a solution is to cache the data in memory. + // Caching is especially helpful when working with iterative algorithms + // which needs many data passes. + dataView = mlContext.Data.Cache(dataView); + + // Define input and output columns of tree-based featurizer. + string labelColumnName = nameof(DataPoint.Label); + string featureColumnName = nameof(DataPoint.Features); + string treesColumnName = nameof(TransformedDataPoint.Trees); + string leavesColumnName = nameof(TransformedDataPoint.Leaves); + string pathsColumnName = nameof(TransformedDataPoint.Paths); + + // Define the configuration of the trainer used to train a tree-based + // model. + var trainerOptions = new FastTreeTweedieTrainer.Options + { + // Only use 80% of features to reduce over-fitting. + FeatureFraction = 0.8, + // Create a simpler model by penalizing usage of new features. + FeatureFirstUsePenalty = 0.1, + // Reduce the number of trees to 3. + NumberOfTrees = 3, + // Number of leaves per tree. + NumberOfLeaves = 6, + LabelColumnName = labelColumnName, + FeatureColumnName = featureColumnName + }; + + // Define the tree-based featurizer's configuration. + var options = new FastTreeTweedieFeaturizationEstimator.Options + { + InputColumnName = featureColumnName, + TreesColumnName = treesColumnName, + LeavesColumnName = leavesColumnName, + PathsColumnName = pathsColumnName, + TrainerOptions = trainerOptions + }; + + // Define the featurizer. + var pipeline = mlContext.Transforms.FeaturizeByFastTreeTweedie( + options); + + // Train the model. + var model = pipeline.Fit(dataView); + + // Create testing data. Use different random seed to make it different + // from training data. + var transformed = model.Transform(dataView); + + // Convert IDataView object to a list. Each element in the resulted list + // corresponds to a row in the IDataView. + var transformedDataPoints = mlContext.Data.CreateEnumerable< + TransformedDataPoint>(transformed, false).ToList(); + + // Print out the transformation of the first 3 data points. + for (int i = 0; i < 3; ++i) + { + var dataPoint = dataPoints[i]; + var transformedDataPoint = transformedDataPoints[i]; + Console.WriteLine("The original feature vector [" + String.Join(",", + dataPoint.Features) + "] is transformed to three different " + + "tree-based feature vectors:"); + + Console.WriteLine(" Trees' output values: [" + String.Join(",", + transformedDataPoint.Trees) + "]."); + + Console.WriteLine(" Leave IDs' 0-1 representation: [" + String + .Join(",", transformedDataPoint.Leaves) + "]."); + + Console.WriteLine(" Paths IDs' 0-1 representation: [" + String + .Join(",", transformedDataPoint.Paths) + "]."); + } + + // Expected output: + // The original feature vector [1.543569,1.494266,1.284405] is + // transformed to three different tree-based feature vectors: + // Trees' output values: [-0.05652997,-0.02312196,-0.01179363]. + // Leave IDs' 0-1 representation: [0,1,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0]. + // Paths IDs' 0-1 representation: [1,0,0,0,0,1,1,0,1,0,1,1,0,0,0]. + // The original feature vector [0.764918,1.11206,0.648211] is + // transformed to three different tree-based feature vectors: + // Trees' output values: [-0.1933938,-0.1042738,-0.2312837]. + // Leave IDs' 0-1 representation: [0,0,1,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0]. + // Paths IDs' 0-1 representation: [1,1,1,0,0,1,1,0,0,0,1,0,0,0,0]. + // The original feature vector [1.251254,1.269456,1.444864] is + // transformed to three different tree-based feature vectors: + // Trees' output values: [-0.05652997,-0.06082304,-0.04528879]. + // Leave IDs' 0-1 representation: [0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0]. + // Paths IDs' 0-1 representation: [1,0,0,0,0,1,1,0,1,0,1,1,1,0,1]. + } + + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) + { + var random = new Random(seed); + for (int i = 0; i < count; i++) + { + float label = (float)random.NextDouble(); + yield return new DataPoint + { + Label = label, + // Create random features that are correlated with the label. + Features = Enumerable.Repeat(label, 3).Select(x => x + + (float)random.NextDouble()).ToArray() + }; + } + } + + // Example with label and 50 feature values. A data set is a collection of + // such examples. + private class DataPoint + { + public float Label { get; set; } + [VectorType(3)] + public float[] Features { get; set; } + } + + // Class used to capture the output of tree-base featurization. + private class TransformedDataPoint : DataPoint + { + // The i-th value is the output value of the i-th decision tree. + public float[] Trees { get; set; } + // The 0-1 encoding of leaves the input feature vector falls into. + public float[] Leaves { get; set; } + // The 0-1 encoding of paths the input feature vector reaches the + // leaves. + public float[] Paths { get; set; } + } + } +} + diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TreeFeaturization/FastTreeTweedieFeaturizationWithOptions.tt b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TreeFeaturization/FastTreeTweedieFeaturizationWithOptions.tt new file mode 100644 index 0000000000..b1ea23e2b2 --- /dev/null +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TreeFeaturization/FastTreeTweedieFeaturizationWithOptions.tt @@ -0,0 +1,57 @@ +<#@ include file="RegressionFeaturization.ttinclude"#> + +<#+ +string ClassHeader = @"// This example requires installation of additional NuGet package + // Microsoft.ML.FastTree. "; + +string ClassName="FastTreeTweedieFeaturizationWithOptions"; +bool CacheData = true; +string ExtraUsing = "using Microsoft.ML.Trainers.FastTree;"; +string Trainer = @"FeaturizeByFastTreeTweedie"; +string TrainerOptions = @"FastTreeTweedieTrainer.Options + { + // Only use 80% of features to reduce over-fitting. + FeatureFraction = 0.8, + // Create a simpler model by penalizing usage of new features. + FeatureFirstUsePenalty = 0.1, + // Reduce the number of trees to 3. + NumberOfTrees = 3, + // Number of leaves per tree. + NumberOfLeaves = 6, + LabelColumnName = labelColumnName, + FeatureColumnName = featureColumnName + }"; + +string Options = @"FastTreeTweedieFeaturizationEstimator.Options + { + InputColumnName = featureColumnName, + TreesColumnName = treesColumnName, + LeavesColumnName = leavesColumnName, + PathsColumnName = pathsColumnName, + TrainerOptions = trainerOptions + }"; + +string ExpectedOutputPerInstance= @"// Expected output: + // Label: 0.985, Prediction: 0.866 + // Label: 0.155, Prediction: 0.171 + // Label: 0.515, Prediction: 0.470 + // Label: 0.566, Prediction: 0.476 + // Label: 0.096, Prediction: 0.140"; + +string ExpectedOutput = @"// Expected output: + // The original feature vector [1.543569,1.494266,1.284405] is + // transformed to three different tree-based feature vectors: + // Trees' output values: [-0.05652997,-0.02312196,-0.01179363]. + // Leave IDs' 0-1 representation: [0,1,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0]. + // Paths IDs' 0-1 representation: [1,0,0,0,0,1,1,0,1,0,1,1,0,0,0]. + // The original feature vector [0.764918,1.11206,0.648211] is + // transformed to three different tree-based feature vectors: + // Trees' output values: [-0.1933938,-0.1042738,-0.2312837]. + // Leave IDs' 0-1 representation: [0,0,1,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0]. + // Paths IDs' 0-1 representation: [1,1,1,0,0,1,1,0,0,0,1,0,0,0,0]. + // The original feature vector [1.251254,1.269456,1.444864] is + // transformed to three different tree-based feature vectors: + // Trees' output values: [-0.05652997,-0.06082304,-0.04528879]. + // Leave IDs' 0-1 representation: [0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0]. + // Paths IDs' 0-1 representation: [1,0,0,0,0,1,1,0,1,0,1,1,1,0,1]."; +#> \ No newline at end of file diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TreeFeaturization/PretrainedTreeEnsembleFeaturizationWithOptions.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TreeFeaturization/PretrainedTreeEnsembleFeaturizationWithOptions.cs new file mode 100644 index 0000000000..b60c74672d --- /dev/null +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TreeFeaturization/PretrainedTreeEnsembleFeaturizationWithOptions.cs @@ -0,0 +1,172 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.ML; +using Microsoft.ML.Data; +using Microsoft.ML.Trainers.FastTree; + +namespace Samples.Dynamic.Transforms.TreeFeaturization +{ + public static class PretrainedTreeEnsembleFeaturizationWithOptions + { + public static void Example() + { + // Create data set + int dataPointCount = 200; + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. + var mlContext = new MLContext(seed: 0); + + // Create a list of training data points. + var dataPoints = GenerateRandomDataPoints(dataPointCount).ToList(); + + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. + var dataView = mlContext.Data.LoadFromEnumerable(dataPoints); + + // Define input and output columns of tree-based featurizer. + string labelColumnName = nameof(DataPoint.Label); + string featureColumnName = nameof(DataPoint.Features); + string treesColumnName = nameof(TransformedDataPoint.Trees); + string leavesColumnName = nameof(TransformedDataPoint.Leaves); + string pathsColumnName = nameof(TransformedDataPoint.Paths); + + // Define a tree model whose trees will be extracted to construct a tree + // featurizer. + var trainer = mlContext.BinaryClassification.Trainers.FastTree( + new FastTreeBinaryTrainer.Options + { + NumberOfThreads = 1, + NumberOfTrees = 1, + NumberOfLeaves = 4, + MinimumExampleCountPerLeaf = 1, + FeatureColumnName = featureColumnName, + LabelColumnName = labelColumnName + }); + + // Train the defined tree model. + var model = trainer.Fit(dataView); + var predicted = model.Transform(dataView); + + // Define the configuration of tree-based featurizer. + var options = new PretrainedTreeFeaturizationEstimator.Options() + { + InputColumnName = featureColumnName, + ModelParameters = model.Model.SubModel, // Pretrained tree model. + TreesColumnName = treesColumnName, + LeavesColumnName = leavesColumnName, + PathsColumnName = pathsColumnName + }; + + // Fit the created featurizer. It doesn't perform actual training + // because a pretrained model is provided. + var treeFeaturizer = mlContext.Transforms + .FeaturizeByPretrainTreeEnsemble(options).Fit(dataView); + + // Apply TreeEnsembleFeaturizer to the input data. + var transformed = treeFeaturizer.Transform(dataView); + + // Convert IDataView object to a list. Each element in the resulted list + // corresponds to a row in the IDataView. + var transformedDataPoints = mlContext.Data.CreateEnumerable< + TransformedDataPoint>(transformed, false).ToList(); + + // Print out the transformation of the first 3 data points. + for (int i = 0; i < 3; ++i) + { + var dataPoint = dataPoints[i]; + var transformedDataPoint = transformedDataPoints[i]; + Console.WriteLine("The original feature vector [" + String.Join( + ",", dataPoint.Features) + "] is transformed to three " + + "different tree-based feature vectors:"); + + Console.WriteLine(" Trees' output values: [" + String.Join(",", + transformedDataPoint.Trees) + "]."); + + Console.WriteLine(" Leave IDs' 0-1 representation: [" + String + .Join(",", transformedDataPoint.Leaves) + "]."); + + Console.WriteLine(" Paths IDs' 0-1 representation: [" + String + .Join(",", transformedDataPoint.Paths) + "]."); + } + + // Expected output: + // The original feature vector[0.8173254, 0.7680227, 0.5581612] is + // transformed to three different tree - based feature vectors: + // Trees' output values: [0.4172185]. + // Leave IDs' 0-1 representation: [1,0,0,0]. + // Paths IDs' 0-1 representation: [1,1,1]. + // The original feature vector[0.7588848, 1.106027, 0.6421779] is + // transformed to three different tree - based feature vectors: + // Trees' output values: [-1]. + // Leave IDs' 0-1 representation: [0,0,1,0]. + // Paths IDs' 0-1 representation: [1,1,0]. + // The original feature vector[0.2737045, 0.2919063, 0.4673147] is + // transformed to three different tree - based feature vectors: + // Trees' output values: [0.4172185]. + // Leave IDs' 0-1 representation: [1,0,0,0]. + // Paths IDs' 0-1 representation: [1,1,1]. + // + // Note that the trained model contains only one tree. + // + // Node 0 + // / \ + // / Leaf -2 + // Node 1 + // / \ + // / Leaf -3 + // Node 2 + // / \ + // / Leaf -4 + // Leaf -1 + // + // Thus, if a data point reaches Leaf indexed by -1, its 0-1 path + // representation may be [1,1,1] because that data point + // went through all Node 0, Node 1, and Node 2. + + } + + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) + { + var random = new Random(seed); + float randomFloat() => (float)random.NextDouble(); + for (int i = 0; i < count; i++) + { + var label = randomFloat() > 0.5; + yield return new DataPoint + { + Label = label, + // Create random features that are correlated with the label. + // For data points with false label, the feature values are + // slightly increased by adding a constant. + Features = Enumerable.Repeat(label, 3).Select(x => x ? + randomFloat() : randomFloat() + 0.2f).ToArray() + }; + } + } + + // Example with label and 3 feature values. A data set is a collection of + // such examples. + private class DataPoint + { + public bool Label { get; set; } + [VectorType(3)] + public float[] Features { get; set; } + } + + // Class used to capture the output of tree-base featurization. + private class TransformedDataPoint : DataPoint + { + // The i-th value is the output value of the i-th decision tree. + public float[] Trees { get; set; } + // The 0-1 encoding of leaves the input feature vector falls into. + public float[] Leaves { get; set; } + // The 0-1 encoding of paths the input feature vector reaches the + // leaves. + public float[] Paths { get; set; } + } + } +} diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TreeFeaturization/RankingFeaturization.ttinclude b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TreeFeaturization/RankingFeaturization.ttinclude new file mode 100644 index 0000000000..a10b9d8b96 --- /dev/null +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TreeFeaturization/RankingFeaturization.ttinclude @@ -0,0 +1,135 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.ML; +using Microsoft.ML.Data; +<# if (TrainerOptions != null) { #> +<#=OptionsInclude#> +<# } #> + +namespace Samples.Dynamic.Transforms.TreeFeaturization +{ + public static class <#=ClassName#> + {<#=Comments#> + public static void Example() + { + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. + var mlContext = new MLContext(seed: 0); + + // Create a list of training data points. + var dataPoints = GenerateRandomDataPoints(100).ToList(); + + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. + var dataView = mlContext.Data.LoadFromEnumerable(dataPoints); +<# if (CacheData) { #> + + // ML.NET doesn't cache data set by default. Therefore, if one reads a + // data set from a file and accesses it many times, it can be slow due + // to expensive featurization and disk operations. When the considered + // data can fit into memory, a solution is to cache the data in memory. + // Caching is especially helpful when working with iterative algorithms + // which needs many data passes. + dataView = mlContext.Data.Cache(dataView); +<# } #> + + // Define input and output columns of tree-based featurizer. + string labelColumnName = nameof(DataPoint.Label); + string featureColumnName = nameof(DataPoint.Features); + string treesColumnName = nameof(TransformedDataPoint.Trees); + string leavesColumnName = nameof(TransformedDataPoint.Leaves); + string pathsColumnName = nameof(TransformedDataPoint.Paths); + + // Define the configuration of the trainer used to train a tree-based + // model. + var trainerOptions = new <#=TrainerOptions#>; + + // Define the tree-based featurizer's configuration. + var options = new <#=Options#>; + + // Define the featurizer. + var pipeline = mlContext.Transforms.<#=Trainer#>( + options); + + // Train the model. + var model = pipeline.Fit(dataView); + + // Apply the trained transformer to the considered data set. + var transformed = model.Transform(dataView); + + // Convert IDataView object to a list. Each element in the resulted list + // corresponds to a row in the IDataView. + var transformedDataPoints = mlContext.Data.CreateEnumerable< + TransformedDataPoint>(transformed, false).ToList(); + + // Print out the transformation of the first 3 data points. + for (int i = 0; i < 3; ++i) + { + var dataPoint = dataPoints[i]; + var transformedDataPoint = transformedDataPoints[i]; + Console.WriteLine("The original feature vector [" + String.Join(",", + dataPoint.Features) + "] is transformed to three different " + + "tree-based feature vectors:"); + + Console.WriteLine(" Trees' output values: [" + String.Join(",", + transformedDataPoint.Trees) + "]."); + + Console.WriteLine(" Leave IDs' 0-1 representation: [" + String + .Join(",", transformedDataPoint.Leaves) + "]."); + + Console.WriteLine(" Paths IDs' 0-1 representation: [" + String + .Join(",", transformedDataPoint.Paths) + "]."); + } + + <#=ExpectedOutput#> + } + + private static IEnumerable GenerateRandomDataPoints(int count, + int seed = 0, int groupSize = 10) + { + var random = new Random(seed); + float randomFloat() => (float)random.NextDouble(); + for (int i = 0; i < count; i++) + { + var label = random.Next(0, 5); + yield return new DataPoint + { + Label = (uint)label, + GroupId = (uint)(i / groupSize), + // Create random features that are correlated with the label. + // For data points with larger labels, the feature values are + // slightly increased by adding a constant. + Features = Enumerable.Repeat(label, 3).Select(x => randomFloat() + + x * 0.1f).ToArray() + }; + } + } + + // Example with label, groupId, and 3 feature values. A data set is a + // collection of such examples. + private class DataPoint + { + [KeyType(5)] + public uint Label { get; set; } + [KeyType(100)] + public uint GroupId { get; set; } + [VectorType(3)] + public float[] Features { get; set; } + } + + // Class used to capture the output of tree-base featurization. + private class TransformedDataPoint : DataPoint + { + // The i-th value is the output value of the i-th decision tree. + public float[] Trees { get; set; } + // The 0-1 encoding of leaves the input feature vector falls into. + public float[] Leaves { get; set; } + // The 0-1 encoding of paths the input feature vector reaches the + // leaves. + public float[] Paths { get; set; } + } + } +} \ No newline at end of file diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TreeFeaturization/RegressionFeaturization.ttinclude b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TreeFeaturization/RegressionFeaturization.ttinclude new file mode 100644 index 0000000000..cb684b8387 --- /dev/null +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TreeFeaturization/RegressionFeaturization.ttinclude @@ -0,0 +1,132 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.ML; +using Microsoft.ML.Data; +<# if (ExtraUsing != null) { #> +<#=ExtraUsing#> +<# } #> + +namespace Samples.Dynamic.Transforms.TreeFeaturization +{ + public static class <#=ClassName#> + { +<# if (ClassHeader != null) { #> + <#=ClassHeader#> +<# } #> + public static void Example() + { + // Create a new context for ML.NET operations. It can be used for + // exception tracking and logging, as a catalog of available operations + // and as the source of randomness. Setting the seed to a fixed number + // in this example to make outputs deterministic. + var mlContext = new MLContext(seed: 0); + + // Create a list of training data points. + var dataPoints = GenerateRandomDataPoints(100).ToList(); + + // Convert the list of data points to an IDataView object, which is + // consumable by ML.NET API. + var dataView = mlContext.Data.LoadFromEnumerable(dataPoints); +<# if (CacheData) { #> + + // ML.NET doesn't cache data set by default. Therefore, if one reads a + // data set from a file and accesses it many times, it can be slow due + // to expensive featurization and disk operations. When the considered + // data can fit into memory, a solution is to cache the data in memory. + // Caching is especially helpful when working with iterative algorithms + // which needs many data passes. + dataView = mlContext.Data.Cache(dataView); +<# } #> + + // Define input and output columns of tree-based featurizer. + string labelColumnName = nameof(DataPoint.Label); + string featureColumnName = nameof(DataPoint.Features); + string treesColumnName = nameof(TransformedDataPoint.Trees); + string leavesColumnName = nameof(TransformedDataPoint.Leaves); + string pathsColumnName = nameof(TransformedDataPoint.Paths); + + // Define the configuration of the trainer used to train a tree-based + // model. + var trainerOptions = new <#=TrainerOptions#>; + + // Define the tree-based featurizer's configuration. + var options = new <#=Options#>; + + // Define the featurizer. + var pipeline = mlContext.Transforms.<#=Trainer#>( + options); + + // Train the model. + var model = pipeline.Fit(dataView); + + // Create testing data. Use different random seed to make it different + // from training data. + var transformed = model.Transform(dataView); + + // Convert IDataView object to a list. Each element in the resulted list + // corresponds to a row in the IDataView. + var transformedDataPoints = mlContext.Data.CreateEnumerable< + TransformedDataPoint>(transformed, false).ToList(); + + // Print out the transformation of the first 3 data points. + for (int i = 0; i < 3; ++i) + { + var dataPoint = dataPoints[i]; + var transformedDataPoint = transformedDataPoints[i]; + Console.WriteLine("The original feature vector [" + String.Join(",", + dataPoint.Features) + "] is transformed to three different " + + "tree-based feature vectors:"); + + Console.WriteLine(" Trees' output values: [" + String.Join(",", + transformedDataPoint.Trees) + "]."); + + Console.WriteLine(" Leave IDs' 0-1 representation: [" + String + .Join(",", transformedDataPoint.Leaves) + "]."); + + Console.WriteLine(" Paths IDs' 0-1 representation: [" + String + .Join(",", transformedDataPoint.Paths) + "]."); + } + + <#=ExpectedOutput#> + } + + private static IEnumerable GenerateRandomDataPoints(int count, + int seed=0) + { + var random = new Random(seed); + for (int i = 0; i < count; i++) + { + float label = (float)random.NextDouble(); + yield return new DataPoint + { + Label = label, + // Create random features that are correlated with the label. + Features = Enumerable.Repeat(label, 3).Select(x => x + + (float)random.NextDouble()).ToArray() + }; + } + } + + // Example with label and 50 feature values. A data set is a collection of + // such examples. + private class DataPoint + { + public float Label { get; set; } + [VectorType(3)] + public float[] Features { get; set; } + } + + // Class used to capture the output of tree-base featurization. + private class TransformedDataPoint : DataPoint + { + // The i-th value is the output value of the i-th decision tree. + public float[] Trees { get; set; } + // The 0-1 encoding of leaves the input feature vector falls into. + public float[] Leaves { get; set; } + // The 0-1 encoding of paths the input feature vector reaches the + // leaves. + public float[] Paths { get; set; } + } + } +} \ No newline at end of file diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/WithOnFitDelegate.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/WithOnFitDelegate.cs index fcb56653c5..72c541b8b3 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/WithOnFitDelegate.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/WithOnFitDelegate.cs @@ -13,27 +13,40 @@ public class WithOnFitDelegate { public static void Example() { - // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, - // as well as the source of randomness. + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. var mlContext = new MLContext(); var samples = new List() { - new DataPoint(){ Features = new float[4] { 8, 1, 3, 0}, Label = true }, - new DataPoint(){ Features = new float[4] { 6, 2, 2, 0}, Label = true }, - new DataPoint(){ Features = new float[4] { 4, 0, 1, 0}, Label = false }, - new DataPoint(){ Features = new float[4] { 2,-1,-1, 1}, Label = false } + new DataPoint(){ Features = new float[4] { 8, 1, 3, 0}, + Label = true }, + + new DataPoint(){ Features = new float[4] { 6, 2, 2, 0}, + Label = true }, + + new DataPoint(){ Features = new float[4] { 4, 0, 1, 0}, + Label = false }, + + new DataPoint(){ Features = new float[4] { 2,-1,-1, 1}, + Label = false } + }; - // Convert training data to IDataView, the general data type used in ML.NET. + // Convert training data to IDataView, the general data type used in + // ML.NET. var data = mlContext.Data.LoadFromEnumerable(samples); - // Create a pipeline to normalize the features and train a binary classifier. - // We use WithOnFitDelegate for the intermediate binning normalization step, - // so that we can inspect the properties of the normalizer after fitting. + // Create a pipeline to normalize the features and train a binary + // classifier. We use WithOnFitDelegate for the intermediate binning + // normalization step, so that we can inspect the properties of the + // normalizer after fitting. NormalizingTransformer binningTransformer = null; var pipeline = - mlContext.Transforms.NormalizeBinning("Features", maximumBinCount: 3) - .WithOnFitDelegate(fittedTransformer => binningTransformer = fittedTransformer) - .Append(mlContext.BinaryClassification.Trainers.LbfgsLogisticRegression()); + mlContext.Transforms + .NormalizeBinning("Features", maximumBinCount: 3) + .WithOnFitDelegate( + fittedTransformer => binningTransformer = fittedTransformer) + .Append(mlContext.BinaryClassification.Trainers + .LbfgsLogisticRegression()); Console.WriteLine(binningTransformer == null); // Expected Output: @@ -56,6 +69,7 @@ public static void Example() Console.WriteLine( $"Bin {i}: Density = {binningParam.Density[i]}, " + $"Upper-bounds = {upperBounds}"); + } // Expected output: // Bin 0: Density = 2, Upper-bounds = 3, 7, Infinity diff --git a/docs/samples/Microsoft.ML.Samples/Microsoft.ML.Samples.csproj b/docs/samples/Microsoft.ML.Samples/Microsoft.ML.Samples.csproj index 61d20ab033..2de093e8e0 100644 --- a/docs/samples/Microsoft.ML.Samples/Microsoft.ML.Samples.csproj +++ b/docs/samples/Microsoft.ML.Samples/Microsoft.ML.Samples.csproj @@ -449,6 +449,30 @@ TextTemplatingFileGenerator SdcaWithOptions.cs + + TextTemplatingFileGenerator + FastForestBinaryFeaturizationWithOptions.cs + + + TextTemplatingFileGenerator + FastForestRegressionFeaturizationWithOptions.cs + + + TextTemplatingFileGenerator + FastTreeBinaryFeaturizationWithOptions.cs + + + TextTemplatingFileGenerator + FastTreeRankingFeaturizationWithOptions.cs + + + TextTemplatingFileGenerator + FastTreeRegressionFeaturizationWithOptions.cs + + + TextTemplatingFileGenerator + FastTreeTweedieFeaturizationWithOptions.cs + @@ -896,6 +920,36 @@ True SdcaWithOptions.tt + + True + True + FastForestBinaryFeaturizationWithOptions.tt + + + FastForestRegressionFeaturizationWithOptions.tt + True + True + + + FastTreeBinaryFeaturizationWithOptions.tt + True + True + + + True + True + FastTreeRankingFeaturizationWithOptions.tt + + + True + True + FastTreeRegressionFeaturizationWithOptions.tt + + + True + True + FastTreeTweedieFeaturizationWithOptions.tt + diff --git a/docs/samples/Microsoft.ML.Samples/Program.cs b/docs/samples/Microsoft.ML.Samples/Program.cs index eb49938927..4c46399421 100644 --- a/docs/samples/Microsoft.ML.Samples/Program.cs +++ b/docs/samples/Microsoft.ML.Samples/Program.cs @@ -1,5 +1,6 @@ -using System; +using System; using System.Reflection; +using Samples.Dynamic; namespace Microsoft.ML.Samples { diff --git a/docs/specs/mlnet-database-loader/mlnet-database-loader-specs.md b/docs/specs/mlnet-database-loader/mlnet-database-loader-specs.md new file mode 100644 index 0000000000..cf9d3e5774 --- /dev/null +++ b/docs/specs/mlnet-database-loader/mlnet-database-loader-specs.md @@ -0,0 +1,467 @@ +# Specs for ML.NET Relational Database Loader + +This specs-doc focuses on the features needed for the base **ML.NET API**, most of all. +The scenarios related to ML.NET **AutoML API**, the **CLI** and **VS Model Builder** will also be considered and covered in this document by in a significantly less detail since there should be different spec docs for those additional tools and APIs. + +# Problem to solve + +Customers (.NET developers in the enterprise, ISVs, etc.) have tolds us through multiple channels that one of their main source of data, including data for datasets to be used in machine learning, comes directly from databases. This is a very important difference between .NET developers in the enterprise and data scientists who are used to work with datasets as text files. + +ML.NET 1.0 and 1.1 only supports the [IDataView LoadFromEnumerable()](https://docs.microsoft.com/en-us/dotnet/api/microsoft.ml.dataoperationscatalog.loadfromenumerable?view=ml-dotnet-1.1.0) method in order to load data into an IDataView from any source that produces an IEnumerable structure, typically a database. However, the code for loading that data from a database (i.e. using Entity Framework or symply System.Data) is responsability of the user/developer. When training with very large tables in databases so training time could be many hours, such code can be complicated in certain cases. + +Within the 'databases scope' problem there are multiple areas. + +The **scope** for this feature is initially limited to **relational databases** with higher priority on SQL Server and Azure SQL Database, but one of the goals is to make this loader/connector compatible with any relational database which is supported by .NET providers. + +- Scope to support in this feature: + - Relational Databases, such as: + - SQL Server and Azure SQL Database + - Other relational databases such as MySQL, Oracle, PostgreSQL, etc. + +- Scope not supported in this feature: + - No-SQL Databases, such as: + - Azure Cosmos DB + - MongoDB + - Any other non-SQL database + - Big Data, such as: + - Big Data: Spark, Hadoop, etc. + + +## Evidence + +Sample feedback and quotes: + +- *"[Rachel Poulsen comment](https://www.quora.com/What-are-the-most-valuable-skills-to-learn-for-a-data-scientist-now), Data Scientist at Silicon Valley Data Science: I have been a Data Scientist in Silicon Valley for the last 4 years. There are A LOT of tools out there for data science as I'm sure you are aware. But very importantly, SQL: I'd argue this is 100% necessary for all data scientists, regardless of whether you are using structured or unstructured data. Too many people and companies still use it and are comfortable with it for it to be thrown out with NoSQL techniques. Even companies using NoSQL databases and unstructured data are still using SQL."* + +- *"forki commented on May 17, 2018 +I did not say EF ;-) - I heard some of the authors of this package worked on EF before, but please please don't bind it to EF, or sql server or any other specific tech. Or course you want to get data into the learning system without going over to csv. But the data is usually already flattened anyway, so no need for a complex OR mapping tool - especially since the target data structure is Immutable"* + +- *Other customers - TBD* + +# Who is the Customer + +- .NET developers used to work with databases who are getting started with machine learning with ML.NET. + + +# Goals + +The business goals are the following, depending on the possible scenarios: + +- Ability for developers to load and automatically stream data from relational databases in order to train/evaluate ML.NET models. +- The code to load from a database should be extremely easy, a single line of code in most cases. +- Tooling (Model Builder in VS and the CLI) and AutoML API should also support this feature. + +# Solution + +The solution is to create an ML.NET database loader classes supporting the above scenarios. + +The main supported features are: + +- **Easily load data into IDataView from databases**: The main goal is to load data into IDataView from databases very easily (a single line of code as one of the approaches) and properly (with good performance when streaming data while training). + +- **Train ML.NET models with data streaming coming from a database**: As the ultimate goal, the main purpose of loading data into an IDataView is to usually train an ML.NET model with data coming from a database table (or view or SQL sentence). + +- **Evaluate ML.NET models with data streaming coming from a database**: As secondary goal is to evaluate the quality/accuracy of an ML.NET model with data coming from a database table (or view or SQL sentence) by comparing it with a set or predictions. This goal should support: + + - Train/test split scenario (one database source as training dataset and one database source as testing-dataset), such as a 80%-20% approach. + + - Train/evaluation/test split scenario (one database source as training dataset, one database source as evaluation-dataset and one database source as test-dataset), such as a 50%-30%-20% approach. + + - Cross-validation scenario. Single database source. Internally it'll be split in multiple folds (such as 5 folds) for multiple trains and tests. This should be transparent from a database connection point of view which only needs one database source. + +- **Additional support for AutoML API, CLI and Model Builder:** Loading data from databases should be supported by AutoML API, Model Builder in VS and the ML.NET CLI. + +-------------------------------------- + +TBD - Include diagram with blocks of AutoML, CLI and Model Buidler using this component/loader + +-------------------------------------- + + +## Design criteria + +- **Supported frameworks in .NET**: The DatabaseLoader component should be supported for the [frameworks supported by ML.NET](https://github.com/dotnet/machinelearning#installation) which include applications running on: + + - .NET Core 2.1 and higher + - .NET Framework 4.6.1 and higher + +The way to support those frameworks would be by creating a **.NET Standard 2.0 library** as the base for this package since .NET Standard is supported by both frameworks. + +- **Supported RDBMS**: The DatabaseLoader component should be supported for most of the [data providers supported by System.Data.Common in .NET Standard 2.0](https://docs.microsoft.com/en-us/dotnet/framework/data/adonet/data-providers) which special focus/support on: + + - **P1 RDBMS support/tested priorities:** + + The following data providers should be tested with higher priority: + + - Data Provider for SQL Server (Microsoft provider) + - Data Provider for Oracle - Test on: + - [Oracle Data Provider for .NET (ODP.NET)](https://www.oracle.com/technetwork/developer-tools/visual-studio/downloads/index-085163.html) + - MySql provider - Test on: + - [MySQL Connector/NET provider](https://dev.mysql.com/doc/connector-net/en/connector-net-versions.html) / [MySql.Data NuGet](https://www.nuget.org/packages/MySql.Data/) + - PostgreSQL providers - Test on: + - Npgsql open source ADO.NET Data Provider for PostgreSQL + + This ML.NET database loader won't probably need Entity Framework, but for a relationship, see [EF providers](https://docs.microsoft.com/en-us/ef/core/providers/) for a relationship to ADO.NET providers. + + - **P2 RDBMS support/tested priorities:** + + Other data providers to test with lower priority. It will also need to select specific RDBMS on OLE DB and ODBC: + + - Data Provider for OLE DB + - Data Provider for ODBC + - Data Provider for EntityClient Provider (Entity Data Model (EDM)) + +- **CRITICAL: Implement support for 'handle and continue' after transient errors happening in Azure SQL Database (or any DB):** When using Azure SQL Database as the source of your training database, because databases in Azure SQL DB can be moved to different servers across the internal Azure SQL Database cluster, transient failures (usually for just a few seconds) in the form of connectivity exceptions can happen. Even further, by design in Azure SQL Database, if a process is blocking too many resources in SQL, sometimes the database connection can be thrown away in favor of other customers/databases. +There are several strategies in order to handle database transient errors (see [Working with SQL Database connection issues and transient errors](https://docs.microsoft.com/en-us/azure/sql-database/sql-database-connectivity-issues)) like doing a 'Retry strategy' and start with a new connection again. But that strategy is only okay for short/fast queries. That simple strategy which throws away all the progress made and start the same query again wouldn't be good when training with a very large table because it could mean that the training operation "never finishes" if you have at least one transient error on every "training try". +We'll need to come up with a reasonably general pattern (probably something that reasons about primary keys), but this scenario is not simple. + + See [related issue](https://github.com/dotnet/machinelearning-samples/issues/507) + +- **Support for remote/network connection and internal connection within the RDBMS server**: The Database loader should not only support remote/network connection to the RDBMS server but also support from C# running within the RDBMS server such as [SQL Server CLR integration](https://docs.microsoft.com/en-us/sql/relational-databases/clr-integration/clr-integration-overview?view=sql-server-2017), [Oracle Database Extensions for .NET](https://docs.oracle.com/cd/B19306_01/win.102/b14306/intro.htm), etc. Scenarios: + + 1. Training-machine-ML.NET-code <--network--> Remote database-in-RDBMS-server + + 2. RDBMS-server running database and .NET code using ML.NET code + +- **NuGet packages and libraries design**: +The implementation of this feature should be packaged following the following approach, which is aligned and consistent to the current approach used by the .NET Framework and .NET Core in the System.Data.Common and System.Data.SqlClient: + + - Implementation code with NO depedencies to specific database providers (such as SQL Server, Oracle, MySQL, etc.) will be packaged in the same NuGet package and library than the existing TextLoader-related classes which is in the Microsoft.ML.Data library. This code is basically the foundational API for the Database loader where the user has to provide any specific database connection (so dependencies are taken in user's code). + - Implementation code WITH dependencies to data proviers (such as SQL Server, Oracle, MySQL, etc.) that might be created when creating additional convenient APIs where the user only needs to provide a connection string and table-name or SQL statement, will be placed in a segregated class library and NuGet package, so that ML.NET core packages don't depend on specific database providers. + +- **Support for sparse data**: The database loader should support sparse data, at least up to the maximum number of columns in SQL Server (1,024 columns per nonwide table, 30,000 columns per wide table or 4,096 columns per SELECT statement). + + ML.NET supports sparse data such as in the following example using a [sparse matrix](https://en.wikipedia.org/wiki/Sparse_matrix) of thousands or even millions of columns even when in this example only 200 columns have real data (sparse data): + + - [ML.NET sample using millions of columns with sparse data](https://github.com/dotnet/machinelearning-samples/tree/master/samples/csharp/getting-started/LargeDatasets) + + SQL Server supports [Sparse columns](https://docs.microsoft.com/en-us/sql/relational-databases/tables/use-sparse-columns?view=sql-server-2017), however, it is just a way to optimize storage for null values. It still needs to have a real column created in the table per each logical column (i.e. 1,000 columns defined in the SQL table) even when it might not have data. + + + + + + +# Gated conditions for first Preview release + +- TBD + +# Gated conditions for GA release + +- TBD + +# API design examples + +The following are a few C# examples on how this API could be used by a developer when loading data from a database into an IDataView. + +Whenever in the samples below it is using a `SqlConnection` type, it could also be using any other supported DbConnection in .NET providers, such as: + +- `SqlConnection` (SQL Server) +- `NpgsqlConnection` (PostgreSQL) +- `OracleConnection` (Oracle) +- `MySqlConnection` (MySql) +- etc. + +The specific type (`SqlConnection`, `NpgsqlConnection`, `OracleConnection`, etc.) is specified as a generic parameter so references to the needed NuGet packages will be provided by the user when trying to compile. + +**1. (Convenient 'must-have' method) Data loading from a database by specifying a SQL query sentence:** + +The signature of the `DataOperationsCatalog.LoadFromDbSqlQuery` method looks like: + +```C# +public IDataView LoadFromDbSqlQuery(string connString, string sqlQuerySentence) where T : class; +``` + +`TRow` is the model's input data class (Observation class) to be used by IDataView and ML.NET API. + +Example code using it: + +```C# +MLContext mlContext = new MLContext(); + +//Example loading from a SQL Server or SQL Azure database with a SQL query sentence +IDataView trainingDataView = mlContext.Data.LoadFromDbSqlQuery(connString: myConnString, sqlQuerySentence: "Select * from InputMLModelDataset where InputMLModelDataset.CompanyName = 'MSFT'"); +``` + +**2. (Foundational method) Data loading from a database with an IDataReader object:** + +This is the foundational or pillar method which will be used by the rest of the higher level or convenient methods: + +The signature of the `DataOperationsCatalog.LoadFromDbDataReader` method looks like: + +```C# +public IDataView LoadFromDbDataReader(Func executeReader) where TRow : class; +``` + +`TRow` is the model's input data class (Observation class) to be used by IDataView and ML.NET API. + +The following code is how the mlContext.Data.LoadFromDbDataReader() method is used: + +```C# +MLContext mlContext = new MLContext(); + +string conString = YOUR_CONNECTION_STRING; +using (SqlConnection connection = new SqlConnection(conString)) +{ + connection.Open(); + using (SqlCommand command = new SqlCommand("Select * from InputMLModelDataset", connection)) + { + mlContext.Data.LoadFromDbDataReader(() => command.ExecuteReader()); + } +} +``` + +**IMPORTANT - Lambda with command.ExecuteReader() as parameter instead of a IDataReader as paramater:** It is important to highlight that because we want to enforce that each query will use a different data reader object (in order to avoid multi-threading issues), the `mlContext.Data.LoadFromDbDataReader()` method is accepting a lambda expression with the `command.ExecuteReader()` line of code. That means the method only accepts a `Func` delegate because the lambda expression has no input parameters and returns an IDataReader object (such as `SqlDataReader`). + + +**3. ('Nice to have') Data loading from a database table (Simplest approach):** + +This is mostly a 'sugar method', but can be a 'nice to have feature' for users. +The signature of the `DataOperationsCatalog.LoadFromDbTable` method looks like: + +```C# +public IDataView LoadFromDbTable(string connString, string tableName) where T : class; +``` + +Example code using it: + +```C# +MLContext mlContext = new MLContext(); + +//Example loading from a SQL Server or SQL Azure database table +IDataView trainingDataView = mlContext.Data.LoadFromDbTable(connString: myConnString, + tableName: "TrainingDataTable"); +``` + +**4. ('Nice to have') Data loading from a database view:** + +This is mostly a 'sugar method', but can be a 'nice to have feature' for users. +The signature of the `DataOperationsCatalog.LoadFromDbView` method looks like: + +```C# +public IDataView LoadFromDbView(string connString, string viewName) where T : class; +``` + +Example code using it: + +```C# +MLContext mlContext = new MLContext(); + +//Example loading from a SQL Server or SQL Azure database view +IDataView trainingDataView = mlContext.Data.LoadFromDbView(connString: myConnString, + viewName: "TrainingDatabaseView"); +``` + +## Support connectivity from .NET assemblies embedded into the RDBMS server + +As introduced, the database loader should not only support remote/network connection to the RDBMS server but also support connectivity from C# running within the RDBMS server such as [SQL Server CLR integration](https://docs.microsoft.com/en-us/sql/relational-databases/clr-integration/clr-integration-overview?view=sql-server-2017) (aka [SQL CLR](https://en.wikipedia.org/wiki/SQL_CLR)), [Oracle Database Extensions for .NET](https://docs.oracle.com/cd/B19306_01/win.102/b14306/intro.htm), etc. + +The only difference is the way you define the connection string, which simply provides **'context' string** (instead of server name, user, etc. when using the network), such as: + +- Code example running on [SQL Server CLR integration](https://docs.microsoft.com/en-us/sql/relational-databases/clr-integration/clr-integration-overview?view=sql-server-2017) + + ``` + //SQL Server + SqlConnection conn = new SqlConnection("context connection=true") + ``` + + - See here an [example of a CLR Scalar-Valued Function](https://docs.microsoft.com/en-us/sql/relational-databases/clr-integration-database-objects-user-defined-functions/clr-scalar-valued-functions?view=sql-server-2017#example-of-a-clr-scalar-valued-function) + + - [Introduction to SQL Server CLR Integration](https://docs.microsoft.com/en-us/dotnet/framework/data/adonet/sql/introduction-to-sql-server-clr-integration) + +- Code example running on [Oracle Database Extensions for .NET](https://docs.oracle.com/cd/B19306_01/win.102/b14306/intro.htm) + ``` + //Oracle + OracleConnection con = new OracleConnection(); + con.ConnectionString = "context connection=true"; + ``` + + - See here an [exampleof a C# stored procedure in Oracle ](https://www.oracle.com/technetwork/articles/dotnet/williams-sps-089817.html?printOnly=1) + +The code should be similar on any other RDBMS supporting .NET assemblies running within the database engine. The only change should be provided within the connection string. + +**Out of scope:** + +ML.NET won't implement components creating concrete database objects such as **CLR Scalar-Valued Functions** or **CLR/C# stored procedures** (which have different implementation for SQL Server, Oracle, etc.). Those components should be created by the user/developer using C# then adding ML.NET API code in order to train a model. + +Also, note that the fact that ML.NET will be supported to be used within user components using CLR integration, that doesn't mean that the user can do it on any RDBMS. There are RDBMS such as Azure SQL Database with single databases and elastic pools and other RDBMS that don't support that feature. Other RDBMS suchas SQL Server on-premises or Azure SQL Database Managed Instances, Oracle, etc. do support it. + +For instance: + +- [Feature comparison: Azure SQL Database versus SQL Server](https://docs.microsoft.com/en-us/azure/sql-database/sql-database-features) + +# Input data classes/types design + +There can be two different approaches here: + +- Use similar input data classes/types to ML.NET 1.x input data classes +- Use similar input data classes/types to Entity Framework POCO entity data model classes + +## Approach A: Using ML.NET input data classes + +A sample ML.NET input data class are the following: + +**A.1 - Simple input data class:** + +```C# +using Microsoft.ML.Data; +//... +private class ModelInputData +{ + [LoadColumn(0)] + public bool SentimentLabel { get; set; } + [LoadColumn(1)] + public string SentimentText { get; set; } +} +``` + +**A.2 - Large number of columns loaded at the same time into a single vector column:** + +When you have hundreds or even thousands of columns, those columns should be grouped in the data model class by using "vector valued columns" which are arrays of types (such as float[]) for the contiguous columns with the same type, as the following code example: + +```C# +using Microsoft.ML.Data; +//... +private class ModelInputData +{ + [LoadColumn(0)] + public float TargetLabel { get; set;} + + [LoadColumn(1, 2000), ColumnName("Features")] + [VectorType(2000)] + public float[] FeatureVector { get; set;} +} +``` + +Pros: +- It uses the same consistent model input classes already used in ML.NET when loading from dataset files +- It supports loading many columns at the same time into a single vector column + +Cons: +- It needs special attributes such as `LoadColumnAttribute`, therefore, these input classes are not POCO classes since they have a dependency on ML.NET packages. + +**A.3 - Support column-map 'by property name' convention:** + +When loading from a database, it has to be able to use a property-name based convention, meaning that if the class is not providing the `LoadColumnAttribute` then it would try to map the column name in the database table to the property's name, such as the following case: + +```C# +private class ModelInputData +{ + public bool SentimentLabel { get; set; } + public string SentimentText { get; set; } +} +``` + +This last approach is similar to the Entity Framework POCO entity class approach, however, Entity Framework POCO entity classes support an additional large number of .NET types including complex-types and navigation/embedded types. + +*In a different page and related to dataset files instead of databases, that feature would also be useful if using dataset files with a header with column names matching property names.* + +## Approach B: Using Entity Framework POCO entity data model classes + +When using Entity Framework, a POCO entity is a class that doesn't depend on any framework-specific base class. This is also why they are persistence-ignorant objects following the [persistence ignorance principle](https://deviq.com/persistence-ignorance/). + +It is like any other normal .NET CLR class, which is why it is called POCO ("Plain Old CLR Object"). + +POCO entities are supported in both EF 6.x and EF Core. + +```C# +public class ModelInputData +{ + public int StudentID { get; set; } + public string StudentName { get; set; } + public DateTime? DateOfBirth { get; set; } + public byte[] Photo { get; set; } + public decimal Height { get; set; } + public float Weight { get; set; } + + public StudentAddress StudentAddress { get; set; } + public Grade Grade { get; set; } +} +``` + +Pros: +- It uses the same data model classes used in Entity Framework entity models, familiar to many .NET developers. +- It doesn't need special attributes with dependency on ML.NET packages since they are POCO classes. + +Cons: +- EF does not support loading many columns at the same time into a single vector column. +- EF requires a mandatory ID property in the POCO class +- ML.NET might not support certain .NET types allowed by EF POCO classes (i.e. DateTime, etc.). +- ML.NET doesn't support embedded/navigation/relationship entity types such as `StudentAddress` in the sample above, neither complex-types in EF. +- Input data classes won't be consistent/similar to ML.NET input data classes when using dataset files. + +### Selected approach for input data class when reading from a database + +*TO BE DISCUSSED/CONFIRMED:* + +Due to the mentioned pros/cons above and additional constraints in ML.NET supported types, the most feasible approach in the short term is **to use ML.NET input data classes with/without ML.NET attributes**, consistent with current support in ML.NET, as ML.NET input data class when reading from a database. + +Supporting the same scope of POCO entities supported by entity Framework seems problematic due to many more additional types supported plus embeded navigation types/classes (complex types), etc. + + +# ML.NET CLI design samples + +**1. CLI training from a database table (Simplest approach):** + +Sample CLI command: + +``` +> mlnet auto-train --task regression --db-conn-string "YOUR-DATABASE-CONNECTION-STRING" --db-table "MyTrainingDbTable" --label-column-name Price +``` + +**2. CLI training from a database view:** + +Sample CLI command: + +``` +> mlnet auto-train --task regression --db-conn-string "YOUR-DATABASE-CONNECTION-STRING" --db-view "MyTrainingDbView" --label-column-name Price +``` + +**3. CLI training from a database with a SQL query sentence:** + +Sample CLI command: + +``` +> mlnet auto-train --task regression --db-conn-string "YOUR-DATABASE-CONNECTION-STRING" --sql-query "SELECT * FROM MyTrainingDbTable WHERE Company = 'MSFT'" --label-column-name Price +``` + + +# ML.NET AutoML API design samples + +For ML.NET AutoML the C# code to use is the same than for regular ML.NET code since the code to load data infor an IDataView should use the same API, such as the following example: + +```C# +MLContext mlContext = new MLContext(); + +//Load train dataset from a database table +IDataView trainDataView = mlContext.Data.LoadFromDatabaseTable(connString: myConnString, tableName: "MyTrainDataTable"); + +//Load train dataset from a database table +IDataView testDataView = mlContext.Data.LoadFromDatabaseTable(connString: myConnString, tableName: "MyTestDataTable"); + +// Run AutoML experiment +var progressHandler = new BinaryExperimentProgressHandler(); + +ExperimentResult experimentResult = mlContext.Auto() + .CreateBinaryClassificationExperiment(ExperimentTime) + .Execute(trainingDataView, progressHandler: progressHandler); +``` + +Therefore, most of the code above is regular AutoML API code and the only pieces of code using the DatabaseLoader are using the same API than when using regular ML.NET code for loading data from a database. + +# Model Builder for Visual Studio mock UI samples + +TBD + + + + +# Open questions + +- QUESTION 1 TBD: + + +# References + +- [TBD](http://TBD) + diff --git a/pkg/Microsoft.ML.OnnxTransformer/Microsoft.ML.OnnxTransformer.nupkgproj b/pkg/Microsoft.ML.OnnxTransformer/Microsoft.ML.OnnxTransformer.nupkgproj index b817e809d1..c924ef4aba 100644 --- a/pkg/Microsoft.ML.OnnxTransformer/Microsoft.ML.OnnxTransformer.nupkgproj +++ b/pkg/Microsoft.ML.OnnxTransformer/Microsoft.ML.OnnxTransformer.nupkgproj @@ -7,6 +7,7 @@ + diff --git a/src/Microsoft.ML.Core/Utilities/Utils.cs b/src/Microsoft.ML.Core/Utilities/Utils.cs index 3f9f17c5dc..fbdb3a791d 100644 --- a/src/Microsoft.ML.Core/Utilities/Utils.cs +++ b/src/Microsoft.ML.Core/Utilities/Utils.cs @@ -841,7 +841,7 @@ public static void BuildSubsetMaps(int lim, IEnumerable c // REVIEW: Better names? List mapList = new List(); - invMap = invMap = Enumerable.Repeat(-1, lim).ToArray(); + invMap = Enumerable.Repeat(-1, lim).ToArray(); foreach (var col in columnsNeeded) { diff --git a/src/Microsoft.ML.Data/Data/SchemaDefinition.cs b/src/Microsoft.ML.Data/Data/SchemaDefinition.cs index 157d35d940..b26cd9554c 100644 --- a/src/Microsoft.ML.Data/Data/SchemaDefinition.cs +++ b/src/Microsoft.ML.Data/Data/SchemaDefinition.cs @@ -28,7 +28,7 @@ public sealed class KeyTypeAttribute : Attribute /// public KeyTypeAttribute() { - + throw Contracts.ExceptNotSupp("Using KeyType without the Count parameter is not supported"); } /// diff --git a/src/Microsoft.ML.Data/Transforms/ColumnCopying.cs b/src/Microsoft.ML.Data/Transforms/ColumnCopying.cs index bc681c5dbb..bd08e9c211 100644 --- a/src/Microsoft.ML.Data/Transforms/ColumnCopying.cs +++ b/src/Microsoft.ML.Data/Transforms/ColumnCopying.cs @@ -49,7 +49,6 @@ namespace Microsoft.ML.Transforms /// ]]> /// /// - /// /// public sealed class ColumnCopyingEstimator : TrivialEstimator { diff --git a/src/Microsoft.ML.Data/Transforms/NormalizeColumnSng.cs b/src/Microsoft.ML.Data/Transforms/NormalizeColumnSng.cs index ce545308dc..0685034dd2 100644 --- a/src/Microsoft.ML.Data/Transforms/NormalizeColumnSng.cs +++ b/src/Microsoft.ML.Data/Transforms/NormalizeColumnSng.cs @@ -1557,7 +1557,7 @@ public static IColumnFunctionBuilder Create(NormalizingEstimator.LogMeanVariance { var lim = column.MaximumExampleCount; host.CheckUserArg(lim > 1, nameof(column.MaximumExampleCount), "Must be greater than 1"); - return new MeanVarOneColumnFunctionBuilder(host, lim, false, getter, true, column.UseCdf); + return new MeanVarOneColumnFunctionBuilder(host, lim, column.EnsureZeroUntouched, getter, true, column.UseCdf); } protected override bool ProcessValue(in TFloat origVal) @@ -1633,7 +1633,7 @@ public static IColumnFunctionBuilder Create(NormalizingEstimator.LogMeanVariance var lim = column.MaximumExampleCount; host.CheckUserArg(lim > 1, nameof(column.MaximumExampleCount), "Must be greater than 1"); var cv = srcType.Size; - return new MeanVarVecColumnFunctionBuilder(host, cv, lim, false, getter, true, column.UseCdf); + return new MeanVarVecColumnFunctionBuilder(host, cv, lim, column.EnsureZeroUntouched, getter, true, column.UseCdf); } protected override bool ProcessValue(in VBuffer buffer) diff --git a/src/Microsoft.ML.Data/Transforms/Normalizer.cs b/src/Microsoft.ML.Data/Transforms/Normalizer.cs index 7f2776b5a6..e33ea8744f 100644 --- a/src/Microsoft.ML.Data/Transforms/Normalizer.cs +++ b/src/Microsoft.ML.Data/Transforms/Normalizer.cs @@ -42,7 +42,7 @@ namespace Microsoft.ML.Transforms /// The resulting NormalizingEstimator will normalize the data in one of the following ways based upon how it was created: /// * Min Max - A linear rescale that is based upon the minimum and maximum values for each row. /// * Mean Variance - Rescale each row to unit variance and, optionally, zero mean. - /// * Log Mean Variance - Rescale each row to unit variance based on a log scale. + /// * Log Mean Variance - Rescale each row to unit variance, optionally, zero mean based on computations in log scale. /// * Binning - Bucketizes the data in each row and performs a linear rescale based on the calculated bins. /// * Supervised Binning - Bucketize the data in each row and performas a linear rescale based on the calculated bins. The bin calculation is based on correlation of the Label column. /// @@ -54,6 +54,13 @@ namespace Microsoft.ML.Transforms /// With Min Max, the distribution depends on how far away the number is from 0, resulting in the number with the largest distance being mapped to 1 if its a positive number /// or -1 if its a negative number. The distance from 0 will affect the distribution with a majority of numbers that are closer together normalizing towards 0. /// + /// The equation for the output $y$ of applying both Mean Variance and Log Mean Variance on input $x$ without + /// using the CDF option is: $y = (x - \text{offset}) \text{scale}$. Where offset and scale are computed during training. + /// + /// Using the CDF option it is: $y = 0.5 * (1 + \text{ERF}((x - \text{mean}) / (\text{standard deviation} * sqrt(2)))$. + /// Where ERF is the [Error Function](https://en.wikipedia.org/wiki/Error_function) used to approximate the CDF of a random variable assumed to + /// normally distributed. The mean and standard deviation are computing during training. + /// /// To create this estimator use one of the following: /// * [NormalizeMinMax](xref:Microsoft.ML.NormalizationCatalog.NormalizeMinMax(Microsoft.ML.TransformsCatalog, System.String, System.String, System.Int64, System.Boolean)) /// * [NormalizeMeanVariance](xref:Microsoft.ML.NormalizationCatalog.NormalizeMeanVariance(Microsoft.ML.TransformsCatalog,System.String,System.String,System.Int64,System.Boolean,System.Boolean)) @@ -183,13 +190,13 @@ internal override IColumnFunctionBuilder MakeBuilder(IHost host, int srcIndex, D } [BestFriend] - internal sealed class LogMeanVarianceColumnOptions : ColumnOptionsBase + internal sealed class LogMeanVarianceColumnOptions : ControlZeroColumnOptionsBase { public readonly bool UseCdf; public LogMeanVarianceColumnOptions(string outputColumnName, string inputColumnName = null, - long maximumExampleCount = Defaults.MaximumExampleCount, bool useCdf = Defaults.LogMeanVarCdf) - : base(outputColumnName, inputColumnName ?? outputColumnName, maximumExampleCount) + long maximumExampleCount = Defaults.MaximumExampleCount, bool useCdf = Defaults.LogMeanVarCdf, bool fixZero = Defaults.EnsureZeroUntouched) + : base(outputColumnName, inputColumnName ?? outputColumnName, maximumExampleCount, fixZero) { UseCdf = useCdf; } diff --git a/src/Microsoft.ML.Data/Transforms/ValueMapping.cs b/src/Microsoft.ML.Data/Transforms/ValueMapping.cs index a9c07decd6..8bab27f412 100644 --- a/src/Microsoft.ML.Data/Transforms/ValueMapping.cs +++ b/src/Microsoft.ML.Data/Transforms/ValueMapping.cs @@ -160,8 +160,6 @@ public sealed override SchemaShape GetOutputSchema(SchemaShape inputSchema) /// /// Specifies the key type. /// Specifies the value type. - /// - /// /// public sealed class ValueMappingEstimator : ValueMappingEstimator { diff --git a/src/Microsoft.ML.DnnAnalyzer/Microsoft.ML.DnnAnalyzer/Microsoft.ML.DnnAnalyzer.csproj b/src/Microsoft.ML.DnnAnalyzer/Microsoft.ML.DnnAnalyzer/Microsoft.ML.DnnAnalyzer.csproj index 44aa3ec001..1fcfc38381 100644 --- a/src/Microsoft.ML.DnnAnalyzer/Microsoft.ML.DnnAnalyzer/Microsoft.ML.DnnAnalyzer.csproj +++ b/src/Microsoft.ML.DnnAnalyzer/Microsoft.ML.DnnAnalyzer/Microsoft.ML.DnnAnalyzer.csproj @@ -14,7 +14,9 @@ - + + $(TensorFlowMajorVersion) + diff --git a/src/Microsoft.ML.DnnImageFeaturizer.ResNet18/ResNet18Extension.cs b/src/Microsoft.ML.DnnImageFeaturizer.ResNet18/ResNet18Extension.cs index aa45783b0b..9572201cc1 100644 --- a/src/Microsoft.ML.DnnImageFeaturizer.ResNet18/ResNet18Extension.cs +++ b/src/Microsoft.ML.DnnImageFeaturizer.ResNet18/ResNet18Extension.cs @@ -30,8 +30,8 @@ public static EstimatorChain ResNet18(this DnnImageMod /// /// This allows a custom model location to be specified. This is useful is a custom model is specified, - /// or if the model is desired to be placed or shipped separately in a different folder from the main application. Note that because Onnx models - /// must be in a directory all by themsleves for the OnnxTransformer to work, this method appends a ResNet18Onnx/ResNetPrepOnnx subdirectory + /// or if the model is desired to be placed or shipped separately in a different folder from the main application. Note that because ONNX models + /// must be in a directory all by themselves for the OnnxTransformer to work, this method appends a ResNet18Onnx/ResNetPrepOnnx subdirectory /// to the passed in directory to prevent having to make that directory manually each time. /// public static EstimatorChain ResNet18(this DnnImageModelSelector dnnModelContext, IHostEnvironment env, string outputColumnName, string inputColumnName, string modelDir) diff --git a/src/Microsoft.ML.Ensemble/OutputCombiners/BaseStacking.cs b/src/Microsoft.ML.Ensemble/OutputCombiners/BaseStacking.cs index cef3eb35d5..5416542bc0 100644 --- a/src/Microsoft.ML.Ensemble/OutputCombiners/BaseStacking.cs +++ b/src/Microsoft.ML.Ensemble/OutputCombiners/BaseStacking.cs @@ -22,10 +22,10 @@ public abstract class ArgumentsBase [TGUI(Label = "Validation Dataset Proportion")] public Single ValidationDatasetProportion = 0.3f; - internal abstract IComponentFactory>> GetPredictorFactory(); + internal abstract IComponentFactory>, IPredictorProducing>> GetPredictorFactory(); } - private protected readonly IComponentFactory>> BasePredictorType; + private protected readonly IComponentFactory>, IPredictorProducing>> BasePredictorType; private protected readonly IHost Host; private protected IPredictorProducing Meta; @@ -140,54 +140,75 @@ public void Train(List> models, RoleMappedData data, maps[i] = m.GetMapper, TOutput>(); } - // REVIEW: Should implement this better.... - var labels = new Single[100]; - var features = new VBuffer[100]; - int count = 0; - // REVIEW: Should this include bad values or filter them? - using (var cursor = new FloatLabelCursor(data, CursOpt.AllFeatures | CursOpt.AllLabels)) + var view = CreateDataView(host, ch, data, maps, models); + var trainer = BasePredictorType.CreateComponent(host); + if (trainer.Info.NeedNormalization) + ch.Warning("The trainer specified for stacking wants normalization, but we do not currently allow this."); + Meta = trainer.Fit(view).Model; + CheckMeta(); + } + } + + private IDataView CreateDataView(IHostEnvironment env, IChannel ch, RoleMappedData data, ValueMapper, + TOutput>[] maps, List> models) + { + switch (data.Schema.Label.Value.Type.GetRawKind()) + { + case InternalDataKind.BL: + return CreateDataView(env, ch, data, maps, models, x => x > 0); + case InternalDataKind.R4: + return CreateDataView(env, ch, data, maps, models, x => x); + case InternalDataKind.U4: + ch.Check(data.Schema.Label.Value.Type is KeyDataViewType); + return CreateDataView(env, ch, data, maps, models, x => float.IsNaN(x) ? 0 : (uint)(x + 1)); + default: + throw ch.Except("Unsupported label type"); + } + } + + private IDataView CreateDataView(IHostEnvironment env, IChannel ch, RoleMappedData data, ValueMapper, TOutput>[] maps, + List> models, Func labelConvert) + { + // REVIEW: Should implement this better.... + var labels = new T[100]; + var features = new VBuffer[100]; + int count = 0; + // REVIEW: Should this include bad values or filter them? + using (var cursor = new FloatLabelCursor(data, CursOpt.AllFeatures | CursOpt.AllLabels)) + { + TOutput[] predictions = new TOutput[maps.Length]; + var vBuffers = new VBuffer[maps.Length]; + while (cursor.MoveNext()) { - TOutput[] predictions = new TOutput[maps.Length]; - var vBuffers = new VBuffer[maps.Length]; - while (cursor.MoveNext()) + Parallel.For(0, maps.Length, i => { - Parallel.For(0, maps.Length, i => + var model = models[i]; + if (model.SelectedFeatures != null) { - var model = models[i]; - if (model.SelectedFeatures != null) - { - EnsembleUtils.SelectFeatures(in cursor.Features, model.SelectedFeatures, model.Cardinality, ref vBuffers[i]); - maps[i](in vBuffers[i], ref predictions[i]); - } - else - maps[i](in cursor.Features, ref predictions[i]); - }); - - Utils.EnsureSize(ref labels, count + 1); - Utils.EnsureSize(ref features, count + 1); - labels[count] = cursor.Label; - FillFeatureBuffer(predictions, ref features[count]); - count++; - } + EnsembleUtils.SelectFeatures(in cursor.Features, model.SelectedFeatures, model.Cardinality, ref vBuffers[i]); + maps[i](in vBuffers[i], ref predictions[i]); + } + else + maps[i](in cursor.Features, ref predictions[i]); + }); + + Utils.EnsureSize(ref labels, count + 1); + Utils.EnsureSize(ref features, count + 1); + labels[count] = labelConvert(cursor.Label); + FillFeatureBuffer(predictions, ref features[count]); + count++; } + } - ch.Info("The number of instances used for stacking trainer is {0}", count); - - var bldr = new ArrayDataViewBuilder(host); - Array.Resize(ref labels, count); - Array.Resize(ref features, count); - bldr.AddColumn(DefaultColumnNames.Label, NumberDataViewType.Single, labels); - bldr.AddColumn(DefaultColumnNames.Features, NumberDataViewType.Single, features); + ch.Info("The number of instances used for stacking trainer is {0}", count); - var view = bldr.GetDataView(); - var rmd = new RoleMappedData(view, DefaultColumnNames.Label, DefaultColumnNames.Features); + var bldr = new ArrayDataViewBuilder(env); + Array.Resize(ref labels, count); + Array.Resize(ref features, count); + bldr.AddColumn(DefaultColumnNames.Label, data.Schema.Label.Value.Type as PrimitiveDataViewType, labels); + bldr.AddColumn(DefaultColumnNames.Features, NumberDataViewType.Single, features); - var trainer = BasePredictorType.CreateComponent(host); - if (trainer.Info.NeedNormalization) - ch.Warning("The trainer specified for stacking wants normalization, but we do not currently allow this."); - Meta = trainer.Train(rmd); - CheckMeta(); - } + return bldr.GetDataView(); } } } diff --git a/src/Microsoft.ML.Ensemble/OutputCombiners/MultiStacking.cs b/src/Microsoft.ML.Ensemble/OutputCombiners/MultiStacking.cs index dcf2b47805..a74923185c 100644 --- a/src/Microsoft.ML.Ensemble/OutputCombiners/MultiStacking.cs +++ b/src/Microsoft.ML.Ensemble/OutputCombiners/MultiStacking.cs @@ -19,7 +19,8 @@ namespace Microsoft.ML.Trainers.Ensemble { - using TVectorPredictor = IPredictorProducing>; + using TVectorTrainer = ITrainerEstimator>>, IPredictorProducing>>; + internal sealed class MultiStacking : BaseStacking>, IMulticlassOutputCombiner { public const string LoadName = "MultiStacking"; @@ -44,9 +45,9 @@ public sealed class Arguments : ArgumentsBase, ISupportMulticlassOutputCombinerF [Argument(ArgumentType.Multiple, HelpText = "Base predictor for meta learning", ShortName = "bp", SortOrder = 50, Visibility = ArgumentAttribute.VisibilityType.CmdLineOnly, SignatureType = typeof(SignatureMulticlassClassifierTrainer))] [TGUI(Label = "Base predictor")] - public IComponentFactory> BasePredictorType; + public IComponentFactory BasePredictorType; - internal override IComponentFactory> GetPredictorFactory() => BasePredictorType; + internal override IComponentFactory GetPredictorFactory() => BasePredictorType; public IMulticlassOutputCombiner CreateComponent(IHostEnvironment env) => new MultiStacking(env, this); } diff --git a/src/Microsoft.ML.Ensemble/OutputCombiners/RegressionStacking.cs b/src/Microsoft.ML.Ensemble/OutputCombiners/RegressionStacking.cs index c42b488d7e..fb77f28795 100644 --- a/src/Microsoft.ML.Ensemble/OutputCombiners/RegressionStacking.cs +++ b/src/Microsoft.ML.Ensemble/OutputCombiners/RegressionStacking.cs @@ -17,7 +17,7 @@ namespace Microsoft.ML.Trainers.Ensemble { - using TScalarPredictor = IPredictorProducing; + using TScalarTrainer = ITrainerEstimator>, IPredictorProducing>; internal sealed class RegressionStacking : BaseScalarStacking, IRegressionOutputCombiner { @@ -43,9 +43,9 @@ public sealed class Arguments : ArgumentsBase, ISupportRegressionOutputCombinerF [Argument(ArgumentType.Multiple, HelpText = "Base predictor for meta learning", ShortName = "bp", SortOrder = 50, Visibility = ArgumentAttribute.VisibilityType.CmdLineOnly, SignatureType = typeof(SignatureRegressorTrainer))] [TGUI(Label = "Base predictor")] - public IComponentFactory> BasePredictorType; + public IComponentFactory BasePredictorType; - internal override IComponentFactory> GetPredictorFactory() => BasePredictorType; + internal override IComponentFactory GetPredictorFactory() => BasePredictorType; public IRegressionOutputCombiner CreateComponent(IHostEnvironment env) => new RegressionStacking(env, this); } diff --git a/src/Microsoft.ML.Ensemble/OutputCombiners/Stacking.cs b/src/Microsoft.ML.Ensemble/OutputCombiners/Stacking.cs index 686aed36f4..6602841bbc 100644 --- a/src/Microsoft.ML.Ensemble/OutputCombiners/Stacking.cs +++ b/src/Microsoft.ML.Ensemble/OutputCombiners/Stacking.cs @@ -15,7 +15,8 @@ namespace Microsoft.ML.Trainers.Ensemble { - using TScalarPredictor = IPredictorProducing; + using TScalarTrainer = ITrainerEstimator>, IPredictorProducing>; + internal sealed class Stacking : BaseScalarStacking, IBinaryOutputCombiner { public const string UserName = "Stacking"; @@ -41,9 +42,9 @@ public sealed class Arguments : ArgumentsBase, ISupportBinaryOutputCombinerFacto [Argument(ArgumentType.Multiple, HelpText = "Base predictor for meta learning", ShortName = "bp", SortOrder = 50, Visibility = ArgumentAttribute.VisibilityType.CmdLineOnly, SignatureType = typeof(SignatureBinaryClassifierTrainer))] [TGUI(Label = "Base predictor")] - public IComponentFactory> BasePredictorType; + public IComponentFactory BasePredictorType; - internal override IComponentFactory> GetPredictorFactory() => BasePredictorType; + internal override IComponentFactory GetPredictorFactory() => BasePredictorType; public IBinaryOutputCombiner CreateComponent(IHostEnvironment env) => new Stacking(env, this); } diff --git a/src/Microsoft.ML.Ensemble/Trainer/Binary/EnsembleTrainer.cs b/src/Microsoft.ML.Ensemble/Trainer/Binary/EnsembleTrainer.cs index 92c3b2e21b..f1b4e9daa7 100644 --- a/src/Microsoft.ML.Ensemble/Trainer/Binary/EnsembleTrainer.cs +++ b/src/Microsoft.ML.Ensemble/Trainer/Binary/EnsembleTrainer.cs @@ -23,10 +23,12 @@ namespace Microsoft.ML.Trainers.Ensemble { using TDistPredictor = IDistPredictorProducing; using TScalarPredictor = IPredictorProducing; + using TScalarTrainer = ITrainerEstimator>, IPredictorProducing>; + /// /// A generic ensemble trainer for binary classification. /// - internal sealed class EnsembleTrainer : EnsembleTrainerBase, IModelCombiner { @@ -47,20 +49,15 @@ public sealed class Arguments : ArgumentsBase // REVIEW: If we make this public again it should be an *estimator* of this type of predictor, rather than the (deprecated) ITrainer. [Argument(ArgumentType.Multiple, HelpText = "Base predictor type", ShortName = "bp,basePredictorTypes", SortOrder = 1, Visibility = ArgumentAttribute.VisibilityType.CmdLineOnly, SignatureType = typeof(SignatureBinaryClassifierTrainer))] - public IComponentFactory>[] BasePredictors; + public IComponentFactory[] BasePredictors; - internal override IComponentFactory>[] GetPredictorFactories() => BasePredictors; + internal override IComponentFactory[] GetPredictorFactories() => BasePredictors; public Arguments() { BasePredictors = new[] { - ComponentFactoryUtils.CreateFromFunction( - env => { - var trainerEstimator = new LinearSvmTrainer(env); - return TrainerUtils.MapTrainerEstimatorToTrainer(env, trainerEstimator); - }) + ComponentFactoryUtils.CreateFromFunction(env => new LinearSvmTrainer(env, LabelColumnName, FeatureColumnName)) }; } } @@ -83,7 +80,7 @@ private EnsembleTrainer(IHostEnvironment env, Arguments args, PredictionKind pre private protected override PredictionKind PredictionKind => PredictionKind.BinaryClassification; - private protected override TScalarPredictor CreatePredictor(List> models) + private protected override IPredictor CreatePredictor(List> models) { if (models.All(m => m.Predictor is TDistPredictor)) return new EnsembleDistributionModelParameters(Host, PredictionKind, CreateModels(models), Combiner); diff --git a/src/Microsoft.ML.Ensemble/Trainer/EnsembleTrainerBase.cs b/src/Microsoft.ML.Ensemble/Trainer/EnsembleTrainerBase.cs index 3ab77000bc..ae44bff5e4 100644 --- a/src/Microsoft.ML.Ensemble/Trainer/EnsembleTrainerBase.cs +++ b/src/Microsoft.ML.Ensemble/Trainer/EnsembleTrainerBase.cs @@ -17,8 +17,7 @@ namespace Microsoft.ML.Trainers.Ensemble { using Stopwatch = System.Diagnostics.Stopwatch; - internal abstract class EnsembleTrainerBase : ITrainer - where TPredictor : class, IPredictorProducing + internal abstract class EnsembleTrainerBase : ITrainer where TSelector : class, ISubModelSelector where TCombiner : class, IOutputCombiner { @@ -51,7 +50,7 @@ public abstract class ArgumentsBase : TrainerInputBaseWithLabel [TGUI(Label = "Show Sub-Model Metrics")] public bool ShowMetrics; - internal abstract IComponentFactory>>[] GetPredictorFactories(); + internal abstract IComponentFactory>, IPredictorProducing>>[] GetPredictorFactories(); #pragma warning restore CS0649 } @@ -62,7 +61,7 @@ public abstract class ArgumentsBase : TrainerInputBaseWithLabel private protected readonly IHost Host; /// Ensemble members - private protected readonly ITrainer>[] Trainers; + private protected readonly ITrainerEstimator>, IPredictorProducing>[] Trainers; private readonly ISubsetSelector _subsetSelector; private protected ISubModelSelector SubModelSelector; @@ -95,7 +94,7 @@ private protected EnsembleTrainerBase(ArgumentsBase args, IHostEnvironment env, _subsetSelector = Args.SamplingType.CreateComponent(Host); - Trainers = new ITrainer>[NumModels]; + Trainers = new ITrainerEstimator>, IPredictorProducing>[NumModels]; for (int i = 0; i < Trainers.Length; i++) Trainers[i] = predictorFactories[i % predictorFactories.Length].CreateComponent(Host); // We infer normalization and calibration preferences from the trainers. However, even if the internal trainers @@ -106,7 +105,7 @@ private protected EnsembleTrainerBase(ArgumentsBase args, IHostEnvironment env, } } - TPredictor ITrainer.Train(TrainContext context) + IPredictor ITrainer.Train(TrainContext context) { Host.CheckValue(context, nameof(context)); @@ -117,9 +116,9 @@ TPredictor ITrainer.Train(TrainContext context) } IPredictor ITrainer.Train(TrainContext context) - => ((ITrainer)this).Train(context); + => ((ITrainer)this).Train(context); - private TPredictor TrainCore(IChannel ch, RoleMappedData data) + private IPredictor TrainCore(IChannel ch, RoleMappedData data) { Host.AssertValue(ch); ch.AssertValue(data); @@ -155,8 +154,9 @@ private TPredictor TrainCore(IChannel ch, RoleMappedData data) { if (EnsureMinimumFeaturesSelected(subset)) { + // REVIEW: How to pass the role mappings to the trainer? var model = new FeatureSubsetModel( - Trainers[(int)index].Train(subset.Data), + Trainers[(int)index].Fit(subset.Data.Data).Model, subset.SelectedFeatures, null); SubModelSelector.CalculateMetrics(model, _subsetSelector, subset, batch, needMetrics); @@ -190,7 +190,7 @@ private TPredictor TrainCore(IChannel ch, RoleMappedData data) return CreatePredictor(models); } - private protected abstract TPredictor CreatePredictor(List> models); + private protected abstract IPredictor CreatePredictor(List> models); private bool EnsureMinimumFeaturesSelected(Subset subset) { diff --git a/src/Microsoft.ML.Ensemble/Trainer/Multiclass/MulticlassDataPartitionEnsembleTrainer.cs b/src/Microsoft.ML.Ensemble/Trainer/Multiclass/MulticlassDataPartitionEnsembleTrainer.cs index 20427668b8..edc4ca53cd 100644 --- a/src/Microsoft.ML.Ensemble/Trainer/Multiclass/MulticlassDataPartitionEnsembleTrainer.cs +++ b/src/Microsoft.ML.Ensemble/Trainer/Multiclass/MulticlassDataPartitionEnsembleTrainer.cs @@ -24,11 +24,13 @@ namespace Microsoft.ML.Trainers.Ensemble { using TVectorPredictor = IPredictorProducing>; + using TVectorTrainer = ITrainerEstimator>>, IPredictorProducing>>; + /// /// A generic ensemble classifier for multi-class classification /// internal sealed class MulticlassDataPartitionEnsembleTrainer : - EnsembleTrainerBase, EnsembleMulticlassModelParameters, + EnsembleTrainerBase, IMulticlassSubModelSelector, IMulticlassOutputCombiner>, IModelCombiner { @@ -48,26 +50,21 @@ public sealed class Arguments : ArgumentsBase // REVIEW: If we make this public again it should be an *estimator* of this type of predictor, rather than the (deprecated) ITrainer. [Argument(ArgumentType.Multiple, HelpText = "Base predictor type", ShortName = "bp,basePredictorTypes", SortOrder = 1, Visibility = ArgumentAttribute.VisibilityType.CmdLineOnly, SignatureType = typeof(SignatureMulticlassClassifierTrainer))] - internal IComponentFactory>[] BasePredictors; + internal IComponentFactory[] BasePredictors; - internal override IComponentFactory>[] GetPredictorFactories() => BasePredictors; + internal override IComponentFactory[] GetPredictorFactories() => BasePredictors; public Arguments() { BasePredictors = new[] { - ComponentFactoryUtils.CreateFromFunction( - env => { - // Note that this illustrates a fundamnetal problem with the mixture of `ITrainer` and `ITrainerEstimator` - // present in this class. The options to the estimator have no way of being communicated to the `ITrainer` - // implementation, so there is a fundamnetal disconnect if someone chooses to ever use the *estimator* with - // non-default column names. Unfortuantely no method of resolving this temporary strikes me as being any - // less laborious than the proper fix, which is that this "meta" component should itself be a trainer - // estimator, as opposed to a regular trainer. - var trainerEstimator = new LbfgsMaximumEntropyMulticlassTrainer(env, LabelColumnName, FeatureColumnName); - return TrainerUtils.MapTrainerEstimatorToTrainer(env, trainerEstimator); - }) + // Note that this illustrates a fundamnetal problem with the mixture of `ITrainer` and `ITrainerEstimator` + // present in this class. The options to the estimator have no way of being communicated to the `ITrainer` + // implementation, so there is a fundamnetal disconnect if someone chooses to ever use the *estimator* with + // non-default column names. Unfortuantely no method of resolving this temporary strikes me as being any + // less laborious than the proper fix, which is that this "meta" component should itself be a trainer + // estimator, as opposed to a regular trainer. + ComponentFactoryUtils.CreateFromFunction(env => new LbfgsMaximumEntropyMulticlassTrainer(env, LabelColumnName, FeatureColumnName)) }; } } @@ -90,7 +87,7 @@ private MulticlassDataPartitionEnsembleTrainer(IHostEnvironment env, Arguments a private protected override PredictionKind PredictionKind => PredictionKind.MulticlassClassification; - private protected override EnsembleMulticlassModelParameters CreatePredictor(List>> models) + private protected override IPredictor CreatePredictor(List>> models) { return new EnsembleMulticlassModelParameters(Host, CreateModels(models), Combiner as IMulticlassOutputCombiner); } diff --git a/src/Microsoft.ML.Ensemble/Trainer/Regression/RegressionEnsembleTrainer.cs b/src/Microsoft.ML.Ensemble/Trainer/Regression/RegressionEnsembleTrainer.cs index ab17f5f28e..490cdd463d 100644 --- a/src/Microsoft.ML.Ensemble/Trainer/Regression/RegressionEnsembleTrainer.cs +++ b/src/Microsoft.ML.Ensemble/Trainer/Regression/RegressionEnsembleTrainer.cs @@ -23,7 +23,9 @@ namespace Microsoft.ML.Trainers.Ensemble { using TScalarPredictor = IPredictorProducing; - internal sealed class RegressionEnsembleTrainer : EnsembleTrainerBase>, IPredictorProducing>; + + internal sealed class RegressionEnsembleTrainer : EnsembleTrainerBase, IModelCombiner { @@ -42,20 +44,15 @@ public sealed class Arguments : ArgumentsBase // REVIEW: If we make this public again it should be an *estimator* of this type of predictor, rather than the (deprecated) ITrainer. [Argument(ArgumentType.Multiple, HelpText = "Base predictor type", ShortName = "bp,basePredictorTypes", SortOrder = 1, Visibility = ArgumentAttribute.VisibilityType.CmdLineOnly, SignatureType = typeof(SignatureRegressorTrainer))] - public IComponentFactory>[] BasePredictors; + public IComponentFactory[] BasePredictors; - internal override IComponentFactory>[] GetPredictorFactories() => BasePredictors; + internal override IComponentFactory[] GetPredictorFactories() => BasePredictors; public Arguments() { BasePredictors = new[] { - ComponentFactoryUtils.CreateFromFunction( - env => { - var trainerEstimator = new OnlineGradientDescentTrainer(env); - return TrainerUtils.MapTrainerEstimatorToTrainer(env, trainerEstimator); - }) + ComponentFactoryUtils.CreateFromFunction(env => new OnlineGradientDescentTrainer(env, LabelColumnName, FeatureColumnName)) }; } } @@ -78,7 +75,7 @@ private RegressionEnsembleTrainer(IHostEnvironment env, Arguments args, Predicti private protected override PredictionKind PredictionKind => PredictionKind.Regression; - private protected override TScalarPredictor CreatePredictor(List> models) + private protected override IPredictor CreatePredictor(List> models) { return new EnsembleModelParameters(Host, PredictionKind, CreateModels(models), Combiner); } diff --git a/src/Microsoft.ML.FastTree/FastTree.cs b/src/Microsoft.ML.FastTree/FastTree.cs index ace3298dd8..82772ff151 100644 --- a/src/Microsoft.ML.FastTree/FastTree.cs +++ b/src/Microsoft.ML.FastTree/FastTree.cs @@ -14,6 +14,7 @@ using Microsoft.ML.CommandLine; using Microsoft.ML.Data; using Microsoft.ML.Data.Conversion; +using Microsoft.ML.FastTree.Utils; using Microsoft.ML.Internal.Internallearn; using Microsoft.ML.Internal.Utilities; using Microsoft.ML.Model; @@ -2791,7 +2792,6 @@ public abstract class TreeEnsembleModelParameters : IPredictorWithFeatureWeights, IFeatureContributionMapper, ICalculateFeatureContribution, - ICanGetSummaryAsIRow, ISingleCanSavePfa, ISingleCanSaveOnnx { @@ -3276,23 +3276,6 @@ internal int GetLeaf(int treeId, in VBuffer features, ref List path) return TrainedEnsemble.GetTreeAt(treeId).GetLeaf(in features, ref path); } - DataViewRow ICanGetSummaryAsIRow.GetSummaryIRowOrNull(RoleMappedSchema schema) - { - var names = default(VBuffer>); - AnnotationUtils.GetSlotNames(schema, RoleMappedSchema.ColumnRole.Feature, NumFeatures, ref names); - var metaBuilder = new DataViewSchema.Annotations.Builder(); - metaBuilder.AddSlotNames(NumFeatures, names.CopyTo); - - var weights = default(VBuffer); - ((IHaveFeatureWeights)this).GetFeatureWeights(ref weights); - var builder = new DataViewSchema.Annotations.Builder(); - builder.Add>("Gains", new VectorDataViewType(NumberDataViewType.Single, NumFeatures), weights.CopyTo, metaBuilder.ToAnnotations()); - - return AnnotationUtils.AnnotationsAsRow(builder.ToAnnotations()); - } - - DataViewRow ICanGetSummaryAsIRow.GetStatsIRowOrNull(RoleMappedSchema schema) => null; - private sealed class Tree : ITree> { private readonly InternalRegressionTree _regTree; @@ -3378,7 +3361,7 @@ public TreeNode(Dictionary keyValues) /// and is the type of /// . /// - public abstract class TreeEnsembleModelParametersBasedOnRegressionTree : TreeEnsembleModelParameters + public abstract class TreeEnsembleModelParametersBasedOnRegressionTree : TreeEnsembleModelParameters, ICanGetSummaryAsIDataView { /// /// An ensemble of trees exposed to users. It is a wrapper on the @@ -3406,6 +3389,12 @@ private RegressionTreeEnsemble CreateTreeEnsembleFromInternalDataStructure() var treeWeights = TrainedEnsemble.Trees.Select(tree => tree.Weight); return new RegressionTreeEnsemble(trees, treeWeights, TrainedEnsemble.Bias); } + + /// + /// Used for the Summarize entrypoint. + /// + IDataView ICanGetSummaryAsIDataView.GetSummaryDataView(RoleMappedSchema schema) + => RegressionTreeBaseUtils.RegressionTreeEnsembleAsIDataView(Host, TrainedTreeEnsemble.Bias, TrainedTreeEnsemble.TreeWeights, TrainedTreeEnsemble.Trees); } /// @@ -3418,7 +3407,7 @@ private RegressionTreeEnsemble CreateTreeEnsembleFromInternalDataStructure() /// and is the type of /// . /// - public abstract class TreeEnsembleModelParametersBasedOnQuantileRegressionTree : TreeEnsembleModelParameters + public abstract class TreeEnsembleModelParametersBasedOnQuantileRegressionTree : TreeEnsembleModelParameters, ICanGetSummaryAsIDataView { /// /// An ensemble of trees exposed to users. It is a wrapper on the @@ -3446,5 +3435,11 @@ private QuantileRegressionTreeEnsemble CreateTreeEnsembleFromInternalDataStructu var treeWeights = TrainedEnsemble.Trees.Select(tree => tree.Weight); return new QuantileRegressionTreeEnsemble(trees, treeWeights, TrainedEnsemble.Bias); } + + /// + /// Used for the Summarize entrypoint. + /// + IDataView ICanGetSummaryAsIDataView.GetSummaryDataView(RoleMappedSchema schema) + => RegressionTreeBaseUtils.RegressionTreeEnsembleAsIDataView(Host, TrainedTreeEnsemble.Bias, TrainedTreeEnsemble.TreeWeights, TrainedTreeEnsemble.Trees); } } diff --git a/src/Microsoft.ML.FastTree/TreeEnsembleFeaturizationEstimator.cs b/src/Microsoft.ML.FastTree/TreeEnsembleFeaturizationEstimator.cs new file mode 100644 index 0000000000..c7248fe3c1 --- /dev/null +++ b/src/Microsoft.ML.FastTree/TreeEnsembleFeaturizationEstimator.cs @@ -0,0 +1,464 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Linq; +using Microsoft.ML.Data; +using Microsoft.ML.Runtime; + +namespace Microsoft.ML.Trainers.FastTree +{ + /// + /// This class encapsulates the common behavior of all tree-based featurizers such as , + /// , , + /// , and . + /// All tree-based featurizers share the same output schema computed by . All tree-based featurizers + /// requires an input feature column name and a suffix for all output columns. The returned by + /// produces three columns: (1) the prediction values of all trees, (2) the IDs of leaves the input feature vector falling into, and (3) + /// the binary vector which encodes the paths to those destination leaves. + /// + public abstract class TreeEnsembleFeaturizationEstimatorBase : IEstimator + { + /// + /// The common options of tree-based featurizations such as , , + /// , , and . + /// + public abstract class OptionsBase + { + /// + /// The name of feature column in the when calling . + /// The column type must be a vector of . The column called would be mapped + /// to columns called , , and in the output + /// of and its derived classes. Note that is not + /// necessary to be the same as the feature column used to train the underlying tree model. + /// + public string InputColumnName; + + /// + /// The name of the column that stores the prediction values of all trees. Its type is a vector of + /// and the i-th vector element is the prediction value predicted by the i-th tree. + /// If is , this output column may not be generated. + /// + public string TreesColumnName; + + /// + /// The 0-1 encoding of all leaf nodes' IDs. Its type is a vector of . If the given feature + /// vector falls into the first leaf of the first tree, the first element in the 0-1 encoding would be 1. + /// If is , this output column may not be generated. + /// + public string LeavesColumnName; + + /// + /// The 0-1 encoding of the paths to the leaves. If the path to the first tree's leaf is node 1 (2nd node in the first tree), + /// node 3 (4th node in the first tree), and node 5 (6th node in the first tree), the 2nd, 4th, and 6th element in that encoding + /// would be 1. + /// If is , this output column may not be generated. + /// + public string PathsColumnName; + }; + + /// + /// See . + /// + private protected readonly string FeatureColumnName; + + /// + /// See . + /// + private protected readonly string TreesColumnName; + + /// + /// See . + /// + private protected readonly string LeavesColumnName; + + /// + /// See . + /// + private protected readonly string PathsColumnName; + + /// + /// Environment of this instance. It controls error throwing and other environment settings. + /// + private protected readonly IHostEnvironment Env; + + private protected TreeEnsembleFeaturizationEstimatorBase(IHostEnvironment env, OptionsBase options) + { + Env = env; + if (options.InputColumnName == null) + throw Env.Except(nameof(options), "The " + nameof(options.InputColumnName) + " cannot be null."); + if (options.TreesColumnName == null && options.LeavesColumnName == null && options.PathsColumnName == null) + throw Env.Except($"{nameof(OptionsBase.TreesColumnName)}, {nameof(OptionsBase.LeavesColumnName)}, and {nameof(OptionsBase.PathsColumnName)} cannot be all null at the same time. " + + $"At least one output column name should be provided so at least one output column may be generated."); + + FeatureColumnName = options.InputColumnName; + TreesColumnName = options.TreesColumnName; + LeavesColumnName = options.LeavesColumnName; + PathsColumnName = options.PathsColumnName; + } + + /// + /// All derived class should implement to tell how to get a + /// out from and parameters inside this or derived classes. + /// + /// Data used to train a tree model. + /// The trees used in . + private protected abstract TreeEnsembleModelParameters PrepareModel(IDataView input); + + /// + /// Produce a which maps the column called in + /// to three output columns. + /// + public TreeEnsembleFeaturizationTransformer Fit(IDataView input) + { + var model = PrepareModel(input); + return new TreeEnsembleFeaturizationTransformer(Env, input.Schema, input.Schema[FeatureColumnName], model, + TreesColumnName, LeavesColumnName, PathsColumnName); + } + + /// + /// adds three float-vector columns into . + /// Given a feature vector column, the added columns are the prediction values of all trees, the leaf IDs the feature + /// vector falls into, and the paths to those leaves. + /// + /// A schema which contains a feature column. Note that feature column name can be specified + /// by . + /// Output produced by . + public SchemaShape GetOutputSchema(SchemaShape inputSchema) + { + Env.CheckValue(inputSchema, nameof(inputSchema)); + + if (!inputSchema.TryFindColumn(FeatureColumnName, out var col)) + throw Env.ExceptSchemaMismatch(nameof(inputSchema), "input", FeatureColumnName); + + var result = inputSchema.ToDictionary(x => x.Name); + + if (TreesColumnName != null) + result[TreesColumnName] = new SchemaShape.Column(TreesColumnName, + SchemaShape.Column.VectorKind.Vector, NumberDataViewType.Single, false); + + if (LeavesColumnName != null) + result[LeavesColumnName] = new SchemaShape.Column(LeavesColumnName, + SchemaShape.Column.VectorKind.Vector, NumberDataViewType.Single, false); + + if (PathsColumnName != null) + result[PathsColumnName] = new SchemaShape.Column(PathsColumnName, + SchemaShape.Column.VectorKind.Vector, NumberDataViewType.Single, false); + + return new SchemaShape(result.Values); + } + } + /// + /// A which contains a pre-trained and calling its + /// produces a featurizer based on the pre-trained model. + /// + /// + /// . + /// The input features column data must be a known-sized vector of. + /// + /// This estimator outputs the following columns: + /// + /// | Output Column Name | Column Type | Description| + /// | -- | -- | -- | + /// | `Trees` | Vector of | The output values of all trees. | + /// | `Leaves` | Vector of | The IDs of all leaves where the input feature vector falls into. | + /// | `Paths` | Vector of | The paths the input feature vector passed through to reach the leaves. | + /// + /// Those output columns are all optional and user can change their names. + /// Please set the names of skipped columns to null so that they would not be produced. + /// + /// [!include[algorithm](~/../docs/samples/docs/api-reference/tree-featurization-prediction.md)] + /// ]]> + /// + /// + /// + public sealed class PretrainedTreeFeaturizationEstimator : TreeEnsembleFeaturizationEstimatorBase + { + /// + /// of as + /// used when calling . + /// + public sealed class Options : OptionsBase + { + /// + /// The pretrained tree model used to do tree-based featurization. Note that contains a collection of decision trees. + /// + public TreeEnsembleModelParameters ModelParameters; + }; + + private TreeEnsembleModelParameters _modelParameters; + + internal PretrainedTreeFeaturizationEstimator(IHostEnvironment env, Options options) : base(env, options) + { + _modelParameters = options.ModelParameters; + } + + /// + /// Produce the for tree-based feature engineering. This function does not + /// invoke training procedure and just returns the pre-trained model passed in via . + /// + private protected override TreeEnsembleModelParameters PrepareModel(IDataView input) => _modelParameters; + } + + /// + /// A to transform input feature vector to tree-based features. + /// + /// + /// + /// + /// + /// + public sealed class FastTreeBinaryFeaturizationEstimator : TreeEnsembleFeaturizationEstimatorBase + { + private readonly FastTreeBinaryTrainer.Options _trainerOptions; + + /// + /// Options for the . + /// + public sealed class Options : OptionsBase + { + /// + /// The configuration of used to train the underlying . + /// + public FastTreeBinaryTrainer.Options TrainerOptions; + } + + internal FastTreeBinaryFeaturizationEstimator(IHostEnvironment env, Options options) + : base(env, options) + { + _trainerOptions = options.TrainerOptions; + } + + private protected override TreeEnsembleModelParameters PrepareModel(IDataView input) + { + var trainer = new FastTreeBinaryTrainer(Env, _trainerOptions); + var trained = trainer.Fit(input); + return trained.Model.SubModel; + } + } + + /// + /// A to transform input feature vector to tree-based features. + /// + /// + /// + /// + /// + /// + public sealed class FastTreeRegressionFeaturizationEstimator : TreeEnsembleFeaturizationEstimatorBase + { + private readonly FastTreeRegressionTrainer.Options _trainerOptions; + + /// + /// Options for the . + /// + public sealed class Options : OptionsBase + { + /// + /// The configuration of used to train the underlying . + /// + public FastTreeRegressionTrainer.Options TrainerOptions; + } + + internal FastTreeRegressionFeaturizationEstimator(IHostEnvironment env, Options options) + : base(env, options) + { + _trainerOptions = options.TrainerOptions; + } + + private protected override TreeEnsembleModelParameters PrepareModel(IDataView input) + { + var trainer = new FastTreeRegressionTrainer(Env, _trainerOptions); + var trained = trainer.Fit(input); + return trained.Model; + } + } + + /// + /// A to transform input feature vector to tree-based features. + /// + /// + /// + /// + /// + /// + public sealed class FastForestBinaryFeaturizationEstimator : TreeEnsembleFeaturizationEstimatorBase + { + private readonly FastForestBinaryTrainer.Options _trainerOptions; + + /// + /// Options for the . + /// + public sealed class Options : OptionsBase + { + /// + /// The configuration of used to train the underlying . + /// + public FastForestBinaryTrainer.Options TrainerOptions; + } + + internal FastForestBinaryFeaturizationEstimator(IHostEnvironment env, Options options) + : base(env, options) + { + _trainerOptions = options.TrainerOptions; + } + + private protected override TreeEnsembleModelParameters PrepareModel(IDataView input) + { + var trainer = new FastForestBinaryTrainer(Env, _trainerOptions); + var trained = trainer.Fit(input); + return trained.Model; + } + } + + /// + /// A to transform input feature vector to tree-based features. + /// + /// + /// + /// + /// + /// + public sealed class FastForestRegressionFeaturizationEstimator : TreeEnsembleFeaturizationEstimatorBase + { + private readonly FastForestRegressionTrainer.Options _trainerOptions; + + /// + /// Options for the . + /// + public sealed class Options : OptionsBase + { + /// + /// The configuration of used to train the underlying . + /// + public FastForestRegressionTrainer.Options TrainerOptions; + } + + internal FastForestRegressionFeaturizationEstimator(IHostEnvironment env, Options options) + : base(env, options) + { + _trainerOptions = options.TrainerOptions; + } + + private protected override TreeEnsembleModelParameters PrepareModel(IDataView input) + { + var trainer = new FastForestRegressionTrainer(Env, _trainerOptions); + var trained = trainer.Fit(input); + return trained.Model; + } + } + + /// + /// A to transform input feature vector to tree-based features. + /// + /// + /// + /// + /// + /// + public sealed class FastTreeRankingFeaturizationEstimator : TreeEnsembleFeaturizationEstimatorBase + { + private readonly FastTreeRankingTrainer.Options _trainerOptions; + + /// + /// Options for the . + /// + public sealed class Options : OptionsBase + { + /// + /// The configuration of used to train the underlying . + /// + public FastTreeRankingTrainer.Options TrainerOptions; + } + + internal FastTreeRankingFeaturizationEstimator(IHostEnvironment env, Options options) + : base(env, options) + { + _trainerOptions = options.TrainerOptions; + } + + private protected override TreeEnsembleModelParameters PrepareModel(IDataView input) + { + var trainer = new FastTreeRankingTrainer(Env, _trainerOptions); + var trained = trainer.Fit(input); + return trained.Model; + } + } + + /// + /// A to transform input feature vector to tree-based features. + /// + /// + /// + /// + /// + /// + public sealed class FastTreeTweedieFeaturizationEstimator : TreeEnsembleFeaturizationEstimatorBase + { + private readonly FastTreeTweedieTrainer.Options _trainerOptions; + + /// + /// Options for the . + /// + public sealed class Options : OptionsBase + { + /// + /// The configuration of used to train the underlying . + /// + public FastTreeTweedieTrainer.Options TrainerOptions; + } + + internal FastTreeTweedieFeaturizationEstimator(IHostEnvironment env, Options options) + : base(env, options) + { + _trainerOptions = options.TrainerOptions; + } + + private protected override TreeEnsembleModelParameters PrepareModel(IDataView input) + { + var trainer = new FastTreeTweedieTrainer(Env, _trainerOptions); + var trained = trainer.Fit(input); + return trained.Model; + } + } +} diff --git a/src/Microsoft.ML.FastTree/TreeEnsembleFeaturizationTransformer.cs b/src/Microsoft.ML.FastTree/TreeEnsembleFeaturizationTransformer.cs new file mode 100644 index 0000000000..d30337b07e --- /dev/null +++ b/src/Microsoft.ML.FastTree/TreeEnsembleFeaturizationTransformer.cs @@ -0,0 +1,184 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections.Generic; +using Microsoft.ML; +using Microsoft.ML.Data; +using Microsoft.ML.Data.IO; +using Microsoft.ML.Runtime; +using Microsoft.ML.Trainers.FastTree; + +[assembly: LoadableClass(typeof(TreeEnsembleFeaturizationTransformer), typeof(TreeEnsembleFeaturizationTransformer), + null, typeof(SignatureLoadModel), "", TreeEnsembleFeaturizationTransformer.LoaderSignature)] + +namespace Microsoft.ML.Trainers.FastTree +{ + /// + /// resulting from fitting any derived class of . + /// The derived classes include, for example, and + /// . + /// + public sealed class TreeEnsembleFeaturizationTransformer : PredictionTransformerBase + { + internal const string LoaderSignature = "TreeEnseFeat"; + private readonly TreeEnsembleFeaturizerBindableMapper.Arguments _scorerArgs; + private readonly DataViewSchema.DetachedColumn _featureDetachedColumn; + /// + /// See . + /// + private readonly string _treesColumnName; + /// + /// See . + /// + private readonly string _leavesColumnName; + /// + /// See . + /// + private readonly string _pathsColumnName; + /// + /// Check if is compatible with . + /// + /// A column checked against . + private void CheckFeatureColumnCompatibility(DataViewSchema.Column inspectedFeatureColumn) + { + string nameErrorMessage = $"The column called {inspectedFeatureColumn.Name} does not match the expected " + + $"feature column with name {_featureDetachedColumn.Name} and type {_featureDetachedColumn.Type}. " + + $"Please rename your column by calling CopyColumns defined in TransformExtensionsCatalog"; + // Check if column names are the same. + Host.Check(_featureDetachedColumn.Name == inspectedFeatureColumn.Name, nameErrorMessage); + + string typeErrorMessage = $"The column called {inspectedFeatureColumn.Name} has a type {inspectedFeatureColumn.Type}, " + + $"which does not match the expected feature column with name {_featureDetachedColumn.Name} and type {_featureDetachedColumn.Type}. " + + $"Please make sure your feature column type is {_featureDetachedColumn.Type}."; + // Check if column types are identical. + Host.Check(_featureDetachedColumn.Type.Equals(inspectedFeatureColumn.Type), typeErrorMessage); + } + + /// + /// Create from by using as the feature role. + /// + /// The original schema to be mapped. + private RoleMappedSchema MakeFeatureRoleMappedSchema(DataViewSchema schema) + { + var roles = new List>(); + roles.Add(new KeyValuePair(RoleMappedSchema.ColumnRole.Feature, _featureDetachedColumn.Name)); + return new RoleMappedSchema(schema, roles); + } + + internal TreeEnsembleFeaturizationTransformer(IHostEnvironment env, DataViewSchema inputSchema, + DataViewSchema.Column featureColumn, TreeEnsembleModelParameters modelParameters, + string treesColumnName, string leavesColumnName, string pathsColumnName) : + base(Contracts.CheckRef(env, nameof(env)).Register(nameof(TreeEnsembleFeaturizationTransformer)), modelParameters, inputSchema) + { + // Store featureColumn as a detached column because a fitted transformer can be applied to different IDataViews and different + // IDataView may have different schemas. + _featureDetachedColumn = new DataViewSchema.DetachedColumn(featureColumn); + + // Check if featureColumn matches a column in inputSchema. The answer is yes if they have the same name and type. + // The indexed column, inputSchema[featureColumn.Index], should match the detached column, _featureDetachedColumn. + CheckFeatureColumnCompatibility(inputSchema[featureColumn.Index]); + + // Store output column names so that this transformer can be saved into a file later. + _treesColumnName = treesColumnName; + _leavesColumnName = leavesColumnName; + _pathsColumnName = pathsColumnName; + + // Create an argument, _scorerArgs, to pass the output column names to the underlying scorer. + _scorerArgs = new TreeEnsembleFeaturizerBindableMapper.Arguments { + TreesColumnName = _treesColumnName, LeavesColumnName = _leavesColumnName, PathsColumnName = _pathsColumnName }; + + // Create a bindable mapper. It provides the core computation and can be attached to any IDataView and produce + // a transformed IDataView. + BindableMapper = new TreeEnsembleFeaturizerBindableMapper(env, _scorerArgs, modelParameters); + + // Create a scorer. + var roleMappedSchema = MakeFeatureRoleMappedSchema(inputSchema); + Scorer = new GenericScorer(Host, _scorerArgs, new EmptyDataView(Host, inputSchema), BindableMapper.Bind(Host, roleMappedSchema), roleMappedSchema); + } + + private TreeEnsembleFeaturizationTransformer(IHostEnvironment host, ModelLoadContext ctx) + : base(Contracts.CheckRef(host, nameof(host)).Register(nameof(TreeEnsembleFeaturizationTransformer)), ctx) + { + // *** Binary format *** + // + // string: feature column's name. + // string: the name of the columns where tree prediction values are stored. + // string: the name of the columns where trees' leave are stored. + // string: the name of the columns where trees' paths are stored. + + // Load stored fields. + string featureColumnName = ctx.LoadString(); + _featureDetachedColumn = new DataViewSchema.DetachedColumn(TrainSchema[featureColumnName]); + _treesColumnName = ctx.LoadStringOrNull(); + _leavesColumnName = ctx.LoadStringOrNull(); + _pathsColumnName = ctx.LoadStringOrNull(); + + // Create an argument to specify output columns' names of this transformer. + _scorerArgs = new TreeEnsembleFeaturizerBindableMapper.Arguments { + TreesColumnName = _treesColumnName, LeavesColumnName = _leavesColumnName, PathsColumnName = _pathsColumnName }; + + // Create a bindable mapper. It provides the core computation and can be attached to any IDataView and produce + // a transformed IDataView. + BindableMapper = new TreeEnsembleFeaturizerBindableMapper(host, _scorerArgs, Model); + + // Create a scorer. + var roleMappedSchema = MakeFeatureRoleMappedSchema(TrainSchema); + Scorer = new GenericScorer(Host, _scorerArgs, new EmptyDataView(Host, TrainSchema), BindableMapper.Bind(Host, roleMappedSchema), roleMappedSchema); + } + + /// + /// appends three columns to the . + /// The three columns are all vectors. The fist column stores the prediction values of all trees and + /// its default name is "Trees". The second column (default name: "Leaves") contains leaf IDs where the given feature vector falls into. + /// The third column (default name: "Paths") encodes the paths to those leaves via a 0-1 vector. + /// + /// of the data to be transformed. + /// of the transformed data if the input schema is . + public override DataViewSchema GetOutputSchema(DataViewSchema inputSchema) => Transform(new EmptyDataView(Host, inputSchema)).Schema; + + private protected override void SaveModel(ModelSaveContext ctx) + { + Host.CheckValue(ctx, nameof(ctx)); + ctx.CheckAtModel(); + ctx.SetVersionInfo(GetVersionInfo()); + + // *** Binary format *** + // model: prediction model. + // stream: empty data view that contains train schema. + // string: feature column name. + // string: the name of the columns where tree prediction values are stored. + // string: the name of the columns where trees' leave are stored. + // string: the name of the columns where trees' paths are stored. + + ctx.SaveModel(Model, DirModel); + ctx.SaveBinaryStream(DirTransSchema, writer => + { + using (var ch = Host.Start("Saving train schema")) + { + var saver = new BinarySaver(Host, new BinarySaver.Arguments { Silent = true }); + DataSaverUtils.SaveDataView(ch, saver, new EmptyDataView(Host, TrainSchema), writer.BaseStream); + } + }); + + ctx.SaveString(_featureDetachedColumn.Name); + ctx.SaveStringOrNull(_treesColumnName); + ctx.SaveStringOrNull(_leavesColumnName); + ctx.SaveStringOrNull(_pathsColumnName); + } + + private static VersionInfo GetVersionInfo() + { + return new VersionInfo( + modelSignature: "TREEFEAT", // "TREE" ensemble "FEAT"urizer. + verWrittenCur: 0x00010001, // Initial + verReadableCur: 0x00010001, + verWeCanReadBack: 0x00010001, + loaderSignature: LoaderSignature, + loaderAssemblyName: typeof(TreeEnsembleFeaturizationTransformer).Assembly.FullName); + } + + private static TreeEnsembleFeaturizationTransformer Create(IHostEnvironment env, ModelLoadContext ctx) + => new TreeEnsembleFeaturizationTransformer(env, ctx); + } +} \ No newline at end of file diff --git a/src/Microsoft.ML.FastTree/TreeEnsembleFeaturizer.cs b/src/Microsoft.ML.FastTree/TreeEnsembleFeaturizer.cs index 38cbeda4f2..bbb69aa222 100644 --- a/src/Microsoft.ML.FastTree/TreeEnsembleFeaturizer.cs +++ b/src/Microsoft.ML.FastTree/TreeEnsembleFeaturizer.cs @@ -44,45 +44,65 @@ namespace Microsoft.ML.Data /// internal sealed class TreeEnsembleFeaturizerBindableMapper : ISchemaBindableMapper, ICanSaveModel { - public static class OutputColumnNames - { - public const string Trees = "Trees"; - public const string Paths = "Paths"; - public const string Leaves = "Leaves"; - } - + /// + /// In addition to options inherited from , + /// adds output columns' names of tree-based featurizer. + /// public sealed class Arguments : ScorerArgumentsBase - { - } - - private sealed class BoundMapper : ISchemaBoundRowMapper { /// - /// Column index of values predicted by all trees in an ensemble in . + /// See . /// - private const int TreeValuesColumnId = 0; + public string TreesColumnName; /// - /// Column index of leaf IDs containing the considered example in . + /// See . /// - private const int LeafIdsColumnId = 1; + public string LeavesColumnName; /// - /// Column index of path IDs which specify the paths the considered example passing through per tree in . + /// See . /// - private const int PathIdsColumnId = 2; - - private readonly TreeEnsembleFeaturizerBindableMapper _owner; - private readonly IExceptionContext _ectx; + public string PathsColumnName; + } + private sealed class BoundMapper : ISchemaBoundRowMapper + { public RoleMappedSchema InputRoleMappedSchema { get; } - public DataViewSchema InputSchema => InputRoleMappedSchema.Schema; public DataViewSchema OutputSchema { get; } + public ISchemaBindableMapper Bindable => _owner; + + private readonly TreeEnsembleFeaturizerBindableMapper _owner; + private readonly IExceptionContext _ectx; + + /// + /// Feature vector to be mapped to tree-based features. + /// private DataViewSchema.Column FeatureColumn => InputRoleMappedSchema.Feature.Value; - public ISchemaBindableMapper Bindable => _owner; + /// + /// The name of the column that stores the prediction values of all trees. Its type is a vector of + /// and the i-th vector element is the prediction value predicted by the i-th tree. + /// If is , this output column may not be generated. + /// + private string _treesColumnName; + + /// + /// The 0-1 encoding of all leaf nodes' IDs. Its type is a vector of . If the given feature + /// vector falls into the first leaf of the first tree, the first element in the 0-1 encoding would be 1. + /// If is , this output column may not be generated. + /// + private string _leavesColumnName; - public BoundMapper(IExceptionContext ectx, TreeEnsembleFeaturizerBindableMapper owner, - RoleMappedSchema schema) + /// + /// The 0-1 encoding of the paths to the leaves. If the path to the first tree's leaf is node 1 (2nd node in the first tree), + /// node 3 (4th node in the first tree), and node 5 (6th node in the first tree), the 2nd, 4th, and 6th element in that encoding + /// would be 1. + /// If is , this output column may not be generated. + /// + private string _pathsColumnName; + + public BoundMapper(IExceptionContext ectx, TreeEnsembleFeaturizerBindableMapper owner, RoleMappedSchema schema, + string treesColumnName, string leavesColumnName, string pathsColumnName) { Contracts.AssertValue(ectx); ectx.AssertValue(owner); @@ -111,37 +131,45 @@ public BoundMapper(IExceptionContext ectx, TreeEnsembleFeaturizerBindableMapper // Start creating output schema with types derived above. var schemaBuilder = new DataViewSchema.Builder(); - // Metadata of tree values. - var treeIdMetadataBuilder = new DataViewSchema.Annotations.Builder(); - treeIdMetadataBuilder.Add(AnnotationUtils.Kinds.SlotNames, AnnotationUtils.GetNamesType(treeValueType.Size), - (ValueGetter>>)owner.GetTreeSlotNames); - // Add the column of trees' output values - schemaBuilder.AddColumn(OutputColumnNames.Trees, treeValueType, treeIdMetadataBuilder.ToAnnotations()); - - // Metadata of leaf IDs. - var leafIdMetadataBuilder = new DataViewSchema.Annotations.Builder(); - leafIdMetadataBuilder.Add(AnnotationUtils.Kinds.SlotNames, AnnotationUtils.GetNamesType(leafIdType.Size), - (ValueGetter>>)owner.GetLeafSlotNames); - leafIdMetadataBuilder.Add(AnnotationUtils.Kinds.IsNormalized, BooleanDataViewType.Instance, (ref bool value) => value = true); - // Add the column of leaves' IDs where the input example reaches. - schemaBuilder.AddColumn(OutputColumnNames.Leaves, leafIdType, leafIdMetadataBuilder.ToAnnotations()); - - // Metadata of path IDs. - var pathIdMetadataBuilder = new DataViewSchema.Annotations.Builder(); - pathIdMetadataBuilder.Add(AnnotationUtils.Kinds.SlotNames, AnnotationUtils.GetNamesType(pathIdType.Size), - (ValueGetter>>)owner.GetPathSlotNames); - pathIdMetadataBuilder.Add(AnnotationUtils.Kinds.IsNormalized, BooleanDataViewType.Instance, (ref bool value) => value = true); - // Add the column of encoded paths which the input example passes. - schemaBuilder.AddColumn(OutputColumnNames.Paths, pathIdType, pathIdMetadataBuilder.ToAnnotations()); + _treesColumnName = treesColumnName; + if (treesColumnName != null) + { + // Metadata of tree values. + var treeIdMetadataBuilder = new DataViewSchema.Annotations.Builder(); + treeIdMetadataBuilder.Add(AnnotationUtils.Kinds.SlotNames, AnnotationUtils.GetNamesType(treeValueType.Size), + (ValueGetter>>)owner.GetTreeSlotNames); - OutputSchema = schemaBuilder.ToSchema(); + // Add the column of trees' output values + schemaBuilder.AddColumn(treesColumnName, treeValueType, treeIdMetadataBuilder.ToAnnotations()); + } - // Tree values must be the first output column. - Contracts.Assert(OutputSchema[OutputColumnNames.Trees].Index == TreeValuesColumnId); - // leaf IDs must be the second output column. - Contracts.Assert(OutputSchema[OutputColumnNames.Leaves].Index == LeafIdsColumnId); - // Path IDs must be the third output column. - Contracts.Assert(OutputSchema[OutputColumnNames.Paths].Index == PathIdsColumnId); + _leavesColumnName = leavesColumnName; + if (leavesColumnName != null) + { + // Metadata of leaf IDs. + var leafIdMetadataBuilder = new DataViewSchema.Annotations.Builder(); + leafIdMetadataBuilder.Add(AnnotationUtils.Kinds.SlotNames, AnnotationUtils.GetNamesType(leafIdType.Size), + (ValueGetter>>)owner.GetLeafSlotNames); + leafIdMetadataBuilder.Add(AnnotationUtils.Kinds.IsNormalized, BooleanDataViewType.Instance, (ref bool value) => value = true); + + // Add the column of leaves' IDs where the input example reaches. + schemaBuilder.AddColumn(leavesColumnName, leafIdType, leafIdMetadataBuilder.ToAnnotations()); + } + + _pathsColumnName = pathsColumnName; + if (pathsColumnName != null) + { + // Metadata of path IDs. + var pathIdMetadataBuilder = new DataViewSchema.Annotations.Builder(); + pathIdMetadataBuilder.Add(AnnotationUtils.Kinds.SlotNames, AnnotationUtils.GetNamesType(pathIdType.Size), + (ValueGetter>>)owner.GetPathSlotNames); + pathIdMetadataBuilder.Add(AnnotationUtils.Kinds.IsNormalized, BooleanDataViewType.Instance, (ref bool value) => value = true); + + // Add the column of encoded paths which the input example passes. + schemaBuilder.AddColumn(pathsColumnName, pathIdType, pathIdMetadataBuilder.ToAnnotations()); + } + + OutputSchema = schemaBuilder.ToSchema(); } DataViewRow ISchemaBoundRowMapper.GetRow(DataViewRow input, IEnumerable activeColumns) @@ -156,40 +184,41 @@ private Delegate[] CreateGetters(DataViewRow input, IEnumerable(); var activeIndices = activeColumns.Select(c => c.Index); - var treeValueActive = activeIndices.Contains(TreeValuesColumnId); - var leafIdActive = activeIndices.Contains(LeafIdsColumnId); - var pathIdActive = activeIndices.Contains(PathIdsColumnId); - - if (!treeValueActive && !leafIdActive && !pathIdActive) - return delegates; - var state = new State(_ectx, input, _owner._ensemble, _owner._totalLeafCount, FeatureColumn.Index); // Get the tree value getter. - if (treeValueActive) + if (_treesColumnName != null) { ValueGetter> fn = state.GetTreeValues; - delegates[TreeValuesColumnId] = fn; + if(activeIndices.Contains(OutputSchema[_treesColumnName].Index)) + delegates.Add(fn); + else + delegates.Add(null); } // Get the leaf indicator getter. - if (leafIdActive) + if (_leavesColumnName != null ) { ValueGetter> fn = state.GetLeafIds; - delegates[LeafIdsColumnId] = fn; + if (activeIndices.Contains(OutputSchema[_leavesColumnName].Index)) + delegates.Add(fn); + else + delegates.Add(null); } // Get the path indicators getter. - if (pathIdActive) + if (_pathsColumnName != null) { ValueGetter> fn = state.GetPathIds; - delegates[PathIdsColumnId] = fn; + if (activeIndices.Contains(OutputSchema[_pathsColumnName].Index)) + delegates.Add(fn); + else + delegates.Add(null); } - return delegates; + return delegates.ToArray(); } private sealed class State @@ -350,9 +379,10 @@ private static VersionInfo GetVersionInfo() return new VersionInfo( modelSignature: "TREEMAPR", // verWrittenCur: 0x00010001, // Initial - verWrittenCur: 0x00010002, // Add _defaultValueForMissing - verReadableCur: 0x00010002, - verWeCanReadBack: 0x00010001, + // verWrittenCur: 0x00010002, // Add _defaultValueForMissing + verWrittenCur: 0x00010003, // Add output column names (_treesColumnName, _leavesColumnName, _pathsColumnName) + verReadableCur: 0x00010003, + verWeCanReadBack: 0x00010002, loaderSignature: LoaderSignature, loaderAssemblyName: typeof(TreeEnsembleFeaturizerBindableMapper).Assembly.FullName); } @@ -360,6 +390,9 @@ private static VersionInfo GetVersionInfo() private readonly IHost _host; private readonly TreeEnsembleModelParameters _ensemble; private readonly int _totalLeafCount; + private readonly string _treesColumnName; + private readonly string _leavesColumnName; + private readonly string _pathsColumnName; public TreeEnsembleFeaturizerBindableMapper(IHostEnvironment env, Arguments args, IPredictor predictor) { @@ -368,6 +401,11 @@ public TreeEnsembleFeaturizerBindableMapper(IHostEnvironment env, Arguments args _host.CheckValue(args, nameof(args)); _host.CheckValue(predictor, nameof(predictor)); + // Store output columns specified by the user. + _treesColumnName = args.TreesColumnName; + _leavesColumnName = args.LeavesColumnName; + _pathsColumnName = args.PathsColumnName; + // This function accepts models trained by FastTreeTrainer family. There are four types that "predictor" can be. // 1. CalibratedPredictorBase // 2. FastTreeRankingModelParameters @@ -387,12 +425,19 @@ public TreeEnsembleFeaturizerBindableMapper(IHostEnvironment env, ModelLoadConte Contracts.CheckValue(env, nameof(env)); _host = env.Register(LoaderSignature); _host.AssertValue(ctx); + ctx.CheckAtModel(GetVersionInfo()); // *** Binary format *** // ensemble + // string: treesColumnName + // string: leavesColumnName + // string: pathsColumnName ctx.LoadModel(env, out _ensemble, "Ensemble"); _totalLeafCount = CountLeaves(_ensemble); + _treesColumnName = ctx.LoadStringOrNull(); + _leavesColumnName = ctx.LoadStringOrNull(); + _pathsColumnName = ctx.LoadStringOrNull(); } void ICanSaveModel.Save(ModelSaveContext ctx) @@ -403,9 +448,15 @@ void ICanSaveModel.Save(ModelSaveContext ctx) // *** Binary format *** // ensemble + // string: treesColumnName + // string: leavesColumnName + // string: pathsColumnName _host.AssertValue(_ensemble); ctx.SaveModel(_ensemble, "Ensemble"); + ctx.SaveStringOrNull(_treesColumnName); + ctx.SaveStringOrNull(_leavesColumnName); + ctx.SaveStringOrNull(_pathsColumnName); } private static int CountLeaves(TreeEnsembleModelParameters ensemble) @@ -474,7 +525,7 @@ ISchemaBoundMapper ISchemaBindableMapper.Bind(IHostEnvironment env, RoleMappedSc env.AssertValue(schema); env.CheckParam(schema.Feature != null, nameof(schema), "Need a feature column"); - return new BoundMapper(env, this, schema); + return new BoundMapper(env, this, schema, _treesColumnName, _leavesColumnName, _pathsColumnName); } } @@ -572,7 +623,8 @@ private static IDataTransform Create(IHostEnvironment env, Arguments args, IData IDataTransform xf; using (var ch = host.Start("Create Tree Ensemble Scorer")) { - var scorerArgs = new TreeEnsembleFeaturizerBindableMapper.Arguments() { Suffix = args.Suffix }; + var scorerArgs = new TreeEnsembleFeaturizerBindableMapper.Arguments() { + Suffix = args.Suffix, TreesColumnName = "Trees", LeavesColumnName = "Leaves", PathsColumnName = "Paths" }; if (!string.IsNullOrWhiteSpace(args.TrainedModelFile)) { if (args.Trainer != null) @@ -644,7 +696,8 @@ public static IDataTransform CreateForEntryPoint(IHostEnvironment env, Arguments using (var ch = host.Start("Create Tree Ensemble Scorer")) { - var scorerArgs = new TreeEnsembleFeaturizerBindableMapper.Arguments() { Suffix = args.Suffix }; + var scorerArgs = new TreeEnsembleFeaturizerBindableMapper.Arguments() { + Suffix = args.Suffix, TreesColumnName = "Trees", LeavesColumnName = "Leaves", PathsColumnName = "Paths" }; var predictor = args.PredictorModel.Predictor; ch.Trace("Prepare data"); RoleMappedData data = null; diff --git a/src/Microsoft.ML.FastTree/TreeTrainersCatalog.cs b/src/Microsoft.ML.FastTree/TreeTrainersCatalog.cs index 3909bca2ff..cdd4f5f87f 100644 --- a/src/Microsoft.ML.FastTree/TreeTrainersCatalog.cs +++ b/src/Microsoft.ML.FastTree/TreeTrainersCatalog.cs @@ -9,9 +9,9 @@ namespace Microsoft.ML { /// - /// Collection of extension methods used by , - /// , , - /// and to create instances of decision tree trainers. + /// Collection of extension methods used by , , + /// , , and + /// to create instances of decision tree trainers and featurizers. /// public static class TreeExtensions { @@ -427,7 +427,6 @@ public static FastForestBinaryTrainer FastForest(this BinaryClassificationCatalo /// ]]> /// /// - public static FastForestBinaryTrainer FastForest(this BinaryClassificationCatalog.BinaryClassificationTrainers catalog, FastForestBinaryTrainer.Options options) { @@ -437,5 +436,152 @@ public static FastForestBinaryTrainer FastForest(this BinaryClassificationCatalo var env = CatalogUtils.GetEnvironment(catalog); return new FastForestBinaryTrainer(env, options); } + + /// + /// Create , which produces tree-based features given a . + /// + /// The context to create . + /// The options to configure . See and + /// for available settings. + /// + /// + /// + /// + /// + public static PretrainedTreeFeaturizationEstimator FeaturizeByPretrainTreeEnsemble(this TransformsCatalog catalog, + PretrainedTreeFeaturizationEstimator.Options options) + { + Contracts.CheckValue(catalog, nameof(catalog)); + var env = CatalogUtils.GetEnvironment(catalog); + return new PretrainedTreeFeaturizationEstimator(env, options); + } + + /// + /// Create , which uses to train to create tree-based features. + /// + /// The context to create . + /// The options to configure . See and + /// for available settings. + /// + /// + /// + /// + /// + public static FastForestRegressionFeaturizationEstimator FeaturizeByFastForestRegression(this TransformsCatalog catalog, + FastForestRegressionFeaturizationEstimator.Options options) + { + Contracts.CheckValue(catalog, nameof(catalog)); + var env = CatalogUtils.GetEnvironment(catalog); + return new FastForestRegressionFeaturizationEstimator(env, options); + } + + /// + /// Create , which uses to train to create tree-based features. + /// + /// The context to create . + /// The options to configure . See and + /// for available settings. + /// + /// + /// + /// + /// + public static FastTreeRegressionFeaturizationEstimator FeaturizeByFastTreeRegression(this TransformsCatalog catalog, + FastTreeRegressionFeaturizationEstimator.Options options) + { + Contracts.CheckValue(catalog, nameof(catalog)); + var env = CatalogUtils.GetEnvironment(catalog); + return new FastTreeRegressionFeaturizationEstimator(env, options); + } + + /// + /// Create , which uses to train to create tree-based features. + /// + /// The context to create . + /// The options to configure . See and + /// for available settings. + /// + /// + /// + /// + /// + public static FastForestBinaryFeaturizationEstimator FeaturizeByFastForestBinary(this TransformsCatalog catalog, + FastForestBinaryFeaturizationEstimator.Options options) + { + Contracts.CheckValue(catalog, nameof(catalog)); + var env = CatalogUtils.GetEnvironment(catalog); + return new FastForestBinaryFeaturizationEstimator(env, options); + } + + /// + /// Create , which uses to train to create tree-based features. + /// + /// The context to create . + /// The options to configure . See and + /// for available settings. + /// + /// + /// + /// + /// + public static FastTreeBinaryFeaturizationEstimator FeaturizeByFastTreeBinary(this TransformsCatalog catalog, + FastTreeBinaryFeaturizationEstimator.Options options) + { + Contracts.CheckValue(catalog, nameof(catalog)); + var env = CatalogUtils.GetEnvironment(catalog); + return new FastTreeBinaryFeaturizationEstimator(env, options); + } + + /// + /// Create , which uses to train to create tree-based features. + /// + /// The context to create . + /// The options to configure . See and + /// for available settings. + /// + /// + /// + /// + /// + public static FastTreeRankingFeaturizationEstimator FeaturizeByFastTreeRanking(this TransformsCatalog catalog, + FastTreeRankingFeaturizationEstimator.Options options) + { + Contracts.CheckValue(catalog, nameof(catalog)); + var env = CatalogUtils.GetEnvironment(catalog); + return new FastTreeRankingFeaturizationEstimator(env, options); + } + + /// + /// Create , which uses to train to create tree-based features. + /// + /// The context to create . + /// The options to configure . See and + /// for available settings. + /// + /// + /// + /// + /// + public static FastTreeTweedieFeaturizationEstimator FeaturizeByFastTreeTweedie(this TransformsCatalog catalog, + FastTreeTweedieFeaturizationEstimator.Options options) + { + Contracts.CheckValue(catalog, nameof(catalog)); + var env = CatalogUtils.GetEnvironment(catalog); + return new FastTreeTweedieFeaturizationEstimator(env, options); + } } } diff --git a/src/Microsoft.ML.FastTree/Utils/RegressionTreeBaseUtils.cs b/src/Microsoft.ML.FastTree/Utils/RegressionTreeBaseUtils.cs new file mode 100644 index 0000000000..042de66e02 --- /dev/null +++ b/src/Microsoft.ML.FastTree/Utils/RegressionTreeBaseUtils.cs @@ -0,0 +1,147 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.ML.Data; +using Microsoft.ML.Runtime; +using Microsoft.ML.Trainers.FastTree; + +namespace Microsoft.ML.FastTree.Utils +{ + internal class RegressionTreeBaseUtils + { + /// + /// Utility method used to represent a tree ensemble as an . + /// Every row in the corresponds to a node in the tree ensemble. The columns are the fields for each node. + /// The column TreeID specifies which tree the node belongs to. The gets + /// special treatment since it has some additional fields ( + /// and ). + /// + public static IDataView RegressionTreeEnsembleAsIDataView(IHost host, double bias, IReadOnlyList treeWeights, IReadOnlyList trees) + { + var builder = new ArrayDataViewBuilder(host); + var numberOfRows = trees.Select(tree => tree.NumberOfNodes).Sum() + trees.Select(tree => tree.NumberOfLeaves).Sum(); + + var treeWeightsList = new List(); + var treeId = new List(); + var isLeaf = new List>(); + var leftChild = new List(); + var rightChild = new List(); + var numericalSplitFeatureIndexes = new List(); + var numericalSplitThresholds = new List(); + var categoricalSplitFlags = new List(); + var leafValues = new List(); + var splitGains = new List(); + var categoricalSplitFeatures = new List>(); + var categoricalCategoricalSplitFeatureRange = new List>(); + + for (int i = 0; i < trees.Count; i++) + { + // TreeWeights column. The TreeWeight value will be repeated for all the notes in the same tree in the IDataView. + treeWeightsList.AddRange(Enumerable.Repeat(treeWeights[i], trees[i].NumberOfNodes + trees[i].NumberOfLeaves)); + + // Tree id indicates which tree the node belongs to. + treeId.AddRange(Enumerable.Repeat(i, trees[i].NumberOfNodes + trees[i].NumberOfLeaves)); + + // IsLeaf column indicates if node is a leaf node. + isLeaf.AddRange(Enumerable.Repeat(new ReadOnlyMemory("Tree node".ToCharArray()), trees[i].NumberOfNodes)); + isLeaf.AddRange(Enumerable.Repeat(new ReadOnlyMemory("Leaf node".ToCharArray()), trees[i].NumberOfLeaves)); + + // LeftChild column. + leftChild.AddRange(trees[i].LeftChild.AsEnumerable()); + leftChild.AddRange(Enumerable.Repeat(0, trees[i].NumberOfLeaves)); + + // RightChild column. + rightChild.AddRange(trees[i].RightChild.AsEnumerable()); + rightChild.AddRange(Enumerable.Repeat(0, trees[i].NumberOfLeaves)); + + // NumericalSplitFeatureIndexes column. + numericalSplitFeatureIndexes.AddRange(trees[i].NumericalSplitFeatureIndexes.AsEnumerable()); + numericalSplitFeatureIndexes.AddRange(Enumerable.Repeat(0, trees[i].NumberOfLeaves)); + + // NumericalSplitThresholds column. + numericalSplitThresholds.AddRange(trees[i].NumericalSplitThresholds.AsEnumerable()); + numericalSplitThresholds.AddRange(Enumerable.Repeat(0f, trees[i].NumberOfLeaves)); + + // CategoricalSplitFlags column. + categoricalSplitFlags.AddRange(trees[i].CategoricalSplitFlags.AsEnumerable()); + categoricalSplitFlags.AddRange(Enumerable.Repeat(false, trees[i].NumberOfLeaves)); + + // LeafValues column. + leafValues.AddRange(Enumerable.Repeat(0d, trees[i].NumberOfNodes)); + leafValues.AddRange(trees[i].LeafValues.AsEnumerable()); + + // SplitGains column. + splitGains.AddRange(trees[i].SplitGains.AsEnumerable()); + splitGains.AddRange(Enumerable.Repeat(0d, trees[i].NumberOfLeaves)); + + for (int j = 0; j < trees[i].NumberOfNodes; j++) + { + // CategoricalSplitFeatures column. + var categoricalSplitFeaturesArray = trees[i].GetCategoricalSplitFeaturesAt(j).ToArray(); + categoricalSplitFeatures.Add(new VBuffer(categoricalSplitFeaturesArray.Length, categoricalSplitFeaturesArray)); + var len = trees[i].GetCategoricalSplitFeaturesAt(j).ToArray().Length; + + // CategoricalCategoricalSplitFeatureRange column. + var categoricalCategoricalSplitFeatureRangeArray = trees[i].GetCategoricalCategoricalSplitFeatureRangeAt(j).ToArray(); + categoricalCategoricalSplitFeatureRange.Add(new VBuffer(categoricalCategoricalSplitFeatureRangeArray.Length, categoricalCategoricalSplitFeatureRangeArray)); + len = trees[i].GetCategoricalCategoricalSplitFeatureRangeAt(j).ToArray().Length; + } + + categoricalSplitFeatures.AddRange(Enumerable.Repeat(new VBuffer(), trees[i].NumberOfLeaves)); + categoricalCategoricalSplitFeatureRange.AddRange(Enumerable.Repeat(new VBuffer(), trees[i].NumberOfLeaves)); + } + + // Bias column. This will be a repeated value for all rows in the resulting IDataView. + builder.AddColumn("Bias", NumberDataViewType.Double, Enumerable.Repeat(bias, numberOfRows).ToArray()); + builder.AddColumn("TreeWeights", NumberDataViewType.Double, treeWeightsList.ToArray()); + builder.AddColumn("TreeID", NumberDataViewType.Int32, treeId.ToArray()); + builder.AddColumn("IsLeaf", TextDataViewType.Instance, isLeaf.ToArray()); + builder.AddColumn(nameof(RegressionTreeBase.LeftChild), NumberDataViewType.Int32, leftChild.ToArray()); + builder.AddColumn(nameof(RegressionTreeBase.RightChild), NumberDataViewType.Int32, rightChild.ToArray()); + builder.AddColumn(nameof(RegressionTreeBase.NumericalSplitFeatureIndexes), NumberDataViewType.Int32, numericalSplitFeatureIndexes.ToArray()); + builder.AddColumn(nameof(RegressionTreeBase.NumericalSplitThresholds), NumberDataViewType.Single, numericalSplitThresholds.ToArray()); + builder.AddColumn(nameof(RegressionTreeBase.CategoricalSplitFlags), BooleanDataViewType.Instance, categoricalSplitFlags.ToArray()); + builder.AddColumn(nameof(RegressionTreeBase.LeafValues), NumberDataViewType.Double, leafValues.ToArray()); + builder.AddColumn(nameof(RegressionTreeBase.SplitGains), NumberDataViewType.Double, splitGains.ToArray()); + builder.AddColumn("CategoricalSplitFeatures", NumberDataViewType.Int32, categoricalSplitFeatures.ToArray()); + builder.AddColumn("CategoricalCategoricalSplitFeatureRange", NumberDataViewType.Int32, categoricalCategoricalSplitFeatureRange.ToArray()); + + // If the input tree array is a quantile regression tree we need to add two more columns. + var quantileTrees = trees as IReadOnlyList; + if (quantileTrees != null) + { + // LeafSamples column. + var leafSamples = new List>(); + + // LeafSampleWeights column. + var leafSampleWeights = new List>(); + for (int i = 0; i < quantileTrees.Count; i++) + { + leafSamples.AddRange(Enumerable.Repeat(new VBuffer(), quantileTrees[i].NumberOfNodes)); + leafSampleWeights.AddRange(Enumerable.Repeat(new VBuffer(), quantileTrees[i].NumberOfNodes)); + for (int j = 0; j < quantileTrees[i].NumberOfLeaves; j++) + { + var leafSamplesArray = quantileTrees[i].GetLeafSamplesAt(j).ToArray(); + leafSamples.Add(new VBuffer(leafSamplesArray.Length, leafSamplesArray)); + var len = quantileTrees[i].GetLeafSamplesAt(j).ToArray().Length; + + var leafSampleWeightsArray = quantileTrees[i].GetLeafSampleWeightsAt(j).ToArray(); + leafSampleWeights.Add(new VBuffer(leafSampleWeightsArray.Length, leafSampleWeightsArray)); + len = quantileTrees[i].GetLeafSampleWeightsAt(j).ToArray().Length; + } + } + + builder.AddColumn("LeafSamples", NumberDataViewType.Double, leafSamples.ToArray()); + builder.AddColumn("LeafSampleWeights", NumberDataViewType.Double, leafSampleWeights.ToArray()); + } + + var data = builder.GetDataView(); + return data; + } + + } +} diff --git a/src/Microsoft.ML.ImageAnalytics/ExtensionsCatalog.cs b/src/Microsoft.ML.ImageAnalytics/ExtensionsCatalog.cs index 78cbc74874..60e92e79a2 100644 --- a/src/Microsoft.ML.ImageAnalytics/ExtensionsCatalog.cs +++ b/src/Microsoft.ML.ImageAnalytics/ExtensionsCatalog.cs @@ -118,6 +118,7 @@ internal static ImageLoadingEstimator LoadImages(this TransformsCatalog catalog, /// /// /// public static ImagePixelExtractingEstimator ExtractPixels(this TransformsCatalog catalog, diff --git a/src/Microsoft.ML.LightGbm/LightGbmArguments.cs b/src/Microsoft.ML.LightGbm/LightGbmArguments.cs index a05d57fd1b..765b0d297a 100644 --- a/src/Microsoft.ML.LightGbm/LightGbmArguments.cs +++ b/src/Microsoft.ML.LightGbm/LightGbmArguments.cs @@ -58,7 +58,6 @@ public BoosterParameterBase(OptionsBase options) public abstract class OptionsBase : IBoosterParameterFactory { internal BoosterParameterBase GetBooster() { return null; } - /// /// The minimum loss reduction required to make a further partition on a leaf node of the tree. /// diff --git a/src/Microsoft.ML.LightGbm/LightGbmBinaryTrainer.cs b/src/Microsoft.ML.LightGbm/LightGbmBinaryTrainer.cs index 22de1c03f7..568e5a0ddb 100644 --- a/src/Microsoft.ML.LightGbm/LightGbmBinaryTrainer.cs +++ b/src/Microsoft.ML.LightGbm/LightGbmBinaryTrainer.cs @@ -162,7 +162,7 @@ public enum EvaluateMetricType [Argument(ArgumentType.AtMostOnce, HelpText = "Evaluation metrics.", ShortName = "em")] - public EvaluateMetricType EvaluationMetric = EvaluateMetricType.Logloss; + public EvaluateMetricType EvaluationMetric = EvaluateMetricType.Default; static Options() { diff --git a/src/Microsoft.ML.LightGbm/LightGbmMulticlassTrainer.cs b/src/Microsoft.ML.LightGbm/LightGbmMulticlassTrainer.cs index 29f19f6b2c..353856e7bd 100644 --- a/src/Microsoft.ML.LightGbm/LightGbmMulticlassTrainer.cs +++ b/src/Microsoft.ML.LightGbm/LightGbmMulticlassTrainer.cs @@ -75,6 +75,12 @@ public enum EvaluateMetricType LogLoss, } + /// + /// Whether training data is unbalanced. + /// + [Argument(ArgumentType.AtMostOnce, HelpText = "Use for multi-class classification when training data is not balanced", ShortName = "us")] + public bool UnbalancedSets = false; + /// /// Whether to use softmax loss. /// @@ -95,7 +101,7 @@ public enum EvaluateMetricType [Argument(ArgumentType.AtMostOnce, HelpText = "Evaluation metrics.", ShortName = "em")] - public EvaluateMetricType EvaluationMetric = EvaluateMetricType.Error; + public EvaluateMetricType EvaluationMetric = EvaluateMetricType.Default; static Options() { @@ -110,6 +116,7 @@ internal override Dictionary ToDictionary(IHost host) { var res = base.ToDictionary(host); + res[GetOptionName(nameof(UnbalancedSets))] = UnbalancedSets; res[GetOptionName(nameof(Sigmoid))] = Sigmoid; res[GetOptionName(nameof(EvaluateMetricType))] = GetOptionName(EvaluationMetric.ToString()); diff --git a/src/Microsoft.ML.LightGbm/LightGbmRankingTrainer.cs b/src/Microsoft.ML.LightGbm/LightGbmRankingTrainer.cs index 12cc0b148e..85dd19f65b 100644 --- a/src/Microsoft.ML.LightGbm/LightGbmRankingTrainer.cs +++ b/src/Microsoft.ML.LightGbm/LightGbmRankingTrainer.cs @@ -143,7 +143,7 @@ public enum EvaluateMetricType [Argument(ArgumentType.AtMostOnce, HelpText = "Evaluation metrics.", ShortName = "em")] - public EvaluateMetricType EvaluationMetric = EvaluateMetricType.NormalizedDiscountedCumulativeGain; + public EvaluateMetricType EvaluationMetric = EvaluateMetricType.Default; static Options() { diff --git a/src/Microsoft.ML.LightGbm/LightGbmRegressionTrainer.cs b/src/Microsoft.ML.LightGbm/LightGbmRegressionTrainer.cs index 90ad8cfa39..618b04274e 100644 --- a/src/Microsoft.ML.LightGbm/LightGbmRegressionTrainer.cs +++ b/src/Microsoft.ML.LightGbm/LightGbmRegressionTrainer.cs @@ -133,7 +133,7 @@ public enum EvaluateMetricType [Argument(ArgumentType.AtMostOnce, HelpText = "Evaluation metrics.", ShortName = "em")] - public EvaluateMetricType EvaluationMetric = EvaluateMetricType.RootMeanSquaredError; + public EvaluateMetricType EvaluationMetric = EvaluateMetricType.Default; static Options() { diff --git a/src/Microsoft.ML.OnnxTransformer/Microsoft.ML.OnnxTransformer.csproj b/src/Microsoft.ML.OnnxTransformer/Microsoft.ML.OnnxTransformer.csproj index ff5388d52b..d2f51a9429 100644 --- a/src/Microsoft.ML.OnnxTransformer/Microsoft.ML.OnnxTransformer.csproj +++ b/src/Microsoft.ML.OnnxTransformer/Microsoft.ML.OnnxTransformer.csproj @@ -10,6 +10,12 @@ + + + + OnnxMl.cs + + diff --git a/src/Microsoft.ML.OnnxTransformer/OnnxCatalog.cs b/src/Microsoft.ML.OnnxTransformer/OnnxCatalog.cs index 40717ec590..c6136ed2e6 100644 --- a/src/Microsoft.ML.OnnxTransformer/OnnxCatalog.cs +++ b/src/Microsoft.ML.OnnxTransformer/OnnxCatalog.cs @@ -45,6 +45,13 @@ public static OnnxScoringEstimator ApplyOnnxModel(this TransformsCatalog catalog /// The path of the file containing the ONNX model. /// Optional GPU device ID to run execution on, to run on CPU. /// If GPU error, raise exception or fallback to CPU. + /// + /// + /// + /// + /// public static OnnxScoringEstimator ApplyOnnxModel(this TransformsCatalog catalog, string outputColumnName, string inputColumnName, diff --git a/src/Microsoft.ML.OnnxTransformer/OnnxMapType.cs b/src/Microsoft.ML.OnnxTransformer/OnnxMapType.cs new file mode 100644 index 0000000000..028fd3f8df --- /dev/null +++ b/src/Microsoft.ML.OnnxTransformer/OnnxMapType.cs @@ -0,0 +1,101 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using Microsoft.ML.Data; +using Microsoft.ML.Internal.Utilities; + +namespace Microsoft.ML.Transforms.Onnx +{ + /// + /// The corresponding of ONNX's map type in 's type system. + /// In other words, if an ONNX model produces a map, a column in may be typed to . + /// Its underlying type is , where the generic type "TKey" and "TValue" are the input arguments of + /// . + /// + public sealed class OnnxMapType : StructuredDataViewType + { + /// + /// Create the corresponding for ONNX map. + /// + /// Key type of the associated ONNX map. + /// Value type of the associated ONNX map. + public OnnxMapType(Type keyType, Type valueType) : base(typeof(IDictionary<,>).MakeGenericType(keyType, valueType)) + { + DataViewTypeManager.Register(this, RawType, new[] { new OnnxMapTypeAttribute(keyType, valueType) }); + } + + public override bool Equals(DataViewType other) + { + if (other is OnnxMapType) + return RawType == other.RawType; + else + return false; + } + + public override int GetHashCode() + { + return RawType.GetHashCode(); + } + } + + /// + /// To declare column in as a field + /// in a , the associated field should be marked with . + /// Its uses are similar to those of and other es derived + /// from . + /// + public sealed class OnnxMapTypeAttribute : DataViewTypeAttribute + { + private Type _keyType; + private Type _valueType; + + /// + /// Create a map (aka dictionary) type. + /// + public OnnxMapTypeAttribute() + { + } + + /// + /// Create a map (aka dictionary) type. A map is a collection of key-value + /// pairs. specifies the type of keys and + /// is the type of values. + /// + public OnnxMapTypeAttribute(Type keyType, Type valueType) + { + _keyType = keyType; + _valueType = valueType; + } + + /// + /// Map types with the same key type and the same value type should be equal. + /// + public override bool Equals(DataViewTypeAttribute other) + { + if (other is OnnxMapTypeAttribute otherSequence) + return _keyType.Equals(otherSequence._keyType) && _valueType.Equals(otherSequence._valueType); + return false; + } + + /// + /// Produce the same hash code for map types with the same key type and the same value type. + /// + public override int GetHashCode() + { + return Hashing.CombineHash(_keyType.GetHashCode(), _valueType.GetHashCode()); + } + + /// + /// An implementation of . + /// + public override void Register() + { + var enumerableType = typeof(IDictionary<,>); + var type = enumerableType.MakeGenericType(_keyType, _valueType); + DataViewTypeManager.Register(new OnnxMapType(_keyType, _valueType), type, new[] { this }); + } + } +} diff --git a/src/Microsoft.ML.OnnxTransformer/OnnxSequenceType.cs b/src/Microsoft.ML.OnnxTransformer/OnnxSequenceType.cs new file mode 100644 index 0000000000..acfca70e47 --- /dev/null +++ b/src/Microsoft.ML.OnnxTransformer/OnnxSequenceType.cs @@ -0,0 +1,102 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using Microsoft.ML.Data; + +namespace Microsoft.ML.Transforms.Onnx +{ + /// + /// The corresponding of ONNX's sequence type in 's type system. + /// In other words, if an ONNX model produces a sequence, a column in may be typed to . + /// Its underlying type is , where the generic type "T" is the input argument of + /// . + /// + public sealed class OnnxSequenceType : StructuredDataViewType + { + private static Type MakeNativeType(Type elementType) + { + var enumerableTypeInfo = typeof(IEnumerable<>); + var enumerableType = enumerableTypeInfo.MakeGenericType(elementType); + return enumerableType; + } + + /// + /// Create the corresponding for ONNX sequence. + /// + /// The element type of a sequence. + public OnnxSequenceType(Type elementType) : base(MakeNativeType(elementType)) + { + DataViewTypeManager.Register(this, RawType, new[] { new OnnxSequenceTypeAttribute(elementType) }); + } + + public override bool Equals(DataViewType other) + { + if (other is OnnxSequenceType) + return RawType == other.RawType; + else + return false; + } + + public override int GetHashCode() + { + return RawType.GetHashCode(); + } + } + + /// + /// To declare column in as a field + /// in a , the associated field should be marked with . + /// Its uses are similar to those of and other es derived + /// from . + /// + public sealed class OnnxSequenceTypeAttribute : DataViewTypeAttribute + { + private Type _elemType; + + /// + /// Create a sequence type. + /// + public OnnxSequenceTypeAttribute() + { + } + + /// + /// Create a -sequence type. + /// + public OnnxSequenceTypeAttribute(Type elemType) + { + _elemType = elemType; + } + + /// + /// Sequence types with the same element type should be equal. + /// + public override bool Equals(DataViewTypeAttribute other) + { + if (other is OnnxSequenceTypeAttribute otherSequence) + return _elemType.Equals(otherSequence._elemType); + return false; + } + + /// + /// Produce the same hash code for sequence types with the same element type. + /// + public override int GetHashCode() + { + return _elemType.GetHashCode(); + } + + /// + /// An implementation of . + /// + public override void Register() + { + var enumerableType = typeof(IEnumerable<>); + var type = enumerableType.MakeGenericType(_elemType); + DataViewTypeManager.Register(new OnnxSequenceType(_elemType), type, new[] { this }); + } + } +} diff --git a/src/Microsoft.ML.OnnxTransformer/OnnxTransform.cs b/src/Microsoft.ML.OnnxTransformer/OnnxTransform.cs index 90f5bcad45..2b09fff813 100644 --- a/src/Microsoft.ML.OnnxTransformer/OnnxTransform.cs +++ b/src/Microsoft.ML.OnnxTransformer/OnnxTransform.cs @@ -46,7 +46,7 @@ namespace Microsoft.ML.Transforms.Onnx /// | Output column data type | The same data type as the input column | /// | Required NuGet in addition to Microsoft.ML | Microsoft.ML.OnnxTransformer | /// - /// Supports inferencing of models in ONNX 1.2 and 1.3 format (opset 7, 8 and 9), using the + /// Supports inferencing of models in ONNX 1.2, 1.3, 1.4, and 1.5 format (opset 7, 8, 9, and 10), using the /// [Microsoft.ML.OnnxRuntime](https://www.nuget.org/packages/Microsoft.ML.OnnxRuntime/) library. /// Models are scored on CPU by default. If GPU execution is needed (optional), use the /// NuGet package available at [Microsoft.ML.OnnxRuntime.Gpu](https://www.nuget.org/packages/Microsoft.ML.OnnxRuntime.Gpu/) @@ -85,6 +85,7 @@ internal sealed class Options : TransformInputBase } private readonly Options _options; + // This field is internal because the associated estimator may access it. internal readonly OnnxModel Model; internal const string Summary = "Transforms the data using the Onnx model."; @@ -92,9 +93,22 @@ internal sealed class Options : TransformInputBase internal const string ShortName = "Onnx"; internal const string LoaderSignature = "OnnxTransform"; - internal readonly string[] Inputs; - internal readonly string[] Outputs; - internal readonly DataViewType[] OutputTypes; + /// + /// Input column names from ML.NET's perspective. It can be ordered differently than ONNX model's input list. + /// It's also possible that the contains less variables than ONNX model's input list. + /// For each name in , an input tensor with the same name can be found in the underlying ONNX model. + /// + internal string[] Inputs { get; } + /// + /// Output column names from ML.NET's perspective. It can be ordered differently than ONNX model's output list. + /// It's also possible that the contains less variables than ONNX model's output list. + /// For each name in , an output tensor with the same name can be found in the underlying ONNX model. + /// + internal string[] Outputs { get; } + /// + /// Types of . The i-th element is the type of the i-th output in . + /// + internal DataViewType[] OutputTypes { get; } private static VersionInfo GetVersionInfo() { @@ -165,16 +179,25 @@ private OnnxTransformer(IHostEnvironment env, Options options, byte[] modelBytes foreach (var col in options.OutputColumns) Host.CheckNonWhiteSpace(col, nameof(options.OutputColumns)); + // Use ONNXRuntime to figure out the right input and output configuration. + // However, ONNXRuntime doesn't provide strongly-typed method to access the produced + // variables, we will inspect the ONNX model file to get information regarding types. try { if (modelBytes == null) { + // Entering this region means that the model file is passed in by the user. Host.CheckNonWhiteSpace(options.ModelFile, nameof(options.ModelFile)); Host.CheckIO(File.Exists(options.ModelFile), "Model file {0} does not exists.", options.ModelFile); - Model = new OnnxModel(options.ModelFile, options.GpuDeviceId, options.FallbackToCpu); + // Because we cannot delete the user file, ownModelFile should be false. + Model = new OnnxModel(options.ModelFile, options.GpuDeviceId, options.FallbackToCpu, ownModelFile: false); } else + { + // Entering this region means that the byte[] is passed as the model. To feed that byte[] to ONNXRuntime, we need + // to create a temporal file to store it and then call ONNXRuntime's API to load that file. Model = OnnxModel.CreateFromBytes(modelBytes, options.GpuDeviceId, options.FallbackToCpu); + } } catch (OnnxRuntimeException e) { @@ -182,20 +205,14 @@ private OnnxTransformer(IHostEnvironment env, Options options, byte[] modelBytes } var modelInfo = Model.ModelInfo; - Inputs = (options.InputColumns.Count() == 0) ? Model.InputNames.ToArray() : options.InputColumns; - Outputs = (options.OutputColumns.Count() == 0) ? Model.OutputNames.ToArray() : options.OutputColumns; + Inputs = (options.InputColumns.Count() == 0) ? Model.ModelInfo.InputNames.ToArray() : options.InputColumns; + Outputs = (options.OutputColumns.Count() == 0) ? Model.ModelInfo.OutputNames.ToArray() : options.OutputColumns; OutputTypes = new DataViewType[Outputs.Length]; var numModelOutputs = Model.ModelInfo.OutputsInfo.Length; for (int i = 0; i < Outputs.Length; i++) { - var idx = Model.OutputNames.IndexOf(Outputs[i]); - if (idx < 0) - throw Host.Except($"Column {Outputs[i]} doesn't match output node names of model"); - - var outputNodeInfo = Model.ModelInfo.OutputsInfo[idx]; - var shape = outputNodeInfo.Shape; - var dims = AdjustDimensions(shape); - OutputTypes[i] = new VectorDataViewType(OnnxUtils.OnnxToMlNetType(outputNodeInfo.Type), dims.ToArray()); + var outputInfo = Model.ModelInfo.GetOutput(Outputs[i]); + OutputTypes[i] = outputInfo.DataViewType; } _options = options; } @@ -272,7 +289,7 @@ private protected override void SaveModel(ModelSaveContext ctx) ctx.CheckAtModel(); ctx.SetVersionInfo(GetVersionInfo()); - ctx.SaveBinaryStream("OnnxModel", w => { w.WriteByteArray(Model.ToByteArray()); }); + ctx.SaveBinaryStream("OnnxModel", w => { w.WriteByteArray(File.ReadAllBytes(Model.ModelFile)); }); Host.CheckNonEmpty(Inputs, nameof(Inputs)); ctx.Writer.Write(Inputs.Length); @@ -286,11 +303,12 @@ private protected override void SaveModel(ModelSaveContext ctx) } private protected override IRowMapper MakeRowMapper(DataViewSchema inputSchema) => new Mapper(this, inputSchema); + /// + /// This design assumes that all unknown dimensions are 1s. It also convert scalar shape [] in ONNX to [1]. + /// [TODO] We should infer the unknown shape from input data instead of forcing them to be 1. + /// private static IEnumerable AdjustDimensions(OnnxShape shape) { - // if the model output is of type Map or Sequence, the shape property - // will not be filled (so count=0). Don't throw an exception here - // it will be runtime exception, util Maps and Sequences become supported. if (shape.Count > 0) { return shape.Select(x => (x <= 0) ? 1 : x); @@ -301,10 +319,19 @@ private static IEnumerable AdjustDimensions(OnnxShape shape) private sealed class Mapper : MapperBase { private readonly OnnxTransformer _parent; + /// + /// 's i-th element value tells the column index to + /// find the i-th ONNX input. + /// private readonly int[] _inputColIndices; - private readonly bool[] _isInputVector; + /// + /// 's i-th element value tells if the i-th ONNX input's shape if it's a tensor. + /// private readonly OnnxShape[] _inputTensorShapes; - private readonly System.Type[] _inputOnnxTypes; + /// + /// 's i-th element value tells if the of the i-th ONNX input. + /// + private readonly Type[] _inputOnnxTypes; public Mapper(OnnxTransformer parent, DataViewSchema inputSchema) : base(Contracts.CheckRef(parent, nameof(parent)).Host.Register(nameof(Mapper)), inputSchema, parent) @@ -312,41 +339,34 @@ public Mapper(OnnxTransformer parent, DataViewSchema inputSchema) : _parent = parent; _inputColIndices = new int[_parent.Inputs.Length]; - _isInputVector = new bool[_parent.Inputs.Length]; _inputTensorShapes = new OnnxShape[_parent.Inputs.Length]; - _inputOnnxTypes = new System.Type[_parent.Inputs.Length]; + _inputOnnxTypes = new Type[_parent.Inputs.Length]; var model = _parent.Model; for (int i = 0; i < _parent.Inputs.Length; i++) { - var idx = model.InputNames.IndexOf(_parent.Inputs[i]); - if (idx < 0) - throw Host.Except($"Column {_parent.Inputs[i]} doesn't match input node names of model"); - - var inputNodeInfo = model.ModelInfo.InputsInfo[idx]; + var inputNodeInfo = model.ModelInfo.GetInput(_parent.Inputs[i]); var shape = inputNodeInfo.Shape; - var inputType = OnnxUtils.OnnxToMlNetType(inputNodeInfo.Type); var inputShape = AdjustDimensions(inputNodeInfo.Shape); _inputTensorShapes[i] = inputShape.ToList(); - _inputOnnxTypes[i] = inputNodeInfo.Type; + _inputOnnxTypes[i] = inputNodeInfo.TypeInOnnxRuntime; var col = inputSchema.GetColumnOrNull(_parent.Inputs[i]); if (!col.HasValue) - throw Host.ExceptSchemaMismatch( nameof(inputSchema),"input", _parent.Inputs[i]); + throw Host.ExceptSchemaMismatch(nameof(inputSchema),"input", _parent.Inputs[i]); _inputColIndices[i] = col.Value.Index; var type = inputSchema[_inputColIndices[i]].Type; var vectorType = type as VectorDataViewType; - _isInputVector[i] = vectorType != null; if (vectorType != null && vectorType.Size == 0) throw Host.Except($"Variable length input columns not supported"); - if (type.GetItemType() != inputType) - throw Host.ExceptSchemaMismatch(nameof(inputSchema), "input", _parent.Inputs[i], inputType.ToString(), type.ToString()); + if (type.GetItemType() != inputNodeInfo.DataViewType.GetItemType()) + throw Host.ExceptSchemaMismatch(nameof(inputSchema), "input", _parent.Inputs[i], inputNodeInfo.DataViewType.GetItemType().ToString(), type.ToString()); // If the column is one dimension we make sure that the total size of the Onnx shape matches. // Compute the total size of the known dimensions of the shape. @@ -355,8 +375,6 @@ public Mapper(OnnxTransformer parent, DataViewSchema inputSchema) : int typeValueCount = type.GetValueCount(); if (typeValueCount % valCount != 0) throw Contracts.Except($"Input shape mismatch: Input '{_parent.Inputs[i]}' has shape {String.Join(",", inputShape)}, but input data is of length {typeValueCount}."); - - //Host.Assert(_outputItemRawType == _outputColType.ItemType.RawType); } } @@ -375,22 +393,42 @@ private protected override Func GetDependenciesCore(Func a private protected override void SaveModel(ModelSaveContext ctx) => _parent.SaveModel(ctx); - private interface INamedOnnxValueGetter + protected override Delegate MakeGetter(DataViewRow input, int iinfo, Func activeOutput, out Action disposer) { - NamedOnnxValue GetNamedOnnxValue(); + disposer = null; + Host.AssertValue(input); + + var activeOutputColNames = _parent.Outputs.Where((x, i) => activeOutput(i)).ToArray(); + + if (_parent.Model.ModelInfo.OutputsInfo[iinfo].DataViewType is VectorDataViewType vectorType) + { + var elemRawType = vectorType.ItemType.RawType; + var srcNamedValueGetters = GetNamedOnnxValueGetters(input, _inputColIndices, _inputOnnxTypes, _inputTensorShapes); + if (vectorType.ItemType is TextDataViewType) + return MakeStringTensorGetter(input, iinfo, srcNamedValueGetters, activeOutputColNames); + else + return Utils.MarshalInvoke(MakeTensorGetter, elemRawType, input, iinfo, srcNamedValueGetters, activeOutputColNames); + } + else + { + var type = _parent.Model.ModelInfo.OutputsInfo[iinfo].DataViewType.RawType; + var srcNamedValueGetters = GetNamedOnnxValueGetters(input, _inputColIndices, _inputOnnxTypes, _inputTensorShapes); + return Utils.MarshalInvoke(MakeObjectGetter, type, input, iinfo, srcNamedValueGetters, activeOutputColNames); + } } - private class OutputCache + + private class OnnxRuntimeOutputCacher { public long Position; public Dictionary Outputs; - public OutputCache() + public OnnxRuntimeOutputCacher() { Position = -1; Outputs = new Dictionary(); } } - private void UpdateCacheIfNeeded(long position, INamedOnnxValueGetter[] srcNamedOnnxValueGetters, string[] activeOutputColNames, OutputCache outputCache) + private void UpdateCacheIfNeeded(long position, INamedOnnxValueGetter[] srcNamedOnnxValueGetters, string[] activeOutputColNames, OnnxRuntimeOutputCacher outputCache) { if (outputCache.Position != position) { @@ -412,103 +450,174 @@ private void UpdateCacheIfNeeded(long position, INamedOnnxValueGetter[] srcNamed } } - protected override Delegate MakeGetter(DataViewRow input, int iinfo, Func activeOutput, out Action disposer) + private Delegate MakeTensorGetter(DataViewRow input, int iinfo, INamedOnnxValueGetter[] srcNamedValueGetters, string[] activeOutputColNames) { - disposer = null; Host.AssertValue(input); - //Host.Assert(typeof(T) == _outputItemRawType); + var outputCacher = new OnnxRuntimeOutputCacher(); + ValueGetter> valueGetter = (ref VBuffer dst) => + { + UpdateCacheIfNeeded(input.Position, srcNamedValueGetters, activeOutputColNames, outputCacher); + var namedOnnxValue = outputCacher.Outputs[_parent.Outputs[iinfo]]; + var tensor = namedOnnxValue.AsTensor() as System.Numerics.Tensors.DenseTensor; + if (tensor == null) + throw Host.Except($"Output column {namedOnnxValue.Name} doesn't contain a DenseTensor of expected type {typeof(T)}"); + var editor = VBufferEditor.Create(ref dst, (int)tensor.Length); + tensor.Buffer.Span.CopyTo(editor.Values); + dst = editor.Commit(); + }; + return valueGetter; + } - var outputCache = new OutputCache(); - var activeOutputColNames = _parent.Outputs.Where((x, i) => activeOutput(i)).ToArray(); - var type = OnnxUtils.OnnxToMlNetType(_parent.Model.ModelInfo.OutputsInfo[iinfo].Type).RawType; - Host.Assert(type == _parent.OutputTypes[iinfo].GetItemType().RawType); - var srcNamedValueGetters = GetNamedOnnxValueGetters(input, _parent.Inputs, _inputColIndices, _isInputVector, _inputOnnxTypes, _inputTensorShapes); - return Utils.MarshalInvoke(MakeGetter, type, input, iinfo, srcNamedValueGetters, activeOutputColNames, outputCache); + private Delegate MakeStringTensorGetter(DataViewRow input, int iinfo, INamedOnnxValueGetter[] srcNamedValueGetters, string[] activeOutputColNames) + { + Host.AssertValue(input); + var outputCacher = new OnnxRuntimeOutputCacher(); + ValueGetter>> valueGetter = (ref VBuffer> dst) => + { + UpdateCacheIfNeeded(input.Position, srcNamedValueGetters, activeOutputColNames, outputCacher); + var namedOnnxValue = outputCacher.Outputs[_parent.Outputs[iinfo]]; + var tensor = namedOnnxValue.AsTensor() as System.Numerics.Tensors.DenseTensor; + if (tensor == null) + throw Host.Except($"Output column {namedOnnxValue.Name} doesn't contain a DenseTensor of expected type {typeof(string)}"); + + // Create VBufferEditor to fill "dst" with the values in "denseTensor". + var editor = VBufferEditor.Create(ref dst, (int)tensor.Length); + for (int i = 0; i < tensor.Length; ++i) + // Cast because string in ML.NET is typed to ReadOnlyMemory. + editor.Values[i] = tensor.GetValue(i).AsMemory(); + dst = editor.Commit(); + }; + return valueGetter; } - private Delegate MakeGetter(DataViewRow input, int iinfo, INamedOnnxValueGetter[] srcNamedValueGetters, string[] activeOutputColNames, OutputCache outputCache) + private Delegate MakeObjectGetter(DataViewRow input, int iinfo, INamedOnnxValueGetter[] srcNamedValueGetters, string[] activeOutputColNames) { Host.AssertValue(input); - ValueGetter> valuegetter = (ref VBuffer dst) => + var outputCache = new OnnxRuntimeOutputCacher(); + ValueGetter valueGetter = (ref T dst) => { UpdateCacheIfNeeded(input.Position, srcNamedValueGetters, activeOutputColNames, outputCache); var namedOnnxValue = outputCache.Outputs[_parent.Outputs[iinfo]]; - var denseTensor = namedOnnxValue.AsTensor() as System.Numerics.Tensors.DenseTensor; - if (denseTensor == null) - throw Host.Except($"Output column {namedOnnxValue.Name} doesn't contain a DenseTensor of expected type {typeof(T)}"); - var editor = VBufferEditor.Create(ref dst, (int)denseTensor.Length); - denseTensor.Buffer.Span.CopyTo(editor.Values); - dst = editor.Commit(); + var trueValue = namedOnnxValue.AsEnumerable().Select(value => value.AsDictionary()); + var caster = _parent.Model.ModelInfo.OutputsInfo[iinfo].Caster; + dst = (T)caster(namedOnnxValue); }; - return valuegetter; + return valueGetter; } + /// + /// Helper function to wrap ML.NET getters to produce ONNXRuntime variables. + /// For each required input of the ONNX model, there will be a , + /// which first invokes a ML.NET getter and casts the obtained value to . + /// private static INamedOnnxValueGetter[] GetNamedOnnxValueGetters(DataViewRow input, - string[] inputColNames, int[] inputColIndices, - bool[] isInputVector, - System.Type[] onnxInputTypes, + Type[] onnxInputTypes, OnnxShape[] onnxInputShapes) { var srcNamedOnnxValueGetters = new INamedOnnxValueGetter[inputColIndices.Length]; for (int i = 0; i < inputColIndices.Length; i++) { int colIndex = inputColIndices[i]; - srcNamedOnnxValueGetters[i] = CreateNamedOnnxValueGetter(input, onnxInputTypes[i], isInputVector[i], inputColNames[i], colIndex, onnxInputShapes[i]); + var isVector = input.Schema[colIndex].Type is VectorDataViewType; + if (!isVector) + srcNamedOnnxValueGetters[i] = CreateNamedOnnxValueGetter(input, onnxInputTypes[i], colIndex, onnxInputShapes[i]); + else + srcNamedOnnxValueGetters[i] = CreateNamedOnnxValueGetterVec(input, onnxInputTypes[i], colIndex, onnxInputShapes[i]); } return srcNamedOnnxValueGetters; } - private static INamedOnnxValueGetter CreateNamedOnnxValueGetter(DataViewRow input, System.Type onnxType, bool isVector, string colName, int colIndex, OnnxShape onnxShape) + /// + /// Wrap ML.NET getter to produce NamedOnnxValue. The wrapper is used to fetch non-vector ML.NET column and cast ML.NET column to + /// NamedOnnxValue which is consumable by ONNXRuntime. + /// + private static INamedOnnxValueGetter CreateNamedOnnxValueGetter(DataViewRow input, Type onnxType, int colIndex, OnnxShape onnxShape) { - var type = OnnxUtils.OnnxToMlNetType(onnxType).RawType; + // This type is column type in ML.NET used to invoke ML.NET + // getter, so we use just use the type provided by the input's Schema. + // This function handles non-tensor types, so we directly access RawType. + // For tensor types, we need to do GetItemType().RawType. + var type = input.Schema[colIndex].Type.RawType; Contracts.AssertValue(type); - return Utils.MarshalInvoke(CreateNameOnnxValueGetter, type, input, isVector, colName, colIndex, onnxShape); + return Utils.MarshalInvoke(CreateNamedOnnxValueGetterCore, type, input, colIndex, onnxShape); } - private static INamedOnnxValueGetter CreateNameOnnxValueGetter(DataViewRow input, bool isVector, string colName, int colIndex, OnnxShape onnxShape) + /// + /// Function needed by reflection in . + /// + private static INamedOnnxValueGetter CreateNamedOnnxValueGetterCore(DataViewRow input, int colIndex, OnnxShape onnxShape) { - if (isVector) - return new NamedOnnxValueGetterVec(input, colName, colIndex, onnxShape); - return new NameOnnxValueGetter(input, colName, colIndex); + return new NameOnnxValueGetter(input, colIndex); + } + + /// + /// Wrap ML.NET getter to produce NamedOnnxValue. The wrapper is used to fetch vector-typed ML.NET column and cast ML.NET column to + /// NamedOnnxValue which is consumable by ONNXRuntime. + /// + private static INamedOnnxValueGetter CreateNamedOnnxValueGetterVec(DataViewRow input, Type onnxType, int colIndex, OnnxShape onnxShape) + { + // This type is column type in ML.NET used to invoke ML.NET + // getter, so we use just use the type provided by the input's Schema. + // This function handles tensor types, so we need to call GetItemType() + // to get the element type in VBuffer. + var type = input.Schema[colIndex].Type.GetItemType().RawType; + Contracts.AssertValue(type); + return Utils.MarshalInvoke(CreateNamedOnnxValueGetterVecCore, type, input, colIndex, onnxShape); + } + + /// + /// Function needed by reflection in . + /// + private static INamedOnnxValueGetter CreateNamedOnnxValueGetterVecCore(DataViewRow input, int colIndex, OnnxShape onnxShape) + { + return new NamedOnnxValueGetterVec(input, colIndex, onnxShape); + } + + /// + /// Common function for wrapping ML.NET getter as a NamedOnnxValue getter. + /// + private interface INamedOnnxValueGetter + { + NamedOnnxValue GetNamedOnnxValue(); } private class NameOnnxValueGetter : INamedOnnxValueGetter { - private readonly ValueGetter _srcgetter; + private readonly ValueGetter _srcGetter; private readonly string _colName; - public NameOnnxValueGetter(DataViewRow input, string colName, int colIndex) + public NameOnnxValueGetter(DataViewRow input, int colIndex) { - _colName = colName; - _srcgetter = input.GetGetter(input.Schema[colIndex]); + _colName = input.Schema[colIndex].Name; + _srcGetter = input.GetGetter(input.Schema[colIndex]); } public NamedOnnxValue GetNamedOnnxValue() { var scalar = default(T); - _srcgetter(ref scalar); + _srcGetter(ref scalar); return OnnxUtils.CreateScalarNamedOnnxValue(_colName, scalar); } } private class NamedOnnxValueGetterVec : INamedOnnxValueGetter { - private readonly ValueGetter> _srcgetter; + private readonly ValueGetter> _srcGetter; private readonly OnnxShape _tensorShape; private readonly string _colName; private VBuffer _vBuffer; private VBuffer _vBufferDense; - public NamedOnnxValueGetterVec(DataViewRow input, string colName, int colIndex, OnnxShape tensorShape) + public NamedOnnxValueGetterVec(DataViewRow input, int colIndex, OnnxShape tensorShape) { - _srcgetter = input.GetGetter>(input.Schema[colIndex]); + _srcGetter = input.GetGetter>(input.Schema[colIndex]); _tensorShape = tensorShape; - _colName = colName; + _colName = input.Schema[colIndex].Name; _vBuffer = default; _vBufferDense = default; } public NamedOnnxValue GetNamedOnnxValue() { - _srcgetter(ref _vBuffer); + _srcGetter(ref _vBuffer); _vBuffer.CopyToDense(ref _vBufferDense); return OnnxUtils.CreateNamedOnnxValue(_colName, _vBufferDense.GetValues(), _tensorShape); } @@ -595,21 +704,30 @@ public override SchemaShape GetOutputSchema(SchemaShape inputSchema) var result = inputSchema.ToDictionary(x => x.Name); var resultDic = inputSchema.ToDictionary(x => x.Name); + // This loop checks if all input columns needed in the underlying transformer can be found + // in inputSchema. + // Since ML.NET can only produces tensors (scalars are converted to tensor with shape [1] before feeding + // ML.NET them into ONNXRuntime), the bridge code in ONNX Transformer assumes that all inputs are tensors. for (var i = 0; i < Transformer.Inputs.Length; i++) { + // Get the i-th IDataView input column's name in the underlying ONNX transformer. var input = Transformer.Inputs[i]; + + // Make sure inputSchema contains the i-th input column. if (!inputSchema.TryFindColumn(input, out var col)) throw Host.ExceptSchemaMismatch(nameof(inputSchema), "input", input); + + // Make sure that the input columns in inputSchema are fixed shape tensors. if (col.Kind == SchemaShape.Column.VectorKind.VariableVector) throw Host.ExceptSchemaMismatch(nameof(inputSchema), "input", input, "vector", col.GetTypeString()); var inputsInfo = Transformer.Model.ModelInfo.InputsInfo; - var idx = Transformer.Model.InputNames.IndexOf(input); + var idx = Transformer.Model.ModelInfo.InputNames.IndexOf(input); if (idx < 0) throw Host.Except($"Column {input} doesn't match input node names of model."); var inputNodeInfo = inputsInfo[idx]; - var expectedType = OnnxUtils.OnnxToMlNetType(inputNodeInfo.Type); + var expectedType = ((VectorDataViewType)inputNodeInfo.DataViewType).ItemType; if (col.ItemType != expectedType) throw Host.ExceptSchemaMismatch(nameof(inputSchema), "input", input, expectedType.ToString(), col.ItemType.ToString()); } @@ -620,6 +738,7 @@ public override SchemaShape GetOutputSchema(SchemaShape inputSchema) Transformer.OutputTypes[i].IsKnownSizeVector() ? SchemaShape.Column.VectorKind.Vector : SchemaShape.Column.VectorKind.VariableVector, Transformer.OutputTypes[i].GetItemType(), false); } + return new SchemaShape(resultDic.Values); } } diff --git a/src/Microsoft.ML.OnnxTransformer/OnnxTypeParser.cs b/src/Microsoft.ML.OnnxTransformer/OnnxTypeParser.cs new file mode 100644 index 0000000000..f5cb773ccb --- /dev/null +++ b/src/Microsoft.ML.OnnxTransformer/OnnxTypeParser.cs @@ -0,0 +1,366 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Numerics.Tensors; +using Microsoft.ML.Data; +using Microsoft.ML.Internal.Utilities; +using Microsoft.ML.Model.OnnxConverter; +using Microsoft.ML.OnnxRuntime; +using Microsoft.ML.Runtime; + +namespace Microsoft.ML.Transforms.Onnx +{ + internal static class OnnxTypeParser + { + /// + /// Derive the corresponding for ONNX tensor's element type specified by . + /// The corresponding should match the type system in ONNXRuntime's C# APIs. + /// This function is used when determining the corresponding of . + /// + /// ONNX's tensor element type. + public static Type GetNativeScalarType(OnnxCSharpToProtoWrapper.TensorProto.Types.DataType dataType) + { + Type scalarType = null; + switch (dataType) + { + case OnnxCSharpToProtoWrapper.TensorProto.Types.DataType.Bool: + scalarType = typeof(System.Boolean); + break; + case OnnxCSharpToProtoWrapper.TensorProto.Types.DataType.Int8: + scalarType = typeof(System.SByte); + break; + case OnnxCSharpToProtoWrapper.TensorProto.Types.DataType.Uint8: + scalarType = typeof(System.Byte); + break; + case OnnxCSharpToProtoWrapper.TensorProto.Types.DataType.Int16: + scalarType = typeof(System.Int16); + break; + case OnnxCSharpToProtoWrapper.TensorProto.Types.DataType.Uint16: + scalarType = typeof(System.UInt16); + break; + case OnnxCSharpToProtoWrapper.TensorProto.Types.DataType.Int32: + scalarType = typeof(System.Int32); + break; + case OnnxCSharpToProtoWrapper.TensorProto.Types.DataType.Uint32: + scalarType = typeof(System.UInt32); + break; + case OnnxCSharpToProtoWrapper.TensorProto.Types.DataType.Int64: + scalarType = typeof(System.Int64); + break; + case OnnxCSharpToProtoWrapper.TensorProto.Types.DataType.Uint64: + scalarType = typeof(System.UInt64); + break; + case OnnxCSharpToProtoWrapper.TensorProto.Types.DataType.Double: + scalarType = typeof(System.Double); + break; + case OnnxCSharpToProtoWrapper.TensorProto.Types.DataType.Float: + scalarType = typeof(System.Single); + break; + case OnnxCSharpToProtoWrapper.TensorProto.Types.DataType.String: + scalarType = typeof(string); + break; + default: + throw Contracts.Except("Unsupported ONNX scalar type: " + dataType.ToString()); + } + return scalarType; + } + + /// + /// Derive the corresponding for ONNX variable typed to . + /// The corresponding should match the type system in ONNXRuntime's C# APIs. + /// + /// ONNX variable's type. + public static Type GetNativeType(OnnxCSharpToProtoWrapper.TypeProto typeProto) + { + if (typeProto.ValueCase == OnnxCSharpToProtoWrapper.TypeProto.ValueOneofCase.TensorType) + { + if (typeProto.TensorType.Shape == null || typeProto.TensorType.Shape.Dim.Count == 0) + { + return GetNativeScalarType(typeProto.TensorType.ElemType); + } + else + { + Type tensorType = typeof(VBuffer<>); + Type elementType = GetNativeScalarType(typeProto.TensorType.ElemType); + return tensorType.MakeGenericType(elementType); + } + } + else if (typeProto.ValueCase == OnnxCSharpToProtoWrapper.TypeProto.ValueOneofCase.SequenceType) + { + var enumerableType = typeof(IEnumerable<>); + var elementType = GetNativeType(typeProto.SequenceType.ElemType); + return enumerableType.MakeGenericType(elementType); + } + else if (typeProto.ValueCase == OnnxCSharpToProtoWrapper.TypeProto.ValueOneofCase.MapType) + { + var dictionaryType = typeof(IDictionary<,>); + Type keyType = GetNativeScalarType(typeProto.MapType.KeyType); + Type valueType = GetNativeType(typeProto.MapType.ValueType); + return dictionaryType.MakeGenericType(keyType, valueType); + } + return null; + } + + /// + /// Derive the corresponding for ONNX tensor's element type specified by . + /// + /// ONNX's tensor element type. + public static DataViewType GetScalarDataViewType(OnnxCSharpToProtoWrapper.TensorProto.Types.DataType dataType) + { + DataViewType scalarType = null; + switch (dataType) + { + case OnnxCSharpToProtoWrapper.TensorProto.Types.DataType.Bool: + scalarType = BooleanDataViewType.Instance; + break; + case OnnxCSharpToProtoWrapper.TensorProto.Types.DataType.Int8: + scalarType = NumberDataViewType.SByte; + break; + case OnnxCSharpToProtoWrapper.TensorProto.Types.DataType.Uint8: + scalarType = NumberDataViewType.Byte; + break; + case OnnxCSharpToProtoWrapper.TensorProto.Types.DataType.Int16: + scalarType = NumberDataViewType.Int16; + break; + case OnnxCSharpToProtoWrapper.TensorProto.Types.DataType.Uint16: + scalarType = NumberDataViewType.UInt16; + break; + case OnnxCSharpToProtoWrapper.TensorProto.Types.DataType.Int32: + scalarType = NumberDataViewType.Int32; + break; + case OnnxCSharpToProtoWrapper.TensorProto.Types.DataType.Uint32: + scalarType = NumberDataViewType.UInt32; + break; + case OnnxCSharpToProtoWrapper.TensorProto.Types.DataType.Int64: + scalarType = NumberDataViewType.Int64; + break; + case OnnxCSharpToProtoWrapper.TensorProto.Types.DataType.Uint64: + scalarType = NumberDataViewType.UInt64; + break; + case OnnxCSharpToProtoWrapper.TensorProto.Types.DataType.Float: + scalarType = NumberDataViewType.Single; + break; + case OnnxCSharpToProtoWrapper.TensorProto.Types.DataType.Double: + scalarType = NumberDataViewType.Double; + break; + case OnnxCSharpToProtoWrapper.TensorProto.Types.DataType.String: + scalarType = TextDataViewType.Instance; + break; + default: + throw Contracts.Except("Unsupported ONNX scalar type: " + dataType.ToString()); + } + return scalarType; + } + + /// + /// Parse the dimension information of a single tensor axis. Note that 2-D ONNX tensors have two axes. + /// + /// ONNX's tensor dimension. + public static int GetDimValue(OnnxCSharpToProtoWrapper.TensorShapeProto.Types.Dimension dim) + { + int value = 0; + switch (dim.ValueCase) + { + case OnnxCSharpToProtoWrapper.TensorShapeProto.Types.Dimension.ValueOneofCase.DimValue: + // The vector length in ML.NET is typed to 32-bit integer, so the check below is added for perverting overflowing. + if (dim.DimValue > int.MaxValue) + throw Contracts.ExceptParamValue(dim.DimValue, nameof(dim), $"Dimension {dim} in ONNX tensor cannot exceed the maximum of 32-bit signed integer."); + // Variable-length dimension is translated to 0. + value = dim.DimValue > 0 ? (int)dim.DimValue : 0; + break; + case OnnxCSharpToProtoWrapper.TensorShapeProto.Types.Dimension.ValueOneofCase.DimParam: + // Variable-length dimension is translated to 0. + value = 0; + break; + default: + throw Contracts.ExceptParamValue(dim.DimValue, nameof(dim), $"Dimension {dim} in ONNX tensor cannot exceed the maximum of 32-bit signed integer."); + } + return value; + } + + /// + /// Parse the shape information of a tensor. + /// + /// ONNX's tensor shape. + public static IEnumerable GetTensorDims(Microsoft.ML.Model.OnnxConverter.OnnxCSharpToProtoWrapper.TensorShapeProto tensorShapeProto) + { + if (tensorShapeProto == null) + // Scalar has null dimensionality. + return null; + + List dims = new List(); + foreach(var d in tensorShapeProto.Dim) + { + var dimValue = GetDimValue(d); + dims.Add(dimValue); + } + return dims; + } + + /// + /// Derive the corresponding for ONNX variable typed to . + /// The returned should match the type system in ONNXRuntime's C# APIs. + /// + /// ONNX variable's type. + public static DataViewType GetDataViewType(OnnxCSharpToProtoWrapper.TypeProto typeProto) + { + var oneOfFieldName = typeProto.ValueCase.ToString(); + if (typeProto.ValueCase == OnnxCSharpToProtoWrapper.TypeProto.ValueOneofCase.TensorType) + { + if (typeProto.TensorType.Shape.Dim.Count == 0) + // ONNX scalar is a tensor without shape information; that is, + // ONNX scalar's shape is an empty list. + return GetScalarDataViewType(typeProto.TensorType.ElemType); + else + { + var shape = GetTensorDims(typeProto.TensorType.Shape); + if (shape == null) + // Scalar has null shape. + return GetScalarDataViewType(typeProto.TensorType.ElemType); + else if (shape.Count() != 0 && shape.Aggregate((x, y) => x * y) > 0) + // Known shape tensor. + return new VectorDataViewType((PrimitiveDataViewType)GetScalarDataViewType(typeProto.TensorType.ElemType), shape.ToArray()); + else + // Tensor with unknown shape. + return new VectorDataViewType((PrimitiveDataViewType)GetScalarDataViewType(typeProto.TensorType.ElemType), 0); + } + } + else if (typeProto.ValueCase == OnnxCSharpToProtoWrapper.TypeProto.ValueOneofCase.SequenceType) + { + var elemTypeProto = typeProto.SequenceType.ElemType; + var elemType = GetNativeType(elemTypeProto); + return new OnnxSequenceType(elemType); + } + else if (typeProto.ValueCase == OnnxCSharpToProtoWrapper.TypeProto.ValueOneofCase.MapType) + { + var keyType = GetNativeScalarType(typeProto.MapType.KeyType); + var valueType = GetNativeType(typeProto.MapType.ValueType); + return new OnnxMapType(keyType, valueType); + } + else + throw Contracts.ExceptParamValue(typeProto, nameof(typeProto), $"Unsupported ONNX variable type {typeProto}"); + } + + /// + /// Class which store casting functions used in . + /// + private class CastHelper + { + public static T CastTo(object o) => (T) o; + + public static IEnumerable CastOnnxSequenceToIEnumerable(IEnumerable o, Func caster) + { + return o.Select(v => (TDst)caster(v)); + } + } + + /// + /// Create a to map a to the associated .NET . + /// The resulted .NET object's actual type is . + /// The returned should match the type system in ONNXRuntime's C# APIs. + /// + /// ONNX variable's type. + /// C# type of . + public static Func GetDataViewValueCasterAndResultedType(OnnxCSharpToProtoWrapper.TypeProto typeProto, out Type resultedType) + { + var oneOfFieldName = typeProto.ValueCase.ToString(); + if (typeProto.ValueCase == OnnxCSharpToProtoWrapper.TypeProto.ValueOneofCase.TensorType) + { + var shape = GetTensorDims(typeProto.TensorType.Shape); + + if (shape == null) + { + // Entering this scope means that an ONNX scalar is found. Note that ONNX scalar is typed to tensor without a shape. + + // Get tensor element type. + var type = GetScalarDataViewType(typeProto.TensorType.ElemType).RawType; + + // Access the first element as a scalar. + var accessInfo = typeof(Tensor<>).GetMethod(nameof(Tensor.GetValue)); + var accessSpecialized = accessInfo.MakeGenericMethod(type); + + // NamedOnnxValue to scalar. + Func caster = (NamedOnnxValue value) => { + var scalar = accessSpecialized.Invoke(value, new object[] { 0 }); + return scalar; + }; + + resultedType = type; + + return caster; + } + else + { + // Entering this scope means an ONNX tensor is found. + + var type = GetScalarDataViewType(typeProto.TensorType.ElemType).RawType; + var methodInfo = typeof(NamedOnnxValue).GetMethod(nameof(NamedOnnxValue.AsTensor)); + var methodSpecialized = methodInfo.MakeGenericMethod(type); + + // NamedOnnxValue to Tensor. + Func caster = (NamedOnnxValue value) => methodSpecialized.Invoke(value, new object[] { }); + + resultedType = typeof(Tensor<>).MakeGenericType(type); + + return caster; + } + } + else if (typeProto.ValueCase == OnnxCSharpToProtoWrapper.TypeProto.ValueOneofCase.SequenceType) + { + // Now, we see a Sequence in ONNX. If its element type is T, the variable produced by + // ONNXRuntime would be typed to IEnumerable. + + // Find a proper caster (a function which maps NamedOnnxValue to a .NET object) for the element in + // the ONNX sequence. Note that ONNX sequence is typed to IEnumerable, so we need + // to convert NamedOnnxValue to a proper type such as IDictionary<>. + var elementCaster = GetDataViewValueCasterAndResultedType(typeProto.SequenceType.ElemType, out Type elementType); + + // Set the .NET type which corresponds to the first input argument, typeProto. + resultedType = typeof(IEnumerable<>).MakeGenericType(elementType); + + // Create the element's caster to map IEnumerable produced by ONNXRuntime to + // IEnumerable. + var methodInfo = typeof(CastHelper).GetMethod(nameof(CastHelper.CastOnnxSequenceToIEnumerable)); + var methodSpecialized = methodInfo.MakeGenericMethod(typeof(NamedOnnxValue), elementType); + + // Use element-level caster to create sequence caster. + Func caster = (NamedOnnxValue value) => + { + var enumerable = value.AsEnumerable(); + return methodSpecialized.Invoke(null, new object[] { enumerable, elementCaster }); + }; + + return caster; + } + else if (typeProto.ValueCase == OnnxCSharpToProtoWrapper.TypeProto.ValueOneofCase.MapType) + { + // Entering this scope means a ONNX Map (equivalent to IDictionary<>) will be produced. + + var keyType = GetNativeScalarType(typeProto.MapType.KeyType); + var valueType = GetNativeType(typeProto.MapType.ValueType); + + // The resulted type of the object returned by the caster below. + resultedType = typeof(IDictionary<,>).MakeGenericType(keyType, valueType); + + // Create a method to convert NamedOnnxValue to IDictionary. + var asDictionaryMethodInfo = typeof(NamedOnnxValue).GetMethod(nameof(NamedOnnxValue.AsDictionary)); + var asDictionaryMethod = asDictionaryMethodInfo.MakeGenericMethod(keyType, valueType); + + // Create a caster to convert NamedOnnxValue to IDictionary. + Func caster = (NamedOnnxValue value) => + { + return asDictionaryMethod.Invoke(value, new object[] { }); + }; + + return caster; + } + else + throw Contracts.ExceptParamValue(typeProto, nameof(typeProto), $"Unsupported ONNX variable type {typeProto}"); + } + } + +} diff --git a/src/Microsoft.ML.OnnxTransformer/OnnxUtils.cs b/src/Microsoft.ML.OnnxTransformer/OnnxUtils.cs index 09822c1433..c7604e1e10 100644 --- a/src/Microsoft.ML.OnnxTransformer/OnnxUtils.cs +++ b/src/Microsoft.ML.OnnxTransformer/OnnxUtils.cs @@ -8,6 +8,7 @@ using System.Linq; using System.Numerics.Tensors; using Microsoft.ML.Data; +using Microsoft.ML.Model.OnnxConverter; using Microsoft.ML.OnnxRuntime; using Microsoft.ML.Runtime; using OnnxShape = System.Collections.Generic.List; @@ -20,57 +21,122 @@ namespace Microsoft.ML.Transforms.Onnx /// It provides API to open a session, score tensors (NamedOnnxValues) and return /// the results. /// - internal sealed class OnnxModel + internal sealed class OnnxModel : IDisposable { - /// /// OnnxModelInfo contains the data that we should get from /// OnnxRuntime API once that functionality is added. /// public sealed class OnnxModelInfo { - public readonly OnnxNodeInfo[] InputsInfo; - public readonly OnnxNodeInfo[] OutputsInfo; + /// + /// InputNames[i] is the name of the i-th element in . + /// + public List InputNames { get; } + /// + /// OutputNames[i] is the name of the i-th element in . + /// + public List OutputNames { get; } + /// + /// Inputs of the containing . + /// + public OnnxVariableInfo[] InputsInfo { get; } + /// + /// Outputs of the containing . + /// + public OnnxVariableInfo[] OutputsInfo { get; } - public OnnxModelInfo(IEnumerable inputsInfo, IEnumerable outputsInfo) + public OnnxModelInfo(IEnumerable inputsInfo, IEnumerable outputsInfo) { + InputNames = inputsInfo.Select(val => val.Name).ToList(); InputsInfo = inputsInfo.ToArray(); + OutputNames = outputsInfo.Select(val => val.Name).ToList(); OutputsInfo = outputsInfo.ToArray(); } + + /// + /// Return the ONNX value for a input column called . + /// + public OnnxVariableInfo GetInput(string name) + { + var index = InputNames.IndexOf(name); + if (index < 0) + throw Contracts.ExceptParamValue(name, nameof(name), $"Input tensor, {name}, does not exist in the ONNX model. " + + $"Available input names are [{string.Join(",", InputNames)}]."); + return InputsInfo[index]; + } + + /// + /// Return the ONNX value for a output column called . + /// + public OnnxVariableInfo GetOutput(string name) + { + var index = OutputNames.IndexOf(name); + if (index < 0) + throw Contracts.ExceptParamValue(name, nameof(name), $"Onput tensor, {name}, does not exist in the ONNX model. " + + $"Available output names are [{string.Join(",", OutputNames)}]."); + return OutputsInfo[index]; + } } /// /// OnnxNodeInfo contains all the information for a given node (e.g. inputs/outputs) /// of an Onnx model. /// - public class OnnxNodeInfo + public class OnnxVariableInfo { /// - /// The Name of the node + /// The Name of the variable. Note that ONNX variable are named. /// - public readonly string Name; + public string Name { get; } /// - /// The shape of the node + /// The shape of the variable if the variable is a tensor. For other + /// types such sequence and dictionary, would be + /// . /// - public readonly OnnxShape Shape; + public OnnxShape Shape { get; } /// - /// The type of the node + /// The type of the variable produced by ONNXRuntime. /// - public readonly System.Type Type; + public Type TypeInOnnxRuntime { get; } + /// + /// The that this ONNX variable corresponds + /// to in 's type system. + /// + public DataViewType DataViewType { get; } + /// + /// A method to case produced by + /// ONNXRuntime to the type specified in . + /// + public Func Caster { get; } - public OnnxNodeInfo(string name, OnnxShape shape, System.Type type) + public OnnxVariableInfo(string name, OnnxShape shape, Type typeInOnnxRuntime, DataViewType mlnetType, Func caster) { Name = name; Shape = shape; - Type = type; + TypeInOnnxRuntime = typeInOnnxRuntime; + DataViewType = mlnetType; + Caster = caster; } } - public readonly OnnxModelInfo ModelInfo; + /// + /// The ONNXRuntime facility to execute the loaded ONNX model. + /// private readonly InferenceSession _session; - private readonly string _modelFile; - public readonly List InputNames; - public readonly List OutputNames; + /// + /// Indicates if is a temporal file created by + /// or . If , should delete . + /// + private bool _ownModelFile; + /// + /// The location where the used ONNX model loaded from. + /// + internal string ModelFile { get; } + /// + /// The ONNX model file that built upon. + /// + internal OnnxModelInfo ModelInfo { get; } /// /// Constructs OnnxModel object from file. @@ -78,9 +144,14 @@ public OnnxNodeInfo(string name, OnnxShape shape, System.Type type) /// Model file path. /// GPU device ID to execute on. Null for CPU. /// If true, resumes CPU execution quitely upon GPU error. - public OnnxModel(string modelFile, int? gpuDeviceId = null, bool fallbackToCpu = false) + /// If true, the will be deleted when is + /// no longer needed. + public OnnxModel(string modelFile, int? gpuDeviceId = null, bool fallbackToCpu = false, bool ownModelFile=false) { - _modelFile = modelFile; + ModelFile = modelFile; + // If we don't own the model file, _disposed should be false to prevent deleting user's file. + _ownModelFile = ownModelFile; + _disposed = false; if (gpuDeviceId != null) { @@ -103,13 +174,55 @@ public OnnxModel(string modelFile, int? gpuDeviceId = null, bool fallbackToCpu = _session = new InferenceSession(modelFile); } - ModelInfo = new OnnxModelInfo(GetInputsInfo(), GetOutputsInfo()); - InputNames = ModelInfo.InputsInfo.Select(i => i.Name).ToList(); - OutputNames = ModelInfo.OutputsInfo.Select(i => i.Name).ToList(); + // Load ONNX model file and parse its input and output schema. The reason of doing so is that ONNXRuntime + // doesn't expose full type information via its C# APIs. + ModelFile = modelFile; + var model = new OnnxCSharpToProtoWrapper.ModelProto(); + using (var modelStream = File.OpenRead(modelFile)) + model = OnnxCSharpToProtoWrapper.ModelProto.Parser.ParseFrom(modelStream); + + // Parse actual input and output types stored in the loaded ONNX model to get their DataViewType's. + var inputTypePool = new Dictionary(); + foreach (var valueInfo in model.Graph.Input) + inputTypePool[valueInfo.Name] = OnnxTypeParser.GetDataViewType(valueInfo.Type); + var outputTypePool = new Dictionary(); + + // Build casters which maps NamedOnnxValue to .NET objects. + var casterPool = new Dictionary>(); + foreach (var valueInfo in model.Graph.Output) + { + outputTypePool[valueInfo.Name] = OnnxTypeParser.GetDataViewType(valueInfo.Type); + casterPool[valueInfo.Name] = OnnxTypeParser.GetDataViewValueCasterAndResultedType(valueInfo.Type, out Type actualType); + } + + var onnxRuntimeInputInfos = new List(); + foreach (var pair in _session.InputMetadata) + { + var name = pair.Key; + var meta = pair.Value; + var dataViewType = inputTypePool[name]; + var info = new OnnxVariableInfo(name, meta.Dimensions.ToList(), meta.ElementType, dataViewType, null); + onnxRuntimeInputInfos.Add(info); + } + + var onnxRuntimeOutputInfos = new List(); + foreach (var pair in _session.OutputMetadata) + { + var name = pair.Key; + var meta = pair.Value; + var dataViewType = outputTypePool[name]; + var caster = casterPool[name]; + var info = new OnnxVariableInfo(name, meta.Dimensions.ToList(), meta.ElementType, dataViewType, caster); + onnxRuntimeOutputInfos.Add(info); + } + + ModelInfo = new OnnxModelInfo(onnxRuntimeInputInfos, onnxRuntimeOutputInfos); } /// - /// Create an OnnxModel from a byte[] + /// Create an OnnxModel from a byte[]. Usually, a ONNX model is consumed by as a file. + /// With and , it's possible + /// to use in-memory model (type: byte[]) to create . /// /// Bytes of the serialized model /// OnnxModel @@ -120,6 +233,9 @@ public static OnnxModel CreateFromBytes(byte[] modelBytes) /// /// Create an OnnxModel from a byte[]. Set execution to GPU if required. + /// Usually, a ONNX model is consumed by as a file. + /// With and , + /// it's possible to use in-memory model (type: byte[]) to create . /// /// Bytes of the serialized model. /// GPU device ID to execute on. Null for CPU. @@ -132,12 +248,7 @@ public static OnnxModel CreateFromBytes(byte[] modelBytes, int? gpuDeviceId = nu var tempModelFile = Path.Combine(tempModelDir, "model.onnx"); File.WriteAllBytes(tempModelFile, modelBytes); - return new OnnxModel(tempModelFile, gpuDeviceId, fallbackToCpu); - - // TODO: - // tempModelFile is needed in case the model needs to be saved - // Either have to save the modelbytes and delete the temp dir/file, - // or keep the dir/file and write proper cleanup when application closes + return new OnnxModel(tempModelFile, gpuDeviceId, fallbackToCpu, ownModelFile: true); } /// @@ -151,37 +262,49 @@ public IReadOnlyCollection Run(List inputNamedOn } /// - /// Convert the model to a byte array. + /// Flag used to indicate if the unmanaged resources (aka the model file + /// and ) have been deleted. /// - /// byte[] - public byte[] ToByteArray() + private bool _disposed; + + public void Dispose() { - return File.ReadAllBytes(_modelFile); + Dispose(true); + GC.SuppressFinalize(this); } /// - /// Returns input metadata of the ONNX model. + /// There are two unmanaged resources we can dispose, and + /// if is . /// - /// OnnxNodeInfo[] - private IEnumerable GetInputsInfo() + /// + private void Dispose(bool disposing) { - return _session.InputMetadata.Select(kv => new OnnxNodeInfo(kv.Key, kv.Value.Dimensions.ToList(), kv.Value.ElementType)); + if (!_disposed) + { + // There are two things to be disposed. + if (disposing) + { + // First, we release the resource token by ONNXRuntime. + _session.Dispose(); + // Second, we delete the model file if that file is not created by the user. + if (_ownModelFile && File.Exists(ModelFile)) + File.Delete(ModelFile); + } + _disposed = true; + } } - /// - /// Returns output metadata of the ONNX model. - /// - /// - private IEnumerable GetOutputsInfo() + ~OnnxModel() { - return _session.OutputMetadata.Select(kv => new OnnxNodeInfo(kv.Key, kv.Value.Dimensions.ToList(), kv.Value.ElementType)); + Dispose(false); } } internal sealed class OnnxUtils { - private static HashSet _onnxTypeMap = - new HashSet + private static HashSet _onnxTypeMap = + new HashSet { typeof(Double), typeof(Single), @@ -192,8 +315,8 @@ internal sealed class OnnxUtils typeof(UInt32), typeof(UInt64) }; - private static Dictionary _typeToKindMap= - new Dictionary + private static Dictionary _typeToKindMap= + new Dictionary { { typeof(Single) , InternalDataKind.R4}, { typeof(Double) , InternalDataKind.R8}, @@ -243,7 +366,7 @@ public static NamedOnnxValue CreateNamedOnnxValue(string name, ReadOnlySpan /// /// - public static PrimitiveDataViewType OnnxToMlNetType(System.Type type) + public static PrimitiveDataViewType OnnxToMlNetType(Type type) { if (!_typeToKindMap.ContainsKey(type)) throw Contracts.ExceptNotSupp("Onnx type not supported", type); diff --git a/src/Microsoft.ML.TensorFlow/TensorFlow/TensorflowUtils.cs b/src/Microsoft.ML.TensorFlow/TensorFlow/TensorflowUtils.cs index 2c71d22f97..8de6347529 100644 --- a/src/Microsoft.ML.TensorFlow/TensorFlow/TensorflowUtils.cs +++ b/src/Microsoft.ML.TensorFlow/TensorFlow/TensorflowUtils.cs @@ -14,7 +14,7 @@ namespace Microsoft.ML.Transforms.TensorFlow { - public static class TensorFlowUtils + internal static class TensorFlowUtils { /// /// Key to access operator's type (a string) in . diff --git a/src/Microsoft.ML.TensorFlow/TensorFlowModel.cs b/src/Microsoft.ML.TensorFlow/TensorFlowModel.cs index f814e387e2..a490965805 100644 --- a/src/Microsoft.ML.TensorFlow/TensorFlowModel.cs +++ b/src/Microsoft.ML.TensorFlow/TensorFlowModel.cs @@ -105,7 +105,7 @@ public TensorFlowEstimator ScoreTensorFlowModel(string[] outputColumnNames, stri /// /// The support for retraining is experimental. /// - public TensorFlowEstimator RetrainTensorFlowModel( + internal TensorFlowEstimator RetrainTensorFlowModel( string[] outputColumnNames, string[] inputColumnNames, string labelColumnName, diff --git a/src/Microsoft.ML.TimeSeries/AdaptiveSingularSpectrumSequenceModeler.cs b/src/Microsoft.ML.TimeSeries/AdaptiveSingularSpectrumSequenceModeler.cs index da14de2bcb..1986eac4bb 100644 --- a/src/Microsoft.ML.TimeSeries/AdaptiveSingularSpectrumSequenceModeler.cs +++ b/src/Microsoft.ML.TimeSeries/AdaptiveSingularSpectrumSequenceModeler.cs @@ -6,1699 +6,1566 @@ using System.Collections.Generic; using System.Numerics; using Microsoft.ML; +using Microsoft.ML.CommandLine; using Microsoft.ML.Data; using Microsoft.ML.Internal.CpuMath; using Microsoft.ML.Internal.Utilities; using Microsoft.ML.Runtime; -using Microsoft.ML.TimeSeries; using Microsoft.ML.Transforms.TimeSeries; -[assembly: LoadableClass(typeof(AdaptiveSingularSpectrumSequenceModeler.AdaptiveSingularSpectrumSequenceModelerInternal), - typeof(AdaptiveSingularSpectrumSequenceModeler.AdaptiveSingularSpectrumSequenceModelerInternal), null, typeof(SignatureLoadModel), +[assembly: LoadableClass(typeof(AdaptiveSingularSpectrumSequenceModelerInternal), + typeof(AdaptiveSingularSpectrumSequenceModelerInternal), null, typeof(SignatureLoadModel), "SSA Sequence Modeler", - AdaptiveSingularSpectrumSequenceModeler.AdaptiveSingularSpectrumSequenceModelerInternal.LoaderSignature)] - -[assembly: LoadableClass(typeof(AdaptiveSingularSpectrumSequenceModeler), - typeof(AdaptiveSingularSpectrumSequenceModeler), null, typeof(SignatureLoadModel), - "SSA Sequence Modeler Wrapper", - AdaptiveSingularSpectrumSequenceModeler.LoaderSignature)] + AdaptiveSingularSpectrumSequenceModelerInternal.LoaderSignature)] namespace Microsoft.ML.Transforms.TimeSeries { + /// + /// Ranking selection method for the signal. + /// + public enum RankSelectionMethod + { + Fixed, + Exact, + Fast + } + + /// + /// Growth ratio. Defined as Growth^(1/TimeSpan). + /// + public struct GrowthRatio + { + [Argument(ArgumentType.AtMostOnce, HelpText = "Time span of growth ratio. Must be strictly positive.", SortOrder = 1)] + public int TimeSpan; + + [Argument(ArgumentType.AtMostOnce, HelpText = "Growth. Must be non-negative.", SortOrder = 2)] + public Double Growth; + + public Double Ratio { get { return Math.Pow(Growth, 1d / TimeSpan); } } + } + /// /// This class implements basic Singular Spectrum Analysis (SSA) model for modeling univariate time-series. /// For the details of the model, refer to http://arxiv.org/pdf/1206.6910.pdf. /// - public sealed class AdaptiveSingularSpectrumSequenceModeler : ICanForecast + internal sealed class AdaptiveSingularSpectrumSequenceModelerInternal : SequenceModelerBase { - /// - /// Ranking selection method for the signal. - /// - public enum RankSelectionMethod + internal const string LoaderSignature = "SSAModel"; + + internal sealed class SsaForecastResult : ForecastResultBase { - Fixed, - Exact, - Fast + public VBuffer ForecastStandardDeviation; + public VBuffer UpperBound; + public VBuffer LowerBound; + public Single ConfidenceLevel; + + internal bool CanComputeForecastIntervals; + internal Single BoundOffset; + + public bool IsVarianceValid { get { return CanComputeForecastIntervals; } } } - /// - /// Growth ratio. Defined as Growth^(1/TimeSpan). - /// - public struct GrowthRatio + public sealed class ModelInfo { - private int _timeSpan; - private Double _growth; + /// + /// The singular values of the SSA of the input time-series + /// + public Single[] Spectrum; /// - /// Time span of growth ratio. Must be strictly positive. + /// The roots of the characteristic polynomial after stabilization (meaningful only if the model is stabilized.) /// - public int TimeSpan - { - get - { - return _timeSpan; - } - set - { - Contracts.CheckParam(value > 0, nameof(TimeSpan), "The time span must be strictly positive."); - _timeSpan = value; - } - } + public Complex[] RootsAfterStabilization; /// - /// Growth. Must be non-negative. + /// The roots of the characteristic polynomial before stabilization (meaningful only if the model is stabilized.) /// - public Double Growth - { - get - { - return _growth; - } - set - { - Contracts.CheckParam(value >= 0, nameof(Growth), "The growth must be non-negative."); - _growth = value; - } - } + public Complex[] RootsBeforeStabilization; - public GrowthRatio(int timeSpan = 1, double growth = Double.PositiveInfinity) - { - Contracts.CheckParam(timeSpan > 0, nameof(TimeSpan), "The time span must be strictly positive."); - Contracts.CheckParam(growth >= 0, nameof(Growth), "The growth must be non-negative."); + /// + /// The rank of the model + /// + public int Rank; - _growth = growth; - _timeSpan = timeSpan; - } + /// + /// The window size used to compute the SSA model + /// + public int WindowSize; - public Double Ratio { get { return Math.Pow(_growth, 1d / _timeSpan); } } - } + /// + /// The auto-regressive coefficients learned by the model + /// + public Single[] AutoRegressiveCoefficients; - private AdaptiveSingularSpectrumSequenceModelerInternal _modeler; + /// + /// The flag indicating whether the model has been trained + /// + public bool IsTrained; - private readonly string _inputColumnName; + /// + /// The flag indicating a naive model is trained instead of SSA + /// + public bool IsNaiveModelTrained; - internal const string LoaderSignature = "ForecastModel"; + /// + /// The flag indicating whether the learned model has an exponential trend (meaningful only if the model is stabilized.) + /// + public bool IsExponentialTrendPresent; - private readonly IHost _host; + /// + /// The flag indicating whether the learned model has a polynomial trend (meaningful only if the model is stabilized.) + /// + public bool IsPolynomialTrendPresent; - private static VersionInfo GetVersionInfo() - { - return new VersionInfo( - modelSignature: "SSAMODLW", - verWrittenCur: 0x00010001, // Initial - verReadableCur: 0x00010001, - verWeCanReadBack: 0x00010001, - loaderSignature: LoaderSignature, - loaderAssemblyName: typeof(AdaptiveSingularSpectrumSequenceModeler).Assembly.FullName); - } + /// + /// The flag indicating whether the learned model has been stabilized + /// + public bool IsStabilized; - internal AdaptiveSingularSpectrumSequenceModeler(IHostEnvironment env, string inputColumnName, int trainSize, int seriesLength, int windowSize, Single discountFactor = 1, - RankSelectionMethod rankSelectionMethod = RankSelectionMethod.Exact, int? rank = null, int? maxRank = null, - bool shouldComputeForecastIntervals = true, bool shouldstablize = true, bool shouldMaintainInfo = false, GrowthRatio? maxGrowth = null) - { - Contracts.CheckValue(env, nameof(env)); - _host = env.Register(LoaderSignature); - _host.CheckParam(!string.IsNullOrEmpty(inputColumnName), nameof(inputColumnName)); + /// + /// The flag indicating whether any artificial seasonality (a seasonality with period greater than the window size) is removed + /// (meaningful only if the model is stabilized.) + /// + public bool IsArtificialSeasonalityRemoved; - _inputColumnName = inputColumnName; - _modeler = new AdaptiveSingularSpectrumSequenceModelerInternal(env, trainSize, seriesLength, windowSize, discountFactor, - rankSelectionMethod, rank, maxRank, shouldComputeForecastIntervals, shouldstablize, shouldMaintainInfo, maxGrowth); + /// + /// The exponential trend magnitude (meaningful only if the model is stabilized.) + /// + public Double ExponentialTrendFactor; } - internal AdaptiveSingularSpectrumSequenceModeler(IHostEnvironment env, ModelLoadContext ctx) - { - Contracts.CheckValue(env, nameof(env)); - _host = env.Register(LoaderSignature); - _inputColumnName = ctx.Reader.ReadString(); - ctx.LoadModel(_host, out _modeler, "ForecastWrapper"); - } + private ModelInfo _info; /// - /// This class implements basic Singular Spectrum Analysis (SSA) model for modeling univariate time-series. - /// For the details of the model, refer to http://arxiv.org/pdf/1206.6910.pdf. + /// Returns the meta information about the learned model. /// - internal sealed class AdaptiveSingularSpectrumSequenceModelerInternal : SequenceModelerBase + public ModelInfo Info { - internal const string LoaderSignature = "SSAModel"; - - internal sealed class SsaForecastResult : ForecastResultBase - { - public VBuffer ForecastStandardDeviation; - public VBuffer UpperBound; - public VBuffer LowerBound; - public Single ConfidenceLevel; + get { return _info; } + } - internal bool CanComputeForecastIntervals; - internal Single BoundOffset; + private Single[] _alpha; + private Single[] _state; + private readonly FixedSizeQueue _buffer; + private CpuAlignedVector _x; + private CpuAlignedVector _xSmooth; + private int _windowSize; + private readonly int _seriesLength; + private readonly RankSelectionMethod _rankSelectionMethod; + private readonly Single _discountFactor; + private readonly int _trainSize; + private int _maxRank; + private readonly Double _maxTrendRatio; + private readonly bool _shouldStablize; + private readonly bool _shouldMaintainInfo; - public bool IsVarianceValid { get { return CanComputeForecastIntervals; } } - } + private readonly IHost _host; - public sealed class ModelInfo - { - /// - /// The singular values of the SSA of the input time-series - /// - public Single[] Spectrum; - - /// - /// The roots of the characteristic polynomial after stabilization (meaningful only if the model is stabilized.) - /// - public Complex[] RootsAfterStabilization; - - /// - /// The roots of the characteristic polynomial before stabilization (meaningful only if the model is stabilized.) - /// - public Complex[] RootsBeforeStabilization; - - /// - /// The rank of the model - /// - public int Rank; - - /// - /// The window size used to compute the SSA model - /// - public int WindowSize; - - /// - /// The auto-regressive coefficients learned by the model - /// - public Single[] AutoRegressiveCoefficients; - - /// - /// The flag indicating whether the model has been trained - /// - public bool IsTrained; - - /// - /// The flag indicating a naive model is trained instead of SSA - /// - public bool IsNaiveModelTrained; - - /// - /// The flag indicating whether the learned model has an exponential trend (meaningful only if the model is stabilized.) - /// - public bool IsExponentialTrendPresent; - - /// - /// The flag indicating whether the learned model has a polynomial trend (meaningful only if the model is stabilized.) - /// - public bool IsPolynomialTrendPresent; - - /// - /// The flag indicating whether the learned model has been stabilized - /// - public bool IsStabilized; - - /// - /// The flag indicating whether any artificial seasonality (a seasonality with period greater than the window size) is removed - /// (meaningful only if the model is stabilized.) - /// - public bool IsArtificialSeasonalityRemoved; - - /// - /// The exponential trend magnitude (meaningful only if the model is stabilized.) - /// - public Double ExponentialTrendFactor; - } + private CpuAlignedMatrixRow _wTrans; + private Single _observationNoiseVariance; + private Single _observationNoiseMean; + private Single _autoregressionNoiseVariance; + private Single _autoregressionNoiseMean; - private ModelInfo _info; + private int _rank; + private CpuAlignedVector _y; + private Single _nextPrediction; - /// - /// Returns the meta information about the learned model. - /// - public ModelInfo Info - { - get { return _info; } - } + /// + /// Determines whether the confidence interval required for forecasting. + /// + public bool ShouldComputeForecastIntervals { get; set; } - private Single[] _alpha; - private Single[] _state; - private readonly FixedSizeQueue _buffer; - private CpuAlignedVector _x; - private CpuAlignedVector _xSmooth; - private int _windowSize; - private readonly int _seriesLength; - private readonly RankSelectionMethod _rankSelectionMethod; - private readonly Single _discountFactor; - private readonly int _trainSize; - private int _maxRank; - private readonly Double _maxTrendRatio; - private readonly bool _shouldStablize; - private readonly bool _shouldMaintainInfo; - - private readonly IHost _host; - - private CpuAlignedMatrixRow _wTrans; - private Single _observationNoiseVariance; - private Single _observationNoiseMean; - private Single _autoregressionNoiseVariance; - private Single _autoregressionNoiseMean; - - private int _rank; - private CpuAlignedVector _y; - private Single _nextPrediction; + /// + /// Returns the rank of the subspace used for SSA projection (parameter r). + /// + public int Rank { get { return _rank; } } - /// - /// Determines whether the confidence interval required for forecasting. - /// - public bool ShouldComputeForecastIntervals { get; set; } + /// + /// Returns the smoothed (via SSA projection) version of the last series observation fed to the model. + /// + public Single LastSmoothedValue { get { return _state[_windowSize - 2]; } } - /// - /// Returns the rank of the subspace used for SSA projection (parameter r). - /// - public int Rank { get { return _rank; } } + /// + /// Returns the last series observation fed to the model. + /// + public Single LastValue { get { return _buffer.Count > 0 ? _buffer[_buffer.Count - 1] : Single.NaN; } } - /// - /// Returns the smoothed (via SSA projection) version of the last series observation fed to the model. - /// - public Single LastSmoothedValue { get { return _state[_windowSize - 2]; } } + private static VersionInfo GetVersionInfo() + { + return new VersionInfo( + modelSignature: "SSAMODLR", + //verWrittenCur: 0x00010001, // Initial + verWrittenCur: 0x00010002, // Added saving _state and _nextPrediction + verReadableCur: 0x00010002, + verWeCanReadBack: 0x00010001, + loaderSignature: LoaderSignature, + loaderAssemblyName: typeof(AdaptiveSingularSpectrumSequenceModelerInternal).Assembly.FullName); + } - /// - /// Returns the last series observation fed to the model. - /// - public Single LastValue { get { return _buffer.Count > 0 ? _buffer[_buffer.Count - 1] : Single.NaN; } } + private const int VersionSavingStateAndPrediction = 0x00010002; - private static VersionInfo GetVersionInfo() + /// + /// The constructor for Adaptive SSA model. + /// + /// The exception context. + /// The length of series from the begining used for training. + /// The length of series that is kept in buffer for modeling (parameter N). + /// The length of the window on the series for building the trajectory matrix (parameter L). + /// The discount factor in [0,1] used for online updates (default = 1). + /// The rank selection method (default = Exact). + /// The desired rank of the subspace used for SSA projection (parameter r). This parameter should be in the range in [1, windowSize]. + /// If set to null, the rank is automatically determined based on prediction error minimization. (default = null) + /// The maximum rank considered during the rank selection process. If not provided (i.e. set to null), it is set to windowSize - 1. + /// The flag determining whether the confidence bounds for the point forecasts should be computed. (default = true) + /// The flag determining whether the model should be stabilized. + /// The flag determining whether the meta information for the model needs to be maintained. + /// The maximum growth on the exponential trend. + public AdaptiveSingularSpectrumSequenceModelerInternal(IHostEnvironment env, int trainSize, int seriesLength, int windowSize, Single discountFactor = 1, + RankSelectionMethod rankSelectionMethod = RankSelectionMethod.Exact, int? rank = null, int? maxRank = null, + bool shouldComputeForecastIntervals = true, bool shouldstablize = true, bool shouldMaintainInfo = false, GrowthRatio? maxGrowth = null) + : base() + { + Contracts.CheckValue(env, nameof(env)); + _host = env.Register(LoaderSignature); + _host.CheckParam(windowSize >= 2, nameof(windowSize), "The window size should be at least 2."); // ...because otherwise we have nothing to autoregress on + _host.CheckParam(seriesLength > windowSize, nameof(seriesLength), "The series length should be greater than the window size."); + _host.Check(trainSize > 2 * windowSize, "The input size for training should be greater than twice the window size."); + _host.CheckParam(0 <= discountFactor && discountFactor <= 1, nameof(discountFactor), "The discount factor should be in [0,1]."); + _host.CheckParam(maxGrowth == null || maxGrowth.Value.TimeSpan > 0, nameof(GrowthRatio.TimeSpan), "The time span must be strictly positive."); + _host.CheckParam(maxGrowth == null || maxGrowth.Value.Growth >= 0, nameof(GrowthRatio.Growth), "The growth must be non-negative."); + + if (maxRank != null) { - return new VersionInfo( - modelSignature: "SSAMODLR", - //verWrittenCur: 0x00010001, // Initial - verWrittenCur: 0x00010002, // Added saving _state and _nextPrediction - verReadableCur: 0x00010002, - verWeCanReadBack: 0x00010001, - loaderSignature: LoaderSignature, - loaderAssemblyName: typeof(AdaptiveSingularSpectrumSequenceModelerInternal).Assembly.FullName); + _maxRank = (int)maxRank; + _host.CheckParam(1 <= _maxRank && _maxRank < windowSize, nameof(maxRank), + "The max rank should be in [1, windowSize)."); } + else + _maxRank = windowSize - 1; - private const int VersionSavingStateAndPrediction = 0x00010002; - - /// - /// The constructor for Adaptive SSA model. - /// - /// The exception context. - /// The length of series from the begining used for training. - /// The length of series that is kept in buffer for modeling (parameter N). - /// The length of the window on the series for building the trajectory matrix (parameter L). - /// The discount factor in [0,1] used for online updates (default = 1). - /// The rank selection method (default = Exact). - /// The desired rank of the subspace used for SSA projection (parameter r). This parameter should be in the range in [1, windowSize]. - /// If set to null, the rank is automatically determined based on prediction error minimization. (default = null) - /// The maximum rank considered during the rank selection process. If not provided (i.e. set to null), it is set to windowSize - 1. - /// The flag determining whether the confidence bounds for the point forecasts should be computed. (default = true) - /// The flag determining whether the model should be stabilized. - /// The flag determining whether the meta information for the model needs to be maintained. - /// The maximum growth on the exponential trend - public AdaptiveSingularSpectrumSequenceModelerInternal(IHostEnvironment env, int trainSize, int seriesLength, int windowSize, Single discountFactor = 1, - RankSelectionMethod rankSelectionMethod = RankSelectionMethod.Exact, int? rank = null, int? maxRank = null, - bool shouldComputeForecastIntervals = true, bool shouldstablize = true, bool shouldMaintainInfo = false, GrowthRatio? maxGrowth = null) - : base() + _rankSelectionMethod = rankSelectionMethod; + if (_rankSelectionMethod == RankSelectionMethod.Fixed) { - Contracts.CheckValue(env, nameof(env)); - _host = env.Register(LoaderSignature); - _host.CheckParam(windowSize >= 2, nameof(windowSize), "The window size should be at least 2."); // ...because otherwise we have nothing to autoregress on - _host.CheckParam(seriesLength > windowSize, nameof(seriesLength), "The series length should be greater than the window size."); - _host.Check(trainSize > 2 * windowSize, "The input series length for training should be greater than twice the window size."); - _host.CheckParam(0 <= discountFactor && discountFactor <= 1, nameof(discountFactor), "The discount factor should be in [0,1]."); - - if (maxRank != null) + if (rank != null) { - _maxRank = (int)maxRank; - _host.CheckParam(1 <= _maxRank && _maxRank < windowSize, nameof(maxRank), - "The max rank should be in [1, windowSize)."); + _rank = (int)rank; + _host.CheckParam(1 <= _rank && _rank < windowSize, nameof(rank), "The rank should be in [1, windowSize)."); } else - _maxRank = windowSize - 1; - - _rankSelectionMethod = rankSelectionMethod; - if (_rankSelectionMethod == RankSelectionMethod.Fixed) - { - if (rank != null) - { - _rank = (int)rank; - _host.CheckParam(1 <= _rank && _rank < windowSize, nameof(rank), "The rank should be in [1, windowSize)."); - } - else - _rank = _maxRank; - } + _rank = _maxRank; + } - _seriesLength = seriesLength; - _windowSize = windowSize; - _trainSize = trainSize; - _discountFactor = discountFactor; + _seriesLength = seriesLength; + _windowSize = windowSize; + _trainSize = trainSize; + _discountFactor = discountFactor; - _buffer = new FixedSizeQueue(seriesLength); + _buffer = new FixedSizeQueue(seriesLength); - _alpha = new Single[windowSize - 1]; - _state = new Single[windowSize - 1]; - _x = new CpuAlignedVector(windowSize, CpuMathUtils.GetVectorAlignment()); - _xSmooth = new CpuAlignedVector(windowSize, CpuMathUtils.GetVectorAlignment()); - ShouldComputeForecastIntervals = shouldComputeForecastIntervals; + _alpha = new Single[windowSize - 1]; + _state = new Single[windowSize - 1]; + _x = new CpuAlignedVector(windowSize, CpuMathUtils.GetVectorAlignment()); + _xSmooth = new CpuAlignedVector(windowSize, CpuMathUtils.GetVectorAlignment()); + ShouldComputeForecastIntervals = shouldComputeForecastIntervals; - _observationNoiseVariance = 0; - _autoregressionNoiseVariance = 0; - _observationNoiseMean = 0; - _autoregressionNoiseMean = 0; - _shouldStablize = shouldstablize; - _maxTrendRatio = maxGrowth == null ? Double.PositiveInfinity : ((GrowthRatio)maxGrowth).Ratio; + _observationNoiseVariance = 0; + _autoregressionNoiseVariance = 0; + _observationNoiseMean = 0; + _autoregressionNoiseMean = 0; + _shouldStablize = shouldstablize; + _maxTrendRatio = maxGrowth == null ? Double.PositiveInfinity : ((GrowthRatio)maxGrowth).Ratio; - _shouldMaintainInfo = shouldMaintainInfo; - if (_shouldMaintainInfo) - { - _info = new ModelInfo(); - _info.WindowSize = _windowSize; - } + _shouldMaintainInfo = shouldMaintainInfo; + if (_shouldMaintainInfo) + { + _info = new ModelInfo(); + _info.WindowSize = _windowSize; } + } - /// - /// The copy constructor. - /// - /// An object whose contents are copied to the current object. - private AdaptiveSingularSpectrumSequenceModelerInternal(AdaptiveSingularSpectrumSequenceModelerInternal model) + /// + /// The copy constructor. + /// + /// An object whose contents are copied to the current object. + private AdaptiveSingularSpectrumSequenceModelerInternal(AdaptiveSingularSpectrumSequenceModelerInternal model) + { + _host = model._host.Register(LoaderSignature); + _host.Assert(model._windowSize >= 2); + _host.Assert(model._seriesLength > model._windowSize); + _host.Assert(model._trainSize > 2 * model._windowSize); + _host.Assert(0 <= model._discountFactor && model._discountFactor <= 1); + _host.Assert(1 <= model._rank && model._rank < model._windowSize); + + _rank = model._rank; + _maxRank = model._maxRank; + _rankSelectionMethod = model._rankSelectionMethod; + _seriesLength = model._seriesLength; + _windowSize = model._windowSize; + _trainSize = model._trainSize; + _discountFactor = model._discountFactor; + ShouldComputeForecastIntervals = model.ShouldComputeForecastIntervals; + _observationNoiseVariance = model._observationNoiseVariance; + _autoregressionNoiseVariance = model._autoregressionNoiseVariance; + _observationNoiseMean = model._observationNoiseMean; + _autoregressionNoiseMean = model._autoregressionNoiseMean; + _nextPrediction = model._nextPrediction; + _maxTrendRatio = model._maxTrendRatio; + _shouldStablize = model._shouldStablize; + _shouldMaintainInfo = model._shouldMaintainInfo; + _info = model._info; + _buffer = model._buffer.Clone(); + _alpha = new Single[_windowSize - 1]; + Array.Copy(model._alpha, _alpha, _windowSize - 1); + _state = new Single[_windowSize - 1]; + Array.Copy(model._state, _state, _windowSize - 1); + + _x = new CpuAlignedVector(_windowSize, CpuMathUtils.GetVectorAlignment()); + _xSmooth = new CpuAlignedVector(_windowSize, CpuMathUtils.GetVectorAlignment()); + + if (model._wTrans != null) { - _host = model._host.Register(LoaderSignature); - _host.Assert(model._windowSize >= 2); - _host.Assert(model._seriesLength > model._windowSize); - _host.Assert(model._trainSize > 2 * model._windowSize); - _host.Assert(0 <= model._discountFactor && model._discountFactor <= 1); - _host.Assert(1 <= model._rank && model._rank < model._windowSize); - - _rank = model._rank; - _maxRank = model._maxRank; - _rankSelectionMethod = model._rankSelectionMethod; - _seriesLength = model._seriesLength; - _windowSize = model._windowSize; - _trainSize = model._trainSize; - _discountFactor = model._discountFactor; - ShouldComputeForecastIntervals = model.ShouldComputeForecastIntervals; - _observationNoiseVariance = model._observationNoiseVariance; - _autoregressionNoiseVariance = model._autoregressionNoiseVariance; - _observationNoiseMean = model._observationNoiseMean; - _autoregressionNoiseMean = model._autoregressionNoiseMean; - _nextPrediction = model._nextPrediction; - _maxTrendRatio = model._maxTrendRatio; - _shouldStablize = model._shouldStablize; - _shouldMaintainInfo = model._shouldMaintainInfo; - _info = model._info; - _buffer = model._buffer.Clone(); - _alpha = new Single[_windowSize - 1]; - Array.Copy(model._alpha, _alpha, _windowSize - 1); - _state = new Single[_windowSize - 1]; - Array.Copy(model._state, _state, _windowSize - 1); + _y = new CpuAlignedVector(_rank, CpuMathUtils.GetVectorAlignment()); + _wTrans = new CpuAlignedMatrixRow(_rank, _windowSize, CpuMathUtils.GetVectorAlignment()); + _wTrans.CopyFrom(model._wTrans); + } + } - _x = new CpuAlignedVector(_windowSize, CpuMathUtils.GetVectorAlignment()); - _xSmooth = new CpuAlignedVector(_windowSize, CpuMathUtils.GetVectorAlignment()); + public AdaptiveSingularSpectrumSequenceModelerInternal(IHostEnvironment env, ModelLoadContext ctx) + { + Contracts.CheckValue(env, nameof(env)); + _host = env.Register(LoaderSignature); - if (model._wTrans != null) - { - _y = new CpuAlignedVector(_rank, CpuMathUtils.GetVectorAlignment()); - _wTrans = new CpuAlignedMatrixRow(_rank, _windowSize, CpuMathUtils.GetVectorAlignment()); - _wTrans.CopyFrom(model._wTrans); - } + // *** Binary format *** + // int: _seriesLength + // int: _windowSize + // int: _trainSize + // int: _rank + // float: _discountFactor + // RankSelectionMethod: _rankSelectionMethod + // bool: isWeightSet + // float[]: _alpha + // float[]: _state + // bool: ShouldComputeForecastIntervals + // float: _observationNoiseVariance + // float: _autoregressionNoiseVariance + // float: _observationNoiseMean + // float: _autoregressionNoiseMean + // float: _nextPrediction + // int: _maxRank + // bool: _shouldStablize + // bool: _shouldMaintainInfo + // double: _maxTrendRatio + // float[]: _wTrans (only if _isWeightSet == true) + + _seriesLength = ctx.Reader.ReadInt32(); + // Do an early check. We'll have the stricter check later. + _host.CheckDecode(_seriesLength > 2); + + _windowSize = ctx.Reader.ReadInt32(); + _host.CheckDecode(_windowSize >= 2); + _host.CheckDecode(_seriesLength > _windowSize); + + _trainSize = ctx.Reader.ReadInt32(); + _host.CheckDecode(_trainSize > 2 * _windowSize); + + _rank = ctx.Reader.ReadInt32(); + _host.CheckDecode(1 <= _rank && _rank < _windowSize); + + _discountFactor = ctx.Reader.ReadSingle(); + _host.CheckDecode(0 <= _discountFactor && _discountFactor <= 1); + + byte temp; + temp = ctx.Reader.ReadByte(); + _rankSelectionMethod = (RankSelectionMethod)temp; + bool isWeightSet = ctx.Reader.ReadBoolByte(); + + _alpha = ctx.Reader.ReadFloatArray(); + _host.CheckDecode(Utils.Size(_alpha) == _windowSize - 1); + + if (ctx.Header.ModelVerReadable >= VersionSavingStateAndPrediction) + { + _state = ctx.Reader.ReadFloatArray(); + _host.CheckDecode(Utils.Size(_state) == _windowSize - 1); } + else + _state = new Single[_windowSize - 1]; - public AdaptiveSingularSpectrumSequenceModelerInternal(IHostEnvironment env, ModelLoadContext ctx) - { - Contracts.CheckValue(env, nameof(env)); - _host = env.Register(LoaderSignature); - - // *** Binary format *** - // int: _seriesLength - // int: _windowSize - // int: _trainSize - // int: _rank - // float: _discountFactor - // RankSelectionMethod: _rankSelectionMethod - // bool: isWeightSet - // float[]: _alpha - // float[]: _state - // bool: ShouldComputeForecastIntervals - // float: _observationNoiseVariance - // float: _autoregressionNoiseVariance - // float: _observationNoiseMean - // float: _autoregressionNoiseMean - // float: _nextPrediction - // int: _maxRank - // bool: _shouldStablize - // bool: _shouldMaintainInfo - // double: _maxTrendRatio - // float[]: _wTrans (only if _isWeightSet == true) - - _seriesLength = ctx.Reader.ReadInt32(); - // Do an early check. We'll have the stricter check later. - _host.CheckDecode(_seriesLength > 2); - - _windowSize = ctx.Reader.ReadInt32(); - _host.CheckDecode(_windowSize >= 2); - _host.CheckDecode(_seriesLength > _windowSize); - - _trainSize = ctx.Reader.ReadInt32(); - _host.CheckDecode(_trainSize > 2 * _windowSize); - - _rank = ctx.Reader.ReadInt32(); - _host.CheckDecode(1 <= _rank && _rank < _windowSize); - - _discountFactor = ctx.Reader.ReadSingle(); - _host.CheckDecode(0 <= _discountFactor && _discountFactor <= 1); - - byte temp; - temp = ctx.Reader.ReadByte(); - _rankSelectionMethod = (RankSelectionMethod)temp; - bool isWeightSet = ctx.Reader.ReadBoolByte(); - - _alpha = ctx.Reader.ReadFloatArray(); - _host.CheckDecode(Utils.Size(_alpha) == _windowSize - 1); - - if (ctx.Header.ModelVerReadable >= VersionSavingStateAndPrediction) - { - _state = ctx.Reader.ReadFloatArray(); - _host.CheckDecode(Utils.Size(_state) == _windowSize - 1); - } - else - _state = new Single[_windowSize - 1]; + ShouldComputeForecastIntervals = ctx.Reader.ReadBoolByte(); - ShouldComputeForecastIntervals = ctx.Reader.ReadBoolByte(); + _observationNoiseVariance = ctx.Reader.ReadSingle(); + _host.CheckDecode(_observationNoiseVariance >= 0); - _observationNoiseVariance = ctx.Reader.ReadSingle(); - _host.CheckDecode(_observationNoiseVariance >= 0); + _autoregressionNoiseVariance = ctx.Reader.ReadSingle(); + _host.CheckDecode(_autoregressionNoiseVariance >= 0); - _autoregressionNoiseVariance = ctx.Reader.ReadSingle(); - _host.CheckDecode(_autoregressionNoiseVariance >= 0); + _observationNoiseMean = ctx.Reader.ReadSingle(); + _autoregressionNoiseMean = ctx.Reader.ReadSingle(); + if (ctx.Header.ModelVerReadable >= VersionSavingStateAndPrediction) + _nextPrediction = ctx.Reader.ReadSingle(); - _observationNoiseMean = ctx.Reader.ReadSingle(); - _autoregressionNoiseMean = ctx.Reader.ReadSingle(); - if (ctx.Header.ModelVerReadable >= VersionSavingStateAndPrediction) - _nextPrediction = ctx.Reader.ReadSingle(); + _maxRank = ctx.Reader.ReadInt32(); + _host.CheckDecode(1 <= _maxRank && _maxRank <= _windowSize - 1); - _maxRank = ctx.Reader.ReadInt32(); - _host.CheckDecode(1 <= _maxRank && _maxRank <= _windowSize - 1); + _shouldStablize = ctx.Reader.ReadBoolByte(); + _shouldMaintainInfo = ctx.Reader.ReadBoolByte(); + if (_shouldMaintainInfo) + { + _info = new ModelInfo(); + _info.AutoRegressiveCoefficients = new Single[_windowSize - 1]; + Array.Copy(_alpha, _info.AutoRegressiveCoefficients, _windowSize - 1); - _shouldStablize = ctx.Reader.ReadBoolByte(); - _shouldMaintainInfo = ctx.Reader.ReadBoolByte(); - if (_shouldMaintainInfo) - { - _info = new ModelInfo(); - _info.AutoRegressiveCoefficients = new Single[_windowSize - 1]; - Array.Copy(_alpha, _info.AutoRegressiveCoefficients, _windowSize - 1); + _info.IsStabilized = _shouldStablize; + _info.Rank = _rank; + _info.WindowSize = _windowSize; + } - _info.IsStabilized = _shouldStablize; - _info.Rank = _rank; - _info.WindowSize = _windowSize; - } + _maxTrendRatio = ctx.Reader.ReadDouble(); + _host.CheckDecode(_maxTrendRatio >= 0); - _maxTrendRatio = ctx.Reader.ReadDouble(); - _host.CheckDecode(_maxTrendRatio >= 0); + if (isWeightSet) + { + var tempArray = ctx.Reader.ReadFloatArray(); + _host.CheckDecode(Utils.Size(tempArray) == _rank * _windowSize); + _wTrans = new CpuAlignedMatrixRow(_rank, _windowSize, CpuMathUtils.GetVectorAlignment()); + int i = 0; + _wTrans.CopyFrom(tempArray, ref i); + tempArray = ctx.Reader.ReadFloatArray(); + i = 0; + _y = new CpuAlignedVector(_rank, CpuMathUtils.GetVectorAlignment()); + _y.CopyFrom(tempArray, ref i); + } - if (isWeightSet) - { - var tempArray = ctx.Reader.ReadFloatArray(); - _host.CheckDecode(Utils.Size(tempArray) == _rank * _windowSize); - _wTrans = new CpuAlignedMatrixRow(_rank, _windowSize, CpuMathUtils.GetVectorAlignment()); - int i = 0; - _wTrans.CopyFrom(tempArray, ref i); - tempArray = ctx.Reader.ReadFloatArray(); - i = 0; - _y = new CpuAlignedVector(_rank, CpuMathUtils.GetVectorAlignment()); - _y.CopyFrom(tempArray, ref i); - } + _buffer = TimeSeriesUtils.DeserializeFixedSizeQueueSingle(ctx.Reader, _host); + _x = new CpuAlignedVector(_windowSize, CpuMathUtils.GetVectorAlignment()); + _xSmooth = new CpuAlignedVector(_windowSize, CpuMathUtils.GetVectorAlignment()); + } - _buffer = TimeSeriesUtils.DeserializeFixedSizeQueueSingle(ctx.Reader, _host); - _x = new CpuAlignedVector(_windowSize, CpuMathUtils.GetVectorAlignment()); - _xSmooth = new CpuAlignedVector(_windowSize, CpuMathUtils.GetVectorAlignment()); - } + private protected override void SaveModel(ModelSaveContext ctx) + { + _host.CheckValue(ctx, nameof(ctx)); + ctx.CheckAtModel(); + ctx.SetVersionInfo(GetVersionInfo()); - private protected override void SaveModel(ModelSaveContext ctx) + _host.Assert(_windowSize >= 2); + _host.Assert(_seriesLength > _windowSize); + _host.Assert(_trainSize > 2 * _windowSize); + _host.Assert(0 <= _discountFactor && _discountFactor <= 1); + _host.Assert(1 <= _rank && _rank < _windowSize); + _host.Assert(Utils.Size(_alpha) == _windowSize - 1); + _host.Assert(_observationNoiseVariance >= 0); + _host.Assert(_autoregressionNoiseVariance >= 0); + _host.Assert(1 <= _maxRank && _maxRank <= _windowSize - 1); + _host.Assert(_maxTrendRatio >= 0); + + // *** Binary format *** + // int: _seriesLength + // int: _windowSize + // int: _trainSize + // int: _rank + // float: _discountFactor + // RankSelectionMethod: _rankSelectionMethod + // bool: _isWeightSet + // float[]: _alpha + // float[]: _state + // bool: ShouldComputeForecastIntervals + // float: _observationNoiseVariance + // float: _autoregressionNoiseVariance + // float: _observationNoiseMean + // float: _autoregressionNoiseMean + // float: _nextPrediction + // int: _maxRank + // bool: _shouldStablize + // bool: _shouldMaintainInfo + // double: _maxTrendRatio + // float[]: _wTrans (only if _isWeightSet == true) + + ctx.Writer.Write(_seriesLength); + ctx.Writer.Write(_windowSize); + ctx.Writer.Write(_trainSize); + ctx.Writer.Write(_rank); + ctx.Writer.Write(_discountFactor); + ctx.Writer.Write((byte)_rankSelectionMethod); + ctx.Writer.WriteBoolByte(_wTrans != null); + ctx.Writer.WriteSingleArray(_alpha); + ctx.Writer.WriteSingleArray(_state); + ctx.Writer.WriteBoolByte(ShouldComputeForecastIntervals); + ctx.Writer.Write(_observationNoiseVariance); + ctx.Writer.Write(_autoregressionNoiseVariance); + ctx.Writer.Write(_observationNoiseMean); + ctx.Writer.Write(_autoregressionNoiseMean); + ctx.Writer.Write(_nextPrediction); + ctx.Writer.Write(_maxRank); + ctx.Writer.WriteBoolByte(_shouldStablize); + ctx.Writer.WriteBoolByte(_shouldMaintainInfo); + ctx.Writer.Write(_maxTrendRatio); + + if (_wTrans != null) { - _host.CheckValue(ctx, nameof(ctx)); - ctx.CheckAtModel(); - ctx.SetVersionInfo(GetVersionInfo()); - - _host.Assert(_windowSize >= 2); - _host.Assert(_seriesLength > _windowSize); - _host.Assert(_trainSize > 2 * _windowSize); - _host.Assert(0 <= _discountFactor && _discountFactor <= 1); - _host.Assert(1 <= _rank && _rank < _windowSize); - _host.Assert(Utils.Size(_alpha) == _windowSize - 1); - _host.Assert(_observationNoiseVariance >= 0); - _host.Assert(_autoregressionNoiseVariance >= 0); - _host.Assert(1 <= _maxRank && _maxRank <= _windowSize - 1); - _host.Assert(_maxTrendRatio >= 0); - - // *** Binary format *** - // int: _seriesLength - // int: _windowSize - // int: _trainSize - // int: _rank - // float: _discountFactor - // RankSelectionMethod: _rankSelectionMethod - // bool: _isWeightSet - // float[]: _alpha - // float[]: _state - // bool: ShouldComputeForecastIntervals - // float: _observationNoiseVariance - // float: _autoregressionNoiseVariance - // float: _observationNoiseMean - // float: _autoregressionNoiseMean - // float: _nextPrediction - // int: _maxRank - // bool: _shouldStablize - // bool: _shouldMaintainInfo - // double: _maxTrendRatio - // float[]: _wTrans (only if _isWeightSet == true) - - ctx.Writer.Write(_seriesLength); - ctx.Writer.Write(_windowSize); - ctx.Writer.Write(_trainSize); - ctx.Writer.Write(_rank); - ctx.Writer.Write(_discountFactor); - ctx.Writer.Write((byte)_rankSelectionMethod); - ctx.Writer.WriteBoolByte(_wTrans != null); - ctx.Writer.WriteSingleArray(_alpha); - ctx.Writer.WriteSingleArray(_state); - ctx.Writer.WriteBoolByte(ShouldComputeForecastIntervals); - ctx.Writer.Write(_observationNoiseVariance); - ctx.Writer.Write(_autoregressionNoiseVariance); - ctx.Writer.Write(_observationNoiseMean); - ctx.Writer.Write(_autoregressionNoiseMean); - ctx.Writer.Write(_nextPrediction); - ctx.Writer.Write(_maxRank); - ctx.Writer.WriteBoolByte(_shouldStablize); - ctx.Writer.WriteBoolByte(_shouldMaintainInfo); - ctx.Writer.Write(_maxTrendRatio); - - if (_wTrans != null) - { - // REVIEW: this may not be the most efficient way for serializing an aligned matrix. - var tempArray = new Single[_rank * _windowSize]; - int iv = 0; - _wTrans.CopyTo(tempArray, ref iv); - ctx.Writer.WriteSingleArray(tempArray); - tempArray = new float[_rank]; - iv = 0; - _y.CopyTo(tempArray, ref iv); - ctx.Writer.WriteSingleArray(tempArray); - } - - TimeSeriesUtils.SerializeFixedSizeQueue(_buffer, ctx.Writer); + // REVIEW: this may not be the most efficient way for serializing an aligned matrix. + var tempArray = new Single[_rank * _windowSize]; + int iv = 0; + _wTrans.CopyTo(tempArray, ref iv); + ctx.Writer.WriteSingleArray(tempArray); + tempArray = new float[_rank]; + iv = 0; + _y.CopyTo(tempArray, ref iv); + ctx.Writer.WriteSingleArray(tempArray); } - private static void ReconstructSignal(TrajectoryMatrix tMat, Single[] singularVectors, int rank, Single[] output) - { - Contracts.Assert(tMat != null); - Contracts.Assert(1 <= rank && rank <= tMat.WindowSize); - Contracts.Assert(Utils.Size(singularVectors) >= tMat.WindowSize * rank); - Contracts.Assert(Utils.Size(output) >= tMat.SeriesLength); + TimeSeriesUtils.SerializeFixedSizeQueue(_buffer, ctx.Writer); + } - var k = tMat.SeriesLength - tMat.WindowSize + 1; - Contracts.Assert(k > 0); + private static void ReconstructSignal(TrajectoryMatrix tMat, Single[] singularVectors, int rank, Single[] output) + { + Contracts.Assert(tMat != null); + Contracts.Assert(1 <= rank && rank <= tMat.WindowSize); + Contracts.Assert(Utils.Size(singularVectors) >= tMat.WindowSize * rank); + Contracts.Assert(Utils.Size(output) >= tMat.SeriesLength); - var v = new Single[k]; - int i; + var k = tMat.SeriesLength - tMat.WindowSize + 1; + Contracts.Assert(k > 0); - for (i = 0; i < tMat.SeriesLength; ++i) - output[i] = 0; + var v = new Single[k]; + int i; - for (i = 0; i < rank; ++i) - { - // Reconstructing the i-th eigen triple component and adding it to output - tMat.MultiplyTranspose(singularVectors, v, false, tMat.WindowSize * i, 0); - tMat.RankOneHankelization(singularVectors, v, 1, output, true, tMat.WindowSize * i, 0, 0); - } - } + for (i = 0; i < tMat.SeriesLength; ++i) + output[i] = 0; - private static void ReconstructSignalTailFast(Single[] series, TrajectoryMatrix tMat, Single[] singularVectors, int rank, Single[] output) + for (i = 0; i < rank; ++i) { - Contracts.Assert(tMat != null); - Contracts.Assert(1 <= rank && rank <= tMat.WindowSize); - Contracts.Assert(Utils.Size(singularVectors) >= tMat.WindowSize * rank); + // Reconstructing the i-th eigen triple component and adding it to output + tMat.MultiplyTranspose(singularVectors, v, false, tMat.WindowSize * i, 0); + tMat.RankOneHankelization(singularVectors, v, 1, output, true, tMat.WindowSize * i, 0, 0); + } + } - int len = 2 * tMat.WindowSize - 1; - Contracts.Assert(Utils.Size(output) >= len); + private static void ReconstructSignalTailFast(Single[] series, TrajectoryMatrix tMat, Single[] singularVectors, int rank, Single[] output) + { + Contracts.Assert(tMat != null); + Contracts.Assert(1 <= rank && rank <= tMat.WindowSize); + Contracts.Assert(Utils.Size(singularVectors) >= tMat.WindowSize * rank); - Single v; - /*var k = tMat.SeriesLength - tMat.WindowSize + 1; - Contracts.Assert(k > 0); - var v = new Single[k];*/ + int len = 2 * tMat.WindowSize - 1; + Contracts.Assert(Utils.Size(output) >= len); - Single temp; - int i; - int j; - int offset1 = tMat.SeriesLength - 2 * tMat.WindowSize + 1; - int offset2 = tMat.SeriesLength - tMat.WindowSize; + Single v; + /*var k = tMat.SeriesLength - tMat.WindowSize + 1; + Contracts.Assert(k > 0); + var v = new Single[k];*/ - for (i = 0; i < len; ++i) - output[i] = 0; + Single temp; + int i; + int j; + int offset1 = tMat.SeriesLength - 2 * tMat.WindowSize + 1; + int offset2 = tMat.SeriesLength - tMat.WindowSize; - for (i = 0; i < rank; ++i) - { - // Reconstructing the i-th eigen triple component and adding it to output - v = 0; - for (j = offset1; j < offset1 + tMat.WindowSize; ++j) - v += series[j] * singularVectors[tMat.WindowSize * i - offset1 + j]; + for (i = 0; i < len; ++i) + output[i] = 0; - for (j = 0; j < tMat.WindowSize - 1; ++j) - output[j] += v * singularVectors[tMat.WindowSize * i + j]; + for (i = 0; i < rank; ++i) + { + // Reconstructing the i-th eigen triple component and adding it to output + v = 0; + for (j = offset1; j < offset1 + tMat.WindowSize; ++j) + v += series[j] * singularVectors[tMat.WindowSize * i - offset1 + j]; - temp = v * singularVectors[tMat.WindowSize * (i + 1) - 1]; + for (j = 0; j < tMat.WindowSize - 1; ++j) + output[j] += v * singularVectors[tMat.WindowSize * i + j]; - v = 0; - for (j = offset2; j < offset2 + tMat.WindowSize; ++j) - v += series[j] * singularVectors[tMat.WindowSize * i - offset2 + j]; + temp = v * singularVectors[tMat.WindowSize * (i + 1) - 1]; - for (j = tMat.WindowSize; j < 2 * tMat.WindowSize - 1; ++j) - output[j] += v * singularVectors[tMat.WindowSize * (i - 1) + j + 1]; + v = 0; + for (j = offset2; j < offset2 + tMat.WindowSize; ++j) + v += series[j] * singularVectors[tMat.WindowSize * i - offset2 + j]; - temp += v * singularVectors[tMat.WindowSize * i]; - output[tMat.WindowSize - 1] += (temp / 2); - } + for (j = tMat.WindowSize; j < 2 * tMat.WindowSize - 1; ++j) + output[j] += v * singularVectors[tMat.WindowSize * (i - 1) + j + 1]; + + temp += v * singularVectors[tMat.WindowSize * i]; + output[tMat.WindowSize - 1] += (temp / 2); } + } - private static void ComputeNoiseMoments(Single[] series, Single[] signal, Single[] alpha, out Single observationNoiseVariance, out Single autoregressionNoiseVariance, - out Single observationNoiseMean, out Single autoregressionNoiseMean, int startIndex = 0) + private static void ComputeNoiseMoments(Single[] series, Single[] signal, Single[] alpha, out Single observationNoiseVariance, out Single autoregressionNoiseVariance, + out Single observationNoiseMean, out Single autoregressionNoiseMean, int startIndex = 0) + { + Contracts.Assert(Utils.Size(alpha) > 0); + Contracts.Assert(Utils.Size(signal) > 2 * Utils.Size(alpha)); // To assure that the autoregression noise variance is unbiased. + Contracts.Assert(Utils.Size(series) >= Utils.Size(signal) + startIndex); + + var signalLength = Utils.Size(signal); + var windowSize = Utils.Size(alpha) + 1; + var k = signalLength - windowSize + 1; + Contracts.Assert(k > 0); + + var y = new Single[k]; + int i; + + observationNoiseMean = 0; + observationNoiseVariance = 0; + autoregressionNoiseMean = 0; + autoregressionNoiseVariance = 0; + + // Computing the observation noise moments + for (i = 0; i < signalLength; ++i) + observationNoiseMean += (series[i + startIndex] - signal[i]); + observationNoiseMean /= signalLength; + + for (i = 0; i < signalLength; ++i) + observationNoiseVariance += (series[i + startIndex] - signal[i]) * (series[i + startIndex] - signal[i]); + observationNoiseVariance /= signalLength; + observationNoiseVariance -= (observationNoiseMean * observationNoiseMean); + + // Computing the auto-regression noise moments + TrajectoryMatrix xTM = new TrajectoryMatrix(null, signal, windowSize - 1, signalLength - 1); + xTM.MultiplyTranspose(alpha, y); + + for (i = 0; i < k; ++i) + autoregressionNoiseMean += (signal[windowSize - 1 + i] - y[i]); + autoregressionNoiseMean /= k; + + for (i = 0; i < k; ++i) { - Contracts.Assert(Utils.Size(alpha) > 0); - Contracts.Assert(Utils.Size(signal) > 2 * Utils.Size(alpha)); // To assure that the autoregression noise variance is unbiased. - Contracts.Assert(Utils.Size(series) >= Utils.Size(signal) + startIndex); - - var signalLength = Utils.Size(signal); - var windowSize = Utils.Size(alpha) + 1; - var k = signalLength - windowSize + 1; - Contracts.Assert(k > 0); + autoregressionNoiseVariance += (signal[windowSize - 1 + i] - y[i] - autoregressionNoiseMean) * + (signal[windowSize - 1 + i] - y[i] - autoregressionNoiseMean); + } - var y = new Single[k]; - int i; + autoregressionNoiseVariance /= (k - windowSize + 1); + Contracts.Assert(autoregressionNoiseVariance >= 0); + } - observationNoiseMean = 0; - observationNoiseVariance = 0; - autoregressionNoiseMean = 0; - autoregressionNoiseVariance = 0; + private static int DetermineSignalRank(Single[] series, TrajectoryMatrix tMat, Single[] singularVectors, Single[] singularValues, + Single[] outputSignal, int maxRank) + { + Contracts.Assert(tMat != null); + Contracts.Assert(Utils.Size(series) >= tMat.SeriesLength); + Contracts.Assert(Utils.Size(outputSignal) >= tMat.SeriesLength); + Contracts.Assert(Utils.Size(singularVectors) >= tMat.WindowSize * tMat.WindowSize); + Contracts.Assert(Utils.Size(singularValues) >= tMat.WindowSize); + Contracts.Assert(1 <= maxRank && maxRank <= tMat.WindowSize - 1); + + var inputSeriesLength = tMat.SeriesLength; + var k = inputSeriesLength - tMat.WindowSize + 1; + Contracts.Assert(k > 0); + + var x = new Single[inputSeriesLength]; + var y = new Single[k]; + var alpha = new Single[tMat.WindowSize - 1]; + var v = new Single[k]; + + Single nu = 0; + Double minErr = Double.MaxValue; + int minIndex = maxRank - 1; + int evaluationLength = Math.Min(Math.Max(tMat.WindowSize, 200), k); + + TrajectoryMatrix xTM = new TrajectoryMatrix(null, x, tMat.WindowSize - 1, inputSeriesLength - 1); + + int i; + int j; + int n; + Single temp; + Double error; + Double sumSingularVals = 0; + Single lambda; + Single observationNoiseMean; + + FixedSizeQueue window = new FixedSizeQueue(tMat.WindowSize - 1); + + for (i = 0; i < tMat.WindowSize; ++i) + sumSingularVals += singularValues[i]; + + for (i = 0; i < maxRank; ++i) + { + // Updating the auto-regressive coefficients + lambda = singularVectors[tMat.WindowSize * i + tMat.WindowSize - 1]; + for (j = 0; j < tMat.WindowSize - 1; ++j) + alpha[j] += lambda * singularVectors[tMat.WindowSize * i + j]; - // Computing the observation noise moments - for (i = 0; i < signalLength; ++i) - observationNoiseMean += (series[i + startIndex] - signal[i]); - observationNoiseMean /= signalLength; + // Updating nu + nu += lambda * lambda; - for (i = 0; i < signalLength; ++i) - observationNoiseVariance += (series[i + startIndex] - signal[i]) * (series[i + startIndex] - signal[i]); - observationNoiseVariance /= signalLength; - observationNoiseVariance -= (observationNoiseMean * observationNoiseMean); + // Reconstructing the i-th eigen triple component and adding it to x + tMat.MultiplyTranspose(singularVectors, v, false, tMat.WindowSize * i, 0); + tMat.RankOneHankelization(singularVectors, v, 1, x, true, tMat.WindowSize * i, 0, 0); - // Computing the auto-regression noise moments - TrajectoryMatrix xTM = new TrajectoryMatrix(null, signal, windowSize - 1, signalLength - 1); - xTM.MultiplyTranspose(alpha, y); + observationNoiseMean = 0; + for (j = 0; j < inputSeriesLength; ++j) + observationNoiseMean += (series[j] - x[j]); + observationNoiseMean /= inputSeriesLength; - for (i = 0; i < k; ++i) - autoregressionNoiseMean += (signal[windowSize - 1 + i] - y[i]); - autoregressionNoiseMean /= k; + for (j = inputSeriesLength - evaluationLength - tMat.WindowSize + 1; j < inputSeriesLength - evaluationLength; ++j) + window.AddLast(x[j]); - for (i = 0; i < k; ++i) + error = 0; + for (j = inputSeriesLength - evaluationLength; j < inputSeriesLength; ++j) { - autoregressionNoiseVariance += (signal[windowSize - 1 + i] - y[i] - autoregressionNoiseMean) * - (signal[windowSize - 1 + i] - y[i] - autoregressionNoiseMean); + temp = 0; + for (n = 0; n < tMat.WindowSize - 1; ++n) + temp += alpha[n] * window[n]; + + temp /= (1 - nu); + temp += observationNoiseMean; + window.AddLast(temp); + error += Math.Abs(series[j] - temp); + if (error > minErr) + break; } - autoregressionNoiseVariance /= (k - windowSize + 1); - Contracts.Assert(autoregressionNoiseVariance >= 0); + if (error < minErr) + { + minErr = error; + minIndex = i; + Array.Copy(x, outputSignal, inputSeriesLength); + } } - private static int DetermineSignalRank(Single[] series, TrajectoryMatrix tMat, Single[] singularVectors, Single[] singularValues, - Single[] outputSignal, int maxRank) - { - Contracts.Assert(tMat != null); - Contracts.Assert(Utils.Size(series) >= tMat.SeriesLength); - Contracts.Assert(Utils.Size(outputSignal) >= tMat.SeriesLength); - Contracts.Assert(Utils.Size(singularVectors) >= tMat.WindowSize * tMat.WindowSize); - Contracts.Assert(Utils.Size(singularValues) >= tMat.WindowSize); - Contracts.Assert(1 <= maxRank && maxRank <= tMat.WindowSize - 1); - - var inputSeriesLength = tMat.SeriesLength; - var k = inputSeriesLength - tMat.WindowSize + 1; - Contracts.Assert(k > 0); - - var x = new Single[inputSeriesLength]; - var y = new Single[k]; - var alpha = new Single[tMat.WindowSize - 1]; - var v = new Single[k]; - - Single nu = 0; - Double minErr = Double.MaxValue; - int minIndex = maxRank - 1; - int evaluationLength = Math.Min(Math.Max(tMat.WindowSize, 200), k); - - TrajectoryMatrix xTM = new TrajectoryMatrix(null, x, tMat.WindowSize - 1, inputSeriesLength - 1); - - int i; - int j; - int n; - Single temp; - Double error; - Double sumSingularVals = 0; - Single lambda; - Single observationNoiseMean; - - FixedSizeQueue window = new FixedSizeQueue(tMat.WindowSize - 1); - - for (i = 0; i < tMat.WindowSize; ++i) - sumSingularVals += singularValues[i]; - - for (i = 0; i < maxRank; ++i) - { - // Updating the auto-regressive coefficients - lambda = singularVectors[tMat.WindowSize * i + tMat.WindowSize - 1]; - for (j = 0; j < tMat.WindowSize - 1; ++j) - alpha[j] += lambda * singularVectors[tMat.WindowSize * i + j]; + return minIndex + 1; + } - // Updating nu - nu += lambda * lambda; + internal override void InitState() + { + for (int i = 0; i < _windowSize - 2; ++i) + _state[i] = 0; - // Reconstructing the i-th eigen triple component and adding it to x - tMat.MultiplyTranspose(singularVectors, v, false, tMat.WindowSize * i, 0); - tMat.RankOneHankelization(singularVectors, v, 1, x, true, tMat.WindowSize * i, 0, 0); + _buffer.Clear(); + } - observationNoiseMean = 0; - for (j = 0; j < inputSeriesLength; ++j) - observationNoiseMean += (series[j] - x[j]); - observationNoiseMean /= inputSeriesLength; + private static int DetermineSignalRankFast(Single[] series, TrajectoryMatrix tMat, Single[] singularVectors, Single[] singularValues, int maxRank) + { + Contracts.Assert(tMat != null); + Contracts.Assert(Utils.Size(series) >= tMat.SeriesLength); + Contracts.Assert(Utils.Size(singularVectors) >= tMat.WindowSize * tMat.WindowSize); + Contracts.Assert(Utils.Size(singularValues) >= tMat.WindowSize); + Contracts.Assert(1 <= maxRank && maxRank <= tMat.WindowSize - 1); + + var inputSeriesLength = tMat.SeriesLength; + var k = inputSeriesLength - tMat.WindowSize + 1; + Contracts.Assert(k > 0); + + var x = new Single[tMat.WindowSize - 1]; + var alpha = new Single[tMat.WindowSize - 1]; + Single v; + + Single nu = 0; + Double minErr = Double.MaxValue; + int minIndex = maxRank - 1; + int evaluationLength = Math.Min(Math.Max(tMat.WindowSize, 200), k); + + int i; + int j; + int n; + int offset; + Single temp; + Double error; + Single lambda; + Single observationNoiseMean; + + FixedSizeQueue window = new FixedSizeQueue(tMat.WindowSize - 1); + + for (i = 0; i < maxRank; ++i) + { + // Updating the auto-regressive coefficients + lambda = singularVectors[tMat.WindowSize * i + tMat.WindowSize - 1]; + for (j = 0; j < tMat.WindowSize - 1; ++j) + alpha[j] += lambda * singularVectors[tMat.WindowSize * i + j]; - for (j = inputSeriesLength - evaluationLength - tMat.WindowSize + 1; j < inputSeriesLength - evaluationLength; ++j) - window.AddLast(x[j]); + // Updating nu + nu += lambda * lambda; - error = 0; - for (j = inputSeriesLength - evaluationLength; j < inputSeriesLength; ++j) - { - temp = 0; - for (n = 0; n < tMat.WindowSize - 1; ++n) - temp += alpha[n] * window[n]; - - temp /= (1 - nu); - temp += observationNoiseMean; - window.AddLast(temp); - error += Math.Abs(series[j] - temp); - if (error > minErr) - break; - } + // Reconstructing the i-th eigen triple component and adding it to x + v = 0; + offset = inputSeriesLength - evaluationLength - tMat.WindowSize + 1; - if (error < minErr) - { - minErr = error; - minIndex = i; - Array.Copy(x, outputSignal, inputSeriesLength); - } - } + for (j = offset; j <= inputSeriesLength - evaluationLength; ++j) + v += series[j] * singularVectors[tMat.WindowSize * i - offset + j]; - return minIndex + 1; - } + for (j = 0; j < tMat.WindowSize - 1; ++j) + x[j] += v * singularVectors[tMat.WindowSize * i + j]; - internal override void InitState() - { - for (int i = 0; i < _windowSize - 2; ++i) - _state[i] = 0; + // Computing the empirical observation noise mean + observationNoiseMean = 0; + for (j = offset; j < inputSeriesLength - evaluationLength; ++j) + observationNoiseMean += (series[j] - x[j - offset]); + observationNoiseMean /= (tMat.WindowSize - 1); - _buffer.Clear(); - } + for (j = 0; j < tMat.WindowSize - 1; ++j) + window.AddLast(x[j]); - private static int DetermineSignalRankFast(Single[] series, TrajectoryMatrix tMat, Single[] singularVectors, Single[] singularValues, int maxRank) - { - Contracts.Assert(tMat != null); - Contracts.Assert(Utils.Size(series) >= tMat.SeriesLength); - Contracts.Assert(Utils.Size(singularVectors) >= tMat.WindowSize * tMat.WindowSize); - Contracts.Assert(Utils.Size(singularValues) >= tMat.WindowSize); - Contracts.Assert(1 <= maxRank && maxRank <= tMat.WindowSize - 1); - - var inputSeriesLength = tMat.SeriesLength; - var k = inputSeriesLength - tMat.WindowSize + 1; - Contracts.Assert(k > 0); - - var x = new Single[tMat.WindowSize - 1]; - var alpha = new Single[tMat.WindowSize - 1]; - Single v; - - Single nu = 0; - Double minErr = Double.MaxValue; - int minIndex = maxRank - 1; - int evaluationLength = Math.Min(Math.Max(tMat.WindowSize, 200), k); - - int i; - int j; - int n; - int offset; - Single temp; - Double error; - Single lambda; - Single observationNoiseMean; - - FixedSizeQueue window = new FixedSizeQueue(tMat.WindowSize - 1); - - for (i = 0; i < maxRank; ++i) + error = 0; + for (j = inputSeriesLength - evaluationLength; j < inputSeriesLength; ++j) { - // Updating the auto-regressive coefficients - lambda = singularVectors[tMat.WindowSize * i + tMat.WindowSize - 1]; - for (j = 0; j < tMat.WindowSize - 1; ++j) - alpha[j] += lambda * singularVectors[tMat.WindowSize * i + j]; - - // Updating nu - nu += lambda * lambda; + temp = 0; + for (n = 0; n < tMat.WindowSize - 1; ++n) + temp += alpha[n] * window[n]; + + temp /= (1 - nu); + temp += observationNoiseMean; + window.AddLast(temp); + error += Math.Abs(series[j] - temp); + if (error > minErr) + break; + } - // Reconstructing the i-th eigen triple component and adding it to x - v = 0; - offset = inputSeriesLength - evaluationLength - tMat.WindowSize + 1; + if (error < minErr) + { + minErr = error; + minIndex = i; + } + } - for (j = offset; j <= inputSeriesLength - evaluationLength; ++j) - v += series[j] * singularVectors[tMat.WindowSize * i - offset + j]; + return minIndex + 1; + } - for (j = 0; j < tMat.WindowSize - 1; ++j) - x[j] += v * singularVectors[tMat.WindowSize * i + j]; + private class SignalComponent + { + public Double Phase; + public int Index; + public int Cluster; - // Computing the empirical observation noise mean - observationNoiseMean = 0; - for (j = offset; j < inputSeriesLength - evaluationLength; ++j) - observationNoiseMean += (series[j] - x[j - offset]); - observationNoiseMean /= (tMat.WindowSize - 1); + public SignalComponent(Double phase, int index) + { + Phase = phase; + Index = index; + } + } - for (j = 0; j < tMat.WindowSize - 1; ++j) - window.AddLast(x[j]); + private bool Stabilize() + { + if (Utils.Size(_alpha) == 1) + { + if (_shouldMaintainInfo) + _info.RootsBeforeStabilization = new[] { new Complex(_alpha[0], 0) }; - error = 0; - for (j = inputSeriesLength - evaluationLength; j < inputSeriesLength; ++j) - { - temp = 0; - for (n = 0; n < tMat.WindowSize - 1; ++n) - temp += alpha[n] * window[n]; - - temp /= (1 - nu); - temp += observationNoiseMean; - window.AddLast(temp); - error += Math.Abs(series[j] - temp); - if (error > minErr) - break; - } + if (_alpha[0] > 1) + _alpha[0] = 1; + else if (_alpha[0] < -1) + _alpha[0] = -1; - if (error < minErr) - { - minErr = error; - minIndex = i; - } + if (_shouldMaintainInfo) + { + _info.IsStabilized = true; + _info.RootsAfterStabilization = new[] { new Complex(_alpha[0], 0) }; + _info.IsExponentialTrendPresent = false; + _info.IsPolynomialTrendPresent = false; + _info.ExponentialTrendFactor = Math.Abs(_alpha[0]); } + return true; + } - return minIndex + 1; + var coeff = new Double[_windowSize - 1]; + Complex[] roots = null; + bool trendFound = false; + bool polynomialTrendFound = false; + Double maxTrendMagnitude = Double.NegativeInfinity; + Double maxNonTrendMagnitude = Double.NegativeInfinity; + Double eps = 1e-9; + Double highFrequenceyBoundry = Math.PI / 2; + var sortedComponents = new List(); + var trendComponents = new List(); + int i; + + // Computing the roots of the characteristic polynomial + for (i = 0; i < _windowSize - 1; ++i) + coeff[i] = -_alpha[i]; + + if (!PolynomialUtils.FindPolynomialRoots(coeff, ref roots)) + return false; + + if (_shouldMaintainInfo) + { + _info.RootsBeforeStabilization = new Complex[_windowSize - 1]; + Array.Copy(roots, _info.RootsBeforeStabilization, _windowSize - 1); } - private class SignalComponent + // Detecting trend components + for (i = 0; i < _windowSize - 1; ++i) { - public Double Phase; - public int Index; - public int Cluster; + if (roots[i].Magnitude > 1 && (Math.Abs(roots[i].Phase) <= eps || Math.Abs(Math.Abs(roots[i].Phase) - Math.PI) <= eps)) + trendComponents.Add(i); + } - public SignalComponent(Double phase, int index) + // Removing unobserved seasonalities and consequently introducing polynomial trend + for (i = 0; i < _windowSize - 1; ++i) + { + if (roots[i].Phase != 0) { - Phase = phase; - Index = index; + if (roots[i].Magnitude >= 1 && 2 * Math.PI / Math.Abs(roots[i].Phase) > _windowSize) + { + /*if (roots[i].Real > 1) + { + polynomialTrendFound = true; + roots[i] = new Complex(Math.Max(1, roots[i].Magnitude), 0); + maxPolynomialTrendMagnitude = Math.Max(maxPolynomialTrendMagnitude, roots[i].Magnitude); + } + else + roots[i] = Complex.FromPolarCoordinates(1, 0); + //roots[i] = Complex.FromPolarCoordinates(0.99, roots[i].Phase);*/ + + /* if (_maxTrendRatio > 1) + { + roots[i] = new Complex(roots[i].Real, 0); + polynomialTrendFound = true; + } + else + roots[i] = roots[i].Imaginary > 0 ? new Complex(roots[i].Real, 0) : new Complex(1, 0);*/ + + roots[i] = new Complex(roots[i].Real, 0); + polynomialTrendFound = true; + + if (_shouldMaintainInfo) + _info.IsArtificialSeasonalityRemoved = true; + } + else if (roots[i].Magnitude > 1) + sortedComponents.Add(new SignalComponent(roots[i].Phase, i)); } } - private bool Stabilize() + if (_maxTrendRatio > 1) { - if (Utils.Size(_alpha) == 1) + // Combining the close exponential-seasonal components + if (sortedComponents.Count > 1 && polynomialTrendFound) { - if (_shouldMaintainInfo) - _info.RootsBeforeStabilization = new[] { new Complex(_alpha[0], 0) }; + sortedComponents.Sort((a, b) => a.Phase.CompareTo(b.Phase)); + var clusterNum = 0; - if (_alpha[0] > 1) - _alpha[0] = 1; - else if (_alpha[0] < -1) - _alpha[0] = -1; - - if (_shouldMaintainInfo) + for (i = 0; i < sortedComponents.Count - 1; ++i) { - _info.IsStabilized = true; - _info.RootsAfterStabilization = new[] { new Complex(_alpha[0], 0) }; - _info.IsExponentialTrendPresent = false; - _info.IsPolynomialTrendPresent = false; - _info.ExponentialTrendFactor = Math.Abs(_alpha[0]); + if ((sortedComponents[i].Phase < 0 && sortedComponents[i + 1].Phase < 0) || + (sortedComponents[i].Phase > 0 && sortedComponents[i + 1].Phase > 0)) + { + sortedComponents[i].Cluster = clusterNum; + if (Math.Abs(sortedComponents[i + 1].Phase - sortedComponents[i].Phase) > 0.05) + clusterNum++; + sortedComponents[i + 1].Cluster = clusterNum; + } + else + clusterNum++; } - return true; - } - var coeff = new Double[_windowSize - 1]; - Complex[] roots = null; - bool trendFound = false; - bool polynomialTrendFound = false; - Double maxTrendMagnitude = Double.NegativeInfinity; - Double maxNonTrendMagnitude = Double.NegativeInfinity; - Double eps = 1e-9; - Double highFrequenceyBoundry = Math.PI / 2; - var sortedComponents = new List(); - var trendComponents = new List(); - int i; - - // Computing the roots of the characteristic polynomial - for (i = 0; i < _windowSize - 1; ++i) - coeff[i] = -_alpha[i]; + int start = 0; + bool polynomialSeasonalityFound = false; + Double largestSeasonalityPhase = 0; + for (i = 1; i < sortedComponents.Count; ++i) + { + if (sortedComponents[i].Cluster != sortedComponents[i - 1].Cluster) + { + if (i - start > 1) // There are more than one point in the cluster + { + Double avgPhase = 0; + Double avgMagnitude = 0; - if (!PolynomialUtils.FindPolynomialRoots(coeff, ref roots)) - return false; - - if (_shouldMaintainInfo) - { - _info.RootsBeforeStabilization = new Complex[_windowSize - 1]; - Array.Copy(roots, _info.RootsBeforeStabilization, _windowSize - 1); - } + for (var j = start; j < i; ++j) + { + avgPhase += sortedComponents[j].Phase; + avgMagnitude += roots[sortedComponents[j].Index].Magnitude; + } + avgPhase /= (i - start); + avgMagnitude /= (i - start); - // Detecting trend components - for (i = 0; i < _windowSize - 1; ++i) - { - if (roots[i].Magnitude > 1 && (Math.Abs(roots[i].Phase) <= eps || Math.Abs(Math.Abs(roots[i].Phase) - Math.PI) <= eps)) - trendComponents.Add(i); - } + for (var j = start; j < i; ++j) + roots[sortedComponents[j].Index] = Complex.FromPolarCoordinates(avgMagnitude, + avgPhase); - // Removing unobserved seasonalities and consequently introducing polynomial trend - for (i = 0; i < _windowSize - 1; ++i) - { - if (roots[i].Phase != 0) - { - if (roots[i].Magnitude >= 1 && 2 * Math.PI / Math.Abs(roots[i].Phase) > _windowSize) - { - /*if (roots[i].Real > 1) - { - polynomialTrendFound = true; - roots[i] = new Complex(Math.Max(1, roots[i].Magnitude), 0); - maxPolynomialTrendMagnitude = Math.Max(maxPolynomialTrendMagnitude, roots[i].Magnitude); + if (!polynomialSeasonalityFound && avgPhase > 0) + { + largestSeasonalityPhase = avgPhase; + polynomialSeasonalityFound = true; + } } - else - roots[i] = Complex.FromPolarCoordinates(1, 0); - //roots[i] = Complex.FromPolarCoordinates(0.99, roots[i].Phase);*/ - - /* if (_maxTrendRatio > 1) - { - roots[i] = new Complex(roots[i].Real, 0); - polynomialTrendFound = true; - } - else - roots[i] = roots[i].Imaginary > 0 ? new Complex(roots[i].Real, 0) : new Complex(1, 0);*/ - - roots[i] = new Complex(roots[i].Real, 0); - polynomialTrendFound = true; - if (_shouldMaintainInfo) - _info.IsArtificialSeasonalityRemoved = true; + start = i; } - else if (roots[i].Magnitude > 1) - sortedComponents.Add(new SignalComponent(roots[i].Phase, i)); } } - if (_maxTrendRatio > 1) + // Combining multiple exponential trends into polynomial ones + if (!polynomialTrendFound) { - // Combining the close exponential-seasonal components - if (sortedComponents.Count > 1 && polynomialTrendFound) - { - sortedComponents.Sort((a, b) => a.Phase.CompareTo(b.Phase)); - var clusterNum = 0; + var ind1 = -1; + var ind2 = -1; - for (i = 0; i < sortedComponents.Count - 1; ++i) - { - if ((sortedComponents[i].Phase < 0 && sortedComponents[i + 1].Phase < 0) || - (sortedComponents[i].Phase > 0 && sortedComponents[i + 1].Phase > 0)) - { - sortedComponents[i].Cluster = clusterNum; - if (Math.Abs(sortedComponents[i + 1].Phase - sortedComponents[i].Phase) > 0.05) - clusterNum++; - sortedComponents[i + 1].Cluster = clusterNum; - } - else - clusterNum++; - } - - int start = 0; - bool polynomialSeasonalityFound = false; - Double largestSeasonalityPhase = 0; - for (i = 1; i < sortedComponents.Count; ++i) - { - if (sortedComponents[i].Cluster != sortedComponents[i - 1].Cluster) - { - if (i - start > 1) // There are more than one point in the cluster - { - Double avgPhase = 0; - Double avgMagnitude = 0; - - for (var j = start; j < i; ++j) - { - avgPhase += sortedComponents[j].Phase; - avgMagnitude += roots[sortedComponents[j].Index].Magnitude; - } - avgPhase /= (i - start); - avgMagnitude /= (i - start); - - for (var j = start; j < i; ++j) - roots[sortedComponents[j].Index] = Complex.FromPolarCoordinates(avgMagnitude, - avgPhase); - - if (!polynomialSeasonalityFound && avgPhase > 0) - { - largestSeasonalityPhase = avgPhase; - polynomialSeasonalityFound = true; - } - } - - start = i; - } - } - } - - // Combining multiple exponential trends into polynomial ones - if (!polynomialTrendFound) + foreach (var ind in trendComponents) { - var ind1 = -1; - var ind2 = -1; - - foreach (var ind in trendComponents) - { - if (Math.Abs(roots[ind].Phase) <= eps) - { - ind1 = ind; - break; - } - } - - for (i = 0; i < _windowSize - 1; ++i) - { - if (Math.Abs(roots[i].Phase) <= eps && 0.9 <= roots[i].Magnitude && i != ind1) - { - ind2 = i; - break; - } - } - - if (ind1 >= 0 && ind2 >= 0 && ind1 != ind2) + if (Math.Abs(roots[ind].Phase) <= eps) { - roots[ind1] = Complex.FromPolarCoordinates(1, 0); - roots[ind2] = Complex.FromPolarCoordinates(1, 0); - polynomialTrendFound = true; + ind1 = ind; + break; } } - } - if (polynomialTrendFound) // Suppress the exponential trend - { - maxTrendMagnitude = Math.Min(1, _maxTrendRatio); - foreach (var ind in trendComponents) - roots[ind] = Complex.FromPolarCoordinates(0.99, roots[ind].Phase); - } - else - { - // Spotting the exponential trend components and finding the max trend magnitude for (i = 0; i < _windowSize - 1; ++i) { - if (roots[i].Magnitude > 1 && Math.Abs(roots[i].Phase) <= eps) + if (Math.Abs(roots[i].Phase) <= eps && 0.9 <= roots[i].Magnitude && i != ind1) { - trendFound = true; - maxTrendMagnitude = Math.Max(maxTrendMagnitude, roots[i].Magnitude); + ind2 = i; + break; } - else - maxNonTrendMagnitude = Math.Max(maxNonTrendMagnitude, roots[i].Magnitude); } - if (!trendFound) - maxTrendMagnitude = 1; - - maxTrendMagnitude = Math.Min(maxTrendMagnitude, _maxTrendRatio); - } - - // Squeezing all components below the maximum trend magnitude - var smallTrendMagnitude = Math.Min(maxTrendMagnitude, (maxTrendMagnitude + 1) / 2); - for (i = 0; i < _windowSize - 1; ++i) - { - if (roots[i].Magnitude >= maxTrendMagnitude) + if (ind1 >= 0 && ind2 >= 0 && ind1 != ind2) { - if ((highFrequenceyBoundry < roots[i].Phase && roots[i].Phase < Math.PI - eps) || - (-Math.PI + eps < roots[i].Phase && roots[i].Phase < -highFrequenceyBoundry)) - roots[i] = Complex.FromPolarCoordinates(smallTrendMagnitude, roots[i].Phase); - else - roots[i] = Complex.FromPolarCoordinates(maxTrendMagnitude, roots[i].Phase); + roots[ind1] = Complex.FromPolarCoordinates(1, 0); + roots[ind2] = Complex.FromPolarCoordinates(1, 0); + polynomialTrendFound = true; } } + } - // Correcting all the other trend components + if (polynomialTrendFound) // Suppress the exponential trend + { + maxTrendMagnitude = Math.Min(1, _maxTrendRatio); + foreach (var ind in trendComponents) + roots[ind] = Complex.FromPolarCoordinates(0.99, roots[ind].Phase); + } + else + { + // Spotting the exponential trend components and finding the max trend magnitude for (i = 0; i < _windowSize - 1; ++i) { - var phase = roots[i].Phase; - if (Math.Abs(phase) <= eps) - roots[i] = new Complex(roots[i].Magnitude, 0); - else if (Math.Abs(phase - Math.PI) <= eps) - roots[i] = new Complex(-roots[i].Magnitude, 0); - else if (Math.Abs(phase + Math.PI) <= eps) - roots[i] = new Complex(-roots[i].Magnitude, 0); + if (roots[i].Magnitude > 1 && Math.Abs(roots[i].Phase) <= eps) + { + trendFound = true; + maxTrendMagnitude = Math.Max(maxTrendMagnitude, roots[i].Magnitude); + } + else + maxNonTrendMagnitude = Math.Max(maxNonTrendMagnitude, roots[i].Magnitude); } - // Computing the characteristic polynomial from the modified roots - try - { - if (!PolynomialUtils.FindPolynomialCoefficients(roots, ref coeff)) - return false; - } - catch (OverflowException) - { - return false; - } + if (!trendFound) + maxTrendMagnitude = 1; - // Updating alpha - for (i = 0; i < _windowSize - 1; ++i) - _alpha[i] = (Single)(-coeff[i]); + maxTrendMagnitude = Math.Min(maxTrendMagnitude, _maxTrendRatio); + } - if (_shouldMaintainInfo) + // Squeezing all components below the maximum trend magnitude + var smallTrendMagnitude = Math.Min(maxTrendMagnitude, (maxTrendMagnitude + 1) / 2); + for (i = 0; i < _windowSize - 1; ++i) + { + if (roots[i].Magnitude >= maxTrendMagnitude) { - _info.RootsAfterStabilization = roots; - _info.IsStabilized = true; - _info.IsPolynomialTrendPresent = polynomialTrendFound; - _info.IsExponentialTrendPresent = maxTrendMagnitude > 1; - _info.ExponentialTrendFactor = maxTrendMagnitude; + if ((highFrequenceyBoundry < roots[i].Phase && roots[i].Phase < Math.PI - eps) || + (-Math.PI + eps < roots[i].Phase && roots[i].Phase < -highFrequenceyBoundry)) + roots[i] = Complex.FromPolarCoordinates(smallTrendMagnitude, roots[i].Phase); + else + roots[i] = Complex.FromPolarCoordinates(maxTrendMagnitude, roots[i].Phase); } + } - return true; + // Correcting all the other trend components + for (i = 0; i < _windowSize - 1; ++i) + { + var phase = roots[i].Phase; + if (Math.Abs(phase) <= eps) + roots[i] = new Complex(roots[i].Magnitude, 0); + else if (Math.Abs(phase - Math.PI) <= eps) + roots[i] = new Complex(-roots[i].Magnitude, 0); + else if (Math.Abs(phase + Math.PI) <= eps) + roots[i] = new Complex(-roots[i].Magnitude, 0); } - /// - /// Feeds the next observation on the series to the model and as a result changes the state of the model. - /// - /// The next observation on the series. - /// Determines whether the model parameters also need to be updated upon consuming the new observation (default = false). - internal override void Consume(ref Single input, bool updateModel = false) + // Computing the characteristic polynomial from the modified roots + try + { + if (!PolynomialUtils.FindPolynomialCoefficients(roots, ref coeff)) + return false; + } + catch (OverflowException) { - if (Single.IsNaN(input)) - return; + return false; + } - int i; + // Updating alpha + for (i = 0; i < _windowSize - 1; ++i) + _alpha[i] = (Single)(-coeff[i]); - if (_wTrans == null) - { - _y = new CpuAlignedVector(_rank, CpuMathUtils.GetVectorAlignment()); - _wTrans = new CpuAlignedMatrixRow(_rank, _windowSize, CpuMathUtils.GetVectorAlignment()); - Single[] vecs = new Single[_rank * _windowSize]; + if (_shouldMaintainInfo) + { + _info.RootsAfterStabilization = roots; + _info.IsStabilized = true; + _info.IsPolynomialTrendPresent = polynomialTrendFound; + _info.IsExponentialTrendPresent = maxTrendMagnitude > 1; + _info.ExponentialTrendFactor = maxTrendMagnitude; + } - for (i = 0; i < _rank; ++i) - vecs[(_windowSize + 1) * i] = 1; + return true; + } - i = 0; - _wTrans.CopyFrom(vecs, ref i); - } + /// + /// Feeds the next observation on the series to the model and as a result changes the state of the model. + /// + /// The next observation on the series. + /// Determines whether the model parameters also need to be updated upon consuming the new observation (default = false). + internal override void Consume(ref Single input, bool updateModel = false) + { + if (Single.IsNaN(input)) + return; - // Forming vector x + int i; - if (_buffer.Count == 0) - { - for (i = 0; i < _windowSize - 1; ++i) - _buffer.AddLast(_state[i]); - } + if (_wTrans == null) + { + _y = new CpuAlignedVector(_rank, CpuMathUtils.GetVectorAlignment()); + _wTrans = new CpuAlignedMatrixRow(_rank, _windowSize, CpuMathUtils.GetVectorAlignment()); + Single[] vecs = new Single[_rank * _windowSize]; - int len = _buffer.Count; - for (i = 0; i < _windowSize - len - 1; ++i) - _x[i] = 0; - for (i = Math.Max(0, len - _windowSize + 1); i < len; ++i) - _x[i - len + _windowSize - 1] = _buffer[i]; - _x[_windowSize - 1] = input; + for (i = 0; i < _rank; ++i) + vecs[(_windowSize + 1) * i] = 1; - // Computing y: Eq. (11) in https://hal-institut-mines-telecom.archives-ouvertes.fr/hal-00479772/file/twocolumns.pdf - CpuAligenedMathUtils.MatTimesSrc(_wTrans, _x, _y); + i = 0; + _wTrans.CopyFrom(vecs, ref i); + } - // Updating the state vector - CpuAligenedMathUtils.MatTranTimesSrc(_wTrans, _y, _xSmooth); + // Forming vector x - _nextPrediction = _autoregressionNoiseMean + _observationNoiseMean; - for (i = 0; i < _windowSize - 2; ++i) - { - _state[i] = ((_windowSize - 2 - i) * _state[i + 1] + _xSmooth[i + 1]) / (_windowSize - 1 - i); - _nextPrediction += _state[i] * _alpha[i]; - } - _state[_windowSize - 2] = _xSmooth[_windowSize - 1]; - _nextPrediction += _state[_windowSize - 2] * _alpha[_windowSize - 2]; + if (_buffer.Count == 0) + { + for (i = 0; i < _windowSize - 1; ++i) + _buffer.AddLast(_state[i]); + } - if (updateModel) - { - // REVIEW: to be implemented in the next version based on the FAPI algorithm - // in https://hal-institut-mines-telecom.archives-ouvertes.fr/hal-00479772/file/twocolumns.pdf. - } + int len = _buffer.Count; + for (i = 0; i < _windowSize - len - 1; ++i) + _x[i] = 0; + for (i = Math.Max(0, len - _windowSize + 1); i < len; ++i) + _x[i - len + _windowSize - 1] = _buffer[i]; + _x[_windowSize - 1] = input; + + // Computing y: Eq. (11) in https://hal-institut-mines-telecom.archives-ouvertes.fr/hal-00479772/file/twocolumns.pdf + CpuAligenedMathUtils.MatTimesSrc(_wTrans, _x, _y); + + // Updating the state vector + CpuAligenedMathUtils.MatTranTimesSrc(_wTrans, _y, _xSmooth); - _buffer.AddLast(input); + _nextPrediction = _autoregressionNoiseMean + _observationNoiseMean; + for (i = 0; i < _windowSize - 2; ++i) + { + _state[i] = ((_windowSize - 2 - i) * _state[i + 1] + _xSmooth[i + 1]) / (_windowSize - 1 - i); + _nextPrediction += _state[i] * _alpha[i]; } + _state[_windowSize - 2] = _xSmooth[_windowSize - 1]; + _nextPrediction += _state[_windowSize - 2] * _alpha[_windowSize - 2]; - /// - /// Train the model parameters based on a training series. - /// - /// The training time-series. - internal override void Train(FixedSizeQueue data) + if (updateModel) { - _host.CheckParam(data != null, nameof(data), "The input series for training cannot be null."); - _host.CheckParam(data.Count >= _trainSize, nameof(data), "The input series for training does not have enough points for training."); + // REVIEW: to be implemented in the next version based on the FAPI algorithm + // in https://hal-institut-mines-telecom.archives-ouvertes.fr/hal-00479772/file/twocolumns.pdf. + } - Single[] dataArray = new Single[_trainSize]; + _buffer.AddLast(input); + } - int i; - int count; - for (i = 0, count = 0; count < _trainSize && i < data.Count; ++i) - if (!Single.IsNaN(data[i])) - dataArray[count++] = data[i]; + /// + /// Train the model parameters based on a training series. + /// + /// The training time-series. + internal override void Train(FixedSizeQueue data) + { + _host.CheckParam(data != null, nameof(data), "The input series for training cannot be null."); + _host.CheckParam(data.Count >= _trainSize, nameof(data), "The input series for training does not have enough points for training."); - if (_shouldMaintainInfo) - { - _info = new ModelInfo(); - _info.WindowSize = _windowSize; - } + Single[] dataArray = new Single[_trainSize]; - if (count <= 2 * _windowSize) - { -#if !TLCSSA - using (var ch = _host.Start("Train")) - ch.Warning( - "Training cannot be completed because the input series for training does not have enough points."); -#endif - } - else - { - if (count != _trainSize) - Array.Resize(ref dataArray, count); + int i; + int count; + for (i = 0, count = 0; count < _trainSize && i < data.Count; ++i) + if (!Single.IsNaN(data[i])) + dataArray[count++] = data[i]; - TrainCore(dataArray, count); - } + if (_shouldMaintainInfo) + { + _info = new ModelInfo(); + _info.WindowSize = _windowSize; } + if (count <= 2 * _windowSize) + { #if !TLCSSA - /// - /// Train the model parameters based on a training series. - /// - /// The training time-series. - internal override void Train(RoleMappedData data) + using (var ch = _host.Start("Train")) + ch.Warning( + "Training cannot be completed because the input series for training does not have enough points."); +#endif + } + else { - _host.CheckValue(data, nameof(data)); - _host.CheckParam(data.Schema.Feature.HasValue, nameof(data), "Must have features column."); - var featureCol = data.Schema.Feature.Value; - if (featureCol.Type != NumberDataViewType.Single) - throw _host.ExceptSchemaMismatch(nameof(data), "feature", featureCol.Name, "Single", featureCol.Type.ToString()); + if (count != _trainSize) + Array.Resize(ref dataArray, count); - Single[] dataArray = new Single[_trainSize]; + TrainCore(dataArray, count); + } + } - int count = 0; - using (var cursor = data.Data.GetRowCursor(featureCol)) - { - var getVal = cursor.GetGetter(featureCol); - Single val = default; - while (cursor.MoveNext() && count < _trainSize) - { - getVal(ref val); - if (!Single.IsNaN(val)) - dataArray[count++] = val; - } - } +#if !TLCSSA + /// + /// Train the model parameters based on a training series. + /// + /// The training time-series. + internal override void Train(RoleMappedData data) + { + _host.CheckValue(data, nameof(data)); + _host.CheckParam(data.Schema.Feature.HasValue, nameof(data), "Must have features column."); + var featureCol = data.Schema.Feature.Value; + if (featureCol.Type != NumberDataViewType.Single) + throw _host.ExceptSchemaMismatch(nameof(data), "feature", featureCol.Name, "Single", featureCol.Type.ToString()); - if (_shouldMaintainInfo) - { - _info = new ModelInfo(); - _info.WindowSize = _windowSize; - } + Single[] dataArray = new Single[_trainSize]; - if (count <= 2 * _windowSize) - { - using (var ch = _host.Start("Train")) - ch.Warning("Training cannot be completed because the input series for training does not have enough points."); - } - else + int count = 0; + using (var cursor = data.Data.GetRowCursor(featureCol)) + { + var getVal = cursor.GetGetter(featureCol); + Single val = default; + while (cursor.MoveNext() && count < _trainSize) { - if (count != _trainSize) - Array.Resize(ref dataArray, count); - - TrainCore(dataArray, count); + getVal(ref val); + if (!Single.IsNaN(val)) + dataArray[count++] = val; } } -#endif - private void TrainCore(Single[] dataArray, int originalSeriesLength) + if (_shouldMaintainInfo) { - _host.Assert(Utils.Size(dataArray) > 0); - Single[] singularVals; - Single[] leftSingularVecs; - var learnNaiveModel = false; + _info = new ModelInfo(); + _info.WindowSize = _windowSize; + } - var signalLength = _rankSelectionMethod == RankSelectionMethod.Exact ? originalSeriesLength : 2 * _windowSize - 1;//originalSeriesLength; - var signal = new Single[signalLength]; + if (count <= 2 * _windowSize) + { + using (var ch = _host.Start("Train")) + ch.Warning("Training cannot be completed because the input series for training does not have enough points."); + } + else + { + if (count != _trainSize) + Array.Resize(ref dataArray, count); - int i; - // Creating the trajectory matrix for the series - TrajectoryMatrix tMat = new TrajectoryMatrix(_host, dataArray, _windowSize, originalSeriesLength); + TrainCore(dataArray, count); + } + } +#endif - // Computing the SVD of the trajectory matrix - if (!tMat.ComputeSvd(out singularVals, out leftSingularVecs)) - learnNaiveModel = true; - else + private void TrainCore(Single[] dataArray, int originalSeriesLength) + { + _host.Assert(Utils.Size(dataArray) > 0); + Single[] singularVals; + Single[] leftSingularVecs; + var learnNaiveModel = false; + + var signalLength = _rankSelectionMethod == RankSelectionMethod.Exact ? originalSeriesLength : 2 * _windowSize - 1;//originalSeriesLength; + var signal = new Single[signalLength]; + + int i; + // Creating the trajectory matrix for the series + TrajectoryMatrix tMat = new TrajectoryMatrix(_host, dataArray, _windowSize, originalSeriesLength); + + // Computing the SVD of the trajectory matrix + if (!tMat.ComputeSvd(out singularVals, out leftSingularVecs)) + learnNaiveModel = true; + else + { + for (i = 0; i < _windowSize * _maxRank; ++i) { - for (i = 0; i < _windowSize * _maxRank; ++i) + if (Single.IsNaN(leftSingularVecs[i])) { - if (Single.IsNaN(leftSingularVecs[i])) - { - learnNaiveModel = true; - break; - } + learnNaiveModel = true; + break; } } + } - // Checking for standard eigenvectors, if found reduce the window size and reset training. - if (!learnNaiveModel) + // Checking for standard eigenvectors, if found reduce the window size and reset training. + if (!learnNaiveModel) + { + for (i = 0; i < _windowSize; ++i) { - for (i = 0; i < _windowSize; ++i) + var v = leftSingularVecs[(i + 1) * _windowSize - 1]; + if (v * v == 1) { - var v = leftSingularVecs[(i + 1) * _windowSize - 1]; - if (v * v == 1) + if (_windowSize > 2) { - if (_windowSize > 2) - { - _windowSize--; - _maxRank = _windowSize / 2; - _alpha = new Single[_windowSize - 1]; - _state = new Single[_windowSize - 1]; - _x = new CpuAlignedVector(_windowSize, CpuMathUtils.GetVectorAlignment()); - _xSmooth = new CpuAlignedVector(_windowSize, CpuMathUtils.GetVectorAlignment()); - - TrainCore(dataArray, originalSeriesLength); - return; - } - else - { - learnNaiveModel = true; - break; - } + _windowSize--; + _maxRank = _windowSize / 2; + _alpha = new Single[_windowSize - 1]; + _state = new Single[_windowSize - 1]; + _x = new CpuAlignedVector(_windowSize, CpuMathUtils.GetVectorAlignment()); + _xSmooth = new CpuAlignedVector(_windowSize, CpuMathUtils.GetVectorAlignment()); + + TrainCore(dataArray, originalSeriesLength); + return; + } + else + { + learnNaiveModel = true; + break; } } } + } - // Learn the naive (averaging) model in case the eigen decomposition is not possible - if (learnNaiveModel) - { + // Learn the naive (averaging) model in case the eigen decomposition is not possible + if (learnNaiveModel) + { #if !TLCSSA - using (var ch = _host.Start("Train")) - ch.Warning("The precise SSA model cannot be trained."); + using (var ch = _host.Start("Train")) + ch.Warning("The precise SSA model cannot be trained."); #endif - _rank = 1; - var temp = (Single)(1f / Math.Sqrt(_windowSize)); - for (i = 0; i < _windowSize; ++i) - leftSingularVecs[i] = temp; - } - else - { - // Computing the signal rank - if (_rankSelectionMethod == RankSelectionMethod.Exact) - _rank = DetermineSignalRank(dataArray, tMat, leftSingularVecs, singularVals, signal, _maxRank); - else if (_rankSelectionMethod == RankSelectionMethod.Fast) - _rank = DetermineSignalRankFast(dataArray, tMat, leftSingularVecs, singularVals, _maxRank); - } + _rank = 1; + var temp = (Single)(1f / Math.Sqrt(_windowSize)); + for (i = 0; i < _windowSize; ++i) + leftSingularVecs[i] = temp; + } + else + { + // Computing the signal rank + if (_rankSelectionMethod == RankSelectionMethod.Exact) + _rank = DetermineSignalRank(dataArray, tMat, leftSingularVecs, singularVals, signal, _maxRank); + else if (_rankSelectionMethod == RankSelectionMethod.Fast) + _rank = DetermineSignalRankFast(dataArray, tMat, leftSingularVecs, singularVals, _maxRank); + } - // Setting the the y vector - _y = new CpuAlignedVector(_rank, CpuMathUtils.GetVectorAlignment()); + // Setting the the y vector + _y = new CpuAlignedVector(_rank, CpuMathUtils.GetVectorAlignment()); - // Setting the weight matrix - _wTrans = new CpuAlignedMatrixRow(_rank, _windowSize, CpuMathUtils.GetVectorAlignment()); - i = 0; - _wTrans.CopyFrom(leftSingularVecs, ref i); + // Setting the weight matrix + _wTrans = new CpuAlignedMatrixRow(_rank, _windowSize, CpuMathUtils.GetVectorAlignment()); + i = 0; + _wTrans.CopyFrom(leftSingularVecs, ref i); - // Setting alpha - Single nu = 0; - for (i = 0; i < _rank; ++i) - { - _y[i] = leftSingularVecs[_windowSize * (i + 1) - 1]; - nu += _y[i] * _y[i]; - } + // Setting alpha + Single nu = 0; + for (i = 0; i < _rank; ++i) + { + _y[i] = leftSingularVecs[_windowSize * (i + 1) - 1]; + nu += _y[i] * _y[i]; + } - CpuAligenedMathUtils.MatTranTimesSrc(_wTrans, _y, _xSmooth); - for (i = 0; i < _windowSize - 1; ++i) - _alpha[i] = _xSmooth[i] / (1 - nu); + CpuAligenedMathUtils.MatTranTimesSrc(_wTrans, _y, _xSmooth); + for (i = 0; i < _windowSize - 1; ++i) + _alpha[i] = _xSmooth[i] / (1 - nu); - // Stabilizing the model - if (_shouldStablize && !learnNaiveModel) + // Stabilizing the model + if (_shouldStablize && !learnNaiveModel) + { + if (!Stabilize()) { - if (!Stabilize()) - { #if !TLCSSA - using (var ch = _host.Start("Train")) - ch.Warning("The trained model cannot be stablized."); + using (var ch = _host.Start("Train")) + ch.Warning("The trained model cannot be stablized."); #endif - } } + } - // Computing the noise moments - if (ShouldComputeForecastIntervals) - { - if (_rankSelectionMethod != RankSelectionMethod.Exact) - ReconstructSignalTailFast(dataArray, tMat, leftSingularVecs, _rank, signal); - - ComputeNoiseMoments(dataArray, signal, _alpha, out _observationNoiseVariance, out _autoregressionNoiseVariance, - out _observationNoiseMean, out _autoregressionNoiseMean, originalSeriesLength - signalLength); - _observationNoiseMean = 0; - _autoregressionNoiseMean = 0; - } + // Computing the noise moments + if (ShouldComputeForecastIntervals) + { + if (_rankSelectionMethod != RankSelectionMethod.Exact) + ReconstructSignalTailFast(dataArray, tMat, leftSingularVecs, _rank, signal); - // Setting the state - _nextPrediction = _autoregressionNoiseMean + _observationNoiseMean; + ComputeNoiseMoments(dataArray, signal, _alpha, out _observationNoiseVariance, out _autoregressionNoiseVariance, + out _observationNoiseMean, out _autoregressionNoiseMean, originalSeriesLength - signalLength); + _observationNoiseMean = 0; + _autoregressionNoiseMean = 0; + } - if (_buffer.Count > 0) // Use the buffer to set the state when there are data points pushed into the buffer using the Consume() method - { - int len = _buffer.Count; - for (i = 0; i < _windowSize - len; ++i) - _x[i] = 0; - for (i = Math.Max(0, len - _windowSize); i < len; ++i) - _x[i - len + _windowSize] = _buffer[i]; - } - else // use the training data points otherwise - { - for (i = originalSeriesLength - _windowSize; i < originalSeriesLength; ++i) - _x[i - originalSeriesLength + _windowSize] = dataArray[i]; - } + // Setting the state + _nextPrediction = _autoregressionNoiseMean + _observationNoiseMean; - CpuAligenedMathUtils.MatTimesSrc(_wTrans, _x, _y); - CpuAligenedMathUtils.MatTranTimesSrc(_wTrans, _y, _xSmooth); + if (_buffer.Count > 0) // Use the buffer to set the state when there are data points pushed into the buffer using the Consume() method + { + int len = _buffer.Count; + for (i = 0; i < _windowSize - len; ++i) + _x[i] = 0; + for (i = Math.Max(0, len - _windowSize); i < len; ++i) + _x[i - len + _windowSize] = _buffer[i]; + } + else // use the training data points otherwise + { + for (i = originalSeriesLength - _windowSize; i < originalSeriesLength; ++i) + _x[i - originalSeriesLength + _windowSize] = dataArray[i]; + } - for (i = 1; i < _windowSize; ++i) - { - _state[i - 1] = _xSmooth[i]; - _nextPrediction += _state[i - 1] * _alpha[i - 1]; - } + CpuAligenedMathUtils.MatTimesSrc(_wTrans, _x, _y); + CpuAligenedMathUtils.MatTranTimesSrc(_wTrans, _y, _xSmooth); - if (_shouldMaintainInfo) - { - _info.IsTrained = true; - _info.WindowSize = _windowSize; - _info.AutoRegressiveCoefficients = new Single[_windowSize - 1]; - Array.Copy(_alpha, _info.AutoRegressiveCoefficients, _windowSize - 1); - _info.Rank = _rank; - _info.IsNaiveModelTrained = learnNaiveModel; - _info.Spectrum = singularVals; - } + for (i = 1; i < _windowSize; ++i) + { + _state[i - 1] = _xSmooth[i]; + _nextPrediction += _state[i - 1] * _alpha[i - 1]; } - /// - /// Forecasts the future values of the series up to the given horizon. - /// - /// The forecast result. - /// The forecast horizon. - internal override void Forecast(ref ForecastResultBase result, int horizon = 1) + if (_shouldMaintainInfo) { - _host.CheckParam(horizon >= 1, nameof(horizon), "The horizon parameter should be greater than 0."); - if (result == null) - result = new SsaForecastResult(); - - var str = "The result argument must be of type " + typeof(SsaForecastResult).ToString(); - _host.CheckParam(result is SsaForecastResult, nameof(result), str); + _info.IsTrained = true; + _info.WindowSize = _windowSize; + _info.AutoRegressiveCoefficients = new Single[_windowSize - 1]; + Array.Copy(_alpha, _info.AutoRegressiveCoefficients, _windowSize - 1); + _info.Rank = _rank; + _info.IsNaiveModelTrained = learnNaiveModel; + _info.Spectrum = singularVals; + } + } - var output = result as SsaForecastResult; + /// + /// Forecasts the future values of the series up to the given horizon. + /// + /// The forecast result. + /// The forecast horizon. + internal override void Forecast(ref ForecastResultBase result, int horizon = 1) + { + _host.CheckParam(horizon >= 1, nameof(horizon), "The horizon parameter should be greater than 0."); + if (result == null) + result = new SsaForecastResult(); - var resEditor = VBufferEditor.Create(ref result.PointForecast, horizon); + var str = "The result argument must be of type " + typeof(SsaForecastResult).ToString(); + _host.CheckParam(result is SsaForecastResult, nameof(result), str); - int i; - int j; - int k; + var output = result as SsaForecastResult; - // Computing the point forecasts - resEditor.Values[0] = _nextPrediction; - for (i = 1; i < horizon; ++i) - { - k = 0; - resEditor.Values[i] = _autoregressionNoiseMean + _observationNoiseMean; - for (j = i; j < _windowSize - 1; ++j, ++k) - resEditor.Values[i] += _state[j] * _alpha[k]; + var resEditor = VBufferEditor.Create(ref result.PointForecast, horizon); - for (j = Math.Max(0, i - _windowSize + 1); j < i; ++j, ++k) - resEditor.Values[i] += resEditor.Values[j] * _alpha[k]; - } + int i; + int j; + int k; - // Computing the forecast variances - if (ShouldComputeForecastIntervals) - { - var sdEditor = VBufferEditor.Create(ref output.ForecastStandardDeviation, horizon); - var lastCol = new FixedSizeQueue(_windowSize - 1); + // Computing the point forecasts + resEditor.Values[0] = _nextPrediction; + for (i = 1; i < horizon; ++i) + { + k = 0; + resEditor.Values[i] = _autoregressionNoiseMean + _observationNoiseMean; + for (j = i; j < _windowSize - 1; ++j, ++k) + resEditor.Values[i] += _state[j] * _alpha[k]; - for (i = 0; i < _windowSize - 3; ++i) - lastCol.AddLast(0); - lastCol.AddLast(1); - lastCol.AddLast(_alpha[_windowSize - 2]); - sdEditor.Values[0] = _autoregressionNoiseVariance + _observationNoiseVariance; + for (j = Math.Max(0, i - _windowSize + 1); j < i; ++j, ++k) + resEditor.Values[i] += resEditor.Values[j] * _alpha[k]; + } - for (i = 1; i < horizon; ++i) - { - Single temp = 0; - for (j = 0; j < _windowSize - 1; ++j) - temp += _alpha[j] * lastCol[j]; - lastCol.AddLast(temp); + // Computing the forecast variances + if (ShouldComputeForecastIntervals) + { + var sdEditor = VBufferEditor.Create(ref output.ForecastStandardDeviation, horizon); + var lastCol = new FixedSizeQueue(_windowSize - 1); - sdEditor.Values[i] = sdEditor.Values[i - 1] + _autoregressionNoiseVariance * temp * temp; - } + for (i = 0; i < _windowSize - 3; ++i) + lastCol.AddLast(0); + lastCol.AddLast(1); + lastCol.AddLast(_alpha[_windowSize - 2]); + sdEditor.Values[0] = _autoregressionNoiseVariance + _observationNoiseVariance; - for (i = 0; i < horizon; ++i) - sdEditor.Values[i] = (float)Math.Sqrt(sdEditor.Values[i]); + for (i = 1; i < horizon; ++i) + { + Single temp = 0; + for (j = 0; j < _windowSize - 1; ++j) + temp += _alpha[j] * lastCol[j]; + lastCol.AddLast(temp); - output.ForecastStandardDeviation = sdEditor.Commit(); + sdEditor.Values[i] = sdEditor.Values[i - 1] + _autoregressionNoiseVariance * temp * temp; } - result.PointForecast = resEditor.Commit(); - output.CanComputeForecastIntervals = ShouldComputeForecastIntervals; - output.BoundOffset = 0; - } - - /// - /// Predicts the next value on the series. - /// - /// The prediction result. - internal override void PredictNext(ref Single output) - { - output = _nextPrediction; - } + for (i = 0; i < horizon; ++i) + sdEditor.Values[i] = (float)Math.Sqrt(sdEditor.Values[i]); - internal override SequenceModelerBase Clone() - { - return new AdaptiveSingularSpectrumSequenceModelerInternal(this); + output.ForecastStandardDeviation = sdEditor.Commit(); } - /// - /// Computes the forecast intervals for the input forecast object at the given confidence level. The results are stored in the forecast object. - /// - /// The input forecast object - /// The confidence level in [0, 1) - internal static void ComputeForecastIntervals(ref SsaForecastResult forecast, Single confidenceLevel = 0.95f) - { - Contracts.CheckParam(0 <= confidenceLevel && confidenceLevel < 1, nameof(confidenceLevel), "The confidence level must be in [0, 1)."); - Contracts.CheckValue(forecast, nameof(forecast)); - Contracts.Check(forecast.CanComputeForecastIntervals, "The forecast intervals cannot be computed for this forecast object."); - - var meanForecast = forecast.PointForecast.GetValues(); - var horizon = meanForecast.Length; - var sdForecast = forecast.ForecastStandardDeviation.GetValues(); - Contracts.Check(sdForecast.Length >= horizon, "The forecast standard deviation values are not available."); + result.PointForecast = resEditor.Commit(); + output.CanComputeForecastIntervals = ShouldComputeForecastIntervals; + output.BoundOffset = 0; + } - forecast.ConfidenceLevel = confidenceLevel; - if (horizon == 0) - return; + /// + /// Predicts the next value on the series. + /// + /// The prediction result. + internal override void PredictNext(ref Single output) + { + output = _nextPrediction; + } - var upper = VBufferEditor.Create(ref forecast.UpperBound, horizon); - var lower = VBufferEditor.Create(ref forecast.LowerBound, horizon); + internal override SequenceModelerBase Clone() + { + return new AdaptiveSingularSpectrumSequenceModelerInternal(this); + } - var z = ProbabilityFunctions.Probit(0.5 + confidenceLevel / 2.0); - double temp; + /// + /// Computes the forecast intervals for the input forecast object at the given confidence level. The results are stored in the forecast object. + /// + /// The input forecast object + /// The confidence level in [0, 1) + internal static void ComputeForecastIntervals(ref SsaForecastResult forecast, Single confidenceLevel = 0.95f) + { + Contracts.CheckParam(0 <= confidenceLevel && confidenceLevel < 1, nameof(confidenceLevel), "The confidence level must be in [0, 1)."); + Contracts.CheckValue(forecast, nameof(forecast)); + Contracts.Check(forecast.CanComputeForecastIntervals, "The forecast intervals cannot be computed for this forecast object."); - for (int i = 0; i < horizon; ++i) - { - temp = z * sdForecast[i]; - upper.Values[i] = (Single)(meanForecast[i] + forecast.BoundOffset + temp); - lower.Values[i] = (Single)(meanForecast[i] + forecast.BoundOffset - temp); - } + var meanForecast = forecast.PointForecast.GetValues(); + var horizon = meanForecast.Length; + var sdForecast = forecast.ForecastStandardDeviation.GetValues(); + Contracts.Check(sdForecast.Length >= horizon, "The forecast standard deviation values are not available."); - forecast.UpperBound = upper.Commit(); - forecast.LowerBound = lower.Commit(); - } + forecast.ConfidenceLevel = confidenceLevel; + if (horizon == 0) + return; - public void Train(IDataView dataView, string inputColumnName) => Train(new RoleMappedData(dataView, null, inputColumnName)); + var upper = VBufferEditor.Create(ref forecast.UpperBound, horizon); + var lower = VBufferEditor.Create(ref forecast.LowerBound, horizon); - public float[] Forecast(int horizon) - { - ForecastResultBase result = null; - Forecast(ref result, horizon); - return result.PointForecast.GetValues().ToArray(); - } + var z = ProbabilityFunctions.Probit(0.5 + confidenceLevel / 2.0); + double temp; - public void ForecastWithConfidenceIntervals(int horizon, out float[] forecast, out float[] confidenceIntervalLowerBounds, out float[] confidenceIntervalUpperBounds, float confidenceLevel = 0.95f) + for (int i = 0; i < horizon; ++i) { - ForecastResultBase result = null; - Forecast(ref result, horizon); - SsaForecastResult ssaResult = (SsaForecastResult)result; - ComputeForecastIntervals(ref ssaResult, confidenceLevel); - forecast = result.PointForecast.GetValues().ToArray(); - confidenceIntervalLowerBounds = ssaResult.LowerBound.GetValues().ToArray(); - confidenceIntervalUpperBounds = ssaResult.UpperBound.GetValues().ToArray(); + temp = z * sdForecast[i]; + upper.Values[i] = (Single)(meanForecast[i] + forecast.BoundOffset + temp); + lower.Values[i] = (Single)(meanForecast[i] + forecast.BoundOffset - temp); } - public void Update(IDataView dataView, string inputColumnName) - { - _host.CheckParam(dataView != null, nameof(dataView), "The input series for updating cannot be null."); + forecast.UpperBound = upper.Commit(); + forecast.LowerBound = lower.Commit(); + } - var data = new RoleMappedData(dataView, null, inputColumnName); - if (data.Schema.Feature.Value.Type != NumberDataViewType.Single) - throw _host.ExceptUserArg(nameof(data.Schema.Feature.Value.Name), "The time series input column has " + - "type '{0}', but must be a float.", data.Schema.Feature.Value.Type); + public void Train(IDataView dataView, string inputColumnName) => Train(new RoleMappedData(dataView, null, inputColumnName)); - var col = data.Schema.Feature.Value; - using (var cursor = data.Data.GetRowCursor(data.Data.Schema)) - { - var getVal = cursor.GetGetter(col); - Single val = default(Single); - while (cursor.MoveNext()) - { - getVal(ref val); - if (!Single.IsNaN(val)) - Consume(ref val); - } - } - } + public float[] Forecast(int horizon) + { + ForecastResultBase result = null; + Forecast(ref result, horizon); + return result.PointForecast.GetValues().ToArray(); } - /// - /// Train a forecasting model from an . - /// - /// Reference to the - public void Train(IDataView dataView) => _modeler.Train(dataView, _inputColumnName); + public void ForecastWithConfidenceIntervals(int horizon, out float[] forecast, out float[] confidenceIntervalLowerBounds, out float[] confidenceIntervalUpperBounds, float confidenceLevel = 0.95f) + { + ForecastResultBase result = null; + Forecast(ref result, horizon); + SsaForecastResult ssaResult = (SsaForecastResult)result; + ComputeForecastIntervals(ref ssaResult, confidenceLevel); + forecast = result.PointForecast.GetValues().ToArray(); + confidenceIntervalLowerBounds = ssaResult.LowerBound.GetValues().ToArray(); + confidenceIntervalUpperBounds = ssaResult.UpperBound.GetValues().ToArray(); + } - /// - /// Update a forecasting model with the new observations in the form of an . - /// - /// Reference to the observations as an - /// Name of the input column to update from. If null then input column name specified at model initiation is taken as default. - public void Update(IDataView dataView, string inputColumnName = null) => _modeler.Update(dataView, inputColumnName ?? _inputColumnName); + public void Update(IDataView dataView, string inputColumnName) + { + _host.CheckParam(dataView != null, nameof(dataView), "The input series for updating cannot be null."); - /// - /// Perform forecasting until a particular . - /// - /// Number of values to forecast. - /// Forecasted values. - public float[] Forecast(int horizon) => _modeler.Forecast(horizon); + var data = new RoleMappedData(dataView, null, inputColumnName); + if (data.Schema.Feature.Value.Type != NumberDataViewType.Single) + throw _host.ExceptUserArg(nameof(data.Schema.Feature.Value.Name), "The time series input column has " + + "type '{0}', but must be a float.", data.Schema.Feature.Value.Type); - /// - /// For saving a model into a repository. - /// - public void Save(ModelSaveContext ctx) - { - _host.CheckValue(ctx, nameof(ctx)); - ctx.CheckAtModel(); - ctx.SetVersionInfo(GetVersionInfo()); - ctx.Writer.Write(_inputColumnName); - ctx.SaveModel(_modeler, "ForecastWrapper"); + var col = data.Schema.Feature.Value; + using (var cursor = data.Data.GetRowCursor(data.Data.Schema)) + { + var getVal = cursor.GetGetter(col); + Single val = default(Single); + while (cursor.MoveNext()) + { + getVal(ref val); + if (!Single.IsNaN(val)) + Consume(ref val); + } + } } - - /// - /// Perform forecasting until a particular and also computes confidence intervals. - /// For confidence intervals to be computed the model must be trained with - /// set to true. - /// - /// Number of values to forecast. - /// Forecasted values - /// Lower bound confidence intervals of forecasted values. - /// Upper bound confidence intervals of forecasted values. - /// Forecast confidence level. - public void ForecastWithConfidenceIntervals(int horizon, out float[] forecast, out float[] confidenceIntervalLowerBounds, out float[] confidenceIntervalUpperBounds, float confidenceLevel = 0.95f) => - _modeler.ForecastWithConfidenceIntervals(horizon, out forecast, out confidenceIntervalLowerBounds, out confidenceIntervalUpperBounds, confidenceLevel); } } diff --git a/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs b/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs index e36633d7b5..11ffa01017 100644 --- a/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs +++ b/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs @@ -2,10 +2,8 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -using System; using Microsoft.ML.Data; using Microsoft.ML.Transforms.TimeSeries; -using static Microsoft.ML.Transforms.TimeSeries.AdaptiveSingularSpectrumSequenceModeler; namespace Microsoft.ML { @@ -149,23 +147,30 @@ public static SrCnnAnomalyEstimator DetectAnomalyBySrCnn(this TransformsCatalog => new SrCnnAnomalyEstimator(CatalogUtils.GetEnvironment(catalog), outputColumnName, windowSize, backAddWindowSize, lookaheadWindowSize, averageingWindowSize, judgementWindowSize, threshold, inputColumnName); /// - /// Singular Spectrum Analysis (SSA) model for modeling univariate time-series. + /// Singular Spectrum Analysis (SSA) model for univariate time-series forecasting. /// For the details of the model, refer to http://arxiv.org/pdf/1206.6910.pdf. /// /// Catalog. - /// The name of the column on which forecasting needs to be performed. + /// Name of the column resulting from the transformation of . + /// Name of column to transform. If set to , the value of the will be used as source. + /// The vector contains Alert, Raw Score, P-Value as first three values. + /// The length of the window on the series for building the trajectory matrix (parameter L). + /// The length of series that is kept in buffer for modeling (parameter N). /// The length of series from the begining used for training. - /// The length of series that is kept in buffer for modeling (parameter N from reference papar). - /// The length of the window on the series for building the trajectory matrix (parameter L from reference papar). - /// The discount factor in [0,1] used for online updates (default = 1). - /// The rank selection method (default = Exact). - /// The desired rank of the subspace used for SSA projection (parameter r from reference papar). This parameter should be in the range in [1, ]. - /// If set to null, the rank is automatically determined based on prediction error minimization. (default = null) - /// The maximum rank considered during the rank selection process. If not provided (i.e. set to null), it is set to - 1. - /// The flag determining whether the confidence bounds for the point forecasts should be computed. (default = ) - /// The flag determining whether the model should be stabilized. + /// The number of values to forecast. + /// The flag determing whether the model is adaptive. + /// The discount factor in [0,1] used for online updates. + /// The rank selection method. + /// The desired rank of the subspace used for SSA projection (parameter r). This parameter should be in the range in [1, windowSize]. + /// If set to null, the rank is automatically determined based on prediction error minimization. + /// The maximum rank considered during the rank selection process. If not provided (i.e. set to null), it is set to windowSize - 1. + /// The flag determining whether the model should be stabilized. /// The flag determining whether the meta information for the model needs to be maintained. - /// The maximum growth on the exponential trend + /// The maximum growth on the exponential trend. + /// The name of the confidence interval lower bound column. If not specified then confidence intervals will not be calculated. + /// The name of the confidence interval upper bound column. If not specified then confidence intervals will not be calculated. + /// The confidence level for forecasting. + /// Set this to true if horizon will change after training(at prediction time). /// /// /// /// /// - public static AdaptiveSingularSpectrumSequenceModeler AdaptiveSingularSpectrumSequenceModeler(this ForecastingCatalog catalog, - string inputColumnName, int trainSize, int seriesLength, int windowSize, Single discountFactor = 1, RankSelectionMethod rankSelectionMethod = RankSelectionMethod.Exact, - int? rank = null, int? maxRank = null, bool shouldComputeForecastIntervals = true, bool shouldstablize = true, bool shouldMaintainInfo = false, GrowthRatio? maxGrowth = null) => - new AdaptiveSingularSpectrumSequenceModeler(CatalogUtils.GetEnvironment(catalog), inputColumnName, trainSize, seriesLength, windowSize, discountFactor, - rankSelectionMethod, rank, maxRank, shouldComputeForecastIntervals, shouldstablize, shouldMaintainInfo, maxGrowth); + public static SsaForecastingEstimator ForecastBySsa( + this ForecastingCatalog catalog, string outputColumnName, string inputColumnName, int windowSize, int seriesLength, int trainSize, int horizon, + bool isAdaptive = false, float discountFactor = 1, RankSelectionMethod rankSelectionMethod = RankSelectionMethod.Exact, int? rank = null, + int? maxRank = null, bool shouldStabilize = true, bool shouldMaintainInfo = false, GrowthRatio? maxGrowth = null, string confidenceLowerBoundColumn = null, + string confidenceUpperBoundColumn = null, float confidenceLevel = 0.95f, bool variableHorizon = false) => + new SsaForecastingEstimator(CatalogUtils.GetEnvironment(catalog), outputColumnName, inputColumnName, windowSize, seriesLength, trainSize, + horizon, isAdaptive, discountFactor, rankSelectionMethod, rank, maxRank, shouldStabilize, shouldMaintainInfo, maxGrowth, confidenceLowerBoundColumn, + confidenceUpperBoundColumn, confidenceLevel, variableHorizon); } } diff --git a/src/Microsoft.ML.TimeSeries/Forecast.cs b/src/Microsoft.ML.TimeSeries/Forecast.cs deleted file mode 100644 index 9321d115f0..0000000000 --- a/src/Microsoft.ML.TimeSeries/Forecast.cs +++ /dev/null @@ -1,93 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System.IO; -using Microsoft.ML.Data; -using static Microsoft.ML.Transforms.TimeSeries.AdaptiveSingularSpectrumSequenceModeler; - -namespace Microsoft.ML.TimeSeries -{ - /// - /// Interface for forecasting models. - /// - /// The type of values that are forecasted. - public interface ICanForecast : ICanSaveModel - { - /// - /// Train a forecasting model from an . - /// - /// Training data. - void Train(IDataView dataView); - - /// - /// Update a forecasting model with the new observations in the form of an . - /// - /// Reference to the observations as an - /// Name of the input column to update from. - void Update(IDataView dataView, string inputColumnName = null); - - /// - /// Perform forecasting until a particular . - /// - /// Number of values to forecast. - /// Forecasted values. - T[] Forecast(int horizon); - - /// - /// Perform forecasting until a particular and also computes confidence intervals. - /// - /// Number of values to forecast. - /// Forecasted values - /// Lower bound confidence intervals of forecasted values. - /// Upper bound confidence intervals of forecasted values. - /// Forecast confidence level. - void ForecastWithConfidenceIntervals(int horizon, out T[] forecast, out float[] confidenceIntervalLowerBounds, out float[] confidenceIntervalUpperBounds, float confidenceLevel = 0.95f); - } - - public static class ForecastExtensions - { - /// - /// Load a model. - /// - /// The type of , usually float. - /// - /// File path to load the model from. - /// model. - public static ICanForecast LoadForecastingModel(this ModelOperationsCatalog catalog, string filePath) - { - var env = CatalogUtils.GetEnvironment(catalog); - using (var file = File.OpenRead(filePath)) - { - using (var rep = RepositoryReader.Open(file, env)) - { - ModelLoadContext.LoadModel, SignatureLoadModel>(env, out var model, rep, LoaderSignature); - return model; - } - } - } - - /// - /// Save a model to a file specified by - /// - /// - /// - /// model to save. - /// File path to save the model to. - public static void SaveForecastingModel(this ModelOperationsCatalog catalog, ICanForecast model, string filePath) - { - var env = CatalogUtils.GetEnvironment(catalog); - using (var file = File.Create(filePath)) - { - using (var ch = env.Start("Saving forecasting model.")) - { - using (var rep = RepositoryWriter.CreateNew(file, ch)) - { - ModelSaveContext.SaveModel(rep, model, LoaderSignature); - rep.Commit(); - } - } - } - } - } -} diff --git a/src/Microsoft.ML.TimeSeries/Microsoft.ML.TimeSeries.csproj b/src/Microsoft.ML.TimeSeries/Microsoft.ML.TimeSeries.csproj index eb00b14674..c5b4550c51 100644 --- a/src/Microsoft.ML.TimeSeries/Microsoft.ML.TimeSeries.csproj +++ b/src/Microsoft.ML.TimeSeries/Microsoft.ML.TimeSeries.csproj @@ -10,5 +10,4 @@ - diff --git a/src/Microsoft.ML.TimeSeries/PredictionFunction.cs b/src/Microsoft.ML.TimeSeries/PredictionEngine.cs similarity index 66% rename from src/Microsoft.ML.TimeSeries/PredictionFunction.cs rename to src/Microsoft.ML.TimeSeries/PredictionEngine.cs index 310be9bd36..75bcf88407 100644 --- a/src/Microsoft.ML.TimeSeries/PredictionFunction.cs +++ b/src/Microsoft.ML.TimeSeries/PredictionEngine.cs @@ -26,6 +26,7 @@ internal interface IStatefulTransformer : ITransformer /// /// Creates a clone of the transfomer. Used for taking the snapshot of the state. + /// This is used to create multiple time series with their own state. /// /// IStatefulTransformer Clone(); @@ -33,14 +34,22 @@ internal interface IStatefulTransformer : ITransformer internal abstract class StatefulRow : DataViewRow { - public abstract Action GetPinger(); + public abstract Action GetPinger(); } internal interface IStatefulRowMapper : IRowMapper { void CloneState(); - Action CreatePinger(DataViewRow input, Func activeOutput, out Action disposer); + Action CreatePinger(DataViewRow input, Func activeOutput, out Action disposer); + } + + internal class PingerArgument + { + public long RowPosition; + public float? ConfidenceLevel; + public int? Horizon; + public bool DontConsumeSource; } /// @@ -51,16 +60,16 @@ internal interface IStatefulRowMapper : IRowMapper /// /// The user-defined type that holds the example. /// The user-defined type that holds the prediction. - public sealed class TimeSeriesPredictionFunction : PredictionEngineBase + public sealed class TimeSeriesPredictionEngine : PredictionEngineBase where TSrc : class where TDst : class, new() { - private Action _pinger; + private Action _pinger; private long _rowPosition; private ITransformer InputTransformer { get; set; } /// - /// Checkpoints to disk with the updated + /// Checkpoints to disk with the updated /// state. /// /// Usually . @@ -83,7 +92,7 @@ public void CheckPoint(IHostEnvironment env, string modelPath) } /// - /// Checkpoints to a with the updated + /// Checkpoints to a with the updated /// state. /// /// Usually . @@ -102,14 +111,14 @@ public void CheckPoint(IHostEnvironment env, Stream stream) env.CheckParam(stream != null, nameof(stream)); if (Transformer is ITransformerChainAccessor) - { + { - new TransformerChain - (((ITransformerChainAccessor)Transformer).Transformers, - ((ITransformerChainAccessor)Transformer).Scopes).SaveTo(env, stream); - } - else - Transformer.SaveTo(env, stream); + new TransformerChain + (((ITransformerChainAccessor)Transformer).Transformers, + ((ITransformerChainAccessor)Transformer).Scopes).SaveTo(env, stream); + } + else + Transformer.SaveTo(env, stream); } private static ITransformer CloneTransformers(ITransformer transformer) @@ -132,10 +141,10 @@ private static ITransformer CloneTransformers(ITransformer transformer) } /// - /// Contructor for creating time series specific prediction engine. It allows update the time series model to be updated with the observations + /// Contructor for creating time series specific prediction engine. It allows the time series model to be updated with the observations /// seen at prediction time via /// - public TimeSeriesPredictionFunction(IHostEnvironment env, ITransformer transformer, bool ignoreMissingColumns, + public TimeSeriesPredictionEngine(IHostEnvironment env, ITransformer transformer, bool ignoreMissingColumns, SchemaDefinition inputSchemaDefinition = null, SchemaDefinition outputSchemaDefinition = null) : base(env, CloneTransformers(transformer), ignoreMissingColumns, inputSchemaDefinition, outputSchemaDefinition) { @@ -188,11 +197,11 @@ internal DataViewRow GetStatefulRows(DataViewRow input, IRowToRowMapper mapper, return result; } - private Action CreatePinger(List rows) + internal Action CreatePinger(List rows) { if (rows.Count == 0) return position => { }; - Action pinger = null; + Action pinger = null; foreach (var row in rows) pinger += row.GetPinger(); return pinger; @@ -255,32 +264,113 @@ private protected override Func TransformerChec } /// - /// Run prediction pipeline on one example. + /// Performs prediction. In the case of forecasting only task can be left as null. + /// If is not null then it could be used to update forecasting models with new obervation. + /// For anomaly detection the model is always updated with . /// - /// The example to run on. - /// The object to store the prediction in. If it's null, a new one will be created, otherwise the old one - /// is reused. - public override void Predict(TSrc example, ref TDst prediction) + /// Input to the prediction engine. + /// Forecasting/Prediction from the engine. + /// Used to indicate the number of values to forecast. + /// Used in forecasting model for confidence. + public void Predict(TSrc example, ref TDst prediction, int? horizon = null, float? confidenceLevel = null) { - Contracts.CheckValue(example, nameof(example)); - ExtractValues(example); - if (prediction == null) - prediction = new TDst(); + if (example != null && prediction != null) + { + //Update models and make a prediction after updating. + Contracts.CheckValue(example, nameof(example)); + ExtractValues(example); + + // Update state. + _pinger(new PingerArgument() + { + RowPosition = _rowPosition, + ConfidenceLevel = confidenceLevel, + Horizon = horizon + }); - // Update state. - _pinger(_rowPosition); + // Predict. + FillValues(prediction); - // Predict. - FillValues(prediction); + _rowPosition++; + } + else if (prediction != null) + { + //Forecast. + + // Signal all time series models to not fetch src values in getters. + _pinger(new PingerArgument() + { + RowPosition = _rowPosition, + DontConsumeSource = true, + ConfidenceLevel = confidenceLevel, + Horizon = horizon + }); + + // Predict. The expectation is user has asked for columns that are + // forecasting columns and hence will not trigger a getter that needs an input. + FillValues(prediction); + } + else if (example != null) + { + //Update models. - _rowPosition++; + //Extract value that needs to propagated to all the models. + Contracts.CheckValue(example, nameof(example)); + ExtractValues(example); + + // Update state. + _pinger(new PingerArgument() + { + RowPosition = _rowPosition, + ConfidenceLevel = confidenceLevel, + Horizon = horizon + }); + + _rowPosition++; + } + } + + /// + /// Performs prediction. In the case of forecasting only task can be left as null. + /// If is not null then it could be used to update forecasting models with new obervation. + /// For anomaly detection the model is always updated with . + /// + /// Input to the prediction engine. + /// Forecasting/Prediction from the engine. + public override void Predict(TSrc example, ref TDst prediction) => Predict(example, ref prediction); + + /// + /// Performs prediction. In the case of forecasting only task can be left as null. + /// If is not null then it could be used to update forecasting models with new obervation. + /// + /// Input to the prediction engine. + /// Number of values to forecast. + /// Confidence level for forecasting. + /// Prediction/Forecasting after the model has been updated with + public TDst Predict(TSrc example, int? horizon = null, float? confidenceLevel = null) + { + TDst dst = new TDst(); + Predict(example, ref dst, horizon, confidenceLevel); + return dst; + } + + /// + /// Forecasting only task. + /// + /// Number of values to forecast. + /// Confidence level for forecasting. + public TDst Predict(int? horizon = null, float? confidenceLevel = null) + { + TDst dst = new TDst(); + Predict(null, ref dst, horizon, confidenceLevel); + return dst; } } public static class PredictionFunctionExtensions { /// - /// creates a prediction function/engine for a time series pipeline + /// creates a prediction engine for a time series pipeline. /// It updates the state of time series model with observations seen at prediction phase and allows checkpointing the model. /// /// Class describing input schema to the model. @@ -290,7 +380,7 @@ public static class PredictionFunctionExtensions /// To ignore missing columns. Default is false. /// Input schema definition. Default is null. /// Output schema definition. Default is null. - ///

Example code can be found by searching for TimeSeriesPredictionFunction in ML.NET.

+ ///

Example code can be found by searching for TimeSeriesPredictionEngine in ML.NET.

/// /// /// /// /// - public static TimeSeriesPredictionFunction CreateTimeSeriesPredictionFunction(this ITransformer transformer, IHostEnvironment env, + public static TimeSeriesPredictionEngine CreateTimeSeriesEngine(this ITransformer transformer, IHostEnvironment env, bool ignoreMissingColumns = false, SchemaDefinition inputSchemaDefinition = null, SchemaDefinition outputSchemaDefinition = null) where TSrc : class where TDst : class, new() @@ -308,7 +398,7 @@ public static TimeSeriesPredictionFunction CreateTimeSeriesPredictio env.CheckValue(transformer, nameof(transformer)); env.CheckValueOrNull(inputSchemaDefinition); env.CheckValueOrNull(outputSchemaDefinition); - return new TimeSeriesPredictionFunction(env, transformer, ignoreMissingColumns, inputSchemaDefinition, outputSchemaDefinition); + return new TimeSeriesPredictionEngine(env, transformer, ignoreMissingColumns, inputSchemaDefinition, outputSchemaDefinition); } } } diff --git a/src/Microsoft.ML.TimeSeries/SSaForecasting.cs b/src/Microsoft.ML.TimeSeries/SSaForecasting.cs new file mode 100644 index 0000000000..8dc9c981b3 --- /dev/null +++ b/src/Microsoft.ML.TimeSeries/SSaForecasting.cs @@ -0,0 +1,361 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Linq; +using Microsoft.ML; +using Microsoft.ML.CommandLine; +using Microsoft.ML.Data; +using Microsoft.ML.Runtime; +using Microsoft.ML.Transforms.TimeSeries; + +[assembly: LoadableClass(SsaForecastingTransformer.Summary, typeof(IDataTransform), typeof(SsaForecastingTransformer), typeof(SsaForecastingTransformer.Options), typeof(SignatureDataTransform), + SsaForecastingTransformer.UserName, SsaForecastingTransformer.LoaderSignature, SsaForecastingTransformer.ShortName)] + +[assembly: LoadableClass(SsaForecastingTransformer.Summary, typeof(IDataTransform), typeof(SsaForecastingTransformer), null, typeof(SignatureLoadDataTransform), + SsaForecastingTransformer.UserName, SsaForecastingTransformer.LoaderSignature)] + +[assembly: LoadableClass(SsaForecastingTransformer.Summary, typeof(SsaForecastingTransformer), null, typeof(SignatureLoadModel), + SsaForecastingTransformer.UserName, SsaForecastingTransformer.LoaderSignature)] + +[assembly: LoadableClass(typeof(IRowMapper), typeof(SsaForecastingTransformer), null, typeof(SignatureLoadRowMapper), + SsaForecastingTransformer.UserName, SsaForecastingTransformer.LoaderSignature)] + +namespace Microsoft.ML.Transforms.TimeSeries +{ + /// + /// resulting from fitting a . + /// + public sealed class SsaForecastingTransformer : SsaForecastingBaseWrapper, IStatefulTransformer + { + internal const string Summary = "This transform forecasts using Singular Spectrum Analysis (SSA)."; + internal const string LoaderSignature = "SsaForecasting"; + internal const string UserName = "SSA Forecasting"; + internal const string ShortName = "ssafcst"; + + internal sealed class Options : TransformInputBase + { + [Argument(ArgumentType.Required, HelpText = "The name of the source column.", ShortName = "src", SortOrder = 1, Purpose = SpecialPurpose.ColumnName)] + public string Source; + + [Argument(ArgumentType.Required, HelpText = "The name of the new column.", SortOrder = 2)] + public string Name; + + [Argument(ArgumentType.AtMostOnce, HelpText = "The name of the confidence interval lower bound column.", ShortName = "cnfminname", SortOrder = 3)] + public string ConfidenceLowerBoundColumn; + + [Argument(ArgumentType.AtMostOnce, HelpText = "The name of the confidence interval upper bound column.", ShortName = "cnfmaxnname", SortOrder = 3)] + public string ConfidenceUpperBoundColumn; + + [Argument(ArgumentType.AtMostOnce, HelpText = "The discount factor in [0,1] used for online updates.", ShortName = "disc", SortOrder = 5)] + public float DiscountFactor = 1; + + [Argument(ArgumentType.AtMostOnce, HelpText = "The flag determing whether the model is adaptive", ShortName = "adp", SortOrder = 6)] + public bool IsAdaptive = false; + + [Argument(ArgumentType.Required, HelpText = "The length of the window on the series for building the trajectory matrix (parameter L).", SortOrder = 2)] + public int WindowSize; + + [Argument(ArgumentType.AtMostOnce, HelpText = "The rank selection method.", SortOrder = 3)] + public RankSelectionMethod RankSelectionMethod = RankSelectionMethod.Exact; + + [Argument(ArgumentType.AtMostOnce, HelpText = "The desired rank of the subspace used for SSA projection (parameter r). This parameter should be in the range in [1, windowSize]. " + + "If set to null, the rank is automatically determined based on prediction error minimization.", SortOrder = 3)] + public int? Rank = null; + + [Argument(ArgumentType.AtMostOnce, HelpText = "The maximum rank considered during the rank selection process. If not provided (i.e. set to null), it is set to windowSize - 1.", SortOrder = 3)] + public int? MaxRank = null; + + [Argument(ArgumentType.AtMostOnce, HelpText = "The flag determining whether the model should be stabilized.", SortOrder = 3)] + public bool ShouldStabilize = true; + + [Argument(ArgumentType.AtMostOnce, HelpText = "The flag determining whether the meta information for the model needs to be maintained.", SortOrder = 3)] + public bool ShouldMaintainInfo = false; + + [Argument(ArgumentType.AtMostOnce, HelpText = "The maximum growth on the exponential trend.", SortOrder = 3)] + public GrowthRatio? MaxGrowth = null; + + [Argument(ArgumentType.Required, HelpText = "The length of series that is kept in buffer for modeling (parameter N).", SortOrder = 2)] + public int SeriesLength; + + [Argument(ArgumentType.Required, HelpText = "The length of series from the begining used for training.", SortOrder = 2)] + public int TrainSize; + + [Argument(ArgumentType.Required, HelpText = "The number of values to forecast.", SortOrder = 2)] + public int Horizon; + + [Argument(ArgumentType.AtMostOnce, HelpText = "The confidence level in [0, 1) for forecasting.", SortOrder = 2)] + public float ConfidenceLevel = 0.95f; + + [Argument(ArgumentType.AtMostOnce, HelpText = "Set this to true horizon will change at prediction time.", SortOrder = 2)] + public bool VariableHorizon; + } + + private sealed class BaseArguments : SsaForecastingOptions + { + public BaseArguments(Options options) + { + Source = options.Source; + Name = options.Name; + ConfidenceLowerBoundColumn = options.ConfidenceLowerBoundColumn; + ConfidenceUpperBoundColumn = options.ConfidenceUpperBoundColumn; + WindowSize = options.WindowSize; + DiscountFactor = options.DiscountFactor; + IsAdaptive = options.IsAdaptive; + RankSelectionMethod = options.RankSelectionMethod; + Rank = options.Rank; + ShouldStablize = options.ShouldStabilize; + MaxGrowth = options.MaxGrowth; + SeriesLength = options.SeriesLength; + TrainSize = options.TrainSize; + Horizon = options.Horizon; + ConfidenceLevel = options.ConfidenceLevel; + VariableHorizon = options.VariableHorizon; + } + } + + private static VersionInfo GetVersionInfo() + { + return new VersionInfo( + modelSignature: "FRCSTRNS", + verWrittenCur: 0x00010001, // Initial + verReadableCur: 0x00010001, + verWeCanReadBack: 0x00010001, + loaderSignature: LoaderSignature, + loaderAssemblyName: typeof(SsaForecastingTransformer).Assembly.FullName); + } + + internal SsaForecastingTransformer(IHostEnvironment env, Options options, IDataView input) + : base(new BaseArguments(options), LoaderSignature, env) + { + InternalTransform.Model.Train(new RoleMappedData(input, null, InternalTransform.InputColumnName)); + } + + // Factory method for SignatureDataTransform. + private static IDataTransform Create(IHostEnvironment env, Options options, IDataView input) + { + Contracts.CheckValue(env, nameof(env)); + env.CheckValue(options, nameof(options)); + env.CheckValue(input, nameof(input)); + + return new SsaForecastingTransformer(env, options, input).MakeDataTransform(input); + } + + internal SsaForecastingTransformer(IHostEnvironment env, Options options) + : base(new BaseArguments(options), LoaderSignature, env) + { + // This constructor is empty. + } + + // Factory method for SignatureLoadDataTransform. + private static IDataTransform Create(IHostEnvironment env, ModelLoadContext ctx, IDataView input) + { + Contracts.CheckValue(env, nameof(env)); + env.CheckValue(ctx, nameof(ctx)); + env.CheckValue(input, nameof(input)); + + return new SsaForecastingTransformer(env, ctx).MakeDataTransform(input); + } + + IStatefulTransformer IStatefulTransformer.Clone() + { + var clone = (SsaForecastingTransformer)MemberwiseClone(); + clone.InternalTransform.Model = clone.InternalTransform.Model.Clone(); + clone.InternalTransform.StateRef = (SsaForecastingBase.State)clone.InternalTransform.StateRef.Clone(); + clone.InternalTransform.StateRef.InitState(clone.InternalTransform, InternalTransform.Host); + return clone; + } + + // Factory method for SignatureLoadModel. + private static SsaForecastingTransformer Create(IHostEnvironment env, ModelLoadContext ctx) + { + Contracts.CheckValue(env, nameof(env)); + env.CheckValue(ctx, nameof(ctx)); + ctx.CheckAtModel(GetVersionInfo()); + + return new SsaForecastingTransformer(env, ctx); + } + + internal SsaForecastingTransformer(IHostEnvironment env, ModelLoadContext ctx) + : base(env, ctx, LoaderSignature) + { + // *** Binary format *** + // + InternalTransform.Host.CheckDecode(InternalTransform.IsAdaptive == false); + } + + private protected override void SaveModel(ModelSaveContext ctx) + { + InternalTransform.Host.CheckValue(ctx, nameof(ctx)); + ctx.CheckAtModel(); + ctx.SetVersionInfo(GetVersionInfo()); + + InternalTransform.Host.Assert(InternalTransform.IsAdaptive == false); + + // *** Binary format *** + // + + base.SaveModel(ctx); + } + + // Factory method for SignatureLoadRowMapper. + private static IRowMapper Create(IHostEnvironment env, ModelLoadContext ctx, DataViewSchema inputSchema) + => Create(env, ctx).MakeRowMapper(inputSchema); + } + + /// + /// Forecasts using Singular Spectrum Analysis. + /// + /// + /// | + /// | Output column data type | Vector of | + /// + /// | | | + /// | -- | -- | + /// | Does this estimator need to look at the data to train its parameters? | Yes | + /// | Input column data type | | + /// | Output column data type | Three vectors of | + /// + /// [!include[io](~/../docs/samples/docs/api-reference/time-series-props.md)] + /// + /// [!include[io](~/../docs/samples/docs/api-reference/time-series-ssa.md)] + /// + /// Check the See Also section for links to usage examples. + /// ]]> + /// + /// + public sealed class SsaForecastingEstimator : IEstimator + { + private readonly IHost _host; + private readonly SsaForecastingTransformer.Options _options; + + /// + /// Create a new instance of + /// + /// Host Environment. + /// Name of the column resulting from the transformation of . + /// Name of column to transform. If set to , the value of the will be used as source. + /// The vector contains Alert, Raw Score, P-Value as first three values. + /// The length of the window on the series for building the trajectory matrix (parameter L). + /// The length of series that is kept in buffer for modeling (parameter N). + /// The length of series from the begining used for training. + /// The number of values to forecast. + /// The flag determing whether the model is adaptive. + /// The discount factor in [0,1] used for online updates. + /// The rank selection method. + /// The desired rank of the subspace used for SSA projection (parameter r). This parameter should be in the range in [1, windowSize]. + /// If set to null, the rank is automatically determined based on prediction error minimization. + /// The maximum rank considered during the rank selection process. If not provided (i.e. set to null), it is set to windowSize - 1. + /// The flag determining whether the model should be stabilized. + /// The flag determining whether the meta information for the model needs to be maintained. + /// The maximum growth on the exponential trend. + /// The name of the confidence interval lower bound column. If not specified then confidence intervals will not be calculated. + /// The name of the confidence interval upper bound column. If not specified then confidence intervals will not be calculated. + /// The confidence level for forecasting. + /// Set this to true if horizon will change after training. + internal SsaForecastingEstimator(IHostEnvironment env, + string outputColumnName, + string inputColumnName, + int windowSize, + int seriesLength, + int trainSize, + int horizon, + bool isAdaptive = false, + float discountFactor = 1, + RankSelectionMethod rankSelectionMethod = RankSelectionMethod.Exact, + int? rank = null, + int? maxRank = null, + bool shouldStabilize = true, + bool shouldMaintainInfo = false, + GrowthRatio? maxGrowth = null, + string confidenceLowerBoundColumn = null, + string confidenceUpperBoundColumn = null, + float confidenceLevel = 0.95f, + bool variableHorizon = false) + : this(env, new SsaForecastingTransformer.Options + { + Source = inputColumnName ?? outputColumnName, + Name = outputColumnName, + DiscountFactor = discountFactor, + IsAdaptive = isAdaptive, + WindowSize = windowSize, + RankSelectionMethod = rankSelectionMethod, + Rank = rank, + MaxRank = maxRank, + ShouldStabilize = shouldStabilize, + ShouldMaintainInfo = shouldMaintainInfo, + MaxGrowth = maxGrowth, + ConfidenceLevel = confidenceLevel, + ConfidenceLowerBoundColumn = confidenceLowerBoundColumn, + ConfidenceUpperBoundColumn = confidenceUpperBoundColumn, + SeriesLength = seriesLength, + TrainSize = trainSize, + VariableHorizon = variableHorizon, + Horizon = horizon + }) + { + } + + internal SsaForecastingEstimator(IHostEnvironment env, SsaForecastingTransformer.Options options) + { + Contracts.CheckValue(env, nameof(env)); + _host = env.Register(nameof(SsaForecastingEstimator)); + + _host.CheckNonEmpty(options.Name, nameof(options.Name)); + _host.CheckNonEmpty(options.Source, nameof(options.Source)); + + _options = options; + } + + /// + /// Train and return a transformer. + /// + public SsaForecastingTransformer Fit(IDataView input) + { + _host.CheckValue(input, nameof(input)); + return new SsaForecastingTransformer(_host, _options, input); + } + + /// + /// Schema propagation for transformers. + /// Returns the output schema of the data, if the input schema is like the one provided. + /// Creates three output columns if confidence intervals are requested otherwise + /// just one. + /// + public SchemaShape GetOutputSchema(SchemaShape inputSchema) + { + _host.CheckValue(inputSchema, nameof(inputSchema)); + + if (!inputSchema.TryFindColumn(_options.Source, out var col)) + throw _host.ExceptSchemaMismatch(nameof(inputSchema), "input", _options.Source); + if (col.ItemType != NumberDataViewType.Single) + throw _host.ExceptSchemaMismatch(nameof(inputSchema), "input", _options.Source, "Single", col.GetTypeString()); + + var resultDic = inputSchema.ToDictionary(x => x.Name); + resultDic[_options.Name] = new SchemaShape.Column( + _options.Name, SchemaShape.Column.VectorKind.Vector, NumberDataViewType.Single, false); + + if (!string.IsNullOrEmpty(_options.ConfidenceUpperBoundColumn)) + { + resultDic[_options.ConfidenceLowerBoundColumn] = new SchemaShape.Column( + _options.ConfidenceLowerBoundColumn, SchemaShape.Column.VectorKind.Vector, + NumberDataViewType.Single, false); + + resultDic[_options.ConfidenceUpperBoundColumn] = new SchemaShape.Column( + _options.ConfidenceUpperBoundColumn, SchemaShape.Column.VectorKind.Vector, + NumberDataViewType.Single, false); + } + + return new SchemaShape(resultDic.Values); + } + } +} diff --git a/src/Microsoft.ML.TimeSeries/SequentialAnomalyDetectionTransformBase.cs b/src/Microsoft.ML.TimeSeries/SequentialAnomalyDetectionTransformBase.cs index 915e74ab39..5048a38dac 100644 --- a/src/Microsoft.ML.TimeSeries/SequentialAnomalyDetectionTransformBase.cs +++ b/src/Microsoft.ML.TimeSeries/SequentialAnomalyDetectionTransformBase.cs @@ -377,28 +377,31 @@ private Delegate MakeGetter(DataViewRow input, AnomalyDetectionStateBase state) return valueGetter; } - public Action CreatePinger(DataViewRow input, Func activeOutput, out Action disposer) + public Action CreatePinger(DataViewRow input, Func activeOutput, out Action disposer) { disposer = null; - Action pinger = null; + Action pinger = null; if (activeOutput(0)) pinger = MakePinger(input, State); return pinger; } - private Action MakePinger(DataViewRow input, AnomalyDetectionStateBase state) + private Action MakePinger(DataViewRow input, AnomalyDetectionStateBase state) + { + _host.AssertValue(input); + var srcGetter = input.GetGetter(input.Schema[_inputColumnIndex]); + Action pinger = (PingerArgument args) => { - _host.AssertValue(input); - var srcGetter = input.GetGetter(input.Schema[_inputColumnIndex]); - Action pinger = (long rowPosition) => - { - TInput src = default; - srcGetter(ref src); - state.UpdateState(ref src, rowPosition, _parent.WindowSize > 0); - }; - return pinger; - } + if (args.DontConsumeSource) + return; + + TInput src = default; + srcGetter(ref src); + state.UpdateState(ref src, args.RowPosition, _parent.WindowSize > 0); + }; + return pinger; + } public void CloneState() { @@ -516,7 +519,7 @@ private protected override void SetNaOutput(ref VBuffer dst) dst = editor.Commit(); } - private protected sealed override void TransformCore(ref TInput input, FixedSizeQueue windowedBuffer, long iteration, ref VBuffer dst) + public sealed override void TransformCore(ref TInput input, FixedSizeQueue windowedBuffer, long iteration, ref VBuffer dst) { var outputLength = Parent.OutputLength; Host.Assert(outputLength >= 2); diff --git a/src/Microsoft.ML.TimeSeries/SequentialForecastingTransformBase.cs b/src/Microsoft.ML.TimeSeries/SequentialForecastingTransformBase.cs new file mode 100644 index 0000000000..5b7813bc11 --- /dev/null +++ b/src/Microsoft.ML.TimeSeries/SequentialForecastingTransformBase.cs @@ -0,0 +1,319 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.IO; +using System.Threading; +using Microsoft.ML.CommandLine; +using Microsoft.ML.Data; +using Microsoft.ML.Runtime; +using static Microsoft.ML.DataViewSchema; + +namespace Microsoft.ML.Transforms.TimeSeries +{ + + /// + /// The base class that can be inherited by the 'Argument' classes in the derived classes containing the shared input parameters. + /// + internal abstract class ForecastingArgumentsBase + { + [Argument(ArgumentType.Required, HelpText = "The name of the source column", ShortName = "src", + SortOrder = 1, Purpose = SpecialPurpose.ColumnName)] + public string Source; + + [Argument(ArgumentType.Required, HelpText = "The name of the new column", ShortName = "name", + SortOrder = 2)] + public string Name; + + [Argument(ArgumentType.Required, HelpText = "The name of the confidence interval lower bound column.", ShortName = "cnfminname", + SortOrder = 2)] + public string ConfidenceLowerBoundColumn; + + [Argument(ArgumentType.Required, HelpText = "The name of the confidence interval upper bound column.", ShortName = "cnfmaxnname", + SortOrder = 2)] + public string ConfidenceUpperBoundColumn; + + [Argument(ArgumentType.AtMostOnce, HelpText = "The length of series from the begining used for training.", ShortName = "wnd", + SortOrder = 3)] + public int TrainSize = 1; + + [Argument(ArgumentType.AtMostOnce, HelpText = "The size of the initial window. The default value " + + "is set to 0, which means there is no initial window considered.", ShortName = "initwnd", SortOrder = 5)] + public int SeriesLength = 0; + } + + /// + /// The base class for forecasting transforms that also supports confidence intervals for each forecasted value. + /// For more details, please refer to http://arxiv.org/pdf/1204.3251.pdf + /// + /// The type of the input sequence + /// The type of the input sequence + internal abstract class SequentialForecastingTransformBase : SequentialTransformerBase, TState> + where TState : SequentialForecastingTransformBase.ForecastingStateBase, new() + { + + // The size of the VBuffer in the dst column. + private readonly int _outputLength; + + private protected SequentialForecastingTransformBase(int windowSize, int initialWindowSize, + string inputColumnName, string outputColumnName, string confidenceLowerBoundColumn, + string confidenceUpperBoundColumn, string name, int outputLength, IHostEnvironment env) + : base(Contracts.CheckRef(env, nameof(env)).Register(name), windowSize, initialWindowSize, + outputColumnName, confidenceLowerBoundColumn, + confidenceUpperBoundColumn, inputColumnName, new VectorDataViewType(NumberDataViewType.Single, outputLength)) + { + _outputLength = outputLength; + } + + private protected SequentialForecastingTransformBase(ForecastingArgumentsBase args, string name, int outputLength, IHostEnvironment env) + : this(args.TrainSize, args.SeriesLength, args.Source, args.ConfidenceLowerBoundColumn, + args.ConfidenceUpperBoundColumn, args.Name, name, outputLength, env) + { + } + + private protected SequentialForecastingTransformBase(IHostEnvironment env, ModelLoadContext ctx, string name) + : base(Contracts.CheckRef(env, nameof(env)).Register(name), ctx) + { + _outputLength = ctx.Reader.ReadInt32(); + // *** Binary format *** + // + } + + private protected override void SaveModel(ModelSaveContext ctx) + { + Host.CheckValue(ctx, nameof(ctx)); + ctx.CheckAtModel(); + + // *** Binary format *** + // + + base.SaveModel(ctx); + ctx.Writer.Write(_outputLength); + } + + internal override IStatefulRowMapper MakeRowMapper(DataViewSchema schema) => new Mapper(Host, this, schema); + + internal sealed class Mapper : IStatefulRowMapper + { + private readonly IHost _host; + private readonly SequentialForecastingTransformBase _parent; + private readonly DataViewSchema _parentSchema; + private readonly int _inputColumnIndex; + private ForecastingStateBase State { get; set; } + private bool _dontFetchSrcValue; + + public Mapper(IHostEnvironment env, SequentialForecastingTransformBase parent, DataViewSchema inputSchema) + { + Contracts.CheckValue(env, nameof(env)); + _host = env.Register(nameof(Mapper)); + _host.CheckValue(inputSchema, nameof(inputSchema)); + _host.CheckValue(parent, nameof(parent)); + + if (!inputSchema.TryGetColumnIndex(parent.InputColumnName, out _inputColumnIndex)) + throw _host.ExceptSchemaMismatch(nameof(inputSchema), "input", parent.InputColumnName); + + var colType = inputSchema[_inputColumnIndex].Type; + if (colType != NumberDataViewType.Single) + throw _host.ExceptSchemaMismatch(nameof(inputSchema), "input", parent.InputColumnName, "Single", colType.ToString()); + + _parent = parent; + _parentSchema = inputSchema; + State = (ForecastingStateBase)_parent.StateRef; + _dontFetchSrcValue = false; + } + + public DataViewSchema.DetachedColumn[] GetOutputColumns() + { + DetachedColumn[] info; + + if (!string.IsNullOrEmpty(_parent.ConfidenceUpperBoundColumn)) + { + info = new DetachedColumn[3]; + info[0] = new DetachedColumn(_parent.OutputColumnName, new VectorDataViewType(NumberDataViewType.Single, _parent._outputLength)); + info[1] = new DetachedColumn(_parent.ConfidenceLowerBoundColumn, new VectorDataViewType(NumberDataViewType.Single, _parent._outputLength)); + info[2] = new DetachedColumn(_parent.ConfidenceUpperBoundColumn, new VectorDataViewType(NumberDataViewType.Single, _parent._outputLength)); + } + else + { + info = new DetachedColumn[1]; + info[0] = new DetachedColumn(_parent.OutputColumnName, new VectorDataViewType(NumberDataViewType.Single, _parent._outputLength)); + } + + return info; + } + + public Func GetDependencies(Func activeOutput) + { + if (activeOutput(0)) + return col => col == _inputColumnIndex; + else + return col => false; + } + + void ICanSaveModel.Save(ModelSaveContext ctx) => _parent.SaveModel(ctx); + + public Delegate[] CreateGetters(DataViewRow input, Func activeOutput, out Action disposer) + { + disposer = null; + var getters = string.IsNullOrEmpty(_parent.ConfidenceUpperBoundColumn) ? new Delegate[1] : new Delegate[3]; + + if (activeOutput(0)) + { + ValueGetter> valueGetter = (ref VBuffer dst) => + { + State.Forecast(ref dst); + }; + + getters[0] = valueGetter; + } + + if (!string.IsNullOrEmpty(_parent.ConfidenceUpperBoundColumn)) + { + if (activeOutput(1)) + { + ValueGetter> valueGetter = (ref VBuffer dst) => + { + State.ConfidenceIntervalLowerBound(ref dst); + }; + + getters[1] = valueGetter; + } + + if (activeOutput(2)) + { + ValueGetter> valueGetter = (ref VBuffer dst) => + { + State.ConfidenceIntervalUpperBound(ref dst); + }; + + getters[2] = valueGetter; + } + } + return getters; + } + + private delegate void ProcessData(ref TInput src, ref VBuffer dst); + + private Delegate MakeGetter(DataViewRow input, ForecastingStateBase state) + { + _host.AssertValue(input); + var srcGetter = input.GetGetter(input.Schema[_inputColumnIndex]); + ProcessData processData = _parent.WindowSize > 0 ? + (ProcessData)state.Process : state.ProcessWithoutBuffer; + + ValueGetter> valueGetter = (ref VBuffer dst) => + { + TInput src = default; + if (_dontFetchSrcValue) + { + state.TransformCore(ref src, null, 0, ref dst); + return; + } + + srcGetter(ref src); + processData(ref src, ref dst); + + }; + return valueGetter; + } + + public Action CreatePinger(DataViewRow input, Func activeOutput, out Action disposer) + { + disposer = null; + Action pinger = null; + if (activeOutput(0)) + pinger = MakePinger(input, State); + + return pinger; + } + + private Action MakePinger(DataViewRow input, ForecastingStateBase state) + { + _host.AssertValue(input); + var srcGetter = input.GetGetter(input.Schema[_inputColumnIndex]); + Action pinger = (PingerArgument args) => + { + state.LocalConfidenceLevel = args.ConfidenceLevel; + state.LocalHorizon = args.Horizon; + + // This means don't call srcGetter in getters. + if (args.DontConsumeSource) + { + _dontFetchSrcValue = true; + return; + } + + _dontFetchSrcValue = false; + TInput src = default; + srcGetter(ref src); + state.UpdateState(ref src, args.RowPosition, _parent.WindowSize > 0); + }; + return pinger; + } + + public void CloneState() + { + if (Interlocked.Increment(ref _parent.StateRefCount) > 1) + { + State = (ForecastingStateBase)_parent.StateRef.Clone(); + } + } + + public ITransformer GetTransformer() + { + return _parent; + } + } + /// + /// The base state class for sequential anomaly detection: this class implements the p-values and martinagle calculations for anomaly detection + /// given that the raw anomaly score calculation is specified by the derived classes. + /// + internal abstract class ForecastingStateBase : SequentialTransformerBase, TState>.StateBase + { + // A reference to the parent transform. + protected SequentialForecastingTransformBase Parent; + internal int? LocalHorizon; + internal float? LocalConfidenceLevel; + + private protected ForecastingStateBase() { } + + private protected override void CloneCore(TState state) + { + base.CloneCore(state); + } + + private protected ForecastingStateBase(BinaryReader reader) : base(reader) + { + } + + internal override void Save(BinaryWriter writer) + { + base.Save(writer); + } + + private protected override void SetNaOutput(ref VBuffer dst) + { + var outputLength = Parent._outputLength; + var editor = VBufferEditor.Create(ref dst, outputLength); + + for (int i = 0; i < outputLength; ++i) + editor.Values[i] = float.NaN; + + dst = editor.Commit(); + } + + private protected sealed override void InitializeStateCore(bool disk = false) + { + Parent = (SequentialForecastingTransformBase)ParentTransform; + Host.Assert(WindowSize >= 0); + InitializeForecaster(); + } + + /// + /// The abstract method that realizes the initialization functionality for the forecaster. + /// + private protected abstract void InitializeForecaster(); + } + } +} diff --git a/src/Microsoft.ML.TimeSeries/SequentialTransformBase.cs b/src/Microsoft.ML.TimeSeries/SequentialTransformBase.cs index a5657e1b56..931cece4b0 100644 --- a/src/Microsoft.ML.TimeSeries/SequentialTransformBase.cs +++ b/src/Microsoft.ML.TimeSeries/SequentialTransformBase.cs @@ -29,6 +29,29 @@ public DataBox(T value) } } + /// + /// The box class that is used to box the TInput and TOutput for the LambdaTransform. + /// This is for the case where there are three output columns. + /// + /// The type to be boxed, e.g. TInput or TOutput + internal sealed class DataBoxForecastingWithConfidenceIntervals + { + public T Forecast; + public T ConfidenceIntervalLowerBound; + public T ConfidenceIntervalUpperBound; + + public DataBoxForecastingWithConfidenceIntervals() + { + } + + public DataBoxForecastingWithConfidenceIntervals(T forecast, T confidenceIntervalLowerBound, T confidenceIntervalUpperBound) + { + Forecast = forecast; + ConfidenceIntervalLowerBound = confidenceIntervalLowerBound; + ConfidenceIntervalUpperBound = confidenceIntervalUpperBound; + } + } + /// /// The base class for sequential processing transforms. This class implements the basic sliding window buffering. The derived classes need to specify the transform logic, /// the initialization logic and the learning logic via implementing the abstract methods TransformCore(), InitializeStateCore() and LearnStateFromDataCore(), respectively diff --git a/src/Microsoft.ML.TimeSeries/SequentialTransformerBase.cs b/src/Microsoft.ML.TimeSeries/SequentialTransformerBase.cs index 7f0c98574f..4ffe27b31f 100644 --- a/src/Microsoft.ML.TimeSeries/SequentialTransformerBase.cs +++ b/src/Microsoft.ML.TimeSeries/SequentialTransformerBase.cs @@ -26,6 +26,7 @@ namespace Microsoft.ML.Transforms.TimeSeries internal abstract class SequentialTransformerBase : IStatefulTransformer where TState : SequentialTransformerBase.StateBase, new() { + public SequentialTransformerBase() { } /// /// The base class for encapsulating the State object for sequential processing. This class implements a windowed buffer. @@ -165,8 +166,45 @@ public void UpdateStateCore(ref TInput input, bool buffer = true) } } + public void Process(ref TInput input, ref TOutput output1, ref TOutput output2, ref TOutput output3) + { + //Using prediction engine will not evaluate the below condition to true. + if (PreviousPosition == -1) + UpdateStateCore(ref input); + + if (InitialWindowedBuffer.Count < InitialWindowSize) + { + SetNaOutput(ref output1); + + if (InitialWindowedBuffer.Count == InitialWindowSize) + LearnStateFromDataCore(InitialWindowedBuffer); + } + else + { + TransformCore(ref input, WindowedBuffer, RowCounter - InitialWindowSize, ref output1, ref output2, ref output3); + } + } + + public void ProcessWithoutBuffer(ref TInput input, ref TOutput output1, ref TOutput output2, ref TOutput output3) + { + //Using prediction engine will not evaluate the below condition to true. + if (PreviousPosition == -1) + UpdateStateCore(ref input); + + if (InitialWindowedBuffer.Count < InitialWindowSize) + { + SetNaOutput(ref output1); + + if (InitialWindowedBuffer.Count == InitialWindowSize) + LearnStateFromDataCore(InitialWindowedBuffer); + } + else + TransformCore(ref input, WindowedBuffer, RowCounter - InitialWindowSize, ref output1, ref output2, ref output3); + } + public void Process(ref TInput input, ref TOutput output) { + //Using prediction engine will not evaluate the below condition to true. if (PreviousPosition == -1) UpdateStateCore(ref input); @@ -185,8 +223,9 @@ public void Process(ref TInput input, ref TOutput output) public void ProcessWithoutBuffer(ref TInput input, ref TOutput output) { + //Using prediction engine will not evaluate the below condition to true. if (PreviousPosition == -1) - UpdateStateCore(ref input, false); + UpdateStateCore(ref input); if (InitialWindowedBuffer.Count < InitialWindowSize) { @@ -212,7 +251,37 @@ private protected virtual void SetNaOutput(ref TOutput dst) { } /// A reference to the dst object. /// A reference to the windowed buffer. /// A long number that indicates the number of times TransformCore has been called so far (starting value = 0). - private protected virtual void TransformCore(ref TInput input, FixedSizeQueue windowedBuffer, long iteration, ref TOutput dst) + public virtual void TransformCore(ref TInput input, FixedSizeQueue windowedBuffer, long iteration, ref TOutput dst) + { + + } + + public virtual void Forecast(ref TOutput dst) + { + + } + + public virtual void ConfidenceIntervalLowerBound(ref TOutput dst) + { + + } + + public virtual void ConfidenceIntervalUpperBound(ref TOutput dst) + { + + } + + /// + /// The abstract method that realizes the main logic for the transform. + /// + /// A reference to the input object. + /// A reference to the dst object. + /// + /// + /// A reference to the windowed buffer. + /// A long number that indicates the number of times TransformCore has been called so far (starting value = 0). + private protected virtual void TransformCore(ref TInput input, FixedSizeQueue windowedBuffer, long iteration, + ref TOutput dst1, ref TOutput dst2, ref TOutput dst3) { } @@ -266,6 +335,8 @@ private protected virtual void CloneCore(TState state) internal readonly string InputColumnName; internal readonly string OutputColumnName; + internal readonly string ConfidenceLowerBoundColumn; + internal readonly string ConfidenceUpperBoundColumn; private protected DataViewType OutputColumnType; bool ITransformer.IsRowToRowMapper => false; @@ -282,7 +353,8 @@ private protected virtual void CloneCore(TState state) /// The name of the dst column. /// The name of the input column. /// - private protected SequentialTransformerBase(IHost host, int windowSize, int initialWindowSize, string outputColumnName, string inputColumnName, DataViewType outputColType) + private protected SequentialTransformerBase(IHost host, int windowSize, int initialWindowSize, + string outputColumnName, string inputColumnName, DataViewType outputColType) { Host = host; Host.CheckParam(initialWindowSize >= 0, nameof(initialWindowSize), "Must be non-negative."); @@ -299,6 +371,15 @@ private protected SequentialTransformerBase(IHost host, int windowSize, int init WindowSize = windowSize; } + private protected SequentialTransformerBase(IHost host, int windowSize, int initialWindowSize, + string outputColumnName, string confidenceLowerBoundColumn, + string confidenceUpperBoundColumn, string inputColumnName, DataViewType outputColType) : + this(host, windowSize, initialWindowSize, outputColumnName, inputColumnName, outputColType) + { + ConfidenceLowerBoundColumn = confidenceLowerBoundColumn; + ConfidenceUpperBoundColumn = confidenceUpperBoundColumn; + } + private protected SequentialTransformerBase(IHost host, ModelLoadContext ctx) { Host = host; @@ -322,6 +403,8 @@ private protected SequentialTransformerBase(IHost host, ModelLoadContext ctx) InputColumnName = inputColumnName; OutputColumnName = outputColumnName; + ConfidenceLowerBoundColumn = ctx.Reader.ReadString(); + ConfidenceUpperBoundColumn = ctx.Reader.ReadString(); InitialWindowSize = initialWindowSize; WindowSize = windowSize; @@ -348,7 +431,8 @@ private protected virtual void SaveModel(ModelSaveContext ctx) ctx.Writer.Write(InitialWindowSize); ctx.SaveNonEmptyString(InputColumnName); ctx.SaveNonEmptyString(OutputColumnName); - + ctx.Writer.Write(ConfidenceLowerBoundColumn ?? string.Empty); + ctx.Writer.Write(ConfidenceUpperBoundColumn ?? string.Empty); var bs = new BinarySaver(Host, new BinarySaver.Arguments()); bs.TryWriteTypeDescription(ctx.Writer.BaseStream, OutputColumnType, out int byteWritten); } @@ -393,10 +477,12 @@ public SequentialDataTransform(IHost host, SequentialTransformerBase 0, _parent.OutputColumnType); + _parent.OutputColumnName, _parent.ConfidenceLowerBoundColumn, + _parent.ConfidenceUpperBoundColumn, InitFunction, _parent.WindowSize > 0, _parent.OutputColumnType); + _mapper = mapper; _bindings = new ColumnBindings(input.Schema, _mapper.GetOutputColumns()); } @@ -404,24 +490,49 @@ public SequentialDataTransform(IHost host, SequentialTransformerBase _mapper.CloneState(); private static IDataTransform CreateLambdaTransform(IHost host, IDataView input, string inputColumnName, - string outputColumnName, Action initFunction, bool hasBuffer, DataViewType outputColTypeOverride) + string outputColumnName, string forecastingConfidenceIntervalMinOutputColumnName, + string forecastingConfidenceIntervalMaxOutputColumnName, Action initFunction, bool hasBuffer, DataViewType outputColTypeOverride) { var inputSchema = SchemaDefinition.Create(typeof(DataBox)); inputSchema[0].ColumnName = inputColumnName; - var outputSchema = SchemaDefinition.Create(typeof(DataBox)); - outputSchema[0].ColumnName = outputColumnName; + SchemaDefinition outputSchema; + + if (!string.IsNullOrEmpty(forecastingConfidenceIntervalMinOutputColumnName)) + { + outputSchema = SchemaDefinition.Create(typeof(DataBoxForecastingWithConfidenceIntervals)); + outputSchema[0].ColumnName = outputColumnName; + + if (outputColTypeOverride != null) + outputSchema[0].ColumnType = outputSchema[1].ColumnType = outputSchema[2].ColumnType = outputColTypeOverride; - if (outputColTypeOverride != null) - outputSchema[0].ColumnType = outputColTypeOverride; + outputSchema[1].ColumnName = forecastingConfidenceIntervalMinOutputColumnName; + outputSchema[2].ColumnName = forecastingConfidenceIntervalMaxOutputColumnName; - Action, DataBox, TState> lambda; - if (hasBuffer) - lambda = MapFunction; + Action, DataBoxForecastingWithConfidenceIntervals, TState> lambda; + if (hasBuffer) + lambda = MapFunction; + else + lambda = MapFunctionWithoutBuffer; + + return LambdaTransform.CreateMap(host, input, lambda, initFunction, inputSchema, outputSchema); + } else - lambda = MapFunctionWithoutBuffer; + { + outputSchema = SchemaDefinition.Create(typeof(DataBox)); + outputSchema[0].ColumnName = outputColumnName; + + if (outputColTypeOverride != null) + outputSchema[0].ColumnType = outputColTypeOverride; - return LambdaTransform.CreateMap(host, input, lambda, initFunction, inputSchema, outputSchema); + Action, DataBox, TState> lambda; + if (hasBuffer) + lambda = MapFunction; + else + lambda = MapFunctionWithoutBuffer; + + return LambdaTransform.CreateMap(host, input, lambda, initFunction, inputSchema, outputSchema); + } } private static void MapFunction(DataBox input, DataBox output, TState state) @@ -429,11 +540,21 @@ private static void MapFunction(DataBox input, DataBox output, state.Process(ref input.Value, ref output.Value); } + private static void MapFunction(DataBox input, DataBoxForecastingWithConfidenceIntervals output, TState state) + { + state.Process(ref input.Value, ref output.Forecast, ref output.ConfidenceIntervalLowerBound, ref output.ConfidenceIntervalUpperBound); + } + private static void MapFunctionWithoutBuffer(DataBox input, DataBox output, TState state) { state.ProcessWithoutBuffer(ref input.Value, ref output.Value); } + private static void MapFunctionWithoutBuffer(DataBox input, DataBoxForecastingWithConfidenceIntervals output, TState state) + { + state.ProcessWithoutBuffer(ref input.Value, ref output.Forecast, ref output.ConfidenceIntervalLowerBound, ref output.ConfidenceIntervalUpperBound); + } + private void InitFunction(TState state) { state.InitState(_parent.WindowSize, _parent.InitialWindowSize, _parent, _parent.Host); @@ -500,7 +621,7 @@ private sealed class RowImpl : StatefulRow private readonly DataViewSchema _schema; private readonly DataViewRow _input; private readonly Delegate[] _getters; - private readonly Action _pinger; + private readonly Action _pinger; private readonly Action _disposer; private bool _disposed; private readonly ColumnBindings _bindings; @@ -511,7 +632,7 @@ private sealed class RowImpl : StatefulRow public override long Batch => _input.Batch; - public RowImpl(ColumnBindings bindings, DataViewRow input, Delegate[] getters, Action pinger, Action disposer) + public RowImpl(ColumnBindings bindings, DataViewRow input, Delegate[] getters, Action pinger, Action disposer) { Contracts.CheckValue(bindings, nameof(bindings)); Contracts.CheckValue(input, nameof(input)); @@ -550,8 +671,8 @@ public override ValueGetter GetGetter(DataViewSchema.Column column) return fn; } - public override Action GetPinger() => - _pinger as Action ?? throw Contracts.Except("Invalid TValue in GetPinger: '{0}'", typeof(long)); + public override Action GetPinger() => + _pinger as Action ?? throw Contracts.Except("Invalid TValue in GetPinger: '{0}'", typeof(PingerArgument)); /// /// Returns whether the given column is active in this row. @@ -824,7 +945,7 @@ private sealed class StatefulRowImpl : StatefulRow { private readonly DataViewRow _input; private readonly Delegate[] _getters; - private readonly Action _pinger; + private readonly Action _pinger; private readonly Action _disposer; private readonly TimeSeriesRowToRowMapperTransform _parent; @@ -836,7 +957,7 @@ private sealed class StatefulRowImpl : StatefulRow public override DataViewSchema Schema { get; } public StatefulRowImpl(DataViewRow input, TimeSeriesRowToRowMapperTransform parent, - DataViewSchema schema, Delegate[] getters, Action pinger, Action disposer) + DataViewSchema schema, Delegate[] getters, Action pinger, Action disposer) { _input = input; _parent = parent; @@ -873,8 +994,8 @@ public override ValueGetter GetGetter(DataViewSchema.Column colu return fn; } - public override Action GetPinger() => - _pinger as Action ?? throw Contracts.Except("Invalid TValue in GetPinger: '{0}'", typeof(long)); + public override Action GetPinger() => + _pinger as Action ?? throw Contracts.Except("Invalid TValue in GetPinger: '{0}'", typeof(PingerArgument)); public override ValueGetter GetIdGetter() => _input.GetIdGetter(); diff --git a/src/Microsoft.ML.TimeSeries/SrCnnTransformBase.cs b/src/Microsoft.ML.TimeSeries/SrCnnTransformBase.cs index 9f64fa2167..c76973bef2 100644 --- a/src/Microsoft.ML.TimeSeries/SrCnnTransformBase.cs +++ b/src/Microsoft.ML.TimeSeries/SrCnnTransformBase.cs @@ -220,25 +220,28 @@ private Delegate MakeGetter(DataViewRow input, SrCnnStateBase state) return valueGetter; } - public Action CreatePinger(DataViewRow input, Func activeOutput, out Action disposer) + public Action CreatePinger(DataViewRow input, Func activeOutput, out Action disposer) { disposer = null; - Action pinger = null; + Action pinger = null; if (activeOutput(0)) pinger = MakePinger(input, State); return pinger; } - private Action MakePinger(DataViewRow input, SrCnnStateBase state) + private Action MakePinger(DataViewRow input, SrCnnStateBase state) { _host.AssertValue(input); var srcGetter = input.GetGetter(input.Schema[_inputColumnIndex]); - Action pinger = (long rowPosition) => + Action pinger = (PingerArgument args) => { + if (args.DontConsumeSource) + return; + TInput src = default; srcGetter(ref src); - state.UpdateState(ref src, rowPosition, _parent.WindowSize > 0); + state.UpdateState(ref src, args.RowPosition, _parent.WindowSize > 0); }; return pinger; } @@ -289,7 +292,7 @@ private protected override void SetNaOutput(ref VBuffer dst) dst = editor.Commit(); } - private protected sealed override void TransformCore(ref TInput input, FixedSizeQueue windowedBuffer, long iteration, ref VBuffer dst) + public sealed override void TransformCore(ref TInput input, FixedSizeQueue windowedBuffer, long iteration, ref VBuffer dst) { var outputLength = Parent.OutputLength; diff --git a/src/Microsoft.ML.TimeSeries/SsaAnomalyDetectionBase.cs b/src/Microsoft.ML.TimeSeries/SsaAnomalyDetectionBase.cs index 0c823f2de1..f93efb6aa7 100644 --- a/src/Microsoft.ML.TimeSeries/SsaAnomalyDetectionBase.cs +++ b/src/Microsoft.ML.TimeSeries/SsaAnomalyDetectionBase.cs @@ -201,8 +201,8 @@ public SsaAnomalyDetectionBase(SsaOptions options, string name, IHostEnvironment ErrorFunc = ErrorFunctionUtils.GetErrorFunction(ErrorFunction); IsAdaptive = options.IsAdaptive; // Creating the master SSA model - Model = new AdaptiveSingularSpectrumSequenceModeler.AdaptiveSingularSpectrumSequenceModelerInternal(Host, options.InitialWindowSize, SeasonalWindowSize + 1, SeasonalWindowSize, - DiscountFactor, AdaptiveSingularSpectrumSequenceModeler.RankSelectionMethod.Exact, null, SeasonalWindowSize / 2, false, false); + Model = new AdaptiveSingularSpectrumSequenceModelerInternal(Host, options.InitialWindowSize, SeasonalWindowSize + 1, SeasonalWindowSize, + DiscountFactor, RankSelectionMethod.Exact, null, SeasonalWindowSize / 2, false, false); StateRef = new State(); StateRef.InitState(WindowSize, InitialWindowSize, this, Host); diff --git a/src/Microsoft.ML.TimeSeries/SsaForecastingBase.cs b/src/Microsoft.ML.TimeSeries/SsaForecastingBase.cs new file mode 100644 index 0000000000..1eb942c775 --- /dev/null +++ b/src/Microsoft.ML.TimeSeries/SsaForecastingBase.cs @@ -0,0 +1,314 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.IO; +using Microsoft.ML.CommandLine; +using Microsoft.ML.Data; +using Microsoft.ML.Internal.Utilities; +using Microsoft.ML.Runtime; + +namespace Microsoft.ML.Transforms.TimeSeries +{ + /// + /// The wrapper to that implements the general anomaly detection transform based on Singular Spectrum modeling of the time-series. + /// For the details of the Singular Spectrum Analysis (SSA), refer to http://arxiv.org/pdf/1206.6910.pdf. + /// + public class SsaForecastingBaseWrapper : IStatefulTransformer, ICanSaveModel + { + /// + /// Whether a call to should succeed, on an + /// appropriate schema. + /// + bool ITransformer.IsRowToRowMapper => ((ITransformer)InternalTransform).IsRowToRowMapper; + + /// + /// Creates a clone of the transformer. Used for taking the snapshot of the state. + /// + /// + IStatefulTransformer IStatefulTransformer.Clone() => InternalTransform.Clone(); + + /// + /// Schema propagation for transformers. + /// Returns the output schema of the data, if the input schema is like the one provided. + /// + public DataViewSchema GetOutputSchema(DataViewSchema inputSchema) => InternalTransform.GetOutputSchema(inputSchema); + + /// + /// Constructs a row-to-row mapper based on an input schema. If + /// is false, then an exception should be thrown. If the input schema is in any way + /// unsuitable for constructing the mapper, an exception should likewise be thrown. + /// + /// The input schema for which we should get the mapper. + /// The row to row mapper. + IRowToRowMapper ITransformer.GetRowToRowMapper(DataViewSchema inputSchema) + => ((ITransformer)InternalTransform).GetRowToRowMapper(inputSchema); + + /// + /// Same as but also supports mechanism to save the state. + /// + /// The input schema for which we should get the mapper. + /// The row to row mapper. + public IRowToRowMapper GetStatefulRowToRowMapper(DataViewSchema inputSchema) + => ((IStatefulTransformer)InternalTransform).GetStatefulRowToRowMapper(inputSchema); + + /// + /// Take the data in, make transformations, output the data. + /// Note that 's are lazy, so no actual transformations happen here, just schema validation. + /// + public IDataView Transform(IDataView input) => InternalTransform.Transform(input); + + /// + /// For saving a model into a repository. + /// + void ICanSaveModel.Save(ModelSaveContext ctx) => SaveModel(ctx); + + private protected virtual void SaveModel(ModelSaveContext ctx) => InternalTransform.SaveThis(ctx); + + /// + /// Creates a row mapper from Schema. + /// + internal IStatefulRowMapper MakeRowMapper(DataViewSchema schema) => InternalTransform.MakeRowMapper(schema); + + /// + /// Creates an IDataTransform from an IDataView. + /// + internal IDataTransform MakeDataTransform(IDataView input) => InternalTransform.MakeDataTransform(input); + + /// + /// Options for SSA Anomaly Detection. + /// + internal abstract class SsaForecastingOptions : ForecastingArgumentsBase + { + [Argument(ArgumentType.AtMostOnce, HelpText = "The discount factor in [0, 1]", ShortName = "disc", SortOrder = 12)] + public Single DiscountFactor = 1; + + [Argument(ArgumentType.AtMostOnce, HelpText = "The function used to compute the error between the expected and the observed value", ShortName = "err", SortOrder = 13)] + public ErrorFunction ErrorFunction = ErrorFunction.SignedDifference; + + [Argument(ArgumentType.AtMostOnce, HelpText = "The flag determing whether the model is adaptive", ShortName = "adp", SortOrder = 14)] + public bool IsAdaptive = false; + public int WindowSize; + public RankSelectionMethod RankSelectionMethod; + public int? Rank; + public int? MaxRank; + public bool ShouldStablize; + public bool ShouldMaintainInfo; + public GrowthRatio? MaxGrowth; + public int Horizon; + public float ConfidenceLevel; + public bool VariableHorizon; + } + + internal SsaForecastingBase InternalTransform; + + internal SsaForecastingBaseWrapper(SsaForecastingOptions options, string name, IHostEnvironment env) + { + InternalTransform = new SsaForecastingBase(options, name, env, this); + } + //string forecastingConfidenceIntervalMinOutputColumnName, string forecastingConfidenceIntervalMaxOutputColumnName, int horizon, bool computeConfidenceIntervals + internal SsaForecastingBaseWrapper(IHostEnvironment env, ModelLoadContext ctx, string name) + { + InternalTransform = new SsaForecastingBase(env, ctx, name); + } + + /// + /// This base class that implements the general anomaly detection transform based on Singular Spectrum modeling of the time-series. + /// For the details of the Singular Spectrum Analysis (SSA), refer to http://arxiv.org/pdf/1206.6910.pdf. + /// + internal sealed class SsaForecastingBase : SequentialForecastingTransformBase + { + internal SsaForecastingBaseWrapper Parent; + internal readonly bool IsAdaptive; + internal readonly int Horizon; + internal readonly float ConfidenceLevel; + internal SequenceModelerBase Model; + + public SsaForecastingBase(SsaForecastingOptions options, string name, IHostEnvironment env, SsaForecastingBaseWrapper parent) + : base(options.TrainSize, 0, options.Source, options.Name, options.ConfidenceLowerBoundColumn, + options.ConfidenceUpperBoundColumn, name, options.VariableHorizon ? 0: options.Horizon, env) + { + Host.CheckUserArg(0 <= options.DiscountFactor && options.DiscountFactor <= 1, nameof(options.DiscountFactor), "Must be in the range [0, 1]."); + IsAdaptive = options.IsAdaptive; + Horizon = options.Horizon; + ConfidenceLevel = options.ConfidenceLevel; + // Creating the master SSA model + Model = new AdaptiveSingularSpectrumSequenceModelerInternal(Host, options.TrainSize, options.SeriesLength, options.WindowSize, + options.DiscountFactor, options.RankSelectionMethod, options.Rank, options.MaxRank, !string.IsNullOrEmpty(options.ConfidenceLowerBoundColumn), + options.ShouldStablize, options.ShouldMaintainInfo, options.MaxGrowth); + + StateRef = new State(); + StateRef.InitState(WindowSize, InitialWindowSize, this, Host); + Parent = parent; + } + + public SsaForecastingBase(IHostEnvironment env, ModelLoadContext ctx, string name) : base(env, ctx, name) + { + // *** Binary format *** + // + // bool: _isAdaptive + // int32: Horizon + // bool: ComputeConfidenceIntervals + // State: StateRef + // AdaptiveSingularSpectrumSequenceModeler: _model + + Host.CheckDecode(InitialWindowSize == 0); + + IsAdaptive = ctx.Reader.ReadBoolean(); + Horizon = ctx.Reader.ReadInt32(); + ConfidenceLevel = ctx.Reader.ReadFloat(); + StateRef = new State(ctx.Reader); + + ctx.LoadModel, SignatureLoadModel>(env, out Model, "SSA"); + Host.CheckDecode(Model != null); + StateRef.InitState(this, Host); + } + + public override DataViewSchema GetOutputSchema(DataViewSchema inputSchema) + { + Host.CheckValue(inputSchema, nameof(inputSchema)); + + if (!inputSchema.TryGetColumnIndex(InputColumnName, out var col)) + throw Host.ExceptSchemaMismatch(nameof(inputSchema), "input", InputColumnName); + + var colType = inputSchema[col].Type; + if (colType != NumberDataViewType.Single) + throw Host.ExceptSchemaMismatch(nameof(inputSchema), "input", InputColumnName, "Single", colType.ToString()); + + return Transform(new EmptyDataView(Host, inputSchema)).Schema; + } + + private protected override void SaveModel(ModelSaveContext ctx) + { + ((ICanSaveModel)Parent).Save(ctx); + } + + internal void SaveThis(ModelSaveContext ctx) + { + Host.CheckValue(ctx, nameof(ctx)); + ctx.CheckAtModel(); + + Host.Assert(InitialWindowSize == 0); + Host.Assert(Model != null); + + // *** Binary format *** + // + // bool: _isAdaptive + // int32: Horizon + // State: StateRef + // AdaptiveSingularSpectrumSequenceModeler: _model + + base.SaveModel(ctx); + ctx.Writer.Write(IsAdaptive); + ctx.Writer.Write(Horizon); + ctx.Writer.Write(ConfidenceLevel); + StateRef.Save(ctx.Writer); + + ctx.SaveModel(Model, "SSA"); + } + + internal sealed class State : ForecastingStateBase + { + private SequenceModelerBase _model; + private SsaForecastingBase _parentForecaster; + + public State() + { + } + + internal State(BinaryReader reader) : base(reader) + { + WindowedBuffer = TimeSeriesUtils.DeserializeFixedSizeQueueSingle(reader, Host); + InitialWindowedBuffer = TimeSeriesUtils.DeserializeFixedSizeQueueSingle(reader, Host); + } + + internal override void Save(BinaryWriter writer) + { + base.Save(writer); + TimeSeriesUtils.SerializeFixedSizeQueue(WindowedBuffer, writer); + TimeSeriesUtils.SerializeFixedSizeQueue(InitialWindowedBuffer, writer); + } + + private protected override void CloneCore(State state) + { + base.CloneCore(state); + Contracts.Assert(state is State); + var stateLocal = state as State; + stateLocal.WindowedBuffer = WindowedBuffer.Clone(); + stateLocal.InitialWindowedBuffer = InitialWindowedBuffer.Clone(); + if (_model != null) + { + _parentForecaster.Model = _parentForecaster.Model.Clone(); + _model = _parentForecaster.Model; + } + } + + private protected override void LearnStateFromDataCore(FixedSizeQueue data) + { + // This method is empty because there is no need to implement a training logic here. + } + + private protected override void InitializeForecaster() + { + _parentForecaster = (SsaForecastingBase)Parent; + _model = _parentForecaster.Model; + } + + public override void TransformCore(ref float input, FixedSizeQueue windowedBuffer, long iteration, ref VBuffer dst) + { + // Forecasting is being done without prediction engine. Update the model + // with the observation. + if (PreviousPosition == -1) + Consume(input); + + dst = new VBuffer(LocalHorizon ?? _parentForecaster.Horizon, + ((AdaptiveSingularSpectrumSequenceModelerInternal)_model).Forecast(_parentForecaster.Horizon)); + } + + private protected override void TransformCore(ref float input, FixedSizeQueue windowedBuffer, long iteration, + ref VBuffer dst1, ref VBuffer dst2, ref VBuffer dst3) + { + // Forecasting is being done without prediction engine. Update the model + // with the observation. + if (PreviousPosition == -1) + Consume(input); + + ((AdaptiveSingularSpectrumSequenceModelerInternal)_model).ForecastWithConfidenceIntervals(LocalHorizon ?? _parentForecaster.Horizon, + out float[] forecast, out float[] min, out float[] max, LocalConfidenceLevel ?? _parentForecaster.ConfidenceLevel); + + dst1 = new VBuffer(_parentForecaster.Horizon, forecast); + dst2 = new VBuffer(_parentForecaster.Horizon, min); + dst3 = new VBuffer(_parentForecaster.Horizon, max); + } + + public override void Forecast( ref VBuffer dst) + { + int horizon = LocalHorizon ?? _parentForecaster.Horizon; + dst = new VBuffer(horizon, ((AdaptiveSingularSpectrumSequenceModelerInternal)_model).Forecast(horizon)); + } + + public override void ConfidenceIntervalLowerBound(ref VBuffer dst) + { + int horizon = LocalHorizon ?? _parentForecaster.Horizon; + ((AdaptiveSingularSpectrumSequenceModelerInternal)_model).ForecastWithConfidenceIntervals(horizon, + out float[] forecast, out float[] min, out float[] max, LocalConfidenceLevel ?? _parentForecaster.ConfidenceLevel); + + dst = new VBuffer(horizon, min); + } + + public override void ConfidenceIntervalUpperBound(ref VBuffer dst) + { + int horizon = LocalHorizon ?? _parentForecaster.Horizon; + ((AdaptiveSingularSpectrumSequenceModelerInternal)_model).ForecastWithConfidenceIntervals(horizon, + out float[] forecast, out float[] min, out float[] max, LocalConfidenceLevel ?? _parentForecaster.ConfidenceLevel); + + dst = new VBuffer(horizon, max); + } + + public override void Consume(Single input) => _model.Consume(ref input, _parentForecaster.IsAdaptive); + } + } + } +} diff --git a/src/Microsoft.ML.TimeSeries/TimeSeriesProcessing.cs b/src/Microsoft.ML.TimeSeries/TimeSeriesProcessing.cs index a7dd59cfc9..9420aad1f3 100644 --- a/src/Microsoft.ML.TimeSeries/TimeSeriesProcessing.cs +++ b/src/Microsoft.ML.TimeSeries/TimeSeriesProcessing.cs @@ -124,5 +124,19 @@ public static CommonOutputs.TransformOutput SsaSpikeDetector(IHostEnvironment en OutputData = view }; } + + [TlcModule.EntryPoint(Desc = TimeSeries.SsaForecastingTransformer.Summary, + UserName = TimeSeries.SsaForecastingTransformer.UserName, + ShortName = TimeSeries.SsaForecastingTransformer.ShortName)] + internal static CommonOutputs.TransformOutput SsaForecasting(IHostEnvironment env, SsaForecastingTransformer.Options options) + { + var h = EntryPointUtils.CheckArgsAndCreateHost(env, "SsaForecasting", options); + var view = new SsaForecastingEstimator(h, options).Fit(options.Data).Transform(options.Data); + return new CommonOutputs.TransformOutput() + { + Model = new TransformModelImpl(h, view, options.Data), + OutputData = view + }; + } } } diff --git a/src/Microsoft.ML.Transforms/NormalizerCatalog.cs b/src/Microsoft.ML.Transforms/NormalizerCatalog.cs index 59ab4b401e..ec6c117d01 100644 --- a/src/Microsoft.ML.Transforms/NormalizerCatalog.cs +++ b/src/Microsoft.ML.Transforms/NormalizerCatalog.cs @@ -141,7 +141,7 @@ public static NormalizingEstimator NormalizeLogMeanVariance(this TransformsCatal long maximumExampleCount = NormalizingEstimator.Defaults.MaximumExampleCount, bool useCdf = NormalizingEstimator.Defaults.LogMeanVarCdf) { - var columnOptions = new NormalizingEstimator.LogMeanVarianceColumnOptions(outputColumnName, inputColumnName, maximumExampleCount, useCdf); + var columnOptions = new NormalizingEstimator.LogMeanVarianceColumnOptions(outputColumnName, inputColumnName, maximumExampleCount, useCdf, false); return new NormalizingEstimator(CatalogUtils.GetEnvironment(catalog), columnOptions); } @@ -159,7 +159,54 @@ public static NormalizingEstimator NormalizeLogMeanVariance(this TransformsCatal bool useCdf = NormalizingEstimator.Defaults.LogMeanVarCdf) => new NormalizingEstimator(CatalogUtils.GetEnvironment(catalog), columns.Select(column => - new NormalizingEstimator.LogMeanVarianceColumnOptions(column.OutputColumnName, column.InputColumnName, maximumExampleCount, useCdf)).ToArray()); + new NormalizingEstimator.LogMeanVarianceColumnOptions(column.OutputColumnName, column.InputColumnName, maximumExampleCount, useCdf, false)).ToArray()); + + /// + /// Create a , which normalizes based on the computed mean and variance of the logarithm of the data. + /// + /// The transform catalog + /// Name of the column resulting from the transformation of . + /// The data type on this column is the same as the input column. + /// Whether to map zero to zero, preserving sparsity. + /// Name of the column to transform. If set to , the value of the will be used as source. + /// The data type on this column should be , or a known-sized vector of those types. + /// Maximum number of examples used to train the normalizer. + /// Whether to use CDF as the output. + /// + /// + /// + /// + /// + public static NormalizingEstimator NormalizeLogMeanVariance(this TransformsCatalog catalog, + string outputColumnName, + bool fixZero, + string inputColumnName = null, + long maximumExampleCount = NormalizingEstimator.Defaults.MaximumExampleCount, + bool useCdf = NormalizingEstimator.Defaults.LogMeanVarCdf) + { + var columnOptions = new NormalizingEstimator.LogMeanVarianceColumnOptions(outputColumnName, inputColumnName, maximumExampleCount, useCdf, fixZero); + return new NormalizingEstimator(CatalogUtils.GetEnvironment(catalog), columnOptions); + } + + /// + /// Create a , which normalizes based on the computed mean and variance of the logarithm of the data. + /// + /// The transform catalog + /// The pairs of input and output columns. + /// The input columns must be of data type , or a known-sized vector of those types. + /// The data type for the output column will be the same as the associated input column. + /// Whether to map zero to zero, preserving sparsity. + /// Maximum number of examples used to train the normalizer. + /// Whether to use CDF as the output. + public static NormalizingEstimator NormalizeLogMeanVariance(this TransformsCatalog catalog, InputOutputColumnPair[] columns, + bool fixZero, + long maximumExampleCount = NormalizingEstimator.Defaults.MaximumExampleCount, + bool useCdf = NormalizingEstimator.Defaults.LogMeanVarCdf) => + new NormalizingEstimator(CatalogUtils.GetEnvironment(catalog), + columns.Select(column => + new NormalizingEstimator.LogMeanVarianceColumnOptions(column.OutputColumnName, column.InputColumnName, maximumExampleCount, useCdf, fixZero)).ToArray()); /// /// Create a , which normalizes by assigning the data into bins with equal density. diff --git a/src/Microsoft.ML.Transforms/RandomFourierFeaturizing.cs b/src/Microsoft.ML.Transforms/RandomFourierFeaturizing.cs index 44cc3564ca..4dc0637eaa 100644 --- a/src/Microsoft.ML.Transforms/RandomFourierFeaturizing.cs +++ b/src/Microsoft.ML.Transforms/RandomFourierFeaturizing.cs @@ -622,7 +622,6 @@ private void TransformFeatures(in VBuffer src, ref VBuffer dst, Tr /// Check the See Also section for links to usage examples. /// ]]> /// - /// /// public sealed class ApproximatedKernelMappingEstimator : IEstimator { diff --git a/src/Native/CMakeLists.txt b/src/Native/CMakeLists.txt index a814277e91..586a58a704 100644 --- a/src/Native/CMakeLists.txt +++ b/src/Native/CMakeLists.txt @@ -8,6 +8,13 @@ set(RESOURCES) include_directories("${CMAKE_BINARY_DIR}/../../") if(WIN32) + # There seems to be a bug in the latest VS2019 + # which is adding /ZI (which conflicts with /guard:cf) instead of /Zi. + message("CMAKE_C_FLAGS_DEBUG is ${CMAKE_C_FLAGS_DEBUG}") + message("In a future version, if the CMake that ships with VS2019 no longer contains the /ZI flag, delete this message block and the two lines below.") + string(REPLACE "/ZI" "/Zi" CMAKE_C_FLAGS_DEBUG ${CMAKE_C_FLAGS_DEBUG}) + string(REPLACE "/ZI" "/Zi" CMAKE_CXX_FLAGS_DEBUG ${CMAKE_CXX_FLAGS_DEBUG}) + add_definitions(-DWIN32) add_definitions(-D_WIN32=1) add_definitions(-DUNICODE -D_UNICODE) @@ -22,7 +29,7 @@ if(WIN32) add_compile_options($<$:/MT>) add_compile_options($<$:/MT>) add_compile_options(/guard:cf) - add_compile_options(/d2Zi+) # make optimized builds debugging easier + add_compile_options(/Zo) # make optimized builds debugging easier. /Zo is the newer documented flag. add_compile_options(/nologo) # Suppress Startup Banner add_compile_options(/W3) # set warning level to 3 add_compile_options(/WX) # treat warnings as errors diff --git a/src/Redist/Microsoft.ML.TensorFlow.Redist/Microsoft.ML.TensorFlow.Redist.proj b/src/Redist/Microsoft.ML.TensorFlow.Redist/Microsoft.ML.TensorFlow.Redist.proj index 3a847b65af..4d8de1db3a 100644 --- a/src/Redist/Microsoft.ML.TensorFlow.Redist/Microsoft.ML.TensorFlow.Redist.proj +++ b/src/Redist/Microsoft.ML.TensorFlow.Redist/Microsoft.ML.TensorFlow.Redist.proj @@ -6,11 +6,26 @@ - - - - - + + + + + + @@ -93,8 +108,6 @@ <_fileFromArchive Include="%(TensorFlowArchive.FilesFromArchive)" ExtractDirectory="%(TensorFlowArchive.ExtractDirectory)" Runtime="%(TensorFlowArchive.Runtime)" /> <_fileFromArchive DestinationFile="%(FileName)%(Extension)"/> - - <_fileFromArchive Condition="'%(Runtime)' == 'osx-x64' AND '%(Extension)' == '.so'" DestinationFile="%(FileName).dylib" /> <_fileFromArchive PackagePath="runtimes\%(_fileFromArchive.Runtime)\native\%(_fileFromArchive.DestinationFile)" /> @@ -120,9 +133,6 @@ - - diff --git a/src/Redist/Microsoft.ML.TensorFlow.Redist/libtensorflow-cpu-darwin-x86_64-1.13.1.tar.gz.sha b/src/Redist/Microsoft.ML.TensorFlow.Redist/libtensorflow-cpu-darwin-x86_64-1.13.1.tar.gz.sha deleted file mode 100644 index b6cfa6d7b8..0000000000 --- a/src/Redist/Microsoft.ML.TensorFlow.Redist/libtensorflow-cpu-darwin-x86_64-1.13.1.tar.gz.sha +++ /dev/null @@ -1 +0,0 @@ -9D748CF9FA3A18BC9456CF3F4E44DE519403FB542A85D1916BB9B1E3AFD90139258936C780E78488721D0872A365BE07CB97A53A6C851BEE5362D5221AE17BF3 \ No newline at end of file diff --git a/src/Redist/Microsoft.ML.TensorFlow.Redist/libtensorflow-cpu-darwin-x86_64-1.14.0.tar.gz.sha b/src/Redist/Microsoft.ML.TensorFlow.Redist/libtensorflow-cpu-darwin-x86_64-1.14.0.tar.gz.sha new file mode 100644 index 0000000000..951c4556e5 --- /dev/null +++ b/src/Redist/Microsoft.ML.TensorFlow.Redist/libtensorflow-cpu-darwin-x86_64-1.14.0.tar.gz.sha @@ -0,0 +1 @@ +7002EF701BD23C5EF5FF94192E935F0DDF960A21BE2531CEE158586830C00E0BA889900F7F6E8AB568BEE0ACF1F5A6A246BB43D11C4109E9DC782B46377D8142 \ No newline at end of file diff --git a/src/Redist/Microsoft.ML.TensorFlow.Redist/libtensorflow-cpu-linux-x86_64-1.13.1.tar.gz.sha b/src/Redist/Microsoft.ML.TensorFlow.Redist/libtensorflow-cpu-linux-x86_64-1.13.1.tar.gz.sha deleted file mode 100644 index 55c2e50b2a..0000000000 --- a/src/Redist/Microsoft.ML.TensorFlow.Redist/libtensorflow-cpu-linux-x86_64-1.13.1.tar.gz.sha +++ /dev/null @@ -1 +0,0 @@ -A09EA6EF85BAEF567AE33538D0FEF648317AC4357A6C4C2AF4890E2C60E16A73014645118FFE3A5A56E03E0C941B7770AB7342532EBF07066784A72443970AE7 \ No newline at end of file diff --git a/src/Redist/Microsoft.ML.TensorFlow.Redist/libtensorflow-cpu-linux-x86_64-1.14.0.tar.gz.sha b/src/Redist/Microsoft.ML.TensorFlow.Redist/libtensorflow-cpu-linux-x86_64-1.14.0.tar.gz.sha new file mode 100644 index 0000000000..784640a022 --- /dev/null +++ b/src/Redist/Microsoft.ML.TensorFlow.Redist/libtensorflow-cpu-linux-x86_64-1.14.0.tar.gz.sha @@ -0,0 +1 @@ +E3F6D0309117E9E45780ECF8BC4D0268B3FC9F12E3E38FFE58496789330A4ACD2DC8FF721F3B8900357F6155F8A54000E45B99495F823486B558E8B42532392D \ No newline at end of file diff --git a/src/Redist/Microsoft.ML.TensorFlow.Redist/libtensorflow-cpu-windows-x86_64-1.13.1.zip.sha b/src/Redist/Microsoft.ML.TensorFlow.Redist/libtensorflow-cpu-windows-x86_64-1.13.1.zip.sha deleted file mode 100644 index 018d84782a..0000000000 --- a/src/Redist/Microsoft.ML.TensorFlow.Redist/libtensorflow-cpu-windows-x86_64-1.13.1.zip.sha +++ /dev/null @@ -1 +0,0 @@ -301762731DFCFFABEC7C5ED95453CC27E56B10E06B3EB4CB83B1F552A5345D00E087F65B9FC99D74219016B4C53CC70A7FFC13C29C1D10FC8EEEFA6B18896144 \ No newline at end of file diff --git a/src/Redist/Microsoft.ML.TensorFlow.Redist/libtensorflow-cpu-windows-x86_64-1.14.0.zip.sha b/src/Redist/Microsoft.ML.TensorFlow.Redist/libtensorflow-cpu-windows-x86_64-1.14.0.zip.sha new file mode 100644 index 0000000000..b7d6402c45 --- /dev/null +++ b/src/Redist/Microsoft.ML.TensorFlow.Redist/libtensorflow-cpu-windows-x86_64-1.14.0.zip.sha @@ -0,0 +1 @@ +59A2B80B441439B851202358CE4A65BA0DDDB319A8A29E87B135DCD9954BC5B0628F2C0C8E72D6942EA3CDCE172805C2BD5421815B3D0210B62BC0936DC59A08 \ No newline at end of file diff --git a/test/BaselineOutput/Common/EntryPoints/Summarize.txt b/test/BaselineOutput/Common/EntryPoints/Summarize.txt new file mode 100644 index 0000000000..2fa47dda99 --- /dev/null +++ b/test/BaselineOutput/Common/EntryPoints/Summarize.txt @@ -0,0 +1,3912 @@ +#@ TextLoader{ +#@ header+ +#@ sep=tab +#@ col=Bias:R8:0 +#@ col=TreeWeights:R8:1 +#@ col=TreeID:I4:2 +#@ col=IsLeaf:TX:3 +#@ col=LeftChild:I4:4 +#@ col=RightChild:I4:5 +#@ col=NumericalSplitFeatureIndexes:I4:6 +#@ col=NumericalSplitThresholds:R4:7 +#@ col=CategoricalSplitFlags:BL:8 +#@ col=LeafValues:R8:9 +#@ col=SplitGains:R8:10 +#@ col=CategoricalSplitFeatures:I4:11-** +#@ col={name=CategoricalCategoricalSplitFeatureRange type=I4 src={ min=-1 var=+}} +#@ } +Bias TreeWeights TreeID IsLeaf LeftChild RightChild NumericalSplitFeatureIndexes NumericalSplitThresholds CategoricalSplitFlags LeafValues SplitGains +0 1 0 Tree node 2 1 0 2.5 0 0 17.785768088177125 +0 1 0 Tree node 5 3 1 2.5 0 0 1.6465267967918438 +0 1 0 Tree node 7 -4 4 4.5 0 0 0.71082434639549064 +0 1 0 Tree node 4 8 0 4.5 0 0 0.42381305215160303 +0 1 0 Tree node -3 6 4 2.5 0 0 0.60804232804232861 +0 1 0 Tree node -2 -7 4 1.5 0 0 0.2260869565217391 +0 1 0 Tree node -6 9 4 7.5 0 0 0.083626844890002827 +0 1 0 Tree node 10 -9 6 2.5 0 0 0.055001052148183405 +0 1 0 Tree node -5 11 2 1.5 0 0 0.050887551204886528 +0 1 0 Tree node -8 15 5 3.5 0 0 0.045714285714285374 +0 1 0 Tree node 13 -12 4 2.5 0 0 0.0095897435897462913 +0 1 0 Tree node 12 -13 6 2.5 0 0 0.0049443308339025549 +0 1 0 Tree node -10 -14 5 5.5 0 0 0.0074074074074073071 +0 1 0 Tree node 14 -15 2 1.5 0 0 3.5527136788005009E-15 +0 1 0 Tree node -1 -16 0 1.5 0 0 1.7763568394002505E-15 +0 1 0 Tree node -11 -17 5 5.5 0 0 2.2204460492503131E-16 +0 1 0 Leaf node 7 5:-1.0000000000000131 +0 1 0 Leaf node 7 5:-0.99999999999999967 +0 1 0 Leaf node 7 5:-0.42857142857142844 +0 1 0 Leaf node 7 5:0.058823529411763879 +0 1 0 Leaf node 7 5:0.63636363636360638 +0 1 0 Leaf node 7 5:0.47368421052631576 +0 1 0 Leaf node 7 5:6.9999999999999976E-45 +0 1 0 Leaf node 7 5:0.60000000000000131 +0 1 0 Leaf node 7 5:-0.6363636363636368 +0 1 0 Leaf node 7 5:0.83333333333333393 +0 1 0 Leaf node 7 5:1.0000000000000002 +0 1 0 Leaf node 7 5:-0.87500000000000033 +0 1 0 Leaf node 7 5:0.99999999999999911 +0 1 0 Leaf node 7 5:0.99999999999999967 +0 1 0 Leaf node 7 5:-0.99999999999999944 +0 1 0 Leaf node 7 5:-1.0000000000000002 +0 1 0 Leaf node 7 5:0.99999999999999978 +0 1 1 Tree node 2 1 0 2.5 0 0 11.454785468643887 +0 1 1 Tree node 5 3 1 2.5 0 0 1.0604358429592122 +0 1 1 Tree node 6 -4 5 4.5 0 0 0.53999191150213477 +0 1 1 Tree node 4 9 0 4.5 0 0 0.27319749841997343 +0 1 1 Tree node -3 7 4 2.5 0 0 0.39050430572805894 +0 1 1 Tree node -2 -7 3 2.5 0 0 0.14928874578956988 +0 1 1 Tree node 13 -8 4 3.5 0 0 0.091127812441879286 +0 1 1 Tree node 8 10 1 4.5 0 0 0.057036823076973442 +0 1 1 Tree node -6 -10 3 3.5 0 0 0.071891750827165313 +0 1 1 Tree node -5 11 2 1.5 0 0 0.033000707096361381 +0 1 1 Tree node -9 -12 3 4.5 0 0 0.010877352897405657 +0 1 1 Tree node 12 18 6 2.5 0 0 0.0032167239171947415 +0 1 1 Tree node -11 -14 2 5.5 0 0 0.0063517593168255226 +0 1 1 Tree node 14 15 6 1.5 0 0 0.00034029474834262885 +0 1 1 Tree node 16 -16 4 2.5 0 0 0.0002265695930052658 +0 1 1 Tree node -15 -17 5 2.5 0 0 5.7263644352878451E-05 +0 1 1 Tree node 17 -18 0 1.5 0 0 1.7763568394002505E-15 +0 1 1 Tree node -1 -19 2 2.5 0 0 1.7763568394002505E-15 +0 1 1 Tree node -13 -20 7 6.5 0 0 8.8817841970012523E-16 +0 1 1 Leaf node 7 5:-0.83516002301781556 +0 1 1 Leaf node 7 5:-0.89662561583167877 +0 1 1 Leaf node 7 5:-0.34559310714370484 +0 1 1 Leaf node 7 5:0.2116089646832488 +0 1 1 Leaf node 7 5:0.51807586153566532 +0 1 1 Leaf node 7 5:0.65195446992533235 +0 1 1 Leaf node 7 5:-0.069329440654477714 +0 1 1 Leaf node 7 5:-0.41213811886748514 +0 1 1 Leaf node 7 5:0.87487722590049721 +0 1 1 Leaf node 7 5:0.1277497640533187 +0 1 1 Leaf node 7 5:0.69368537418188003 +0 1 1 Leaf node 7 5:0.65941447318530344 +0 1 1 Leaf node 7 5:0.83516002301782055 +0 1 1 Leaf node 7 5:0.84944487669860125 +0 1 1 Leaf node 7 5:-0.85292753769564333 +0 1 1 Leaf node 7 5:-0.85234404485935344 +0 1 1 Leaf node 7 5:-0.84280916364700587 +0 1 1 Leaf node 7 5:-0.83516002301781966 +0 1 1 Leaf node 7 5:-0.83516002301782022 +0 1 1 Leaf node 7 5:0.83516002301781933 +0 1 2 Tree node 1 3 0 3.5 0 0 7.5392691652446091 +0 1 2 Tree node 2 8 4 5.5 0 0 1.4864804823138398 +0 1 2 Tree node 6 -4 6 3.5 0 0 0.38516316430086661 +0 1 2 Tree node 4 9 0 4.5 0 0 0.070925408875513796 +0 1 2 Tree node 5 -6 2 5.5 0 0 0.13371058874908101 +0 1 2 Tree node -2 -7 4 7.5 0 0 0.11646966989787789 +0 1 2 Tree node 10 7 4 2.5 0 0 0.063942632416533307 +0 1 2 Tree node -8 -9 3 2.5 0 0 0.1709599061264315 +0 1 2 Tree node -3 -10 5 4.5 0 0 0.029485557104393323 +0 1 2 Tree node -5 11 2 1.5 0 0 0.022435424337950316 +0 1 2 Tree node 14 -12 0 2.5 0 0 0.011429117108978204 +0 1 2 Tree node 12 13 5 4.5 0 0 0.0031877499206554027 +0 1 2 Tree node 18 -14 0 8.5 0 0 0.010203714727515578 +0 1 2 Tree node -13 17 4 2.5 0 0 0.00051196003400599821 +0 1 2 Tree node 16 15 0 1.5 0 0 0.00022181222864414707 +0 1 2 Tree node -16 -17 5 2.5 0 0 0.00032292226043484362 +0 1 2 Tree node -1 -18 4 1.5 0 0 0.00028323012481745735 +0 1 2 Tree node -15 -19 6 2.5 0 0 0.00020791606832148979 +0 1 2 Tree node -11 -20 6 3.5 0 0 8.8967040437648581E-05 +0 1 2 Leaf node 7 5:-0.74025552192789046 +0 1 2 Leaf node 7 5:-0.16326816242738612 +0 1 2 Leaf node 7 5:0.49765682775397591 +0 1 2 Leaf node 7 5:0.18185272435245958 +0 1 2 Leaf node 7 5:0.4317133410486772 +0 1 2 Leaf node 7 5:0.84127318282102337 +0 1 2 Leaf node 7 5:0.59016153047042985 +0 1 2 Leaf node 7 5:-0.736878123608942 +0 1 2 Leaf node 7 5:0.12105478692687226 +0 1 2 Leaf node 7 5:0.87235047104793451 +0 1 2 Leaf node 7 5:0.75079481734277076 +0 1 2 Leaf node 7 5:-0.82019128072142011 +0 1 2 Leaf node 7 5:0.76945383961054514 +0 1 2 Leaf node 7 5:0.53805401463060321 +0 1 2 Leaf node 7 5:0.75069895633750217 +0 1 2 Leaf node 7 5:-0.74285713806706444 +0 1 2 Leaf node 7 5:-0.76693406123396701 +0 1 2 Leaf node 7 5:-0.75599062089691682 +0 1 2 Leaf node 7 5:0.73997724088872519 +0 1 2 Leaf node 7 5:0.73997724088872474 +0 1 3 Tree node 1 3 0 3.5 0 0 4.9569209843676543 +0 1 3 Tree node 2 8 4 5.5 0 0 0.97160634848663996 +0 1 3 Tree node 6 -4 6 3.5 0 0 0.25182784904621602 +0 1 3 Tree node 4 11 5 4.5 0 0 0.050266809736952522 +0 1 3 Tree node 5 13 7 1.5 0 0 0.061867421220459295 +0 1 3 Tree node 9 -7 2 5.5 0 0 0.089116438212435922 +0 1 3 Tree node 10 7 4 2.5 0 0 0.042662415170788393 +0 1 3 Tree node -8 -9 5 1.5 0 0 0.12256247030301171 +0 1 3 Tree node -3 -10 3 3.5 0 0 0.023115817607411215 +0 1 3 Tree node -2 -11 4 5.5 0 0 0.021957723027224714 +0 1 3 Tree node 17 -12 0 2.5 0 0 0.0072345752337517055 +0 1 3 Tree node 12 14 1 4.5 0 0 0.0061956406432162048 +0 1 3 Tree node -5 -14 4 8.5 0 0 0.019466983610124139 +0 1 3 Tree node -6 -15 7 4.5 0 0 0.0047273118137041159 +0 1 3 Tree node 15 16 0 5.5 0 0 0.0020477401569514164 +0 1 3 Tree node -13 -17 4 8.5 0 0 0.0025123366467380559 +0 1 3 Tree node -16 -18 6 1.5 0 0 0.00040219369675043026 +0 1 3 Tree node -1 18 0 1.5 0 0 0.00014131210384249471 +0 1 3 Tree node -19 -20 5 2.5 0 0 0.00020550203233732356 +0 1 3 Leaf node 7 5:-0.67910265966523431 +0 1 3 Leaf node 7 5:-0.12614821028001405 +0 1 3 Leaf node 7 5:0.44560858046361068 +0 1 3 Leaf node 7 5:0.14599188791913748 +0 1 3 Leaf node 7 5:0.32364264101523998 +0 1 3 Leaf node 7 5:0.59936575580278251 +0 1 3 Leaf node 7 5:0.71743938923772466 +0 1 3 Leaf node 7 5:-0.76141512320614002 +0 1 3 Leaf node 7 5:-0.02243913091672789 +0 1 3 Leaf node 7 5:0.78248097666996363 +0 1 3 Leaf node 7 5:0.2197434511660977 +0 1 3 Leaf node 7 5:-0.7316562073292483 +0 1 3 Leaf node 7 5:0.746845173229854 +0 1 3 Leaf node 7 5:0.69620607347363428 +0 1 3 Leaf node 7 5:0.74683941512752361 +0 1 3 Leaf node 7 5:0.69449413494951118 +0 1 3 Leaf node 7 5:0.68169788372500384 +0 1 3 Leaf node 7 5:0.678341889992506 +0 1 3 Leaf node 7 5:-0.68046529007978385 +0 1 3 Leaf node 7 5:-0.69732734228970483 +0 1 4 Tree node 4 1 1 2.5 0 0 3.3648318772250714 +0 1 4 Tree node -2 2 0 1.5 0 0 0.48157521456886243 +0 1 4 Tree node 3 5 4 2.5 0 0 0.11006413457692643 +0 1 4 Tree node -3 11 3 3.5 0 0 0.31728462756785558 +0 1 4 Tree node 12 -6 7 1.5 0 0 0.099151179424424196 +0 1 4 Tree node 8 6 0 3.5 0 0 0.015687558770881571 +0 1 4 Tree node 7 13 4 8.5 0 0 0.030466565744893481 +0 1 4 Tree node -7 9 0 4.5 0 0 0.062479942707049901 +0 1 4 Tree node -4 -10 6 4.5 0 0 0.030451624831607904 +0 1 4 Tree node 10 17 3 5.5 0 0 0.021198610669313778 +0 1 4 Tree node -9 -12 5 4.5 0 0 0.04094035396287074 +0 1 4 Tree node -5 -13 6 7.5 0 0 0.0062844681665965296 +0 1 4 Tree node 15 -14 6 2.5 0 0 0.0055881937603312615 +0 1 4 Tree node 18 14 3 6.5 0 0 0.0048528406235339716 +0 1 4 Tree node -15 -16 6 4.5 0 0 0.0082348933991172601 +0 1 4 Tree node -1 16 4 2.5 0 0 0.0031639346679037672 +0 1 4 Tree node -17 -18 5 1.5 0 0 0.0031172038997350771 +0 1 4 Tree node -11 -19 5 6.5 0 0 0.0024250625447380381 +0 1 4 Tree node -8 -20 0 4.5 0 0 0.0014648330184170444 +0 1 4 Leaf node 7 5:-0.63712598823665623 +0 1 4 Leaf node 7 5:-0.55144966622089264 +0 1 4 Leaf node 7 5:-0.60278790814868133 +0 1 4 Leaf node 7 5:0.78629221620683021 +0 1 4 Leaf node 7 5:0.49170713377344383 +0 1 4 Leaf node 7 5:-0.058637990163642371 +0 1 4 Leaf node 7 5:-0.053016412554148497 +0 1 4 Leaf node 7 5:0.66527569631899708 +0 1 4 Leaf node 7 5:0.062370133341687319 +0 1 4 Leaf node 7 5:0.38692293939070216 +0 1 4 Leaf node 7 5:0.6855710922697501 +0 1 4 Leaf node 7 5:0.63955865207899876 +0 1 4 Leaf node 7 5:0.73349176825284179 +0 1 4 Leaf node 7 5:-0.64598421106675052 +0 1 4 Leaf node 7 5:0.36984105442940551 +0 1 4 Leaf node 7 5:0.63748237804682961 +0 1 4 Leaf node 7 5:-0.69465824479698379 +0 1 4 Leaf node 7 5:-0.47697027144893539 +0 1 4 Leaf node 7 5:0.63706381852564919 +0 1 4 Leaf node 7 5:0.64146046424417491 +0 1 5 Tree node 2 1 1 2.5 0 0 2.2266955762236385 +0 1 5 Tree node 3 4 5 2.5 0 0 0.32113830414610589 +0 1 5 Tree node 11 -4 5 3.5 0 0 0.081731277346908326 +0 1 5 Tree node -2 -5 0 2.5 0 0 0.072925347172220489 +0 1 5 Tree node 5 6 4 1.5 0 0 0.059870173322468356 +0 1 5 Tree node -3 -7 2 3.5 0 0 0.13863930144313646 +0 1 5 Tree node 7 10 6 8.5 0 0 0.020065184839636596 +0 1 5 Tree node 8 12 5 4.5 0 0 0.02893292904044209 +0 1 5 Tree node 9 -10 6 4.5 0 0 0.032618217174464267 +0 1 5 Tree node 16 -11 3 4.5 0 0 0.060032472228908951 +0 1 5 Tree node -8 -12 0 4.5 0 0 0.022219311865554703 +0 1 5 Tree node 14 -13 0 2.5 0 0 0.011090899375911123 +0 1 5 Tree node -9 13 2 2.5 0 0 0.0096478024952059149 +0 1 5 Tree node -14 -15 4 7.5 0 0 0.018931599937353938 +0 1 5 Tree node 17 15 3 1.5 0 0 0.003138538348598674 +0 1 5 Tree node -16 -17 4 2.5 0 0 0.0061811982097792484 +0 1 5 Tree node -6 -18 4 9.5 0 0 0.0029134260186311489 +0 1 5 Tree node -1 18 5 1.5 0 0 0.0015794275655390266 +0 1 5 Tree node -19 -20 5 2.5 0 0 0.0021785356729120825 +0 1 5 Leaf node 7 5:-0.60942513365043893 +0 1 5 Leaf node 7 5:-0.61350930057376085 +0 1 5 Leaf node 7 5:-0.35806418345529634 +0 1 5 Leaf node 7 5:-0.0033188441089130895 +0 1 5 Leaf node 7 5:0.18634691143945928 +0 1 5 Leaf node 7 5:0.62003008502460211 +0 1 5 Leaf node 7 5:0.72009768914666872 +0 1 5 Leaf node 7 5:0.75038293342126672 +0 1 5 Leaf node 7 5:0.66555398911559138 +0 1 5 Leaf node 7 5:0.016537097335072071 +0 1 5 Leaf node 7 5:-0.0027223300409208047 +0 1 5 Leaf node 7 5:0.61176391358487181 +0 1 5 Leaf node 7 5:-0.69589474088240499 +0 1 5 Leaf node 7 5:0.26494061574521649 +0 1 5 Leaf node 7 5:0.61196227517430624 +0 1 5 Leaf node 7 5:-0.60695746169983589 +0 1 5 Leaf node 7 5:-0.58915404361370138 +0 1 5 Leaf node 7 5:0.617119607358383 +0 1 5 Leaf node 7 5:-0.43169152505237196 +0 1 5 Leaf node 7 5:-0.605419794416029 +0 1 6 Tree node 2 1 1 2.5 0 0 1.4768072087055806 +0 1 6 Tree node 3 4 5 2.5 0 0 0.21367161878941165 +0 1 6 Tree node 14 -4 7 1.5 0 0 0.062813914804945581 +0 1 6 Tree node -2 -5 0 2.5 0 0 0.048024075046444487 +0 1 6 Tree node 5 7 5 4.5 0 0 0.039909160293157608 +0 1 6 Tree node 6 13 6 7.5 0 0 0.069043844351630246 +0 1 6 Tree node -3 8 2 1.5 0 0 0.053906432070409252 +0 1 6 Tree node -6 10 2 1.5 0 0 0.029391811507596732 +0 1 6 Tree node 9 -10 2 5.5 0 0 0.025563318887974612 +0 1 6 Tree node -8 -11 1 4.5 0 0 0.026382947115488865 +0 1 6 Tree node 11 12 1 4.5 0 0 0.019317607375981649 +0 1 6 Tree node -9 -13 4 8.5 0 0 0.043296093411369627 +0 1 6 Tree node -12 18 0 4.5 0 0 0.0058628276886528163 +0 1 6 Tree node -7 -15 7 2.5 0 0 0.0053709020037477628 +0 1 6 Tree node 15 -16 4 3.5 0 0 0.0040438484726235124 +0 1 6 Tree node 16 -17 6 2.5 0 0 0.01563690569968057 +0 1 6 Tree node 17 -18 4 2.5 0 0 0.0031877058780001022 +0 1 6 Tree node -1 -19 3 1.5 0 0 0.00038064552842143051 +0 1 6 Tree node -14 -20 6 5.5 0 0 0.00026244522316837271 +0 1 6 Leaf node 7 5:-0.59096041322772475 +0 1 6 Leaf node 7 5:-0.58894587595718872 +0 1 6 Leaf node 7 5:-0.2441969205303969 +0 1 6 Leaf node 7 5:0.025215554503243455 +0 1 6 Leaf node 7 5:0.1523305965313744 +0 1 6 Leaf node 7 5:0.73925478157361546 +0 1 6 Leaf node 7 5:0.66255149152754766 +0 1 6 Leaf node 7 5:-0.14301179390189941 +0 1 6 Leaf node 7 5:-0.15608286338200414 +0 1 6 Leaf node 7 5:0.60857221306508269 +0 1 6 Leaf node 7 5:0.27760999298009492 +0 1 6 Leaf node 7 5:0.64923498311631966 +0 1 6 Leaf node 7 5:0.58958832526557259 +0 1 6 Leaf node 7 5:0.58964811148741347 +0 1 6 Leaf node 7 5:0.59247183699744066 +0 1 6 Leaf node 7 5:-0.26981291103800642 +0 1 6 Leaf node 7 5:-0.69469744311210224 +0 1 6 Leaf node 7 5:-0.62210180824874162 +0 1 6 Leaf node 7 5:-0.58339331131815875 +0 1 6 Leaf node 7 5:0.58338059901738826 +0 1 7 Tree node 3 1 1 2.5 0 0 0.98267624126066111 +0 1 7 Tree node 2 4 4 2.5 0 0 0.15641463742075218 +0 1 7 Tree node 16 15 3 3.5 0 0 0.17999739728518938 +0 1 7 Tree node 8 -5 5 3.5 0 0 0.052269725017739188 +0 1 7 Tree node 7 5 0 3.5 0 0 0.009113419809487433 +0 1 7 Tree node 6 9 0 4.5 0 0 0.02420997773693917 +0 1 7 Tree node -6 13 4 8.5 0 0 0.03764094896277221 +0 1 7 Tree node -3 -9 4 8.5 0 0 0.015832270922302327 +0 1 7 Tree node 17 -10 0 2.5 0 0 0.0065281927367447645 +0 1 7 Tree node 14 10 1 6.5 0 0 0.005334317994526272 +0 1 7 Tree node 11 12 1 8.5 0 0 0.012423680703875617 +0 1 7 Tree node -11 -13 5 4.5 0 0 0.036299232949423853 +0 1 7 Tree node -12 -14 5 4.5 0 0 0.0053428763348583597 +0 1 7 Tree node -8 -15 3 3.5 0 0 0.0036028148317047426 +0 1 7 Tree node -7 -16 4 5.5 0 0 0.0030910763873618352 +0 1 7 Tree node -4 -17 6 7.5 0 0 0.0026533424742291728 +0 1 7 Tree node -2 -18 0 1.5 0 0 0.0025381496804612047 +0 1 7 Tree node -1 18 3 1.5 0 0 0.002319573230862737 +0 1 7 Tree node -19 -20 4 2.5 0 0 0.0041680645345724132 +0 1 7 Leaf node 7 5:-0.45967359032024091 +0 1 7 Leaf node 7 5:-0.57191282405294586 +0 1 7 Leaf node 7 5:0.51486793580060419 +0 1 7 Leaf node 7 5:0.34648823848695115 +0 1 7 Leaf node 7 5:0.054131124611757162 +0 1 7 Leaf node 7 5:-0.19930331974063303 +0 1 7 Leaf node 7 5:0.61275766519257269 +0 1 7 Leaf node 7 5:0.5862712625279265 +0 1 7 Leaf node 7 5:0.33473190498296673 +0 1 7 Leaf node 7 5:-0.63237482520358357 +0 1 7 Leaf node 7 5:-0.12163508560840822 +0 1 7 Leaf node 7 5:0.62520434633236432 +0 1 7 Leaf node 7 5:0.56724385740745986 +0 1 7 Leaf node 7 5:0.56738830226919812 +0 1 7 Leaf node 7 5:0.25768952864781652 +0 1 7 Leaf node 7 5:0.57617337155810211 +0 1 7 Leaf node 7 5:0.62612277664389548 +0 1 7 Leaf node 7 5:-0.5550635189965929 +0 1 7 Leaf node 7 5:-0.56718349139165458 +0 1 7 Leaf node 7 5:-0.50254944451819639 +0 1 8 Tree node 1 3 0 3.5 0 0 0.66536258125299119 +0 1 8 Tree node 2 6 4 5.5 0 0 0.21156612850914802 +0 1 8 Tree node 7 -4 1 3.5 0 0 0.035347045455051851 +0 1 8 Tree node 4 10 0 4.5 0 0 0.021173250918402775 +0 1 8 Tree node -2 5 5 3.5 0 0 0.039388041882774387 +0 1 8 Tree node -6 -7 4 7.5 0 0 0.011956248029585063 +0 1 8 Tree node -3 -8 5 4.5 0 0 0.010779537101287567 +0 1 8 Tree node 9 8 4 2.5 0 0 0.010512711956295662 +0 1 8 Tree node -9 -10 5 1.5 0 0 0.033981403375574371 +0 1 8 Tree node 15 -11 5 3.5 0 0 0.021304258822453015 +0 1 8 Tree node 11 12 4 3.5 0 0 0.0065543612240061 +0 1 8 Tree node -5 -13 5 7.5 0 0 0.0042881320819726093 +0 1 8 Tree node 13 16 3 5.5 0 0 0.0039757543845597432 +0 1 8 Tree node 14 17 4 8.5 0 0 0.014451702497454241 +0 1 8 Tree node -12 -16 5 4.5 0 0 0.029656051134302919 +0 1 8 Tree node -1 -17 0 2.5 0 0 0.0029428024641653394 +0 1 8 Tree node -14 18 5 4.5 0 0 0.0026885308938822405 +0 1 8 Tree node -15 -19 5 4.5 0 0 0.0017925633233363408 +0 1 8 Tree node -18 -20 6 5.5 0 0 0.00012556052649784211 +0 1 8 Leaf node 7 5:-0.55408254832985404 +0 1 8 Leaf node 7 5:-0.28789479031165482 +0 1 8 Leaf node 7 5:0.26234655626027176 +0 1 8 Leaf node 7 5:0.059731505121539206 +0 1 8 Leaf node 7 5:0.62476646693244353 +0 1 8 Leaf node 7 5:0.053863106410461575 +0 1 8 Leaf node 7 5:0.58332984358332363 +0 1 8 Leaf node 7 5:0.68054061996126081 +0 1 8 Leaf node 7 5:-0.59072693795367659 +0 1 8 Leaf node 7 5:0.061392772182986034 +0 1 8 Leaf node 7 5:-0.65234469819990226 +0 1 8 Leaf node 7 5:-0.22208214402922027 +0 1 8 Leaf node 7 5:0.55339681310819977 +0 1 8 Leaf node 7 5:0.58889800112411428 +0 1 8 Leaf node 7 5:0.57934396210814187 +0 1 8 Leaf node 7 5:0.55982216917928118 +0 1 8 Leaf node 7 5:-0.58755171835672737 +0 1 8 Leaf node 7 5:0.5580141923199754 +0 1 8 Leaf node 7 5:0.55278778799464079 +0 1 8 Leaf node 7 5:0.55227197686411889 +0 1 9 Tree node 1 2 6 2.5 0 0 0.46211195382233078 +0 1 9 Tree node 5 4 4 5.5 0 0 0.11364064475415986 +0 1 9 Tree node 3 6 4 1.5 0 0 0.023407128582424108 +0 1 9 Tree node -2 -5 1 3.5 0 0 0.051822612932553018 +0 1 9 Tree node -3 18 5 4.5 0 0 0.012563483256531151 +0 1 9 Tree node 12 11 2 2.5 0 0 0.0096764554786254453 +0 1 9 Tree node 7 8 2 1.5 0 0 0.0084174825576821266 +0 1 9 Tree node -4 -9 1 4.5 0 0 0.051792571786919671 +0 1 9 Tree node 9 15 0 4.5 0 0 0.017365562478503127 +0 1 9 Tree node -8 10 4 6.5 0 0 0.032375454149245177 +0 1 9 Tree node -11 -12 7 1.5 0 0 0.021062634309595405 +0 1 9 Tree node -7 -13 5 1.5 0 0 0.0071173825329110818 +0 1 9 Tree node 13 14 3 2.5 0 0 0.0036613075522242688 +0 1 9 Tree node -1 -15 4 2.5 0 0 0.023114039771071848 +0 1 9 Tree node -14 -16 4 1.5 0 0 0.01594193426181878 +0 1 9 Tree node 16 17 5 4.5 0 0 0.0033461805382599963 +0 1 9 Tree node -10 -18 6 8.5 0 0 0.002596509490379581 +0 1 9 Tree node -17 -19 1 4.5 0 0 0.0022198283426134446 +0 1 9 Tree node -6 -20 0 5.5 0 0 0.00072197085345742512 +0 1 9 Leaf node 7 5:-0.53944970971230977 +0 1 9 Leaf node 7 5:-0.60440304857439919 +0 1 9 Leaf node 7 5:-0.0053669646141650479 +0 1 9 Leaf node 7 5:0.67444525441391723 +0 1 9 Leaf node 7 5:0.34587860871731263 +0 1 9 Leaf node 7 5:0.58428097372510623 +0 1 9 Leaf node 7 5:-0.54262569786920289 +0 1 9 Leaf node 7 5:-0.2026826272047059 +0 1 9 Leaf node 7 5:-0.029843260707607084 +0 1 9 Leaf node 7 5:0.58440711845439119 +0 1 9 Leaf node 7 5:0.61463492852757617 +0 1 9 Leaf node 7 5:0.006675582403521038 +0 1 9 Leaf node 7 5:-0.05257196512448873 +0 1 9 Leaf node 7 5:-0.57390345932906783 +0 1 9 Leaf node 7 5:-0.60915231508775991 +0 1 9 Leaf node 7 5:0.091288990386104135 +0 1 9 Leaf node 7 5:0.59167977454141141 +0 1 9 Leaf node 7 5:0.54638527704219375 +0 1 9 Leaf node 7 5:0.5426083350724733 +0 1 9 Leaf node 7 5:0.54562130194229053 +0 1 10 Tree node 1 2 4 3.5 0 0 0.30789455065013699 +0 1 10 Tree node 3 15 0 4.5 0 0 0.10940037591044327 +0 1 10 Tree node -2 5 0 1.5 0 0 0.03045259743356539 +0 1 10 Tree node 4 -5 2 3.5 0 0 0.025405318634106722 +0 1 10 Tree node 10 -6 1 3.5 0 0 0.018517533118444396 +0 1 10 Tree node 6 7 0 3.5 0 0 0.013919742117652012 +0 1 10 Tree node -4 -8 6 4.5 0 0 0.030101488852239788 +0 1 10 Tree node 8 9 1 4.5 0 0 0.010531835457834871 +0 1 10 Tree node -7 -10 0 4.5 0 0 0.027429139420893117 +0 1 10 Tree node -9 11 0 4.5 0 0 0.013750779633581581 +0 1 10 Tree node 16 -12 4 2.5 0 0 0.0057504814446144858 +0 1 10 Tree node 12 14 0 9.5 0 0 0.0050860137512775405 +0 1 10 Tree node 13 18 4 8.5 0 0 0.016342913596589407 +0 1 10 Tree node -11 -15 7 1.5 0 0 0.033333683644683448 +0 1 10 Tree node -13 -16 5 4.5 0 0 0.0043060924487002133 +0 1 10 Tree node -3 -17 5 7.5 0 0 0.0031064711526102018 +0 1 10 Tree node 17 -18 5 3.5 0 0 0.0022544544202355299 +0 1 10 Tree node -1 -19 0 2.5 0 0 0.0014075164497784898 +0 1 10 Tree node -14 -20 5 4.5 0 0 0.00065099159393832567 +0 1 10 Leaf node 7 5:-0.53532763461312483 +0 1 10 Leaf node 7 5:-0.15291558816113995 +0 1 10 Leaf node 7 5:0.59696869858668045 +0 1 10 Leaf node 7 5:0.6910534782776756 +0 1 10 Leaf node 7 5:0.20987376889830989 +0 1 10 Leaf node 7 5:-0.50667599462483714 +0 1 10 Leaf node 7 5:-0.24554533133325326 +0 1 10 Leaf node 7 5:0.06969654390899814 +0 1 10 Leaf node 7 5:0.60498587294634831 +0 1 10 Leaf node 7 5:0.56131472681193095 +0 1 10 Leaf node 7 5:-0.46670194019273203 +0 1 10 Leaf node 7 5:-0.074852306135570118 +0 1 10 Leaf node 7 5:0.58151845081865428 +0 1 10 Leaf node 7 5:0.54820674743086217 +0 1 10 Leaf node 7 5:0.55368829741411296 +0 1 10 Leaf node 7 5:0.53436688029504109 +0 1 10 Leaf node 7 5:0.53672230865263593 +0 1 10 Leaf node 7 5:-0.37878951618426387 +0 1 10 Leaf node 7 5:-0.55723751684751865 +0 1 10 Leaf node 7 5:0.5343768719384725 +0 1 11 Tree node 1 3 6 2.5 0 0 0.22626816903925392 +0 1 11 Tree node 2 13 1 4.5 0 0 0.050850795324268311 +0 1 11 Tree node 14 -4 4 5.5 0 0 0.014402581733509279 +0 1 11 Tree node 4 5 4 1.5 0 0 0.0097156660280974527 +0 1 11 Tree node -2 -6 3 3.5 0 0 0.029846285793087287 +0 1 11 Tree node 6 7 2 1.5 0 0 0.008208514191130542 +0 1 11 Tree node -5 -8 1 4.5 0 0 0.03930673379936804 +0 1 11 Tree node 8 10 1 4.5 0 0 0.011630114507080985 +0 1 11 Tree node 9 -10 2 5.5 0 0 0.028465018701186815 +0 1 11 Tree node -7 -11 3 3.5 0 0 0.017175906713295365 +0 1 11 Tree node 11 12 3 4.5 0 0 0.0056087602061532654 +0 1 11 Tree node -9 -13 4 7.5 0 0 0.0061990483146454056 +0 1 11 Tree node -12 -14 4 4.5 0 0 0.0055296896065546164 +0 1 11 Tree node -3 -15 0 7.5 0 0 0.0052209466501699983 +0 1 11 Tree node 18 15 3 1.5 0 0 0.0017015766102715935 +0 1 11 Tree node -16 16 4 1.5 0 0 0.0053099415573156228 +0 1 11 Tree node 17 -18 3 2.5 0 0 0.018312958411079702 +0 1 11 Tree node -17 -19 4 2.5 0 0 0.0086135043904270736 +0 1 11 Tree node -1 -20 5 2.5 0 0 0.0015769507670384715 +0 1 11 Leaf node 7 5:-0.26961114638364003 +0 1 11 Leaf node 7 5:-0.583549237407753 +0 1 11 Leaf node 7 5:0.46497128821381539 +0 1 11 Leaf node 7 5:0.12336134672059156 +0 1 11 Leaf node 7 5:0.62796913989424075 +0 1 11 Leaf node 7 5:0.32309133849965038 +0 1 11 Leaf node 7 5:0.077469251279986245 +0 1 11 Leaf node 7 5:-0.060250197216972445 +0 1 11 Leaf node 7 5:0.58848130357847583 +0 1 11 Leaf node 7 5:0.44209327868916243 +0 1 11 Leaf node 7 5:-0.54081951713312248 +0 1 11 Leaf node 7 5:-0.031283472133162869 +0 1 11 Leaf node 7 5:0.53677255006165248 +0 1 11 Leaf node 7 5:0.53402587487795372 +0 1 11 Leaf node 7 5:0.070040792637224819 +0 1 11 Leaf node 7 5:-0.53027778978358542 +0 1 11 Leaf node 7 5:-0.5424972937929653 +0 1 11 Leaf node 7 5:-0.014760001803255296 +0 1 11 Leaf node 7 5:-0.64427460953313231 +0 1 11 Leaf node 7 5:-0.54019340326486431 +0 1 12 Tree node 1 4 5 3.5 0 0 0.15614538106077411 +0 1 12 Tree node 2 -3 6 7.5 0 0 0.041096152599663371 +0 1 12 Tree node 9 3 4 4.5 0 0 0.019998563921278217 +0 1 12 Tree node -4 -5 4 7.5 0 0 0.027668714508952946 +0 1 12 Tree node 6 5 3 2.5 0 0 0.010142435130639796 +0 1 12 Tree node -6 7 2 1.5 0 0 0.040778538880145321 +0 1 12 Tree node -2 -8 1 2.5 0 0 0.02548237418944807 +0 1 12 Tree node 8 11 2 5.5 0 0 0.008865407752897278 +0 1 12 Tree node -7 12 1 4.5 0 0 0.02547820286656529 +0 1 12 Tree node 10 -11 7 1.5 0 0 0.0083394579515667644 +0 1 12 Tree node 15 13 4 2.5 0 0 0.011169333448184382 +0 1 12 Tree node -9 17 1 3.5 0 0 0.0071121424044018264 +0 1 12 Tree node -10 14 5 4.5 0 0 0.0054713812403121252 +0 1 12 Tree node -12 -15 1 1.5 0 0 0.0023904831842125746 +0 1 12 Tree node -14 -16 4 7.5 0 0 0.0015695417262545121 +0 1 12 Tree node 18 16 0 2.5 0 0 0.0012826848436582883 +0 1 12 Tree node -17 -18 1 2.5 0 0 0.0011505115941559327 +0 1 12 Tree node -13 -19 4 3.5 0 0 0.00053978733616190377 +0 1 12 Tree node -1 -20 4 1.5 0 0 0.00027651925904334518 +0 1 12 Leaf node 7 5:-0.52318445554083692 +0 1 12 Leaf node 7 5:-0.40756931869872531 +0 1 12 Leaf node 7 5:0.57663002357232107 +0 1 12 Leaf node 7 5:0.37341354199561883 +0 1 12 Leaf node 7 5:-0.18572597812748576 +0 1 12 Leaf node 7 5:0.70485447365705389 +0 1 12 Leaf node 7 5:-0.29630573735653426 +0 1 12 Leaf node 7 5:0.47476256663167732 +0 1 12 Leaf node 7 5:0.59642023952503043 +0 1 12 Leaf node 7 5:0.014275306923851813 +0 1 12 Leaf node 7 5:0.076685210177186908 +0 1 12 Leaf node 7 5:-0.5468995729178433 +0 1 12 Leaf node 7 5:0.5432497684841372 +0 1 12 Leaf node 7 5:0.5568737044416987 +0 1 12 Leaf node 7 5:-0.43555855651595199 +0 1 12 Leaf node 7 5:0.52409558964838221 +0 1 12 Leaf node 7 5:-0.52740704979530739 +0 1 12 Leaf node 7 5:-0.36975782176742489 +0 1 12 Leaf node 7 5:0.52507335235142838 +0 1 12 Leaf node 7 5:-0.53291325569631554 +0 1 13 Tree node 1 6 6 2.5 0 0 0.10976432500911738 +0 1 13 Tree node 2 5 0 4.5 0 0 0.025580917620604049 +0 1 13 Tree node 4 3 5 3.5 0 0 0.013206540993223992 +0 1 13 Tree node -4 -5 3 2.5 0 0 0.031741720511075766 +0 1 13 Tree node 15 -6 1 3.5 0 0 0.013913362070583843 +0 1 13 Tree node -3 14 0 6.5 0 0 0.0066102605600364605 +0 1 13 Tree node 7 10 6 8.5 0 0 0.0057814210948923389 +0 1 13 Tree node 11 8 6 4.5 0 0 0.01143565455751884 +0 1 13 Tree node 9 17 0 4.5 0 0 0.02835334587583397 +0 1 13 Tree node -9 -11 4 6.5 0 0 0.040613029357732713 +0 1 13 Tree node -8 -12 0 4.5 0 0 0.0093700335443936342 +0 1 13 Tree node 12 -13 7 2.5 0 0 0.0052544392793956764 +0 1 13 Tree node -2 13 4 2.5 0 0 0.0089338527218209632 +0 1 13 Tree node -14 -15 3 4.5 0 0 0.021867570298709046 +0 1 13 Tree node -7 -16 3 5.5 0 0 0.0046647722237541517 +0 1 13 Tree node 16 -17 4 4.5 0 0 0.0026477041129318452 +0 1 13 Tree node -1 -18 4 2.5 0 0 0.0045648797767236932 +0 1 13 Tree node -10 18 1 7.5 0 0 0.0020688040371609553 +0 1 13 Tree node -19 -20 3 5.5 0 0 0.0031045947335267225 +0 1 13 Leaf node 7 5:-0.51986865350638722 +0 1 13 Leaf node 7 5:-0.11040410506852875 +0 1 13 Leaf node 7 5:0.57268517709278788 +0 1 13 Leaf node 7 5:-0.47794315950627336 +0 1 13 Leaf node 7 5:0.52969828982641054 +0 1 13 Leaf node 7 5:-0.59291448151184489 +0 1 13 Leaf node 7 5:-0.20585137616315158 +0 1 13 Leaf node 7 5:0.57631786371468052 +0 1 13 Leaf node 7 5:-0.56942420962051921 +0 1 13 Leaf node 7 5:0.54102819256282986 +0 1 13 Leaf node 7 5:0.086637015758144434 +0 1 13 Leaf node 7 5:0.52567855096280236 +0 1 13 Leaf node 7 5:0.60012811707891789 +0 1 13 Leaf node 7 5:0.59980474967717967 +0 1 13 Leaf node 7 5:-0.163669081046947 +0 1 13 Leaf node 7 5:0.5285796662139729 +0 1 13 Leaf node 7 5:0.0087206217048215635 +0 1 13 Leaf node 7 5:-0.55262587005832275 +0 1 13 Leaf node 7 5:-0.062278296234288807 +0 1 13 Leaf node 7 5:0.53087963763241142 +0 1 14 Tree node 1 9 6 2.5 0 0 0.073664541454314877 +0 1 14 Tree node 2 5 0 4.5 0 0 0.017191198396735739 +0 1 14 Tree node 4 3 5 3.5 0 0 0.0087712365000271603 +0 1 14 Tree node -4 -5 3 2.5 0 0 0.020826534731069608 +0 1 14 Tree node 6 -6 3 3.5 0 0 0.010978205932861182 +0 1 14 Tree node -3 -7 3 4.5 0 0 0.0051531902121699511 +0 1 14 Tree node 7 8 2 2.5 0 0 0.0046008918998156623 +0 1 14 Tree node 18 -9 4 2.5 0 0 0.0081403544084001356 +0 1 14 Tree node -8 -10 5 1.5 0 0 0.0057685104722513692 +0 1 14 Tree node 10 14 6 8.5 0 0 0.0037898214095373091 +0 1 14 Tree node 11 12 4 5.5 0 0 0.0076259624779356407 +0 1 14 Tree node 13 -13 6 5.5 0 0 0.029691049603581644 +0 1 14 Tree node -12 15 3 2.5 0 0 0.0068336581738681415 +0 1 14 Tree node -2 -15 6 3.5 0 0 0.0068198668979555163 +0 1 14 Tree node -11 -16 0 4.5 0 0 0.0062325520726422244 +0 1 14 Tree node 17 16 2 4.5 0 0 0.0042938045490877302 +0 1 14 Tree node -17 -18 2 6.5 0 0 0.011052590787260487 +0 1 14 Tree node -14 -19 0 5.5 0 0 0.0034047397736560285 +0 1 14 Tree node -1 -20 0 2.5 0 0 0.00066999120683295527 +0 1 14 Leaf node 7 5:-0.51561869960975903 +0 1 14 Leaf node 7 5:0.064965489279098662 +0 1 14 Leaf node 7 5:0.54672553394811341 +0 1 14 Leaf node 7 5:-0.42794255710570928 +0 1 14 Leaf node 7 5:0.47723415266950997 +0 1 14 Leaf node 7 5:-0.57997271525713234 +0 1 14 Leaf node 7 5:-0.026878550870769707 +0 1 14 Leaf node 7 5:-0.51706575812488298 +0 1 14 Leaf node 7 5:-0.42202885266646972 +0 1 14 Leaf node 7 5:0.29902428292017508 +0 1 14 Leaf node 7 5:0.56089726024265174 +0 1 14 Leaf node 7 5:0.59297259819488046 +0 1 14 Leaf node 7 5:-0.40072862909976958 +0 1 14 Leaf node 7 5:0.55722741175196289 +0 1 14 Leaf node 7 5:0.43592366056422038 +0 1 14 Leaf node 7 5:0.52092228948001518 +0 1 14 Leaf node 7 5:-0.33126445296649076 +0 1 14 Leaf node 7 5:0.5260318530802458 +0 1 14 Leaf node 7 5:0.19646547609909421 +0 1 14 Leaf node 7 5:-0.53613078749954124 +0 1 15 Tree node 1 6 5 3.5 0 0 0.050005428138913066 +0 1 15 Tree node 4 2 1 3.5 0 0 0.016557202478245235 +0 1 15 Tree node 3 -4 7 1.5 0 0 0.015595979747119293 +0 1 15 Tree node -3 5 6 2.5 0 0 0.015440120074788278 +0 1 15 Tree node 11 -6 2 3.5 0 0 0.012629088966815562 +0 1 15 Tree node -5 -7 2 1.5 0 0 0.012257885441092938 +0 1 15 Tree node 7 12 4 8.5 0 0 0.0051562434717637301 +0 1 15 Tree node 8 13 7 2.5 0 0 0.01058109640068222 +0 1 15 Tree node 9 10 1 4.5 0 0 0.014589261486559816 +0 1 15 Tree node -2 -11 4 4.5 0 0 0.013713378230676872 +0 1 15 Tree node -10 14 5 4.5 0 0 0.010599093172183952 +0 1 15 Tree node 15 -13 4 3.5 0 0 0.0049522757658672922 +0 1 15 Tree node -8 16 1 3.5 0 0 0.0034970325367616878 +0 1 15 Tree node -9 -15 1 6.5 0 0 0.0033012248935417824 +0 1 15 Tree node -12 18 0 6.5 0 0 0.0014063932939876282 +0 1 15 Tree node -1 -17 4 1.5 0 0 0.00063813835519541812 +0 1 15 Tree node -14 17 5 4.5 0 0 0.00051174752651069605 +0 1 15 Tree node -18 -19 1 4.5 0 0 0.00031072370211207394 +0 1 15 Tree node -16 -20 6 8.5 0 0 0.00026222586413785984 +0 1 15 Leaf node 7 5:-0.51339738856293449 +0 1 15 Leaf node 7 5:-0.02426698880298082 +0 1 15 Leaf node 7 5:-0.49334650742449315 +0 1 15 Leaf node 7 5:0.56481960286895749 +0 1 15 Leaf node 7 5:-0.094952890238406931 +0 1 15 Leaf node 7 5:-0.54326021716146011 +0 1 15 Leaf node 7 5:0.59065079019206201 +0 1 15 Leaf node 7 5:0.56978708664851263 +0 1 15 Leaf node 7 5:0.59642953706560553 +0 1 15 Leaf node 7 5:-0.1839475716406358 +0 1 15 Leaf node 7 5:-0.43299026032990323 +0 1 15 Leaf node 7 5:0.5439815453655269 +0 1 15 Leaf node 7 5:0.09811949056835112 +0 1 15 Leaf node 7 5:0.52674333583354316 +0 1 15 Leaf node 7 5:0.51633072937038049 +0 1 15 Leaf node 7 5:0.52330486812375832 +0 1 15 Leaf node 7 5:-0.43149915483121271 +0 1 15 Leaf node 7 5:0.52577498512317622 +0 1 15 Leaf node 7 5:0.51429071196982468 +0 1 15 Leaf node 7 5:0.51306045121323574 +0 1 16 Tree node 1 5 6 2.5 0 0 0.034922180292757038 +0 1 16 Tree node 2 -3 1 7.5 0 0 0.0083223183202063983 +0 1 16 Tree node 3 -4 7 1.5 0 0 0.0047971127476390645 +0 1 16 Tree node 4 -5 2 4.5 0 0 0.0089085260885463126 +0 1 16 Tree node 14 -6 3 4.5 0 0 0.011078276507295447 +0 1 16 Tree node -2 6 0 2.5 0 0 0.0029566503350601826 +0 1 16 Tree node 7 12 0 8.5 0 0 0.0036751512578696004 +0 1 16 Tree node 8 13 6 8.5 0 0 0.0059848726521023179 +0 1 16 Tree node 9 11 4 8.5 0 0 0.0078583449644792584 +0 1 16 Tree node 10 -11 6 5.5 0 0 0.022991477677359592 +0 1 16 Tree node -7 -12 4 4.5 0 0 0.0057047016773697362 +0 1 16 Tree node -10 -13 7 2.5 0 0 0.0050871685526441262 +0 1 16 Tree node -8 -14 5 4.5 0 0 0.0029861919934006491 +0 1 16 Tree node -9 -15 0 4.5 0 0 0.0024827401489719743 +0 1 16 Tree node 15 16 2 2.5 0 0 0.0017641011558221786 +0 1 16 Tree node 17 -17 4 4.5 0 0 0.0076070837805875952 +0 1 16 Tree node -16 -18 4 1.5 0 0 0.0051551566812635665 +0 1 16 Tree node 18 -19 1 2.5 0 0 0.0012355323424063612 +0 1 16 Tree node -1 -20 0 2.5 0 0 0.0015064247629036319 +0 1 16 Leaf node 7 5:-0.51191893450905124 +0 1 16 Leaf node 7 5:0.29019842488311814 +0 1 16 Leaf node 7 5:0.5255640515471518 +0 1 16 Leaf node 7 5:0.15669068632421074 +0 1 16 Leaf node 7 5:-0.50111679931034747 +0 1 16 Leaf node 7 5:0.46909914212190967 +0 1 16 Leaf node 7 5:0.22643921142398804 +0 1 16 Leaf node 7 5:0.56243454747629318 +0 1 16 Leaf node 7 5:0.54457670131682889 +0 1 16 Leaf node 7 5:0.52812482862226084 +0 1 16 Leaf node 7 5:-0.46702945097047871 +0 1 16 Leaf node 7 5:-0.096463221998153437 +0 1 16 Leaf node 7 5:-0.21201960400216255 +0 1 16 Leaf node 7 5:0.51898810437384124 +0 1 16 Leaf node 7 5:0.51477035270057747 +0 1 16 Leaf node 7 5:-0.51408170728461933 +0 1 16 Leaf node 7 5:-0.50311065672176392 +0 1 16 Leaf node 7 5:0.35326841600395303 +0 1 16 Leaf node 7 5:-0.48706778743675477 +0 1 16 Leaf node 7 5:0.11560697958169701 +0 1 17 Tree node 1 3 4 3.5 0 0 0.026199460202249732 +0 1 17 Tree node 2 10 2 3.5 0 0 0.013812601462603366 +0 1 17 Tree node 11 -4 7 2.5 0 0 0.0065256636036647997 +0 1 17 Tree node 5 4 2 4.5 0 0 0.0041491933000817067 +0 1 17 Tree node 9 15 2 5.5 0 0 0.021472977884773738 +0 1 17 Tree node 7 6 2 1.5 0 0 0.0062679780531134915 +0 1 17 Tree node -7 12 5 3.5 0 0 0.017431788033337468 +0 1 17 Tree node -2 8 1 1.5 0 0 0.01356467167840044 +0 1 17 Tree node -9 -10 6 3.5 0 0 0.017513443963996442 +0 1 17 Tree node -5 -11 1 6.5 0 0 0.0043236920175540282 +0 1 17 Tree node -3 -12 3 4.5 0 0 0.0040974800518205736 +0 1 17 Tree node 13 -13 1 3.5 0 0 0.0039489480201614045 +0 1 17 Tree node -8 17 4 7.5 0 0 0.0030701152575358893 +0 1 17 Tree node 14 -15 6 2.5 0 0 0.0010550782638396247 +0 1 17 Tree node 18 -16 0 2.5 0 0 0.0012641364611505994 +0 1 17 Tree node -6 16 4 6.5 0 0 0.00065159790451085539 +0 1 17 Tree node -17 -18 0 4.5 0 0 0.0018248089917160346 +0 1 17 Tree node -14 -19 5 6.5 0 0 0.00048841227187443412 +0 1 17 Tree node -1 -20 1 2.5 0 0 0.0002164612927133755 +0 1 17 Leaf node 7 5:-0.51028982763868502 +0 1 17 Leaf node 7 5:-0.42277316428413669 +0 1 17 Leaf node 7 5:0.019320773294559633 +0 1 17 Leaf node 7 5:0.31156797536281611 +0 1 17 Leaf node 7 5:-0.48659105477631959 +0 1 17 Leaf node 7 5:0.064281516586084872 +0 1 17 Leaf node 7 5:0.64741500559038356 +0 1 17 Leaf node 7 5:-0.096320958403999735 +0 1 17 Leaf node 7 5:0.44445955453559471 +0 1 17 Leaf node 7 5:-0.23116636026869331 +0 1 17 Leaf node 7 5:-0.28969748977531329 +0 1 17 Leaf node 7 5:0.59723274359376055 +0 1 17 Leaf node 7 5:-0.37878851033466465 +0 1 17 Leaf node 7 5:0.53005093338098419 +0 1 17 Leaf node 7 5:-0.29406869347719866 +0 1 17 Leaf node 7 5:-0.54848794865282813 +0 1 17 Leaf node 7 5:0.53984573614176667 +0 1 17 Leaf node 7 5:0.51132138865418397 +0 1 17 Leaf node 7 5:0.51274512568913 +0 1 17 Leaf node 7 5:-0.21614877965036547 +0 1 18 Tree node 1 5 5 3.5 0 0 0.021112280277619702 +0 1 18 Tree node 2 -3 6 7.5 0 0 0.0072426158891725489 +0 1 18 Tree node 3 -4 2 4.5 0 0 0.0076852841327901163 +0 1 18 Tree node 13 4 4 4.5 0 0 0.014187055201277939 +0 1 18 Tree node -5 -6 2 2.5 0 0 0.028021692501024001 +0 1 18 Tree node 6 7 2 1.5 0 0 0.0030468784982390171 +0 1 18 Tree node -2 -8 3 2.5 0 0 0.011739583575037712 +0 1 18 Tree node 8 9 2 3.5 0 0 0.0034360588258430143 +0 1 18 Tree node -7 17 0 4.5 0 0 0.012085805660976178 +0 1 18 Tree node -9 10 1 3.5 0 0 0.0044276691641317971 +0 1 18 Tree node -11 11 1 4.5 0 0 0.0037936938511448064 +0 1 18 Tree node -12 12 0 4.5 0 0 0.0065228729975793471 +0 1 18 Tree node -13 16 6 2.5 0 0 0.0020661283318547871 +0 1 18 Tree node 14 -15 7 1.5 0 0 0.0011068223959918924 +0 1 18 Tree node 15 -16 1 3.5 0 0 0.0056301714272222747 +0 1 18 Tree node -1 18 4 1.5 0 0 0.00088934004047698176 +0 1 18 Tree node -14 -18 2 5.5 0 0 0.00041181809233768654 +0 1 18 Tree node -10 -19 4 7.5 0 0 0.00037696629897412005 +0 1 18 Tree node -17 -20 5 2.5 0 0 0.00035392922972881258 +0 1 18 Leaf node 7 5:-0.50767822966285159 +0 1 18 Leaf node 7 5:-0.17295558776101061 +0 1 18 Leaf node 7 5:0.53904949376236988 +0 1 18 Leaf node 7 5:-0.42917784518720897 +0 1 18 Leaf node 7 5:-0.13695983025434957 +0 1 18 Leaf node 7 5:0.75127197060561313 +0 1 18 Leaf node 7 5:-0.31252913743253513 +0 1 18 Leaf node 7 5:0.59776428443976437 +0 1 18 Leaf node 7 5:0.56389253822138763 +0 1 18 Leaf node 7 5:0.53297860584467716 +0 1 18 Leaf node 7 5:-0.20958286723473954 +0 1 18 Leaf node 7 5:0.57881874984774861 +0 1 18 Leaf node 7 5:-0.17123486409166658 +0 1 18 Leaf node 7 5:0.51903575497466992 +0 1 18 Leaf node 7 5:0.058433678961001261 +0 1 18 Leaf node 7 5:-0.44029695400379548 +0 1 18 Leaf node 7 5:-0.52944059309296676 +0 1 18 Leaf node 7 5:0.50826530129270375 +0 1 18 Leaf node 7 5:0.51155038802476882 +0 1 18 Leaf node 7 5:-0.25508466122784074 +0 1 19 Tree node 1 8 5 3.5 0 0 0.014190199768306239 +0 1 19 Tree node 2 3 1 3.5 0 0 0.0055077660299448321 +0 1 19 Tree node 6 -4 2 3.5 0 0 0.0081959995001485778 +0 1 19 Tree node 4 -5 7 1.5 0 0 0.0050914731131662585 +0 1 19 Tree node -3 5 6 2.5 0 0 0.0043363227309953702 +0 1 19 Tree node -6 -7 2 1.5 0 0 0.0049769170058882612 +0 1 19 Tree node 13 7 2 2.5 0 0 0.002133850218479897 +0 1 19 Tree node -8 -9 5 1.5 0 0 0.0024311646100819814 +0 1 19 Tree node 9 10 2 1.5 0 0 0.0020209412243530371 +0 1 19 Tree node -2 -11 3 2.5 0 0 0.0077694407581772587 +0 1 19 Tree node 11 14 4 8.5 0 0 0.0025986605929219806 +0 1 19 Tree node -10 12 1 4.5 0 0 0.0093093382415621491 +0 1 19 Tree node -13 15 5 4.5 0 0 0.0038272890901722627 +0 1 19 Tree node 16 -15 4 3.5 0 0 0.0016960439122046828 +0 1 19 Tree node 18 -16 0 4.5 0 0 0.0014890156339094915 +0 1 19 Tree node -14 -17 3 4.5 0 0 0.0010957508418878246 +0 1 19 Tree node 17 -18 7 1.5 0 0 0.00033034016828489311 +0 1 19 Tree node -1 -19 4 1.5 0 0 0.00036494291543407234 +0 1 19 Tree node -12 -20 2 8.5 0 0 0.00029222910450662012 +0 1 19 Leaf node 7 5:-0.50609387423444407 +0 1 19 Leaf node 7 5:-0.14192085538615329 +0 1 19 Leaf node 7 5:-0.34603898228179691 +0 1 19 Leaf node 7 5:-0.46699454633165033 +0 1 19 Leaf node 7 5:0.54615244936276774 +0 1 19 Leaf node 7 5:-0.078188516896581808 +0 1 19 Leaf node 7 5:0.5538234615342047 +0 1 19 Leaf node 7 5:-0.50607299498804292 +0 1 19 Leaf node 7 5:0.38202874048234969 +0 1 19 Leaf node 7 5:-0.25400576917316048 +0 1 19 Leaf node 7 5:0.57839588978842604 +0 1 19 Leaf node 7 5:0.53878402507588119 +0 1 19 Leaf node 7 5:-0.1212847805097932 +0 1 19 Leaf node 7 5:0.52952899957688537 +0 1 19 Leaf node 7 5:-0.24470367691430886 +0 1 19 Leaf node 7 5:0.50873919600530271 +0 1 19 Leaf node 7 5:0.50927566714919525 +0 1 19 Leaf node 7 5:-0.28440547303266195 +0 1 19 Leaf node 7 5:-0.51668495583616403 +0 1 19 Leaf node 7 5:0.51843122366064986 +0 1 20 Tree node 1 2 1 2.5 0 0 0.009674849364679862 +0 1 20 Tree node 10 -3 4 3.5 0 0 0.0027435181413555275 +0 1 20 Tree node 3 9 5 4.5 0 0 0.0019154054355676429 +0 1 20 Tree node 4 -5 0 9.5 0 0 0.005137610635238125 +0 1 20 Tree node 5 -6 0 6.5 0 0 0.017571945561059375 +0 1 20 Tree node 6 -7 3 6.5 0 0 0.0076624798758601828 +0 1 20 Tree node 7 -8 3 4.5 0 0 0.010215085733546701 +0 1 20 Tree node 14 8 4 2.5 0 0 0.0066079085433918125 +0 1 20 Tree node 11 -10 6 4.5 0 0 0.0077318736514283039 +0 1 20 Tree node -4 12 2 1.5 0 0 0.0020017297944750264 +0 1 20 Tree node 16 -12 4 2.5 0 0 0.0018850361759946005 +0 1 20 Tree node -9 -13 4 4.5 0 0 0.0014167048802566223 +0 1 20 Tree node 13 15 0 4.5 0 0 0.0007747823193042765 +0 1 20 Tree node -11 -15 1 4.5 0 0 0.0034864109677124259 +0 1 20 Tree node -2 -16 0 1.5 0 0 0.00070376716888741371 +0 1 20 Tree node -14 18 1 4.5 0 0 0.00057303865185990861 +0 1 20 Tree node 17 -18 5 3.5 0 0 0.00049711603090177477 +0 1 20 Tree node -1 -19 6 1.5 0 0 0.00016734002131534639 +0 1 20 Tree node -17 -20 4 7.5 0 0 0.00016218215607818496 +0 1 20 Leaf node 7 5:-0.50572527772382725 +0 1 20 Leaf node 7 5:-0.50855313478407871 +0 1 20 Leaf node 7 5:0.093306778109122773 +0 1 20 Leaf node 7 5:0.5441449346359355 +0 1 20 Leaf node 7 5:0.54776147197261305 +0 1 20 Leaf node 7 5:-0.61056705751485219 +0 1 20 Leaf node 7 5:-0.38450585350215011 +0 1 20 Leaf node 7 5:0.57265828541475927 +0 1 20 Leaf node 7 5:0.37618006185346875 +0 1 20 Leaf node 7 5:-0.14018895144911214 +0 1 20 Leaf node 7 5:-0.19632502741589752 +0 1 20 Leaf node 7 5:-0.55242845300201282 +0 1 20 Leaf node 7 5:0.52878405509374049 +0 1 20 Leaf node 7 5:0.52440253357193889 +0 1 20 Leaf node 7 5:0.52599945616346411 +0 1 20 Leaf node 7 5:-0.54911194842364974 +0 1 20 Leaf node 7 5:0.5132558640019943 +0 1 20 Leaf node 7 5:-0.23205688461773452 +0 1 20 Leaf node 7 5:-0.51247593858256357 +0 1 20 Leaf node 7 5:0.50628443917753685 +0 1 21 Tree node 2 1 6 2.5 0 0 0.0069388101312888219 +0 1 21 Tree node -2 7 0 2.5 0 0 0.0029491720913770916 +0 1 21 Tree node 3 -4 1 7.5 0 0 0.0021479877444496222 +0 1 21 Tree node 4 5 6 1.5 0 0 0.0019765621296212871 +0 1 21 Tree node 13 6 4 5.5 0 0 0.0074307863988658639 +0 1 21 Tree node -5 -7 2 1.5 0 0 0.0036270023128129967 +0 1 21 Tree node -6 -8 5 4.5 0 0 0.0016751828933784657 +0 1 21 Tree node 8 12 6 7.5 0 0 0.0014067809030424705 +0 1 21 Tree node 10 9 6 4.5 0 0 0.0033826721589015861 +0 1 21 Tree node -10 17 5 4.5 0 0 0.01777195712396203 +0 1 21 Tree node -3 11 4 4.5 0 0 0.003275468876311376 +0 1 21 Tree node -12 -13 3 4.5 0 0 0.0023470209289690922 +0 1 21 Tree node 16 18 4 6.5 0 0 0.0011948409327821557 +0 1 21 Tree node -1 14 4 1.5 0 0 0.0010090985894475281 +0 1 21 Tree node 15 -16 1 1.5 0 0 0.0017392677205339902 +0 1 21 Tree node -15 -17 5 1.5 0 0 0.0021689776616968955 +0 1 21 Tree node -9 -18 1 6.5 0 0 0.00091296143501088398 +0 1 21 Tree node -11 -19 4 7.5 0 0 0.00071923336888012469 +0 1 21 Tree node -14 -20 4 8.5 0 0 0.000648723378337161 +0 1 21 Leaf node 7 5:-0.47408129558203665 +0 1 21 Leaf node 7 5:0.3083100418138317 +0 1 21 Leaf node 7 5:0.28879649406952024 +0 1 21 Leaf node 7 5:0.5168392261149054 +0 1 21 Leaf node 7 5:-0.25512457465758942 +0 1 21 Leaf node 7 5:0.35724891155727895 +0 1 21 Leaf node 7 5:-0.51451713869565452 +0 1 21 Leaf node 7 5:0.51386552069297731 +0 1 21 Leaf node 7 5:0.46394216046544262 +0 1 21 Leaf node 7 5:-0.43216040444569442 +0 1 21 Leaf node 7 5:0.52760616222085222 +0 1 21 Leaf node 7 5:0.51890685874083997 +0 1 21 Leaf node 7 5:-0.15976147056729298 +0 1 21 Leaf node 7 5:-0.16635197479803401 +0 1 21 Leaf node 7 5:-0.52455234797493688 +0 1 21 Leaf node 7 5:-0.27878159791583668 +0 1 21 Leaf node 7 5:0.15354714181196524 +0 1 21 Leaf node 7 5:0.51580102266603567 +0 1 21 Leaf node 7 5:0.50654285455157988 +0 1 21 Leaf node 7 5:0.50821896102881647 +0 1 22 Tree node 17 1 0 1.5 0 0 0.0052687414186369007 +0 1 22 Tree node -2 2 1 1.5 0 0 0.0018258990479662694 +0 1 22 Tree node 3 16 3 9.5 0 0 0.0014744384789479661 +0 1 22 Tree node 4 5 0 2.5 0 0 0.0017130289754866343 +0 1 22 Tree node -3 -6 4 2.5 0 0 0.010238616020861711 +0 1 22 Tree node 6 9 5 3.5 0 0 0.0022146777703388951 +0 1 22 Tree node -5 7 1 3.5 0 0 0.0035949146801500294 +0 1 22 Tree node 8 -9 7 1.5 0 0 0.0056163082389426317 +0 1 22 Tree node -8 -10 2 2.5 0 0 0.0074018163593124481 +0 1 22 Tree node 12 10 2 2.5 0 0 0.003118735911498696 +0 1 22 Tree node 11 13 2 3.5 0 0 0.0021616099563501216 +0 1 22 Tree node -11 -13 4 6.5 0 0 0.0049134808374007967 +0 1 22 Tree node -7 -14 4 9.5 0 0 0.0020813376768551587 +0 1 22 Tree node -12 14 4 3.5 0 0 0.0016723940098449853 +0 1 22 Tree node 15 -16 4 8.5 0 0 0.0021969154322548554 +0 1 22 Tree node -15 -17 2 5.5 0 0 0.0043953256925163588 +0 1 22 Tree node -4 -18 4 9.5 0 0 0.0013564940780908153 +0 1 22 Tree node -1 18 3 1.5 0 0 0.0010813212187634536 +0 1 22 Tree node -19 -20 4 3.5 0 0 0.0032270490202797944 +0 1 22 Leaf node 7 5:0.16205529544947281 +0 1 22 Leaf node 7 5:-0.53479235958830451 +0 1 22 Leaf node 7 5:-0.26444873436918059 +0 1 22 Leaf node 7 5:0.54970464907922689 +0 1 22 Leaf node 7 5:-0.35766753286471398 +0 1 22 Leaf node 7 5:0.50707089765412183 +0 1 22 Leaf node 7 5:0.51744390310223121 +0 1 22 Leaf node 7 5:-0.38708801420082217 +0 1 22 Leaf node 7 5:0.54071541798978029 +0 1 22 Leaf node 7 5:0.20306863289973004 +0 1 22 Leaf node 7 5:-0.37687368744142735 +0 1 22 Leaf node 7 5:0.55641621184610379 +0 1 22 Leaf node 7 5:0.51662873586584979 +0 1 22 Leaf node 7 5:0.50866166426103343 +0 1 22 Leaf node 7 5:-0.34448983257064592 +0 1 22 Leaf node 7 5:0.51074107848439254 +0 1 22 Leaf node 7 5:0.51049313660240281 +0 1 22 Leaf node 7 5:0.50487528793200742 +0 1 22 Leaf node 7 5:-0.50932883062089152 +0 1 22 Leaf node 7 5:-0.29971840429052798 +0 1 23 Tree node 6 1 6 2.5 0 0 0.004113679285837564 +0 1 23 Tree node -2 2 0 2.5 0 0 0.0020241898755614354 +0 1 23 Tree node -3 3 3 2.5 0 0 0.0012901836215787317 +0 1 23 Tree node 4 15 0 4.5 0 0 0.0021983221443093743 +0 1 23 Tree node -4 5 2 2.5 0 0 0.0071560782520069307 +0 1 23 Tree node -6 -7 1 4.5 0 0 0.012783731763570439 +0 1 23 Tree node 7 11 7 1.5 0 0 0.0012463871251592088 +0 1 23 Tree node 9 8 6 1.5 0 0 0.0021553495587349949 +0 1 23 Tree node -9 -10 1 1.5 0 0 0.0033133718842122094 +0 1 23 Tree node 10 -11 3 5.5 0 0 0.0029836803061305175 +0 1 23 Tree node 12 -12 0 4.5 0 0 0.0011680142303919345 +0 1 23 Tree node -8 -13 1 2.5 0 0 0.00091243056530642666 +0 1 23 Tree node 13 -14 5 4.5 0 0 0.00077803758216762452 +0 1 23 Tree node -1 14 3 1.5 0 0 0.00080294827809346964 +0 1 23 Tree node -15 -16 4 2.5 0 0 0.0024266399197029522 +0 1 23 Tree node -5 16 1 6.5 0 0 0.00060133212870694397 +0 1 23 Tree node 17 18 0 8.5 0 0 0.0012779715408162478 +0 1 23 Tree node -17 -19 2 3.5 0 0 0.0032381997710710896 +0 1 23 Tree node -18 -20 5 4.5 0 0 0.0016289987662522314 +0 1 23 Leaf node 7 5:0.17065597943797647 +0 1 23 Leaf node 7 5:0.2791799358246696 +0 1 23 Leaf node 7 5:0.30067341524840707 +0 1 23 Leaf node 7 5:0.34298940954398821 +0 1 23 Leaf node 7 5:0.51618650898669582 +0 1 23 Leaf node 7 5:-0.50195531598360799 +0 1 23 Leaf node 7 5:0.10629455536309641 +0 1 23 Leaf node 7 5:-0.016375593705214492 +0 1 23 Leaf node 7 5:-0.50319932406158219 +0 1 23 Leaf node 7 5:-0.61965363355150194 +0 1 23 Leaf node 7 5:0.51813298819989928 +0 1 23 Leaf node 7 5:0.51731930611814669 +0 1 23 Leaf node 7 5:0.53953855316560007 +0 1 23 Leaf node 7 5:-0.36153805822304697 +0 1 23 Leaf node 7 5:-0.5032547784346233 +0 1 23 Leaf node 7 5:-0.33036477694558147 +0 1 23 Leaf node 7 5:-0.43954584694241866 +0 1 23 Leaf node 7 5:0.53907595157373933 +0 1 23 Leaf node 7 5:0.50460665476367206 +0 1 23 Leaf node 7 5:0.50430011746931225 +0 1 24 Tree node 1 2 4 3.5 0 0 0.003188355353022935 +0 1 24 Tree node 4 10 2 3.5 0 0 0.0032541714525674229 +0 1 24 Tree node 3 5 0 2.5 0 0 0.0022314549049747131 +0 1 24 Tree node -2 -5 1 1.5 0 0 0.0030393961410826328 +0 1 24 Tree node 13 -6 1 4.5 0 0 0.0013343210662027255 +0 1 24 Tree node 16 6 6 1.5 0 0 0.001126720387579358 +0 1 24 Tree node -7 7 3 2.5 0 0 0.0022487928874878917 +0 1 24 Tree node 8 9 1 4.5 0 0 0.0041605013578677281 +0 1 24 Tree node -8 -10 0 4.5 0 0 0.0097976506867170528 +0 1 24 Tree node -9 11 0 4.5 0 0 0.0013476581058373053 +0 1 24 Tree node -3 -12 6 3.5 0 0 0.00099285936172616596 +0 1 24 Tree node 12 17 3 5.5 0 0 0.00093299432751450502 +0 1 24 Tree node -11 -14 5 4.5 0 0 0.0033094223079058131 +0 1 24 Tree node 14 15 2 1.5 0 0 0.00071422047595863253 +0 1 24 Tree node 18 -16 7 1.5 0 0 0.0011163474007485688 +0 1 24 Tree node -15 -17 4 1.5 0 0 0.00064909861663366348 +0 1 24 Tree node -4 -18 5 4.5 0 0 0.00053048337537873916 +0 1 24 Tree node -13 -19 2 3.5 0 0 0.0003533212177838312 +0 1 24 Tree node -1 -20 6 2.5 0 0 0.00021198241564584792 +0 1 24 Leaf node 7 5:-0.46536336839744757 +0 1 24 Leaf node 7 5:0.017148247322651751 +0 1 24 Leaf node 7 5:0.46754486456057387 +0 1 24 Leaf node 7 5:0.28409503355585525 +0 1 24 Leaf node 7 5:0.36235543885939925 +0 1 24 Leaf node 7 5:-0.31664215207350771 +0 1 24 Leaf node 7 5:0.54638259210413698 +0 1 24 Leaf node 7 5:-0.44663459406170647 +0 1 24 Leaf node 7 5:0.52127916815634356 +0 1 24 Leaf node 7 5:0.51346395717754367 +0 1 24 Leaf node 7 5:-0.30809620052355796 +0 1 24 Leaf node 7 5:0.20274933731184686 +0 1 24 Leaf node 7 5:0.52919434714836877 +0 1 24 Leaf node 7 5:0.50471807118529821 +0 1 24 Leaf node 7 5:-0.52314476573569613 +0 1 24 Leaf node 7 5:0.28323838361007614 +0 1 24 Leaf node 7 5:-0.31661071416976116 +0 1 24 Leaf node 7 5:0.50878331471799831 +0 1 24 Leaf node 7 5:0.50397089332129252 +0 1 24 Leaf node 7 5:-0.27880259908186261 +0 1 25 Tree node 1 3 1 2.5 0 0 0.0023878845101652319 +0 1 25 Tree node 2 -3 4 3.5 0 0 0.0012560117850237413 +0 1 25 Tree node 14 -4 6 2.5 0 0 0.0012943657452632884 +0 1 25 Tree node 4 5 6 2.5 0 0 0.00055965976804946724 +0 1 25 Tree node 6 -6 6 1.5 0 0 0.0020191861875794772 +0 1 25 Tree node -5 8 3 2.5 0 0 0.0019893375673103912 +0 1 25 Tree node 7 -8 3 4.5 0 0 0.0014885069507657512 +0 1 25 Tree node 17 -9 4 7.5 0 0 0.0010020426411569696 +0 1 25 Tree node 9 10 6 3.5 0 0 0.00074911487757524777 +0 1 25 Tree node -7 -11 5 4.5 0 0 0.0022635302181886539 +0 1 25 Tree node 11 13 5 4.5 0 0 0.0017898893293444523 +0 1 25 Tree node -10 12 6 7.5 0 0 0.0088386093809107221 +0 1 25 Tree node -13 -14 7 2.5 0 0 0.00076867504674675916 +0 1 25 Tree node -12 15 0 4.5 0 0 0.00059729560595599271 +0 1 25 Tree node -1 -16 4 2.5 0 0 0.00026305202894017074 +0 1 25 Tree node 16 -17 4 8.5 0 0 0.00010306572289209683 +0 1 25 Tree node -15 -18 0 9.5 0 0 7.6616508318383967E-05 +0 1 25 Tree node 18 -19 2 1.5 0 0 7.4875645116677381E-05 +0 1 25 Tree node -2 -20 5 1.5 0 0 9.0469291179437526E-05 +0 1 25 Leaf node 7 5:-0.50341783595653478 +0 1 25 Leaf node 7 5:-0.52592481941589131 +0 1 25 Leaf node 7 5:0.084240328730647171 +0 1 25 Leaf node 7 5:-0.3424023204399872 +0 1 25 Leaf node 7 5:0.44769847776091687 +0 1 25 Leaf node 7 5:-0.31151269388618613 +0 1 25 Leaf node 7 5:0.36666584426130899 +0 1 25 Leaf node 7 5:0.52512475069147768 +0 1 25 Leaf node 7 5:0.51310552808656351 +0 1 25 Leaf node 7 5:-0.35523583387573804 +0 1 25 Leaf node 7 5:-0.13851212891250447 +0 1 25 Leaf node 7 5:0.27784109139060653 +0 1 25 Leaf node 7 5:0.5256386411005981 +0 1 25 Leaf node 7 5:0.50483201616586859 +0 1 25 Leaf node 7 5:0.50834752414200091 +0 1 25 Leaf node 7 5:-0.51407436777599091 +0 1 25 Leaf node 7 5:0.50224739003905339 +0 1 25 Leaf node 7 5:0.50293782259909836 +0 1 25 Leaf node 7 5:-0.30817835926080606 +0 1 25 Leaf node 7 5:-0.22391334040953703 +0 1 26 Tree node 4 1 0 1.5 0 0 0.00174264980117466 +0 1 26 Tree node -2 2 1 1.5 0 0 0.00067262834202387806 +0 1 26 Tree node 3 6 0 2.5 0 0 0.00064796381527998121 +0 1 26 Tree node -3 -5 4 2.5 0 0 0.0039909254313393568 +0 1 26 Tree node 17 5 3 1.5 0 0 0.00063208525009441469 +0 1 26 Tree node -6 -7 4 3.5 0 0 0.0024148592633293844 +0 1 26 Tree node 8 7 7 3.5 0 0 0.000558775141992736 +0 1 26 Tree node -8 -9 4 4.5 0 0 0.00075347182600321507 +0 1 26 Tree node 9 15 0 9.5 0 0 0.00073227343781693773 +0 1 26 Tree node 10 -11 2 5.5 0 0 0.0012201574746084202 +0 1 26 Tree node 11 -12 2 4.5 0 0 0.0046254489942950984 +0 1 26 Tree node 12 16 4 8.5 0 0 0.0013960215077554047 +0 1 26 Tree node 14 13 0 3.5 0 0 0.0029770977275349123 +0 1 26 Tree node -14 -15 5 4.5 0 0 0.0055401070186584644 +0 1 26 Tree node -4 -16 4 2.5 0 0 0.0033868414112124897 +0 1 26 Tree node -10 -17 5 4.5 0 0 0.00071995919971457479 +0 1 26 Tree node -13 -18 5 4.5 0 0 0.00060952734813890909 +0 1 26 Tree node -1 18 5 1.5 0 0 0.00052650252696052186 +0 1 26 Tree node -19 -20 5 2.5 0 0 0.00070528494252137359 +0 1 26 Leaf node 7 5:-0.5074427200709396 +0 1 26 Leaf node 7 5:-0.52887970936865991 +0 1 26 Leaf node 7 5:-0.1966982431003759 +0 1 26 Leaf node 7 5:-0.29351949956171014 +0 1 26 Leaf node 7 5:0.43226642879883248 +0 1 26 Leaf node 7 5:-0.50748182680137377 +0 1 26 Leaf node 7 5:-0.27756599206625693 +0 1 26 Leaf node 7 5:0.53579615007775094 +0 1 26 Leaf node 7 5:0.51201100701077007 +0 1 26 Leaf node 7 5:0.52324527706577895 +0 1 26 Leaf node 7 5:0.37376592195625885 +0 1 26 Leaf node 7 5:-0.41593283507322687 +0 1 26 Leaf node 7 5:0.52385856570206135 +0 1 26 Leaf node 7 5:-0.32345958580550727 +0 1 26 Leaf node 7 5:0.13224925387151962 +0 1 26 Leaf node 7 5:0.33143772353608847 +0 1 26 Leaf node 7 5:0.50433363589155689 +0 1 26 Leaf node 7 5:0.50850688834149849 +0 1 26 Leaf node 7 5:0.52563148194635179 +0 1 26 Leaf node 7 5:-0.50294849647875639 +0 1 27 Tree node 2 1 6 2.5 0 0 0.0013813732983522951 +0 1 27 Tree node -2 11 0 2.5 0 0 0.0012404598378504216 +0 1 27 Tree node 3 6 7 1.5 0 0 0.00039588894567763988 +0 1 27 Tree node 5 4 6 1.5 0 0 0.00065442109519415852 +0 1 27 Tree node -5 -6 1 1.5 0 0 0.0010757933502994019 +0 1 27 Tree node 7 -7 3 4.5 0 0 0.00095697383889281997 +0 1 27 Tree node -4 -8 0 3.5 0 0 0.00054069580321755443 +0 1 27 Tree node 8 -9 1 4.5 0 0 0.00039448852671083827 +0 1 27 Tree node -1 9 4 1.5 0 0 0.0005452406155066295 +0 1 27 Tree node 10 -11 2 2.5 0 0 0.001730197436140792 +0 1 27 Tree node -10 -12 4 3.5 0 0 0.0010675665705001561 +0 1 27 Tree node -3 12 3 2.5 0 0 0.00035262987011112993 +0 1 27 Tree node 13 15 0 4.5 0 0 0.00088857108533813554 +0 1 27 Tree node -13 14 2 2.5 0 0 0.0033436380786260917 +0 1 27 Tree node -15 -16 1 4.5 0 0 0.0057545822074275797 +0 1 27 Tree node -14 16 1 6.5 0 0 0.0003171732554201319 +0 1 27 Tree node 17 18 0 8.5 0 0 0.00042733300002772181 +0 1 27 Tree node -17 -19 6 7.5 0 0 0.0012385024842074596 +0 1 27 Tree node -18 -20 5 4.5 0 0 0.00054451824432518857 +0 1 27 Leaf node 7 5:-0.50198887916700341 +0 1 27 Leaf node 7 5:0.2724743444458318 +0 1 27 Leaf node 7 5:0.23787921618392135 +0 1 27 Leaf node 7 5:0.26657024180329408 +0 1 27 Leaf node 7 5:-0.50172859659387281 +0 1 27 Leaf node 7 5:-0.48925799542294474 +0 1 27 Leaf node 7 5:0.46964127529880206 +0 1 27 Leaf node 7 5:-0.11773290341774639 +0 1 27 Leaf node 7 5:0.41957018728383971 +0 1 27 Leaf node 7 5:-0.51901970826071531 +0 1 27 Leaf node 7 5:0.13783522754452196 +0 1 27 Leaf node 7 5:-0.3474772616897383 +0 1 27 Leaf node 7 5:0.31854087622699118 +0 1 27 Leaf node 7 5:0.51718052203730203 +0 1 27 Leaf node 7 5:-0.39592756029499171 +0 1 27 Leaf node 7 5:0.084977102011627934 +0 1 27 Leaf node 7 5:-0.42229445434921153 +0 1 27 Leaf node 7 5:0.51999333665664205 +0 1 27 Leaf node 7 5:0.50848530510620948 +0 1 27 Leaf node 7 5:0.50270851577443432 +0 1 28 Tree node 1 3 4 3.5 0 0 0.0010709988206946716 +0 1 28 Tree node 2 10 2 3.5 0 0 0.001293335462693929 +0 1 28 Tree node 11 -4 1 4.5 0 0 0.00099587210672884676 +0 1 28 Tree node 4 6 0 3.5 0 0 0.00092272957205976572 +0 1 28 Tree node 5 -6 4 8.5 0 0 0.00098054427532636942 +0 1 28 Tree node -2 -7 0 1.5 0 0 0.0032479830697777472 +0 1 28 Tree node -5 7 2 1.5 0 0 0.00094093330778075262 +0 1 28 Tree node 17 8 2 3.5 0 0 0.0005822596310954063 +0 1 28 Tree node 9 15 1 4.5 0 0 0.00080172835357945058 +0 1 28 Tree node -9 -11 0 4.5 0 0 0.0013807719535843122 +0 1 28 Tree node -3 -12 6 3.5 0 0 0.00056233131481540086 +0 1 28 Tree node 12 14 2 1.5 0 0 0.0003396203849986533 +0 1 28 Tree node 18 13 3 2.5 0 0 0.00052130607980047853 +0 1 28 Tree node -14 -15 5 2.5 0 0 0.0011857908649767573 +0 1 28 Tree node -13 -16 4 1.5 0 0 0.00037660880231749224 +0 1 28 Tree node -10 16 4 7.5 0 0 0.00031754920797770882 +0 1 28 Tree node -17 -18 4 9.5 0 0 0.00048883451346723613 +0 1 28 Tree node -8 -19 5 6.5 0 0 0.00023628878176680458 +0 1 28 Tree node -1 -20 6 1.5 0 0 0.00013243284626704237 +0 1 28 Leaf node 7 5:-0.50229786607716365 +0 1 28 Leaf node 7 5:-0.03256923346847803 +0 1 28 Leaf node 7 5:0.45987417211701487 +0 1 28 Leaf node 7 5:-0.30530129955458518 +0 1 28 Leaf node 7 5:-0.18267573614911325 +0 1 28 Leaf node 7 5:0.0070500617604225104 +0 1 28 Leaf node 7 5:0.34780705277039442 +0 1 28 Leaf node 7 5:0.51879091623290474 +0 1 28 Leaf node 7 5:-0.38687424018769734 +0 1 28 Leaf node 7 5:0.51268958775689899 +0 1 28 Leaf node 7 5:0.50704577272513396 +0 1 28 Leaf node 7 5:0.12596703135672396 +0 1 28 Leaf node 7 5:-0.5176275898852043 +0 1 28 Leaf node 7 5:-0.50450700531237136 +0 1 28 Leaf node 7 5:0.44259000778841462 +0 1 28 Leaf node 7 5:-0.29315365176824065 +0 1 28 Leaf node 7 5:-0.39628036110317894 +0 1 28 Leaf node 7 5:0.50442826599948964 +0 1 28 Leaf node 7 5:0.5051086527758698 +0 1 28 Leaf node 7 5:-0.51057091514977282 +0 1 29 Tree node 1 4 5 3.5 0 0 0.00086384541964705964 +0 1 29 Tree node 2 -3 2 4.5 0 0 0.0012018691459386645 +0 1 29 Tree node 11 3 4 4.5 0 0 0.0017750045202973006 +0 1 29 Tree node -4 -5 2 2.5 0 0 0.0053453897630884551 +0 1 29 Tree node 5 7 1 3.5 0 0 0.00087806203230262811 +0 1 29 Tree node -2 6 4 2.5 0 0 0.0029027092179605401 +0 1 29 Tree node -7 -8 6 3.5 0 0 0.0012472352868222263 +0 1 29 Tree node 8 9 1 4.5 0 0 0.00052101901334615096 +0 1 29 Tree node -6 -10 6 6.5 0 0 0.00049010396676387553 +0 1 29 Tree node 10 14 5 4.5 0 0 0.00016082209847320997 +0 1 29 Tree node -9 -12 6 7.5 0 0 0.00042083539410256823 +0 1 29 Tree node 12 -13 7 1.5 0 0 0.00014524990940365846 +0 1 29 Tree node 13 15 4 2.5 0 0 0.0004434110614482587 +0 1 29 Tree node 18 -15 6 2.5 0 0 0.00027669436691844589 +0 1 29 Tree node -11 16 0 4.5 0 0 0.0001229639762594219 +0 1 29 Tree node -14 -17 1 1.5 0 0 0.00012209701062280628 +0 1 29 Tree node 17 -18 4 6.5 0 0 4.9729183256088511E-05 +0 1 29 Tree node -16 -19 2 5.5 0 0 8.3391177550794368E-05 +0 1 29 Tree node -1 -20 3 1.5 0 0 2.1245897515842981E-05 +0 1 29 Leaf node 7 5:-0.50217865900960412 +0 1 29 Leaf node 7 5:-0.2899983747612892 +0 1 29 Leaf node 7 5:-0.29183129144145992 +0 1 29 Leaf node 7 5:-0.088390905865797106 +0 1 29 Leaf node 7 5:0.66070089752976691 +0 1 29 Leaf node 7 5:0.032210347522702923 +0 1 29 Leaf node 7 5:0.16005833680427159 +0 1 29 Leaf node 7 5:0.55412125577622107 +0 1 29 Leaf node 7 5:-0.13343317838177407 +0 1 29 Leaf node 7 5:-0.35143517377847872 +0 1 29 Leaf node 7 5:0.5106149667818255 +0 1 29 Leaf node 7 5:0.51154179952155543 +0 1 29 Leaf node 7 5:0.11448787668283851 +0 1 29 Leaf node 7 5:-0.50929524632137324 +0 1 29 Leaf node 7 5:-0.26522554874865645 +0 1 29 Leaf node 7 5:0.51099577392239592 +0 1 29 Leaf node 7 5:-0.22306329064577277 +0 1 29 Leaf node 7 5:0.50214781863597768 +0 1 29 Leaf node 7 5:0.50145833300748199 +0 1 29 Leaf node 7 5:-0.50146751744416773 +0 1 30 Tree node 1 3 4 5.5 0 0 0.00063570402167771535 +0 1 30 Tree node 2 11 0 4.5 0 0 0.00096837587495170153 +0 1 30 Tree node 7 -4 0 3.5 0 0 0.0015177249342108123 +0 1 30 Tree node -2 4 4 7.5 0 0 0.00076354720409310651 +0 1 30 Tree node 6 5 5 3.5 0 0 0.0013248442321059063 +0 1 30 Tree node 10 12 0 4.5 0 0 0.0015140886097549226 +0 1 30 Tree node -5 -8 1 4.5 0 0 0.0012538668664115182 +0 1 30 Tree node 8 -9 5 4.5 0 0 0.00068297928022617474 +0 1 30 Tree node 9 -10 3 3.5 0 0 0.00073084619977319262 +0 1 30 Tree node 13 -11 4 3.5 0 0 0.0021902950862769531 +0 1 30 Tree node -6 -12 6 3.5 0 0 0.00053796492281329381 +0 1 30 Tree node -3 16 0 7.5 0 0 0.0004995593553487011 +0 1 30 Tree node -7 -14 4 8.5 0 0 0.00031484231776541345 +0 1 30 Tree node 14 15 1 2.5 0 0 0.00016131383201122793 +0 1 30 Tree node 18 -16 6 2.5 0 0 0.00057459258885346725 +0 1 30 Tree node -15 -17 5 2.5 0 0 0.00054695208237955908 +0 1 30 Tree node -13 17 0 8.5 0 0 0.0001165144072580875 +0 1 30 Tree node -18 -19 6 8.5 0 0 8.3850941343013039E-05 +0 1 30 Tree node -1 -20 4 1.5 0 0 5.573133157639446E-05 +0 1 30 Leaf node 7 5:-0.5020337915042733 +0 1 30 Leaf node 7 5:0.33131609415250546 +0 1 30 Leaf node 7 5:0.51761103876170844 +0 1 30 Leaf node 7 5:-0.20879853127744549 +0 1 30 Leaf node 7 5:-0.30040215532568737 +0 1 30 Leaf node 7 5:0.51177727395977679 +0 1 30 Leaf node 7 5:-0.35678012963352079 +0 1 30 Leaf node 7 5:0.011519860926091792 +0 1 30 Leaf node 7 5:-0.26286814724418628 +0 1 30 Leaf node 7 5:-0.34972525024679102 +0 1 30 Leaf node 7 5:0.26855813405198059 +0 1 30 Leaf node 7 5:0.53962341847758433 +0 1 30 Leaf node 7 5:-0.099995024042295488 +0 1 30 Leaf node 7 5:0.50282589538715716 +0 1 30 Leaf node 7 5:-0.50671252266215139 +0 1 30 Leaf node 7 5:-0.52434880179322052 +0 1 30 Leaf node 7 5:0.38031781770380951 +0 1 30 Leaf node 7 5:0.5130053097864743 +0 1 30 Leaf node 7 5:0.5018275752592517 +0 1 30 Leaf node 7 5:-0.50992725402047323 +0 1 31 Tree node 1 6 1 2.5 0 0 0.00049364523699367308 +0 1 31 Tree node 4 2 3 1.5 0 0 0.00027718839901712944 +0 1 31 Tree node 3 -4 4 4.5 0 0 0.00059793285866783998 +0 1 31 Tree node 18 -5 6 2.5 0 0 0.00037458538638710339 +0 1 31 Tree node -1 5 5 1.5 0 0 0.00019093478428735397 +0 1 31 Tree node -6 -7 5 2.5 0 0 0.00033359808941728353 +0 1 31 Tree node 8 7 6 8.5 0 0 0.00016708800415198694 +0 1 31 Tree node -8 -9 0 4.5 0 0 0.00059967244780709489 +0 1 31 Tree node 9 15 2 5.5 0 0 0.00042905377517151561 +0 1 31 Tree node 10 -11 2 4.5 0 0 0.0017145020552411195 +0 1 31 Tree node 11 14 7 1.5 0 0 0.0013760846622101512 +0 1 31 Tree node 12 -13 2 3.5 0 0 0.0008825108275374667 +0 1 31 Tree node 13 -14 6 3.5 0 0 0.0015233240721457513 +0 1 31 Tree node 16 -15 3 4.5 0 0 0.00087281684244552252 +0 1 31 Tree node -12 -16 5 3.5 0 0 0.00053728963594973389 +0 1 31 Tree node -10 -17 5 3.5 0 0 0.00042236371603696738 +0 1 31 Tree node 17 -18 4 3.5 0 0 0.00033597877751368453 +0 1 31 Tree node -2 -19 0 2.5 0 0 0.00049825159064973822 +0 1 31 Tree node -3 -20 0 2.5 0 0 0.00015608215527210961 +0 1 31 Leaf node 7 5:-0.50557772834727976 +0 1 31 Leaf node 7 5:-0.50495794023391771 +0 1 31 Leaf node 7 5:-0.50718442261917462 +0 1 31 Leaf node 7 5:-0.13616014510512767 +0 1 31 Leaf node 7 5:-0.27267647037908482 +0 1 31 Leaf node 7 5:0.48299312557136104 +0 1 31 Leaf node 7 5:-0.50199616099417288 +0 1 31 Leaf node 7 5:0.51930904461990557 +0 1 31 Leaf node 7 5:0.50416698368637136 +0 1 31 Leaf node 7 5:0.52197148925971448 +0 1 31 Leaf node 7 5:-0.30617820288084263 +0 1 31 Leaf node 7 5:0.52410922533791271 +0 1 31 Leaf node 7 5:0.24199072166824326 +0 1 31 Leaf node 7 5:-0.29478635975214829 +0 1 31 Leaf node 7 5:0.51794363215686878 +0 1 31 Leaf node 7 5:0.51392380685828476 +0 1 31 Leaf node 7 5:0.50400158667600126 +0 1 31 Leaf node 7 5:0.2318132364896745 +0 1 31 Leaf node 7 5:-0.42430720977765024 +0 1 31 Leaf node 7 5:0.30825446538480328 +0 1 32 Tree node 1 3 0 1.5 0 0 0.00040051713584148029 +0 1 32 Tree node 15 2 3 1.5 0 0 0.00022774139910397594 +0 1 32 Tree node 18 -4 4 3.5 0 0 0.0014247288566795828 +0 1 32 Tree node 5 4 7 3.5 0 0 0.00021250286619202335 +0 1 32 Tree node -5 -6 1 4.5 0 0 0.00060014042753528873 +0 1 32 Tree node 7 6 4 5.5 0 0 0.00020358674355784741 +0 1 32 Tree node -7 10 3 2.5 0 0 0.0010369711257441001 +0 1 32 Tree node 8 12 0 4.5 0 0 0.0005755841895158548 +0 1 32 Tree node 9 -10 0 3.5 0 0 0.001477058357784346 +0 1 32 Tree node 17 -11 4 2.5 0 0 0.00054635676711904636 +0 1 32 Tree node 13 11 2 4.5 0 0 0.00032170407999397463 +0 1 32 Tree node -12 -13 2 5.5 0 0 0.0015037553436215048 +0 1 32 Tree node -9 16 6 6.5 0 0 0.00028897500345708999 +0 1 32 Tree node -8 14 0 4.5 0 0 0.00027802396787524882 +0 1 32 Tree node -15 -16 0 7.5 0 0 0.00030146883080512814 +0 1 32 Tree node -1 -17 5 1.5 0 0 0.00015010647968637556 +0 1 32 Tree node -14 -18 4 3.5 0 0 0.00012147232571323615 +0 1 32 Tree node -2 -19 2 1.5 0 0 9.5517028111960609E-05 +0 1 32 Tree node -3 -20 6 1.5 0 0 7.7743951905674108E-05 +0 1 32 Leaf node 7 5:-0.50456701605804155 +0 1 32 Leaf node 7 5:-0.1994828773846857 +0 1 32 Leaf node 7 5:-0.50124939599324358 +0 1 32 Leaf node 7 5:-0.26772845975895282 +0 1 32 Leaf node 7 5:0.52485911996382806 +0 1 32 Leaf node 7 5:0.50378665623261809 +0 1 32 Leaf node 7 5:0.57732476858811921 +0 1 32 Leaf node 7 5:0.51539195926988313 +0 1 32 Leaf node 7 5:0.51333397116934443 +0 1 32 Leaf node 7 5:-0.2354106365743783 +0 1 32 Leaf node 7 5:0.13363193317818647 +0 1 32 Leaf node 7 5:-0.30716103767263342 +0 1 32 Leaf node 7 5:0.5044806890860043 +0 1 32 Leaf node 7 5:0.50984781370268462 +0 1 32 Leaf node 7 5:-0.2131749663243257 +0 1 32 Leaf node 7 5:0.51369080025238378 +0 1 32 Leaf node 7 5:0.34859165052828878 +0 1 32 Leaf node 7 5:-0.27443371470791506 +0 1 32 Leaf node 7 5:-0.51712962290950182 +0 1 32 Leaf node 7 5:-0.51449447835638218 +0 1 33 Tree node 4 1 5 3.5 0 0 0.00038260125189930506 +0 1 33 Tree node 2 7 1 3.5 0 0 0.00068387016673990123 +0 1 33 Tree node -2 3 4 2.5 0 0 0.0010969961253008713 +0 1 33 Tree node -4 -5 6 3.5 0 0 0.00081791947758433326 +0 1 33 Tree node 5 6 4 7.5 0 0 0.0004675700895557755 +0 1 33 Tree node 8 -7 4 4.5 0 0 0.0006753487348740816 +0 1 33 Tree node -6 -8 1 4.5 0 0 0.00060894201038403089 +0 1 33 Tree node 9 13 1 4.5 0 0 0.00041179325832402393 +0 1 33 Tree node 10 -10 3 4.5 0 0 0.00028645030148681381 +0 1 33 Tree node -3 -11 5 5.5 0 0 0.00025865374294922619 +0 1 33 Tree node 12 11 4 2.5 0 0 9.7231061330985081E-05 +0 1 33 Tree node -12 -13 1 2.5 0 0 0.00048941330071883152 +0 1 33 Tree node 18 -14 6 2.5 0 0 0.00011476469716328617 +0 1 33 Tree node 14 17 0 8.5 0 0 9.4933606777981492E-05 +0 1 33 Tree node 15 16 2 3.5 0 0 0.00022710694493681445 +0 1 33 Tree node -9 -17 2 2.5 0 0 0.00054997038823854309 +0 1 33 Tree node -16 -18 0 4.5 0 0 0.00053320718861395318 +0 1 33 Tree node -15 -19 6 3.5 0 0 0.00015029145305090957 +0 1 33 Tree node -1 -20 3 1.5 0 0 1.8100376740521639E-05 +0 1 33 Leaf node 7 5:-0.50196202209397733 +0 1 33 Leaf node 7 5:-0.17715130284752148 +0 1 33 Leaf node 7 5:-0.021299099439444167 +0 1 33 Leaf node 7 5:0.11650277666481643 +0 1 33 Leaf node 7 5:0.54224672211193847 +0 1 33 Leaf node 7 5:-0.28085894492008762 +0 1 33 Leaf node 7 5:0.16239471671564562 +0 1 33 Leaf node 7 5:-0.033625163435426722 +0 1 33 Leaf node 7 5:0.50825416715871863 +0 1 33 Leaf node 7 5:0.24371443893814759 +0 1 33 Leaf node 7 5:-0.37012737881691282 +0 1 33 Leaf node 7 5:-0.52194676497254822 +0 1 33 Leaf node 7 5:0.095477136984004368 +0 1 33 Leaf node 7 5:-0.50639431723484052 +0 1 33 Leaf node 7 5:-0.38351069559098278 +0 1 33 Leaf node 7 5:0.53434586980978416 +0 1 33 Leaf node 7 5:-0.44545008797413305 +0 1 33 Leaf node 7 5:0.50307117313090322 +0 1 33 Leaf node 7 5:0.50385095211902142 +0 1 33 Leaf node 7 5:-0.50141778380410396 +0 1 34 Tree node 8 1 6 2.5 0 0 0.00028052039432903094 +0 1 34 Tree node -2 2 0 2.5 0 0 0.00054218228681408788 +0 1 34 Tree node -3 3 3 2.5 0 0 0.00020000034230784863 +0 1 34 Tree node 6 4 4 2.5 0 0 0.00034915391872970993 +0 1 34 Tree node 5 7 0 4.5 0 0 0.00050960800560438386 +0 1 34 Tree node -5 -7 1 4.5 0 0 0.00096961117278994804 +0 1 34 Tree node -4 -8 2 3.5 0 0 0.00030188562692881707 +0 1 34 Tree node -6 16 2 2.5 0 0 0.00016538226284503442 +0 1 34 Tree node 14 9 3 1.5 0 0 0.00012023001632879329 +0 1 34 Tree node 11 10 4 4.5 0 0 0.00036488719062987109 +0 1 34 Tree node -11 12 1 2.5 0 0 0.002104471163541254 +0 1 34 Tree node 13 -13 5 3.5 0 0 0.00030969003365800298 +0 1 34 Tree node 18 -14 0 7.5 0 0 0.00017276108977108863 +0 1 34 Tree node -10 -15 4 2.5 0 0 0.0001346996121454622 +0 1 34 Tree node -1 15 5 1.5 0 0 6.3214826487096475E-05 +0 1 34 Tree node -16 -17 5 2.5 0 0 0.00014161130388637002 +0 1 34 Tree node 17 -18 4 7.5 0 0 5.6359618710973006E-05 +0 1 34 Tree node -9 -19 2 5.5 0 0 4.783810180751012E-05 +0 1 34 Tree node -12 -20 3 3.5 0 0 2.3091486780013311E-05 +0 1 34 Leaf node 7 5:-0.50298246623878629 +0 1 34 Leaf node 7 5:0.30738750618985539 +0 1 34 Leaf node 7 5:0.25701215788304538 +0 1 34 Leaf node 7 5:-0.0065893566095374364 +0 1 34 Leaf node 7 5:-0.25847109492327885 +0 1 34 Leaf node 7 5:-0.14234345818248495 +0 1 34 Leaf node 7 5:0.12794124699450396 +0 1 34 Leaf node 7 5:0.53762472935991779 +0 1 34 Leaf node 7 5:0.50833935792412066 +0 1 34 Leaf node 7 5:-0.42393721281086383 +0 1 34 Leaf node 7 5:-0.30670506705477063 +0 1 34 Leaf node 7 5:0.51743508285132289 +0 1 34 Leaf node 7 5:0.3093922685088224 +0 1 34 Leaf node 7 5:-0.29379695894699426 +0 1 34 Leaf node 7 5:-0.24775613528412227 +0 1 34 Leaf node 7 5:0.35449721841488652 +0 1 34 Leaf node 7 5:-0.5014154490360514 +0 1 34 Leaf node 7 5:0.50155427808906972 +0 1 34 Leaf node 7 5:0.5020918201686454 +0 1 34 Leaf node 7 5:0.12767403229952909 +0 1 35 Tree node 1 2 4 5.5 0 0 0.000289916406401744 +0 1 35 Tree node 3 8 7 2.5 0 0 0.00046314301936120872 +0 1 35 Tree node -2 12 3 2.5 0 0 0.00044265692108919671 +0 1 35 Tree node 5 4 4 4.5 0 0 0.00034535618091019438 +0 1 35 Tree node -5 -6 1 4.5 0 0 0.0013840440625592987 +0 1 35 Tree node 6 7 2 3.5 0 0 0.00026669923933521372 +0 1 35 Tree node 9 -8 1 4.5 0 0 0.0010370303460596373 +0 1 35 Tree node -7 -9 3 3.5 0 0 0.000285140871005207 +0 1 35 Tree node -3 -10 0 6.5 0 0 0.0001998572508947767 +0 1 35 Tree node 11 10 4 2.5 0 0 0.00014694470873063541 +0 1 35 Tree node -11 -12 5 2.5 0 0 0.0010450898053320514 +0 1 35 Tree node 18 -13 5 3.5 0 0 0.00042619700930880808 +0 1 35 Tree node 15 13 2 4.5 0 0 0.00013714927336318979 +0 1 35 Tree node -14 14 2 5.5 0 0 0.00083080136911415823 +0 1 35 Tree node -15 -16 0 4.5 0 0 0.00019139690630366822 +0 1 35 Tree node -4 16 1 4.5 0 0 0.0001891230483357448 +0 1 35 Tree node 17 -18 0 7.5 0 0 0.00010870089232687598 +0 1 35 Tree node -17 -19 0 5.5 0 0 0.00030549751449966732 +0 1 35 Tree node -1 -20 3 1.5 0 0 1.9237294682239777E-05 +0 1 35 Leaf node 7 5:-0.50195491223525934 +0 1 35 Leaf node 7 5:0.28723002470661074 +0 1 35 Leaf node 7 5:0.39285849631719505 +0 1 35 Leaf node 7 5:0.51747167902360491 +0 1 35 Leaf node 7 5:-0.23319098423488394 +0 1 35 Leaf node 7 5:0.50939516103007942 +0 1 35 Leaf node 7 5:-0.092962010919365415 +0 1 35 Leaf node 7 5:-0.37842618308124204 +0 1 35 Leaf node 7 5:0.52659627987778657 +0 1 35 Leaf node 7 5:0.50611710588676506 +0 1 35 Leaf node 7 5:-0.51629959084159904 +0 1 35 Leaf node 7 5:0.42583401492031325 +0 1 35 Leaf node 7 5:-0.51895411409725245 +0 1 35 Leaf node 7 5:-0.26965571825162932 +0 1 35 Leaf node 7 5:0.51217225241661857 +0 1 35 Leaf node 7 5:0.50160933850738909 +0 1 35 Leaf node 7 5:0.50443515805401651 +0 1 35 Leaf node 7 5:0.51212775464840754 +0 1 35 Leaf node 7 5:-0.33520160816396294 +0 1 35 Leaf node 7 5:-0.34299854743234121 +0 1 36 Tree node 5 1 4 5.5 0 0 0.0001939296532241513 +0 1 36 Tree node -2 2 4 7.5 0 0 0.00031256493139802629 +0 1 36 Tree node 4 3 5 3.5 0 0 0.0004855625745178719 +0 1 36 Tree node -4 11 0 3.5 0 0 0.00062073981636333482 +0 1 36 Tree node -3 -6 1 4.5 0 0 0.00037854898286151676 +0 1 36 Tree node 6 10 7 2.5 0 0 0.00030480270510444604 +0 1 36 Tree node 7 -8 6 8.5 0 0 0.0003278517612722387 +0 1 36 Tree node 8 -9 6 5.5 0 0 0.0010039039243146665 +0 1 36 Tree node 9 -10 4 4.5 0 0 0.00031524525344263425 +0 1 36 Tree node 13 -11 3 4.5 0 0 0.00037626178412903722 +0 1 36 Tree node -7 -12 0 6.5 0 0 0.0001304704110389305 +0 1 36 Tree node -5 12 0 4.5 0 0 7.8726320854212168E-05 +0 1 36 Tree node -13 -14 5 4.5 0 0 8.6017324702726986E-05 +0 1 36 Tree node 16 14 2 1.5 0 0 6.662465393197906E-05 +0 1 36 Tree node 15 -16 3 2.5 0 0 0.00022492757253278695 +0 1 36 Tree node 17 -17 1 2.5 0 0 0.00016726302832403329 +0 1 36 Tree node 18 -18 4 2.5 0 0 9.1042753948135057E-05 +0 1 36 Tree node -15 -19 0 1.5 0 0 8.4770925332004518E-05 +0 1 36 Tree node -1 -20 6 1.5 0 0 1.1431028543833875E-05 +0 1 36 Leaf node 7 5:-0.50085557721934348 +0 1 36 Leaf node 7 5:0.31735655244027605 +0 1 36 Leaf node 7 5:-0.24514187655692926 +0 1 36 Leaf node 7 5:0.52780319927196218 +0 1 36 Leaf node 7 5:0.50640801417940673 +0 1 36 Leaf node 7 5:-0.016916657813448529 +0 1 36 Leaf node 7 5:0.35291407727182905 +0 1 36 Leaf node 7 5:0.51792218774185184 +0 1 36 Leaf node 7 5:-0.36433633028842755 +0 1 36 Leaf node 7 5:-0.13828059905491502 +0 1 36 Leaf node 7 5:0.48153620614268727 +0 1 36 Leaf node 7 5:0.5050122687972417 +0 1 36 Leaf node 7 5:-0.32339385736523241 +0 1 36 Leaf node 7 5:0.50211456992966941 +0 1 36 Leaf node 7 5:-0.50142540537475966 +0 1 36 Leaf node 7 5:-0.47084921659739432 +0 1 36 Leaf node 7 5:0.23402876830562139 +0 1 36 Leaf node 7 5:0.19054695984747813 +0 1 36 Leaf node 7 5:-0.51682779108628252 +0 1 36 Leaf node 7 5:-0.50440521901667756 +0 1 37 Tree node 1 5 0 1.5 0 0 0.00013159170814251475 +0 1 37 Tree node 3 2 3 1.5 0 0 0.00017710360684575032 +0 1 37 Tree node -3 -4 4 3.5 0 0 0.00093222631447983202 +0 1 37 Tree node -1 4 5 1.5 0 0 9.7323072622346291E-05 +0 1 37 Tree node -5 -6 5 2.5 0 0 0.00014196128567454778 +0 1 37 Tree node 6 7 0 2.5 0 0 8.3691965612788549E-05 +0 1 37 Tree node -2 -8 4 2.5 0 0 0.00057430985305128831 +0 1 37 Tree node 8 14 4 8.5 0 0 8.7241985040034567E-05 +0 1 37 Tree node 11 9 4 4.5 0 0 0.00037630852088953518 +0 1 37 Tree node 10 -11 1 7.5 0 0 0.00064714962581228336 +0 1 37 Tree node -10 -12 6 4.5 0 0 0.00069615246922458487 +0 1 37 Tree node 13 12 6 4.5 0 0 0.00019380795092773651 +0 1 37 Tree node -13 -14 6 7.5 0 0 0.00061413835707436126 +0 1 37 Tree node 18 -15 4 2.5 0 0 0.00052254206657036193 +0 1 37 Tree node 16 15 7 2.5 0 0 0.00015424408974999205 +0 1 37 Tree node -16 -17 5 4.5 0 0 0.00022986853296745054 +0 1 37 Tree node 17 -18 5 4.5 0 0 0.00015295618487322558 +0 1 37 Tree node -9 -19 2 3.5 0 0 0.00013742257131883214 +0 1 37 Tree node -7 -20 3 2.5 0 0 5.7649734867415498E-05 +0 1 37 Leaf node 7 5:-0.50386898566272442 +0 1 37 Leaf node 7 5:-0.11908019225988946 +0 1 37 Leaf node 7 5:-0.50395557579832206 +0 1 37 Leaf node 7 5:-0.29602310754135497 +0 1 37 Leaf node 7 5:0.41831972348024526 +0 1 37 Leaf node 7 5:-0.5004543835491212 +0 1 37 Leaf node 7 5:-0.24071474331391474 +0 1 37 Leaf node 7 5:0.34187786733980591 +0 1 37 Leaf node 7 5:0.51756744002681354 +0 1 37 Leaf node 7 5:-0.42347552733058841 +0 1 37 Leaf node 7 5:0.50891377820003925 +0 1 37 Leaf node 7 5:-0.056655759303368039 +0 1 37 Leaf node 7 5:-0.38462117325935918 +0 1 37 Leaf node 7 5:0.43965302475380152 +0 1 37 Leaf node 7 5:0.39675599602841066 +0 1 37 Leaf node 7 5:-0.47160892359258727 +0 1 37 Leaf node 7 5:0.50476274041635194 +0 1 37 Leaf node 7 5:0.5036166386588119 +0 1 37 Leaf node 7 5:0.21318691037833218 +0 1 37 Leaf node 7 5:0.12045666497879298 +0 1 38 Tree node 8 1 6 2.5 0 0 0.00012613598913324884 +0 1 38 Tree node -2 2 0 2.5 0 0 0.000273567263644968 +0 1 38 Tree node -3 3 3 2.5 0 0 0.0001081790763007431 +0 1 38 Tree node 17 4 4 2.5 0 0 0.00013390906795486389 +0 1 38 Tree node 5 6 1 4.5 0 0 0.00025347110643359529 +0 1 38 Tree node -5 -7 2 6.5 0 0 0.00048301766726057478 +0 1 38 Tree node -6 7 4 4.5 0 0 0.00010478669961989072 +0 1 38 Tree node 15 13 3 4.5 0 0 9.6928871281203738E-05 +0 1 38 Tree node 18 9 3 1.5 0 0 9.5786240267518941E-05 +0 1 38 Tree node 11 10 4 4.5 0 0 0.00018462589735697885 +0 1 38 Tree node -11 16 1 2.5 0 0 0.00097425196989744424 +0 1 38 Tree node 12 -13 5 3.5 0 0 0.00012004095282700601 +0 1 38 Tree node -10 -14 4 2.5 0 0 0.00011618255300821364 +0 1 38 Tree node 14 -15 3 5.5 0 0 9.144518302619075E-05 +0 1 38 Tree node -9 -16 1 7.5 0 0 0.00010305648983642141 +0 1 38 Tree node -8 -17 4 8.5 0 0 7.5870084976836267E-05 +0 1 38 Tree node -12 -18 3 4.5 0 0 6.1921177937430814E-05 +0 1 38 Tree node -4 -19 2 3.5 0 0 5.2790343061181013E-05 +0 1 38 Tree node -1 -20 5 1.5 0 0 4.6313932017696163E-05 +0 1 38 Leaf node 7 5:-0.50333846995523768 +0 1 38 Leaf node 7 5:0.30302250772114486 +0 1 38 Leaf node 7 5:0.25193530826746818 +0 1 38 Leaf node 7 5:0.039088938304104162 +0 1 38 Leaf node 7 5:-0.27995298132412838 +0 1 38 Leaf node 7 5:-0.27926046041320779 +0 1 38 Leaf node 7 5:0.50570888886280885 +0 1 38 Leaf node 7 5:0.50839254158429492 +0 1 38 Leaf node 7 5:-0.36559748291914401 +0 1 38 Leaf node 7 5:-0.4016704731510678 +0 1 38 Leaf node 7 5:-0.27962488961709003 +0 1 38 Leaf node 7 5:0.50967871873667792 +0 1 38 Leaf node 7 5:0.27035669070873003 +0 1 38 Leaf node 7 5:-0.27627857751194579 +0 1 38 Leaf node 7 5:0.50567698465406907 +0 1 38 Leaf node 7 5:0.50056612402131306 +0 1 38 Leaf node 7 5:0.50693079578437439 +0 1 38 Leaf node 7 5:-0.12705903155329795 +0 1 38 Leaf node 7 5:0.51967450520950265 +0 1 38 Leaf node 7 5:0.29845009754386409 +0 1 39 Tree node 2 1 4 5.5 0 0 0.00011139188094297153 +0 1 39 Tree node -2 10 3 2.5 0 0 0.00023956904845042311 +0 1 39 Tree node 3 9 7 2.5 0 0 0.00018684389057642019 +0 1 39 Tree node 4 -5 6 8.5 0 0 0.00017311308110404565 +0 1 39 Tree node 5 -6 6 5.5 0 0 0.00047730483660791105 +0 1 39 Tree node 6 -7 6 3.5 0 0 0.0002268880457125851 +0 1 39 Tree node 7 -8 4 4.5 0 0 0.00027619831791225795 +0 1 39 Tree node 8 -9 3 3.5 0 0 0.00012450852930426346 +0 1 39 Tree node 15 -10 6 2.5 0 0 0.00015892829507431968 +0 1 39 Tree node -4 -11 0 6.5 0 0 9.5150317397848979E-05 +0 1 39 Tree node 12 11 2 4.5 0 0 8.0535755953645019E-05 +0 1 39 Tree node -12 16 2 5.5 0 0 0.00040869137394858226 +0 1 39 Tree node 14 13 1 6.5 0 0 0.00011440743636252542 +0 1 39 Tree node -14 -15 0 7.5 0 0 0.00014313604910948818 +0 1 39 Tree node -3 -16 5 4.5 0 0 0.00011389959839186616 +0 1 39 Tree node 17 -17 4 2.5 0 0 7.2262544583689471E-05 +0 1 39 Tree node -13 -18 0 4.5 0 0 5.8750304304286676E-05 +0 1 39 Tree node -1 18 3 1.5 0 0 1.5323009917522291E-05 +0 1 39 Tree node -19 -20 3 2.5 0 0 8.9871442473558241E-06 +0 1 39 Leaf node 7 5:-0.50174743983055681 +0 1 39 Leaf node 7 5:0.26043736608076101 +0 1 39 Leaf node 7 5:0.51403275593320663 +0 1 39 Leaf node 7 5:0.3463750007834735 +0 1 39 Leaf node 7 5:0.51258542931718587 +0 1 39 Leaf node 7 5:-0.31189355522658735 +0 1 39 Leaf node 7 5:0.33462704330143439 +0 1 39 Leaf node 7 5:-0.15031106402241262 +0 1 39 Leaf node 7 5:0.35982413465367341 +0 1 39 Leaf node 7 5:-0.32865524137183982 +0 1 39 Leaf node 7 5:0.50282721948260423 +0 1 39 Leaf node 7 5:-0.25838410355485975 +0 1 39 Leaf node 7 5:0.50739797635580564 +0 1 39 Leaf node 7 5:-0.34217197741034555 +0 1 39 Leaf node 7 5:0.50711334615494097 +0 1 39 Leaf node 7 5:0.50409827201874247 +0 1 39 Leaf node 7 5:-0.22460073482132586 +0 1 39 Leaf node 7 5:0.50111198421441083 +0 1 39 Leaf node 7 5:-0.32953098643041145 +0 1 39 Leaf node 7 5:-0.51103038796752587 +0 1 40 Tree node 5 1 4 5.5 0 0 7.4403890111841858E-05 +0 1 40 Tree node -2 2 4 7.5 0 0 0.00016181040891769666 +0 1 40 Tree node 4 3 5 3.5 0 0 0.00025771673321861013 +0 1 40 Tree node -4 11 0 3.5 0 0 0.00029247102743030953 +0 1 40 Tree node -3 -6 1 4.5 0 0 0.00019081978302980639 +0 1 40 Tree node 6 10 0 4.5 0 0 0.00015270997498010929 +0 1 40 Tree node 7 -8 0 3.5 0 0 0.00034207876140642511 +0 1 40 Tree node 8 -9 5 4.5 0 0 0.00014440043075625595 +0 1 40 Tree node 9 -10 3 3.5 0 0 0.00017212502026100272 +0 1 40 Tree node 14 -11 4 3.5 0 0 0.00053209905734188131 +0 1 40 Tree node -7 13 0 7.5 0 0 7.7836384902658375E-05 +0 1 40 Tree node -5 12 0 4.5 0 0 3.277654609681453E-05 +0 1 40 Tree node -13 -14 4 8.5 0 0 3.2152527433384768E-05 +0 1 40 Tree node -12 17 4 2.5 0 0 2.5157020125131694E-05 +0 1 40 Tree node 15 16 1 2.5 0 0 2.5027699939542428E-05 +0 1 40 Tree node 18 -17 6 2.5 0 0 9.4128823951335116E-05 +0 1 40 Tree node -16 -18 5 2.5 0 0 7.132156397152404E-05 +0 1 40 Tree node -15 -19 4 4.5 0 0 1.3061329552467537E-05 +0 1 40 Tree node -1 -20 3 1.5 0 0 1.0361448339408382E-05 +0 1 40 Leaf node 7 5:-0.50192618558547553 +0 1 40 Leaf node 7 5:0.27816425343450768 +0 1 40 Leaf node 7 5:-0.2283160602774745 +0 1 40 Leaf node 7 5:0.52051626102471571 +0 1 40 Leaf node 7 5:0.50497732548759233 +0 1 40 Leaf node 7 5:-0.033028627289406622 +0 1 40 Leaf node 7 5:0.50911864885622082 +0 1 40 Leaf node 7 5:-0.19534585858567635 +0 1 40 Leaf node 7 5:-0.25832808760183507 +0 1 40 Leaf node 7 5:-0.3091200958967586 +0 1 40 Leaf node 7 5:0.23895058023149579 +0 1 40 Leaf node 7 5:0.50988680763548866 +0 1 40 Leaf node 7 5:-0.29562095128979682 +0 1 40 Leaf node 7 5:0.50168743676574945 +0 1 40 Leaf node 7 5:-0.14223208456429592 +0 1 40 Leaf node 7 5:-0.50309155027895502 +0 1 40 Leaf node 7 5:-0.51033818791332985 +0 1 40 Leaf node 7 5:0.35591804594417631 +0 1 40 Leaf node 7 5:0.50265957303303921 +0 1 40 Leaf node 7 5:-0.50251632472703955 +0 1 41 Tree node 2 1 6 2.5 0 0 6.2026972925424489E-05 +0 1 41 Tree node -2 7 0 2.5 0 0 0.00014222530865585991 +0 1 41 Tree node 18 3 3 1.5 0 0 8.0028892360045801E-05 +0 1 41 Tree node 5 4 4 4.5 0 0 0.00012706359094449908 +0 1 41 Tree node -5 15 1 2.5 0 0 0.00063415250090609549 +0 1 41 Tree node 6 -7 5 3.5 0 0 9.1175190421632368E-05 +0 1 41 Tree node -4 -8 4 2.5 0 0 8.260988786655947E-05 +0 1 41 Tree node -3 8 3 2.5 0 0 4.6984286716654727E-05 +0 1 41 Tree node 14 9 4 2.5 0 0 8.9192521885064823E-05 +0 1 41 Tree node 10 11 1 4.5 0 0 0.00014584748215034686 +0 1 41 Tree node -10 -12 4 7.5 0 0 0.00027980700442721519 +0 1 41 Tree node 16 12 3 4.5 0 0 5.2873058474622368E-05 +0 1 41 Tree node 13 -14 2 3.5 0 0 6.4930352775133097E-05 +0 1 41 Tree node -13 -15 1 8.5 0 0 0.00021875935203926353 +0 1 41 Tree node -9 -16 5 4.5 0 0 4.1224373793591264E-05 +0 1 41 Tree node -6 -17 0 7.5 0 0 3.8773613464366501E-05 +0 1 41 Tree node 17 -18 1 7.5 0 0 3.6330792767862776E-05 +0 1 41 Tree node -11 -19 4 8.5 0 0 4.9263400323709911E-05 +0 1 41 Tree node -1 -20 5 1.5 0 0 3.6213316697919849E-05 +0 1 41 Leaf node 7 5:-0.50281461520594084 +0 1 41 Leaf node 7 5:0.27904100047040653 +0 1 41 Leaf node 7 5:0.21018833753696833 +0 1 41 Leaf node 7 5:-0.38134216178747943 +0 1 41 Leaf node 7 5:-0.26166474024945419 +0 1 41 Leaf node 7 5:0.1914876737674574 +0 1 41 Leaf node 7 5:0.27683864043569578 +0 1 41 Leaf node 7 5:-0.27713900387599788 +0 1 41 Leaf node 7 5:0.16776581440362082 +0 1 41 Leaf node 7 5:-0.31340751128623978 +0 1 41 Leaf node 7 5:0.50725341192101747 +0 1 41 Leaf node 7 5:0.040712694930568755 +0 1 41 Leaf node 7 5:-0.43111772867789522 +0 1 41 Leaf node 7 5:0.50194058295046495 +0 1 41 Leaf node 7 5:0.50611083834400039 +0 1 41 Leaf node 7 5:0.50278734158213811 +0 1 41 Leaf node 7 5:-0.29643829800359128 +0 1 41 Leaf node 7 5:-0.048456471614651449 +0 1 41 Leaf node 7 5:0.50491374040348092 +0 1 41 Leaf node 7 5:0.32489655023976871 +0 1 42 Tree node 3 1 7 1.5 0 0 6.9233279329553797E-05 +0 1 42 Tree node -2 2 2 1.5 0 0 0.00042259251041876873 +0 1 42 Tree node -3 -4 0 4.5 0 0 0.00012165476439961443 +0 1 42 Tree node 5 4 6 8.5 0 0 7.7196234590056647E-05 +0 1 42 Tree node -5 -6 5 6.5 0 0 8.0129517922005079E-05 +0 1 42 Tree node 6 -7 6 6.5 0 0 7.6286503765055232E-05 +0 1 42 Tree node 7 9 2 2.5 0 0 7.3241778985804395E-05 +0 1 42 Tree node 14 8 4 4.5 0 0 0.00017629741172551291 +0 1 42 Tree node -9 -10 5 3.5 0 0 0.0003496207189683493 +0 1 42 Tree node 10 11 3 2.5 0 0 0.00012490917117140396 +0 1 42 Tree node -8 -12 4 3.5 0 0 0.00045650786546756099 +0 1 42 Tree node 12 13 4 6.5 0 0 7.9011702805356222E-05 +0 1 42 Tree node -11 -14 0 5.5 0 0 0.00030970183047003774 +0 1 42 Tree node 15 -15 0 7.5 0 0 5.6986698461702266E-05 +0 1 42 Tree node 16 -16 6 2.5 0 0 3.9799734801043013E-05 +0 1 42 Tree node -13 -17 6 1.5 0 0 2.2796273949159347E-05 +0 1 42 Tree node 17 18 5 2.5 0 0 1.1469421853825973E-05 +0 1 42 Tree node -1 -19 4 1.5 0 0 5.6051563075830044E-05 +0 1 42 Tree node -18 -20 4 1.5 0 0 5.4641741153433609E-05 +0 1 42 Leaf node 7 5:-0.50076633426078132 +0 1 42 Leaf node 7 5:0.32202638303529046 +0 1 42 Leaf node 7 5:-0.22640293128353747 +0 1 42 Leaf node 7 5:0.50189913940633701 +0 1 42 Leaf node 7 5:0.51162338900010784 +0 1 42 Leaf node 7 5:0.50187913754398139 +0 1 42 Leaf node 7 5:-0.29420960976051652 +0 1 42 Leaf node 7 5:-0.074157674849783903 +0 1 42 Leaf node 7 5:-0.3339627403788894 +0 1 42 Leaf node 7 5:0.035466951021177931 +0 1 42 Leaf node 7 5:-0.2588345674899985 +0 1 42 Leaf node 7 5:0.42680512148199118 +0 1 42 Leaf node 7 5:0.50743947991661487 +0 1 42 Leaf node 7 5:0.5050983406186752 +0 1 42 Leaf node 7 5:-0.29261443375256985 +0 1 42 Leaf node 7 5:-0.24752478934924635 +0 1 42 Leaf node 7 5:0.11708572878506632 +0 1 42 Leaf node 7 5:-0.50088078819497395 +0 1 42 Leaf node 7 5:-0.5108630909553511 +0 1 42 Leaf node 7 5:0.36738821892699558 +0 1 43 Tree node 2 1 6 2.5 0 0 4.8919547274266192E-05 +0 1 43 Tree node -2 9 0 2.5 0 0 8.9772863513630779E-05 +0 1 43 Tree node 4 3 7 1.5 0 0 5.2774099037836966E-05 +0 1 43 Tree node -4 -5 0 3.5 0 0 0.00017170731357991234 +0 1 43 Tree node 16 5 3 1.5 0 0 5.6098193282386207E-05 +0 1 43 Tree node -6 6 4 2.5 0 0 0.00021468858441141768 +0 1 43 Tree node 8 7 0 2.5 0 0 0.00046735680812614119 +0 1 43 Tree node 18 -9 2 4.5 0 0 4.7803833328000371E-05 +0 1 43 Tree node -7 -10 5 1.5 0 0 3.6914403624499471E-05 +0 1 43 Tree node 10 11 6 3.5 0 0 2.8219102781169564E-05 +0 1 43 Tree node -3 -12 5 4.5 0 0 0.00016976443114345165 +0 1 43 Tree node 12 15 6 8.5 0 0 3.6068535200759112E-05 +0 1 43 Tree node 13 14 0 4.5 0 0 9.5711912070914324E-05 +0 1 43 Tree node -11 -15 6 5.5 0 0 0.00033468266164561508 +0 1 43 Tree node -14 -16 6 4.5 0 0 6.5629426656917876E-05 +0 1 43 Tree node -13 -17 0 4.5 0 0 5.1004335712894863E-05 +0 1 43 Tree node 17 -18 5 2.5 0 0 2.1669540222029022E-05 +0 1 43 Tree node -1 -19 5 1.5 0 0 3.3514410361934217E-05 +0 1 43 Tree node -8 -20 1 5.5 0 0 1.6083642211416006E-05 +0 1 43 Leaf node 7 5:-0.50012984079545741 +0 1 43 Leaf node 7 5:0.24866870796896851 +0 1 43 Leaf node 7 5:0.27757524567668818 +0 1 43 Leaf node 7 5:0.35219510497452966 +0 1 43 Leaf node 7 5:-0.21749723963950257 +0 1 43 Leaf node 7 5:-0.25716456454288417 +0 1 43 Leaf node 7 5:-0.51129537734391206 +0 1 43 Leaf node 7 5:0.2548297960250745 +0 1 43 Leaf node 7 5:-0.14348014133528791 +0 1 43 Leaf node 7 5:-0.46973749191605441 +0 1 43 Leaf node 7 5:0.10706833327564304 +0 1 43 Leaf node 7 5:-0.22801245036856596 +0 1 43 Leaf node 7 5:0.50854190274312272 +0 1 43 Leaf node 7 5:-0.30968466860134697 +0 1 43 Leaf node 7 5:-0.43148988759805462 +0 1 43 Leaf node 7 5:0.29727381628279537 +0 1 43 Leaf node 7 5:0.50098847611533637 +0 1 43 Leaf node 7 5:-0.5005757810870044 +0 1 43 Leaf node 7 5:0.31056624279562117 +0 1 43 Leaf node 7 5:0.50066241928899335 +0 1 44 Tree node 2 1 4 5.5 0 0 4.2221198492693976E-05 +0 1 44 Tree node -2 7 3 2.5 0 0 0.00012079817528041039 +0 1 44 Tree node 3 9 7 2.5 0 0 7.6595304825192737E-05 +0 1 44 Tree node 4 -5 3 6.5 0 0 5.7874289248939709E-05 +0 1 44 Tree node 5 -6 6 7.5 0 0 7.4101969461530653E-05 +0 1 44 Tree node 6 -7 4 4.5 0 0 7.9779887755245004E-05 +0 1 44 Tree node 11 -8 3 3.5 0 0 8.1073968772525404E-05 +0 1 44 Tree node 10 8 2 4.5 0 0 4.7155354204277079E-05 +0 1 44 Tree node -9 14 2 5.5 0 0 0.0002366465558855337 +0 1 44 Tree node -4 -11 0 6.5 0 0 4.604137686069997E-05 +0 1 44 Tree node -3 15 1 4.5 0 0 4.5230894839272734E-05 +0 1 44 Tree node 13 12 6 2.5 0 0 4.3907155328503767E-05 +0 1 44 Tree node -13 -14 1 2.5 0 0 4.1638486560682719E-05 +0 1 44 Tree node 18 -15 4 2.5 0 0 3.4221459925399425E-05 +0 1 44 Tree node -10 -16 0 4.5 0 0 2.5555394857315466E-05 +0 1 44 Tree node 17 16 0 6.5 0 0 2.2300328000574113E-05 +0 1 44 Tree node -17 -18 1 8.5 0 0 4.5654784305746197E-05 +0 1 44 Tree node -12 -19 5 4.5 0 0 1.3400002191204575E-05 +0 1 44 Tree node -1 -20 3 1.5 0 0 7.8089159469886119E-06 +0 1 44 Leaf node 7 5:-0.50137513751995189 +0 1 44 Leaf node 7 5:0.25416607678447289 +0 1 44 Leaf node 7 5:0.51137888627348449 +0 1 44 Leaf node 7 5:0.30683891837941463 +0 1 44 Leaf node 7 5:-0.31807204238270625 +0 1 44 Leaf node 7 5:0.41516914109259423 +0 1 44 Leaf node 7 5:-0.113966370508065 +0 1 44 Leaf node 7 5:0.39388655835809327 +0 1 44 Leaf node 7 5:-0.26608377207402195 +0 1 44 Leaf node 7 5:0.50468942410142104 +0 1 44 Leaf node 7 5:0.50133133741790636 +0 1 44 Leaf node 7 5:0.50395032092231462 +0 1 44 Leaf node 7 5:-0.50870577205058187 +0 1 44 Leaf node 7 5:-0.014902485572511812 +0 1 44 Leaf node 7 5:-0.21719680073051584 +0 1 44 Leaf node 7 5:0.5007892714040868 +0 1 44 Leaf node 7 5:-0.32431674720478038 +0 1 44 Leaf node 7 5:0.5051581748324534 +0 1 44 Leaf node 7 5:0.50165456111548401 +0 1 44 Leaf node 7 5:-0.32161162198141668 +0 1 45 Tree node 1 3 0 1.5 0 0 3.4694101135723904E-05 +0 1 45 Tree node 13 2 3 1.5 0 0 4.6035500896245511E-05 +0 1 45 Tree node 15 -4 4 3.5 0 0 0.00024696331246231617 +0 1 45 Tree node 4 7 2 1.5 0 0 3.6933917810825381E-05 +0 1 45 Tree node 5 -6 7 1.5 0 0 0.00026500476757277932 +0 1 45 Tree node 6 -7 6 3.5 0 0 0.00015057148942538155 +0 1 45 Tree node -2 -8 4 2.5 0 0 0.0001569173878019314 +0 1 45 Tree node 8 12 1 4.5 0 0 7.0208418004873502E-05 +0 1 45 Tree node 9 -10 2 6.5 0 0 0.00013364023974343873 +0 1 45 Tree node 11 10 3 2.5 0 0 0.00011028268167952954 +0 1 45 Tree node -11 -12 0 4.5 0 0 0.00025356661489160377 +0 1 45 Tree node -5 -13 6 1.5 0 0 9.8548446429403216E-05 +0 1 45 Tree node -9 16 5 3.5 0 0 3.4993132562945493E-05 +0 1 45 Tree node -1 14 5 1.5 0 0 2.5254202476004698E-05 +0 1 45 Tree node -15 -16 5 2.5 0 0 3.5411816257396354E-05 +0 1 45 Tree node -3 -17 6 1.5 0 0 1.4116620793874895E-05 +0 1 45 Tree node 17 18 5 4.5 0 0 1.3656936877578947E-05 +0 1 45 Tree node -14 -19 4 8.5 0 0 3.7634649472026153E-05 +0 1 45 Tree node -18 -20 3 3.5 0 0 1.8887479695550758E-05 +0 1 45 Leaf node 7 5:-0.50282363545092379 +0 1 45 Leaf node 7 5:-0.50370979274284444 +0 1 45 Leaf node 7 5:-0.50112209279142683 +0 1 45 Leaf node 7 5:-0.28486234956629197 +0 1 45 Leaf node 7 5:-0.51065048202388996 +0 1 45 Leaf node 7 5:0.29669889937459715 +0 1 45 Leaf node 7 5:-0.25124572951572011 +0 1 45 Leaf node 7 5:0.42128258107530897 +0 1 45 Leaf node 7 5:0.50453109443224819 +0 1 45 Leaf node 7 5:0.50510904992461847 +0 1 45 Leaf node 7 5:-0.2955977747690588 +0 1 45 Leaf node 7 5:0.50205318283931888 +0 1 45 Leaf node 7 5:0.26680989807583771 +0 1 45 Leaf node 7 5:-0.15821554547448269 +0 1 45 Leaf node 7 5:0.31715338102032231 +0 1 45 Leaf node 7 5:-0.5001945015213618 +0 1 45 Leaf node 7 5:-0.50733845747796924 +0 1 45 Leaf node 7 5:0.50845890595813048 +0 1 45 Leaf node 7 5:0.50377292430882603 +0 1 45 Leaf node 7 5:0.50140699557530677 +0 1 46 Tree node 1 4 1 2.5 0 0 3.1568315013965149E-05 +0 1 46 Tree node 15 2 3 1.5 0 0 2.8248497447595707E-05 +0 1 46 Tree node 3 -4 4 4.5 0 0 9.3971519913522674E-05 +0 1 46 Tree node -3 -5 6 2.5 0 0 4.0249796775217609E-05 +0 1 46 Tree node 5 8 0 3.5 0 0 1.5310756497915625E-05 +0 1 46 Tree node 7 6 4 2.5 0 0 6.0500832076776892E-05 +0 1 46 Tree node -7 -8 6 4.5 0 0 0.00012238220172940492 +0 1 46 Tree node 18 -9 1 3.5 0 0 2.6738684303037999E-05 +0 1 46 Tree node 9 10 0 4.5 0 0 1.8774187625077764E-05 +0 1 46 Tree node -6 -11 4 7.5 0 0 0.00011928669667657688 +0 1 46 Tree node 11 12 4 3.5 0 0 2.9887010039033664E-05 +0 1 46 Tree node -10 -13 6 8.5 0 0 2.5097201370246887E-05 +0 1 46 Tree node 17 13 1 6.5 0 0 2.0007775906613777E-05 +0 1 46 Tree node 14 -15 5 4.5 0 0 3.7457541195616117E-05 +0 1 46 Tree node -14 -16 1 8.5 0 0 0.00013189464174154339 +0 1 46 Tree node -1 16 5 1.5 0 0 1.4024179239911968E-05 +0 1 46 Tree node -17 -18 5 2.5 0 0 2.5294178089767301E-05 +0 1 46 Tree node -12 -19 4 7.5 0 0 1.1050045678482397E-05 +0 1 46 Tree node -2 -20 0 1.5 0 0 7.2828694631119549E-06 +0 1 46 Leaf node 7 5:-0.50231046678473901 +0 1 46 Leaf node 7 5:-0.50347253365950151 +0 1 46 Leaf node 7 5:-0.25443110549471715 +0 1 46 Leaf node 7 5:-0.12804112842756221 +0 1 46 Leaf node 7 5:-0.2739061384156925 +0 1 46 Leaf node 7 5:-0.17205815161219845 +0 1 46 Leaf node 7 5:0.41550163384212419 +0 1 46 Leaf node 7 5:-0.043223933510909196 +0 1 46 Leaf node 7 5:0.3784380206220288 +0 1 46 Leaf node 7 5:0.50694361599703797 +0 1 46 Leaf node 7 5:0.2033837817148578 +0 1 46 Leaf node 7 5:0.50381691634932535 +0 1 46 Leaf node 7 5:0.50040366552989946 +0 1 46 Leaf node 7 5:-0.38279278452278076 +0 1 46 Leaf node 7 5:0.50120199251246633 +0 1 46 Leaf node 7 5:0.50337510740757574 +0 1 46 Leaf node 7 5:0.26845457853034232 +0 1 46 Leaf node 7 5:-0.50031540606916203 +0 1 46 Leaf node 7 5:0.50212090279481703 +0 1 46 Leaf node 7 5:-0.50591829977178648 +0 1 47 Tree node 1 3 5 3.5 0 0 2.8306934950473175E-05 +0 1 47 Tree node 2 -3 4 9.5 0 0 9.5598675380478745E-05 +0 1 47 Tree node 5 -4 4 5.5 0 0 0.00011910773219949363 +0 1 47 Tree node 4 8 2 1.5 0 0 8.3809587301791682E-05 +0 1 47 Tree node -2 -6 3 2.5 0 0 8.9283978445667768E-05 +0 1 47 Tree node 6 7 2 2.5 0 0 6.136834983222341E-05 +0 1 47 Tree node 17 -8 4 3.5 0 0 0.00016009023001212894 +0 1 47 Tree node 16 -9 4 2.5 0 0 7.6907484710711483E-05 +0 1 47 Tree node 9 10 2 3.5 0 0 4.3625395550501197E-05 +0 1 47 Tree node -5 18 0 4.5 0 0 0.00011605381015356614 +0 1 47 Tree node 11 12 3 3.5 0 0 4.1020096795561906E-05 +0 1 47 Tree node -10 -13 0 4.5 0 0 6.1547717210724213E-05 +0 1 47 Tree node -12 13 4 2.5 0 0 2.5688306198998298E-05 +0 1 47 Tree node 14 15 4 8.5 0 0 3.4295173741760578E-05 +0 1 47 Tree node -14 -16 2 5.5 0 0 8.1939140548690581E-05 +0 1 47 Tree node -15 -17 0 4.5 0 0 1.8692649527805954E-05 +0 1 47 Tree node -7 -18 1 1.5 0 0 8.44099274933372E-06 +0 1 47 Tree node -1 -19 3 1.5 0 0 5.9200126756591386E-06 +0 1 47 Tree node -11 -20 4 7.5 0 0 4.4728733507388432E-06 +0 1 47 Leaf node 7 5:-0.50131423992561075 +0 1 47 Leaf node 7 5:-0.017012707009303987 +0 1 47 Leaf node 7 5:-0.22430361499832221 +0 1 47 Leaf node 7 5:0.26905265648684895 +0 1 47 Leaf node 7 5:-0.30265871252687954 +0 1 47 Leaf node 7 5:0.51357589438965812 +0 1 47 Leaf node 7 5:-0.50236014317352851 +0 1 47 Leaf node 7 5:-0.3216280466376259 +0 1 47 Leaf node 7 5:0.21443730351787296 +0 1 47 Leaf node 7 5:0.51126257282494336 +0 1 47 Leaf node 7 5:0.50452138901179966 +0 1 47 Leaf node 7 5:0.51651316659801971 +0 1 47 Leaf node 7 5:0.5009478792237021 +0 1 47 Leaf node 7 5:-0.34349344218467182 +0 1 47 Leaf node 7 5:0.5046499497307384 +0 1 47 Leaf node 7 5:0.50069248076325656 +0 1 47 Leaf node 7 5:0.50141298530712464 +0 1 47 Leaf node 7 5:0.27314798711370114 +0 1 47 Leaf node 7 5:-0.2092825465225622 +0 1 47 Leaf node 7 5:0.50049029076953755 +0 1 48 Tree node 8 1 6 2.5 0 0 2.5660819516587463E-05 +0 1 48 Tree node -2 2 0 2.5 0 0 7.5522791386649382E-05 +0 1 48 Tree node -3 3 3 2.5 0 0 1.8351119092213465E-05 +0 1 48 Tree node 4 5 2 2.5 0 0 2.9595398700054529E-05 +0 1 48 Tree node -4 6 1 4.5 0 0 0.00014258938752394106 +0 1 48 Tree node -5 7 0 3.5 0 0 0.00011496685255403811 +0 1 48 Tree node -6 -8 6 7.5 0 0 8.9891653429098695E-05 +0 1 48 Tree node -7 16 0 4.5 0 0 2.3590173753226992E-05 +0 1 48 Tree node 14 9 3 1.5 0 0 1.7736287666752172E-05 +0 1 48 Tree node 11 10 4 4.5 0 0 5.4633051570698284E-05 +0 1 48 Tree node -11 13 1 2.5 0 0 0.00021891068298589842 +0 1 48 Tree node 12 -13 5 3.5 0 0 2.7215341072321705E-05 +0 1 48 Tree node -10 -14 4 2.5 0 0 1.8292484763090326E-05 +0 1 48 Tree node -12 -15 3 4.5 0 0 1.4211275343146278E-05 +0 1 48 Tree node -1 15 5 1.5 0 0 7.2545319060572739E-06 +0 1 48 Tree node -16 -17 5 2.5 0 0 1.4405959200250595E-05 +0 1 48 Tree node 17 18 2 5.5 0 0 4.1735551625509644E-06 +0 1 48 Tree node -9 -19 6 7.5 0 0 4.3994630573664427E-06 +0 1 48 Tree node -18 -20 1 4.5 0 0 1.1269753239339285E-06 +0 1 48 Leaf node 7 5:-0.50171222282087902 +0 1 48 Leaf node 7 5:0.28721182070073886 +0 1 48 Leaf node 7 5:0.19538073078348106 +0 1 48 Leaf node 7 5:0.49516007136302492 +0 1 48 Leaf node 7 5:-0.30664548168776246 +0 1 48 Leaf node 7 5:-0.31285173243061254 +0 1 48 Leaf node 7 5:-0.1038557354873844 +0 1 48 Leaf node 7 5:0.50593733462948343 +0 1 48 Leaf node 7 5:0.50203225308677168 +0 1 48 Leaf node 7 5:-0.34997829173828332 +0 1 48 Leaf node 7 5:-0.24098500428103214 +0 1 48 Leaf node 7 5:0.50599743484740944 +0 1 48 Leaf node 7 5:0.27038299021812323 +0 1 48 Leaf node 7 5:-0.20874127734160999 +0 1 48 Leaf node 7 5:-0.14493744585563406 +0 1 48 Leaf node 7 5:0.24016480393216999 +0 1 48 Leaf node 7 5:-0.50025870260726002 +0 1 48 Leaf node 7 5:0.50097370241071604 +0 1 48 Leaf node 7 5:0.50073925555323362 +0 1 48 Leaf node 7 5:0.50028715579321203 +0 1 49 Tree node 2 1 4 5.5 0 0 2.3954833063895036E-05 +0 1 49 Tree node -2 12 3 2.5 0 0 8.6610047625230229E-05 +0 1 49 Tree node 3 11 7 2.5 0 0 3.2394353092530666E-05 +0 1 49 Tree node 4 -5 7 1.5 0 0 3.1366087049824242E-05 +0 1 49 Tree node 5 -6 6 8.5 0 0 2.5927285531251635E-05 +0 1 49 Tree node 6 -7 6 5.5 0 0 0.00010524932146151135 +0 1 49 Tree node 7 -8 6 3.5 0 0 3.4266438491743539E-05 +0 1 49 Tree node 8 -9 5 4.5 0 0 3.1678423024586135E-05 +0 1 49 Tree node 9 -10 4 3.5 0 0 2.901228129933124E-05 +0 1 49 Tree node 10 -11 3 3.5 0 0 3.947924342401721E-05 +0 1 49 Tree node 18 -12 0 2.5 0 0 3.766776622673674E-05 +0 1 49 Tree node -4 -13 0 6.5 0 0 2.3738050636777017E-05 +0 1 49 Tree node 14 13 2 4.5 0 0 1.9503734598001358E-05 +0 1 49 Tree node -14 15 2 5.5 0 0 8.7530953052516623E-05 +0 1 49 Tree node -3 16 0 4.5 0 0 2.8035792385962124E-05 +0 1 49 Tree node -15 -17 0 4.5 0 0 1.1301869946354512E-05 +0 1 49 Tree node 17 -18 0 7.5 0 0 9.3278773463790056E-06 +0 1 49 Tree node -16 -19 3 4.5 0 0 2.3009544282032526E-05 +0 1 49 Tree node -1 -20 3 1.5 0 0 5.4421897836388179E-06 +0 1 49 Leaf node 7 5:-0.50136343879236378 +0 1 49 Leaf node 7 5:0.25895770863314782 +0 1 49 Leaf node 7 5:0.50607800881665366 +0 1 49 Leaf node 7 5:0.31140452815544833 +0 1 49 Leaf node 7 5:-0.28811406930894623 +0 1 49 Leaf node 7 5:0.50612035095133134 +0 1 49 Leaf node 7 5:-0.31790055767177711 +0 1 49 Leaf node 7 5:0.30023080572397892 +0 1 49 Leaf node 7 5:-0.21886512917198192 +0 1 49 Leaf node 7 5:0.097173862273557496 +0 1 49 Leaf node 7 5:0.35641376129769597 +0 1 49 Leaf node 7 5:-0.51282602084754325 +0 1 49 Leaf node 7 5:0.50071424611584758 +0 1 49 Leaf node 7 5:-0.23201368404663253 +0 1 49 Leaf node 7 5:0.50334460718653951 +0 1 49 Leaf node 7 5:0.50311281209237879 +0 1 49 Leaf node 7 5:0.50042356869604587 +0 1 49 Leaf node 7 5:0.50265344131498246 +0 1 49 Leaf node 7 5:-0.35569797397729397 +0 1 49 Leaf node 7 5:-0.50178916913559923 +0 1 50 Tree node 1 4 1 2.5 0 0 1.9702155476891668E-05 +0 1 50 Tree node 2 -3 2 3.5 0 0 2.9690026590648268E-05 +0 1 50 Tree node 3 -4 4 3.5 0 0 3.8353524808221949E-05 +0 1 50 Tree node -1 -5 6 2.5 0 0 1.7726430616083453E-05 +0 1 50 Tree node 6 5 3 6.5 0 0 9.0717210494210901E-06 +0 1 50 Tree node -6 18 0 5.5 0 0 5.5920289443607537E-05 +0 1 50 Tree node 7 14 0 6.5 0 0 2.1873110330084474E-05 +0 1 50 Tree node 9 8 3 4.5 0 0 2.4415560677845212E-05 +0 1 50 Tree node -9 -10 5 4.5 0 0 0.00011033223832356367 +0 1 50 Tree node 16 10 4 2.5 0 0 5.0610711459420387E-05 +0 1 50 Tree node -11 11 4 6.5 0 0 6.7132462875292884E-05 +0 1 50 Tree node 13 12 7 1.5 0 0 7.2075986204855238E-05 +0 1 50 Tree node -13 -14 0 4.5 0 0 4.201702081501812E-05 +0 1 50 Tree node -12 -15 1 4.5 0 0 3.7769922470801694E-05 +0 1 50 Tree node 15 17 6 7.5 0 0 2.1399786161833962E-05 +0 1 50 Tree node -8 -17 5 5.5 0 0 4.4976682286938628E-05 +0 1 50 Tree node -2 -18 0 2.5 0 0 1.791251673402117E-05 +0 1 50 Tree node -16 -19 5 5.5 0 0 8.2940781591653485E-06 +0 1 50 Tree node -7 -20 6 3.5 0 0 6.2523619698619213E-06 +0 1 50 Leaf node 7 5:-0.50150617789453522 +0 1 50 Leaf node 7 5:-0.50268064552183156 +0 1 50 Leaf node 7 5:-0.29598132519204146 +0 1 50 Leaf node 7 5:0.070278462489748617 +0 1 50 Leaf node 7 5:-0.28676173884719758 +0 1 50 Leaf node 7 5:-0.40301542066184842 +0 1 50 Leaf node 7 5:0.50283129852022634 +0 1 50 Leaf node 7 5:-0.40650766173618152 +0 1 50 Leaf node 7 5:0.50821778977827403 +0 1 50 Leaf node 7 5:-0.12383649821846804 +0 1 50 Leaf node 7 5:0.26114879343778052 +0 1 50 Leaf node 7 5:0.50722916881933922 +0 1 50 Leaf node 7 5:-0.4031013153324009 +0 1 50 Leaf node 7 5:0.50065569628534812 +0 1 50 Leaf node 7 5:0.50228048613114062 +0 1 50 Leaf node 7 5:0.50635361319129235 +0 1 50 Leaf node 7 5:0.50080148803015734 +0 1 50 Leaf node 7 5:-0.33423688068700558 +0 1 50 Leaf node 7 5:0.50059830924647697 +0 1 50 Leaf node 7 5:0.50048007817295614 +0 1 51 Tree node 3 1 0 1.5 0 0 1.5831074873926999E-05 +0 1 51 Tree node 2 5 0 2.5 0 0 2.4259776836087173E-05 +0 1 51 Tree node -2 -4 4 2.5 0 0 0.00012339501590106491 +0 1 51 Tree node 16 4 3 1.5 0 0 1.9625192485552219E-05 +0 1 51 Tree node 18 -6 4 3.5 0 0 0.00014601447857466926 +0 1 51 Tree node 6 7 1 2.5 0 0 1.7978377394927988E-05 +0 1 51 Tree node -3 -8 3 2.5 0 0 9.2583259705372961E-05 +0 1 51 Tree node -7 8 5 2.5 0 0 2.8597994525830508E-05 +0 1 51 Tree node -9 9 3 2.5 0 0 2.0233157868301647E-05 +0 1 51 Tree node 10 13 0 4.5 0 0 9.6125943783219943E-06 +0 1 51 Tree node 11 -12 6 8.5 0 0 4.4443873092997451E-05 +0 1 51 Tree node 12 -13 6 4.5 0 0 0.00013503192285495329 +0 1 51 Tree node -10 -14 0 3.5 0 0 2.1181881842649842E-05 +0 1 51 Tree node 15 14 1 6.5 0 0 8.6356981897570648E-06 +0 1 51 Tree node -15 -16 2 2.5 0 0 1.3604804087843683E-05 +0 1 51 Tree node -11 -17 5 3.5 0 0 1.2373604219374155E-05 +0 1 51 Tree node -1 17 5 1.5 0 0 8.3669568357281027E-06 +0 1 51 Tree node -18 -19 5 2.5 0 0 1.2462188285953137E-05 +0 1 51 Tree node -5 -20 6 1.5 0 0 4.5965684612126104E-06 +0 1 51 Leaf node 7 5:-0.50120119475812219 +0 1 51 Leaf node 7 5:-0.085770372569926492 +0 1 51 Leaf node 7 5:-0.51962633480509035 +0 1 51 Leaf node 7 5:0.32728156174939055 +0 1 51 Leaf node 7 5:-0.50071945349178393 +0 1 51 Leaf node 7 5:-0.27034535563790674 +0 1 51 Leaf node 7 5:0.32178049143371473 +0 1 51 Leaf node 7 5:0.20932377475155814 +0 1 51 Leaf node 7 5:0.35744713689586494 +0 1 51 Leaf node 7 5:0.30725220658817176 +0 1 51 Leaf node 7 5:0.50321624342785054 +0 1 51 Leaf node 7 5:0.50341695244328111 +0 1 51 Leaf node 7 5:-0.37091135624107574 +0 1 51 Leaf node 7 5:-0.05963729349544554 +0 1 51 Leaf node 7 5:-0.24857742214362383 +0 1 51 Leaf node 7 5:0.14055412834360526 +0 1 51 Leaf node 7 5:0.5006806917762483 +0 1 51 Leaf node 7 5:0.2760853758758563 +0 1 51 Leaf node 7 5:-0.50005845897974166 +0 1 51 Leaf node 7 5:-0.50426587287966218 +0 1 52 Tree node 2 1 6 2.5 0 0 1.5797248283262285E-05 +0 1 52 Tree node -2 8 0 2.5 0 0 5.1544672464462004E-05 +0 1 52 Tree node 4 3 7 1.5 0 0 1.6349120952964714E-05 +0 1 52 Tree node -4 -5 0 3.5 0 0 5.7336895582328344E-05 +0 1 52 Tree node 15 5 4 2.5 0 0 2.2462432678566701E-05 +0 1 52 Tree node -6 6 2 1.5 0 0 6.3517409138252895E-05 +0 1 52 Tree node -7 7 2 3.5 0 0 7.3048508278560905E-05 +0 1 52 Tree node -8 -9 5 4.5 0 0 5.0730896999124416E-05 +0 1 52 Tree node 10 9 6 4.5 0 0 6.5329631371446177E-06 +0 1 52 Tree node -10 13 0 3.5 0 0 2.8343499108906334E-05 +0 1 52 Tree node 11 12 2 3.5 0 0 1.7121798360851594E-05 +0 1 52 Tree node -3 -13 1 5.5 0 0 3.6749558387128118E-05 +0 1 52 Tree node -12 -14 5 4.5 0 0 3.269075709739949E-05 +0 1 52 Tree node -11 14 6 5.5 0 0 1.0478437303983366E-05 +0 1 52 Tree node -15 17 0 4.5 0 0 1.4043255738432927E-05 +0 1 52 Tree node -1 16 3 1.5 0 0 3.4728759022363661E-06 +0 1 52 Tree node -17 -18 1 3.5 0 0 3.0829212905322019E-06 +0 1 52 Tree node -16 18 3 3.5 0 0 2.7828159783649031E-06 +0 1 52 Tree node -19 -20 5 3.5 0 0 1.2109362595842182E-05 +0 1 52 Leaf node 7 5:-0.50103062337469784 +0 1 52 Leaf node 7 5:0.30670917113612189 +0 1 52 Leaf node 7 5:0.21524357185469659 +0 1 52 Leaf node 7 5:0.33891118902544831 +0 1 52 Leaf node 7 5:-0.22258784481396793 +0 1 52 Leaf node 7 5:-0.31835746244664759 +0 1 52 Leaf node 7 5:0.31465585361801834 +0 1 52 Leaf node 7 5:-0.30733595628042415 +0 1 52 Leaf node 7 5:0.50123610919069683 +0 1 52 Leaf node 7 5:-0.1953189465824198 +0 1 52 Leaf node 7 5:0.50621808497870846 +0 1 52 Leaf node 7 5:0.50680817183677485 +0 1 52 Leaf node 7 5:-0.30771493254834098 +0 1 52 Leaf node 7 5:-0.025122635671511705 +0 1 52 Leaf node 7 5:-0.26261613237766501 +0 1 52 Leaf node 7 5:-0.21610023454030178 +0 1 52 Leaf node 7 5:-0.50218306251250655 +0 1 52 Leaf node 7 5:0.30033483233280112 +0 1 52 Leaf node 7 5:0.50422201112310128 +0 1 52 Leaf node 7 5:0.50053205323886263 +0 1 53 Tree node 1 3 5 3.5 0 0 1.3620788817741634E-05 +0 1 53 Tree node 2 -3 4 9.5 0 0 4.3744880802939672E-05 +0 1 53 Tree node 5 -4 4 5.5 0 0 5.8407858700615041E-05 +0 1 53 Tree node 4 10 2 1.5 0 0 4.0998644660010266E-05 +0 1 53 Tree node -2 -6 3 2.5 0 0 3.0844695732634838E-05 +0 1 53 Tree node 6 7 1 3.5 0 0 2.6131562238295168E-05 +0 1 53 Tree node 8 -8 7 1.5 0 0 3.5804567174606954E-05 +0 1 53 Tree node -7 -9 6 1.5 0 0 2.5276450910753231E-05 +0 1 53 Tree node 14 9 3 2.5 0 0 1.7641631727874709E-05 +0 1 53 Tree node -10 -11 4 1.5 0 0 3.454779030780018E-05 +0 1 53 Tree node 11 12 2 3.5 0 0 1.4621401206475665E-05 +0 1 53 Tree node -5 18 0 4.5 0 0 3.8796785524027619E-05 +0 1 53 Tree node 13 -14 0 4.5 0 0 1.7540696989630343E-05 +0 1 53 Tree node -12 -15 3 3.5 0 0 2.5771730081379729E-05 +0 1 53 Tree node 16 15 2 2.5 0 0 1.0734673695771196E-05 +0 1 53 Tree node -16 -17 5 1.5 0 0 4.3144311329906888E-05 +0 1 53 Tree node 17 -18 4 2.5 0 0 1.2232804077149076E-05 +0 1 53 Tree node -1 -19 6 1.5 0 0 1.8820175778868774E-06 +0 1 53 Tree node -13 -20 4 7.5 0 0 1.8741002581148008E-06 +0 1 53 Leaf node 7 5:-0.50050907055381266 +0 1 53 Leaf node 7 5:0.027208199721733713 +0 1 53 Leaf node 7 5:-0.24719040983180418 +0 1 53 Leaf node 7 5:0.28592293394126506 +0 1 53 Leaf node 7 5:-0.27362638247288479 +0 1 53 Leaf node 7 5:0.50937960555589101 +0 1 53 Leaf node 7 5:-0.11391316584689537 +0 1 53 Leaf node 7 5:-0.40787633644881649 +0 1 53 Leaf node 7 5:0.27961229205512955 +0 1 53 Leaf node 7 5:-0.50053091794715254 +0 1 53 Leaf node 7 5:-0.50028601061764855 +0 1 53 Leaf node 7 5:0.50709812699848622 +0 1 53 Leaf node 7 5:0.50343263892245715 +0 1 53 Leaf node 7 5:0.14095336609228587 +0 1 53 Leaf node 7 5:0.040618549872889806 +0 1 53 Leaf node 7 5:-0.50965010772777297 +0 1 53 Leaf node 7 5:0.33493703485684512 +0 1 53 Leaf node 7 5:-0.50281046598120338 +0 1 53 Leaf node 7 5:-0.50252692884097894 +0 1 53 Leaf node 7 5:0.50024990894610644 +0 1 54 Tree node 3 1 4 5.5 0 0 1.0449564769477031E-05 +0 1 54 Tree node 2 10 0 3.5 0 0 3.4781725747317309E-05 +0 1 54 Tree node -2 -4 5 4.5 0 0 1.8008442985210418E-05 +0 1 54 Tree node 4 13 7 2.5 0 0 1.2888566230966494E-05 +0 1 54 Tree node 5 -6 6 8.5 0 0 1.2756360806495522E-05 +0 1 54 Tree node 6 -7 6 5.5 0 0 3.8581145606157572E-05 +0 1 54 Tree node 7 -8 6 3.5 0 0 2.0656844204362079E-05 +0 1 54 Tree node 8 -9 4 4.5 0 0 3.1380514255623248E-05 +0 1 54 Tree node 9 -10 3 3.5 0 0 1.4621438747110433E-05 +0 1 54 Tree node 12 -11 6 2.5 0 0 2.2900025931054471E-05 +0 1 54 Tree node 11 14 4 8.5 0 0 1.1647635998317103E-05 +0 1 54 Tree node -3 -13 3 5.5 0 0 3.3952258130381222E-05 +0 1 54 Tree node 18 -14 4 2.5 0 0 9.843011031214237E-06 +0 1 54 Tree node -5 -15 0 6.5 0 0 8.7756268829535145E-06 +0 1 54 Tree node -12 15 2 2.5 0 0 6.727350838025471E-06 +0 1 54 Tree node 17 16 3 6.5 0 0 3.2889131747738532E-06 +0 1 54 Tree node -17 -18 0 7.5 0 0 4.3365464222854534E-06 +0 1 54 Tree node -16 -19 0 4.5 0 0 4.2734461189730226E-06 +0 1 54 Tree node -1 -20 3 1.5 0 0 1.6913689291905516E-06 +0 1 54 Leaf node 7 5:-0.50086037201040301 +0 1 54 Leaf node 7 5:0.033843088681397751 +0 1 54 Leaf node 7 5:-0.37448190389649366 +0 1 54 Leaf node 7 5:0.51217522627340795 +0 1 54 Leaf node 7 5:0.27490575485899016 +0 1 54 Leaf node 7 5:0.50299174955362114 +0 1 54 Leaf node 7 5:-0.31133769441091408 +0 1 54 Leaf node 7 5:0.29303199700863053 +0 1 54 Leaf node 7 5:-0.13093316951279399 +0 1 54 Leaf node 7 5:0.32715171399224641 +0 1 54 Leaf node 7 5:-0.3081456904926218 +0 1 54 Leaf node 7 5:0.50395183333856142 +0 1 54 Leaf node 7 5:0.50192566432662944 +0 1 54 Leaf node 7 5:-0.19693493789291011 +0 1 54 Leaf node 7 5:0.5006812797787904 +0 1 54 Leaf node 7 5:0.5025220948639596 +0 1 54 Leaf node 7 5:-0.3633096529803474 +0 1 54 Leaf node 7 5:0.5004484780049423 +0 1 54 Leaf node 7 5:0.50053194221395725 +0 1 54 Leaf node 7 5:-0.26148094746045764 +0 1 55 Tree node 1 3 5 3.5 0 0 7.6754809568079242E-06 +0 1 55 Tree node 2 -3 4 9.5 0 0 3.2260340324571028E-05 +0 1 55 Tree node 5 -4 4 5.5 0 0 3.877093221261113E-05 +0 1 55 Tree node 4 12 2 1.5 0 0 2.1674084834884014E-05 +0 1 55 Tree node -2 -6 3 2.5 0 0 2.2807083018813761E-05 +0 1 55 Tree node 6 7 1 3.5 0 0 1.6135729535572015E-05 +0 1 55 Tree node 8 -8 7 1.5 0 0 2.4612048937528781E-05 +0 1 55 Tree node -7 -9 6 1.5 0 0 1.6950192728962213E-05 +0 1 55 Tree node 10 9 1 1.5 0 0 1.2753481305092247E-05 +0 1 55 Tree node -10 -11 4 1.5 0 0 5.0164319913798383E-05 +0 1 55 Tree node 17 11 2 2.5 0 0 2.3252807483982962E-05 +0 1 55 Tree node -12 -13 5 1.5 0 0 2.8710856121697285E-05 +0 1 55 Tree node 13 14 4 7.5 0 0 7.6492109351287803E-06 +0 1 55 Tree node -5 15 1 4.5 0 0 6.7543445731199861E-05 +0 1 55 Tree node -14 18 0 3.5 0 0 2.0100289969587733E-05 +0 1 55 Tree node -15 16 2 3.5 0 0 1.0059192938347607E-05 +0 1 55 Tree node -17 -18 0 7.5 0 0 1.5607428290254392E-05 +0 1 55 Tree node -1 -19 4 2.5 0 0 7.2519185676168518E-06 +0 1 55 Tree node -16 -20 0 4.5 0 0 4.3936649845212124E-06 +0 1 55 Leaf node 7 5:-0.50042652465240589 +0 1 55 Leaf node 7 5:-0.011396072232286019 +0 1 55 Leaf node 7 5:-0.2307950204425066 +0 1 55 Leaf node 7 5:0.26363849891762331 +0 1 55 Leaf node 7 5:-0.33860478957445045 +0 1 55 Leaf node 7 5:0.50830851427470181 +0 1 55 Leaf node 7 5:-0.11236827921312202 +0 1 55 Leaf node 7 5:-0.38355227901716343 +0 1 55 Leaf node 7 5:0.25376676222053929 +0 1 55 Leaf node 7 5:-0.50181955040462822 +0 1 55 Leaf node 7 5:-0.4224856069939491 +0 1 55 Leaf node 7 5:-0.50003113305723368 +0 1 55 Leaf node 7 5:0.45962968866933651 +0 1 55 Leaf node 7 5:0.5040186292582417 +0 1 55 Leaf node 7 5:-0.15315951127303382 +0 1 55 Leaf node 7 5:0.50200879569139123 +0 1 55 Leaf node 7 5:0.50654593115288815 +0 1 55 Leaf node 7 5:0.50093096217321864 +0 1 55 Leaf node 7 5:-0.5021721382757558 +0 1 55 Leaf node 7 5:-8.7353629973190898E-05 +0 1 56 Tree node 4 1 7 1.5 0 0 6.808810651486715E-06 +0 1 56 Tree node -2 2 2 1.5 0 0 3.6261955728698741E-05 +0 1 56 Tree node 3 15 1 4.5 0 0 9.828350137357733E-06 +0 1 56 Tree node -3 -5 2 5.5 0 0 2.9245198782829602E-05 +0 1 56 Tree node 17 5 3 1.5 0 0 6.3933054881416991E-06 +0 1 56 Tree node 6 9 5 3.5 0 0 7.9474911314295252E-06 +0 1 56 Tree node 13 7 4 2.5 0 0 5.1034947281470278E-05 +0 1 56 Tree node 8 -9 6 2.5 0 0 5.3499118837810493E-05 +0 1 56 Tree node -8 -10 4 4.5 0 0 2.1420286960448481E-05 +0 1 56 Tree node 10 12 3 3.5 0 0 1.420706353845876E-05 +0 1 56 Tree node 11 -12 1 5.5 0 0 2.1457333166906432E-05 +0 1 56 Tree node -7 -13 6 2.5 0 0 3.4851711429227048E-05 +0 1 56 Tree node -11 14 1 4.5 0 0 1.1643222885224496E-05 +0 1 56 Tree node -6 -15 3 4.5 0 0 4.3066364907912546E-06 +0 1 56 Tree node -14 16 0 4.5 0 0 3.8611705313286135E-06 +0 1 56 Tree node -4 -17 5 3.5 0 0 3.6269171053915347E-06 +0 1 56 Tree node -16 -18 5 5.5 0 0 2.3651788282706703E-06 +0 1 56 Tree node 18 -19 5 2.5 0 0 2.2196686899889765E-06 +0 1 56 Tree node -1 -20 5 1.5 0 0 4.8424765773883128E-06 +0 1 56 Leaf node 7 5:-0.50021183390578705 +0 1 56 Leaf node 7 5:0.25926167461688454 +0 1 56 Leaf node 7 5:-0.35642750493633574 +0 1 56 Leaf node 7 5:0.50155534555629011 +0 1 56 Leaf node 7 5:0.5018618622533263 +0 1 56 Leaf node 7 5:-0.50125098218115149 +0 1 56 Leaf node 7 5:0.039765854394361963 +0 1 56 Leaf node 7 5:-0.50475128155427618 +0 1 56 Leaf node 7 5:0.021724709049755483 +0 1 56 Leaf node 7 5:-0.43019068201029159 +0 1 56 Leaf node 7 5:-0.24828354981737374 +0 1 56 Leaf node 7 5:-0.2011863911246814 +0 1 56 Leaf node 7 5:0.44411864082423902 +0 1 56 Leaf node 7 5:0.26270861139285234 +0 1 56 Leaf node 7 5:0.179561628219851 +0 1 56 Leaf node 7 5:-0.25322578607195095 +0 1 56 Leaf node 7 5:0.5005473745089315 +0 1 56 Leaf node 7 5:0.50089744503447675 +0 1 56 Leaf node 7 5:-0.50011786301147398 +0 1 56 Leaf node 7 5:0.27824629649848731 +0 1 57 Tree node 5 1 4 5.5 0 0 5.2788866275212542E-06 +0 1 57 Tree node -2 2 4 7.5 0 0 2.254276326755358E-05 +0 1 57 Tree node 4 3 5 3.5 0 0 2.2660139650341648E-05 +0 1 57 Tree node -4 13 0 3.5 0 0 2.4273552092472707E-05 +0 1 57 Tree node -3 -6 1 4.5 0 0 1.8067063141286299E-05 +0 1 57 Tree node 6 9 0 4.5 0 0 7.7869670286057865E-06 +0 1 57 Tree node 7 -8 0 3.5 0 0 3.7016727116630509E-05 +0 1 57 Tree node 8 -9 5 4.5 0 0 1.9578194196982894E-05 +0 1 57 Tree node 10 -10 4 3.5 0 0 1.8107124783015346E-05 +0 1 57 Tree node -7 15 6 3.5 0 0 8.4725187800980116E-06 +0 1 57 Tree node 12 11 1 2.5 0 0 4.1657203563104591E-06 +0 1 57 Tree node -12 -13 0 2.5 0 0 1.199745740929299E-05 +0 1 57 Tree node 17 -14 6 2.5 0 0 8.588989418962142E-06 +0 1 57 Tree node -5 14 0 4.5 0 0 2.807224796115264E-06 +0 1 57 Tree node -15 -16 4 8.5 0 0 2.1017135158638742E-06 +0 1 57 Tree node -11 16 0 8.5 0 0 1.7621937353959486E-06 +0 1 57 Tree node -17 -18 6 8.5 0 0 2.3146389233158184E-06 +0 1 57 Tree node 18 -19 4 2.5 0 0 1.7324297580953139E-06 +0 1 57 Tree node -1 -20 3 1.5 0 0 9.9406691418534794E-07 +0 1 57 Leaf node 7 5:-0.50098768757121104 +0 1 57 Leaf node 7 5:0.33592529506356011 +0 1 57 Leaf node 7 5:-0.22632914191720077 +0 1 57 Leaf node 7 5:0.50759238888470437 +0 1 57 Leaf node 7 5:0.50151496860780098 +0 1 57 Leaf node 7 5:-0.035027868231243001 +0 1 57 Leaf node 7 5:0.50301835895377822 +0 1 57 Leaf node 7 5:-0.19300564555460212 +0 1 57 Leaf node 7 5:-0.30453402858704648 +0 1 57 Leaf node 7 5:0.13434541161335922 +0 1 57 Leaf node 7 5:-0.10704584933057056 +0 1 57 Leaf node 7 5:-0.50150217454786838 +0 1 57 Leaf node 7 5:0.33686652765920067 +0 1 57 Leaf node 7 5:-0.50403025322642947 +0 1 57 Leaf node 7 5:-0.33903619297780035 +0 1 57 Leaf node 7 5:0.50057280647628455 +0 1 57 Leaf node 7 5:0.50308935189056969 +0 1 57 Leaf node 7 5:0.50014968883336275 +0 1 57 Leaf node 7 5:-0.501652513382429 +0 1 57 Leaf node 7 5:-0.50116489872016734 +0 1 58 Tree node 4 1 7 1.5 0 0 3.7981976401604232E-06 +0 1 58 Tree node -2 2 2 1.5 0 0 2.1969398732093908E-05 +0 1 58 Tree node 3 18 1 4.5 0 0 6.5765359168172444E-06 +0 1 58 Tree node -3 -5 2 5.5 0 0 1.7041150830876382E-05 +0 1 58 Tree node -1 5 3 1.5 0 0 4.3677256315809393E-06 +0 1 58 Tree node 6 16 1 9.5 0 0 4.5729533546500386E-06 +0 1 58 Tree node 7 -8 3 6.5 0 0 8.1962628777227458E-06 +0 1 58 Tree node 8 13 1 6.5 0 0 1.1020589751883477E-05 +0 1 58 Tree node 10 9 6 3.5 0 0 1.5466017639175555E-05 +0 1 58 Tree node -10 17 6 5.5 0 0 1.4760622070941788E-05 +0 1 58 Tree node 14 11 4 4.5 0 0 1.2011147344679413E-05 +0 1 58 Tree node -12 12 1 3.5 0 0 7.0100951387939333E-05 +0 1 58 Tree node -13 -14 3 3.5 0 0 1.2536126448259707E-05 +0 1 58 Tree node -9 -15 2 5.5 0 0 1.117888840745567E-05 +0 1 58 Tree node 15 -16 5 3.5 0 0 9.0045713877873542E-06 +0 1 58 Tree node -6 -17 4 2.5 0 0 6.775898242710055E-06 +0 1 58 Tree node -7 -18 2 5.5 0 0 4.3086621638734036E-06 +0 1 58 Tree node -11 -19 6 6.5 0 0 3.5377216328350827E-06 +0 1 58 Tree node -4 -20 5 3.5 0 0 2.3063393640591503E-06 +0 1 58 Leaf node 7 5:0.24648329014839784 +0 1 58 Leaf node 7 5:0.22536644222781457 +0 1 58 Leaf node 7 5:-0.32853330183952217 +0 1 58 Leaf node 7 5:0.50127370948604477 +0 1 58 Leaf node 7 5:0.50123966408606768 +0 1 58 Leaf node 7 5:-0.23978583232362458 +0 1 58 Leaf node 7 5:0.50236009470755782 +0 1 58 Leaf node 7 5:-0.32791546800138699 +0 1 58 Leaf node 7 5:-0.39132626968375056 +0 1 58 Leaf node 7 5:0.34273575797162731 +0 1 58 Leaf node 7 5:0.44076279685284786 +0 1 58 Leaf node 7 5:-0.42820985413122759 +0 1 58 Leaf node 7 5:0.50367469039503554 +0 1 58 Leaf node 7 5:-0.20668188935479601 +0 1 58 Leaf node 7 5:0.50032495723932513 +0 1 58 Leaf node 7 5:0.16281445249264614 +0 1 58 Leaf node 7 5:-0.2576233675648677 +0 1 58 Leaf node 7 5:0.50016580339326777 +0 1 58 Leaf node 7 5:-0.054791853141499584 +0 1 58 Leaf node 7 5:0.50043724917574761 +0 1 59 Tree node 1 3 1 2.5 0 0 3.7722240623060257E-06 +0 1 59 Tree node 2 -3 2 3.5 0 0 7.3034442374982024E-06 +0 1 59 Tree node 14 -4 4 3.5 0 0 6.4047092985112703E-06 +0 1 59 Tree node 4 13 0 6.5 0 0 3.1239490559937154E-06 +0 1 59 Tree node 5 -6 3 6.5 0 0 8.3506457010260012E-06 +0 1 59 Tree node 7 6 3 4.5 0 0 6.380966095088575E-06 +0 1 59 Tree node -7 -8 5 4.5 0 0 3.0882401994534679E-05 +0 1 59 Tree node 10 8 4 2.5 0 0 1.239655961641342E-05 +0 1 59 Tree node -9 9 4 4.5 0 0 1.8683134258034524E-05 +0 1 59 Tree node 12 11 7 1.5 0 0 1.0838905144538606E-05 +0 1 59 Tree node -2 -12 0 2.5 0 0 5.4504220689978247E-06 +0 1 59 Tree node -11 -13 0 4.5 0 0 5.0754368545744609E-06 +0 1 59 Tree node 15 -14 3 3.5 0 0 5.0550929159781968E-06 +0 1 59 Tree node -5 18 5 3.5 0 0 4.4681167093267396E-06 +0 1 59 Tree node 16 -16 6 2.5 0 0 2.2517462417309237E-06 +0 1 59 Tree node -10 -17 3 2.5 0 0 1.086813585523651E-06 +0 1 59 Tree node 17 -18 4 2.5 0 0 8.9930256928481664E-07 +0 1 59 Tree node -1 -19 3 1.5 0 0 8.272580663460004E-07 +0 1 59 Tree node -15 -20 5 4.5 0 0 7.392216722579011E-07 +0 1 59 Leaf node 7 5:-0.50086856937194368 +0 1 59 Leaf node 7 5:-0.50110323055792028 +0 1 59 Leaf node 7 5:-0.32307017235104624 +0 1 59 Leaf node 7 5:0.056004014426473674 +0 1 59 Leaf node 7 5:-0.16997592784542745 +0 1 59 Leaf node 7 5:-0.25858608631154084 +0 1 59 Leaf node 7 5:0.50404459366870558 +0 1 59 Leaf node 7 5:-0.13334456765897706 +0 1 59 Leaf node 7 5:0.30067690405177516 +0 1 59 Leaf node 7 5:0.5019913514036014 +0 1 59 Leaf node 7 5:-0.2940394334580273 +0 1 59 Leaf node 7 5:-0.30917562511622754 +0 1 59 Leaf node 7 5:0.50038463622056839 +0 1 59 Leaf node 7 5:0.0077974372764916195 +0 1 59 Leaf node 7 5:-0.18754482076619497 +0 1 59 Leaf node 7 5:-0.24307311887372199 +0 1 59 Leaf node 7 5:0.50307099945600453 +0 1 59 Leaf node 7 5:-0.50121704978622839 +0 1 59 Leaf node 7 5:-0.50094267755459287 +0 1 59 Leaf node 7 5:0.50057026607499 +0 1 60 Tree node 5 1 4 5.5 0 0 3.0866008460840525E-06 +0 1 60 Tree node -2 2 4 7.5 0 0 1.2734346097634908E-05 +0 1 60 Tree node 4 3 5 3.5 0 0 1.0702103080735938E-05 +0 1 60 Tree node -4 14 0 3.5 0 0 1.0573603776241525E-05 +0 1 60 Tree node -3 -6 6 4.5 0 0 9.4933819955333483E-06 +0 1 60 Tree node 6 13 7 2.5 0 0 5.5389314546763715E-06 +0 1 60 Tree node 7 -8 7 1.5 0 0 7.1349189017398671E-06 +0 1 60 Tree node 9 8 6 4.5 0 0 4.8185729864086527E-06 +0 1 60 Tree node -9 -10 6 8.5 0 0 1.6176618919158473E-05 +0 1 60 Tree node 10 -11 5 4.5 0 0 6.7431831454753814E-06 +0 1 60 Tree node 11 -12 4 3.5 0 0 1.7299319712463309E-05 +0 1 60 Tree node 12 -13 2 3.5 0 0 8.9550161668036473E-06 +0 1 60 Tree node 16 -14 1 3.5 0 0 1.1674927554651676E-05 +0 1 60 Tree node -7 -15 0 6.5 0 0 4.5443060223474172E-06 +0 1 60 Tree node -5 15 0 4.5 0 0 1.7050362143347379E-06 +0 1 60 Tree node -16 -17 4 8.5 0 0 8.8352689240388461E-07 +0 1 60 Tree node -1 17 3 1.5 0 0 6.2851834506093899E-07 +0 1 60 Tree node 18 -19 4 2.5 0 0 2.5320069756584962E-07 +0 1 60 Tree node -18 -20 6 1.5 0 0 4.9096293430389229E-07 +0 1 60 Leaf node 7 5:-0.50069908987915024 +0 1 60 Leaf node 7 5:0.31342256241015881 +0 1 60 Leaf node 7 5:-0.21704848843440225 +0 1 60 Leaf node 7 5:0.50527930842410518 +0 1 60 Leaf node 7 5:0.50119362873845186 +0 1 60 Leaf node 7 5:0.03908386893466307 +0 1 60 Leaf node 7 5:0.2905736403548792 +0 1 60 Leaf node 7 5:-0.27107388409947952 +0 1 60 Leaf node 7 5:-0.27690946240191289 +0 1 60 Leaf node 7 5:0.50206913333719172 +0 1 60 Leaf node 7 5:-0.17950520849029644 +0 1 60 Leaf node 7 5:0.16749834766575164 +0 1 60 Leaf node 7 5:0.48778713000847557 +0 1 60 Leaf node 7 5:-0.50574985096833469 +0 1 60 Leaf node 7 5:0.50036656983030836 +0 1 60 Leaf node 7 5:-0.29160232912289225 +0 1 60 Leaf node 7 5:0.50047586560851209 +0 1 60 Leaf node 7 5:-0.5000435070944399 +0 1 60 Leaf node 7 5:0.11485461891876052 +0 1 60 Leaf node 7 5:-0.50117770727779531 +0 1 61 Tree node 1 7 1 2.5 0 0 2.9149336361067739E-06 +0 1 61 Tree node 2 -3 2 3.5 0 0 5.5674675125710497E-06 +0 1 61 Tree node 3 4 2 2.5 0 0 5.5846658431529207E-06 +0 1 61 Tree node 5 -5 4 4.5 0 0 2.5107533203962156E-05 +0 1 61 Tree node -4 -6 5 1.5 0 0 5.8984791979070776E-06 +0 1 61 Tree node -1 6 3 2.5 0 0 2.1116717195839145E-06 +0 1 61 Tree node -7 -8 5 1.5 0 0 4.95646674579786E-06 +0 1 61 Tree node 8 10 2 4.5 0 0 1.8643625939414287E-06 +0 1 61 Tree node 12 9 7 1.5 0 0 9.56794976887578E-06 +0 1 61 Tree node -10 -11 0 4.5 0 0 1.1659462604234572E-05 +0 1 61 Tree node 11 18 2 5.5 0 0 7.4814198468633276E-06 +0 1 61 Tree node -9 -13 5 6.5 0 0 6.2904465949455914E-06 +0 1 61 Tree node 13 -14 0 8.5 0 0 5.7022325756254181E-06 +0 1 61 Tree node 14 -15 1 6.5 0 0 9.6724880708705994E-06 +0 1 61 Tree node -2 15 4 1.5 0 0 4.9342494175238078E-06 +0 1 61 Tree node -16 16 5 3.5 0 0 5.449971094960553E-06 +0 1 61 Tree node -17 17 3 3.5 0 0 1.0382324594368633E-05 +0 1 61 Tree node -18 -19 4 9.5 0 0 6.4530203153677139E-06 +0 1 61 Tree node -12 -20 5 3.5 0 0 3.9998383011244877E-06 +0 1 61 Leaf node 7 5:-0.50098998577637077 +0 1 61 Leaf node 7 5:-0.27903485610463885 +0 1 61 Leaf node 7 5:-0.31530825523718947 +0 1 61 Leaf node 7 5:-0.50001238709015827 +0 1 61 Leaf node 7 5:-0.15785854281399986 +0 1 61 Leaf node 7 5:0.27834067881025781 +0 1 61 Leaf node 7 5:-0.50115369736560422 +0 1 61 Leaf node 7 5:0.40557124085651941 +0 1 61 Leaf node 7 5:-0.29656703925821742 +0 1 61 Leaf node 7 5:0.50269493037186141 +0 1 61 Leaf node 7 5:0.5011021287522659 +0 1 61 Leaf node 7 5:0.50165448749002761 +0 1 61 Leaf node 7 5:-0.048466277998377001 +0 1 61 Leaf node 7 5:0.50235662837554951 +0 1 61 Leaf node 7 5:-0.401129823055898 +0 1 61 Leaf node 7 5:0.21126587658613796 +0 1 61 Leaf node 7 5:0.35180608186720314 +0 1 61 Leaf node 7 5:-0.2406521478878231 +0 1 61 Leaf node 7 5:0.50078308765073265 +0 1 61 Leaf node 7 5:0.50022649397252728 +0 1 62 Tree node 5 1 4 5.5 0 0 2.5242325862350476E-06 +0 1 62 Tree node -2 2 4 7.5 0 0 9.4009163268480225E-06 +0 1 62 Tree node 4 3 5 3.5 0 0 6.8415432386400714E-06 +0 1 62 Tree node -4 15 0 3.5 0 0 8.2103412251338091E-06 +0 1 62 Tree node -3 -6 1 4.5 0 0 6.8238380505403828E-06 +0 1 62 Tree node 6 10 0 4.5 0 0 3.8990997365372336E-06 +0 1 62 Tree node 7 -8 0 3.5 0 0 1.9440062498048654E-05 +0 1 62 Tree node 8 -9 5 4.5 0 0 5.8934326740464927E-06 +0 1 62 Tree node 9 -10 3 3.5 0 0 5.2022414948210941E-06 +0 1 62 Tree node 12 -11 4 3.5 0 0 1.7467844291630943E-05 +0 1 62 Tree node -7 11 0 7.5 0 0 3.2027141288841485E-06 +0 1 62 Tree node -12 16 0 8.5 0 0 1.1058261008069783E-06 +0 1 62 Tree node 13 14 1 2.5 0 0 1.0535106415463863E-06 +0 1 62 Tree node 18 -15 6 2.5 0 0 3.0900839067466766E-06 +0 1 62 Tree node -14 -16 5 2.5 0 0 2.7277418385174741E-06 +0 1 62 Tree node -5 17 0 4.5 0 0 1.0268929546846594E-06 +0 1 62 Tree node -13 -18 6 8.5 0 0 8.5845156586463797E-07 +0 1 62 Tree node -17 -19 5 4.5 0 0 5.6109695458361351E-07 +0 1 62 Tree node -1 -20 3 1.5 0 0 3.2546534169687651E-07 +0 1 62 Leaf node 7 5:-0.50069375312802289 +0 1 62 Leaf node 7 5:0.28185051764512559 +0 1 62 Leaf node 7 5:-0.18862814324366894 +0 1 62 Leaf node 7 5:0.50461932830792322 +0 1 62 Leaf node 7 5:0.50101709241053005 +0 1 62 Leaf node 7 5:0.0004950304929622108 +0 1 62 Leaf node 7 5:0.50228765259603203 +0 1 62 Leaf node 7 5:-0.18989098696956255 +0 1 62 Leaf node 7 5:-0.22183837337847689 +0 1 62 Leaf node 7 5:-0.2988666872298994 +0 1 62 Leaf node 7 5:0.19285341841615258 +0 1 62 Leaf node 7 5:-0.19033080172909217 +0 1 62 Leaf node 7 5:0.5019086312839014 +0 1 62 Leaf node 7 5:-0.50078247264507036 +0 1 62 Leaf node 7 5:-0.50268343442653274 +0 1 62 Leaf node 7 5:0.35346361989070163 +0 1 62 Leaf node 7 5:-0.276376624920944 +0 1 62 Leaf node 7 5:0.50009591319669611 +0 1 62 Leaf node 7 5:0.50036647217045205 +0 1 62 Leaf node 7 5:-0.5009833066921715 +0 1 63 Tree node 3 1 0 1.5 0 0 2.0432969704582747E-06 +0 1 63 Tree node 2 5 0 2.5 0 0 3.7108667725077336E-06 +0 1 63 Tree node -2 -4 4 2.5 0 0 1.5929728767456032E-05 +0 1 63 Tree node 15 4 3 1.5 0 0 3.3658056778695058E-06 +0 1 63 Tree node -5 -6 4 3.5 0 0 2.5948878725516592E-05 +0 1 63 Tree node 6 7 1 2.5 0 0 2.4674298761335816E-06 +0 1 63 Tree node -3 -8 3 2.5 0 0 1.5612158733365691E-05 +0 1 63 Tree node -7 8 5 2.5 0 0 3.2719383395316858E-06 +0 1 63 Tree node -9 9 3 2.5 0 0 3.1505712201302192E-06 +0 1 63 Tree node 10 11 6 3.5 0 0 2.3172559299226132E-06 +0 1 63 Tree node -10 13 5 3.5 0 0 6.8090041205281539E-06 +0 1 63 Tree node 12 17 5 4.5 0 0 4.8834169295247763E-06 +0 1 63 Tree node -11 14 6 7.5 0 0 2.4028192851052152E-05 +0 1 63 Tree node -12 -15 4 8.5 0 0 3.467989957647923E-06 +0 1 63 Tree node -14 -16 7 2.5 0 0 1.6613763049398458E-06 +0 1 63 Tree node -1 16 5 1.5 0 0 1.2715205044633866E-06 +0 1 63 Tree node -17 -18 5 2.5 0 0 2.0313191635624418E-06 +0 1 63 Tree node -13 18 3 3.5 0 0 1.1572573023600365E-06 +0 1 63 Tree node -19 -20 1 4.5 0 0 8.0938643576822898E-07 +0 1 63 Leaf node 7 5:-0.50037676666652864 +0 1 63 Leaf node 7 5:0.082274506532981379 +0 1 63 Leaf node 7 5:-0.51007288301019249 +0 1 63 Leaf node 7 5:0.28826965418420741 +0 1 63 Leaf node 7 5:-0.50122061993266964 +0 1 63 Leaf node 7 5:-0.27608324218620234 +0 1 63 Leaf node 7 5:0.23462751068207152 +0 1 63 Leaf node 7 5:0.21684357620322328 +0 1 63 Leaf node 7 5:0.33737672660418971 +0 1 63 Leaf node 7 5:0.29864731278308043 +0 1 63 Leaf node 7 5:-0.34635585333335067 +0 1 63 Leaf node 7 5:-0.1424565358298088 +0 1 63 Leaf node 7 5:0.5017303104694868 +0 1 63 Leaf node 7 5:0.50128753350963606 +0 1 63 Leaf node 7 5:0.50110905347140799 +0 1 63 Leaf node 7 5:0.50009930033734384 +0 1 63 Leaf node 7 5:0.30730229461666186 +0 1 63 Leaf node 7 5:-0.50002073849288764 +0 1 63 Leaf node 7 5:-0.22500357277871716 +0 1 63 Leaf node 7 5:0.50028557771999072 +0 1 64 Tree node 2 1 6 2.5 0 0 2.3317744748222354E-06 +0 1 64 Tree node -2 7 0 2.5 0 0 7.8982775747526316E-06 +0 1 64 Tree node 17 3 3 1.5 0 0 1.9557628984878756E-06 +0 1 64 Tree node 9 4 4 3.5 0 0 5.9982482263352334E-06 +0 1 64 Tree node -5 5 0 1.5 0 0 4.9810979834783045E-05 +0 1 64 Tree node -6 6 2 3.5 0 0 1.0450581380810976E-05 +0 1 64 Tree node -7 -8 5 5.5 0 0 3.6116109723653595E-06 +0 1 64 Tree node 8 11 6 3.5 0 0 1.3127972963088118E-06 +0 1 64 Tree node -3 -10 5 4.5 0 0 4.6039809725909405E-06 +0 1 64 Tree node 10 -11 2 3.5 0 0 1.2538806986799369E-06 +0 1 64 Tree node -4 -12 0 2.5 0 0 1.2208893780871142E-06 +0 1 64 Tree node 13 12 2 4.5 0 0 1.2068873355177789E-06 +0 1 64 Tree node -13 -14 0 4.5 0 0 8.5764592547968731E-06 +0 1 64 Tree node 14 16 0 6.5 0 0 3.7936366352692781E-06 +0 1 64 Tree node -9 15 6 5.5 0 0 1.0032139972793415E-05 +0 1 64 Tree node -16 -17 1 5.5 0 0 3.9892404076591649E-06 +0 1 64 Tree node -15 -18 0 8.5 0 0 3.1409281260584296E-06 +0 1 64 Tree node -1 18 5 1.5 0 0 6.2191828731933257E-07 +0 1 64 Tree node -19 -20 5 2.5 0 0 1.2767903728892398E-06 +0 1 64 Leaf node 7 5:-0.5003482260880312 +0 1 64 Leaf node 7 5:0.3380818250235384 +0 1 64 Leaf node 7 5:0.24281453761315522 +0 1 64 Leaf node 7 5:-0.18599193287158558 +0 1 64 Leaf node 7 5:-0.50536339894703319 +0 1 64 Leaf node 7 5:0.22210591123450257 +0 1 64 Leaf node 7 5:-0.29830400292958592 +0 1 64 Leaf node 7 5:0.5003877871477137 +0 1 64 Leaf node 7 5:0.5023780664061257 +0 1 64 Leaf node 7 5:-0.12430679227249682 +0 1 64 Leaf node 7 5:0.47193087776947179 +0 1 64 Leaf node 7 5:-0.24256540211546027 +0 1 64 Leaf node 7 5:-0.35528668610055819 +0 1 64 Leaf node 7 5:0.50043528928827019 +0 1 64 Leaf node 7 5:-0.33107341068301605 +0 1 64 Leaf node 7 5:-0.19494389691428199 +0 1 64 Leaf node 7 5:0.50139768032825283 +0 1 64 Leaf node 7 5:0.50172677463188942 +0 1 64 Leaf node 7 5:0.26031285434617352 +0 1 64 Leaf node 7 5:-0.50006030580601157 +0 1 65 Tree node 2 1 4 5.5 0 0 1.7621999888760707E-06 +0 1 65 Tree node -2 6 3 2.5 0 0 5.8101420348013316E-06 +0 1 65 Tree node 3 9 0 4.5 0 0 2.1845055115986708E-06 +0 1 65 Tree node 4 -5 0 3.5 0 0 1.0500138589277345E-05 +0 1 65 Tree node 5 -6 1 3.5 0 0 2.511016722851089E-06 +0 1 65 Tree node 11 -7 5 4.5 0 0 2.664034466951169E-06 +0 1 65 Tree node 8 7 2 4.5 0 0 1.9365387191277474E-06 +0 1 65 Tree node -8 14 2 5.5 0 0 7.503549978217956E-06 +0 1 65 Tree node -3 15 0 4.5 0 0 2.6944306100440481E-06 +0 1 65 Tree node 10 17 6 6.5 0 0 1.8994052599810116E-06 +0 1 65 Tree node -4 -12 3 4.5 0 0 1.3521855056679827E-06 +0 1 65 Tree node 18 12 3 1.5 0 0 1.2574611067546792E-06 +0 1 65 Tree node 13 -14 4 3.5 0 0 3.9442168180710069E-06 +0 1 65 Tree node -13 -15 6 2.5 0 0 8.8171171456497785E-07 +0 1 65 Tree node -9 -16 0 4.5 0 0 8.6524701165392146E-07 +0 1 65 Tree node 16 -17 6 4.5 0 0 7.3647272853204598E-07 +0 1 65 Tree node -10 -18 0 7.5 0 0 2.1437867035018364E-06 +0 1 65 Tree node -11 -19 0 8.5 0 0 6.6733573612797936E-07 +0 1 65 Tree node -1 -20 5 1.5 0 0 4.850718629112009E-07 +0 1 65 Leaf node 7 5:-0.50024722650471642 +0 1 65 Leaf node 7 5:0.22826491589247352 +0 1 65 Leaf node 7 5:0.5018092870466021 +0 1 65 Leaf node 7 5:0.50172401842097702 +0 1 65 Leaf node 7 5:-0.15778485926886063 +0 1 65 Leaf node 7 5:0.19461323574626069 +0 1 65 Leaf node 7 5:-0.19396355619205624 +0 1 65 Leaf node 7 5:-0.28887797994443715 +0 1 65 Leaf node 7 5:0.50097941559086434 +0 1 65 Leaf node 7 5:-0.32297791110159924 +0 1 65 Leaf node 7 5:-0.3120086525538911 +0 1 65 Leaf node 7 5:0.50056647499481355 +0 1 65 Leaf node 7 5:-0.50079086784021898 +0 1 65 Leaf node 7 5:-0.18794535137002583 +0 1 65 Leaf node 7 5:-0.1858737299397866 +0 1 65 Leaf node 7 5:0.50018556118340496 +0 1 65 Leaf node 7 5:0.50090352797172577 +0 1 65 Leaf node 7 5:0.50111884894051839 +0 1 65 Leaf node 7 5:0.50130647974089182 +0 1 65 Leaf node 7 5:0.21649051988876714 +0 1 66 Tree node 3 1 5 3.5 0 0 1.7525884904102254E-06 +0 1 66 Tree node 2 12 2 1.5 0 0 8.1668807752611825E-06 +0 1 66 Tree node -2 -4 3 2.5 0 0 8.7858138056263146E-06 +0 1 66 Tree node 4 -5 6 7.5 0 0 2.8353383328870907E-06 +0 1 66 Tree node 6 5 0 3.5 0 0 3.9806701624069466E-06 +0 1 66 Tree node -6 -7 2 2.5 0 0 1.2847697973308427E-05 +0 1 66 Tree node 7 -8 1 3.5 0 0 3.6086851643220024E-06 +0 1 66 Tree node 8 -9 2 3.5 0 0 8.8614387398270705E-06 +0 1 66 Tree node 9 -10 7 1.5 0 0 6.0486831411279646E-06 +0 1 66 Tree node 10 11 2 2.5 0 0 3.2162471249414742E-06 +0 1 66 Tree node 18 -12 4 2.5 0 0 1.3898719632678485E-05 +0 1 66 Tree node -11 -13 5 1.5 0 0 3.047163094854295E-06 +0 1 66 Tree node 13 14 1 4.5 0 0 2.785121837695039E-06 +0 1 66 Tree node -3 16 4 7.5 0 0 1.4985847436758993E-05 +0 1 66 Tree node 15 17 0 4.5 0 0 1.3215931873954004E-06 +0 1 66 Tree node -14 -17 6 3.5 0 0 1.1443821928825903E-06 +0 1 66 Tree node -15 -18 0 4.5 0 0 7.8829647404981153E-07 +0 1 66 Tree node -16 -19 2 2.5 0 0 4.59705128427562E-07 +0 1 66 Tree node -1 -20 0 2.5 0 0 2.1225989894537882E-07 +0 1 66 Leaf node 7 5:-0.50019896777901929 +0 1 66 Leaf node 7 5:-0.032749856999420887 +0 1 66 Leaf node 7 5:-0.35391577568403054 +0 1 66 Leaf node 7 5:0.50404697716287117 +0 1 66 Leaf node 7 5:0.50124481196031234 +0 1 66 Leaf node 7 5:-0.26160176407310604 +0 1 66 Leaf node 7 5:0.17055222484642898 +0 1 66 Leaf node 7 5:0.39067341646149406 +0 1 66 Leaf node 7 5:-0.37925616544101803 +0 1 66 Leaf node 7 5:0.33255183920564557 +0 1 66 Leaf node 7 5:-0.50000466019045575 +0 1 66 Leaf node 7 5:-0.50539863016764619 +0 1 66 Leaf node 7 5:0.28466272207783339 +0 1 66 Leaf node 7 5:0.21563368573124461 +0 1 66 Leaf node 7 5:0.5010861275997518 +0 1 66 Leaf node 7 5:0.50213386100968538 +0 1 66 Leaf node 7 5:0.077059634322364887 +0 1 66 Leaf node 7 5:0.50009682421614776 +0 1 66 Leaf node 7 5:0.14191048922448199 +0 1 66 Leaf node 7 5:-0.50157356073609383 +0 1 67 Tree node 2 1 6 2.5 0 0 1.5265117448221589E-06 +0 1 67 Tree node -2 9 0 2.5 0 0 3.6906988694926946E-06 +0 1 67 Tree node 4 3 0 3.5 0 0 1.3114706999089233E-06 +0 1 67 Tree node -4 5 0 4.5 0 0 1.25018199702938E-05 +0 1 67 Tree node 6 -6 7 1.5 0 0 4.1029682536265683E-06 +0 1 67 Tree node -5 16 0 6.5 0 0 1.7518318865100631E-06 +0 1 67 Tree node 7 -8 5 4.5 0 0 1.3751634113580136E-06 +0 1 67 Tree node 8 -9 0 2.5 0 0 2.1063128357414165E-06 +0 1 67 Tree node 17 -10 4 3.5 0 0 4.9418575791183166E-06 +0 1 67 Tree node -3 10 3 2.5 0 0 1.1174429974322993E-06 +0 1 67 Tree node 11 12 6 3.5 0 0 1.7471421392077611E-06 +0 1 67 Tree node -11 -13 5 4.5 0 0 2.0438599893802077E-06 +0 1 67 Tree node -12 13 0 3.5 0 0 1.573224370218058E-06 +0 1 67 Tree node -14 14 2 1.5 0 0 1.7261976985317537E-06 +0 1 67 Tree node -15 15 0 4.5 0 0 1.1811322357142552E-06 +0 1 67 Tree node -16 -17 5 3.5 0 0 5.4667264411407847E-07 +0 1 67 Tree node -7 -18 3 5.5 0 0 4.6200893640788495E-07 +0 1 67 Tree node 18 -19 3 1.5 0 0 3.4497307285163903E-07 +0 1 67 Tree node -1 -20 5 1.5 0 0 1.6874343476008547E-07 +0 1 67 Leaf node 7 5:-0.5000058631308979 +0 1 67 Leaf node 7 5:0.30072487558391814 +0 1 67 Leaf node 7 5:0.21474584877411118 +0 1 67 Leaf node 7 5:-0.39443187081322617 +0 1 67 Leaf node 7 5:0.50121417202091834 +0 1 67 Leaf node 7 5:0.29705511823567432 +0 1 67 Leaf node 7 5:-0.37467973042086816 +0 1 67 Leaf node 7 5:-0.24637647788823294 +0 1 67 Leaf node 7 5:0.35195021075633742 +0 1 67 Leaf node 7 5:-0.16386338503699913 +0 1 67 Leaf node 7 5:0.22691270801879176 +0 1 67 Leaf node 7 5:-0.17607610441663271 +0 1 67 Leaf node 7 5:-0.023240283940045441 +0 1 67 Leaf node 7 5:-0.15169841128490275 +0 1 67 Leaf node 7 5:-0.22089285700755842 +0 1 67 Leaf node 7 5:0.50089715878675622 +0 1 67 Leaf node 7 5:0.50031853139888527 +0 1 67 Leaf node 7 5:0.50034689316795233 +0 1 67 Leaf node 7 5:-0.50077639409257912 +0 1 67 Leaf node 7 5:-0.50097792050864554 +0 1 68 Tree node 2 1 4 5.5 0 0 1.0297722838226496E-06 +0 1 68 Tree node -2 11 3 2.5 0 0 2.9956992420392849E-06 +0 1 68 Tree node 3 10 6 8.5 0 0 1.4930696400725119E-06 +0 1 68 Tree node 4 -5 6 5.5 0 0 3.5183453509428504E-06 +0 1 68 Tree node 5 -6 6 3.5 0 0 2.5992219459847951E-06 +0 1 68 Tree node 6 -7 4 4.5 0 0 2.8013666130653726E-06 +0 1 68 Tree node 7 -8 3 4.5 0 0 6.0967895812327435E-06 +0 1 68 Tree node 9 8 2 2.5 0 0 2.7130734666754643E-06 +0 1 68 Tree node -9 -10 1 1.5 0 0 3.9689171198375201E-06 +0 1 68 Tree node 14 -11 5 3.5 0 0 2.9950638492486706E-06 +0 1 68 Tree node -4 -12 0 7.5 0 0 1.6067609464749252E-06 +0 1 68 Tree node 13 12 2 4.5 0 0 1.1347668855364562E-06 +0 1 68 Tree node -13 17 2 5.5 0 0 3.5303801279211027E-06 +0 1 68 Tree node -3 15 0 4.5 0 0 1.6746505085063214E-06 +0 1 68 Tree node -1 18 0 2.5 0 0 6.0461810366399953E-07 +0 1 68 Tree node -15 16 6 3.5 0 0 4.2607982823809696E-07 +0 1 68 Tree node -17 -18 3 4.5 0 0 1.3955716313178785E-06 +0 1 68 Tree node -14 -19 0 4.5 0 0 3.7157382630098579E-07 +0 1 68 Tree node -16 -20 1 2.5 0 0 1.9085646903606497E-07 +0 1 68 Leaf node 7 5:-0.27951883731399874 +0 1 68 Leaf node 7 5:0.20887560320027304 +0 1 68 Leaf node 7 5:0.50149347428437074 +0 1 68 Leaf node 7 5:0.50134758498132426 +0 1 68 Leaf node 7 5:-0.29938953942762392 +0 1 68 Leaf node 7 5:0.30143578287238593 +0 1 68 Leaf node 7 5:-0.10222521899132596 +0 1 68 Leaf node 7 5:0.48521158232501432 +0 1 68 Leaf node 7 5:-0.5015838144502669 +0 1 68 Leaf node 7 5:-0.40015799248723161 +0 1 68 Leaf node 7 5:0.30922872597793882 +0 1 68 Leaf node 7 5:0.50002515416827054 +0 1 68 Leaf node 7 5:-0.2511581098579026 +0 1 68 Leaf node 7 5:0.50060677709147494 +0 1 68 Leaf node 7 5:0.50082488918323265 +0 1 68 Leaf node 7 5:-0.50099399610730133 +0 1 68 Leaf node 7 5:0.50068328328492395 +0 1 68 Leaf node 7 5:-0.40396745323096273 +0 1 68 Leaf node 7 5:0.50012109588204423 +0 1 68 Leaf node 7 5:-0.20597431542893357 +0 1 69 Tree node 7 1 3 1.5 0 0 8.5486320931480235E-07 +0 1 69 Tree node 2 4 1 2.5 0 0 1.291966433394323E-06 +0 1 69 Tree node 3 -4 4 4.5 0 0 5.1494664099015344E-06 +0 1 69 Tree node 9 -5 6 2.5 0 0 1.430747507754498E-06 +0 1 69 Tree node 5 11 0 3.5 0 0 4.8661147123984355E-07 +0 1 69 Tree node -3 6 4 2.5 0 0 3.2805094430389129E-06 +0 1 69 Tree node -7 -8 6 4.5 0 0 3.5185859144819042E-06 +0 1 69 Tree node -1 8 5 1.5 0 0 4.6980093195098245E-07 +0 1 69 Tree node -9 -10 5 2.5 0 0 8.9594370692681997E-07 +0 1 69 Tree node 10 -11 0 2.5 0 0 4.4750668718590686E-07 +0 1 69 Tree node -2 -12 4 2.5 0 0 1.2080038061817847E-06 +0 1 69 Tree node 12 15 4 8.5 0 0 3.9111111821104339E-07 +0 1 69 Tree node -6 13 0 4.5 0 0 1.9934363394195444E-06 +0 1 69 Tree node 14 -15 4 7.5 0 0 1.2205935114104782E-06 +0 1 69 Tree node -14 -16 0 6.5 0 0 8.1459638839923228E-07 +0 1 69 Tree node -13 16 2 2.5 0 0 7.3789020566056415E-07 +0 1 69 Tree node 17 18 3 6.5 0 0 3.0664971114524928E-07 +0 1 69 Tree node -17 -19 5 4.5 0 0 4.9131980560885916E-07 +0 1 69 Tree node -18 -20 3 8.5 0 0 2.9119480074186094E-07 +0 1 69 Leaf node 7 5:-0.50023973273604283 +0 1 69 Leaf node 7 5:-0.50038605873475939 +0 1 69 Leaf node 7 5:-0.24890700166860316 +0 1 69 Leaf node 7 5:-0.14481323255878972 +0 1 69 Leaf node 7 5:-0.27832200442063693 +0 1 69 Leaf node 7 5:-0.10734008174968099 +0 1 69 Leaf node 7 5:0.37234853345627678 +0 1 69 Leaf node 7 5:-0.008954062367071549 +0 1 69 Leaf node 7 5:0.26793703558785881 +0 1 69 Leaf node 7 5:-0.50004843107077068 +0 1 69 Leaf node 7 5:0.2687653108970996 +0 1 69 Leaf node 7 5:-0.50187865101125484 +0 1 69 Leaf node 7 5:0.50143516101862473 +0 1 69 Leaf node 7 5:0.50090753592012849 +0 1 69 Leaf node 7 5:-0.21885147004274749 +0 1 69 Leaf node 7 5:0.17590666210686923 +0 1 69 Leaf node 7 5:0.50094724990789918 +0 1 69 Leaf node 7 5:-0.43333567928762362 +0 1 69 Leaf node 7 5:0.50029657398389749 +0 1 69 Leaf node 7 5:0.50043297563664291 +0 1 70 Tree node 2 1 6 2.5 0 0 8.4582531205353644E-07 +0 1 70 Tree node -2 7 0 2.5 0 0 2.5880392009796287E-06 +0 1 70 Tree node 16 3 3 1.5 0 0 8.6974468953421618E-07 +0 1 70 Tree node 14 4 4 3.5 0 0 2.1413973189635853E-06 +0 1 70 Tree node -5 5 0 1.5 0 0 1.8848233045853905E-05 +0 1 70 Tree node -6 6 2 3.5 0 0 2.3120814464960311E-06 +0 1 70 Tree node -7 -8 5 5.5 0 0 5.9611448668864427E-07 +0 1 70 Tree node 8 11 6 4.5 0 0 5.2480269176617958E-07 +0 1 70 Tree node 9 10 2 3.5 0 0 1.6691433494768417E-06 +0 1 70 Tree node -3 -11 1 5.5 0 0 2.9918054738215836E-06 +0 1 70 Tree node -10 -12 5 4.5 0 0 1.7822797163639954E-06 +0 1 70 Tree node -9 12 0 3.5 0 0 1.2327261781094224E-06 +0 1 70 Tree node -13 13 6 5.5 0 0 6.1019125620827094E-07 +0 1 70 Tree node -14 -15 0 4.5 0 0 1.1335413559953937E-06 +0 1 70 Tree node 15 -16 2 3.5 0 0 3.2599276116444825E-07 +0 1 70 Tree node -4 18 0 2.5 0 0 6.8822894282611376E-07 +0 1 70 Tree node -1 17 5 1.5 0 0 2.816332915111119E-07 +0 1 70 Tree node -18 -19 5 2.5 0 0 5.8098102148082641E-07 +0 1 70 Tree node -17 -20 1 2.5 0 0 2.3180711475400779E-07 +0 1 70 Leaf node 7 5:-0.50022700382491436 +0 1 70 Leaf node 7 5:0.29138808706518132 +0 1 70 Leaf node 7 5:0.21187899707132404 +0 1 70 Leaf node 7 5:-0.11241556179200669 +0 1 70 Leaf node 7 5:-0.50383878388931169 +0 1 70 Leaf node 7 5:0.17786745633741044 +0 1 70 Leaf node 7 5:-0.18883459577911532 +0 1 70 Leaf node 7 5:0.50025114503638035 +0 1 70 Leaf node 7 5:-0.16111231963657272 +0 1 70 Leaf node 7 5:0.50220904515755649 +0 1 70 Leaf node 7 5:-0.29592305431970634 +0 1 70 Leaf node 7 5:0.035432347919630451 +0 1 70 Leaf node 7 5:0.50147347376464702 +0 1 70 Leaf node 7 5:-0.24988242212945316 +0 1 70 Leaf node 7 5:0.24017830412289307 +0 1 70 Leaf node 7 5:0.46870124605579605 +0 1 70 Leaf node 7 5:-0.50083340604123039 +0 1 70 Leaf node 7 5:0.22621500731746336 +0 1 70 Leaf node 7 5:-0.50003965154396501 +0 1 70 Leaf node 7 5:-0.24893660488482089 +0 1 71 Tree node 2 1 4 5.5 0 0 7.8856896796924533E-07 +0 1 71 Tree node -2 10 3 2.5 0 0 2.3389467091566195E-06 +0 1 71 Tree node 3 8 6 8.5 0 0 1.0859154953183673E-06 +0 1 71 Tree node 4 -5 6 5.5 0 0 2.3668875155747782E-06 +0 1 71 Tree node 5 -6 1 4.5 0 0 1.8284813520253767E-06 +0 1 71 Tree node 6 -7 0 3.5 0 0 5.7749600500350861E-06 +0 1 71 Tree node 9 7 4 2.5 0 0 1.5008216545585902E-06 +0 1 71 Tree node -8 -9 5 1.5 0 0 9.755776327248367E-06 +0 1 71 Tree node -4 -10 0 7.5 0 0 1.1722524179499587E-06 +0 1 71 Tree node 16 -11 0 2.5 0 0 1.1558627150149795E-06 +0 1 71 Tree node 13 11 6 1.5 0 0 6.8481188799657473E-07 +0 1 71 Tree node 14 12 2 4.5 0 0 7.691170779648191E-07 +0 1 71 Tree node -13 17 2 5.5 0 0 3.897096176863966E-06 +0 1 71 Tree node -3 -15 5 5.5 0 0 7.4578304117205761E-07 +0 1 71 Tree node -12 15 0 4.5 0 0 6.3688590890547728E-07 +0 1 71 Tree node -16 -17 0 7.5 0 0 4.2254782792801235E-07 +0 1 71 Tree node 18 -18 3 1.5 0 0 2.5141824972274051E-07 +0 1 71 Tree node -14 -19 1 4.5 0 0 1.8873209493771301E-07 +0 1 71 Tree node -1 -20 5 2.5 0 0 1.1483471455262349E-07 +0 1 71 Leaf node 7 5:-0.50077124135300011 +0 1 71 Leaf node 7 5:0.21454820976546601 +0 1 71 Leaf node 7 5:0.50116952181852692 +0 1 71 Leaf node 7 5:0.50110432018304618 +0 1 71 Leaf node 7 5:-0.28367388945034006 +0 1 71 Leaf node 7 5:0.21800427275204043 +0 1 71 Leaf node 7 5:-0.2398827935227183 +0 1 71 Leaf node 7 5:-0.50153766347697393 +0 1 71 Leaf node 7 5:0.24081568173111584 +0 1 71 Leaf node 7 5:0.50001743703765134 +0 1 71 Leaf node 7 5:-0.50116300359096644 +0 1 71 Leaf node 7 5:0.50082498411299936 +0 1 71 Leaf node 7 5:-0.37928999701433169 +0 1 71 Leaf node 7 5:0.50048175832788711 +0 1 71 Leaf node 7 5:0.50007795224433704 +0 1 71 Leaf node 7 5:-0.16488576422006007 +0 1 71 Leaf node 7 5:0.50083511011521908 +0 1 71 Leaf node 7 5:-0.32558867914912876 +0 1 71 Leaf node 7 5:0.50005047424597238 +0 1 71 Leaf node 7 5:-0.5000324636280421 +0 1 72 Tree node 3 1 5 3.5 0 0 5.5116305444667185E-07 +0 1 72 Tree node 2 7 2 1.5 0 0 2.679292371545614E-06 +0 1 72 Tree node -2 -4 3 2.5 0 0 3.0367915839257383E-06 +0 1 72 Tree node 4 -5 4 9.5 0 0 1.3118971367527312E-06 +0 1 72 Tree node 5 -6 4 5.5 0 0 3.0832820254884793E-06 +0 1 72 Tree node 6 -7 0 3.5 0 0 2.2869001232443612E-06 +0 1 72 Tree node 11 -8 4 3.5 0 0 1.8434796646617916E-06 +0 1 72 Tree node 8 9 1 4.5 0 0 1.1601024435028415E-06 +0 1 72 Tree node -3 15 4 7.5 0 0 5.6314369847630915E-06 +0 1 72 Tree node 10 12 1 5.5 0 0 8.8549997437470243E-07 +0 1 72 Tree node -9 -12 6 4.5 0 0 1.8391187798062101E-06 +0 1 72 Tree node 16 -13 6 2.5 0 0 4.5227657151420826E-07 +0 1 72 Tree node 13 14 2 3.5 0 0 4.3362838664120746E-07 +0 1 72 Tree node -11 -15 6 3.5 0 0 9.594613106132323E-07 +0 1 72 Tree node -14 17 2 4.5 0 0 5.4444654861151322E-07 +0 1 72 Tree node -10 -17 2 6.5 0 0 2.7531091047195163E-07 +0 1 72 Tree node -1 -18 3 1.5 0 0 1.6398455678284963E-07 +0 1 72 Tree node -16 18 5 4.5 0 0 7.9007373111057127E-08 +0 1 72 Tree node -19 -20 2 5.5 0 0 1.3472588450316461E-07 +0 1 72 Leaf node 7 5:-0.5006153385897204 +0 1 72 Leaf node 7 5:-0.034171356915143385 +0 1 72 Leaf node 7 5:-0.33391979238939656 +0 1 72 Leaf node 7 5:0.50213328120914791 +0 1 72 Leaf node 7 5:-0.18512058189368191 +0 1 72 Leaf node 7 5:0.26670738125583776 +0 1 72 Leaf node 7 5:-0.20412266503728094 +0 1 72 Leaf node 7 5:0.146077412601052 +0 1 72 Leaf node 7 5:0.50297428027889424 +0 1 72 Leaf node 7 5:0.50080014396886652 +0 1 72 Leaf node 7 5:-0.45686735249813698 +0 1 72 Leaf node 7 5:-0.31058873550188121 +0 1 72 Leaf node 7 5:-0.22124973298777384 +0 1 72 Leaf node 7 5:0.50199608887667868 +0 1 72 Leaf node 7 5:0.50022894015007535 +0 1 72 Leaf node 7 5:-0.21391019789457538 +0 1 72 Leaf node 7 5:0.50025690433604719 +0 1 72 Leaf node 7 5:-0.32195902670383414 +0 1 72 Leaf node 7 5:0.50031968359891377 +0 1 72 Leaf node 7 5:0.50007899550249724 +0 1 73 Tree node 7 1 6 2.5 0 0 5.5927986993119278E-07 +0 1 73 Tree node 2 4 2 1.5 0 0 1.3173895834007121E-06 +0 1 73 Tree node 3 -4 1 4.5 0 0 3.4929872557467734E-06 +0 1 73 Tree node -2 -5 4 1.5 0 0 5.9725683542886735E-06 +0 1 73 Tree node 5 11 1 4.5 0 0 1.0192347301791457E-06 +0 1 73 Tree node 6 -7 2 6.5 0 0 2.3332503984345572E-06 +0 1 73 Tree node -3 -8 3 3.5 0 0 1.5473309171932533E-06 +0 1 73 Tree node 18 8 3 1.5 0 0 6.0238575974708714E-07 +0 1 73 Tree node 16 9 4 3.5 0 0 1.7864933487399669E-06 +0 1 73 Tree node -10 10 0 1.5 0 0 1.1977401517740352E-05 +0 1 73 Tree node -11 15 2 3.5 0 0 1.0400974336627862E-06 +0 1 73 Tree node -6 12 5 3.5 0 0 4.5801343021523537E-07 +0 1 73 Tree node 14 13 2 3.5 0 0 8.1215800276031154E-07 +0 1 73 Tree node -14 -15 0 4.5 0 0 2.8075229147866417E-06 +0 1 73 Tree node -13 -16 6 6.5 0 0 1.0505048836946928E-06 +0 1 73 Tree node -12 -17 2 5.5 0 0 3.3870578726986968E-07 +0 1 73 Tree node 17 -18 2 3.5 0 0 2.1861316389608384E-07 +0 1 73 Tree node -9 -19 0 2.5 0 0 2.4713519084505676E-07 +0 1 73 Tree node -1 -20 5 1.5 0 0 1.7882029691492769E-07 +0 1 73 Leaf node 7 5:-0.50012446518588738 +0 1 73 Leaf node 7 5:-0.26916403436870012 +0 1 73 Leaf node 7 5:-0.044928174750895486 +0 1 73 Leaf node 7 5:-0.14039551889095392 +0 1 73 Leaf node 7 5:0.49110078957460807 +0 1 73 Leaf node 7 5:0.50060593700462974 +0 1 73 Leaf node 7 5:0.50088107150995675 +0 1 73 Leaf node 7 5:-0.38709003427129457 +0 1 73 Leaf node 7 5:-0.12266720131523688 +0 1 73 Leaf node 7 5:-0.5032196115484695 +0 1 73 Leaf node 7 5:0.14091273986033595 +0 1 73 Leaf node 7 5:-0.19120856439315714 +0 1 73 Leaf node 7 5:-0.46787290592983855 +0 1 73 Leaf node 7 5:0.50252111154859436 +0 1 73 Leaf node 7 5:0.50020579274608257 +0 1 73 Leaf node 7 5:0.50019586961118201 +0 1 73 Leaf node 7 5:0.50010495179658254 +0 1 73 Leaf node 7 5:0.46573497552892096 +0 1 73 Leaf node 7 5:-0.23464057669997007 +0 1 73 Leaf node 7 5:0.233743680972092 +0 1 74 Tree node 1 5 1 2.5 0 0 4.7926260796487016E-07 +0 1 74 Tree node 3 2 0 2.5 0 0 1.213617534748077E-06 +0 1 74 Tree node -3 -4 3 2.5 0 0 3.1103427233433183E-06 +0 1 74 Tree node 4 -5 4 4.5 0 0 2.2176726635996379E-06 +0 1 74 Tree node -1 -6 2 2.5 0 0 4.4331616590150832E-07 +0 1 74 Tree node 6 10 0 3.5 0 0 2.6321222999811316E-07 +0 1 74 Tree node 7 -8 3 4.5 0 0 1.7244525584890406E-06 +0 1 74 Tree node 9 8 4 2.5 0 0 9.7055642628222416E-07 +0 1 74 Tree node -9 -10 6 4.5 0 0 1.6423572394142492E-06 +0 1 74 Tree node -2 -11 5 2.5 0 0 2.0886988568557984E-07 +0 1 74 Tree node -7 11 1 3.5 0 0 1.7719429570386644E-07 +0 1 74 Tree node 12 13 1 4.5 0 0 4.3474281239919765E-07 +0 1 74 Tree node -12 -14 4 7.5 0 0 1.2598701669897982E-06 +0 1 74 Tree node -13 14 2 1.5 0 0 5.434099542176816E-07 +0 1 74 Tree node 15 16 1 5.5 0 0 5.4335755817048735E-07 +0 1 74 Tree node -15 -17 6 3.5 0 0 1.1455096293089319E-06 +0 1 74 Tree node -16 17 5 3.5 0 0 2.4218785198137752E-07 +0 1 74 Tree node 18 -19 2 3.5 0 0 2.3658916367099781E-07 +0 1 74 Tree node -18 -20 6 3.5 0 0 3.7269445625704216E-07 +0 1 74 Leaf node 7 5:-0.50017647268186316 +0 1 74 Leaf node 7 5:-0.50050312655620466 +0 1 74 Leaf node 7 5:-0.50573749389009026 +0 1 74 Leaf node 7 5:0.10730159964388571 +0 1 74 Leaf node 7 5:0.10887030320148121 +0 1 74 Leaf node 7 5:-0.50133196575313999 +0 1 74 Leaf node 7 5:0.50059984737270791 +0 1 74 Leaf node 7 5:0.5011015723509975 +0 1 74 Leaf node 7 5:0.32844709331343047 +0 1 74 Leaf node 7 5:-0.069261279453533137 +0 1 74 Leaf node 7 5:-0.500772107974089 +0 1 74 Leaf node 7 5:-0.26397443716880192 +0 1 74 Leaf node 7 5:-0.11843109789836255 +0 1 74 Leaf node 7 5:0.13089392051337129 +0 1 74 Leaf node 7 5:0.50186007997565829 +0 1 74 Leaf node 7 5:0.50052691545805394 +0 1 74 Leaf node 7 5:-0.21829057099315291 +0 1 74 Leaf node 7 5:-0.42581862594552577 +0 1 74 Leaf node 7 5:0.26409710579636697 +0 1 74 Leaf node 7 5:0.5000742460263139 +0 1 75 Tree node 2 1 6 2.5 0 0 3.463407173433119E-07 +0 1 75 Tree node -2 7 0 2.5 0 0 1.0206724822755099E-06 +0 1 75 Tree node 16 3 3 1.5 0 0 4.3817698068815368E-07 +0 1 75 Tree node 5 4 4 4.5 0 0 1.3643669749271553E-06 +0 1 75 Tree node -5 14 1 2.5 0 0 6.1853602252620831E-06 +0 1 75 Tree node 6 -7 5 3.5 0 0 9.8016338488378942E-07 +0 1 75 Tree node -4 -8 4 2.5 0 0 4.4711130038676047E-07 +0 1 75 Tree node -3 8 3 2.5 0 0 3.5999523321717953E-07 +0 1 75 Tree node 13 9 6 3.5 0 0 5.1763403372808721E-07 +0 1 75 Tree node 10 12 6 7.5 0 0 8.6763006205134677E-07 +0 1 75 Tree node 11 15 5 4.5 0 0 3.2383311381055006E-06 +0 1 75 Tree node -10 -13 0 4.5 0 0 7.4347635450979032E-07 +0 1 75 Tree node -11 17 5 3.5 0 0 7.2077748320380365E-07 +0 1 75 Tree node -9 -15 5 4.5 0 0 6.6635181445474689E-07 +0 1 75 Tree node -6 -16 0 7.5 0 0 2.0887702906450243E-07 +0 1 75 Tree node -12 -17 4 7.5 0 0 1.2862810503165552E-07 +0 1 75 Tree node -1 -18 5 1.5 0 0 1.2826854591210687E-07 +0 1 75 Tree node -14 18 4 6.5 0 0 6.3773928996506154E-08 +0 1 75 Tree node -19 -20 6 8.5 0 0 1.0304604035049152E-07 +0 1 75 Leaf node 7 5:-0.50008342302050446 +0 1 75 Leaf node 7 5:0.28328433414888227 +0 1 75 Leaf node 7 5:0.20660588536539168 +0 1 75 Leaf node 7 5:-0.39156281715319741 +0 1 75 Leaf node 7 5:-0.24933999655822436 +0 1 75 Leaf node 7 5:0.1819705849576839 +0 1 75 Leaf node 7 5:0.33148871309315558 +0 1 75 Leaf node 7 5:-0.21872964591371324 +0 1 75 Leaf node 7 5:0.23603984999947247 +0 1 75 Leaf node 7 5:-0.34478722838753412 +0 1 75 Leaf node 7 5:0.50111821458024841 +0 1 75 Leaf node 7 5:0.50041939894792276 +0 1 75 Leaf node 7 5:-0.242120972978978 +0 1 75 Leaf node 7 5:0.50024637472666056 +0 1 75 Leaf node 7 5:-0.040504896034416822 +0 1 75 Leaf node 7 5:-0.38457986951445178 +0 1 75 Leaf node 7 5:0.50017214935589172 +0 1 75 Leaf node 7 5:0.2277001020458792 +0 1 75 Leaf node 7 5:-0.37274840907996115 +0 1 75 Leaf node 7 5:0.50022290898507527 +0 1 76 Tree node 5 1 4 5.5 0 0 3.4161370897487198E-07 +0 1 76 Tree node -2 2 4 7.5 0 0 1.82143896202816E-06 +0 1 76 Tree node 3 4 5 3.5 0 0 9.3196733238944479E-07 +0 1 76 Tree node -3 -5 1 4.5 0 0 1.3827307816651441E-06 +0 1 76 Tree node -4 17 0 3.5 0 0 9.584824210966582E-07 +0 1 76 Tree node 6 7 7 2.5 0 0 6.0268956205144434E-07 +0 1 76 Tree node 8 -8 7 1.5 0 0 7.9097611590307148E-07 +0 1 76 Tree node -7 -9 0 6.5 0 0 5.3971672764715975E-07 +0 1 76 Tree node 9 -10 1 8.5 0 0 3.1026657017134585E-07 +0 1 76 Tree node 11 10 6 4.5 0 0 7.9901265772980947E-07 +0 1 76 Tree node -11 -12 6 7.5 0 0 9.0702602542790656E-07 +0 1 76 Tree node 12 -13 1 4.5 0 0 3.2287567494471032E-07 +0 1 76 Tree node 13 -14 2 3.5 0 0 6.587572441516971E-07 +0 1 76 Tree node 15 14 2 2.5 0 0 7.0449596048094928E-07 +0 1 76 Tree node -15 -16 5 1.5 0 0 6.4950931536728238E-07 +0 1 76 Tree node 16 -17 4 3.5 0 0 5.1326452492446264E-07 +0 1 76 Tree node -1 -18 0 2.5 0 0 3.4638291296949208E-07 +0 1 76 Tree node -6 18 0 4.5 0 0 1.2941690663358684E-07 +0 1 76 Tree node -19 -20 4 8.5 0 0 6.3040977404911327E-08 +0 1 76 Leaf node 7 5:-0.20987462224210279 +0 1 76 Leaf node 7 5:0.34370617229324868 +0 1 76 Leaf node 7 5:-0.23354171944516264 +0 1 76 Leaf node 7 5:0.50184740694924701 +0 1 76 Leaf node 7 5:0.041332088325804567 +0 1 76 Leaf node 7 5:0.50050986772537609 +0 1 76 Leaf node 7 5:0.32776569515652265 +0 1 76 Leaf node 7 5:-0.28506755930083194 +0 1 76 Leaf node 7 5:0.50011127318379955 +0 1 76 Leaf node 7 5:0.5010251701184939 +0 1 76 Leaf node 7 5:-0.29346629772507882 +0 1 76 Leaf node 7 5:0.032076891108427666 +0 1 76 Leaf node 7 5:0.12943233524104761 +0 1 76 Leaf node 7 5:-0.22434188745086178 +0 1 76 Leaf node 7 5:-0.50000087090314016 +0 1 76 Leaf node 7 5:0.23195121643612615 +0 1 76 Leaf node 7 5:-0.16506056698393748 +0 1 76 Leaf node 7 5:-0.50103426577919818 +0 1 76 Leaf node 7 5:-0.31471401198737381 +0 1 76 Leaf node 7 5:0.50013605138993122 +0 1 77 Tree node 7 1 0 1.5 0 0 2.8534991622199639E-07 +0 1 77 Tree node 2 3 0 2.5 0 0 5.9810603378145958E-07 +0 1 77 Tree node -2 -4 4 2.5 0 0 2.6040947462517039E-06 +0 1 77 Tree node 4 5 1 2.5 0 0 7.2168597901970492E-07 +0 1 77 Tree node -3 -6 3 2.5 0 0 1.5468023379133301E-06 +0 1 77 Tree node -5 6 5 2.5 0 0 9.1391754329258556E-07 +0 1 77 Tree node -7 9 3 2.5 0 0 3.9769375868942747E-07 +0 1 77 Tree node -1 8 3 1.5 0 0 2.8101200583689361E-07 +0 1 77 Tree node 18 -10 4 3.5 0 0 2.6231707177187681E-06 +0 1 77 Tree node 10 17 3 9.5 0 0 2.362103816262747E-07 +0 1 77 Tree node -8 11 2 1.5 0 0 4.20289860344214E-07 +0 1 77 Tree node 12 14 1 4.5 0 0 6.4343578877180548E-07 +0 1 77 Tree node 13 -14 2 5.5 0 0 1.4639785657056745E-06 +0 1 77 Tree node -12 -15 2 3.5 0 0 1.0675298570667091E-06 +0 1 77 Tree node -13 15 4 3.5 0 0 2.2201987987245962E-07 +0 1 77 Tree node -16 16 0 4.5 0 0 5.5957560287293005E-07 +0 1 77 Tree node -17 -18 5 3.5 0 0 2.0378748359550056E-07 +0 1 77 Tree node -11 -19 5 6.5 0 0 1.9316595146780063E-07 +0 1 77 Tree node -9 -20 6 1.5 0 0 1.2307417469728375E-07 +0 1 77 Leaf node 7 5:0.17047796333880072 +0 1 77 Leaf node 7 5:0.023787229293561234 +0 1 77 Leaf node 7 5:-0.50367606713836499 +0 1 77 Leaf node 7 5:0.32414054265339248 +0 1 77 Leaf node 7 5:0.2974100362942147 +0 1 77 Leaf node 7 5:0.1002931687345802 +0 1 77 Leaf node 7 5:0.31585122222414491 +0 1 77 Leaf node 7 5:-0.1794141470193619 +0 1 77 Leaf node 7 5:-0.50020292577928205 +0 1 77 Leaf node 7 5:-0.26944266353975294 +0 1 77 Leaf node 7 5:0.50065846558513416 +0 1 77 Leaf node 7 5:-0.09800199569124575 +0 1 77 Leaf node 7 5:-0.15245375494570906 +0 1 77 Leaf node 7 5:0.50101084070192292 +0 1 77 Leaf node 7 5:-0.32999580651943666 +0 1 77 Leaf node 7 5:0.50143213543163367 +0 1 77 Leaf node 7 5:0.50051296496676334 +0 1 77 Leaf node 7 5:0.039070457515700104 +0 1 77 Leaf node 7 5:0.50018388224240606 +0 1 77 Leaf node 7 5:-0.50087361179490131 +0 1 78 Tree node 2 1 7 1.5 0 0 2.6413630880789821E-07 +0 1 78 Tree node -2 15 2 1.5 0 0 1.397980117274507E-06 +0 1 78 Tree node 3 10 1 9.5 0 0 2.5852975312374154E-07 +0 1 78 Tree node 4 -5 3 6.5 0 0 3.9341960740929335E-07 +0 1 78 Tree node 5 9 1 6.5 0 0 4.4012819317239404E-07 +0 1 78 Tree node 8 6 6 3.5 0 0 8.1444654154845885E-07 +0 1 78 Tree node 7 -8 2 5.5 0 0 1.0368630855770452E-06 +0 1 78 Tree node -7 -9 6 6.5 0 0 1.0191445341118097E-06 +0 1 78 Tree node 11 -10 3 4.5 0 0 5.9055692474228941E-07 +0 1 78 Tree node -6 -11 2 5.5 0 0 4.0730749144447351E-07 +0 1 78 Tree node -4 -12 2 5.5 0 0 2.9456901876967741E-07 +0 1 78 Tree node 12 -13 5 6.5 0 0 2.0469213250318352E-07 +0 1 78 Tree node 13 -14 0 3.5 0 0 2.795675381881938E-07 +0 1 78 Tree node 14 -15 4 4.5 0 0 2.5483442676882372E-07 +0 1 78 Tree node 18 17 2 2.5 0 0 3.69429411505106E-07 +0 1 78 Tree node 16 -17 1 4.5 0 0 1.9963061567787952E-07 +0 1 78 Tree node -3 -18 2 5.5 0 0 5.7635208758253616E-07 +0 1 78 Tree node -16 -19 5 1.5 0 0 1.1972137749282429E-07 +0 1 78 Tree node -1 -20 4 2.5 0 0 1.0220980254711328E-07 +0 1 78 Leaf node 7 5:-0.5003047097765988 +0 1 78 Leaf node 7 5:0.22444573510348623 +0 1 78 Leaf node 7 5:-0.30611785697608207 +0 1 78 Leaf node 7 5:0.50068689757375695 +0 1 78 Leaf node 7 5:-0.36945599868508372 +0 1 78 Leaf node 7 5:-0.38006399653780715 +0 1 78 Leaf node 7 5:0.48138713791666798 +0 1 78 Leaf node 7 5:-0.37635426134082273 +0 1 78 Leaf node 7 5:0.018716065753386548 +0 1 78 Leaf node 7 5:0.20049105693519281 +0 1 78 Leaf node 7 5:0.50005385507482036 +0 1 78 Leaf node 7 5:0.50007024298529412 +0 1 78 Leaf node 7 5:0.33965464746161761 +0 1 78 Leaf node 7 5:-0.23197000568980974 +0 1 78 Leaf node 7 5:-0.075460959843985098 +0 1 78 Leaf node 7 5:-0.50132435434796685 +0 1 78 Leaf node 7 5:0.50020782237059669 +0 1 78 Leaf node 7 5:0.50025495882791449 +0 1 78 Leaf node 7 5:-0.50091071337400805 +0 1 78 Leaf node 7 5:0.12451263436641456 +0 1 79 Tree node 1 3 1 2.5 0 0 1.9652172375847879E-07 +0 1 79 Tree node 2 -3 2 3.5 0 0 4.7751190792221228E-07 +0 1 79 Tree node 7 -4 4 3.5 0 0 3.4963437562173931E-07 +0 1 79 Tree node 4 9 0 3.5 0 0 1.3424489393147716E-07 +0 1 79 Tree node 5 -6 3 4.5 0 0 7.3159553194513154E-07 +0 1 79 Tree node 6 -7 2 3.5 0 0 5.1661635108417546E-07 +0 1 79 Tree node 18 -8 6 2.5 0 0 1.7054793274507079E-07 +0 1 79 Tree node -1 8 2 1.5 0 0 1.2707127023421031E-07 +0 1 79 Tree node 16 -10 6 1.5 0 0 1.4751088712625173E-07 +0 1 79 Tree node 10 11 0 4.5 0 0 9.5373161760610683E-08 +0 1 79 Tree node -5 -12 4 7.5 0 0 8.1607536193974377E-07 +0 1 79 Tree node 12 13 4 3.5 0 0 1.9159586297360454E-07 +0 1 79 Tree node -11 -14 5 6.5 0 0 1.7521465300963001E-07 +0 1 79 Tree node 15 14 1 6.5 0 0 1.4123656220859786E-07 +0 1 79 Tree node -15 17 2 2.5 0 0 2.7403388985120262E-07 +0 1 79 Tree node -13 -17 4 7.5 0 0 9.9577562888267682E-08 +0 1 79 Tree node -9 -18 5 1.5 0 0 3.5487351662805076E-08 +0 1 79 Tree node -16 -19 6 2.5 0 0 2.9902051456854484E-08 +0 1 79 Tree node -2 -20 5 1.5 0 0 2.9348252984278801E-08 +0 1 79 Leaf node 7 5:-0.26932826074612359 +0 1 79 Leaf node 7 5:-0.50142924247413267 +0 1 79 Leaf node 7 5:-0.35706805320944346 +0 1 79 Leaf node 7 5:0.050855763430576996 +0 1 79 Leaf node 7 5:-0.13794666391292915 +0 1 79 Leaf node 7 5:0.50073427712899576 +0 1 79 Leaf node 7 5:0.23308885456402725 +0 1 79 Leaf node 7 5:0.0768663313562883 +0 1 79 Leaf node 7 5:-0.50006356484689518 +0 1 79 Leaf node 7 5:-0.50065106237069734 +0 1 79 Leaf node 7 5:0.50060690709560285 +0 1 79 Leaf node 7 5:0.22638975401300659 +0 1 79 Leaf node 7 5:0.50043941593822372 +0 1 79 Leaf node 7 5:0.50008441776054213 +0 1 79 Leaf node 7 5:-0.24697613193711918 +0 1 79 Leaf node 7 5:-0.23102869684303057 +0 1 79 Leaf node 7 5:0.50021270887683311 +0 1 79 Leaf node 7 5:-0.5008472634108837 +0 1 79 Leaf node 7 5:0.50011612260057614 +0 1 79 Leaf node 7 5:-0.49820402721211787 +0 1 80 Tree node 9 1 3 1.5 0 0 1.7149711414308706E-07 +0 1 80 Tree node 2 5 1 2.5 0 0 2.1885580779252217E-07 +0 1 80 Tree node 3 -4 4 4.5 0 0 1.3146638068496281E-06 +0 1 80 Tree node 4 12 2 2.5 0 0 2.3200850383230923E-07 +0 1 80 Tree node -2 -6 4 2.5 0 0 1.1813778093575595E-07 +0 1 80 Tree node 6 13 0 3.5 0 0 9.230214106634061E-08 +0 1 80 Tree node 7 -8 3 4.5 0 0 4.8650588502614482E-07 +0 1 80 Tree node 11 8 4 2.5 0 0 3.7974243981568658E-07 +0 1 80 Tree node -9 -10 6 4.5 0 0 7.0093079902479239E-07 +0 1 80 Tree node -1 10 5 1.5 0 0 8.1862137539782799E-08 +0 1 80 Tree node -11 -12 5 2.5 0 0 1.7193744869252243E-07 +0 1 80 Tree node -3 -13 5 2.5 0 0 7.6702427378180459E-08 +0 1 80 Tree node -5 -14 5 1.5 0 0 7.2893747669720469E-08 +0 1 80 Tree node -7 14 1 3.5 0 0 7.1474576720331159E-08 +0 1 80 Tree node 15 16 0 4.5 0 0 1.3708705616186673E-07 +0 1 80 Tree node -15 -17 4 7.5 0 0 7.4642844557915107E-07 +0 1 80 Tree node 17 18 4 3.5 0 0 1.3260539885546281E-07 +0 1 80 Tree node -16 -19 5 6.5 0 0 1.174442307139436E-07 +0 1 80 Tree node -18 -20 1 6.5 0 0 8.3893355503443415E-08 +0 1 80 Leaf node 7 5:-0.50005144239373067 +0 1 80 Leaf node 7 5:-0.18936813735960839 +0 1 80 Leaf node 7 5:-0.500393517418247 +0 1 80 Leaf node 7 5:-0.14088564777374035 +0 1 80 Leaf node 7 5:-0.50094063056812821 +0 1 80 Leaf node 7 5:0.27914851367773225 +0 1 80 Leaf node 7 5:0.50048602169791689 +0 1 80 Leaf node 7 5:0.50060104285893336 +0 1 80 Leaf node 7 5:0.30917122997899432 +0 1 80 Leaf node 7 5:-0.07021052236445316 +0 1 80 Leaf node 7 5:0.22607191596679979 +0 1 80 Leaf node 7 5:-0.50002608097320012 +0 1 80 Leaf node 7 5:-0.50053025044609489 +0 1 80 Leaf node 7 5:-0.45203547367223207 +0 1 80 Leaf node 7 5:-0.14517063231991262 +0 1 80 Leaf node 7 5:0.50049681568937254 +0 1 80 Leaf node 7 5:0.17871767722007278 +0 1 80 Leaf node 7 5:0.50031105382082286 +0 1 80 Leaf node 7 5:0.50006911381169838 +0 1 80 Leaf node 7 5:-0.12317467759431192 +0 1 81 Tree node 2 1 6 2.5 0 0 1.6521784181584784E-07 +0 1 81 Tree node -2 10 0 2.5 0 0 5.265690480915879E-07 +0 1 81 Tree node 4 3 0 3.5 0 0 2.7091657963828689E-07 +0 1 81 Tree node -4 9 0 4.5 0 0 1.9961871912804334E-06 +0 1 81 Tree node 5 -6 7 1.5 0 0 1.1982063697499955E-06 +0 1 81 Tree node 6 -7 5 4.5 0 0 2.8523040353855747E-07 +0 1 81 Tree node 7 8 2 2.5 0 0 2.617522222579717E-07 +0 1 81 Tree node -1 -9 4 3.5 0 0 1.4125226640815833E-06 +0 1 81 Tree node -8 -10 5 1.5 0 0 4.5191388306113135E-07 +0 1 81 Tree node -5 -11 0 7.5 0 0 1.7387753039967975E-07 +0 1 81 Tree node 11 12 6 3.5 0 0 1.5217829237743494E-07 +0 1 81 Tree node -3 -13 5 4.5 0 0 3.7661917189299852E-07 +0 1 81 Tree node 13 16 6 8.5 0 0 1.0383291718648434E-07 +0 1 81 Tree node 14 17 5 4.5 0 0 2.4571274504583285E-07 +0 1 81 Tree node 15 -16 3 5.5 0 0 2.2174747112548573E-07 +0 1 81 Tree node -12 -17 2 3.5 0 0 3.226563977831221E-07 +0 1 81 Tree node -14 -18 0 4.5 0 0 1.4948138570646556E-07 +0 1 81 Tree node -15 18 1 3.5 0 0 9.4179559414572244E-08 +0 1 81 Tree node -19 -20 6 7.5 0 0 7.7252487732129463E-08 +0 1 81 Leaf node 7 5:-0.50012085437151599 +0 1 81 Leaf node 7 5:0.30169673918732476 +0 1 81 Leaf node 7 5:0.19264905444482844 +0 1 81 Leaf node 7 5:-0.40320348702163306 +0 1 81 Leaf node 7 5:0.50035597817457778 +0 1 81 Leaf node 7 5:0.35221054831684045 +0 1 81 Leaf node 7 5:-0.30806346424126479 +0 1 81 Leaf node 7 5:-0.50076909030815397 +0 1 81 Leaf node 7 5:-0.291018274687808 +0 1 81 Leaf node 7 5:0.23950174806826338 +0 1 81 Leaf node 7 5:-0.25443727370711261 +0 1 81 Leaf node 7 5:-0.24617322807687789 +0 1 81 Leaf node 7 5:-0.079094402600980038 +0 1 81 Leaf node 7 5:0.50028813094281954 +0 1 81 Leaf node 7 5:0.50035784290438179 +0 1 81 Leaf node 7 5:0.05865530361483981 +0 1 81 Leaf node 7 5:-0.061079209229963405 +0 1 81 Leaf node 7 5:0.50006426897776945 +0 1 81 Leaf node 7 5:0.5001110445451854 +0 1 81 Leaf node 7 5:-0.44922363270566795 +0 1 82 Tree node 5 1 4 5.5 0 0 1.2976407718785445E-07 +0 1 82 Tree node -2 2 4 7.5 0 0 6.8182302627538224E-07 +0 1 82 Tree node 3 4 5 3.5 0 0 4.4618318162906033E-07 +0 1 82 Tree node -3 -5 1 4.5 0 0 6.2938783334529116E-07 +0 1 82 Tree node -4 16 0 3.5 0 0 4.1779571968581127E-07 +0 1 82 Tree node 6 -7 7 3.5 0 0 2.1843784217979942E-07 +0 1 82 Tree node 7 14 6 8.5 0 0 1.6802030021310906E-07 +0 1 82 Tree node 8 -9 6 5.5 0 0 5.5017055978326341E-07 +0 1 82 Tree node 9 -10 7 1.5 0 0 3.8892371715440543E-07 +0 1 82 Tree node 11 10 4 3.5 0 0 1.9048905812269384E-07 +0 1 82 Tree node -11 -12 2 2.5 0 0 1.0444188396364954E-06 +0 1 82 Tree node 12 -13 3 4.5 0 0 8.1308997622300191E-07 +0 1 82 Tree node 13 -14 6 2.5 0 0 7.7612133364754796E-07 +0 1 82 Tree node 15 -15 4 2.5 0 0 1.4553647628271174E-07 +0 1 82 Tree node -8 -16 0 7.5 0 0 1.2550446035001099E-07 +0 1 82 Tree node 17 -17 3 1.5 0 0 7.0065196263798414E-08 +0 1 82 Tree node -6 -18 0 4.5 0 0 5.1657242064466276E-08 +0 1 82 Tree node 18 -19 5 2.5 0 0 2.9839696553407008E-08 +0 1 82 Tree node -1 -20 5 1.5 0 0 6.2064910346192951E-08 +0 1 82 Leaf node 7 5:-0.50000040456796646 +0 1 82 Leaf node 7 5:0.33344538073442964 +0 1 82 Leaf node 7 5:-0.22543665320036813 +0 1 82 Leaf node 7 5:0.50130466989315325 +0 1 82 Leaf node 7 5:0.03670403318723172 +0 1 82 Leaf node 7 5:0.50031455477139053 +0 1 82 Leaf node 7 5:0.32939203201811917 +0 1 82 Leaf node 7 5:0.500436987051326 +0 1 82 Leaf node 7 5:-0.29656347218307627 +0 1 82 Leaf node 7 5:-0.26817460552450961 +0 1 82 Leaf node 7 5:-0.12441900924996784 +0 1 82 Leaf node 7 5:0.18225908473309477 +0 1 82 Leaf node 7 5:0.483946695200479 +0 1 82 Leaf node 7 5:-0.32149428309008943 +0 1 82 Leaf node 7 5:-0.50068598353221572 +0 1 82 Leaf node 7 5:0.50000895837735149 +0 1 82 Leaf node 7 5:-0.18245691146873272 +0 1 82 Leaf node 7 5:0.032103110480648969 +0 1 82 Leaf node 7 5:-0.50001882578940071 +0 1 82 Leaf node 7 5:-0.50056596702418321 +0 1 83 Tree node 10 1 0 1.5 0 0 1.1791807084658507E-07 +0 1 83 Tree node 2 5 2 1.5 0 0 2.5506840101312773E-07 +0 1 83 Tree node 3 -4 7 1.5 0 0 7.605205903511661E-07 +0 1 83 Tree node 4 -5 6 3.5 0 0 7.3346299022552215E-07 +0 1 83 Tree node -2 -6 4 2.5 0 0 1.1200860751056134E-06 +0 1 83 Tree node 6 12 1 4.5 0 0 2.8123199221205423E-07 +0 1 83 Tree node 7 -8 2 6.5 0 0 4.1042914664640936E-07 +0 1 83 Tree node 9 8 3 2.5 0 0 6.6064346698159958E-07 +0 1 83 Tree node -9 -10 0 4.5 0 0 1.0054351803312469E-06 +0 1 83 Tree node -3 -11 6 1.5 0 0 4.7367407587549981E-07 +0 1 83 Tree node 17 11 3 1.5 0 0 1.719371139512927E-07 +0 1 83 Tree node -12 -13 4 3.5 0 0 1.3678238014566987E-06 +0 1 83 Tree node 13 14 1 5.5 0 0 1.6162859649604728E-07 +0 1 83 Tree node -7 -15 3 3.5 0 0 1.9626278951011703E-07 +0 1 83 Tree node 15 16 3 3.5 0 0 7.9654471776927995E-08 +0 1 83 Tree node -14 -17 6 3.5 0 0 2.2461064041064852E-07 +0 1 83 Tree node -16 -18 1 6.5 0 0 1.1764125000309574E-07 +0 1 83 Tree node -1 18 5 1.5 0 0 5.7104231294091928E-08 +0 1 83 Tree node -19 -20 5 2.5 0 0 9.5319362218265565E-08 +0 1 83 Leaf node 7 5:-0.50005674428077129 +0 1 83 Leaf node 7 5:-0.50037826188766588 +0 1 83 Leaf node 7 5:-0.50064543702144515 +0 1 83 Leaf node 7 5:0.22306222579006935 +0 1 83 Leaf node 7 5:-0.28936436729341758 +0 1 83 Leaf node 7 5:0.45367433251131473 +0 1 83 Leaf node 7 5:0.50124956477800597 +0 1 83 Leaf node 7 5:0.50055243046544062 +0 1 83 Leaf node 7 5:-0.31481306076009691 +0 1 83 Leaf node 7 5:0.50009744107023091 +0 1 83 Leaf node 7 5:0.34047045558894101 +0 1 83 Leaf node 7 5:-0.50034545047249879 +0 1 83 Leaf node 7 5:-0.2678423118504748 +0 1 83 Leaf node 7 5:-0.39519752933218183 +0 1 83 Leaf node 7 5:0.1951712222843914 +0 1 83 Leaf node 7 5:0.5008360659419363 +0 1 83 Leaf node 7 5:0.50011733235577693 +0 1 83 Leaf node 7 5:0.11705251784120796 +0 1 83 Leaf node 7 5:0.21165281927784185 +0 1 83 Leaf node 7 5:-0.50000350670769567 +0 1 84 Tree node 2 1 4 5.5 0 0 9.8623811840821389E-08 +0 1 84 Tree node -2 11 3 2.5 0 0 4.493056431290799E-07 +0 1 84 Tree node 3 4 7 2.5 0 0 1.57194925602467E-07 +0 1 84 Tree node 5 -5 7 1.5 0 0 3.5603660292060862E-07 +0 1 84 Tree node -4 -6 0 6.5 0 0 1.4312174380935356E-07 +0 1 84 Tree node 6 -7 6 8.5 0 0 1.1423061673181555E-07 +0 1 84 Tree node 7 -8 6 5.5 0 0 3.2533121375555563E-07 +0 1 84 Tree node 8 -9 5 4.5 0 0 1.5631253931038355E-07 +0 1 84 Tree node 9 -10 4 3.5 0 0 6.480744792771216E-07 +0 1 84 Tree node 10 -11 3 3.5 0 0 1.3264907244299856E-07 +0 1 84 Tree node 16 -12 1 3.5 0 0 3.9544834940869827E-07 +0 1 84 Tree node 13 12 2 4.5 0 0 9.8597334054982166E-08 +0 1 84 Tree node -13 -14 2 5.5 0 0 3.1078348580046114E-07 +0 1 84 Tree node -3 14 0 4.5 0 0 1.2960619626715443E-07 +0 1 84 Tree node 15 -16 0 9.5 0 0 8.0712719405797963E-08 +0 1 84 Tree node -15 -17 2 2.5 0 0 1.1919847540360884E-07 +0 1 84 Tree node 18 17 3 1.5 0 0 5.0618961028119551E-08 +0 1 84 Tree node -18 -19 3 2.5 0 0 2.6291634379069903E-08 +0 1 84 Tree node -1 -20 5 1.5 0 0 2.3876697070824616E-08 +0 1 84 Leaf node 7 5:-0.50001782392788829 +0 1 84 Leaf node 7 5:0.2090930093400887 +0 1 84 Leaf node 7 5:0.50052230559997968 +0 1 84 Leaf node 7 5:0.27134781123562202 +0 1 84 Leaf node 7 5:-0.31217548027406605 +0 1 84 Leaf node 7 5:0.50005937788577259 +0 1 84 Leaf node 7 5:0.50046280567314583 +0 1 84 Leaf node 7 5:-0.25662028513318896 +0 1 84 Leaf node 7 5:-0.1468111717897288 +0 1 84 Leaf node 7 5:0.17577512912376878 +0 1 84 Leaf node 7 5:0.31657225266682837 +0 1 84 Leaf node 7 5:-0.48632214587391454 +0 1 84 Leaf node 7 5:-0.23503412281320268 +0 1 84 Leaf node 7 5:0.50016195127382979 +0 1 84 Leaf node 7 5:-0.30798385122775568 +0 1 84 Leaf node 7 5:0.50039202487807699 +0 1 84 Leaf node 7 5:0.50009226646481653 +0 1 84 Leaf node 7 5:-0.060298868218235643 +0 1 84 Leaf node 7 5:-0.50082332989686262 +0 1 84 Leaf node 7 5:-0.5005038451384799 +0 1 85 Tree node 7 1 6 2.5 0 0 1.0633390718415723E-07 +0 1 85 Tree node 2 4 2 1.5 0 0 2.8888849272687071E-07 +0 1 85 Tree node 3 -4 1 4.5 0 0 5.060459443049953E-07 +0 1 85 Tree node -2 -5 4 1.5 0 0 1.0719785183714886E-06 +0 1 85 Tree node -3 5 3 2.5 0 0 2.0100501798309196E-07 +0 1 85 Tree node 6 12 1 4.5 0 0 3.9098784152105638E-07 +0 1 85 Tree node -6 -8 2 6.5 0 0 7.1097555056280393E-07 +0 1 85 Tree node 16 8 3 1.5 0 0 1.020524894389669E-07 +0 1 85 Tree node 10 9 4 4.5 0 0 4.118806031293427E-07 +0 1 85 Tree node -10 14 1 2.5 0 0 1.9850411339799545E-06 +0 1 85 Tree node 11 -12 5 3.5 0 0 3.112506427159747E-07 +0 1 85 Tree node -9 -13 4 2.5 0 0 1.0382770871159951E-07 +0 1 85 Tree node -7 13 4 3.5 0 0 7.1973065059214647E-08 +0 1 85 Tree node -14 15 0 4.5 0 0 1.9965126496869657E-07 +0 1 85 Tree node 18 -16 0 7.5 0 0 5.3598785115962936E-08 +0 1 85 Tree node -15 -17 5 3.5 0 0 4.1513567022198326E-08 +0 1 85 Tree node -1 17 5 1.5 0 0 3.0094574500888323E-08 +0 1 85 Tree node -18 -19 5 2.5 0 0 6.3599028182751416E-08 +0 1 85 Tree node -11 -20 3 3.5 0 0 7.0683765031220676E-09 +0 1 85 Leaf node 7 5:-0.50006173266479959 +0 1 85 Leaf node 7 5:-0.32343120914465157 +0 1 85 Leaf node 7 5:0.28865381582966115 +0 1 85 Leaf node 7 5:-0.10338932297378564 +0 1 85 Leaf node 7 5:0.48902842090901433 +0 1 85 Leaf node 7 5:-0.36502244926413019 +0 1 85 Leaf node 7 5:-0.1507497800059322 +0 1 85 Leaf node 7 5:0.5004103301092433 +0 1 85 Leaf node 7 5:-0.41917102269991641 +0 1 85 Leaf node 7 5:-0.25944291241205819 +0 1 85 Leaf node 7 5:0.50044772416273131 +0 1 85 Leaf node 7 5:0.34129388525829768 +0 1 85 Leaf node 7 5:-0.20945672567711643 +0 1 85 Leaf node 7 5:0.50112847258686111 +0 1 85 Leaf node 7 5:0.50026304901472451 +0 1 85 Leaf node 7 5:-0.32361675903612513 +0 1 85 Leaf node 7 5:0.50007234688676061 +0 1 85 Leaf node 7 5:0.20157546579352167 +0 1 85 Leaf node 7 5:-0.50001208172209954 +0 1 85 Leaf node 7 5:0.087732178196726046 +0 1 86 Tree node 6 1 0 1.5 0 0 9.6938266128234783E-08 +0 1 86 Tree node 2 3 0 2.5 0 0 1.7123311997900054E-07 +0 1 86 Tree node -2 -4 4 2.5 0 0 7.7139515684502012E-07 +0 1 86 Tree node 4 5 1 2.5 0 0 1.5233103477604699E-07 +0 1 86 Tree node -3 -6 3 2.5 0 0 5.9195306249601815E-07 +0 1 86 Tree node -5 8 5 2.5 0 0 2.5358097239323928E-07 +0 1 86 Tree node 7 -8 6 2.5 0 0 1.2607164448670526E-07 +0 1 86 Tree node 14 -9 4 3.5 0 0 6.2718020724222526E-07 +0 1 86 Tree node -7 9 3 2.5 0 0 1.0524764609835391E-07 +0 1 86 Tree node 10 12 6 3.5 0 0 6.1048179188351255E-08 +0 1 86 Tree node -10 11 5 3.5 0 0 2.0928649450455056E-07 +0 1 86 Tree node -12 15 4 8.5 0 0 1.8451852396608367E-07 +0 1 86 Tree node 13 17 5 4.5 0 0 1.3960325583700501E-07 +0 1 86 Tree node -11 16 6 7.5 0 0 5.4939476061229463E-07 +0 1 86 Tree node 18 -16 3 1.5 0 0 4.5370725333399717E-08 +0 1 86 Tree node -13 -17 0 4.5 0 0 3.2194634818032757E-08 +0 1 86 Tree node -15 -18 7 2.5 0 0 2.8900490325430976E-08 +0 1 86 Tree node -14 -19 2 4.5 0 0 2.2977036953109358E-08 +0 1 86 Tree node -1 -20 5 2.5 0 0 2.0023542026519874E-08 +0 1 86 Leaf node 7 5:-0.50044106098035712 +0 1 86 Leaf node 7 5:0.026785099761221547 +0 1 86 Leaf node 7 5:-0.50216878871557225 +0 1 86 Leaf node 7 5:0.32638832324475436 +0 1 86 Leaf node 7 5:0.33368717776548174 +0 1 86 Leaf node 7 5:0.1802297002021713 +0 1 86 Leaf node 7 5:0.30010899097717958 +0 1 86 Leaf node 7 5:0.20385377016434209 +0 1 86 Leaf node 7 5:-0.17517577772739898 +0 1 86 Leaf node 7 5:0.26081626773397226 +0 1 86 Leaf node 7 5:-0.34830179695384722 +0 1 86 Leaf node 7 5:-0.14283910177671444 +0 1 86 Leaf node 7 5:0.50041492040472124 +0 1 86 Leaf node 7 5:0.50024536697531807 +0 1 86 Leaf node 7 5:0.5001657639794469 +0 1 86 Leaf node 7 5:-0.50017101951797582 +0 1 86 Leaf node 7 5:0.50013289828759089 +0 1 86 Leaf node 7 5:0.50001852870674934 +0 1 86 Leaf node 7 5:-0.13402204575577628 +0 1 86 Leaf node 7 5:-0.50000192412829092 +0 1 87 Tree node 16 1 3 1.5 0 0 7.4743088596029515E-08 +0 1 87 Tree node 2 3 0 1.5 0 0 1.0780736691677393E-07 +0 1 87 Tree node 17 -4 4 3.5 0 0 8.795777362800232E-07 +0 1 87 Tree node 4 7 2 1.5 0 0 1.3402357684021664E-07 +0 1 87 Tree node 5 -6 7 1.5 0 0 4.4402084783473152E-07 +0 1 87 Tree node 6 -7 6 3.5 0 0 2.3238660274033053E-07 +0 1 87 Tree node -3 -8 4 2.5 0 0 3.9071858215350057E-07 +0 1 87 Tree node 18 8 1 2.5 0 0 1.6690303282704655E-07 +0 1 87 Tree node -9 9 3 2.5 0 0 1.0026509189088029E-07 +0 1 87 Tree node 10 12 1 4.5 0 0 1.3852798309290869E-07 +0 1 87 Tree node 11 -12 2 5.5 0 0 2.3358552299105462E-07 +0 1 87 Tree node -10 -13 2 3.5 0 0 1.9718949483683027E-07 +0 1 87 Tree node 13 14 1 5.5 0 0 5.823055213500791E-08 +0 1 87 Tree node -11 -15 6 3.5 0 0 1.0932185146964405E-07 +0 1 87 Tree node -14 15 3 3.5 0 0 5.2929287094596592E-08 +0 1 87 Tree node -16 -17 1 6.5 0 0 8.0348176500209526E-08 +0 1 87 Tree node -1 -18 5 1.5 0 0 3.5227493732257485E-08 +0 1 87 Tree node -2 -19 6 1.5 0 0 2.2232667457279099E-08 +0 1 87 Tree node -5 -20 2 2.5 0 0 1.6052314903649521E-08 +0 1 87 Leaf node 7 5:-0.50003452859927944 +0 1 87 Leaf node 7 5:-0.50014371259025503 +0 1 87 Leaf node 7 5:-0.50031718359333577 +0 1 87 Leaf node 7 5:-0.27133146495451005 +0 1 87 Leaf node 7 5:-0.50033013292696649 +0 1 87 Leaf node 7 5:0.23027915846350855 +0 1 87 Leaf node 7 5:-0.22533531694226122 +0 1 87 Leaf node 7 5:0.40390304870154192 +0 1 87 Leaf node 7 5:0.26989998762107648 +0 1 87 Leaf node 7 5:-0.087291767039752513 +0 1 87 Leaf node 7 5:0.50098673740380284 +0 1 87 Leaf node 7 5:0.50032821304886865 +0 1 87 Leaf node 7 5:-0.24557365678812618 +0 1 87 Leaf node 7 5:-0.27343625673696448 +0 1 87 Leaf node 7 5:-0.027212478945688651 +0 1 87 Leaf node 7 5:0.50068290590305087 +0 1 87 Leaf node 7 5:0.1050286695577401 +0 1 87 Leaf node 7 5:0.21983322709316741 +0 1 87 Leaf node 7 5:-0.50032963431174793 +0 1 87 Leaf node 7 5:-0.41449885681255144 +0 1 88 Tree node 2 1 6 2.5 0 0 7.032189524298391E-08 +0 1 88 Tree node -2 9 0 2.5 0 0 1.924690560109403E-07 +0 1 88 Tree node 4 3 0 3.5 0 0 1.0694063460950355E-07 +0 1 88 Tree node -4 8 0 4.5 0 0 6.9747534143405728E-07 +0 1 88 Tree node 5 -6 7 1.5 0 0 4.2449690056575676E-07 +0 1 88 Tree node 6 -7 5 4.5 0 0 1.0943159726414383E-07 +0 1 88 Tree node 7 -8 0 2.5 0 0 1.1916131782968988E-07 +0 1 88 Tree node 17 -9 4 3.5 0 0 1.6776876167309958E-07 +0 1 88 Tree node -5 -10 0 7.5 0 0 5.9508278357900559E-08 +0 1 88 Tree node -3 10 3 2.5 0 0 5.3568420227521052E-08 +0 1 88 Tree node 13 11 2 2.5 0 0 7.2628457942665095E-08 +0 1 88 Tree node 12 15 1 4.5 0 0 2.8144436812484241E-07 +0 1 88 Tree node -12 -14 2 6.5 0 0 6.571274426131276E-07 +0 1 88 Tree node -11 14 1 4.5 0 0 1.6019864727698897E-07 +0 1 88 Tree node -15 -16 6 7.5 0 0 1.7844189377970419E-07 +0 1 88 Tree node -13 16 2 3.5 0 0 7.608040911280191E-08 +0 1 88 Tree node -17 -18 0 4.5 0 0 2.5294963516393587E-07 +0 1 88 Tree node 18 -19 3 1.5 0 0 3.2552768212930497E-08 +0 1 88 Tree node -1 -20 5 1.5 0 0 1.5807761892467773E-08 +0 1 88 Leaf node 7 5:-0.5000001545848678 +0 1 88 Leaf node 7 5:0.2932977081763174 +0 1 88 Leaf node 7 5:0.21159518360533264 +0 1 88 Leaf node 7 5:-0.39731111898523563 +0 1 88 Leaf node 7 5:0.50020905773664837 +0 1 88 Leaf node 7 5:0.33439936572410583 +0 1 88 Leaf node 7 5:-0.32465326040384052 +0 1 88 Leaf node 7 5:0.40463840872347134 +0 1 88 Leaf node 7 5:-0.1081419662064592 +0 1 88 Leaf node 7 5:-0.22331452975735031 +0 1 88 Leaf node 7 5:0.47451884394670329 +0 1 88 Leaf node 7 5:-0.44122524078461445 +0 1 88 Leaf node 7 5:-0.25090234330601208 +0 1 88 Leaf node 7 5:0.50023403343496697 +0 1 88 Leaf node 7 5:-0.22217229431315938 +0 1 88 Leaf node 7 5:0.50051828899011552 +0 1 88 Leaf node 7 5:0.50077848287677529 +0 1 88 Leaf node 7 5:0.50006734480098169 +0 1 88 Leaf node 7 5:-0.50027161241242113 +0 1 88 Leaf node 7 5:-0.50039928878651418 +0 1 89 Tree node 9 1 3 1.5 0 0 7.4584265482924211E-08 +0 1 89 Tree node 2 5 6 2.5 0 0 7.2963195054474842E-08 +0 1 89 Tree node 7 3 4 3.5 0 0 1.9057945605888095E-07 +0 1 89 Tree node -4 4 0 1.5 0 0 1.5055391860433347E-06 +0 1 89 Tree node -5 6 2 3.5 0 0 1.9302831642249282E-07 +0 1 89 Tree node -3 11 0 2.5 0 0 1.3583240283859078E-07 +0 1 89 Tree node -6 -8 5 5.5 0 0 5.7047369972336433E-08 +0 1 89 Tree node 8 -9 2 3.5 0 0 4.4684231528250082E-08 +0 1 89 Tree node -2 -10 0 2.5 0 0 3.5238879868090802E-08 +0 1 89 Tree node -1 10 5 1.5 0 0 3.4698296049042902E-08 +0 1 89 Tree node -11 -12 5 2.5 0 0 7.047526315874291E-08 +0 1 89 Tree node -7 12 3 2.5 0 0 3.4237929058261255E-08 +0 1 89 Tree node 15 13 2 2.5 0 0 4.9145838118642711E-08 +0 1 89 Tree node 14 17 1 4.5 0 0 1.8690092123225219E-07 +0 1 89 Tree node -14 -16 2 6.5 0 0 4.365681160807792E-07 +0 1 89 Tree node -13 16 1 4.5 0 0 1.057008857143762E-07 +0 1 89 Tree node -17 -18 3 5.5 0 0 1.2463488035067682E-07 +0 1 89 Tree node -15 18 2 3.5 0 0 4.9814232997043746E-08 +0 1 89 Tree node -19 -20 0 4.5 0 0 1.6954704732250495E-07 +0 1 89 Leaf node 7 5:-0.50003245790199657 +0 1 89 Leaf node 7 5:-0.18995237471715695 +0 1 89 Leaf node 7 5:0.25459021329988341 +0 1 89 Leaf node 7 5:-0.50110249790175043 +0 1 89 Leaf node 7 5:0.17710893564309446 +0 1 89 Leaf node 7 5:-0.20937047837335676 +0 1 89 Leaf node 7 5:0.17563096119046076 +0 1 89 Leaf node 7 5:0.50008851260192022 +0 1 89 Leaf node 7 5:0.48974221720774613 +0 1 89 Leaf node 7 5:-0.29786158925178541 +0 1 89 Leaf node 7 5:0.23707057847969204 +0 1 89 Leaf node 7 5:-0.50000987415882137 +0 1 89 Leaf node 7 5:0.46295495308988482 +0 1 89 Leaf node 7 5:-0.41792424643995624 +0 1 89 Leaf node 7 5:-0.21140530190295501 +0 1 89 Leaf node 7 5:0.5001916096371678 +0 1 89 Leaf node 7 5:-0.26872556501942979 +0 1 89 Leaf node 7 5:0.28707552652158957 +0 1 89 Leaf node 7 5:0.50063725502973 +0 1 89 Leaf node 7 5:0.50005513673436852 +0 1 90 Tree node 8 1 4 2.5 0 0 6.0705385390164452E-08 +0 1 90 Tree node 4 2 0 3.5 0 0 2.7122531451769137E-07 +0 1 90 Tree node 3 10 1 4.5 0 0 1.8767150110076609E-07 +0 1 90 Tree node -3 7 4 7.5 0 0 4.5851320815781186E-07 +0 1 90 Tree node 5 -6 7 1.5 0 0 1.4906865973297329E-07 +0 1 90 Tree node -2 6 5 1.5 0 0 4.6943405113964637E-07 +0 1 90 Tree node -7 -8 2 2.5 0 0 3.3780933621613899E-07 +0 1 90 Tree node -5 -9 2 4.5 0 0 6.8262997423848811E-08 +0 1 90 Tree node 9 -10 0 4.5 0 0 5.419297466797672E-08 +0 1 90 Tree node 12 11 6 2.5 0 0 1.0566671664538702E-07 +0 1 90 Tree node -4 14 0 4.5 0 0 5.3023256900028947E-08 +0 1 90 Tree node -11 -13 3 2.5 0 0 4.5919011071600981E-08 +0 1 90 Tree node -1 13 3 1.5 0 0 2.3369283908439663E-08 +0 1 90 Tree node -14 -15 0 2.5 0 0 2.3142742189944413E-08 +0 1 90 Tree node 17 15 0 6.5 0 0 1.9238840075423949E-08 +0 1 90 Tree node 16 18 1 7.5 0 0 4.6417558936504102E-08 +0 1 90 Tree node -16 -18 6 5.5 0 0 1.1006080382367731E-07 +0 1 90 Tree node -12 -19 2 3.5 0 0 3.3752622695025364E-08 +0 1 90 Tree node -17 -20 6 4.5 0 0 2.4328217765725802E-08 +0 1 90 Leaf node 7 5:-0.50035513095454698 +0 1 90 Leaf node 7 5:-0.5004243206374952 +0 1 90 Leaf node 7 5:-0.35650193333408853 +0 1 90 Leaf node 7 5:0.3399281269163883 +0 1 90 Leaf node 7 5:0.50020166779829511 +0 1 90 Leaf node 7 5:0.34853403155822099 +0 1 90 Leaf node 7 5:0.0296134893966635 +0 1 90 Leaf node 7 5:0.27114919419572003 +0 1 90 Leaf node 7 5:-0.32226813644791374 +0 1 90 Leaf node 7 5:0.5003638338429589 +0 1 90 Leaf node 7 5:-0.50031860468345979 +0 1 90 Leaf node 7 5:0.50024802504541321 +0 1 90 Leaf node 7 5:-0.06516299833957688 +0 1 90 Leaf node 7 5:-0.50011319467746929 +0 1 90 Leaf node 7 5:-0.50021198893896845 +0 1 90 Leaf node 7 5:-0.47600902637623205 +0 1 90 Leaf node 7 5:0.500199869403398 +0 1 90 Leaf node 7 5:0.50005847192328245 +0 1 90 Leaf node 7 5:0.50000964216739319 +0 1 90 Leaf node 7 5:-0.22113503908123691 +0 1 91 Tree node 17 1 3 1.5 0 0 4.4504471298019954E-08 +0 1 91 Tree node 2 4 4 5.5 0 0 5.2893076352733901E-08 +0 1 91 Tree node 8 3 4 4.5 0 0 2.4263824548846505E-07 +0 1 91 Tree node -4 -5 1 4.5 0 0 9.1741724330055489E-07 +0 1 91 Tree node -3 5 4 7.5 0 0 1.5032199195842562E-07 +0 1 91 Tree node 6 7 5 3.5 0 0 1.2399116156504532E-07 +0 1 91 Tree node -6 -8 1 4.5 0 0 1.9336235459981351E-07 +0 1 91 Tree node -7 18 0 3.5 0 0 1.4516880880915952E-07 +0 1 91 Tree node 9 -10 7 3.5 0 0 5.2864963354347156E-08 +0 1 91 Tree node 10 -11 3 6.5 0 0 5.5643956618755779E-08 +0 1 91 Tree node 11 -12 3 4.5 0 0 1.6189149408919818E-07 +0 1 91 Tree node 12 -13 1 4.5 0 0 5.1401100638708003E-08 +0 1 91 Tree node 13 -14 5 3.5 0 0 6.6644779411212054E-08 +0 1 91 Tree node 15 14 4 2.5 0 0 4.936279467821135E-08 +0 1 91 Tree node -15 -16 5 2.5 0 0 1.2358066696007413E-07 +0 1 91 Tree node -2 16 6 1.5 0 0 2.6475850468141918E-08 +0 1 91 Tree node -17 -18 1 1.5 0 0 4.1768271972088194E-08 +0 1 91 Tree node -1 -19 5 1.5 0 0 2.0949809689146536E-08 +0 1 91 Tree node -9 -20 0 4.5 0 0 1.8159970146215028E-08 +0 1 91 Leaf node 7 5:-0.50003134612097855 +0 1 91 Leaf node 7 5:-0.5000422435283306 +0 1 91 Leaf node 7 5:0.34626576985034635 +0 1 91 Leaf node 7 5:-0.37777995278660342 +0 1 91 Leaf node 7 5:0.50040241790982609 +0 1 91 Leaf node 7 5:-0.25310173747131864 +0 1 91 Leaf node 7 5:0.5007402545927504 +0 1 91 Leaf node 7 5:0.066276847244085002 +0 1 91 Leaf node 7 5:0.50020231799863601 +0 1 91 Leaf node 7 5:0.33114476544856714 +0 1 91 Leaf node 7 5:-0.45138061850374744 +0 1 91 Leaf node 7 5:0.48260761094832944 +0 1 91 Leaf node 7 5:-0.26827821014508429 +0 1 91 Leaf node 7 5:0.18973194225976603 +0 1 91 Leaf node 7 5:-0.5004174200464524 +0 1 91 Leaf node 7 5:0.099413688835052083 +0 1 91 Leaf node 7 5:-0.50005773975093526 +0 1 91 Leaf node 7 5:-0.50027433958328793 +0 1 91 Leaf node 7 5:0.21445069275517786 +0 1 91 Leaf node 7 5:-0.0076066263168828099 +0 1 92 Tree node 8 1 4 2.5 0 0 3.9487375896676879E-08 +0 1 92 Tree node 2 5 0 3.5 0 0 1.5618765419120981E-07 +0 1 92 Tree node 3 -4 2 4.5 0 0 1.6486902092079148E-07 +0 1 92 Tree node 4 -5 2 2.5 0 0 2.6688830431002112E-07 +0 1 92 Tree node -2 -6 0 1.5 0 0 5.2884886907212607E-07 +0 1 92 Tree node 6 11 1 4.5 0 0 8.2537416119474441E-08 +0 1 92 Tree node -3 7 4 7.5 0 0 2.4133108832109085E-07 +0 1 92 Tree node -8 -9 2 4.5 0 0 4.1985763076299017E-08 +0 1 92 Tree node 9 17 2 3.5 0 0 3.3597503574767419E-08 +0 1 92 Tree node -1 10 2 2.5 0 0 9.9775036110798371E-08 +0 1 92 Tree node -11 -12 5 1.5 0 0 1.0339702632864259E-07 +0 1 92 Tree node 14 12 0 6.5 0 0 2.6657119106598316E-08 +0 1 92 Tree node 13 18 1 7.5 0 0 3.1754158648216479E-08 +0 1 92 Tree node -13 -15 6 5.5 0 0 7.6010625802708581E-08 +0 1 92 Tree node 15 -16 3 5.5 0 0 2.5566234178370938E-08 +0 1 92 Tree node -7 16 4 9.5 0 0 3.0414162595330756E-08 +0 1 92 Tree node -17 -18 5 4.5 0 0 1.7806351620078801E-08 +0 1 92 Tree node -10 -19 6 4.5 0 0 1.5447916074247633E-08 +0 1 92 Tree node -14 -20 6 4.5 0 0 1.4822901292423676E-08 +0 1 92 Leaf node 7 5:-0.20089400622094059 +0 1 92 Leaf node 7 5:-0.21083496375361926 +0 1 92 Leaf node 7 5:-0.30869347941371339 +0 1 92 Leaf node 7 5:-0.15650189919699342 +0 1 92 Leaf node 7 5:0.3183482299269394 +0 1 92 Leaf node 7 5:0.42777608745636486 +0 1 92 Leaf node 7 5:0.50032232329175663 +0 1 92 Leaf node 7 5:0.50017858880328381 +0 1 92 Leaf node 7 5:-0.25134102865086422 +0 1 92 Leaf node 7 5:0.49126266480261721 +0 1 92 Leaf node 7 5:-0.50000004889385041 +0 1 92 Leaf node 7 5:-0.47328770938200132 +0 1 92 Leaf node 7 5:-0.46631935912354316 +0 1 92 Leaf node 7 5:0.50016077994101737 +0 1 92 Leaf node 7 5:0.50004766341008622 +0 1 92 Leaf node 7 5:-0.28472916764968981 +0 1 92 Leaf node 7 5:0.50017913244734236 +0 1 92 Leaf node 7 5:0.50000395825656774 +0 1 92 Leaf node 7 5:0.50002914772126483 +0 1 92 Leaf node 7 5:-0.18878119704474811 +0 1 93 Tree node 12 1 4 1.5 0 0 2.7552343242359659E-08 +0 1 93 Tree node 3 2 2 4.5 0 0 4.3124824753396414E-08 +0 1 93 Tree node -3 15 5 3.5 0 0 1.5103980495792349E-07 +0 1 93 Tree node 4 7 6 1.5 0 0 5.514735626600018E-08 +0 1 93 Tree node 5 -6 2 2.5 0 0 2.1815509683094828E-07 +0 1 93 Tree node 6 -7 0 1.5 0 0 1.9694094425434933E-07 +0 1 93 Tree node -2 -8 4 3.5 0 0 2.9801039295294515E-07 +0 1 93 Tree node 8 10 3 3.5 0 0 1.3398546956766094E-07 +0 1 93 Tree node 9 -10 1 5.5 0 0 1.3991821167363765E-07 +0 1 93 Tree node -5 -11 4 3.5 0 0 8.883591310458659E-08 +0 1 93 Tree node -9 11 6 4.5 0 0 5.889075845043163E-08 +0 1 93 Tree node -12 14 2 2.5 0 0 5.0789794230138945E-08 +0 1 93 Tree node 13 -14 2 3.5 0 0 2.5211569110402429E-08 +0 1 93 Tree node 16 -15 5 3.5 0 0 7.9219376700682148E-08 +0 1 93 Tree node -13 -16 1 5.5 0 0 2.049529534762803E-08 +0 1 93 Tree node -4 18 6 1.5 0 0 1.6895784900971111E-08 +0 1 93 Tree node -1 17 3 1.5 0 0 1.4554267574165872E-08 +0 1 93 Tree node -18 -19 0 2.5 0 0 9.3423771082212253E-09 +0 1 93 Tree node -17 -20 3 3.5 0 0 7.6331749170403796E-09 +0 1 93 Leaf node 7 5:-0.5002532847410921 +0 1 93 Leaf node 7 5:-0.50002577193820785 +0 1 93 Leaf node 7 5:-0.34621832998859781 +0 1 93 Leaf node 7 5:0.50021620364087938 +0 1 93 Leaf node 7 5:0.24246981409566692 +0 1 93 Leaf node 7 5:0.28418995479820208 +0 1 93 Leaf node 7 5:0.084489793945398337 +0 1 93 Leaf node 7 5:-0.50081582279889769 +0 1 93 Leaf node 7 5:-0.26276621939305417 +0 1 93 Leaf node 7 5:-0.013227502050724978 +0 1 93 Leaf node 7 5:0.50055627435726346 +0 1 93 Leaf node 7 5:0.50029013949578549 +0 1 93 Leaf node 7 5:-0.1961466794214809 +0 1 93 Leaf node 7 5:0.48764511271960748 +0 1 93 Leaf node 7 5:-0.39531589317642896 +0 1 93 Leaf node 7 5:0.50013853358795235 +0 1 93 Leaf node 7 5:0.50020264027066685 +0 1 93 Leaf node 7 5:-0.50009540119554652 +0 1 93 Leaf node 7 5:-0.20139119591646898 +0 1 93 Leaf node 7 5:-0.078838282973367396 +0 1 94 Tree node 2 1 4 5.5 0 0 2.4998463710383089E-08 +0 1 94 Tree node -2 9 3 2.5 0 0 8.0021726465431779E-08 +0 1 94 Tree node 3 8 0 4.5 0 0 3.8627196530472239E-08 +0 1 94 Tree node 4 -5 0 3.5 0 0 2.0403330968655699E-07 +0 1 94 Tree node 6 5 4 3.5 0 0 4.7859664776284664E-08 +0 1 94 Tree node -6 -7 1 1.5 0 0 3.7086275482250245E-08 +0 1 94 Tree node 7 -8 1 3.5 0 0 2.3754135442972478E-08 +0 1 94 Tree node 14 -9 6 2.5 0 0 4.7389777153653234E-08 +0 1 94 Tree node -4 16 0 7.5 0 0 1.9638740416186966E-08 +0 1 94 Tree node 11 10 2 4.5 0 0 1.7890625809736304E-08 +0 1 94 Tree node -11 18 2 5.5 0 0 4.6334523144211503E-08 +0 1 94 Tree node 12 13 3 5.5 0 0 1.795881401559105E-08 +0 1 94 Tree node -3 -14 3 4.5 0 0 2.2555476123080801E-08 +0 1 94 Tree node -13 -15 5 6.5 0 0 1.5758650093696194E-08 +0 1 94 Tree node -1 15 2 2.5 0 0 1.1859763133109748E-08 +0 1 94 Tree node -16 -17 5 1.5 0 0 2.0811567998180897E-08 +0 1 94 Tree node -10 17 4 2.5 0 0 1.1598676476832006E-08 +0 1 94 Tree node -18 -19 4 4.5 0 0 5.1749875602895935E-09 +0 1 94 Tree node -12 -20 0 4.5 0 0 4.9182535064251092E-09 +0 1 94 Leaf node 7 5:-0.32447019713388769 +0 1 94 Leaf node 7 5:0.20519275466039361 +0 1 94 Leaf node 7 5:0.50007270624781985 +0 1 94 Leaf node 7 5:0.50016874385053556 +0 1 94 Leaf node 7 5:-0.20775246430004152 +0 1 94 Leaf node 7 5:0.17745794154717812 +0 1 94 Leaf node 7 5:0.0072883166506370421 +0 1 94 Leaf node 7 5:0.39145164593927989 +0 1 94 Leaf node 7 5:-0.25667575485429917 +0 1 94 Leaf node 7 5:0.50035327259244378 +0 1 94 Leaf node 7 5:-0.2039215160620593 +0 1 94 Leaf node 7 5:0.50013332794258136 +0 1 94 Leaf node 7 5:0.50021307582609476 +0 1 94 Leaf node 7 5:-0.29309562586985011 +0 1 94 Leaf node 7 5:0.50007505771747107 +0 1 94 Leaf node 7 5:-0.50000287254500553 +0 1 94 Leaf node 7 5:-0.50031456052772649 +0 1 94 Leaf node 7 5:-0.30830458408394573 +0 1 94 Leaf node 7 5:0.50005571364016976 +0 1 94 Leaf node 7 5:0.50001518919984722 +0 1 95 Tree node 3 1 5 3.5 0 0 2.4064171250306823E-08 +0 1 95 Tree node 2 8 2 1.5 0 0 1.2284838740101199E-07 +0 1 95 Tree node -2 -4 3 2.5 0 0 1.0943760835343147E-07 +0 1 95 Tree node 4 -5 4 9.5 0 0 4.4967922078482722E-08 +0 1 95 Tree node 5 -6 4 5.5 0 0 9.2147169978089791E-08 +0 1 95 Tree node 6 -7 0 3.5 0 0 5.2769470141000756E-08 +0 1 95 Tree node 7 -8 4 3.5 0 0 5.3065843691512457E-08 +0 1 95 Tree node 15 -9 6 2.5 0 0 2.3182639296780344E-08 +0 1 95 Tree node 9 10 4 2.5 0 0 1.8176487981737609E-08 +0 1 95 Tree node -3 -11 2 3.5 0 0 7.2734880461683603E-08 +0 1 95 Tree node 11 12 3 3.5 0 0 5.4943648448480572E-08 +0 1 95 Tree node -10 17 4 8.5 0 0 6.0007547544614698E-08 +0 1 95 Tree node 13 14 4 7.5 0 0 3.789205738628137E-08 +0 1 95 Tree node -12 -15 0 6.5 0 0 1.2356679013800408E-07 +0 1 95 Tree node -14 16 0 4.5 0 0 1.275742795557454E-08 +0 1 95 Tree node 18 -17 3 1.5 0 0 7.0945238836714973E-09 +0 1 95 Tree node -16 -18 4 8.5 0 0 4.6657626991895159E-09 +0 1 95 Tree node -13 -19 5 6.5 0 0 3.8433227703429624E-09 +0 1 95 Tree node -1 -20 5 1.5 0 0 3.5575683090975748E-09 +0 1 95 Leaf node 7 5:-0.50001981301819687 +0 1 95 Leaf node 7 5:-0.0026488382120977124 +0 1 95 Leaf node 7 5:-0.34867122936565498 +0 1 95 Leaf node 7 5:0.50043260345368235 +0 1 95 Leaf node 7 5:-0.17735778541987804 +0 1 95 Leaf node 7 5:0.27018278308036503 +0 1 95 Leaf node 7 5:-0.18135568963780183 +0 1 95 Leaf node 7 5:0.12132846132017647 +0 1 95 Leaf node 7 5:-0.30631767699634133 +0 1 95 Leaf node 7 5:0.50032841578357279 +0 1 95 Leaf node 7 5:0.50028352651928543 +0 1 95 Leaf node 7 5:-0.42492983939029111 +0 1 95 Leaf node 7 5:0.50021192296086803 +0 1 95 Leaf node 7 5:0.50013511134829303 +0 1 95 Leaf node 7 5:0.50005003110007262 +0 1 95 Leaf node 7 5:-0.33081828121978712 +0 1 95 Leaf node 7 5:-0.29235057804061437 +0 1 95 Leaf node 7 5:0.50004177242720715 +0 1 95 Leaf node 7 5:0.50000225983329027 +0 1 95 Leaf node 7 5:-0.50016640036719173 +0 1 96 Tree node 6 1 3 1.5 0 0 2.2318282253877449E-08 +0 1 96 Tree node 2 8 1 2.5 0 0 3.1513039041555034E-08 +0 1 96 Tree node 3 -4 4 4.5 0 0 1.4871180498254495E-07 +0 1 96 Tree node 5 4 2 1.5 0 0 2.3196484755131828E-08 +0 1 96 Tree node -5 -6 4 1.5 0 0 6.4276349141750336E-08 +0 1 96 Tree node -2 -7 0 2.5 0 0 2.0576386485026255E-08 +0 1 96 Tree node -1 7 5 1.5 0 0 9.9624265821995369E-09 +0 1 96 Tree node -8 -9 5 2.5 0 0 2.0705336048473962E-08 +0 1 96 Tree node -3 9 5 1.5 0 0 8.6947406342206335E-09 +0 1 96 Tree node 10 12 0 3.5 0 0 1.8621943247159752E-08 +0 1 96 Tree node -10 11 4 2.5 0 0 6.9467354293861732E-08 +0 1 96 Tree node -12 -13 4 6.5 0 0 7.3037119864063976E-08 +0 1 96 Tree node -11 13 3 2.5 0 0 1.1477590266317004E-08 +0 1 96 Tree node 14 15 0 4.5 0 0 1.2621175119724309E-08 +0 1 96 Tree node -14 -16 3 5.5 0 0 4.2385442240695735E-08 +0 1 96 Tree node -15 16 4 1.5 0 0 9.4278866837936247E-09 +0 1 96 Tree node -17 17 1 6.5 0 0 1.0132254870670809E-08 +0 1 96 Tree node 18 -19 1 7.5 0 0 1.1775817237449643E-08 +0 1 96 Tree node -18 -20 6 6.5 0 0 3.4063288096737414E-08 +0 1 96 Leaf node 7 5:-0.50001374141717347 +0 1 96 Leaf node 7 5:-0.50011950522050685 +0 1 96 Leaf node 7 5:-0.31521439671905876 +0 1 96 Leaf node 7 5:-0.15847195128530106 +0 1 96 Leaf node 7 5:-0.50015131255900513 +0 1 96 Leaf node 7 5:-0.48514168471497199 +0 1 96 Leaf node 7 5:0.29477107337870595 +0 1 96 Leaf node 7 5:0.24931692186427518 +0 1 96 Leaf node 7 5:-0.50000513880043795 +0 1 96 Leaf node 7 5:-0.23862375224438651 +0 1 96 Leaf node 7 5:0.25569783882889424 +0 1 96 Leaf node 7 5:0.32249439638249638 +0 1 96 Leaf node 7 5:0.13563009683493693 +0 1 96 Leaf node 7 5:0.066577874168635948 +0 1 96 Leaf node 7 5:0.50033684089729713 +0 1 96 Leaf node 7 5:-0.23613871616789692 +0 1 96 Leaf node 7 5:0.5001048139109473 +0 1 96 Leaf node 7 5:-0.40139257552076635 +0 1 96 Leaf node 7 5:0.19356393961429894 +0 1 96 Leaf node 7 5:0.50004039106774245 +0 1 97 Tree node 2 1 4 5.5 0 0 1.6877572435138917E-08 +0 1 97 Tree node -2 11 3 2.5 0 0 5.9930194337119825E-08 +0 1 97 Tree node 3 4 7 2.5 0 0 2.315088070850296E-08 +0 1 97 Tree node 5 -5 7 1.5 0 0 4.4250790040958102E-08 +0 1 97 Tree node -4 -6 0 6.5 0 0 2.1458216423838454E-08 +0 1 97 Tree node 16 6 3 1.5 0 0 2.0999681230847958E-08 +0 1 97 Tree node 8 7 4 4.5 0 0 5.5617374355241576E-08 +0 1 97 Tree node -8 -9 6 3.5 0 0 9.4324436392832796E-08 +0 1 97 Tree node 9 10 3 4.5 0 0 1.7496498372227332E-08 +0 1 97 Tree node -7 -11 0 3.5 0 0 2.8174991262757485E-08 +0 1 97 Tree node -10 -12 6 5.5 0 0 1.6115315377992569E-08 +0 1 97 Tree node 12 13 6 1.5 0 0 1.2392805242166598E-08 +0 1 97 Tree node -3 -14 5 5.5 0 0 1.3457293693674866E-08 +0 1 97 Tree node 15 14 2 4.5 0 0 1.142418524783015E-08 +0 1 97 Tree node -15 -16 2 5.5 0 0 6.193430884564962E-08 +0 1 97 Tree node 18 -17 0 6.5 0 0 8.8309059665216196E-09 +0 1 97 Tree node 17 -18 5 2.5 0 0 6.0012583738537737E-09 +0 1 97 Tree node -1 -19 5 1.5 0 0 1.3141785000535894E-08 +0 1 97 Tree node -13 -20 5 4.5 0 0 4.8999983442954278E-09 +0 1 97 Leaf node 7 5:-0.50000346510442384 +0 1 97 Leaf node 7 5:0.19329952269616671 +0 1 97 Leaf node 7 5:0.50017557655779477 +0 1 97 Leaf node 7 5:0.30124536500048771 +0 1 97 Leaf node 7 5:-0.29723829707419508 +0 1 97 Leaf node 7 5:0.50001472270725811 +0 1 97 Leaf node 7 5:-0.1617764292136662 +0 1 97 Leaf node 7 5:-0.36038299946729574 +0 1 97 Leaf node 7 5:0.061140973203705859 +0 1 97 Leaf node 7 5:0.461793638549366 +0 1 97 Leaf node 7 5:-0.2483951840376678 +0 1 97 Leaf node 7 5:-0.027241200459365601 +0 1 97 Leaf node 7 5:0.50011723301208333 +0 1 97 Leaf node 7 5:0.50000782109289976 +0 1 97 Leaf node 7 5:-0.33523455508818295 +0 1 97 Leaf node 7 5:0.50008403561336767 +0 1 97 Leaf node 7 5:-0.068121513197205052 +0 1 97 Leaf node 7 5:-0.50000420728705308 +0 1 97 Leaf node 7 5:0.20984583965375234 +0 1 97 Leaf node 7 5:0.50005758800369604 +0 1 98 Tree node 5 1 6 2.5 0 0 1.676569765600765E-08 +0 1 98 Tree node -2 2 0 2.5 0 0 5.3569738911815594E-08 +0 1 98 Tree node -3 3 3 2.5 0 0 1.4924699397380577E-08 +0 1 98 Tree node -4 4 0 3.5 0 0 2.1648590419928613E-08 +0 1 98 Tree node -5 10 1 3.5 0 0 2.3402291718219022E-08 +0 1 98 Tree node 17 6 3 1.5 0 0 1.4848673502910349E-08 +0 1 98 Tree node 8 7 4 4.5 0 0 5.4435595352696325E-08 +0 1 98 Tree node -8 15 1 2.5 0 0 2.6064265657171911E-07 +0 1 98 Tree node 9 -10 5 3.5 0 0 3.8324156936950322E-08 +0 1 98 Tree node -7 -11 4 2.5 0 0 2.2446271236306693E-08 +0 1 98 Tree node 11 14 0 8.5 0 0 9.8508956597380473E-09 +0 1 98 Tree node 12 13 2 3.5 0 0 2.6538408178814426E-08 +0 1 98 Tree node -6 -14 4 9.5 0 0 6.7708864254988361E-08 +0 1 98 Tree node -13 -15 4 4.5 0 0 1.8451077848205352E-08 +0 1 98 Tree node -12 -16 5 4.5 0 0 1.7893538007448652E-08 +0 1 98 Tree node -9 16 2 3.5 0 0 4.8510502969571526E-09 +0 1 98 Tree node -17 -18 3 4.5 0 0 9.4200936057043808E-09 +0 1 98 Tree node -1 18 5 1.5 0 0 3.9950234778740002E-09 +0 1 98 Tree node -19 -20 5 2.5 0 0 8.6576726300250783E-09 +0 1 98 Leaf node 7 5:-0.50001502851539414 +0 1 98 Leaf node 7 5:0.30534306726931743 +0 1 98 Leaf node 7 5:0.20110357717674848 +0 1 98 Leaf node 7 5:-0.19848676133486526 +0 1 98 Leaf node 7 5:0.50020451976867286 +0 1 98 Leaf node 7 5:-0.39620591283114126 +0 1 98 Leaf node 7 5:-0.4381906236604402 +0 1 98 Leaf node 7 5:-0.25815530391240216 +0 1 98 Leaf node 7 5:0.50012762689755619 +0 1 98 Leaf node 7 5:0.32019056000426999 +0 1 98 Leaf node 7 5:-0.23577138472486453 +0 1 98 Leaf node 7 5:0.50021179149559236 +0 1 98 Leaf node 7 5:0.50015619179926163 +0 1 98 Leaf node 7 5:0.5000978234780793 +0 1 98 Leaf node 7 5:-0.083026108048373334 +0 1 98 Leaf node 7 5:0.50003774820097247 +0 1 98 Leaf node 7 5:0.50012943694121237 +0 1 98 Leaf node 7 5:-0.22335979270150108 +0 1 98 Leaf node 7 5:0.17398594449161725 +0 1 98 Leaf node 7 5:-0.50000344463067226 +0 1 99 Tree node 6 1 4 2.5 0 0 1.2432949371334979E-08 +0 1 99 Tree node 2 5 0 3.5 0 0 4.0743738078627011E-08 +0 1 99 Tree node 3 -4 7 1.5 0 0 6.1566513613659533E-08 +0 1 99 Tree node 4 -5 6 2.5 0 0 7.870819167250575E-08 +0 1 99 Tree node -2 -6 2 1.5 0 0 1.3090061336440947E-07 +0 1 99 Tree node -3 9 3 2.5 0 0 2.4251648116491026E-08 +0 1 99 Tree node 7 -8 0 4.5 0 0 1.5032576169008183E-08 +0 1 99 Tree node 12 8 6 2.5 0 0 1.6491538176824472E-08 +0 1 99 Tree node -9 -10 3 2.5 0 0 1.831007411191046E-08 +0 1 99 Tree node 10 11 3 3.5 0 0 1.1542697347778237E-08 +0 1 99 Tree node -7 -12 1 5.5 0 0 2.328433790320584E-08 +0 1 99 Tree node -11 16 0 4.5 0 0 1.9702768192485041E-08 +0 1 99 Tree node 14 13 3 1.5 0 0 8.0066290366757683E-09 +0 1 99 Tree node -14 -15 0 2.5 0 0 5.0088703556982524E-09 +0 1 99 Tree node 15 -16 5 2.5 0 0 4.3006442755813825E-09 +0 1 99 Tree node -1 -17 5 1.5 0 0 8.284459332793366E-09 +0 1 99 Tree node -13 17 1 6.5 0 0 2.7656797989166588E-09 +0 1 99 Tree node 18 -19 1 7.5 0 0 1.0786058231325664E-08 +0 1 99 Tree node -18 -20 7 1.5 0 0 1.7313120777385077E-08 +0 1 99 Leaf node 7 5:-0.50000003147647765 +0 1 99 Leaf node 7 5:-0.28454340944217604 +0 1 99 Leaf node 7 5:-0.22043275817162536 +0 1 99 Leaf node 7 5:0.33365423637402225 +0 1 99 Leaf node 7 5:0.23779694750806021 +0 1 99 Leaf node 7 5:0.18485304106202394 +0 1 99 Leaf node 7 5:0.50025857648062877 +0 1 99 Leaf node 7 5:0.50019172128197398 +0 1 99 Leaf node 7 5:-0.50015671609933343 +0 1 99 Leaf node 7 5:-0.014693996155191378 +0 1 99 Leaf node 7 5:-0.23609861604079316 +0 1 99 Leaf node 7 5:-0.035264857523295404 +0 1 99 Leaf node 7 5:0.50006736836973453 +0 1 99 Leaf node 7 5:-0.50005016213391207 +0 1 99 Leaf node 7 5:-0.50009346966740154 +0 1 99 Leaf node 7 5:-0.50000282022196418 +0 1 99 Leaf node 7 5:-0.50017584479624944 +0 1 99 Leaf node 7 5:-0.47254493986274426 +0 1 99 Leaf node 7 5:0.50008526629450401 +0 1 99 Leaf node 7 5:0.50002435596756223 diff --git a/test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv b/test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv index f18a9e9e2b..6288a254ae 100644 --- a/test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv +++ b/test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv @@ -38,6 +38,7 @@ TimeSeriesProcessingEntryPoints.PercentileThresholdTransform Detects the values TimeSeriesProcessingEntryPoints.PValueTransform This P-Value transform calculates the p-value of the current input in the sequence with regard to the values in the sliding window. Microsoft.ML.Transforms.TimeSeries.TimeSeriesProcessingEntryPoints PValueTransform Microsoft.ML.Transforms.TimeSeries.PValueTransform+Arguments Microsoft.ML.EntryPoints.CommonOutputs+TransformOutput TimeSeriesProcessingEntryPoints.SlidingWindowTransform Returns the last values for a time series [y(t-d-l+1), y(t-d-l+2), ..., y(t-l-1), y(t-l)] where d is the size of the window, l the lag and y is a Float. Microsoft.ML.Transforms.TimeSeries.TimeSeriesProcessingEntryPoints SlidingWindowTransform Microsoft.ML.Transforms.TimeSeries.SlidingWindowTransformBase`1+Arguments[System.Single] Microsoft.ML.EntryPoints.CommonOutputs+TransformOutput TimeSeriesProcessingEntryPoints.SsaChangePointDetector This transform detects the change-points in a seasonal time-series using Singular Spectrum Analysis (SSA). Microsoft.ML.Transforms.TimeSeries.TimeSeriesProcessingEntryPoints SsaChangePointDetector Microsoft.ML.Transforms.TimeSeries.SsaChangePointDetector+Options Microsoft.ML.EntryPoints.CommonOutputs+TransformOutput +TimeSeriesProcessingEntryPoints.SsaForecasting This transform forecasts using Singular Spectrum Analysis (SSA). Microsoft.ML.Transforms.TimeSeries.TimeSeriesProcessingEntryPoints SsaForecasting Microsoft.ML.Transforms.TimeSeries.SsaForecastingTransformer+Options Microsoft.ML.EntryPoints.CommonOutputs+TransformOutput TimeSeriesProcessingEntryPoints.SsaSpikeDetector This transform detects the spikes in a seasonal time-series using Singular Spectrum Analysis (SSA). Microsoft.ML.Transforms.TimeSeries.TimeSeriesProcessingEntryPoints SsaSpikeDetector Microsoft.ML.Transforms.TimeSeries.SsaSpikeDetector+Options Microsoft.ML.EntryPoints.CommonOutputs+TransformOutput Trainers.AveragedPerceptronBinaryClassifier Averaged Perceptron Binary Classifier. Microsoft.ML.Trainers.AveragedPerceptronTrainer TrainBinary Microsoft.ML.Trainers.AveragedPerceptronTrainer+Options Microsoft.ML.EntryPoints.CommonOutputs+BinaryClassificationOutput Trainers.EnsembleBinaryClassifier Train binary ensemble. Microsoft.ML.Trainers.Ensemble.Ensemble CreateBinaryEnsemble Microsoft.ML.Trainers.Ensemble.EnsembleTrainer+Arguments Microsoft.ML.EntryPoints.CommonOutputs+BinaryClassificationOutput diff --git a/test/BaselineOutput/Common/EntryPoints/core_manifest.json b/test/BaselineOutput/Common/EntryPoints/core_manifest.json index 99b0d3a8ab..6d9e968d58 100644 --- a/test/BaselineOutput/Common/EntryPoints/core_manifest.json +++ b/test/BaselineOutput/Common/EntryPoints/core_manifest.json @@ -4007,6 +4007,244 @@ "ITransformOutput" ] }, + { + "Name": "TimeSeriesProcessingEntryPoints.SsaForecasting", + "Desc": "This transform forecasts using Singular Spectrum Analysis (SSA).", + "FriendlyName": "SSA Forecasting", + "ShortName": "ssafcst", + "Inputs": [ + { + "Name": "Source", + "Type": "String", + "Desc": "The name of the source column.", + "Aliases": [ + "src" + ], + "Required": true, + "SortOrder": 1.0, + "IsNullable": false + }, + { + "Name": "Data", + "Type": "DataView", + "Desc": "Input dataset", + "Required": true, + "SortOrder": 1.0, + "IsNullable": false + }, + { + "Name": "Name", + "Type": "String", + "Desc": "The name of the new column.", + "Required": true, + "SortOrder": 2.0, + "IsNullable": false + }, + { + "Name": "WindowSize", + "Type": "Int", + "Desc": "The length of the window on the series for building the trajectory matrix (parameter L).", + "Required": true, + "SortOrder": 2.0, + "IsNullable": false, + "Default": 0 + }, + { + "Name": "SeriesLength", + "Type": "Int", + "Desc": "The length of series that is kept in buffer for modeling (parameter N).", + "Required": true, + "SortOrder": 2.0, + "IsNullable": false, + "Default": 0 + }, + { + "Name": "TrainSize", + "Type": "Int", + "Desc": "The length of series from the begining used for training.", + "Required": true, + "SortOrder": 2.0, + "IsNullable": false, + "Default": 0 + }, + { + "Name": "Horizon", + "Type": "Int", + "Desc": "The number of values to forecast.", + "Required": true, + "SortOrder": 2.0, + "IsNullable": false, + "Default": 0 + }, + { + "Name": "ConfidenceLevel", + "Type": "Float", + "Desc": "The confidence level in [0, 1) for forecasting.", + "Required": false, + "SortOrder": 2.0, + "IsNullable": false, + "Default": 0.95 + }, + { + "Name": "VariableHorizon", + "Type": "Bool", + "Desc": "Set this to true horizon will change at prediction time.", + "Required": false, + "SortOrder": 2.0, + "IsNullable": false, + "Default": false + }, + { + "Name": "ConfidenceLowerBoundColumn", + "Type": "String", + "Desc": "The name of the confidence interval lower bound column.", + "Aliases": [ + "cnfminname" + ], + "Required": false, + "SortOrder": 3.0, + "IsNullable": false, + "Default": null + }, + { + "Name": "ConfidenceUpperBoundColumn", + "Type": "String", + "Desc": "The name of the confidence interval upper bound column.", + "Aliases": [ + "cnfmaxnname" + ], + "Required": false, + "SortOrder": 3.0, + "IsNullable": false, + "Default": null + }, + { + "Name": "RankSelectionMethod", + "Type": { + "Kind": "Enum", + "Values": [ + "Fixed", + "Exact", + "Fast" + ] + }, + "Desc": "The rank selection method.", + "Required": false, + "SortOrder": 3.0, + "IsNullable": false, + "Default": "Exact" + }, + { + "Name": "Rank", + "Type": "Int", + "Desc": "The desired rank of the subspace used for SSA projection (parameter r). This parameter should be in the range in [1, windowSize]. If set to null, the rank is automatically determined based on prediction error minimization.", + "Required": false, + "SortOrder": 3.0, + "IsNullable": true, + "Default": null + }, + { + "Name": "MaxRank", + "Type": "Int", + "Desc": "The maximum rank considered during the rank selection process. If not provided (i.e. set to null), it is set to windowSize - 1.", + "Required": false, + "SortOrder": 3.0, + "IsNullable": true, + "Default": null + }, + { + "Name": "ShouldStabilize", + "Type": "Bool", + "Desc": "The flag determining whether the model should be stabilized.", + "Required": false, + "SortOrder": 3.0, + "IsNullable": false, + "Default": true + }, + { + "Name": "ShouldMaintainInfo", + "Type": "Bool", + "Desc": "The flag determining whether the meta information for the model needs to be maintained.", + "Required": false, + "SortOrder": 3.0, + "IsNullable": false, + "Default": false + }, + { + "Name": "MaxGrowth", + "Type": { + "Kind": "Struct", + "Fields": [ + { + "Name": "TimeSpan", + "Type": "Int", + "Desc": "Time span of growth ratio. Must be strictly positive.", + "Required": false, + "SortOrder": 1.0, + "IsNullable": false, + "Default": 0 + }, + { + "Name": "Growth", + "Type": "Float", + "Desc": "Growth. Must be non-negative.", + "Required": false, + "SortOrder": 2.0, + "IsNullable": false, + "Default": 0.0 + } + ] + }, + "Desc": "The maximum growth on the exponential trend.", + "Required": false, + "SortOrder": 3.0, + "IsNullable": true, + "Default": null + }, + { + "Name": "DiscountFactor", + "Type": "Float", + "Desc": "The discount factor in [0,1] used for online updates.", + "Aliases": [ + "disc" + ], + "Required": false, + "SortOrder": 5.0, + "IsNullable": false, + "Default": 1.0 + }, + { + "Name": "IsAdaptive", + "Type": "Bool", + "Desc": "The flag determing whether the model is adaptive", + "Aliases": [ + "adp" + ], + "Required": false, + "SortOrder": 6.0, + "IsNullable": false, + "Default": false + } + ], + "Outputs": [ + { + "Name": "OutputData", + "Type": "DataView", + "Desc": "Transformed dataset" + }, + { + "Name": "Model", + "Type": "TransformModel", + "Desc": "Transform model" + } + ], + "InputKind": [ + "ITransformInput" + ], + "OutputKind": [ + "ITransformOutput" + ] + }, { "Name": "TimeSeriesProcessingEntryPoints.SsaSpikeDetector", "Desc": "This transform detects the spikes in a seasonal time-series using Singular Spectrum Analysis (SSA).", @@ -11297,7 +11535,7 @@ "Required": false, "SortOrder": 150.0, "IsNullable": false, - "Default": "Logloss" + "Default": "Default" }, { "Name": "MaximumBinCountPerFeature", @@ -11736,6 +11974,18 @@ "IsNullable": false, "Default": "Auto" }, + { + "Name": "UnbalancedSets", + "Type": "Bool", + "Desc": "Use for multi-class classification when training data is not balanced", + "Aliases": [ + "us" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": false + }, { "Name": "UseSoftmax", "Type": "Bool", @@ -11782,7 +12032,7 @@ "Required": false, "SortOrder": 150.0, "IsNullable": false, - "Default": "Error" + "Default": "Default" }, { "Name": "MaximumBinCountPerFeature", @@ -12279,7 +12529,7 @@ "Required": false, "SortOrder": 150.0, "IsNullable": false, - "Default": "NormalizedDiscountedCumulativeGain" + "Default": "Default" }, { "Name": "MaximumBinCountPerFeature", @@ -12737,7 +12987,7 @@ "Required": false, "SortOrder": 150.0, "IsNullable": false, - "Default": "RootMeanSquaredError" + "Default": "Default" }, { "Name": "MaximumBinCountPerFeature", diff --git a/test/BaselineOutput/Common/EntryPoints/ensemble-model1-summary.txt b/test/BaselineOutput/Common/EntryPoints/ensemble-model1-summary.txt index c2b010213d..bde4e6c44b 100644 --- a/test/BaselineOutput/Common/EntryPoints/ensemble-model1-summary.txt +++ b/test/BaselineOutput/Common/EntryPoints/ensemble-model1-summary.txt @@ -1,7 +1,36 @@ #@ TextLoader{ #@ header+ #@ sep=tab -#@ col=Gains:R4:0-16 +#@ col=Bias:R8:0 +#@ col=TreeWeights:R8:1 +#@ col=TreeID:I4:2 +#@ col=IsLeaf:TX:3 +#@ col=LeftChild:I4:4 +#@ col=RightChild:I4:5 +#@ col=NumericalSplitFeatureIndexes:I4:6 +#@ col=NumericalSplitThresholds:R4:7 +#@ col=CategoricalSplitFlags:BL:8 +#@ col=LeafValues:R8:9 +#@ col=SplitGains:R8:10 +#@ col=CategoricalSplitFeatures:I4:11-** +#@ col={name=CategoricalCategoricalSplitFeatureRange type=I4 src={ min=-1 var=+}} #@ } -Cat.1 Cat.4 Cat.2 Cat.5 Cat.10 Cat.3 Cat.7 Cat.8 Cat.6 Features.thickness Features.uniform_size Features.uniform_shape Features.adhesion Features.epit_size Features.bare_nuclei Features.bland_chromatin Features.normal_nucleoli -0.0607880056 0 0.0249023773 0 0 4.10026857E-09 0 0 0 0.190965369 1 0.7112387 0.14315024 0.222178861 0.413435966 0.254190356 0.2604484 +Bias TreeWeights TreeID IsLeaf LeftChild RightChild NumericalSplitFeatureIndexes NumericalSplitThresholds CategoricalSplitFlags LeafValues SplitGains +0 1 0 Tree node 2 1 10 2.5 0 0 13.415869372871798 +0 1 0 Tree node 3 -3 14 1.5 0 0 0.92083542188806611 +0 1 0 Tree node -1 -4 16 2.5 0 0 0.67201267815728727 +0 1 0 Tree node -2 -5 13 3.5 0 0 0.5363809523809524 +0 1 0 Leaf node 7 5:-0.98625429553265687 +0 1 0 Leaf node 7 5:-0.99999999999999978 +0 1 0 Leaf node 7 5:0.87134502923976442 +0 1 0 Leaf node 7 5:0.27272727272727287 +0 1 0 Leaf node 7 5:0.59999999999999976 +0 1 1 Tree node 3 1 10 2.5 0 0 8.6352053527178452 +0 1 1 Tree node 2 -3 14 2.5 0 0 0.63992190088294487 +0 1 1 Tree node -2 -4 10 3.5 0 0 0.71162415170614624 +0 1 1 Tree node -1 -5 15 4.5 0 0 0.43452626844448261 +0 1 1 Leaf node 7 5:-0.82036054531830893 +0 1 1 Leaf node 7 5:-0.8186475459992979 +0 1 1 Leaf node 7 5:0.75043188004013617 +0 1 1 Leaf node 7 5:0.74116698906550615 +0 1 1 Leaf node 7 5:0.27575806003985165 diff --git a/test/BaselineOutput/Common/EntryPoints/ensemble-model3-summary.txt b/test/BaselineOutput/Common/EntryPoints/ensemble-model3-summary.txt index b8a21ad39b..099bc366a8 100644 --- a/test/BaselineOutput/Common/EntryPoints/ensemble-model3-summary.txt +++ b/test/BaselineOutput/Common/EntryPoints/ensemble-model3-summary.txt @@ -1,7 +1,36 @@ #@ TextLoader{ #@ header+ #@ sep=tab -#@ col=Gains:R4:0-16 +#@ col=Bias:R8:0 +#@ col=TreeWeights:R8:1 +#@ col=TreeID:I4:2 +#@ col=IsLeaf:TX:3 +#@ col=LeftChild:I4:4 +#@ col=RightChild:I4:5 +#@ col=NumericalSplitFeatureIndexes:I4:6 +#@ col=NumericalSplitThresholds:R4:7 +#@ col=CategoricalSplitFlags:BL:8 +#@ col=LeafValues:R8:9 +#@ col=SplitGains:R8:10 +#@ col=CategoricalSplitFeatures:I4:11-** +#@ col={name=CategoricalCategoricalSplitFeatureRange type=I4 src={ min=-1 var=+}} #@ } -Cat.1 Cat.5 Cat.2 Cat.4 Cat.3 Cat.7 Cat.10 Cat.8 Cat.6 Features.thickness Features.uniform_size Features.uniform_shape Features.adhesion Features.epit_size Features.bare_nuclei Features.bland_chromatin Features.normal_nucleoli -0.009761757 0 0.0203766879 0 0.000928933 0 0 0 0 0.308038682 1 0.5590685 0.125412315 0.118880585 0.488731444 0.308761537 0.132577017 +Bias TreeWeights TreeID IsLeaf LeftChild RightChild NumericalSplitFeatureIndexes NumericalSplitThresholds CategoricalSplitFlags LeafValues SplitGains +0 1 0 Tree node 1 3 10 3.5 0 0 13.360986434197844 +0 1 0 Tree node 2 -3 14 5.5 0 0 2.3140017316017349 +0 1 0 Tree node -1 -4 16 3.5 0 0 0.37346969696970156 +0 1 0 Tree node -2 -5 10 4.5 0 0 0.26028321340685778 +0 1 0 Leaf node 7 5:-0.98125000000001306 +0 1 0 Leaf node 7 5:0.4482758620689532 +0 1 0 Leaf node 7 5:0.79999999999999771 +0 1 0 Leaf node 7 5:-1.5265566588595898E-15 +0 1 0 Leaf node 7 5:0.97037037037036966 +0 1 1 Tree node 1 3 10 3.5 0 0 8.6031446225300847 +0 1 1 Tree node -1 2 14 4.5 0 0 1.5404623416657568 +0 1 1 Tree node -3 -4 10 1.5 0 0 0.26130389109429525 +0 1 1 Tree node -2 -5 10 4.5 0 0 0.16830943804733955 +0 1 1 Leaf node 7 5:-0.82068788481267385 +0 1 1 Leaf node 7 5:0.36175241006652414 +0 1 1 Leaf node 7 5:-0.25170035507676081 +0 1 1 Leaf node 7 5:0.69526012254866087 +0 1 1 Leaf node 7 5:0.80839621873313372 diff --git a/test/BaselineOutput/Common/EntryPoints/ensemble-summary-key-value-pairs.txt b/test/BaselineOutput/Common/EntryPoints/ensemble-summary-key-value-pairs.txt index d89d7a7619..debc87d6c1 100644 --- a/test/BaselineOutput/Common/EntryPoints/ensemble-summary-key-value-pairs.txt +++ b/test/BaselineOutput/Common/EntryPoints/ensemble-summary-key-value-pairs.txt @@ -27,16 +27,10 @@ Cat.1: System.Single[] Partition model 1 summary: Per-feature gain summary for the boosted tree ensemble: Features.uniform_size: 1 -Features.uniform_shape: 0.711238682354263 -Features.bare_nuclei: 0.413435971399054 -Features.normal_nucleoli: 0.260448393604327 -Features.bland_chromatin: 0.254190368593018 -Features.epit_size: 0.222178863469679 -Features.thickness: 0.190965373645692 -Features.adhesion: 0.143150245168852 -Cat.1: 0.0607880054395048 -Cat.2: 0.0249023775790133 -Cat.3: 4.10026871732935E-09 +Features.bare_nuclei: 0.261851950948993 +Features.normal_nucleoli: 0.171821243048215 +Features.epit_size: 0.153505802680442 +Features.bland_chromatin: 0.138164395549383 Partition model 2 summary: Linear Binary Classification Predictor non-zero weights (Bias): -4.860323 @@ -66,13 +60,5 @@ Cat.1: System.Single[] Partition model 3 summary: Per-feature gain summary for the boosted tree ensemble: Features.uniform_size: 1 -Features.uniform_shape: 0.559068504082849 -Features.bare_nuclei: 0.488731457203164 -Features.bland_chromatin: 0.308761540884501 -Features.thickness: 0.308038677882308 -Features.normal_nucleoli: 0.132577017456797 -Features.adhesion: 0.125412316945858 -Features.epit_size: 0.118880587537871 -Cat.2: 0.0203766881332348 -Cat.1: 0.00976175711400017 -Cat.3: 0.000928932959407758 +Features.bare_nuclei: 0.412486071655037 +Features.normal_nucleoli: 0.128397028220685 diff --git a/test/BaselineOutput/Common/EntryPoints/ensemble-summary.txt b/test/BaselineOutput/Common/EntryPoints/ensemble-summary.txt index 12c0c05472..0079938c4c 100644 --- a/test/BaselineOutput/Common/EntryPoints/ensemble-summary.txt +++ b/test/BaselineOutput/Common/EntryPoints/ensemble-summary.txt @@ -36,16 +36,10 @@ Partition model 1 summary: Per-feature gain summary for the boosted tree ensemble: Features.uniform_size 1 - Features.uniform_shape 0.711238682354263 - Features.bare_nuclei 0.413435971399054 - Features.normal_nucleoli 0.260448393604327 - Features.bland_chromatin 0.254190368593018 - Features.epit_size 0.222178863469679 - Features.thickness 0.190965373645692 - Features.adhesion 0.143150245168852 - Cat.1 0.0607880054395048 - Cat.2 0.0249023775790133 - Cat.3 4.10026871732935E-09 + Features.bare_nuclei 0.261851950948993 + Features.normal_nucleoli 0.171821243048215 + Features.epit_size 0.153505802680442 + Features.bland_chromatin 0.138164395549383 Partition model 2 summary: Linear Binary Classification Predictor non-zero weights @@ -84,13 +78,5 @@ Partition model 3 summary: Per-feature gain summary for the boosted tree ensemble: Features.uniform_size 1 - Features.uniform_shape 0.559068504082849 - Features.bare_nuclei 0.488731457203164 - Features.bland_chromatin 0.308761540884501 - Features.thickness 0.308038677882308 - Features.normal_nucleoli 0.132577017456797 - Features.adhesion 0.125412316945858 - Features.epit_size 0.118880587537871 - Cat.2 0.0203766881332348 - Cat.1 0.00976175711400017 - Cat.3 0.000928932959407758 + Features.bare_nuclei 0.412486071655037 + Features.normal_nucleoli 0.128397028220685 diff --git a/test/BaselineOutput/Common/LightGBMR/LightGBMReg-CV-generatedRegressionDataset-out.txt b/test/BaselineOutput/Common/LightGBMR/LightGBMReg-CV-generatedRegressionDataset-out.txt index c81802feb6..29dfd1294a 100644 --- a/test/BaselineOutput/Common/LightGBMR/LightGBMReg-CV-generatedRegressionDataset-out.txt +++ b/test/BaselineOutput/Common/LightGBMR/LightGBMReg-CV-generatedRegressionDataset-out.txt @@ -35,10 +35,10 @@ Virtual memory usage(MB): %Number% [1] 'Loading data for LightGBM' started. [1] 'Loading data for LightGBM' finished in %Time%. [2] 'Training with LightGBM' started. -[2] (%Time%) Iteration: 50 Training-rmse: 6.09160118577349 +[2] (%Time%) Iteration: 50 Training-: 37.107605006517 [2] 'Training with LightGBM' finished in %Time%. [3] 'Loading data for LightGBM #2' started. [3] 'Loading data for LightGBM #2' finished in %Time%. [4] 'Training with LightGBM #2' started. -[4] (%Time%) Iteration: 50 Training-rmse: 5.26343689176522 +[4] (%Time%) Iteration: 50 Training-: 27.7037679135951 [4] 'Training with LightGBM #2' finished in %Time%. diff --git a/test/BaselineOutput/Common/LightGBMR/LightGBMReg-TrainTest-generatedRegressionDataset-out.txt b/test/BaselineOutput/Common/LightGBMR/LightGBMReg-TrainTest-generatedRegressionDataset-out.txt index c88cad62b7..668caf5901 100644 --- a/test/BaselineOutput/Common/LightGBMR/LightGBMReg-TrainTest-generatedRegressionDataset-out.txt +++ b/test/BaselineOutput/Common/LightGBMR/LightGBMReg-TrainTest-generatedRegressionDataset-out.txt @@ -26,7 +26,7 @@ Virtual memory usage(MB): %Number% [1] 'Loading data for LightGBM' started. [1] 'Loading data for LightGBM' finished in %Time%. [2] 'Training with LightGBM' started. -[2] (%Time%) Iteration: 50 Training-rmse: 5.10533343749577 +[2] (%Time%) Iteration: 50 Training-: 26.0644295080124 [2] 'Training with LightGBM' finished in %Time%. [3] 'Saving model' started. [3] 'Saving model' finished in %Time%. diff --git a/test/BaselineOutput/Common/LightGBMR/LightGBMRegRmse-CV-generatedRegressionDataset.RMSE-rp.txt b/test/BaselineOutput/Common/LightGBMR/LightGBMRegRmse-CV-generatedRegressionDataset.RMSE-rp.txt index d855ef6c79..ba3cb14b0d 100644 --- a/test/BaselineOutput/Common/LightGBMR/LightGBMRegRmse-CV-generatedRegressionDataset.RMSE-rp.txt +++ b/test/BaselineOutput/Common/LightGBMR/LightGBMRegRmse-CV-generatedRegressionDataset.RMSE-rp.txt @@ -1,4 +1,4 @@ LightGBMR -L1(avg) L2(avg) RMS(avg) Loss-fn(avg) R Squared /iter /lr /nl /mil /v /nt Learner Name Train Dataset Test Dataset Results File Run Time Physical Memory Virtual Memory Command Line Settings -26.59978 1393.326 37.32081 1393.326 0.923402 50 0.2 20 10 + 1 LightGBMR %Data% %Output% 99 0 0 maml.exe CV tr=LightGBMR{nt=1 iter=50 em=RootMeanSquaredError v=+ lr=0.2 mil=10 nl=20} threads=- dout=%Output% loader=Text{col=Label:R4:11 col=Features:R4:0-10 sep=; header+} data=%Data% seed=1 /iter:50;/lr:0.2;/nl:20;/mil:10;/v:+;/nt:1 +L1(avg) L2(avg) RMS(avg) Loss-fn(avg) R Squared /em /iter /lr /nl /mil /v /nt Learner Name Train Dataset Test Dataset Results File Run Time Physical Memory Virtual Memory Command Line Settings +26.59978 1393.326 37.32081 1393.326 0.923402 RootMeanSquaredError 50 0.2 20 10 + 1 LightGBMR %Data% %Output% 99 0 0 maml.exe CV tr=LightGBMR{nt=1 iter=50 em=RootMeanSquaredError v=+ lr=0.2 mil=10 nl=20} threads=- dout=%Output% loader=Text{col=Label:R4:11 col=Features:R4:0-10 sep=; header+} data=%Data% seed=1 /em:RootMeanSquaredError;/iter:50;/lr:0.2;/nl:20;/mil:10;/v:+;/nt:1 diff --git a/test/BaselineOutput/Common/LightGBMR/LightGBMRegRmse-TrainTest-generatedRegressionDataset.RMSE-rp.txt b/test/BaselineOutput/Common/LightGBMR/LightGBMRegRmse-TrainTest-generatedRegressionDataset.RMSE-rp.txt index bf64ad5ed2..cdff8dbb38 100644 --- a/test/BaselineOutput/Common/LightGBMR/LightGBMRegRmse-TrainTest-generatedRegressionDataset.RMSE-rp.txt +++ b/test/BaselineOutput/Common/LightGBMR/LightGBMRegRmse-TrainTest-generatedRegressionDataset.RMSE-rp.txt @@ -1,4 +1,4 @@ LightGBMR -L1(avg) L2(avg) RMS(avg) Loss-fn(avg) R Squared /iter /lr /nl /mil /v /nt Learner Name Train Dataset Test Dataset Results File Run Time Physical Memory Virtual Memory Command Line Settings -3.428896 25.23601 5.023546 25.23601 0.998616 50 0.2 20 10 + 1 LightGBMR %Data% %Data% %Output% 99 0 0 maml.exe TrainTest test=%Data% tr=LightGBMR{nt=1 iter=50 em=RootMeanSquaredError v=+ lr=0.2 mil=10 nl=20} dout=%Output% loader=Text{col=Label:R4:11 col=Features:R4:0-10 sep=; header+} data=%Data% out=%Output% seed=1 /iter:50;/lr:0.2;/nl:20;/mil:10;/v:+;/nt:1 +L1(avg) L2(avg) RMS(avg) Loss-fn(avg) R Squared /em /iter /lr /nl /mil /v /nt Learner Name Train Dataset Test Dataset Results File Run Time Physical Memory Virtual Memory Command Line Settings +3.428896 25.23601 5.023546 25.23601 0.998616 RootMeanSquaredError 50 0.2 20 10 + 1 LightGBMR %Data% %Data% %Output% 99 0 0 maml.exe TrainTest test=%Data% tr=LightGBMR{nt=1 iter=50 em=RootMeanSquaredError v=+ lr=0.2 mil=10 nl=20} dout=%Output% loader=Text{col=Label:R4:11 col=Features:R4:0-10 sep=; header+} data=%Data% out=%Output% seed=1 /em:RootMeanSquaredError;/iter:50;/lr:0.2;/nl:20;/mil:10;/v:+;/nt:1 diff --git a/test/BaselineOutput/Common/WeightedEnsemble/WE-All-TrainTest-breast-cancer-out.txt b/test/BaselineOutput/Common/WeightedEnsemble/WE-All-TrainTest-breast-cancer-out.txt new file mode 100644 index 0000000000..ad0a480605 --- /dev/null +++ b/test/BaselineOutput/Common/WeightedEnsemble/WE-All-TrainTest-breast-cancer-out.txt @@ -0,0 +1,98 @@ +maml.exe TrainTest test=%Data% tr=WeightedEnsemble{nm=20 st=AllInstanceSelector tp=-} dout=%Output% loader=Text{col=Label:BL:0 col=Features:R4:1-9} data=%Data% out=%Output% seed=1 +Automatically adding a MinMax normalization transform, use 'norm=Warn' or 'norm=No' to turn this behavior off. +Training 20 learners for the batch 1 +Beginning training model 1 of 20 +Warning: Skipped 16 instances with missing features during training (over 1 iterations; 16 inst/iter) +Trainer 1 of 20 finished in %Time% +Beginning training model 2 of 20 +Warning: Skipped 16 instances with missing features during training (over 1 iterations; 16 inst/iter) +Trainer 2 of 20 finished in %Time% +Beginning training model 3 of 20 +Warning: Skipped 16 instances with missing features during training (over 1 iterations; 16 inst/iter) +Trainer 3 of 20 finished in %Time% +Beginning training model 4 of 20 +Warning: Skipped 16 instances with missing features during training (over 1 iterations; 16 inst/iter) +Trainer 4 of 20 finished in %Time% +Beginning training model 5 of 20 +Warning: Skipped 16 instances with missing features during training (over 1 iterations; 16 inst/iter) +Trainer 5 of 20 finished in %Time% +Beginning training model 6 of 20 +Warning: Skipped 16 instances with missing features during training (over 1 iterations; 16 inst/iter) +Trainer 6 of 20 finished in %Time% +Beginning training model 7 of 20 +Warning: Skipped 16 instances with missing features during training (over 1 iterations; 16 inst/iter) +Trainer 7 of 20 finished in %Time% +Beginning training model 8 of 20 +Warning: Skipped 16 instances with missing features during training (over 1 iterations; 16 inst/iter) +Trainer 8 of 20 finished in %Time% +Beginning training model 9 of 20 +Warning: Skipped 16 instances with missing features during training (over 1 iterations; 16 inst/iter) +Trainer 9 of 20 finished in %Time% +Beginning training model 10 of 20 +Warning: Skipped 16 instances with missing features during training (over 1 iterations; 16 inst/iter) +Trainer 10 of 20 finished in %Time% +Beginning training model 11 of 20 +Warning: Skipped 16 instances with missing features during training (over 1 iterations; 16 inst/iter) +Trainer 11 of 20 finished in %Time% +Beginning training model 12 of 20 +Warning: Skipped 16 instances with missing features during training (over 1 iterations; 16 inst/iter) +Trainer 12 of 20 finished in %Time% +Beginning training model 13 of 20 +Warning: Skipped 16 instances with missing features during training (over 1 iterations; 16 inst/iter) +Trainer 13 of 20 finished in %Time% +Beginning training model 14 of 20 +Warning: Skipped 16 instances with missing features during training (over 1 iterations; 16 inst/iter) +Trainer 14 of 20 finished in %Time% +Beginning training model 15 of 20 +Warning: Skipped 16 instances with missing features during training (over 1 iterations; 16 inst/iter) +Trainer 15 of 20 finished in %Time% +Beginning training model 16 of 20 +Warning: Skipped 16 instances with missing features during training (over 1 iterations; 16 inst/iter) +Trainer 16 of 20 finished in %Time% +Beginning training model 17 of 20 +Warning: Skipped 16 instances with missing features during training (over 1 iterations; 16 inst/iter) +Trainer 17 of 20 finished in %Time% +Beginning training model 18 of 20 +Warning: Skipped 16 instances with missing features during training (over 1 iterations; 16 inst/iter) +Trainer 18 of 20 finished in %Time% +Beginning training model 19 of 20 +Warning: Skipped 16 instances with missing features during training (over 1 iterations; 16 inst/iter) +Trainer 19 of 20 finished in %Time% +Beginning training model 20 of 20 +Warning: Skipped 16 instances with missing features during training (over 1 iterations; 16 inst/iter) +Trainer 20 of 20 finished in %Time% +Training calibrator. +Warning: The predictor produced non-finite prediction values on 16 instances during testing. Possible causes: abnormal data or the predictor is numerically unstable. +TEST POSITIVE RATIO: 0.3499 (239.0/(239.0+444.0)) +Confusion table + ||====================== +PREDICTED || positive | negative | Recall +TRUTH ||====================== + positive || 232 | 7 | 0.9707 + negative || 11 | 433 | 0.9752 + ||====================== +Precision || 0.9547 | 0.9841 | +OVERALL 0/1 ACCURACY: 0.973646 +LOG LOSS/instance: 0.117326 +Test-set entropy (prior Log-Loss/instance): 0.934003 +LOG-LOSS REDUCTION (RIG): 0.874384 +AUC: 0.995995 + +OVERALL RESULTS +--------------------------------------- +AUC: 0.995995 (0.0000) +Accuracy: 0.973646 (0.0000) +Positive precision: 0.954733 (0.0000) +Positive recall: 0.970711 (0.0000) +Negative precision: 0.984091 (0.0000) +Negative recall: 0.975225 (0.0000) +Log-loss: 0.117326 (0.0000) +Log-loss reduction: 0.874384 (0.0000) +F1 Score: 0.962656 (0.0000) +AUPRC: 0.991908 (0.0000) + +--------------------------------------- +Physical memory usage(MB): %Number% +Virtual memory usage(MB): %Number% +%DateTime% Time elapsed(s): %Number% + diff --git a/test/BaselineOutput/Common/WeightedEnsemble/WE-All-TrainTest-breast-cancer-rp.txt b/test/BaselineOutput/Common/WeightedEnsemble/WE-All-TrainTest-breast-cancer-rp.txt new file mode 100644 index 0000000000..853ec2e2eb --- /dev/null +++ b/test/BaselineOutput/Common/WeightedEnsemble/WE-All-TrainTest-breast-cancer-rp.txt @@ -0,0 +1,4 @@ +WeightedEnsemble +AUC Accuracy Positive precision Positive recall Negative precision Negative recall Log-loss Log-loss reduction F1 Score AUPRC /bp /nm /st Learner Name Train Dataset Test Dataset Results File Run Time Physical Memory Virtual Memory Command Line Settings +0.995995 0.973646 0.954733 0.970711 0.984091 0.975225 0.117326 0.874384 0.962656 0.991908 Microsoft.ML.Runtime.ComponentFactoryUtils+SimpleComponentFactory`1[Microsoft.ML.Trainers.LinearSvmTrainer] 20 AllInstanceSelector WeightedEnsemble %Data% %Data% %Output% 99 0 0 maml.exe TrainTest test=%Data% tr=WeightedEnsemble{nm=20 st=AllInstanceSelector tp=-} dout=%Output% loader=Text{col=Label:BL:0 col=Features:R4:1-9} data=%Data% out=%Output% seed=1 /bp:Microsoft.ML.Runtime.ComponentFactoryUtils+SimpleComponentFactory`1[Microsoft.ML.Trainers.LinearSvmTrainer];/nm:20;/st:AllInstanceSelector + diff --git a/test/BaselineOutput/Common/WeightedEnsemble/WE-All-TrainTest-breast-cancer.txt b/test/BaselineOutput/Common/WeightedEnsemble/WE-All-TrainTest-breast-cancer.txt new file mode 100644 index 0000000000..6b46036c9f --- /dev/null +++ b/test/BaselineOutput/Common/WeightedEnsemble/WE-All-TrainTest-breast-cancer.txt @@ -0,0 +1,700 @@ +Instance Label Score Probability Log-loss Assigned +0 0 -2.72854328 0.0153081212 0.022255735289143007 0 +1 0 2.20268083 0.917555869 3.6004393879063104 1 +2 0 -2.865943 0.0127788186 0.018554746224242847 0 +3 0 2.35591817 0.931748569 3.8729968952719278 1 +4 0 -2.55102944 0.019316094 0.028139892924164366 0 +5 1 7.342854 0.999905 0.00013707668465523314 1 +6 0 -0.834810734 0.16253081 0.25589197873851016 0 +7 0 -3.17788553 0.008468157 0.012268989138756122 0 +8 0 -3.238492 0.007816049 0.011320474070784866 0 +9 0 -2.865943 0.0127788186 0.018554746224242847 0 +10 0 -3.66511059 0.004440975 0.0064212413418911849 0 +11 0 -3.55087972 0.005167652 0.0074746757385812587 0 +12 1 -0.26072526 0.2943737 1.7642793550195373 0 +13 0 -2.962865 0.0112474067 0.016318521416795649 0 +14 1 5.13357639 0.9981972 0.0026032408987607877 1 +15 1 0.795347452 0.630311966 0.66586204354368062 1 +16 0 -3.07316566 0.00972443 0.014098046031275298 0 +17 0 -2.9662137 0.0111978734 0.016246248849105433 0 +18 1 4.68633938 0.996732354 0.0047219354575893813 1 +19 0 -2.46727276 0.0215486 0.031427902667595507 0 +20 1 3.75242281 0.988743246 0.016332160745959907 1 +21 1 4.512779 0.995885253 0.0059485720047010244 1 +22 0 -3.31544065 0.007059439 0.010220736690982309 0 +23 1 ? ? ? 0 +24 0 -3.57815838 0.004984033 0.0072084178999092668 0 +25 1 0.996296167 0.69028 0.53474636798823416 1 +26 0 -3.15139437 0.00876987353 0.012708058893065321 0 +27 0 -2.835495 0.0133011211 0.01931822515825387 0 +28 0 -3.55087972 0.005167652 0.0074746757385812587 0 +29 0 -3.55087972 0.005167652 0.0074746757385812587 0 +30 0 -3.396992 0.00633684872 0.0091712293529778808 0 +31 0 -3.38022614 0.0064791413 0.0093778381082593662 0 +32 1 4.239458 0.9940872 0.0085556583489895793 1 +33 0 -3.17676878 0.008480665 0.012287189461582367 0 +34 0 -3.11044979 0.009257291 0.013417650275024486 0 +35 0 -3.55087972 0.005167652 0.0074746757385812587 0 +36 1 5.48185349 0.998866 0.0016369132539510332 1 +37 0 -1.6649096 0.0603087023 0.089741206617439145 0 +38 1 3.45343351 0.9833229 0.024262826548882763 1 +39 1 1.5258646 0.81867063 0.28864495484536951 1 +40 0 ? ? ? 0 +41 1 1.38819432 0.7898209 0.34040253307438006 1 +42 1 5.458683 0.9988305 0.0016882231796356319 1 +43 1 0.580491543 0.5614732 0.83271095640340953 1 +44 1 5.29709339 0.998549759 0.0020937735440191363 1 +45 0 -3.678948 0.004360162 0.0063041379062197594 0 +46 1 3.468822 0.983656 0.023774240190126871 1 +47 0 -3.77104545 0.00385836349 0.0055772082517802756 0 +48 0 -2.55102944 0.019316094 0.028139892924164366 0 +49 1 3.98990417 0.9917727 0.011918564884792524 1 +50 1 1.47508955 0.8084053 0.30684935015105891 1 +51 1 0.031583786 0.381172359 1.3914845900002191 1 +52 1 2.97328234 0.9688362 0.045675340786728726 1 +53 1 4.095047 0.992841 0.010365393442236737 1 +54 1 3.90287971 0.9907701 0.013377761830287676 1 +55 1 2.99762869 0.9698013 0.04423889724328689 1 +56 1 4.527055 0.9959625 0.0058366711251104444 1 +57 1 0.0076406 0.373672783 1.4201526066842074 1 +58 1 1.30337119 0.77043575 0.37625344486023793 1 +59 1 0.7160735 0.605367243 0.72411748252619823 1 +60 1 1.52246261 0.817996442 0.28983352636751908 1 +61 0 -3.44117737 0.00597654749 0.0086482044429181011 0 +62 1 4.83182669 0.9973069 0.0038905873144013418 1 +63 1 0.251530647 0.4523014 1.1446435924701022 1 +64 0 -3.77104545 0.00385836349 0.0055772082517802756 0 +65 1 1.052484 0.706061065 0.50213513143755195 1 +66 0 -2.9662137 0.0111978734 0.016246248849105433 0 +67 1 2.90392375 0.9659203 0.050023897090518764 1 +68 1 5.48265076 0.9988672 0.0016351914760097265 1 +69 0 -3.423378 0.00611917255 0.0088552209274113513 0 +70 0 -2.425513 0.0227541048 0.033206475233163314 0 +71 1 4.19593763 0.993736267 0.0090650767484431877 1 +72 0 -1.8075124 0.0503942035 0.074599353529820764 0 +73 1 4.958601 0.997724652 0.0032863730972366185 1 +74 1 1.5664475 0.826563537 0.27480237256766699 1 +75 0 -2.75931168 0.0147019951 0.021367958932079885 0 +76 0 -3.158723 0.008685354 0.01258504961606172 0 +77 0 -2.19452333 0.03070709 0.044995395131039846 0 +78 0 -2.55102944 0.019316094 0.028139892924164366 0 +79 0 -3.50334549 0.005503838 0.0079622914531571748 0 +80 0 -2.37792158 0.0242083557 0.035354965355093713 0 +81 0 -2.88169169 0.0125166625 0.01817169008021928 0 +82 0 -2.49842167 0.0206902958 0.030162914333484676 0 +83 0 -2.14587235 0.0326972865 0.047960648840875912 0 +84 1 5.550816 0.9989655 0.0014932379534251326 1 +85 1 4.104134 0.9929266 0.010241024867369912 1 +86 1 1.53414536 0.820303559 0.28577020654764657 1 +87 1 4.303641 0.9945694 0.0078560203900149474 1 +88 0 -2.9662137 0.0111978734 0.016246248849105433 0 +89 0 -3.30726719 0.007136225 0.010332306922777893 0 +90 0 -3.57815838 0.004984033 0.0072084178999092668 0 +91 0 -3.29167366 0.007285025 0.010548539407608692 0 +92 0 -2.9662137 0.0111978734 0.016246248849105433 0 +93 0 -3.77104545 0.00385836349 0.0055772082517802756 0 +94 0 -3.38022614 0.0064791413 0.0093778381082593662 0 +95 0 -3.57815838 0.004984033 0.0072084178999092668 0 +96 0 -3.541967 0.005229093 0.0075637798682153127 0 +97 0 -2.72854328 0.0153081212 0.022255735289143007 0 +98 1 4.969613 0.997757733 0.0032385398655591464 1 +99 1 5.979218 0.999415338 0.00084373558524890779 1 +100 1 2.79339528 0.960722744 0.05780795202304527 1 +101 1 -0.2318275 0.302438259 1.7252874395936495 0 +102 0 -2.76122284 0.0146651361 0.021313990249911121 0 +103 1 0.04465413 0.385290623 1.3759810210952059 1 +104 1 6.60187626 0.999744952 0.000368003812884943 1 +105 1 0.8556795 0.6488483 0.62404688835707656 1 +106 1 5.770711 0.9992281 0.0011140176637295268 1 +107 1 4.22342634 0.993960261 0.0087399210743085431 1 +108 0 -3.45308685 0.00588296773 0.0085123920555465731 0 +109 1 3.8569653 0.990193248 0.014217983403619341 1 +110 0 -1.93793654 0.04269534 0.062949962384238681 0 +111 1 2.511653 0.9438248 0.083408972271702769 1 +112 1 4.558591 0.9961281 0.00559683875538245 1 +113 1 5.746858 0.9992032 0.0011499902524127333 1 +114 0 -1.85905993 0.04720563 0.069763207495935814 0 +115 0 -2.6717186 0.01649289 0.023992613054936204 0 +116 0 -0.709249258 0.186617866 0.2979947929259279 0 +117 1 4.78990459 0.9971525 0.0041139235571519325 1 +118 0 -3.18429565 0.00839671 0.012165037187509468 0 +119 0 -2.61044717 0.017871445 0.026016217539325714 0 +120 0 -3.20053577 0.008218363 0.011905580918919188 0 +121 0 -2.40842676 0.0232661217 0.03396255707495445 0 +122 1 6.248937 0.9995918 0.000588988809282307 1 +123 1 2.2530992 0.9224995 0.1163800002825931 1 +124 1 4.360087 0.9949611 0.0072879979287919072 1 +125 0 -3.77104545 0.00385836349 0.0055772082517802756 0 +126 1 4.38522053 0.9951263 0.0070484426651911411 1 +127 0 -3.1808567 0.008434965 0.012220695909847559 0 +128 1 3.0595746 0.9721281 0.040781670654789444 1 +129 0 -3.55608225 0.00513212057 0.0074231494771251193 0 +130 0 -2.425513 0.0227541048 0.033206475233163314 0 +131 0 -3.38022614 0.0064791413 0.0093778381082593662 0 +132 1 5.601917 0.9990336 0.0013949375136970248 1 +133 0 -3.17389774 0.008512909 0.012334105372298619 0 +134 0 -3.2619648 0.00757709425 0.010973060955492279 0 +135 0 -2.135426 0.033140596 0.048621979644840232 0 +136 0 -3.07316566 0.00972443 0.014098046031275298 0 +137 0 -3.46695566 0.00577582652 0.0083569136151318106 0 +138 0 -2.86817455 0.012741344 0.018499983083470218 0 +139 0 ? ? ? 0 +140 0 -3.46695566 0.00577582652 0.0083569136151318106 0 +141 0 -3.72542262 0.004099309 0.0059262077288994046 0 +142 1 2.42541027 0.937409163 0.093249198662777005 1 +143 0 -2.6717186 0.01649289 0.023992613054936204 0 +144 0 -3.55087972 0.005167652 0.0074746757385812587 0 +145 0 ? ? ? 0 +146 1 0.871638536 0.653679967 0.61334361074366694 1 +147 0 -3.387481 0.00641718367 0.00928787202494262 0 +148 0 -1.00809836 0.13348192 0.20669824279844359 0 +149 1 6.76884174 0.999795854 0.00029455035625064327 1 +150 0 -3.66511059 0.004440975 0.0064212413418911849 0 +151 1 2.55708146 0.9469506 0.078638906936588593 1 +152 1 5.90378475 0.9993535 0.000932963515516277 1 +153 0 -2.569498 0.0188552048 0.027462032964967031 0 +154 0 -3.96086168 0.00299838162 0.0043322484118234967 0 +155 1 2.38984656 0.9345692 0.097626633735752158 1 +156 0 -3.30132556 0.007192563 0.010414172034410071 0 +157 0 -3.38022614 0.0064791413 0.0093778381082593662 0 +158 0 ? ? ? 0 +159 1 7.45218039 0.999917865 0.00011850091305922384 1 +160 1 5.478899 0.998861551 0.0016433699395319865 1 +161 0 -2.81278753 0.0137043 0.019907850847150646 0 +162 0 -3.1808567 0.008434965 0.012220695909847559 0 +163 0 -2.53396678 0.0197517015 0.028780862357180272 0 +164 0 ? ? ? 0 +165 0 -2.50678635 0.0204655658 0.029831885589057073 0 +166 1 5.18005133 0.9983053 0.0024469793733906718 1 +167 1 4.43091154 0.995412946 0.0066329449187538334 1 +168 0 -3.1808567 0.008434965 0.012220695909847559 0 +169 0 -3.84224653 0.00351021881 0.0050730843089374219 0 +170 0 -3.46695566 0.00577582652 0.0083569136151318106 0 +171 0 -3.57815838 0.004984033 0.0072084178999092668 0 +172 0 -3.77104545 0.00385836349 0.0055772082517802756 0 +173 1 8.774223 0.999985933 2.0294095537584014E-05 1 +174 1 3.41777229 0.9825251 0.025433816307306862 1 +175 1 5.06418657 0.9980228 0.0028553279630928199 1 +176 0 -3.38022614 0.0064791413 0.0093778381082593662 0 +177 1 3.73497438 0.9884814 0.016714271999944966 1 +178 0 -2.9662137 0.0111978734 0.016246248849105433 0 +179 1 1.43747234 0.800518155 0.32099397262954793 1 +180 0 -3.66511059 0.004440975 0.0064212413418911849 0 +181 0 -3.96086168 0.00299838162 0.0043322484118234967 0 +182 0 -2.46727276 0.0215486 0.031427902667595507 0 +183 1 5.65242147 0.999096453 0.0013041317630261005 1 +184 1 3.6821804 0.987651944 0.017925379901204307 1 +185 0 -3.4560318 0.005860053 0.0084791378411025239 0 +186 1 3.79264784 0.9893247 0.015484014543185919 1 +187 1 7.109213 0.9998703 0.0001871292597061575 1 +188 1 5.37026024 0.998684347 0.0018993362344133192 1 +189 0 -2.924244 0.0118345981 0.017175550585455768 0 +190 1 7.41118431 0.9999133 0.00012512280420358602 1 +191 1 6.48724 0.9997029 0.00042873045458892818 1 +192 0 -2.835495 0.0133011211 0.01931822515825387 0 +193 0 -3.57815838 0.004984033 0.0072084178999092668 0 +194 0 -3.1808567 0.008434965 0.012220695909847559 0 +195 0 -2.9662137 0.0111978734 0.016246248849105433 0 +196 0 3.63146925 0.9867997 6.2432873244919023 1 +197 0 -2.15677047 0.03224091 0.047280139214237252 0 +198 0 -3.96086168 0.00299838162 0.0043322484118234967 0 +199 0 -3.31544065 0.007059439 0.010220736690982309 0 +200 1 6.04754925 0.9994662 0.000770343941054464 1 +201 1 5.56266356 0.9989817 0.0014698242813019167 1 +202 0 -3.57815838 0.004984033 0.0072084178999092668 0 +203 0 -2.72854328 0.0153081212 0.022255735289143007 0 +204 0 -3.57815838 0.004984033 0.0072084178999092668 0 +205 1 7.505656 0.9999235 0.00011033108924367516 1 +206 1 4.45749474 0.995572 0.0064024812373708224 1 +207 0 -3.66511059 0.004440975 0.0064212413418911849 0 +208 0 -3.66511059 0.004440975 0.0064212413418911849 0 +209 0 -2.78515744 0.014211162 0.020649449284747601 0 +210 1 8.509815 0.99998 2.8893374672045653E-05 1 +211 1 5.90197563 0.999352 0.00093520073801267969 1 +212 0 -3.57815838 0.004984033 0.0072084178999092668 0 +213 1 8.667868 0.9999838 2.3389830121236442E-05 1 +214 1 7.77660275 0.9999467 7.6878293243468792E-05 1 +215 1 4.73319054 0.9969296 0.0044364855615384585 1 +216 0 -3.77104545 0.00385836349 0.0055772082517802756 0 +217 0 -3.57815838 0.004984033 0.0072084178999092668 0 +218 1 5.02794027 0.997925162 0.0029964677064135293 1 +219 0 -2.08089066 0.0355507471 0.052222765858833918 0 +220 0 -3.40849757 0.006241001 0.0090320754706031993 0 +221 1 5.98039627 0.999416232 0.00084244496136570062 1 +222 1 -1.57286942 0.06764871 3.8857937913639682 0 +223 1 3.10161 0.9736069 0.038588725347070051 1 +224 1 5.56522369 0.9989852 0.0014648317091916819 1 +225 0 -3.77104545 0.00385836349 0.0055772082517802756 0 +226 1 5.70770645 0.9991606 0.0012115243914031775 1 +227 1 4.619732 0.996430039 0.0051595796830069731 1 +228 0 -3.66511059 0.004440975 0.0064212413418911849 0 +229 1 7.93523645 0.999956846 6.2259062978417255E-05 1 +230 1 3.318484 0.9801014 0.02899706893187087 1 +231 1 5.3583045 0.998663247 0.0019298175880904135 1 +232 0 0.5565145 0.5535885 1.1635539328905338 1 +233 1 3.60196638 0.9862775 0.019934442868153161 1 +234 0 -1.67357576 0.0596573241 0.088741501157431082 0 +235 0 ? ? ? 0 +236 1 6.774187 0.9997973 0.00029248614451634898 1 +237 1 4.2103157 0.9938544 0.0088935778480629321 1 +238 1 7.43732643 0.999916255 0.00012082287142878181 1 +239 1 3.15491748 0.975372732 0.035974455107190835 1 +240 0 -1.4892621 0.07502612 0.11251546415675362 0 +241 0 -2.881044 0.01252734 0.018187289961738219 0 +242 0 -3.38022614 0.0064791413 0.0093778381082593662 0 +243 0 -2.09123087 0.0350811519 0.051520481343590753 0 +244 0 -3.57815838 0.004984033 0.0072084178999092668 0 +245 0 -2.23104239 0.0292907618 0.04288887279935924 0 +246 1 7.06359053 0.9998622 0.00019882564438936136 1 +247 1 1.97520256 0.8915192 0.1656622444230566 1 +248 0 -2.22128725 0.0296627749 0.043441874943560728 0 +249 0 ? ? ? 0 +250 0 -3.488491 0.005613291 0.0081210817168373139 0 +251 1 4.650688 0.996573865 0.0049513542318431174 1 +252 0 2.5789237 0.9483944 4.2763290576456479 1 +253 1 5.458683 0.9988305 0.0016882231796356319 1 +254 1 4.83182669 0.9973069 0.0038905873144013418 1 +255 1 2.97119951 0.968752265 0.045800316537980092 1 +256 0 -3.46695566 0.00577582652 0.0083569136151318106 0 +257 0 -3.31544065 0.007059439 0.010220736690982309 0 +258 0 -3.1808567 0.008434965 0.012220695909847559 0 +259 0 2.49374032 0.9425453 4.1214311429250063 1 +260 1 5.30725336 0.99856925 0.0020656138166485821 1 +261 1 7.14677572 0.9998766 0.00017801302563611457 1 +262 1 5.994073 0.9994268 0.00082721568673726261 1 +263 1 4.79329062 0.9971653 0.0040953827461603044 1 +264 1 3.24409437 0.9780725 0.031986649916454328 1 +265 0 -1.98512161 0.0401971564 0.059190007520612681 0 +266 1 4.852043 0.9973783 0.0037872952166773687 1 +267 1 1.53848267 0.821154237 0.28427486708860261 1 +268 1 4.85146046 0.997376263 0.0037902266099763627 1 +269 0 -3.57815838 0.004984033 0.0072084178999092668 0 +270 1 3.98194313 0.9916857 0.012045159256748898 1 +271 0 -2.72854328 0.0153081212 0.022255735289143007 0 +272 1 1.53848267 0.821154237 0.28427486708860261 1 +273 1 -0.15505743 0.324454337 1.6239126425549066 0 +274 0 -3.00886583 0.0105855074 0.015353063107842574 0 +275 0 ? ? ? 0 +276 0 -3.31544065 0.007059439 0.010220736690982309 0 +277 0 -3.77104545 0.00385836349 0.0055772082517802756 0 +278 0 -3.57815838 0.004984033 0.0072084178999092668 0 +279 1 3.86520982 0.990299344 0.01406341119260114 1 +280 0 -3.1808567 0.008434965 0.012220695909847559 0 +281 0 -3.17676878 0.008480665 0.012287189461582367 0 +282 1 2.461159 0.9401476 0.08904085472422596 1 +283 1 3.383401 0.9817208 0.026615304837962196 1 +284 1 4.31391859 0.9946429 0.0077494180907182793 1 +285 1 8.10043 0.99996537 4.9961825177101689E-05 1 +286 1 8.968391 0.9999891 1.5736498378211235E-05 1 +287 0 -3.2619648 0.00757709425 0.010973060955492279 0 +288 1 0.9694412 0.6825748 0.55094092419803919 1 +289 1 5.20975971 0.998371 0.0023520591908501027 1 +290 0 -3.96086168 0.00299838162 0.0043322484118234967 0 +291 0 -3.57815838 0.004984033 0.0072084178999092668 0 +292 1 ? ? ? 0 +293 1 3.40588856 0.982251048 0.025836292808480849 1 +294 0 ? ? ? 0 +295 1 4.11647463 0.993041158 0.010074581754464447 1 +296 0 1.19525409 0.743959 1.9655533008943613 1 +297 0 ? ? ? 0 +298 0 -1.84992671 0.04775626 0.070597193841832967 0 +299 1 3.7101984 0.9880992 0.017272181330847947 1 +300 1 4.209749 0.9938498 0.0089002401391625394 1 +301 0 -3.57815838 0.004984033 0.0072084178999092668 0 +302 1 9.011894 0.999989748 1.4790583790855475E-05 1 +303 0 -3.57815838 0.004984033 0.0072084178999092668 0 +304 1 3.334587 0.9805158 0.028387249015995583 1 +305 1 5.532424 0.9989399 0.001530252989699247 1 +306 0 -3.57815838 0.004984033 0.0072084178999092668 0 +307 0 -3.57815838 0.004984033 0.0072084178999092668 0 +308 1 4.14736 0.99332 0.0096695522623947427 1 +309 0 -1.586181 0.0665380359 0.099336856879818441 0 +310 0 -3.50334549 0.005503838 0.0079622914531571748 0 +311 0 -3.96086168 0.00299838162 0.0043322484118234967 0 +312 1 2.21150637 0.9184415 0.12274030303208396 1 +313 0 -3.96086168 0.00299838162 0.0043322484118234967 0 +314 0 -3.84529161 0.00349604874 0.0050525693482786306 0 +315 0 ? ? ? 0 +316 1 2.32279563 0.9288866 0.10642562592908095 1 +317 1 5.58706236 0.999014258 0.0014228259246157244 1 +318 0 -3.44392776 0.0059548053 0.0086166488410348817 0 +319 0 1.25488234 0.7588043 2.0517240349053663 1 +320 1 3.84496284 0.9900366 0.014446224790024031 1 +321 0 ? ? ? 0 +322 0 -3.1808567 0.008434965 0.012220695909847559 0 +323 1 3.05829477 0.97208184 0.040850314752739043 1 +324 0 -3.57815838 0.004984033 0.0072084178999092668 0 +325 0 -2.59425545 0.0182542633 0.026578667025882345 0 +326 1 2.05792737 0.901733 0.14922780522257925 1 +327 0 -3.77104545 0.00385836349 0.0055772082517802756 0 +328 1 3.057047 0.97203666 0.040917369744188303 1 +329 1 3.970356 0.99155736 0.012231862872406163 1 +330 1 2.9581387 0.9682209 0.046591873905363942 1 +331 0 -2.60156059 0.0180805624 0.026323432542080995 0 +332 0 -2.102426 0.03457947 0.05077058949808224 0 +333 1 2.74529266 0.9582303 0.061555636486182085 1 +334 1 3.7646997 0.988923967 0.016068490804133879 1 +335 0 -3.96086168 0.00299838162 0.0043322484118234967 0 +336 1 3.31676149 0.9800566 0.029063048793749918 1 +337 0 -3.57815838 0.004984033 0.0072084178999092668 0 +338 0 -3.84529161 0.00349604874 0.0050525693482786306 0 +339 1 3.301907 0.9796658 0.029638394707112933 1 +340 1 3.463615 0.983544 0.023938511949178617 1 +341 0 -3.57815838 0.004984033 0.0072084178999092668 0 +342 0 -3.72542262 0.004099309 0.0059262077288994046 0 +343 0 -3.96086168 0.00299838162 0.0043322484118234967 0 +344 1 5.28700352 0.998530149 0.0021221060568469914 1 +345 0 -3.96086168 0.00299838162 0.0043322484118234967 0 +346 0 -2.092176 0.03503853 0.051456758042617047 0 +347 0 -3.71749568 0.00414267555 0.0059890313601239659 0 +348 1 -0.1384623 0.329321831 1.6024299403729965 0 +349 1 1.90731454 0.882451832 0.1804105623465225 1 +350 0 -2.62119436 0.0176217146 0.025649423352103927 0 +351 0 -3.38022614 0.0064791413 0.0093778381082593662 0 +352 0 0.629153 0.577376 1.2425534248501178 1 +353 1 5.572207 0.9989945 0.001451317419643085 1 +354 0 -3.77104545 0.00385836349 0.0055772082517802756 0 +355 0 -2.899517 0.0122263413 0.017747598491742872 0 +356 1 -0.6585767 0.197088882 2.3430816978259372 0 +357 1 7.385506 0.999910235 0.00012950874883685672 1 +358 1 4.16058159 0.9934359 0.0095011841931019435 1 +359 1 2.88886786 0.9652534 0.051020344958120654 1 +360 1 8.8277 0.9999869 1.8918215632667518E-05 1 +361 1 3.82925034 0.9898279 0.014750429068757108 1 +362 0 -2.10720563 0.0343674 0.050453713946388694 0 +363 0 -0.95842576 0.14132829 0.21982143387654562 0 +364 0 -3.38022614 0.0064791413 0.0093778381082593662 0 +365 0 -3.55087972 0.005167652 0.0074746757385812587 0 +366 1 8.194901 0.9999695 4.4028230441777205E-05 1 +367 1 6.75759029 0.999792755 0.00029902282514038188 1 +368 0 -3.55087972 0.005167652 0.0074746757385812587 0 +369 0 -3.43820667 0.00600011973 0.0086824168721458487 0 +370 0 -2.325242 0.0259238239 0.037893494520334511 0 +371 0 -3.55087972 0.005167652 0.0074746757385812587 0 +372 0 -2.86817455 0.012741344 0.018499983083470218 0 +373 0 -2.64238739 0.01713926 0.024941076641326766 0 +374 0 -3.11044979 0.009257291 0.013417650275024486 0 +375 0 -3.96086168 0.00299838162 0.0043322484118234967 0 +376 0 -3.77104545 0.00385836349 0.0055772082517802756 0 +377 0 -3.84529161 0.00349604874 0.0050525693482786306 0 +378 0 -2.56472182 0.0189733524 0.027635770006354959 0 +379 0 -1.49042392 0.07491871 0.11234794875412171 0 +380 0 -3.96086168 0.00299838162 0.0043322484118234967 0 +381 1 5.51768875 0.998918831 0.0015606404618770881 1 +382 0 -2.36386371 0.0246550012 0.036015476061441752 0 +383 0 -3.72542262 0.004099309 0.0059262077288994046 0 +384 0 -3.72542262 0.004099309 0.0059262077288994046 0 +385 0 -2.25260377 0.0284845009 0.041691082883248826 0 +386 1 3.23263741 0.977742553 0.032473452567425891 1 +387 0 -1.56172526 0.06859173 0.10251440109409105 0 +388 0 -3.39077234 0.006389269 0.0092473403351480687 0 +389 0 -2.33341575 0.0256501 0.037488142686586956 0 +390 0 -3.57761765 0.00498760864 0.0072136025574007509 0 +391 1 6.082037 0.9994902 0.00073567134443573202 1 +392 0 -3.31544065 0.007059439 0.010220736690982309 0 +393 0 -4.017849 0.00277965143 0.0040157731521621729 0 +394 0 -3.09633446 0.00943147 0.013671307351114586 0 +395 0 -3.31544065 0.007059439 0.010220736690982309 0 +396 0 -3.1808567 0.008434965 0.012220695909847559 0 +397 0 -3.208489 0.008132397 0.011780536298829664 0 +398 0 -2.929965 0.0117457407 0.017045826865115216 0 +399 0 -3.28350019 0.0073642456 0.010663673931054517 0 +400 1 5.59894562 0.999029756 0.0014004462928796737 1 +401 0 -3.46695566 0.00577582652 0.0083569136151318106 0 +402 0 -1.998445 0.0395174921 0.058168754741571183 0 +403 0 -2.51305151 0.02029881 0.029586303226256196 0 +404 0 -3.23840714 0.007816927 0.011321749726755549 0 +405 0 -3.77104545 0.00385836349 0.0055772082517802756 0 +406 0 -2.660952 0.016727319 0.02433653489159535 0 +407 0 -3.77104545 0.00385836349 0.0055772082517802756 0 +408 0 -2.2173748 0.02981326 0.043665632975899607 0 +409 0 -3.11044979 0.009257291 0.013417650275024486 0 +410 0 -3.77104545 0.00385836349 0.0055772082517802756 0 +411 0 ? ? ? 0 +412 1 5.62802267 0.9990666 0.00134725302298929 1 +413 0 -2.52355218 0.0200223 0.029179173440852326 0 +414 1 3.89722729 0.9907009 0.013478531338153847 1 +415 0 -0.4339671 0.248771548 0.41267639080041052 0 +416 1 5.004137 0.9978584 0.0030929814628056409 1 +417 0 -3.77104545 0.00385836349 0.0055772082517802756 0 +418 0 -1.54389989 0.07012546 0.10489201697967264 0 +419 0 -3.06068373 0.009885975 0.014333414544704173 0 +420 0 -1.79550922 0.0511654466 0.075771545949084612 0 +421 1 6.86263037 0.9998198 0.00025997519927756718 1 +422 0 -1.77265787 0.0526646748 0.078052912755774154 0 +423 0 -2.425513 0.0227541048 0.033206475233163314 0 +424 0 -3.46695566 0.00577582652 0.0083569136151318106 0 +425 1 8.762936 0.9999857 2.0638065718838105E-05 1 +426 0 -1.52166367 0.0720831454 0.10793255548434401 0 +427 1 3.055027 0.9719634 0.041026097455560531 1 +428 0 -3.77104545 0.00385836349 0.0055772082517802756 0 +429 0 -3.55087972 0.005167652 0.0074746757385812587 0 +430 0 -3.15302753 0.008750969 0.012680543819513564 0 +431 0 -2.07732153 0.03571424 0.052467349827131367 0 +432 0 -2.71963072 0.0154882455 0.022519663750552844 0 +433 0 -2.67167354 0.0164938644 0.023994042044525644 0 +434 0 3.333221 0.980480969 5.6789747525091601 1 +435 1 4.777939 0.9971069 0.0041798962820680978 1 +436 1 3.44759536 0.983194768 0.024450855724695516 1 +437 0 -3.208489 0.008132397 0.011780536298829664 0 +438 0 -2.468914 0.021502519 0.031359958078789663 0 +439 0 -2.966979 0.011186582 0.016229774482531711 0 +440 1 4.73981476 0.9969565 0.0043974983005904116 1 +441 0 -1.11423659 0.117948353 0.18106496260780325 0 +442 0 -3.02309179 0.0103887226 0.015066153896093903 0 +443 0 -3.622181 0.004701302 0.0067985385098899512 0 +444 0 -1.87156582 0.0464614555 0.068636837733176956 0 +445 0 -3.72542262 0.004099309 0.0059262077288994046 0 +446 0 -3.96086168 0.00299838162 0.0043322484118234967 0 +447 0 -2.966979 0.011186582 0.016229774482531711 0 +448 0 -4.017849 0.00277965143 0.0040157731521621729 0 +449 1 6.273336 0.9996049 0.00057014914213751806 1 +450 0 -2.801452 0.0139100533 0.020208846111157287 0 +451 0 -2.966979 0.011186582 0.016229774482531711 0 +452 0 -3.11953735 0.009146843 0.013256827099540507 0 +453 1 4.70173836 0.9967986 0.0046260890764819595 1 +454 0 -3.15575218 0.008719519 0.012634771470188627 0 +455 1 0.449859619 0.518243134 0.94829899706618082 1 +456 1 5.57815 0.999002457 0.001439869107795693 1 +457 1 5.216879 0.9983864 0.0023298374005677226 1 +458 0 -2.745031 0.01498032 0.021775545833862112 0 +459 0 -2.54004 0.01959556 0.028551076610933101 0 +460 0 -2.62119436 0.0176217146 0.025649423352103927 0 +461 0 -2.20826054 0.0301666968 0.044191299393516507 0 +462 0 -2.3835237 0.0240325723 0.035095095313588072 0 +463 0 -3.104508 0.00933022052 0.013523851702734787 0 +464 0 -3.208489 0.008132397 0.011780536298829664 0 +465 1 5.826476 0.9992834 0.001034244333462264 1 +466 1 5.24466038 0.9984449 0.0022452599185463717 1 +467 1 4.419505 0.995343 0.0067342811207059834 1 +468 0 -3.208489 0.008132397 0.011780536298829664 0 +469 0 -3.56870484 0.00504692 0.0072996021268706733 0 +470 0 -3.396992 0.00633684872 0.0091712293529778808 0 +471 0 -2.3835237 0.0240325723 0.035095095313588072 0 +472 0 -2.771068 0.0144766942 0.021038106101994528 0 +473 0 -3.208489 0.008132397 0.011780536298829664 0 +474 0 -2.966979 0.011186582 0.016229774482531711 0 +475 0 -3.46695566 0.00577582652 0.0083569136151318106 0 +476 0 -3.003498 0.0106607126 0.015462726283116499 0 +477 0 -3.208489 0.008132397 0.011780536298829664 0 +478 0 -2.74636841 0.014954037 0.021737051539210239 0 +479 1 4.83839941 0.9973303 0.003856701862908137 1 +480 0 -3.04141855 0.01014054 0.014704388752658356 0 +481 0 -1.81833625 0.0497081876 0.073557495459187813 0 +482 1 8.325621 0.99997437 3.6976743794448992E-05 1 +483 1 6.48310661 0.999701262 0.00043105291231427129 1 +484 0 -2.745031 0.01498032 0.021775545833862112 0 +485 0 -3.11370373 0.009217593 0.013359843377174928 0 +486 0 -3.396992 0.00633684872 0.0091712293529778808 0 +487 1 7.43303967 0.9999158 0.0001215108598115232 1 +488 1 0.7742307 0.6237289 0.68100905453176719 1 +489 1 -0.7092824 0.186611161 2.4218928229922314 0 +490 0 -3.96086168 0.00299838162 0.0043322484118234967 0 +491 1 3.84166765 0.9899932 0.014509457862826895 1 +492 0 -3.15932131 0.008678491 0.012575061807727175 0 +493 1 6.05943251 0.9994746 0.00075821273927700959 1 +494 0 0.0942180157 0.401052684 0.73949898619610765 1 +495 0 -3.396992 0.00633684872 0.0091712293529778808 0 +496 0 -4.017849 0.00277965143 0.0040157731521621729 0 +497 0 -3.000527 0.0107025672 0.01552376161803456 0 +498 0 -3.07316566 0.00972443 0.014098046031275298 0 +499 0 -3.07316566 0.00972443 0.014098046031275298 0 +500 0 -2.46727276 0.0215486 0.031427902667595507 0 +501 0 -3.07316566 0.00972443 0.014098046031275298 0 +502 0 -2.88169169 0.0125166625 0.01817169008021928 0 +503 0 -2.9662137 0.0111978734 0.016246248849105433 0 +504 0 -3.96086168 0.00299838162 0.0043322484118234967 0 +505 0 -2.98047 0.0109893922 0.015942099916508405 0 +506 1 6.16387653 0.999542832 0.00065970427526997554 1 +507 0 -2.98819137 0.0108780833 0.015779739945598364 0 +508 0 -2.966979 0.011186582 0.016229774482531711 0 +509 0 -3.72542262 0.004099309 0.0059262077288994046 0 +510 0 -3.96086168 0.00299838162 0.0043322484118234967 0 +511 0 -2.835495 0.0133011211 0.01931822515825387 0 +512 0 -2.966979 0.011186582 0.016229774482531711 0 +513 0 -3.396992 0.00633684872 0.0091712293529778808 0 +514 1 6.133228 0.9995238 0.00068714831551900063 1 +515 1 5.495037 0.998885751 0.0016084180934291903 1 +516 0 -4.017849 0.00277965143 0.0040157731521621729 0 +517 0 -3.84529161 0.00349604874 0.0050525693482786306 0 +518 0 -3.0866828 0.009552434 0.013847493724459507 0 +519 1 3.52229786 0.9847634 0.022150975325307833 1 +520 0 -3.91332746 0.003193897 0.0046151934644252475 0 +521 0 -3.155013 0.00872803852 0.012647171026249994 0 +522 1 2.51115465 0.9437896 0.083462818939279745 1 +523 1 4.19020128 0.993688464 0.0091344781621669185 1 +524 0 -3.31544065 0.007059439 0.010220736690982309 0 +525 0 -3.29167366 0.007285025 0.010548539407608692 0 +526 0 -3.208489 0.008132397 0.011780536298829664 0 +527 0 -2.9662137 0.0111978734 0.016246248849105433 0 +528 0 -2.106291 0.0344078839 0.050514198086734907 0 +529 0 -3.15932131 0.008678491 0.012575061807727175 0 +530 1 3.85399461 0.9901547 0.014274171901286254 1 +531 0 -2.660952 0.016727319 0.02433653489159535 0 +532 0 -3.66511059 0.004440975 0.0064212413418911849 0 +533 0 -3.31544065 0.007059439 0.010220736690982309 0 +534 0 -3.55087972 0.005167652 0.0074746757385812587 0 +535 0 -2.99257421 0.0108153978 0.015688312251019798 0 +536 0 -2.72854328 0.0153081212 0.022255735289143007 0 +537 0 -2.52355218 0.0200223 0.029179173440852326 0 +538 0 -3.07316566 0.00972443 0.014098046031275298 0 +539 0 -2.59875679 0.0181470383 0.026421106297944578 0 +540 0 -2.53543568 0.0197138246 0.028725117558180274 0 +541 0 -3.46695566 0.00577582652 0.0083569136151318106 0 +542 0 -2.55029 0.0193347726 0.028167371490982439 0 +543 0 -3.07316566 0.00972443 0.014098046031275298 0 +544 0 -2.94481969 0.0115180928 0.016713535275056118 0 +545 0 -2.835495 0.0133011211 0.01931822515825387 0 +546 1 7.2210474 0.999888241 0.00016124274549225162 1 +547 0 -3.78208971 0.00380218937 0.0054958545229201947 0 +548 0 -3.52043152 0.005380564 0.0077834714222184374 0 +549 1 3.78554416 0.9892242 0.015630567783235471 1 +550 0 -3.31544065 0.007059439 0.010220736690982309 0 +551 0 -3.57815838 0.004984033 0.0072084178999092668 0 +552 0 -2.221508 0.0296543036 0.043429279875101832 0 +553 0 -0.8815942 0.154219717 0.24164516616272672 0 +554 0 -3.46695566 0.00577582652 0.0083569136151318106 0 +555 0 -1.28434277 0.0963231847 0.14612118539267491 0 +556 0 -2.32524228 0.0259238165 0.03789348348534987 0 +557 0 -2.62119436 0.0176217146 0.025649423352103927 0 +558 0 -3.55087972 0.005167652 0.0074746757385812587 0 +559 0 -2.835495 0.0133011211 0.01931822515825387 0 +560 0 -2.72854328 0.0153081212 0.022255735289143007 0 +561 0 -2.72854328 0.0153081212 0.022255735289143007 0 +562 0 -3.57815838 0.004984033 0.0072084178999092668 0 +563 0 -3.31544065 0.007059439 0.010220736690982309 0 +564 0 -2.81278753 0.0137043 0.019907850847150646 0 +565 1 7.007144 0.9998514 0.00021439230344064893 1 +566 0 -2.97586584 0.0110562993 0.016039702379854602 0 +567 0 -2.45182514 0.0219870787 0.032074569018795282 0 +568 1 2.7840333 0.9602491 0.058519349307142761 1 +569 1 5.680218 0.999129236 0.0012567945385539854 1 +570 1 4.69728231 0.996779561 0.0046536086728484602 1 +571 1 6.432601 0.999680459 0.00046107316533662714 1 +572 0 -3.31544065 0.007059439 0.010220736690982309 0 +573 0 -3.77104545 0.00385836349 0.0055772082517802756 0 +574 1 3.61557317 0.9865208 0.019578586333053878 1 +575 0 -2.52355218 0.0200223 0.029179173440852326 0 +576 0 -2.835495 0.0133011211 0.01931822515825387 0 +577 0 -3.77104545 0.00385836349 0.0055772082517802756 0 +578 0 -3.77104545 0.00385836349 0.0055772082517802756 0 +579 0 -3.57815838 0.004984033 0.0072084178999092668 0 +580 0 -2.630504 0.0174081642 0.025335842899848166 0 +581 1 4.95723438 0.99772054 0.0032923200422882576 1 +582 1 5.53655672 0.998945653 0.0015219030032144873 1 +583 0 -3.46695566 0.00577582652 0.0083569136151318106 0 +584 0 -2.04167128 0.0373874865 0.05497291719135828 0 +585 0 -3.96086168 0.00299838162 0.0043322484118234967 0 +586 1 8.320274 0.9999742 3.7234724405850826E-05 1 +587 0 -2.71963072 0.0154882455 0.022519663750552844 0 +588 1 3.21735668 0.9772949 0.033134099612461547 1 +589 0 -2.966979 0.011186582 0.016229774482531711 0 +590 1 2.198461 0.917129338 0.1248028912448389 1 +591 1 3.5133853 0.9845841 0.022413663256857307 1 +592 1 3.13301015 0.9746614 0.037026970549206631 1 +593 0 -2.745031 0.01498032 0.021775545833862112 0 +594 1 3.0187335 0.970614433 0.043029781375990617 1 +595 0 -2.835495 0.0133011211 0.01931822515825387 0 +596 0 -2.86817455 0.012741344 0.018499983083470218 0 +597 0 -2.31856132 0.0261496622 0.03822802023960048 0 +598 0 -3.31544065 0.007059439 0.010220736690982309 0 +599 0 -2.21090865 0.0300635919 0.0440379318376743 0 +600 0 -3.31544065 0.007059439 0.010220736690982309 0 +601 0 -3.84529161 0.00349604874 0.0050525693482786306 0 +602 0 -3.07316566 0.00972443 0.014098046031275298 0 +603 1 2.5055778 0.9433939 0.084067843751878188 1 +604 1 2.78688526 0.960393965 0.058301756645443528 1 +605 1 5.62686062 0.999065161 0.0013493187444396134 1 +606 0 -3.033207 0.010251008 0.014865401891949757 0 +607 0 -3.96086168 0.00299838162 0.0043322484118234967 0 +608 1 6.105055 0.9995056 0.0007134744371953837 1 +609 0 -2.966979 0.011186582 0.016229774482531711 0 +610 1 4.45461035 0.995555 0.0064270979778478406 1 +611 1 4.0742507 0.9926412 0.01065574398933648 1 +612 1 9.3684 0.9999936 9.2011011621090467E-06 1 +613 0 -3.040134 0.0101577435 0.014729462475731883 0 +614 0 -3.61975074 0.004716485 0.0068205464175125014 0 +615 0 -2.66318369 0.01667846 0.024264848825346654 0 +616 0 -3.31544065 0.007059439 0.010220736690982309 0 +617 0 ? ? ? 0 +618 0 -3.07316566 0.00972443 0.014098046031275298 0 +619 0 -2.835495 0.0133011211 0.01931822515825387 0 +620 0 -3.31544065 0.007059439 0.010220736690982309 0 +621 0 -0.4626994 0.2416827 0.39912644579911988 0 +622 0 -1.79068446 0.0514785871 0.076247751984433357 0 +623 0 -3.96086168 0.00299838162 0.0043322484118234967 0 +624 0 -2.72854328 0.0153081212 0.022255735289143007 0 +625 0 -2.10306931 0.0345508568 0.050727830309042196 0 +626 1 3.4206562 0.982591033 0.025337021615257107 1 +627 0 -2.43998957 0.02232892 0.032578918077598676 0 +628 0 -3.72542262 0.004099309 0.0059262077288994046 0 +629 0 -3.208489 0.008132397 0.011780536298829664 0 +630 0 -2.09871578 0.0347449668 0.051017922913125124 0 +631 0 -2.835495 0.0133011211 0.01931822515825387 0 +632 0 -3.96086168 0.00299838162 0.0043322484118234967 0 +633 1 2.92632771 0.966889858 0.04857653818694381 1 +634 0 -3.46695566 0.00577582652 0.0083569136151318106 0 +635 0 -2.851983 0.0130157238 0.018900993803036854 0 +636 1 5.70410347 0.999156535 0.0012173767259136749 1 +637 0 -1.5962137 0.06571217 0.098061017484216279 0 +638 0 -3.208489 0.008132397 0.011780536298829664 0 +639 0 -2.62119436 0.0176217146 0.025649423352103927 0 +640 0 -2.86107063 0.0128610171 0.018674873660404948 0 +641 0 -3.31544065 0.007059439 0.010220736690982309 0 +642 0 -3.31544065 0.007059439 0.010220736690982309 0 +643 0 -3.96086168 0.00299838162 0.0043322484118234967 0 +644 0 -3.72542262 0.004099309 0.0059262077288994046 0 +645 0 -3.31544065 0.007059439 0.010220736690982309 0 +646 0 -3.488491 0.005613291 0.0081210817168373139 0 +647 0 -3.571877 0.00502573047 0.0072688774713218383 0 +648 1 6.552003 0.9997274 0.00039329193382559838 1 +649 0 -3.31544065 0.007059439 0.010220736690982309 0 +650 0 -2.31083 0.0264134 0.03861878288697667 0 +651 0 -3.20869827 0.008130147 0.011777263514393659 0 +652 0 -2.71963072 0.0154882455 0.022519663750552844 0 +653 0 -3.07316566 0.00972443 0.014098046031275298 0 +654 0 -3.1808567 0.008434965 0.012220695909847559 0 +655 0 -3.31544065 0.007059439 0.010220736690982309 0 +656 0 -2.835495 0.0133011211 0.01931822515825387 0 +657 0 -0.134364128 0.330529571 0.57890776192269999 0 +658 1 4.97314835 0.9977683 0.0032232852765622551 1 +659 0 -3.96086168 0.00299838162 0.0043322484118234967 0 +660 0 -3.77104545 0.00385836349 0.0055772082517802756 0 +661 0 -2.9662137 0.0111978734 0.016246248849105433 0 +662 0 -3.36371422 0.00662238058 0.0095858513447582352 0 +663 0 -3.36371422 0.00662238058 0.0095858513447582352 0 +664 0 -2.94796562 0.0114704445 0.016643993945319978 0 +665 0 -3.96086168 0.00299838162 0.0043322484118234967 0 +666 0 -2.2578063 0.02829321 0.041407043609767206 0 +667 0 -3.1808567 0.008434965 0.012220695909847559 0 +668 1 1.71023417 0.8523483 0.23048496037992794 1 +669 1 5.366621 0.998677969 0.0019085494570210317 1 +670 1 4.08973646 0.9927905 0.010438755142753803 1 +671 0 -2.72061634 0.015468224 0.022490324608770131 0 +672 0 -3.38022614 0.0064791413 0.0093778381082593662 0 +673 0 -2.27563167 0.0276472 0.040448229876771359 0 +674 0 -3.77104545 0.00385836349 0.0055772082517802756 0 +675 0 -2.59351587 0.01827194 0.026604643234467276 0 +676 0 -3.56870484 0.00504692 0.0072996021268706733 0 +677 0 -2.966979 0.011186582 0.016229774482531711 0 +678 0 -3.96086168 0.00299838162 0.0043322484118234967 0 +679 0 -3.72542262 0.004099309 0.0059262077288994046 0 +680 1 9.424847 0.9999941 8.5131663352343685E-06 1 +681 1 6.10102463 0.9995029 0.00071734596619533813 1 +682 0 -2.60866475 0.0179131981 0.026077551876483297 0 +683 0 -3.96086168 0.00299838162 0.0043322484118234967 0 +684 0 -3.96086168 0.00299838162 0.0043322484118234967 0 +685 0 -3.96086168 0.00299838162 0.0043322484118234967 0 +686 0 -3.96086168 0.00299838162 0.0043322484118234967 0 +687 0 -3.01049924 0.0105627263 0.01531984570989138 0 +688 0 -3.208489 0.008132397 0.011780536298829664 0 +689 0 -3.12261677 0.009109714 0.013202767753088126 0 +690 0 -3.571877 0.00502573047 0.0072688774713218383 0 +691 1 3.37906361 0.9816168 0.026768161751618318 1 +692 0 -3.46695566 0.00577582652 0.0083569136151318106 0 +693 0 -3.21145988 0.008100514 0.011734162611424938 0 +694 0 -3.08965349 0.00951504 0.013793025632202006 0 +695 0 -3.72542262 0.004099309 0.0059262077288994046 0 +696 1 4.598934 0.9963301 0.0053043110546258845 1 +697 1 2.802182 0.9611624 0.05714790005987188 1 +698 1 3.39275622 0.98194325 0.026288446863364429 1 diff --git a/test/BaselineOutput/Common/WeightedEnsemble/WE-Average-TrainTest-breast-cancer-out.txt b/test/BaselineOutput/Common/WeightedEnsemble/WE-Average-TrainTest-breast-cancer-out.txt new file mode 100644 index 0000000000..6137deb3e9 --- /dev/null +++ b/test/BaselineOutput/Common/WeightedEnsemble/WE-Average-TrainTest-breast-cancer-out.txt @@ -0,0 +1,118 @@ +maml.exe TrainTest test=%Data% tr=WeightedEnsemble{nm=20 oc=Average tp=-} dout=%Output% loader=Text{col=Label:BL:0 col=Features:R4:1-9} data=%Data% out=%Output% seed=1 +Automatically adding a MinMax normalization transform, use 'norm=Warn' or 'norm=No' to turn this behavior off. +Training 20 learners for the batch 1 +Beginning training model 1 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 26 instances with missing features during training (over 1 iterations; 26 inst/iter) +Trainer 1 of 20 finished in %Time% +Beginning training model 2 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 15 instances with missing features during training (over 1 iterations; 15 inst/iter) +Trainer 2 of 20 finished in %Time% +Beginning training model 3 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 11 instances with missing features during training (over 1 iterations; 11 inst/iter) +Trainer 3 of 20 finished in %Time% +Beginning training model 4 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 12 instances with missing features during training (over 1 iterations; 12 inst/iter) +Trainer 4 of 20 finished in %Time% +Beginning training model 5 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 18 instances with missing features during training (over 1 iterations; 18 inst/iter) +Trainer 5 of 20 finished in %Time% +Beginning training model 6 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 15 instances with missing features during training (over 1 iterations; 15 inst/iter) +Trainer 6 of 20 finished in %Time% +Beginning training model 7 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 24 instances with missing features during training (over 1 iterations; 24 inst/iter) +Trainer 7 of 20 finished in %Time% +Beginning training model 8 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 18 instances with missing features during training (over 1 iterations; 18 inst/iter) +Trainer 8 of 20 finished in %Time% +Beginning training model 9 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 15 instances with missing features during training (over 1 iterations; 15 inst/iter) +Trainer 9 of 20 finished in %Time% +Beginning training model 10 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 18 instances with missing features during training (over 1 iterations; 18 inst/iter) +Trainer 10 of 20 finished in %Time% +Beginning training model 11 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 14 instances with missing features during training (over 1 iterations; 14 inst/iter) +Trainer 11 of 20 finished in %Time% +Beginning training model 12 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 13 instances with missing features during training (over 1 iterations; 13 inst/iter) +Trainer 12 of 20 finished in %Time% +Beginning training model 13 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 14 instances with missing features during training (over 1 iterations; 14 inst/iter) +Trainer 13 of 20 finished in %Time% +Beginning training model 14 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 17 instances with missing features during training (over 1 iterations; 17 inst/iter) +Trainer 14 of 20 finished in %Time% +Beginning training model 15 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 24 instances with missing features during training (over 1 iterations; 24 inst/iter) +Trainer 15 of 20 finished in %Time% +Beginning training model 16 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 12 instances with missing features during training (over 1 iterations; 12 inst/iter) +Trainer 16 of 20 finished in %Time% +Beginning training model 17 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 12 instances with missing features during training (over 1 iterations; 12 inst/iter) +Trainer 17 of 20 finished in %Time% +Beginning training model 18 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 19 instances with missing features during training (over 1 iterations; 19 inst/iter) +Trainer 18 of 20 finished in %Time% +Beginning training model 19 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 23 instances with missing features during training (over 1 iterations; 23 inst/iter) +Trainer 19 of 20 finished in %Time% +Beginning training model 20 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 18 instances with missing features during training (over 1 iterations; 18 inst/iter) +Trainer 20 of 20 finished in %Time% +Training calibrator. +Warning: The predictor produced non-finite prediction values on 16 instances during testing. Possible causes: abnormal data or the predictor is numerically unstable. +TEST POSITIVE RATIO: 0.3499 (239.0/(239.0+444.0)) +Confusion table + ||====================== +PREDICTED || positive | negative | Recall +TRUTH ||====================== + positive || 231 | 8 | 0.9665 + negative || 10 | 434 | 0.9775 + ||====================== +Precision || 0.9585 | 0.9819 | +OVERALL 0/1 ACCURACY: 0.973646 +LOG LOSS/instance: 0.116559 +Test-set entropy (prior Log-Loss/instance): 0.934003 +LOG-LOSS REDUCTION (RIG): 0.875205 +AUC: 0.995920 + +OVERALL RESULTS +--------------------------------------- +AUC: 0.995920 (0.0000) +Accuracy: 0.973646 (0.0000) +Positive precision: 0.958506 (0.0000) +Positive recall: 0.966527 (0.0000) +Negative precision: 0.981900 (0.0000) +Negative recall: 0.977477 (0.0000) +Log-loss: 0.116559 (0.0000) +Log-loss reduction: 0.875205 (0.0000) +F1 Score: 0.962500 (0.0000) +AUPRC: 0.991714 (0.0000) + +--------------------------------------- +Physical memory usage(MB): %Number% +Virtual memory usage(MB): %Number% +%DateTime% Time elapsed(s): %Number% + diff --git a/test/BaselineOutput/Common/WeightedEnsemble/WE-Average-TrainTest-breast-cancer-rp.txt b/test/BaselineOutput/Common/WeightedEnsemble/WE-Average-TrainTest-breast-cancer-rp.txt new file mode 100644 index 0000000000..9b3a12e295 --- /dev/null +++ b/test/BaselineOutput/Common/WeightedEnsemble/WE-Average-TrainTest-breast-cancer-rp.txt @@ -0,0 +1,4 @@ +WeightedEnsemble +AUC Accuracy Positive precision Positive recall Negative precision Negative recall Log-loss Log-loss reduction F1 Score AUPRC /oc /bp /nm Learner Name Train Dataset Test Dataset Results File Run Time Physical Memory Virtual Memory Command Line Settings +0.99592 0.973646 0.958506 0.966527 0.9819 0.977477 0.116559 0.875205 0.9625 0.991714 Average Microsoft.ML.Runtime.ComponentFactoryUtils+SimpleComponentFactory`1[Microsoft.ML.Trainers.LinearSvmTrainer] 20 WeightedEnsemble %Data% %Data% %Output% 99 0 0 maml.exe TrainTest test=%Data% tr=WeightedEnsemble{nm=20 oc=Average tp=-} dout=%Output% loader=Text{col=Label:BL:0 col=Features:R4:1-9} data=%Data% out=%Output% seed=1 /oc:Average;/bp:Microsoft.ML.Runtime.ComponentFactoryUtils+SimpleComponentFactory`1[Microsoft.ML.Trainers.LinearSvmTrainer];/nm:20 + diff --git a/test/BaselineOutput/Common/WeightedEnsemble/WE-Average-TrainTest-breast-cancer.txt b/test/BaselineOutput/Common/WeightedEnsemble/WE-Average-TrainTest-breast-cancer.txt new file mode 100644 index 0000000000..ec1f5268a8 --- /dev/null +++ b/test/BaselineOutput/Common/WeightedEnsemble/WE-Average-TrainTest-breast-cancer.txt @@ -0,0 +1,700 @@ +Instance Label Score Probability Log-loss Assigned +0 0 -2.6282382 0.01783696 0.0259655617646397 0 +1 0 1.96705842 0.9275491 3.7868530516332033 1 +2 0 -2.85387278 0.0129900519 0.018863469141483385 0 +3 0 1.93591881 0.924505353 3.7274818364991082 1 +4 0 -2.598872 0.0185861979 0.027066533968906802 0 +5 1 6.95403957 0.99993664 9.1411674874021938E-05 1 +6 0 -1.23903906 0.116509691 0.17871378474240457 0 +7 0 -3.165355 0.008367277 0.012122214657951928 0 +8 0 -3.266156 0.00725429226 0.010503876792076317 0 +9 0 -2.83666182 0.013308771 0.019329410426821694 0 +10 0 -3.686479 0.003994874 0.0057749279345070377 0 +11 0 -3.51190233 0.00511935074 0.0074046315896624214 0 +12 1 -0.526532054 0.267163843 1.904203323507824 0 +13 0 -3.07950687 0.009447551 0.01369472862738292 0 +14 1 4.862514 0.9987481 0.00180720724421216 1 +15 1 0.542482 0.626345456 0.6749695117196477 1 +16 0 -3.023344 0.0102279028 0.014831723310481116 0 +17 0 -2.87251735 0.0126532819 0.018371302375292598 0 +18 1 4.35022163 0.997402847 0.0037517741006701621 1 +19 0 -2.38395858 0.02509029 0.036659483516313734 0 +20 1 3.25616026 0.987743437 0.017791739126429732 1 +21 1 4.06150436 0.996083736 0.0056610664098227671 1 +22 0 -3.26762271 0.007239233 0.010481991952021335 0 +23 1 ? ? ? 0 +24 0 -3.605355 0.004483021 0.0064821727077381958 0 +25 1 0.5821989 0.6395132 0.64495396950781159 1 +26 0 -3.16206551 0.008406319 0.012179016707027899 0 +27 0 -2.77906466 0.0144325038 0.020973417897420254 0 +28 0 -3.51190233 0.00511935074 0.0074046315896624214 0 +29 0 -3.51556587 0.00509279035 0.0073661163875401612 0 +30 0 -3.34874725 0.006452909 0.0093397468928123339 0 +31 0 -3.36107564 0.006341081 0.0091773743618866234 0 +32 1 3.887834 0.9949878 0.0072492792316429184 1 +33 0 -3.22425318 0.007697899 0.011148686558077768 0 +34 0 -3.071902 0.009549661 0.013843453855619456 0 +35 0 -3.51190233 0.00511935074 0.0074046315896624214 0 +36 1 5.102052 0.9991103 0.0012841638716619209 1 +37 0 -1.6750958 0.06609887 0.098658269947196717 0 +38 1 2.925821 0.9805053 0.028402684315024945 1 +39 1 1.07152641 0.78101474 0.35657831758532987 1 +40 0 ? ? ? 0 +41 1 1.21868157 0.8148142 0.2954569545094401 1 +42 1 5.106862 0.999116361 0.0012753849724049479 1 +43 1 0.421162069 0.5850235 0.77343346175640615 1 +44 1 4.84587049 0.9987181 0.0018506018487151699 1 +45 0 -3.65703821 0.00416558562 0.0060222214801234535 0 +46 1 2.90657687 0.9799733 0.02918562843571906 1 +47 0 -3.75618124 0.00361798 0.0052291069954759368 0 +48 0 -2.598872 0.0185861979 0.027066533968906802 0 +49 1 3.48212361 0.9910919 0.012909245689328891 1 +50 1 1.23724663 0.818778753 0.28845442893829887 1 +51 1 -0.2776118 0.34213 1.547383458524106 0 +52 1 2.624481 0.970340669 0.043436754375858859 1 +53 1 3.80447841 0.9943582 0.0081624702155899485 1 +54 1 3.73740077 0.9937949 0.0089799304474630897 1 +55 1 2.6370635 0.9708531 0.042675091673916553 1 +56 1 3.97285128 0.9955579 0.0064228655960759853 1 +57 1 0.107479952 0.473964185 1.0771500500334781 1 +58 1 0.942351162 0.747861445 0.41915708573584748 1 +59 1 0.5798892 0.638752937 0.64667007456270209 1 +60 1 0.992840946 0.761204541 0.39364392730009157 1 +61 0 -3.49325752 0.005256673 0.0076037787042423175 0 +62 1 4.537686 0.9980113 0.0028719572640921601 1 +63 1 0.113746926 0.4761946 1.070376863502621 1 +64 0 -3.75618124 0.00361798 0.0052291069954759368 0 +65 1 0.993808866 0.761455536 0.39316829983703278 1 +66 0 -2.87251735 0.0126532819 0.018371302375292598 0 +67 1 2.36705875 0.9577302 0.062308839103444078 1 +68 1 4.866312 0.998754859 0.0017974780774891569 1 +69 0 -3.476495 0.005383256 0.00778737615473325 0 +70 0 -2.38762283 0.0249626935 0.036470675060072594 0 +71 1 3.759193 0.993983746 0.0087058350214784704 1 +72 0 -1.97156119 0.0443060733 0.065379444451594285 0 +73 1 4.6668725 0.998345554 0.0023888378671193221 1 +74 1 1.31077349 0.8338314 0.2621723424309863 1 +75 0 -2.86702347 0.01275161 0.018514984966823078 0 +76 0 -3.16901946 0.008323995 0.012059247361003817 0 +77 0 -2.268878 0.0294368826 0.043106057919297044 0 +78 0 -2.59094834 0.0187935922 0.027371439438077672 0 +79 0 -3.43077779 0.005744112 0.0083108944429044138 0 +80 0 -2.456492 0.0226789284 0.033095497649791375 0 +81 0 -2.886521 0.0124060186 0.018010050404334042 0 +82 0 -2.44155622 0.0231562126 0.033800223813446564 0 +83 0 -2.25056744 0.0301927254 0.044230019285598346 0 +84 1 4.71005964 0.9984443 0.002246121171378877 1 +85 1 3.671128 0.993183553 0.0098677237129992188 1 +86 1 1.26688719 0.824971 0.27758465397345933 1 +87 1 3.641807 0.992894351 0.01028787834334325 1 +88 0 -2.87251735 0.0126532819 0.018371302375292598 0 +89 0 -3.29395533 0.006974091 0.01009673508444856 0 +90 0 -3.605355 0.004483021 0.0064821727077381958 0 +91 0 -3.281627 0.00709700258 0.010275315686992475 0 +92 0 -2.87251735 0.0126532819 0.018371302375292598 0 +93 0 -3.75618124 0.00361798 0.0052291069954759368 0 +94 0 -3.36107564 0.006341081 0.0091773743618866234 0 +95 0 -3.605355 0.004483021 0.0064821727077381958 0 +96 0 -3.52590561 0.00501856627 0.0072584895473940658 0 +97 0 -2.6282382 0.01783696 0.0259655617646397 0 +98 1 4.72083664 0.998468041 0.002211843705621662 1 +99 1 5.35836172 0.9993827 0.0008908871693913803 1 +100 1 2.70898128 0.973621249 0.038567439798580988 1 +101 1 -0.5737708 0.254173428 1.9761148810504365 0 +102 0 -2.67679667 0.0166626219 0.024241611946502373 0 +103 1 0.136373684 0.484254658 1.0461621681822364 1 +104 1 6.265337 0.9998307 0.00024423603660357661 1 +105 1 0.562006056 0.632843256 0.6600798808555296 1 +106 1 5.509683 0.99950254 0.00071786217084697653 1 +107 1 3.85737944 0.9947663 0.0075704684375963466 1 +108 0 -3.49846077 0.00521798525 0.0075476704302718653 0 +109 1 3.24852943 0.987610936 0.017985282846270094 1 +110 0 -2.17372823 0.0335745364 0.049269627327238129 0 +111 1 2.245582 0.9501276 0.073806815267649539 1 +112 1 4.014995 0.9958161 0.0060487375609611909 1 +113 1 5.552022 0.9995317 0.00067579209761534022 1 +114 0 -2.021864 0.0413638242 0.060944711239919215 0 +115 0 -2.85531187 0.0129637467 0.018825019797801996 0 +116 0 -0.738810539 0.212150559 0.34400813935501734 0 +117 1 4.22980976 0.9969174 0.0044540819270349804 1 +118 0 -3.218911 0.007756356 0.011233679300276442 0 +119 0 -2.667191 0.0168887321 0.02457338577148253 0 +120 0 -3.27281141 0.007186208 0.010404937490447979 0 +121 0 -2.42291141 0.0237658 0.034700801714631864 0 +122 1 5.94457531 0.9997325 0.00038598069684593736 1 +123 1 1.94044363 0.924954832 0.11254517879081385 1 +124 1 3.79063559 0.994246244 0.0083248873430871381 1 +125 0 -3.75618124 0.00361798 0.0052291069954759368 0 +126 1 3.86330938 0.994810164 0.0075068472430012143 1 +127 0 -3.11679673 0.008962306 0.012988163981361843 0 +128 1 2.72279549 0.9741229 0.037824320618519751 1 +129 0 -3.258014 0.00733845448 0.010626189655786903 0 +130 0 -2.38762283 0.0249626935 0.036470675060072594 0 +131 0 -3.36107564 0.006341081 0.0091773743618866234 0 +132 1 5.348998 0.9993744 0.00090284739652280073 1 +133 0 -3.13876224 0.008688118 0.012589072408717294 0 +134 0 -3.18649888 0.008120583 0.01176335290875357 0 +135 0 -2.10759354 0.0367755629 0.054056101026741368 0 +136 0 -3.023344 0.0102279028 0.014831723310481116 0 +137 0 -3.4184494 0.005845474 0.0084579810747969289 0 +138 0 -2.82762337 0.0134792253 0.019578662603528554 0 +139 0 ? ? ? 0 +140 0 -3.4184494 0.005845474 0.0084579810747969289 0 +141 0 -3.662728 0.00413203659 0.005973618813146799 0 +142 1 2.10973549 0.940097034 0.089118419785166547 1 +143 0 -2.85531187 0.0129637467 0.018825019797801996 0 +144 0 -3.51190233 0.00511935074 0.0074046315896624214 0 +145 0 ? ? ? 0 +146 1 0.6787464 0.6706306 0.57640983740540519 1 +147 0 -3.41213346 0.00589808961 0.0085343375943574083 0 +148 0 -1.1527307 0.12979959 0.20058039747005499 0 +149 1 6.111663 0.999789238 0.00030409737394024848 1 +150 0 -3.686479 0.003994874 0.0057749279345070377 0 +151 1 2.21149755 0.947771549 0.077388740952378951 1 +152 1 5.45527363 0.999462366 0.00077585033580392744 1 +153 0 -2.648322 0.017341705 0.025238267200559113 0 +154 0 -3.907008 0.00291936542 0.00421791382857469 0 +155 1 2.032926 0.9336179 0.099095888816917208 1 +156 0 -3.37377882 0.006227868 0.0090130096265379489 0 +157 0 -3.36107564 0.006341081 0.0091773743618866234 0 +158 0 ? ? ? 0 +159 1 6.827044 0.999924064 0.00010955710833393459 1 +160 1 5.08168936 0.999084055 0.0013220342454223198 1 +161 0 -2.743657 0.0151691921 0.022052201361735652 0 +162 0 -3.11679673 0.008962306 0.012988163981361843 0 +163 0 -2.73060226 0.015450052 0.022463696353557282 0 +164 0 ? ? ? 0 +165 0 -2.48073268 0.0219246764 0.031982520468804329 0 +166 1 4.648345 0.9983013 0.0024527505840693247 1 +167 1 4.109608 0.996342659 0.0052861001670183186 1 +168 0 -3.11679673 0.008962306 0.012988163981361843 0 +169 0 -3.8513093 0.003160134 0.0045663285005650535 0 +170 0 -3.4184494 0.005845474 0.0084579810747969289 0 +171 0 -3.605355 0.004483021 0.0064821727077381958 0 +172 0 -3.75618124 0.00361798 0.0052291069954759368 0 +173 1 7.952517 0.999984741 2.2013947263955502E-05 1 +174 1 3.13127947 0.985387266 0.021237267296347721 1 +175 1 4.763921 0.998559237 0.0020800811308642544 1 +176 0 -3.36107564 0.006341081 0.0091773743618866234 0 +177 1 3.23345876 0.987345 0.018373841692781227 1 +178 0 -2.87251735 0.0126532819 0.018371302375292598 0 +179 1 1.10102582 0.788129866 0.34349472183649948 1 +180 0 -3.686479 0.003994874 0.0057749279345070377 0 +181 0 -3.907008 0.00291936542 0.00421791382857469 0 +182 0 -2.38395858 0.02509029 0.036659483516313734 0 +183 1 5.121356 0.9991344 0.001249306792594143 1 +184 1 3.44398 0.9905982 0.013628092859358327 1 +185 0 -3.44219971 0.005651763 0.0081768994816438147 0 +186 1 3.04242015 0.9834442 0.024084877319812924 1 +187 1 6.509462 0.999880552 0.00017233690899385432 1 +188 1 4.64564276 0.9982948 0.0024622257561456589 1 +189 0 -3.02723169 0.0101718893 0.014750080211259314 0 +190 1 6.920555 0.999933541 9.588351406111715E-05 1 +191 1 6.12410355 0.999792933 0.00029876479771217244 1 +192 0 -2.77906466 0.0144325038 0.020973417897420254 0 +193 0 -3.605355 0.004483021 0.0064821727077381958 0 +194 0 -3.11679673 0.008962306 0.012988163981361843 0 +195 0 -2.87251735 0.0126532819 0.018371302375292598 0 +196 0 3.14336181 0.9856335 6.1211468016904362 1 +197 0 -2.21777034 0.03159381 0.046315795034904308 0 +198 0 -3.907008 0.00291936542 0.00421791382857469 0 +199 0 -3.26762271 0.007239233 0.010481991952021335 0 +200 1 5.50388956 0.999498367 0.00072388457209681734 1 +201 1 5.449887 0.9994582 0.00078187297912553995 1 +202 0 -3.605355 0.004483021 0.0064821727077381958 0 +203 0 -2.6282382 0.01783696 0.0259655617646397 0 +204 0 -3.605355 0.004483021 0.0064821727077381958 0 +205 1 6.758401 0.999916255 0.00012082287142878181 1 +206 1 4.08546543 0.9962149 0.0054710678808922303 1 +207 0 -3.686479 0.003994874 0.0057749279345070377 0 +208 0 -3.686479 0.003994874 0.0057749279345070377 0 +209 0 -2.70936251 0.0159179773 0.023149526492909567 0 +210 1 7.79690647 0.999981 2.7431493603031304E-05 1 +211 1 5.49172449 0.9994896 0.00073653169655269271 1 +212 0 -3.605355 0.004483021 0.0064821727077381958 0 +213 1 8.098601 0.9999876 1.7886306565083914E-05 1 +214 1 7.4137125 0.9999671 4.7467992532759307E-05 1 +215 1 4.27127934 0.997094035 0.0041985244211640484 1 +216 0 -3.75618124 0.00361798 0.0052291069954759368 0 +217 0 -3.605355 0.004483021 0.0064821727077381958 0 +218 1 4.417726 0.9976408 0.0034076439105688978 1 +219 0 -1.99251723 0.04305683 0.06349484613809675 0 +220 0 -3.468532 0.00544444658 0.0078761361892400912 0 +221 1 5.467387 0.9994716 0.00076251457220611978 1 +222 1 -1.77912676 0.0575035475 4.1202052279640053 0 +223 1 2.77077627 0.9757941 0.035351368771114847 1 +224 1 4.98765326 0.9989526 0.0015118314343332228 1 +225 0 -3.75618124 0.00361798 0.0052291069954759368 0 +226 1 5.23825741 0.99926734 0.0010573927743703264 1 +227 1 4.22040033 0.9968759 0.0045142044618752585 1 +228 0 -3.686479 0.003994874 0.0057749279345070377 0 +229 1 7.388439 0.9999659 4.9187876653739729E-05 1 +230 1 2.911434 0.9801089 0.028986014090586303 1 +231 1 4.693553 0.9984073 0.0022996059796464349 1 +232 0 0.406565845 0.5799576 1.2513931459884398 1 +233 1 3.24389219 0.9875297 0.018103964194603205 1 +234 0 -1.822286 0.0542549044 0.08047670503653874 0 +235 0 ? ? ? 0 +236 1 6.204999 0.9998155 0.00026616770379175322 1 +237 1 3.76840854 0.9940619 0.0085924225063566447 1 +238 1 6.98390341 0.999939263 8.7627821772873972E-05 1 +239 1 2.78742528 0.976349 0.034531163251991751 1 +240 0 -1.59333324 0.07367699 0.11041274173012067 0 +241 0 -2.92427588 0.0117629105 0.017070892369396344 0 +242 0 -3.36107564 0.006341081 0.0091773743618866234 0 +243 0 -2.25200653 0.0301326457 0.044140647008885238 0 +244 0 -3.605355 0.004483021 0.0064821727077381958 0 +245 0 -2.22849154 0.03112902 0.045623532391940731 0 +246 1 6.61899948 0.999897838 0.00014739666111309571 1 +247 1 1.61292076 0.8853637 0.17565787412932554 1 +248 0 -2.22821569 0.0311408937 0.045641213992158133 0 +249 0 ? ? ? 0 +250 0 -3.52460527 0.00502784131 0.007271938158916337 0 +251 1 4.09574556 0.9962698 0.0053915711511858559 1 +252 0 2.08930516 0.9384339 4.0217196700397011 1 +253 1 5.106862 0.999116361 0.0012753849724049479 1 +254 1 4.537686 0.9980113 0.0028719572640921601 1 +255 1 2.59346437 0.969039857 0.045372088821018751 1 +256 0 -3.4184494 0.005845474 0.0084579810747969289 0 +257 0 -3.26762271 0.007239233 0.010481991952021335 0 +258 0 -3.11679673 0.008962306 0.012988163981361843 0 +259 0 2.1366 0.942220032 4.1132867902440298 1 +260 1 4.888714 0.998794 0.0017409124520544692 1 +261 1 6.60946941 0.9998964 0.00014946066526327797 1 +262 1 5.50262356 0.9994975 0.00072517508992112317 1 +263 1 4.47967672 0.99784 0.0031196100552426183 1 +264 1 3.10484362 0.984833956 0.022047590001510155 1 +265 0 -1.91221583 0.0480339229 0.071017930269060142 0 +266 1 4.4806757 0.9978431 0.003115128833769432 1 +267 1 1.22701848 0.816602767 0.29229363970610811 1 +268 1 4.497992 0.9978956 0.003039208716284136 1 +269 0 -3.605355 0.004483021 0.0064821727077381958 0 +270 1 3.41643643 0.990224957 0.014171783681750615 1 +271 0 -2.6282382 0.01783696 0.0259655617646397 0 +272 1 1.22701848 0.816602767 0.29229363970610811 1 +273 1 -0.238218457 0.354893684 1.494541194155254 0 +274 0 -2.987936 0.0107523222 0.015596321256459572 0 +275 0 ? ? ? 0 +276 0 -3.26762271 0.007239233 0.010481991952021335 0 +277 0 -3.75618124 0.00361798 0.0052291069954759368 0 +278 0 -3.605355 0.004483021 0.0064821727077381958 0 +279 1 3.39824677 0.989970446 0.014542638963041896 1 +280 0 -3.11679673 0.008962306 0.012988163981361843 0 +281 0 -3.22425318 0.007697899 0.011148686558077768 0 +282 1 2.05205441 0.93528986 0.096514548658585747 1 +283 1 3.0574975 0.983790934 0.023576334611297014 1 +284 1 3.80213475 0.994339347 0.0081897979096641673 1 +285 1 7.65792227 0.9999768 3.3451013395372324E-05 1 +286 1 8.484354 0.999992847 1.0318995955417813E-05 1 +287 0 -3.18649888 0.008120583 0.01176335290875357 0 +288 1 0.6828229 0.671914339 0.573650776292008 1 +289 1 4.540345 0.9980188 0.0028611008075221564 1 +290 0 -3.907008 0.00291936542 0.00421791382857469 0 +291 0 -3.605355 0.004483021 0.0064821727077381958 0 +292 1 ? ? ? 0 +293 1 2.89768958 0.979722857 0.029554395345901693 1 +294 0 ? ? ? 0 +295 1 3.93127179 0.995287657 0.0068145430617047335 1 +296 0 0.743866265 0.6908238 1.6934987974785298 1 +297 0 ? ? ? 0 +298 0 -1.76699829 0.05844886 0.086888636267376015 0 +299 1 3.232888 0.9873348 0.018388734757354586 1 +300 1 3.68749166 0.993339837 0.0096407248697918859 1 +301 0 -3.605355 0.004483021 0.0064821727077381958 0 +302 1 8.445241 0.99999243 1.0920939664449809E-05 1 +303 0 -3.605355 0.004483021 0.0064821727077381958 0 +304 1 2.86486816 0.97877115 0.03095651717616256 1 +305 1 4.88329935 0.998784661 0.0017544294546381638 1 +306 0 -3.605355 0.004483021 0.0064821727077381958 0 +307 0 -3.605355 0.004483021 0.0064821727077381958 0 +308 1 3.667212 0.993145645 0.0099227906002395463 1 +309 0 -1.76344776 0.05872834 0.087316937808402426 0 +310 0 -3.43077779 0.005744112 0.0083108944429044138 0 +311 0 -3.907008 0.00291936542 0.00421791382857469 0 +312 1 2.02101088 0.9325562 0.10073740517096717 1 +313 0 -3.907008 0.00291936542 0.00421791382857469 0 +314 0 -3.83730555 0.00322371977 0.0046583571907886281 0 +315 0 ? ? ? 0 +316 1 2.05643129 0.935666859 0.095933140082733404 1 +317 1 5.004877 0.9989781 0.0014750751085414963 1 +318 0 -3.36198235 0.00633293344 0.0091655447866802828 0 +319 0 0.922377944 0.742448568 1.9570675352683911 1 +320 1 3.41728544 0.9902367 0.014154676265356153 1 +321 0 ? ? ? 0 +322 0 -3.11679673 0.008962306 0.012988163981361843 0 +323 1 2.65895724 0.9717244 0.041380912891786976 1 +324 0 -3.605355 0.004483021 0.0064821727077381958 0 +325 0 -2.62749934 0.0178554431 0.025992711659710378 0 +326 1 1.6477077 0.8903069 0.16762537411179193 1 +327 0 -3.75618124 0.00361798 0.0052291069954759368 0 +328 1 2.586014 0.968719244 0.045849493203621694 1 +329 1 3.75527 0.9939502 0.0087545419884464525 1 +330 1 2.75729823 0.975335538 0.036029469573371053 1 +331 0 -2.529095 0.02049245 0.029871481243772795 0 +332 0 -2.13097882 0.0356114544 0.052313579264758756 0 +333 1 2.531636 0.966280162 0.049486551571838115 1 +334 1 3.4191432 0.99026227 0.014117422745733861 1 +335 0 -3.907008 0.00291936542 0.00421791382857469 0 +336 1 3.01855564 0.9828805 0.024912111991267676 1 +337 0 -3.605355 0.004483021 0.0064821727077381958 0 +338 0 -3.83730555 0.00322371977 0.0046583571907886281 0 +339 1 2.95636368 0.9813212 0.027202644128254239 1 +340 1 2.9718864 0.981723 0.026612063921145013 1 +341 0 -3.605355 0.004483021 0.0064821727077381958 0 +342 0 -3.662728 0.00413203659 0.005973618813146799 0 +343 0 -3.907008 0.00291936542 0.00421791382857469 0 +344 1 4.97264671 0.99893 0.0015445427694137865 1 +345 0 -3.907008 0.00291936542 0.00421791382857469 0 +346 0 -2.07182074 0.0386274 0.056832410448320131 0 +347 0 -3.73041153 0.00375300646 0.0054246295197604971 0 +348 1 -0.2370915 0.355262 1.4930446675586599 0 +349 1 1.422096 0.8546939 0.22652028659059675 1 +350 0 -2.65624571 0.0171500538 0.024956920775152557 0 +351 0 -3.36107564 0.006341081 0.0091773743618866234 0 +352 0 0.226374939 0.5163547 1.0479786563646041 1 +353 1 5.002982 0.9989753 0.0014791208409251203 1 +354 0 -3.75618124 0.00361798 0.0052291069954759368 0 +355 0 -2.813359 0.0137526244 0.019978538750463886 0 +356 1 -0.8004532 0.197818756 2.3377488737717447 0 +357 1 6.93749857 0.99993515 9.3561595830050562E-05 1 +358 1 3.625639 0.9927297 0.010527105943757411 1 +359 1 2.53226852 0.966309547 0.049442679121077485 1 +360 1 8.110542 0.999987841 1.7542337039908934E-05 1 +361 1 3.47608185 0.991015434 0.01302056849035485 1 +362 0 -2.19712377 0.0325078778 0.047678180715352736 0 +363 0 -1.21346474 0.12031953 0.18494851271674775 0 +364 0 -3.36107564 0.006341081 0.0091773743618866234 0 +365 0 -3.51190233 0.00511935074 0.0074046315896624214 0 +366 1 7.500038 0.9999709 4.1964377123801209E-05 1 +367 1 6.00183439 0.9997535 0.00035570396871277961 1 +368 0 -3.51556587 0.00509279035 0.0073661163875401612 0 +369 0 -3.4458642 0.005622449 0.0081343681191021963 0 +370 0 -2.37313437 0.0254709367 0.03722288334413755 0 +371 0 -3.51556587 0.00509279035 0.0073661163875401612 0 +372 0 -2.82762337 0.0134792253 0.019578662603528554 0 +373 0 -2.642242 0.0174901951 0.02545629016755618 0 +374 0 -3.071902 0.009549661 0.013843453855619456 0 +375 0 -3.907008 0.00291936542 0.00421791382857469 0 +376 0 -3.75618124 0.00361798 0.0052291069954759368 0 +377 0 -3.83730555 0.00322371977 0.0046583571907886281 0 +378 0 -2.64717054 0.0173697341 0.02527941883761909 0 +379 0 -1.50546336 0.082706444 0.12454459000807398 0 +380 0 -3.907008 0.00291936542 0.00421791382857469 0 +381 1 4.933444 0.9988685 0.001633297522647636 1 +382 0 -2.46871066 0.0222956259 0.032529787649107397 0 +383 0 -3.662728 0.00413203659 0.005973618813146799 0 +384 0 -3.662728 0.00413203659 0.005973618813146799 0 +385 0 -2.304271 0.0280275717 0.04101270510313771 0 +386 1 3.0156002 0.982809365 0.025016490260214026 1 +387 0 -1.64548588 0.06875573 0.10276845291259031 0 +388 0 -3.38304186 0.00614658231 0.008895008859090418 0 +389 0 -2.306156 0.027954381 0.040904072394333073 0 +390 0 -3.61935925 0.00439470261 0.0063541882997191811 0 +391 1 5.56635332 0.999541163 0.00066211313564514278 1 +392 0 -3.26762271 0.007239233 0.010481991952021335 0 +393 0 -3.988132 0.002601037 0.0037573919100924264 0 +394 0 -3.17805815 0.008218187 0.01190532487151834 0 +395 0 -3.26762271 0.007239233 0.010481991952021335 0 +396 0 -3.11679673 0.008962306 0.012988163981361843 0 +397 0 -3.17417026 0.008263535 0.011971291770850049 0 +398 0 -2.95208049 0.0113104619 0.016410528576542627 0 +399 0 -3.3288846 0.00663721375 0.0096073939151328971 0 +400 1 5.038892 0.9990264 0.0014052664919195742 1 +401 0 -3.4184494 0.005845474 0.0084579810747969289 0 +402 0 -2.014259 0.0417963341 0.061595761918872241 0 +403 0 -2.62967777 0.0178010035 0.025912746381212329 0 +404 0 -3.31782031 0.006742137 0.0097597856433223086 0 +405 0 -3.75618124 0.00361798 0.0052291069954759368 0 +406 0 -2.64094114 0.0175221246 0.025503175426540282 0 +407 0 -3.75618124 0.00361798 0.0052291069954759368 0 +408 0 -2.39224434 0.0248026624 0.036233907693558484 0 +409 0 -3.071902 0.009549661 0.013843453855619456 0 +410 0 -3.75618124 0.00361798 0.0052291069954759368 0 +411 0 ? ? ? 0 +412 1 5.263987 0.999293745 0.0010192711903285418 1 +413 0 -2.43251729 0.0234498065 0.034233896082456736 0 +414 1 3.51097441 0.991448164 0.012390749074656452 1 +415 0 -0.7411851 0.211584687 0.34297229790916123 0 +416 1 4.88950253 0.998795331 0.0017390183598981097 1 +417 0 -3.75618124 0.00361798 0.0052291069954759368 0 +418 0 -1.61808527 0.0713021457 0.10671879325550195 0 +419 0 -3.04599214 0.009905825 0.014362338572597992 0 +420 0 -1.93490088 0.04657501 0.068808654572963493 0 +421 1 6.75402069 0.9999157 0.00012159685838243442 1 +422 0 -1.8499912 0.0522614941 0.077439040280427335 0 +423 0 -2.38762283 0.0249626935 0.036470675060072594 0 +424 0 -3.4184494 0.005845474 0.0084579810747969289 0 +425 1 8.252745 0.999990046 1.4360622819808225E-05 1 +426 0 -1.57792318 0.07519208 0.11277434020029603 0 +427 1 2.61812758 0.9700786 0.043826467948240723 1 +428 0 -3.75618124 0.00361798 0.0052291069954759368 0 +429 0 -3.51190233 0.00511935074 0.0074046315896624214 0 +430 0 -3.253957 0.00738075143 0.010687663674474941 0 +431 0 -2.063484 0.03907166 0.057499247745450677 0 +432 0 -2.65020418 0.0172959939 0.025171157655520787 0 +433 0 -2.76240134 0.0147747025 0.021474422669082938 0 +434 0 3.19463944 0.9866337 6.2252579738556983 1 +435 1 4.410512 0.99761647 0.0034428117673751093 1 +436 1 3.11047578 0.9849535 0.021872445625870953 1 +437 0 -3.17417026 0.008263535 0.011971291770850049 0 +438 0 -2.50908351 0.0210736189 0.030727727010326666 0 +439 0 -2.92989063 0.0116701266 0.016935446480853662 0 +440 1 4.360398 0.9974402 0.003697718158147352 1 +441 0 -1.34824824 0.101400226 0.15424939643653701 0 +442 0 -3.10751438 0.009080736 0.013160577458947661 0 +443 0 -3.64408374 0.00424298458 0.0061343560208671445 0 +444 0 -2.01382756 0.041821003 0.061632904458828297 0 +445 0 -3.662728 0.00413203659 0.005973618813146799 0 +446 0 -3.907008 0.00291936542 0.00421791382857469 0 +447 0 -2.92989063 0.0116701266 0.016935446480853662 0 +448 0 -3.988132 0.002601037 0.0037573919100924264 0 +449 1 5.620066 0.999575 0.000613248469067489 1 +450 0 -2.830823 0.01341864 0.019490064907595788 0 +451 0 -2.92989063 0.0116701266 0.016935446480853662 0 +452 0 -3.144804 0.00861417 0.012481457081208893 0 +453 1 4.134612 0.9964704 0.0051011561647871066 1 +454 0 -3.20824 0.007874443 0.011405384843361465 0 +455 1 0.0876712054 0.466921568 1.0987478641825004 1 +456 1 5.275331 0.999305069 0.0010029213839365773 1 +457 1 4.727125 0.9984817 0.0021921216132669563 1 +458 0 -2.73417 0.0153727932 0.022350490813957406 0 +459 0 -2.53844929 0.02022619 0.029479366744597127 0 +460 0 -2.65624571 0.0171500538 0.024956920775152557 0 +461 0 -2.36675334 0.0256979633 0.037559013254144957 0 +462 0 -2.41196656 0.0241309032 0.035240457386936527 0 +463 0 -3.04531026 0.009915374 0.01437625254749182 0 +464 0 -3.17417026 0.008263535 0.011971291770850049 0 +465 1 5.352198 0.999377251 0.0008987172349384933 1 +466 1 4.84867859 0.9987232 0.0018431971214599131 1 +467 1 3.95066 0.9954156 0.0066290574824416091 1 +468 0 -3.17417026 0.008263535 0.011971291770850049 0 +469 0 -3.56046057 0.00477823429 0.0069100570261159339 0 +470 0 -3.34874725 0.006452909 0.0093397468928123339 0 +471 0 -2.41196656 0.0241309032 0.035240457386936527 0 +472 0 -2.766736 0.014684923 0.021342961841691981 0 +473 0 -3.17417026 0.008263535 0.011971291770850049 0 +474 0 -2.92989063 0.0116701266 0.016935446480853662 0 +475 0 -3.4184494 0.005845474 0.0084579810747969289 0 +476 0 -2.97844934 0.0108972881 0.015807751601481245 0 +477 0 -3.17417026 0.008263535 0.011971291770850049 0 +478 0 -2.74320912 0.0151787465 0.022066197885298722 0 +479 1 4.48729563 0.9978633 0.0030859150580126917 1 +480 0 -3.01101518 0.0104075661 0.015093624894872421 0 +481 0 -1.95033813 0.0456064157 0.067343750048687132 0 +482 1 7.830415 0.9999819 2.614159977229895E-05 1 +483 1 5.81641674 0.99967885 0.00046339567512805407 1 +484 0 -2.73417 0.0153727932 0.022350490813957406 0 +485 0 -3.188639 0.00809602 0.011727625387859621 0 +486 0 -3.34874725 0.006452909 0.0093397468928123339 0 +487 1 7.107704 0.9999491 7.3438461030010931E-05 1 +488 1 0.6287323 0.6546769 0.61114498862723421 1 +489 1 -0.7941514 0.199249789 2.3273498975848921 0 +490 0 -3.907008 0.00291936542 0.00421791382857469 0 +491 1 3.70551538 0.9935079 0.0093966240953033617 1 +492 0 -3.10446858 0.00911993347 0.013217646854947663 0 +493 1 5.802666 0.9996725 0.00047259973218091044 1 +494 0 -0.137995735 0.3882759 0.70904696011169721 0 +495 0 -3.34874725 0.006452909 0.0093397468928123339 0 +496 0 -3.988132 0.002601037 0.0037573919100924264 0 +497 0 -2.987488 0.0107591245 0.015606241718216556 0 +498 0 -3.023344 0.0102279028 0.014831723310481116 0 +499 0 -3.023344 0.0102279028 0.014831723310481116 0 +500 0 -2.38395858 0.02509029 0.036659483516313734 0 +501 0 -3.023344 0.0102279028 0.014831723310481116 0 +502 0 -2.886521 0.0124060186 0.018010050404334042 0 +503 0 -2.87251735 0.0126532819 0.018371302375292598 0 +504 0 -3.907008 0.00291936542 0.00421791382857469 0 +505 0 -3.04508519 0.009918528 0.01438084759491281 0 +506 1 5.882653 0.999707758 0.00042167708737945704 1 +507 0 -3.11823583 0.008944083 0.01296163596674773 0 +508 0 -2.92989063 0.0116701266 0.016935446480853662 0 +509 0 -3.662728 0.00413203659 0.005973618813146799 0 +510 0 -3.907008 0.00291936542 0.00421791382857469 0 +511 0 -2.77906466 0.0144325038 0.020973417897420254 0 +512 0 -2.92989063 0.0116701266 0.016935446480853662 0 +513 0 -3.34874725 0.006452909 0.0093397468928123339 0 +514 1 5.41563749 0.9994311 0.00082102077356410697 1 +515 1 4.72786045 0.9984833 0.0021897963188356523 1 +516 0 -3.988132 0.002601037 0.0037573919100924264 0 +517 0 -3.83730555 0.00322371977 0.0046583571907886281 0 +518 0 -3.08590579 0.009362471 0.013570818697650193 0 +519 1 3.32478762 0.988874257 0.016141012626201393 1 +520 0 -3.82588339 0.0032765267 0.0047347898899868287 0 +521 0 -3.09304571 0.009268436 0.013433878283479358 0 +522 1 2.34040117 0.9561629 0.064671621446964631 1 +523 1 3.904993 0.9951084 0.0070743666400943962 1 +524 0 -3.26762271 0.007239233 0.010481991952021335 0 +525 0 -3.281627 0.00709700258 0.010275315686992475 0 +526 0 -3.17417026 0.008263535 0.011971291770850049 0 +527 0 -2.87251735 0.0126532819 0.018371302375292598 0 +528 0 -2.15734744 0.0343414061 0.050414876750893345 0 +529 0 -3.10446858 0.00911993347 0.013217646854947663 0 +530 1 3.29309988 0.988365531 0.016883397018183529 1 +531 0 -2.64094114 0.0175221246 0.025503175426540282 0 +532 0 -3.686479 0.003994874 0.0057749279345070377 0 +533 0 -3.26762271 0.007239233 0.010481991952021335 0 +534 0 -3.51190233 0.00511935074 0.0074046315896624214 0 +535 0 -3.077091 0.009479872 0.013741802921144131 0 +536 0 -2.6282382 0.01783696 0.0259655617646397 0 +537 0 -2.43251729 0.0234498065 0.034233896082456736 0 +538 0 -3.023344 0.0102279028 0.014831723310481116 0 +539 0 -2.534785 0.0203300826 0.029632354621199147 0 +540 0 -2.5161407 0.0208668467 0.030423028120182851 0 +541 0 -3.4184494 0.005845474 0.0084579810747969289 0 +542 0 -2.55652666 0.01972121 0.028735986734009608 0 +543 0 -3.023344 0.0102279028 0.014831723310481116 0 +544 0 -2.983638 0.0108177615 0.015691759633405213 0 +545 0 -2.77906466 0.0144325038 0.020973417897420254 0 +546 1 6.580682 0.999892056 0.00015573869603731518 1 +547 0 -3.74385333 0.00368196028 0.0053217490984873526 0 +548 0 -3.49957347 0.005209749 0.0075357258985943474 0 +549 1 3.197887 0.9866947 0.019324344772168506 1 +550 0 -3.26762271 0.007239233 0.010481991952021335 0 +551 0 -3.605355 0.004483021 0.0064821727077381958 0 +552 0 -2.31788421 0.02750314 0.04023450335953014 0 +553 0 -1.11673784 0.1357127 0.21041712957650402 0 +554 0 -3.4184494 0.005845474 0.0084579810747969289 0 +555 0 -1.44630456 0.08934146 0.13501789419422672 0 +556 0 -2.34096384 0.02663574 0.038948292120745368 0 +557 0 -2.65624571 0.0171500538 0.024956920775152557 0 +558 0 -3.51190233 0.00511935074 0.0074046315896624214 0 +559 0 -2.77906466 0.0144325038 0.020973417897420254 0 +560 0 -2.6282382 0.01783696 0.0259655617646397 0 +561 0 -2.6282382 0.01783696 0.0259655617646397 0 +562 0 -3.605355 0.004483021 0.0064821727077381958 0 +563 0 -3.26762271 0.007239233 0.010481991952021335 0 +564 0 -2.743657 0.0151691921 0.022052201361735652 0 +565 1 6.30995941 0.999841154 0.00022918508536702239 1 +566 0 -2.92107558 0.0118161216 0.017148575629624945 0 +567 0 -2.497496 0.02141749 0.031234595803580383 0 +568 1 2.40637922 0.9599445 0.058977117383329433 1 +569 1 5.56444263 0.9995399 0.00066391978356599986 1 +570 1 4.518421 0.997956 0.0029519184450818659 1 +571 1 5.71379232 0.9996282 0.00053651363580302608 1 +572 0 -3.26762271 0.007239233 0.010481991952021335 0 +573 0 -3.75618124 0.00361798 0.0052291069954759368 0 +574 1 3.3362236 0.989052355 0.015881203106942656 1 +575 0 -2.43251729 0.0234498065 0.034233896082456736 0 +576 0 -2.77906466 0.0144325038 0.020973417897420254 0 +577 0 -3.75618124 0.00361798 0.0052291069954759368 0 +578 0 -3.75618124 0.00361798 0.0052291069954759368 0 +579 0 -3.605355 0.004483021 0.0064821727077381958 0 +580 0 -2.58334374 0.0189947709 0.027667268419522113 0 +581 1 4.665889 0.9983432 0.0023921970903694311 1 +582 1 4.76713943 0.998565853 0.0020705223534249356 1 +583 0 -3.4184494 0.005845474 0.0084579810747969289 0 +584 0 -2.1646533 0.0339973159 0.049900897220732682 0 +585 0 -3.907008 0.00291936542 0.00421791382857469 0 +586 1 7.68396854 0.999977648 3.2247107427628552E-05 1 +587 0 -2.65020418 0.0172959939 0.025171157655520787 0 +588 1 3.12145 0.9851839 0.021535051420384654 1 +589 0 -2.92989063 0.0116701266 0.016935446480853662 0 +590 1 1.94298911 0.9252066 0.11215253482543601 1 +591 1 3.020261 0.9829214 0.024852008233365874 1 +592 1 2.69924116 0.9732619 0.039100026162632168 1 +593 0 -2.73417 0.0153727932 0.022350490813957406 0 +594 1 2.818376 0.9773478 0.033056055369218275 1 +595 0 -2.77906466 0.0144325038 0.020973417897420254 0 +596 0 -2.82762337 0.0134792253 0.019578662603528554 0 +597 0 -2.23679686 0.0307735167 0.045094269194716763 0 +598 0 -3.26762271 0.007239233 0.010481991952021335 0 +599 0 -2.237171 0.0307575986 0.0450705751766904 0 +600 0 -3.26762271 0.007239233 0.010481991952021335 0 +601 0 -3.83730555 0.00322371977 0.0046583571907886281 0 +602 0 -3.023344 0.0102279028 0.014831723310481116 0 +603 1 1.94429326 0.9253353 0.11195188519632764 1 +604 1 2.36156869 0.9574119 0.062788379177033268 1 +605 1 5.16247272 0.9991837 0.0011781321133732907 1 +606 0 -3.03734756 0.0100275576 0.014539729071631006 0 +607 0 -3.907008 0.00291936542 0.00421791382857469 0 +608 1 6.1158843 0.9997905 0.00030229117655877947 1 +609 0 -2.92989063 0.0116701266 0.016935446480853662 0 +610 1 3.92593765 0.9952518 0.0068665558750097494 1 +611 1 3.820659 0.9944863 0.0079766383289292028 1 +612 1 8.576258 0.999993742 9.0291174246371028E-06 1 +613 0 -3.1385386 0.008690868 0.012593073526285688 0 +614 0 -3.5930264 0.004562233 0.0065969710101612513 0 +615 0 -2.631902 0.01774559 0.0258313547227354 0 +616 0 -3.26762271 0.007239233 0.010481991952021335 0 +617 0 ? ? ? 0 +618 0 -3.023344 0.0102279028 0.014831723310481116 0 +619 0 -2.77906466 0.0144325038 0.020973417897420254 0 +620 0 -3.26762271 0.007239233 0.010481991952021335 0 +621 0 -0.548980832 0.260938257 0.43623319921784254 0 +622 0 -1.82113993 0.05433889 0.080604828917890892 0 +623 0 -3.907008 0.00291936542 0.00421791382857469 0 +624 0 -2.723366 0.01560793 0.022695058701067979 0 +625 0 -2.15711141 0.03435258 0.050431568045726675 0 +626 1 2.881782 0.979266942 0.03022591253341152 1 +627 0 -2.61103249 0.0182722658 0.026605122252189296 0 +628 0 -3.662728 0.00413203659 0.005973618813146799 0 +629 0 -3.17417026 0.008263535 0.011971291770850049 0 +630 0 -2.10748839 0.03678088 0.054064063213128129 0 +631 0 -2.77906466 0.0144325038 0.020973417897420254 0 +632 0 -3.907008 0.00291936542 0.00421791382857469 0 +633 1 2.452938 0.9624227 0.055257381339905799 1 +634 0 -3.4184494 0.005845474 0.0084579810747969289 0 +635 0 -2.831287 0.0134098772 0.01947725094939378 0 +636 1 5.22481346 0.999253154 0.0010778738607734989 1 +637 0 -1.71293044 0.06284283 0.093637074858110653 0 +638 0 -3.17417026 0.008263535 0.011971291770850049 0 +639 0 -2.65624571 0.0171500538 0.024956920775152557 0 +640 0 -2.90052462 0.012163532 0.017655865106099827 0 +641 0 -3.26762271 0.007239233 0.010481991952021335 0 +642 0 -3.26762271 0.007239233 0.010481991952021335 0 +643 0 -3.907008 0.00291936542 0.00421791382857469 0 +644 0 -3.662728 0.00413203659 0.005973618813146799 0 +645 0 -3.26762271 0.007239233 0.010481991952021335 0 +646 0 -3.52460527 0.00502784131 0.007271938158916337 0 +647 0 -3.633363 0.00430811942 0.0062287292265427417 0 +648 1 6.0645237 0.9997746 0.00032525585452162109 1 +649 0 -3.26762271 0.007239233 0.010481991952021335 0 +650 0 -2.41173053 0.0241388362 0.035252185347710706 0 +651 0 -3.295854 0.00695534935 0.010069507224441127 0 +652 0 -2.65020418 0.0172959939 0.025171157655520787 0 +653 0 -3.023344 0.0102279028 0.014831723310481116 0 +654 0 -3.11679673 0.008962306 0.012988163981361843 0 +655 0 -3.26762271 0.007239233 0.010481991952021335 0 +656 0 -2.77906466 0.0144325038 0.020973417897420254 0 +657 0 -0.154261857 0.382776678 0.69613551829486686 0 +658 1 4.735967 0.9985007 0.0021646489296086387 1 +659 0 -3.907008 0.00291936542 0.00421791382857469 0 +660 0 -3.75618124 0.00361798 0.0052291069954759368 0 +661 0 -2.87251735 0.0126532819 0.018371302375292598 0 +662 0 -3.36474 0.0063082166 0.0091296591126258566 0 +663 0 -3.36474 0.0063082166 0.0091296591126258566 0 +664 0 -2.99397779 0.0106609939 0.015463136427168929 0 +665 0 -3.907008 0.00291936542 0.00421791382857469 0 +666 0 -2.31152248 0.02774702 0.040596343480657826 0 +667 0 -3.11679673 0.008962306 0.012988163981361843 0 +668 1 1.3648479 0.844250441 0.24425706774053077 1 +669 1 4.82926226 0.9986873 0.0018950310103747121 1 +670 1 3.769151 0.994068146 0.0085833395097514083 1 +671 0 -2.79329181 0.0141465226 0.02055485311534401 0 +672 0 -3.36107564 0.006341081 0.0091773743618866234 0 +673 0 -2.27968168 0.0289995559 0.042456139341540605 0 +674 0 -3.75618124 0.00361798 0.0052291069954759368 0 +675 0 -2.587008 0.0188975725 0.027524332667381482 0 +676 0 -3.56046057 0.00477823429 0.0069100570261159339 0 +677 0 -2.92989063 0.0116701266 0.016935446480853662 0 +678 0 -3.907008 0.00291936542 0.00421791382857469 0 +679 0 -3.662728 0.00413203659 0.005973618813146799 0 +680 1 8.686262 0.999994636 7.739240047104371E-06 1 +681 1 5.717595 0.9996302 0.00053358884622493365 1 +682 0 -2.49937773 0.0213612728 0.031151719882885828 0 +683 0 -3.907008 0.00291936542 0.00421791382857469 0 +684 0 -3.907008 0.00291936542 0.00421791382857469 0 +685 0 -3.907008 0.00291936542 0.00421791382857469 0 +686 0 -3.907008 0.00291936542 0.00421791382857469 0 +687 0 -3.00990248 0.0104239332 0.015117486107810333 0 +688 0 -3.17417026 0.008263535 0.011971291770850049 0 +689 0 -3.213005 0.007821493 0.011328389385335143 0 +690 0 -3.633363 0.00430811942 0.0062287292265427417 0 +691 1 3.1120286 0.984986365 0.021824341396277779 1 +692 0 -3.4184494 0.005845474 0.0084579810747969289 0 +693 0 -3.16847944 0.00833036 0.012068506726605562 0 +694 0 -3.07440114 0.009515986 0.013794403859028146 0 +695 0 -3.662728 0.00413203659 0.005973618813146799 0 +696 1 4.03151941 0.995913267 0.0059079896641200755 1 +697 1 2.39331079 0.9592212 0.060064574083243262 1 +698 1 3.007902 0.982622743 0.025290464456233838 1 diff --git a/test/BaselineOutput/Common/WeightedEnsemble/WE-AvgPer-TrainTest-breast-cancer-out.txt b/test/BaselineOutput/Common/WeightedEnsemble/WE-AvgPer-TrainTest-breast-cancer-out.txt new file mode 100644 index 0000000000..cf6af2aa28 --- /dev/null +++ b/test/BaselineOutput/Common/WeightedEnsemble/WE-AvgPer-TrainTest-breast-cancer-out.txt @@ -0,0 +1,50 @@ +maml.exe TrainTest test=%Data% tr=WeightedEnsemble{bp=AvgPer nm=3 tp=-} dout=%Output% loader=Text{col=Label:BL:0 col=Features:R4:1-9} data=%Data% out=%Output% seed=1 +Automatically adding a MinMax normalization transform, use 'norm=Warn' or 'norm=No' to turn this behavior off. +Training 3 learners for the batch 1 +Beginning training model 1 of 3 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 26 instances with missing features during training (over 1 iterations; 26 inst/iter) +Trainer 1 of 3 finished in %Time% +Beginning training model 2 of 3 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 15 instances with missing features during training (over 1 iterations; 15 inst/iter) +Trainer 2 of 3 finished in %Time% +Beginning training model 3 of 3 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 11 instances with missing features during training (over 1 iterations; 11 inst/iter) +Trainer 3 of 3 finished in %Time% +Training calibrator. +Warning: The predictor produced non-finite prediction values on 16 instances during testing. Possible causes: abnormal data or the predictor is numerically unstable. +TEST POSITIVE RATIO: 0.3499 (239.0/(239.0+444.0)) +Confusion table + ||====================== +PREDICTED || positive | negative | Recall +TRUTH ||====================== + positive || 234 | 5 | 0.9791 + negative || 11 | 433 | 0.9752 + ||====================== +Precision || 0.9551 | 0.9886 | +OVERALL 0/1 ACCURACY: 0.976574 +LOG LOSS/instance: 0.120266 +Test-set entropy (prior Log-Loss/instance): 0.934003 +LOG-LOSS REDUCTION (RIG): 0.871236 +AUC: 0.995797 + +OVERALL RESULTS +--------------------------------------- +AUC: 0.995797 (0.0000) +Accuracy: 0.976574 (0.0000) +Positive precision: 0.955102 (0.0000) +Positive recall: 0.979079 (0.0000) +Negative precision: 0.988584 (0.0000) +Negative recall: 0.975225 (0.0000) +Log-loss: 0.120266 (0.0000) +Log-loss reduction: 0.871236 (0.0000) +F1 Score: 0.966942 (0.0000) +AUPRC: 0.991415 (0.0000) + +--------------------------------------- +Physical memory usage(MB): %Number% +Virtual memory usage(MB): %Number% +%DateTime% Time elapsed(s): %Number% + diff --git a/test/BaselineOutput/Common/WeightedEnsemble/WE-AvgPer-TrainTest-breast-cancer-rp.txt b/test/BaselineOutput/Common/WeightedEnsemble/WE-AvgPer-TrainTest-breast-cancer-rp.txt new file mode 100644 index 0000000000..9e65852a06 --- /dev/null +++ b/test/BaselineOutput/Common/WeightedEnsemble/WE-AvgPer-TrainTest-breast-cancer-rp.txt @@ -0,0 +1,4 @@ +WeightedEnsemble +AUC Accuracy Positive precision Positive recall Negative precision Negative recall Log-loss Log-loss reduction F1 Score AUPRC /bp /nm Learner Name Train Dataset Test Dataset Results File Run Time Physical Memory Virtual Memory Command Line Settings +0.995797 0.976574 0.955102 0.979079 0.988584 0.975225 0.120266 0.871236 0.966942 0.991415 AvgPer 3 WeightedEnsemble %Data% %Data% %Output% 99 0 0 maml.exe TrainTest test=%Data% tr=WeightedEnsemble{bp=AvgPer nm=3 tp=-} dout=%Output% loader=Text{col=Label:BL:0 col=Features:R4:1-9} data=%Data% out=%Output% seed=1 /bp:AvgPer;/nm:3 + diff --git a/test/BaselineOutput/Common/WeightedEnsemble/WE-AvgPer-TrainTest-breast-cancer.txt b/test/BaselineOutput/Common/WeightedEnsemble/WE-AvgPer-TrainTest-breast-cancer.txt new file mode 100644 index 0000000000..cd6e2281db --- /dev/null +++ b/test/BaselineOutput/Common/WeightedEnsemble/WE-AvgPer-TrainTest-breast-cancer.txt @@ -0,0 +1,700 @@ +Instance Label Score Probability Log-loss Assigned +0 0 -2.64225721 0.01699872 0.024734798891773106 0 +1 0 2.53032255 0.9175873 3.6009891670480099 1 +2 0 -2.85259032 0.0131193595 0.019052488192279435 0 +3 0 3.18263674 0.961787164 4.7097988651720648 1 +4 0 -2.602043 0.01785964 0.025998876067635845 0 +5 1 8.328943 0.9999362 9.2013652418763235E-05 1 +6 0 -0.8214984 0.1441913 0.22463974108554102 0 +7 0 -3.162962 0.008937426 0.012951945167532284 0 +8 0 -3.40084314 0.00665333029 0.0096308007078662172 0 +9 0 -2.78199434 0.01431271 0.020798071268316794 0 +10 0 -3.76753688 0.00421679672 0.0060964144371177219 0 +11 0 -3.57733583 0.00534285465 0.0077287752075301903 0 +12 1 -0.145943642 0.2816618 1.827964137977282 0 +13 0 -3.11203718 0.009519416 0.013799399942310069 0 +14 1 5.416985 0.997574568 0.0035034093749688354 1 +15 1 1.0272851 0.62964654 0.66738591475385589 1 +16 0 -2.990669 0.0110621145 0.016048185727640486 0 +17 0 -2.88647676 0.0125819575 0.018267088068968144 0 +18 1 5.44985533 0.997672 0.0033624786068579497 1 +19 0 -2.38444281 0.0233138688 0.034033084074584205 0 +20 1 3.60034323 0.9769754 0.033605886446636479 1 +21 1 4.960816 0.9957176 0.0061914854956740973 1 +22 0 -3.2840023 0.00769185647 0.011139901588193415 0 +23 1 ? ? ? 0 +24 0 -3.73906183 0.004368966 0.0063168947718665724 0 +25 1 1.21789026 0.683309257 0.54938942324601914 1 +26 0 -3.081393 0.009887559 0.014335722854623315 0 +27 0 -2.79879785 0.0140193077 0.020368699164028413 0 +28 0 -3.57733583 0.00534285465 0.0077287752075301903 0 +29 0 -3.415572 0.006532717 0.0094556377041636937 0 +30 0 -3.319243 0.00736273173 0.010661473684257657 0 +31 0 -3.468138 0.00611970061 0.0088559874474810514 0 +32 1 4.548428 0.992849 0.010353787564560591 1 +33 0 -3.26104116 0.007914111 0.011463068787094601 0 +34 0 -3.03546929 0.0104658064 0.015178533994777435 0 +35 0 -3.57733583 0.00534285465 0.0077287752075301903 0 +36 1 5.937471 0.998733342 0.0018285599817242965 1 +37 0 -1.63580728 0.057374537 0.085243441689761862 0 +38 1 3.68594313 0.9792639 0.030230390949068951 1 +39 1 1.35960436 0.7203528 0.47322438381733362 1 +40 0 ? ? ? 0 +41 1 1.74324036 0.806259751 0.31068339066315775 1 +42 1 6.358771 0.9992516 0.0010801113079973799 1 +43 1 1.0734849 0.6430131 0.63707992105182798 1 +44 1 6.11202765 0.9989815 0.0014701685972567185 1 +45 0 -3.69810843 0.0045974534 0.0066480169323248264 0 +46 1 3.998488 0.9858768 0.020520725273315703 1 +47 0 -3.87066913 0.00370853045 0.0053602239211324705 0 +48 0 -2.602043 0.01785964 0.025998876067635845 0 +49 1 4.710529 0.994153261 0.0084598164329324575 1 +50 1 1.9853282 0.849230647 0.23577165863846977 1 +51 1 0.1653862 0.366568744 1.4478443164704415 1 +52 1 3.46802759 0.972945035 0.039569791102444213 1 +53 1 4.57408667 0.9930732 0.01002799497830687 1 +54 1 4.258186 0.989752531 0.014860243285211676 1 +55 1 3.68305063 0.97919035 0.030338755283402914 1 +56 1 5.63595438 0.9981544 0.0026650955056132996 1 +57 1 0.35532856 0.423243374 1.2404406136594532 1 +58 1 1.34439707 0.716506541 0.48094822040550095 1 +59 1 1.21442556 0.6823711 0.55137159087131804 1 +60 1 1.57916451 0.772193253 0.37296614595310351 1 +61 0 -3.5434494 0.00557280472 0.0080623435838360086 0 +62 1 5.78627539 0.9984701 0.0022088293945760644 1 +63 1 0.614236355 0.503562331 0.98975772808148565 1 +64 0 -3.87066913 0.00370853045 0.0053602239211324705 0 +65 1 0.9237709 0.5989934 0.7393879384819777 1 +66 0 -2.88647676 0.0125819575 0.018267088068968144 0 +67 1 3.07211256 0.9563712 0.064357427112375312 1 +68 1 5.753329 0.998405933 0.0023015869365507805 1 +69 0 -3.60256147 0.00517782662 0.0074894310519443095 0 +70 0 -2.20915484 0.0288618673 0.042251578604554092 0 +71 1 4.148643 0.988265932 0.017028787302248289 1 +72 0 -1.67027807 0.055087544 0.081747421828535796 0 +73 1 5.66154242 0.9982124 0.0025812736751607704 1 +74 1 2.10521221 0.867435932 0.20517088909179132 1 +75 0 -2.927059 0.0119669056 0.017368728796317911 0 +76 0 -2.966965 0.0113910912 0.016528187518831179 0 +77 0 -2.08903861 0.03338281 0.048983444456252731 0 +78 0 -2.5253706 0.0196212754 0.028588918513221071 0 +79 0 -3.567583 0.00540805142 0.0078233426086431572 0 +80 0 -2.32998538 0.02491568 0.036401114621345175 0 +81 0 -2.87897158 0.012699076 0.018438217469620204 0 +82 0 -2.36763954 0.0237970911 0.034747044186087268 0 +83 0 -2.36414957 0.0238986723 0.034897175209471852 0 +84 1 6.079429 0.9989391 0.0015313720637253649 1 +85 1 4.79667139 0.9947471 0.0075983035924925202 1 +86 1 1.87611532 0.830901 0.26725145514292187 1 +87 1 4.736424 0.994338453 0.0081910951232013873 1 +88 0 -2.88647676 0.0125819575 0.018267088068968144 0 +89 0 -3.3759973 0.006861824 0.009933640123403616 0 +90 0 -3.73906183 0.004368966 0.0063168947718665724 0 +91 0 -3.27697968 0.007759167 0.011237766032611151 0 +92 0 -2.88647676 0.0125819575 0.018267088068968144 0 +93 0 -3.87066913 0.00370853045 0.0053602239211324705 0 +94 0 -3.468138 0.00611970061 0.0088559874474810514 0 +95 0 -3.73906183 0.004368966 0.0063168947718665724 0 +96 0 -3.5479033 0.005542028 0.0080176943660935727 0 +97 0 -2.64225721 0.01699872 0.024734798891773106 0 +98 1 5.660414 0.9982099 0.0025848917830979144 1 +99 1 6.38035727 0.9992716 0.0010512829267501594 1 +100 1 3.25557566 0.965001345 0.051397142182343647 1 +101 1 -0.0533137321 0.3056757 1.7099262954909895 0 +102 0 -2.62111449 0.017446138 0.025391599107700061 0 +103 1 0.261789322 0.394979417 1.3401506195677144 1 +104 1 6.76045656 0.999547 0.00065368214192851781 1 +105 1 0.5283122 0.476720661 1.0687839421712391 1 +106 1 7.092652 0.9997009 0.00043156901453873987 1 +107 1 5.104385 0.9964188 0.0051758903638738172 1 +108 0 -3.542838 0.00557704223 0.0080684913026905624 0 +109 1 4.0640626 0.986974061 0.018915926344150846 1 +110 0 -2.02161264 0.03621286 0.053213546144801796 0 +111 1 3.08591032 0.957085431 0.063280387695452245 1 +112 1 4.47331953 0.9921504 0.011369221993802024 1 +113 1 6.521157 0.999389052 0.00088168044338409401 1 +114 0 -1.76835728 0.04904162 0.072545896054594564 0 +115 0 -2.66598177 0.0165100824 0.024017832333444792 0 +116 0 -0.295293331 0.245460585 0.40633182940849571 0 +117 1 4.6507287 0.9937022 0.0091145746721560828 1 +118 0 -3.30091643 0.00753210625 0.010907662931260033 0 +119 0 -2.59560347 0.0180014167 0.026207151676105784 0 +120 0 -3.226789 0.00825755 0.011962585788190934 0 +121 0 -2.32638574 0.02502526 0.036563252969620229 0 +122 1 7.0959115 0.999702156 0.00042976265756084565 1 +123 1 2.436901 0.9083122 0.13873983466191542 1 +124 1 4.368197 0.991057634 0.012959135983775204 1 +125 0 -3.87066913 0.00370853045 0.0053602239211324705 0 +126 1 4.289799 0.990145743 0.014287198913392396 1 +127 0 -3.17981029 0.008752749 0.012683135492315652 0 +128 1 3.590857 0.976707041 0.034002197479628134 1 +129 0 -3.79812336 0.004059219 0.0058681329266010423 0 +130 0 -2.20915484 0.0288618673 0.042251578604554092 0 +131 0 -3.468138 0.00611970061 0.0088559874474810514 0 +132 1 6.496856 0.9993702 0.00090887057002802098 1 +133 0 -3.120087 0.00942498352 0.013661860004676191 0 +134 0 -3.27424979 0.00778549025 0.01127604010215829 0 +135 0 -1.93809247 0.0400394276 0.058952942324420055 0 +136 0 -2.990669 0.0110621145 0.016048185727640486 0 +137 0 -3.38819456 0.006758672 0.0097838029729429409 0 +138 0 -2.74657869 0.0149510521 0.02173267988754999 0 +139 0 ? ? ? 0 +140 0 -3.38819456 0.006758672 0.0097838029729429409 0 +141 0 -3.681528 0.004693306 0.0067869484659852783 0 +142 1 2.68703222 0.9312434 0.10276976876170422 1 +143 0 -2.66598177 0.0165100824 0.024017832333444792 0 +144 0 -3.57733583 0.00534285465 0.0077287752075301903 0 +145 0 ? ? ? 0 +146 1 1.166482 0.669238746 0.57940712099393721 1 +147 0 -3.53369665 0.00564079126 0.0081609804765763131 0 +148 0 -1.57685542 0.0614934824 0.091561330845792765 0 +149 1 7.39111376 0.999794066 0.0002971306250718485 1 +150 0 -3.76753688 0.00421679672 0.0060964144371177219 0 +151 1 2.72605658 0.9343026 0.098038168875059173 1 +152 1 6.691239 0.999506056 0.00071278616646084866 1 +153 0 -2.53030038 0.0195030551 0.02841495977104639 0 +154 0 -3.97486115 0.00325702247 0.0047065590202725379 0 +155 1 2.83751249 0.9423561 0.085655747897856641 1 +156 0 -3.28958941 0.007638719 0.011062648151736489 0 +157 0 -3.468138 0.00611970061 0.0088559874474810514 0 +158 0 ? ? ? 0 +159 1 8.068571 0.9999117 0.00012735877431133445 1 +160 1 6.358648 0.999251544 0.0010801973637291345 1 +161 0 -2.72256112 0.0153998248 0.022390098641937944 0 +162 0 -3.17981029 0.008752749 0.012683135492315652 0 +163 0 -2.973286 0.011302432 0.016398811453640787 0 +164 0 ? ? ? 0 +165 0 -2.3953414 0.023005588 0.033577784253893704 0 +166 1 5.857083 0.9985996 0.0020217821847968117 1 +167 1 4.02336264 0.9863034 0.019896603877564437 1 +168 0 -3.17981029 0.008752749 0.012683135492315652 0 +169 0 -3.847302 0.003818061 0.0055188399780066854 0 +170 0 -3.38819456 0.006758672 0.0097838029729429409 0 +171 0 -3.73906183 0.004368966 0.0063168947718665724 0 +172 0 -3.87066913 0.00370853045 0.0053602239211324705 0 +173 1 9.356342 0.999982357 2.5453656867469965E-05 1 +174 1 4.09843 0.987515152 0.018125211189011923 1 +175 1 5.89401865 0.9986627 0.0019305925461549594 1 +176 0 -3.468138 0.00611970061 0.0088559874474810514 0 +177 1 4.712257 0.9941658 0.0084416521665511574 1 +178 0 -2.88647676 0.0125819575 0.018267088068968144 0 +179 1 1.581646 0.7727386 0.37194767425837882 1 +180 0 -3.76753688 0.00421679672 0.0060964144371177219 0 +181 0 -3.97486115 0.00325702247 0.0047065590202725379 0 +182 0 -2.38444281 0.0233138688 0.034033084074584205 0 +183 1 6.548143 0.9994093 0.00085242581612282196 1 +184 1 4.50506735 0.9924536 0.010928477256498946 1 +185 0 -3.482896 0.006008484 0.0086945567651920217 0 +186 1 3.97749281 0.9855066 0.02106257028477352 1 +187 1 6.711457 0.9995184 0.00069497727537140704 1 +188 1 5.818916 0.9985312 0.0021205559353738917 1 +189 0 -2.84098244 0.013308607 0.019329170761039398 0 +190 1 8.265038 0.9999309 9.9667388815113643E-05 1 +191 1 7.6413064 0.9998494 0.0002173164459794357 1 +192 0 -2.79879785 0.0140193077 0.020368699164028413 0 +193 0 -3.73906183 0.004368966 0.0063168947718665724 0 +194 0 -3.17981029 0.008752749 0.012683135492315652 0 +195 0 -2.88647676 0.0125819575 0.018267088068968144 0 +196 0 3.85068941 0.9830587 5.8833108405414283 1 +197 0 -2.310174 0.0255246218 0.037302361163987448 0 +198 0 -3.97486115 0.00325702247 0.0047065590202725379 0 +199 0 -3.2840023 0.00769185647 0.011139901588193415 0 +200 1 6.56151533 0.9994191 0.00083831497269821261 1 +201 1 6.433042 0.999317944 0.00098433446067189745 1 +202 0 -3.73906183 0.004368966 0.0063168947718665724 0 +203 0 -2.64225721 0.01699872 0.024734798891773106 0 +204 0 -3.73906183 0.004368966 0.0063168947718665724 0 +205 1 8.17239952 0.999922454 0.00011187905230883949 1 +206 1 5.34107828 0.9973337 0.0038517872451913665 1 +207 0 -3.76753688 0.00421679672 0.0060964144371177219 0 +208 0 -3.76753688 0.00421679672 0.0060964144371177219 0 +209 0 -2.683842 0.0161513668 0.023491723291569067 0 +210 1 8.779142 0.9999637 5.2369667682746913E-05 1 +211 1 6.67893934 0.999498367 0.00072388457209681734 1 +212 0 -3.73906183 0.004368966 0.0063168947718665724 0 +213 1 9.241756 0.9999796 2.9409333050075951E-05 1 +214 1 8.735781 0.9999617 5.5293481842253808E-05 1 +215 1 5.324557 0.9972782 0.0039320614310995611 1 +216 0 -3.87066913 0.00370853045 0.0053602239211324705 0 +217 0 -3.73906183 0.004368966 0.0063168947718665724 0 +218 1 5.644956 0.998175 0.0026352878013322168 1 +219 0 -1.81162953 0.0465790369 0.06881474818415885 0 +220 0 -3.531965 0.00565294828 0.0081786189500913237 0 +221 1 6.202768 0.9990906 0.0013125665587861025 1 +222 1 -1.64664221 0.05664624 4.1418760172799773 0 +223 1 3.63937521 0.978047967 0.032022873069621122 1 +224 1 6.04423857 0.9988915 0.0016001537412842447 1 +225 0 -3.87066913 0.00370853045 0.0053602239211324705 0 +226 1 6.441114 0.9993248 0.00097443874272921157 1 +227 1 5.391661 0.9974967 0.0036159914882246754 1 +228 0 -3.76753688 0.00421679672 0.0060964144371177219 0 +229 1 8.791424 0.999964237 5.1595717867672461E-05 1 +230 1 3.820557 0.98241967 0.025588648805799773 1 +231 1 5.937156 0.998732865 0.0018292487849703239 1 +232 0 0.9086909 0.5944561 1.3020699175400769 1 +233 1 4.076566 0.987173557 0.018624344332146187 1 +234 0 -1.59241557 0.0603802 0.089850982981327321 0 +235 0 ? ? ? 0 +236 1 6.99668264 0.9996628 0.00048653495834995717 1 +237 1 4.55212831 0.9928818 0.010306152477496833 1 +238 1 7.855024 0.999884665 0.00016640281092730855 1 +239 1 3.41797686 0.971248031 0.042088326624438219 1 +240 0 -1.365458 0.07863495 0.11815521758392736 0 +241 0 -2.94108582 0.0117613031 0.017068545688932395 0 +242 0 -3.468138 0.00611970061 0.0088559874474810514 0 +243 0 -2.24921322 0.0274906419 0.040215962230060673 0 +244 0 -3.73906183 0.004368966 0.0063168947718665724 0 +245 0 -2.1889298 0.0295791738 0.043317582381079828 0 +246 1 7.99522829 0.9999032 0.00013965667184894127 1 +247 1 2.2151475 0.8824597 0.18039769954320503 1 +248 0 -2.066899 0.0342876948 0.050334634335326013 0 +249 0 ? ? ? 0 +250 0 -3.43302059 0.006392639 0.0092522334759092793 0 +251 1 4.76601267 0.994542956 0.0078944095217020963 1 +252 0 2.749187 0.93605566 3.9670395300086332 1 +253 1 6.358771 0.9992516 0.0010801113079973799 1 +254 1 5.78627539 0.9984701 0.0022088293945760644 1 +255 1 3.790628 0.9817616 0.02655539284974652 1 +256 0 -3.38819456 0.006758672 0.0097838029729429409 0 +257 0 -3.2840023 0.00769185647 0.011139901588193415 0 +258 0 -3.17981029 0.008752749 0.012683135492315652 0 +259 0 3.2001853 0.962585449 4.7402567363092185 1 +260 1 5.68128252 0.9982559 0.0025183889627922629 1 +261 1 8.199528 0.999925 0.00010818114329742547 1 +262 1 6.8721137 0.999605954 0.00056860068728618158 1 +263 1 5.0797143 0.996306956 0.0053377989740299161 1 +264 1 3.525185 0.974764 0.036875140095723347 1 +265 0 -1.68991446 0.0538234077 0.079818624850797534 0 +266 1 6.01183558 0.9988457 0.0016662698840470586 1 +267 1 1.77634811 0.8126442 0.29930428855214908 1 +268 1 5.17829752 0.996733844 0.004719778628306334 1 +269 0 -3.73906183 0.004368966 0.0063168947718665724 0 +270 1 4.61221361 0.99339354 0.0095627293167596539 1 +271 0 -2.64225721 0.01699872 0.024734798891773106 0 +272 1 1.77634811 0.8126442 0.29930428855214908 1 +273 1 0.140956879 0.359505683 1.4759135174464615 1 +274 0 -3.01589465 0.0107223364 0.015552591359388794 0 +275 0 ? ? ? 0 +276 0 -3.2840023 0.00769185647 0.011139901588193415 0 +277 0 -3.87066913 0.00370853045 0.0053602239211324705 0 +278 0 -3.73906183 0.004368966 0.0063168947718665724 0 +279 1 4.06260633 0.986950636 0.018950167357454934 1 +280 0 -3.17981029 0.008752749 0.012683135492315652 0 +281 0 -3.26104116 0.007914111 0.011463068787094601 0 +282 1 2.78699446 0.9388275 0.091067970681622645 1 +283 1 3.913549 0.98431915 0.022801931644451701 1 +284 1 4.711718 0.9941619 0.0084472744150402741 1 +285 1 8.92367 0.999969661 4.3770248615567833E-05 1 +286 1 9.36437 0.999982536 2.5195678362732549E-05 1 +287 0 -3.27424979 0.00778549025 0.01127604010215829 0 +288 1 1.20037937 0.678552449 0.55946776173928858 1 +289 1 5.82277155 0.9985383 0.0021103079519827128 1 +290 0 -3.97486115 0.00325702247 0.0047065590202725379 0 +291 0 -3.73906183 0.004368966 0.0063168947718665724 0 +292 1 ? ? ? 0 +293 1 3.90838766 0.984219253 0.022948356485633058 1 +294 0 ? ? ? 0 +295 1 5.08335733 0.9963237 0.0053135460475590181 1 +296 0 1.25781727 0.694012165 1.7084537976374921 1 +297 0 ? ? ? 0 +298 0 -1.75630522 0.04974919 0.073619747846277589 0 +299 1 3.69021845 0.9793722 0.030070844989444904 1 +300 1 4.923652 0.9955148 0.0064853160810323541 1 +301 0 -3.73906183 0.004368966 0.0063168947718665724 0 +302 1 9.410986 0.9999835 2.3819793783233171E-05 1 +303 0 -3.73906183 0.004368966 0.0063168947718665724 0 +304 1 3.570249 0.976113558 0.034879098976964652 1 +305 1 6.113725 0.9989836 0.0014670697566217968 1 +306 0 -3.73906183 0.004368966 0.0063168947718665724 0 +307 0 -3.73906183 0.004368966 0.0063168947718665724 0 +308 1 4.296971 0.9902329 0.014160233982445864 1 +309 0 -1.66254663 0.0555929057 0.082519215932313594 0 +310 0 -3.567583 0.00540805142 0.0078233426086431572 0 +311 0 -3.97486115 0.00325702247 0.0047065590202725379 0 +312 1 2.62645674 0.9262326 0.11055359616867837 1 +313 0 -3.97486115 0.00325702247 0.0047065590202725379 0 +314 0 -3.88042164 0.00366374827 0.0052953778600079247 0 +315 0 ? ? ? 0 +316 1 2.91174841 0.947196 0.078265096076527124 1 +317 1 6.374063 0.9992658 0.0010596301898306959 1 +318 0 -3.65363669 0.00485905726 0.0070272245084438368 0 +319 0 1.40300608 0.731152833 1.8951418230764641 1 +320 1 3.919786 0.9844391 0.022626171569145127 1 +321 0 ? ? ? 0 +322 0 -3.17981029 0.008752749 0.012683135492315652 0 +323 1 3.29634714 0.966682851 0.048885446016359933 1 +324 0 -3.73906183 0.004368966 0.0063168947718665724 0 +325 0 -2.455094 0.021384865 0.031186499675284536 0 +326 1 2.0131712 0.853634059 0.22831035500751037 1 +327 0 -3.87066913 0.00370853045 0.0053602239211324705 0 +328 1 3.5962863 0.976861 0.033774802692909008 1 +329 1 4.53891039 0.992764 0.010477299679813668 1 +330 1 3.80646467 0.982112765 0.026039411878349145 1 +331 0 -2.56356573 0.0187233817 0.027268210614595282 0 +332 0 -1.93801951 0.0400429368 0.058958216237415963 0 +333 1 3.366857 0.9694084 0.044823521820101658 1 +334 1 4.694679 0.9940369 0.0086286685574627053 1 +335 0 -3.97486115 0.00325702247 0.0047065590202725379 0 +336 1 3.820992 0.9824291 0.025574819110896988 1 +337 0 -3.73906183 0.004368966 0.0063168947718665724 0 +338 0 -3.88042164 0.00366374827 0.0052953778600079247 0 +339 1 4.039209 0.986568451 0.019508942176747292 1 +340 1 3.63125753 0.977829 0.032345932382251898 1 +341 0 -3.73906183 0.004368966 0.0063168947718665724 0 +342 0 -3.681528 0.004693306 0.0067869484659852783 0 +343 0 -3.97486115 0.00325702247 0.0047065590202725379 0 +344 1 5.890162 0.9986563 0.0019398920753992185 1 +345 0 -3.97486115 0.00325702247 0.0047065590202725379 0 +346 0 -1.86854362 0.04351881 0.064191500046396052 0 +347 0 -3.65678263 0.004840074 0.006999704153136937 0 +348 1 0.148870945 0.361787319 1.4667862531855564 1 +349 1 1.77262115 0.811933637 0.30056628122867551 1 +350 0 -2.61288142 0.0176234785 0.025652013808458211 0 +351 0 -3.468138 0.00611970061 0.0088559874474810514 0 +352 0 0.6161294 0.504154 1.0120360544962548 1 +353 1 6.694899 0.9995083 0.00070951688495590249 1 +354 0 -3.87066913 0.00370853045 0.0053602239211324705 0 +355 0 -2.81700087 0.0137081565 0.019913492074579597 0 +356 1 -0.533400059 0.194554165 2.3617562291038281 0 +357 1 7.642427 0.999849558 0.0002170584331640775 1 +358 1 4.7509 0.9944394 0.0080446038461501317 1 +359 1 3.33471727 0.9681939 0.046632107093894461 1 +360 1 9.4182 0.999983668 2.3561815570658182E-05 1 +361 1 3.849914 0.983042538 0.024674248745318209 1 +362 0 -2.10898018 0.032587532 0.047796963479285831 0 +363 0 -0.738674164 0.15744701 0.2471606719707819 0 +364 0 -3.468138 0.00611970061 0.0088559874474810514 0 +365 0 -3.57733583 0.00534285465 0.0077287752075301903 0 +366 1 8.829243 0.9999659 4.9187876653739729E-05 1 +367 1 7.417814 0.9998008 0.00028741163655769122 1 +368 0 -3.415572 0.006532717 0.0094556377041636937 0 +369 0 -3.30061579 0.007534917 0.010911748740188923 0 +370 0 -2.22285223 0.0283856876 0.041544353114114883 0 +371 0 -3.415572 0.006532717 0.0094556377041636937 0 +372 0 -2.74657869 0.0149510521 0.02173267988754999 0 +373 0 -2.60301232 0.01783839 0.025967663038287066 0 +374 0 -3.03546929 0.0104658064 0.015178533994777435 0 +375 0 -3.97486115 0.00325702247 0.0047065590202725379 0 +376 0 -3.87066913 0.00370853045 0.0053602239211324705 0 +377 0 -3.88042164 0.00366374827 0.0052953778600079247 0 +378 0 -2.58872437 0.0181540959 0.026431476432137755 0 +379 0 -1.23155785 0.09165277 0.13868420436398607 0 +380 0 -3.97486115 0.00325702247 0.0047065590202725379 0 +381 1 5.73564959 0.998370349 0.002353006639128899 1 +382 0 -2.29505873 0.0259989444 0.038004759043758912 0 +383 0 -3.681528 0.004693306 0.0067869484659852783 0 +384 0 -3.681528 0.004693306 0.0067869484659852783 0 +385 0 -2.201686 0.0291247759 0.042642201036212003 0 +386 1 4.04930353 0.9867347 0.019265867706910304 1 +387 0 -1.48691368 0.06831274 0.10208233183703019 0 +388 0 -3.41342 0.00655020261 0.0094810301694396859 0 +389 0 -2.19644761 0.0293105487 0.042918280909058033 0 +390 0 -3.675396 0.00472925836 0.0068390622093441548 0 +391 1 6.9191947 0.9996285 0.00053608351931672734 1 +392 0 -3.2840023 0.00769185647 0.011139901588193415 0 +393 0 -3.984614 0.00321767386 0.0046496065986144043 0 +394 0 -2.98441339 0.0111480094 0.016173497541192971 0 +395 0 -3.2840023 0.00769185647 0.011139901588193415 0 +396 0 -3.17981029 0.008752749 0.012683135492315652 0 +397 0 -3.15590024 0.009015975 0.013066293424780577 0 +398 0 -2.86309981 0.0129503114 0.018805382372919104 0 +399 0 -3.12784457 0.009334855 0.01353060051180181 0 +400 1 5.67829943 0.9982494 0.002527778423844856 1 +401 0 -3.38819456 0.006758672 0.0097838029729429409 0 +402 0 -1.69518113 0.0534890331 0.079308872895756902 0 +403 0 -2.406535 0.0226931069 0.033116427654792162 0 +404 0 -3.22184753 0.008308303 0.012036417601674301 0 +405 0 -3.87066913 0.00370853045 0.0053602239211324705 0 +406 0 -2.4895916 0.020500578 0.029883453704483483 0 +407 0 -3.87066913 0.00370853045 0.0053602239211324705 0 +408 0 -2.14217162 0.03130428 0.045884529271691343 0 +409 0 -3.03546929 0.0104658064 0.015178533994777435 0 +410 0 -3.87066913 0.00370853045 0.0053602239211324705 0 +411 0 ? ? ? 0 +412 1 6.123529 0.99899596 0.0014492515521410023 1 +413 0 -2.35019088 0.0243092123 0.035504088280376593 0 +414 1 4.46620035 0.9920808 0.011470458045774687 1 +415 0 -0.325322151 0.2385733 0.39322293085530013 0 +416 1 6.08612967 0.998948 0.001518545805775124 1 +417 0 -3.87066913 0.00370853045 0.0053602239211324705 0 +418 0 -1.4770298 0.0691035 0.10330732425013402 0 +419 0 -3.09412026 0.009732979 0.014110501588453093 0 +420 0 -1.73121428 0.0512534119 0.075905302625184065 0 +421 1 7.77754259 0.9998729 0.00018334515555207742 1 +422 0 -1.53781438 0.0643717051 0.095992603056749282 0 +423 0 -2.20915484 0.0288618673 0.042251578604554092 0 +424 0 -3.38819456 0.006758672 0.0097838029729429409 0 +425 1 10.0773563 0.999992847 1.0318995955417813E-05 1 +426 0 -1.06710243 0.110269189 0.16855918151236909 0 +427 1 3.68332815 0.9791974 0.030328392701741962 1 +428 0 -3.87066913 0.00370853045 0.0053602239211324705 0 +429 0 -3.57733583 0.00534285465 0.0077287752075301903 0 +430 0 -3.19277978 0.00861317 0.012480001501394462 0 +431 0 -2.08206773 0.03366521 0.049404992594263168 0 +432 0 -2.59208584 0.0180793311 0.026321623577786946 0 +433 0 -2.69847083 0.01586327 0.023069325447678323 0 +434 0 4.185824 0.988793 6.4794580475221997 1 +435 1 6.03236055 0.9988749 0.001624086057658322 1 +436 1 4.465445 0.992073357 0.011481292804353144 1 +437 0 -3.15590024 0.009015975 0.013066293424780577 0 +438 0 -2.41803432 0.0223764051 0.032648989896200666 0 +439 0 -2.94222879 0.0117447041 0.017044313649163523 0 +440 1 4.675401 0.9938923 0.0088385502305568572 1 +441 0 -1.03582191 0.114165284 0.17489055668950451 0 +442 0 -3.09304976 0.009745888 0.014129308598108592 0 +443 0 -3.64764166 0.004895438 0.0070799680351424314 0 +444 0 -2.1350143 0.0315767974 0.046290449334290169 0 +445 0 -3.681528 0.004693306 0.0067869484659852783 0 +446 0 -3.97486115 0.00325702247 0.0047065590202725379 0 +447 0 -2.94222879 0.0117447041 0.017044313649163523 0 +448 0 -3.984614 0.00321767386 0.0046496065986144043 0 +449 1 7.0466 0.9996832 0.0004571163054142658 1 +450 0 -2.81177545 0.0137967728 0.020043121052069519 0 +451 0 -2.94222879 0.0117447041 0.017044313649163523 0 +452 0 -3.1156857 0.009476499 0.013736889781926455 0 +453 1 4.959913 0.995712757 0.0061984807665729754 1 +454 0 -3.15107036 0.009070091 0.013145079293411164 0 +455 1 0.486377716 0.4636599 1.1088610970020061 1 +456 1 6.109856 0.9989787 0.001474214315881076 1 +457 1 6.0406146 0.9988864 0.0016074711340110803 1 +458 0 -2.63705277 0.0171077978 0.024894895880045932 0 +459 0 -2.331877 0.0248582829 0.036316194250453245 0 +460 0 -2.61288142 0.0176234785 0.025652013808458211 0 +461 0 -2.07931519 0.03377735 0.049572420404281323 0 +462 0 -2.31954813 0.0252347048 0.036873207580538685 0 +463 0 -2.93094563 0.0119095827 0.017285030110952879 0 +464 0 -3.15590024 0.009015975 0.013066293424780577 0 +465 1 6.21495676 0.9991043 0.0012927706874989533 1 +466 1 5.88577127 0.9986489 0.0019505693866031825 1 +467 1 4.98309946 0.995834768 0.0060217094456027857 1 +468 0 -3.15590024 0.009015975 0.013066293424780577 0 +469 0 -3.57731676 0.005342982 0.0077289595960768854 0 +470 0 -3.319243 0.00736273173 0.010661473684257657 0 +471 0 -2.31954813 0.0252347048 0.036873207580538685 0 +472 0 -2.69978023 0.01583773 0.023031887440822525 0 +473 0 -3.15590024 0.009015975 0.013066293424780577 0 +474 0 -2.94222879 0.0117447041 0.017044313649163523 0 +475 0 -3.38819456 0.006758672 0.0097838029729429409 0 +476 0 -2.90797687 0.0122523177 0.017785538952055393 0 +477 0 -3.15590024 0.009015975 0.013066293424780577 0 +478 0 -2.65450168 0.0167447813 0.024362156466683743 0 +479 1 5.49178553 0.997790635 0.0031909667649362879 1 +480 0 -2.970704 0.0113385627 0.016451533820027363 0 +481 0 -1.76049423 0.0495021679 0.073244758440750346 0 +482 1 8.692329 0.9999595 5.8389291528570374E-05 1 +483 1 7.04786062 0.9996837 0.00045642815697035847 1 +484 0 -2.63705277 0.0171077978 0.024894895880045932 0 +485 0 -3.121525 0.00940821 0.013637431474631581 0 +486 0 -3.319243 0.00736273173 0.010661473684257657 0 +487 1 7.855387 0.9998847 0.00016631680968548799 1 +488 1 0.977370739 0.614979148 0.70139060104572637 1 +489 1 -0.553885 0.190571889 2.3915927677982021 0 +490 0 -3.97486115 0.00325702247 0.0047065590202725379 0 +491 1 4.81295633 0.9948525 0.0074454761948489351 1 +492 0 -3.09819674 0.009683976 0.014039111537691676 0 +493 1 6.908913 0.9996237 0.00054296539848543808 1 +494 0 -0.0599432 0.303919256 0.52267342876117118 0 +495 0 -3.319243 0.00736273173 0.010661473684257657 0 +496 0 -3.984614 0.00321767386 0.0046496065986144043 0 +497 0 -2.92542553 0.011991078 0.017404025124045725 0 +498 0 -2.990669 0.0110621145 0.016048185727640486 0 +499 0 -2.990669 0.0110621145 0.016048185727640486 0 +500 0 -2.38444281 0.0233138688 0.034033084074584205 0 +501 0 -2.990669 0.0110621145 0.016048185727640486 0 +502 0 -2.87897158 0.012699076 0.018438217469620204 0 +503 0 -2.88647676 0.0125819575 0.018267088068968144 0 +504 0 -3.97486115 0.00325702247 0.0047065590202725379 0 +505 0 -2.90862179 0.0122425631 0.01777129143863106 0 +506 1 6.43408537 0.999318838 0.00098304371100394524 1 +507 0 -2.99320173 0.0110275242 0.015997725215742106 0 +508 0 -2.94222879 0.0117447041 0.017044313649163523 0 +509 0 -3.681528 0.004693306 0.0067869484659852783 0 +510 0 -3.97486115 0.00325702247 0.0047065590202725379 0 +511 0 -2.79879785 0.0140193077 0.020368699164028413 0 +512 0 -2.94222879 0.0117447041 0.017044313649163523 0 +513 0 -3.319243 0.00736273173 0.010661473684257657 0 +514 1 6.63378572 0.9994693 0.00076587001079550445 1 +515 1 5.963886 0.9987744 0.0017692380359633547 1 +516 0 -3.984614 0.00321767386 0.0046496065986144043 0 +517 0 -3.88042164 0.00366374827 0.0052953778600079247 0 +518 0 -2.97475743 0.01128189 0.016368836871694589 0 +519 1 4.307943 0.9903647 0.01396824463027458 1 +520 0 -3.96510887 0.00329684885 0.004764205244298333 0 +521 0 -3.08510852 0.009842183 0.014269607046958815 0 +522 1 2.96704721 0.9505492 0.073166815180372324 1 +523 1 4.59153032 0.993221641 0.0098123991929780774 1 +524 0 -3.2840023 0.00769185647 0.011139901588193415 0 +525 0 -3.27697968 0.007759167 0.011237766032611151 0 +526 0 -3.15590024 0.009015975 0.013066293424780577 0 +527 0 -2.88647676 0.0125819575 0.018267088068968144 0 +528 0 -1.92569065 0.04063971 0.059855369904849146 0 +529 0 -3.09819674 0.009683976 0.014039111537691676 0 +530 1 4.273786 0.989948452 0.014574691588158144 1 +531 0 -2.4895916 0.020500578 0.029883453704483483 0 +532 0 -3.76753688 0.00421679672 0.0060964144371177219 0 +533 0 -3.2840023 0.00769185647 0.011139901588193415 0 +534 0 -3.57733583 0.00534285465 0.0077287752075301903 0 +535 0 -2.921613 0.0120476848 0.01748668487360899 0 +536 0 -2.64225721 0.01699872 0.024734798891773106 0 +537 0 -2.35019088 0.0243092123 0.035504088280376593 0 +538 0 -2.990669 0.0110621145 0.016048185727640486 0 +539 0 -2.52787375 0.01956116 0.028500457834611762 0 +540 0 -2.50515723 0.0201133937 0.029313286417103342 0 +541 0 -3.38819456 0.006758672 0.0097838029729429409 0 +542 0 -2.36677432 0.0238222349 0.034784203848312246 0 +543 0 -2.990669 0.0110621145 0.016048185727640486 0 +544 0 -2.79412 0.014100384 0.020487335529911339 0 +545 0 -2.79879785 0.0140193077 0.020368699164028413 0 +546 1 8.00272751 0.9999041 0.00013836667767535985 1 +547 0 -3.69128036 0.00463668862 0.0067048839531781302 0 +548 0 -3.49494314 0.00591918826 0.0085649574296908868 0 +549 1 3.83969927 0.9828283 0.024988666982595317 1 +550 0 -3.2840023 0.00769185647 0.011139901588193415 0 +551 0 -3.73906183 0.004368966 0.0063168947718665724 0 +552 0 -2.15162778 0.0309477244 0.045353600788481219 0 +553 0 -0.679618835 0.167491838 0.26446367887826999 0 +554 0 -3.38819456 0.006758672 0.0097838029729429409 0 +555 0 -1.12090206 0.103840515 0.15817259059744809 0 +556 0 -2.111688 0.03248097 0.047638057216391763 0 +557 0 -2.61288142 0.0176234785 0.025652013808458211 0 +558 0 -3.57733583 0.00534285465 0.0077287752075301903 0 +559 0 -2.79879785 0.0140193077 0.020368699164028413 0 +560 0 -2.64225721 0.01699872 0.024734798891773106 0 +561 0 -2.64225721 0.01699872 0.024734798891773106 0 +562 0 -3.73906183 0.004368966 0.0063168947718665724 0 +563 0 -3.2840023 0.00769185647 0.011139901588193415 0 +564 0 -2.72256112 0.0153998248 0.022390098641937944 0 +565 1 8.024355 0.999906659 0.00013466870077045523 1 +566 0 -2.892038 0.01249586 0.018141299029212434 0 +567 0 -2.35460019 0.0241787918 0.035311256176439453 0 +568 1 3.3369565 0.96828 0.046503773358084505 1 +569 1 6.149011 0.999027431 0.0014038032155086271 1 +570 1 5.34282827 0.997339547 0.0038433375907094549 1 +571 1 7.192057 0.999735832 0.00038116390214088962 1 +572 0 -3.2840023 0.00769185647 0.011139901588193415 0 +573 0 -3.87066913 0.00370853045 0.0053602239211324705 0 +574 1 4.329594 0.9906196 0.013596929313576941 1 +575 0 -2.35019088 0.0243092123 0.035504088280376593 0 +576 0 -2.79879785 0.0140193077 0.020368699164028413 0 +577 0 -3.87066913 0.00370853045 0.0053602239211324705 0 +578 0 -3.87066913 0.00370853045 0.0053602239211324705 0 +579 0 -3.73906183 0.004368966 0.0063168947718665724 0 +580 0 -2.49362159 0.0203996412 0.029734792634007537 0 +581 1 5.98707151 0.9988094 0.0017187000731817429 1 +582 1 6.11009932 0.998979 0.0014737839197434645 1 +583 0 -3.38819456 0.006758672 0.0097838029729429409 0 +584 0 -2.31071448 0.0255078189 0.037277484924287833 0 +585 0 -3.97486115 0.00325702247 0.0047065590202725379 0 +586 1 9.207914 0.9999788 3.0613236649743679E-05 1 +587 0 -2.59208584 0.0180793311 0.026321623577786946 0 +588 1 3.639958 0.978063643 0.031999749932399228 1 +589 0 -2.94222879 0.0117447041 0.017044313649163523 0 +590 1 2.36684227 0.9007514 0.15079908682988977 1 +591 1 4.02482557 0.986328065 0.019860509544129948 1 +592 1 3.52459 0.9747457 0.036902223148631855 1 +593 0 -2.63705277 0.0171077978 0.024894895880045932 0 +594 1 3.70278215 0.979687154 0.029606971176003354 1 +595 0 -2.79879785 0.0140193077 0.020368699164028413 0 +596 0 -2.74657869 0.0149510521 0.02173267988754999 0 +597 0 -2.10496283 0.03274626 0.048033691550600131 0 +598 0 -3.2840023 0.00769185647 0.011139901588193415 0 +599 0 -1.82202268 0.0460053273 0.067946884867431043 0 +600 0 -3.2840023 0.00769185647 0.011139901588193415 0 +601 0 -3.88042164 0.00366374827 0.0052953778600079247 0 +602 0 -2.990669 0.0110621145 0.016048185727640486 0 +603 1 2.87687731 0.9449723 0.081656090285904329 1 +604 1 2.88311434 0.9453764 0.081039250436542046 1 +605 1 6.255757 0.999148846 0.0012284790139721639 1 +606 0 -3.006056 0.0108536119 0.015744047301215536 0 +607 0 -3.97486115 0.00325702247 0.0047065590202725379 0 +608 1 7.39554167 0.9997952 0.00029549645428258596 1 +609 0 -2.94222879 0.0117447041 0.017044313649163523 0 +610 1 4.38265 0.991216362 0.012728093352343199 1 +611 1 4.56122828 0.9929617 0.010190016066295809 1 +612 1 9.386885 0.999983 2.4507735908961852E-05 1 +613 0 -3.03611445 0.0104574552 0.01516635842762787 0 +614 0 -3.587088 0.00527844345 0.0076353533571779132 0 +615 0 -2.49588585 0.0203431416 0.029651585881844127 0 +616 0 -3.2840023 0.00769185647 0.011139901588193415 0 +617 0 ? ? ? 0 +618 0 -2.990669 0.0110621145 0.016048185727640486 0 +619 0 -2.79879785 0.0140193077 0.020368699164028413 0 +620 0 -3.2840023 0.00769185647 0.011139901588193415 0 +621 0 -0.445914268 0.212269649 0.3442262313480513 0 +622 0 -1.6779325 0.05459148 0.080990229577983963 0 +623 0 -3.97486115 0.00325702247 0.0047065590202725379 0 +624 0 -2.620176 0.0174662638 0.025421150470708472 0 +625 0 -1.90860176 0.041480992 0.061121053196586103 0 +626 1 3.861115 0.98327446 0.024333924932683069 1 +627 0 -2.37264848 0.023652032 0.034532682535721816 0 +628 0 -3.681528 0.004693306 0.0067869484659852783 0 +629 0 -3.15590024 0.009015975 0.013066293424780577 0 +630 0 -1.77322578 0.04875851 0.072116453615565554 0 +631 0 -2.79879785 0.0140193077 0.020368699164028413 0 +632 0 -3.97486115 0.00325702247 0.0047065590202725379 0 +633 1 3.23095369 0.9639466 0.052974896175934497 1 +634 0 -3.38819456 0.006758672 0.0097838029729429409 0 +635 0 -2.59167171 0.0180885252 0.02633513202553334 0 +636 1 6.00484228 0.998835564 0.0016809053773215632 1 +637 0 -1.48554516 0.06842173 0.10225110556923336 0 +638 0 -3.15590024 0.009015975 0.013066293424780577 0 +639 0 -2.61288142 0.0176234785 0.025652013808458211 0 +640 0 -2.906215 0.0122790076 0.017824522413232794 0 +641 0 -3.2840023 0.00769185647 0.011139901588193415 0 +642 0 -3.2840023 0.00769185647 0.011139901588193415 0 +643 0 -3.97486115 0.00325702247 0.0047065590202725379 0 +644 0 -3.681528 0.004693306 0.0067869484659852783 0 +645 0 -3.2840023 0.00769185647 0.011139901588193415 0 +646 0 -3.43302059 0.006392639 0.0092522334759092793 0 +647 0 -3.61173034 0.005119108 0.0074042797770863325 0 +648 1 7.104957 0.9997055 0.00042494571667690465 1 +649 0 -3.2840023 0.00769185647 0.011139901588193415 0 +650 0 -2.30245924 0.0257656518 0.037659246331383393 0 +651 0 -3.18057179 0.008744492 0.012671117868884718 0 +652 0 -2.59208584 0.0180793311 0.026321623577786946 0 +653 0 -2.990669 0.0110621145 0.016048185727640486 0 +654 0 -3.17981029 0.008752749 0.012683135492315652 0 +655 0 -3.2840023 0.00769185647 0.011139901588193415 0 +656 0 -2.79879785 0.0140193077 0.020368699164028413 0 +657 0 0.110147 0.3506842 0.62300777012778197 1 +658 1 5.661574 0.998212457 0.0025811875298442886 1 +659 0 -3.97486115 0.00325702247 0.0047065590202725379 0 +660 0 -3.87066913 0.00370853045 0.0053602239211324705 0 +661 0 -2.88647676 0.0125819575 0.018267088068968144 0 +662 0 -3.27214074 0.007805887 0.011305697804660998 0 +663 0 -3.27214074 0.007805887 0.011305697804660998 0 +664 0 -2.93998575 0.0117773 0.017091899816281272 0 +665 0 -3.97486115 0.00325702247 0.0047065590202725379 0 +666 0 -2.17654967 0.0300267469 0.043983129208117804 0 +667 0 -3.17981029 0.008752749 0.012683135492315652 0 +668 1 1.96231651 0.8455096 0.24210697890608038 1 +669 1 6.25082827 0.9991436 0.001236052716862961 1 +670 1 4.78304243 0.994657338 0.0077284962607992108 1 +671 0 -2.70239019 0.0157869458 0.022957443003682965 0 +672 0 -3.468138 0.00611970061 0.0088559874474810514 0 +673 0 -2.03371119 0.0356885865 0.05242897113699075 0 +674 0 -3.87066913 0.00370853045 0.0053602239211324705 0 +675 0 -2.36259 0.0239442009 0.034964468907911599 0 +676 0 -3.57731676 0.005342982 0.0077289595960768854 0 +677 0 -2.94222879 0.0117447041 0.017044313649163523 0 +678 0 -3.97486115 0.00325702247 0.0047065590202725379 0 +679 0 -3.681528 0.004693306 0.0067869484659852783 0 +680 1 9.588415 0.999986768 1.9090200549023394E-05 1 +681 1 6.817062 0.99957794 0.00060903310882859201 1 +682 0 -2.42922783 0.022072278 0.032200254364630927 0 +683 0 -3.97486115 0.00325702247 0.0047065590202725379 0 +684 0 -3.97486115 0.00325702247 0.0047065590202725379 0 +685 0 -3.97486115 0.00325702247 0.0047065590202725379 0 +686 0 -3.97486115 0.00325702247 0.0047065590202725379 0 +687 0 -2.95617127 0.0115440777 0.01675146078438388 0 +688 0 -3.15590024 0.009015975 0.013066293424780577 0 +689 0 -3.5034287 0.005857085 0.0084748305070051697 0 +690 0 -3.61173034 0.005119108 0.0074042797770863325 0 +691 1 4.218167 0.989232361 0.015618658690216777 1 +692 0 -3.38819456 0.006758672 0.0097838029729429409 0 +693 0 -3.156261 0.009011946 0.013060428078630799 0 +694 0 -3.051222 0.01026378 0.01488401918556947 0 +695 0 -3.681528 0.004693306 0.0067869484659852783 0 +696 1 5.57695055 0.9980134 0.002868941573516763 1 +697 1 3.34162855 0.968458951 0.046237195400023308 1 +698 1 3.96502829 0.9852823 0.021390951837600855 1 diff --git a/test/BaselineOutput/Common/WeightedEnsemble/WE-BestPerf-TrainTest-breast-cancer-out.txt b/test/BaselineOutput/Common/WeightedEnsemble/WE-BestPerf-TrainTest-breast-cancer-out.txt new file mode 100644 index 0000000000..d11eb7b9dc --- /dev/null +++ b/test/BaselineOutput/Common/WeightedEnsemble/WE-BestPerf-TrainTest-breast-cancer-out.txt @@ -0,0 +1,141 @@ +maml.exe TrainTest test=%Data% tr=WeightedEnsemble{nm=20 pt=BestPerformanceSelector tp=-} dout=%Output% loader=Text{col=Label:BL:0 col=Features:R4:1-9} data=%Data% out=%Output% seed=1 +Automatically adding a MinMax normalization transform, use 'norm=Warn' or 'norm=No' to turn this behavior off. +Training 20 learners for the batch 1 +Beginning training model 1 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 12 instances with missing features during training (over 1 iterations; 12 inst/iter) +Trainer 1 of 20 finished in %Time% +Beginning training model 2 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 18 instances with missing features during training (over 1 iterations; 18 inst/iter) +Trainer 2 of 20 finished in %Time% +Beginning training model 3 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 15 instances with missing features during training (over 1 iterations; 15 inst/iter) +Trainer 3 of 20 finished in %Time% +Beginning training model 4 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 24 instances with missing features during training (over 1 iterations; 24 inst/iter) +Trainer 4 of 20 finished in %Time% +Beginning training model 5 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 18 instances with missing features during training (over 1 iterations; 18 inst/iter) +Trainer 5 of 20 finished in %Time% +Beginning training model 6 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 15 instances with missing features during training (over 1 iterations; 15 inst/iter) +Trainer 6 of 20 finished in %Time% +Beginning training model 7 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 18 instances with missing features during training (over 1 iterations; 18 inst/iter) +Trainer 7 of 20 finished in %Time% +Beginning training model 8 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 14 instances with missing features during training (over 1 iterations; 14 inst/iter) +Trainer 8 of 20 finished in %Time% +Beginning training model 9 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 13 instances with missing features during training (over 1 iterations; 13 inst/iter) +Trainer 9 of 20 finished in %Time% +Beginning training model 10 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 14 instances with missing features during training (over 1 iterations; 14 inst/iter) +Trainer 10 of 20 finished in %Time% +Beginning training model 11 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 17 instances with missing features during training (over 1 iterations; 17 inst/iter) +Trainer 11 of 20 finished in %Time% +Beginning training model 12 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 24 instances with missing features during training (over 1 iterations; 24 inst/iter) +Trainer 12 of 20 finished in %Time% +Beginning training model 13 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 12 instances with missing features during training (over 1 iterations; 12 inst/iter) +Trainer 13 of 20 finished in %Time% +Beginning training model 14 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 12 instances with missing features during training (over 1 iterations; 12 inst/iter) +Trainer 14 of 20 finished in %Time% +Beginning training model 15 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 19 instances with missing features during training (over 1 iterations; 19 inst/iter) +Trainer 15 of 20 finished in %Time% +Beginning training model 16 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 23 instances with missing features during training (over 1 iterations; 23 inst/iter) +Trainer 16 of 20 finished in %Time% +Beginning training model 17 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 18 instances with missing features during training (over 1 iterations; 18 inst/iter) +Trainer 17 of 20 finished in %Time% +Beginning training model 18 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 20 instances with missing features during training (over 1 iterations; 20 inst/iter) +Trainer 18 of 20 finished in %Time% +Beginning training model 19 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 13 instances with missing features during training (over 1 iterations; 13 inst/iter) +Trainer 19 of 20 finished in %Time% +Beginning training model 20 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 15 instances with missing features during training (over 1 iterations; 15 inst/iter) +Trainer 20 of 20 finished in %Time% +List of models and the metrics after sorted +| AUC(Sorted) || Name of Model | +| 0.996916752312436 |LinearBinaryModelParameters +| 0.996659815005139 |LinearBinaryModelParameters +| 0.996659815005139 |LinearBinaryModelParameters +| 0.996659815005139 |LinearBinaryModelParameters +| 0.99653134635149 |LinearBinaryModelParameters +| 0.996402877697842 |LinearBinaryModelParameters +| 0.996402877697842 |LinearBinaryModelParameters +| 0.996274409044193 |LinearBinaryModelParameters +| 0.996274409044193 |LinearBinaryModelParameters +| 0.996145940390545 |LinearBinaryModelParameters +| 0.996145940390545 |LinearBinaryModelParameters +| 0.996145940390545 |LinearBinaryModelParameters +| 0.995889003083248 |LinearBinaryModelParameters +| 0.995760534429599 |LinearBinaryModelParameters +| 0.995632065775951 |LinearBinaryModelParameters +| 0.995503597122302 |LinearBinaryModelParameters +| 0.995246659815005 |LinearBinaryModelParameters +| 0.994732785200411 |LinearBinaryModelParameters +| 0.994218910585817 |LinearBinaryModelParameters +| 0.994026207605344 |LinearBinaryModelParameters +Warning: 10 of 20 trainings failed. +Training calibrator. +Warning: The predictor produced non-finite prediction values on 16 instances during testing. Possible causes: abnormal data or the predictor is numerically unstable. +TEST POSITIVE RATIO: 0.3499 (239.0/(239.0+444.0)) +Confusion table + ||====================== +PREDICTED || positive | negative | Recall +TRUTH ||====================== + positive || 229 | 10 | 0.9582 + negative || 9 | 435 | 0.9797 + ||====================== +Precision || 0.9622 | 0.9775 | +OVERALL 0/1 ACCURACY: 0.972182 +LOG LOSS/instance: 0.117306 +Test-set entropy (prior Log-Loss/instance): 0.934003 +LOG-LOSS REDUCTION (RIG): 0.874405 +AUC: 0.996042 + +OVERALL RESULTS +--------------------------------------- +AUC: 0.996042 (0.0000) +Accuracy: 0.972182 (0.0000) +Positive precision: 0.962185 (0.0000) +Positive recall: 0.958159 (0.0000) +Negative precision: 0.977528 (0.0000) +Negative recall: 0.979730 (0.0000) +Log-loss: 0.117306 (0.0000) +Log-loss reduction: 0.874405 (0.0000) +F1 Score: 0.960168 (0.0000) +AUPRC: 0.991960 (0.0000) + +--------------------------------------- +Physical memory usage(MB): %Number% +Virtual memory usage(MB): %Number% +%DateTime% Time elapsed(s): %Number% + diff --git a/test/BaselineOutput/Common/WeightedEnsemble/WE-BestPerf-TrainTest-breast-cancer-rp.txt b/test/BaselineOutput/Common/WeightedEnsemble/WE-BestPerf-TrainTest-breast-cancer-rp.txt new file mode 100644 index 0000000000..c15ac44dee --- /dev/null +++ b/test/BaselineOutput/Common/WeightedEnsemble/WE-BestPerf-TrainTest-breast-cancer-rp.txt @@ -0,0 +1,4 @@ +WeightedEnsemble +AUC Accuracy Positive precision Positive recall Negative precision Negative recall Log-loss Log-loss reduction F1 Score AUPRC /pt /bp /nm Learner Name Train Dataset Test Dataset Results File Run Time Physical Memory Virtual Memory Command Line Settings +0.996042 0.972182 0.962185 0.958159 0.977528 0.97973 0.117306 0.874405 0.960168 0.99196 BestPerformanceSelector Microsoft.ML.Runtime.ComponentFactoryUtils+SimpleComponentFactory`1[Microsoft.ML.Trainers.LinearSvmTrainer] 20 WeightedEnsemble %Data% %Data% %Output% 99 0 0 maml.exe TrainTest test=%Data% tr=WeightedEnsemble{nm=20 pt=BestPerformanceSelector tp=-} dout=%Output% loader=Text{col=Label:BL:0 col=Features:R4:1-9} data=%Data% out=%Output% seed=1 /pt:BestPerformanceSelector;/bp:Microsoft.ML.Runtime.ComponentFactoryUtils+SimpleComponentFactory`1[Microsoft.ML.Trainers.LinearSvmTrainer];/nm:20 + diff --git a/test/BaselineOutput/Common/WeightedEnsemble/WE-BestPerf-TrainTest-breast-cancer.txt b/test/BaselineOutput/Common/WeightedEnsemble/WE-BestPerf-TrainTest-breast-cancer.txt new file mode 100644 index 0000000000..481ee4c0a6 --- /dev/null +++ b/test/BaselineOutput/Common/WeightedEnsemble/WE-BestPerf-TrainTest-breast-cancer.txt @@ -0,0 +1,700 @@ +Instance Label Score Probability Log-loss Assigned +0 0 -2.66742277 0.0173032172 0.025181762183421101 0 +1 0 1.92188334 0.935199559 3.9478525524547492 1 +2 0 -2.84293485 0.013440121 0.01952147722305005 0 +3 0 1.9052124 0.933706939 3.9149983255128973 1 +4 0 -2.64747477 0.0178061072 0.025920242852290207 0 +5 1 6.54004669 0.999918938 0.00011695294288901403 1 +6 0 -0.895662665 0.1900987 0.30418199784314165 0 +7 0 -3.2027173 0.00798696 0.01156900994272678 0 +8 0 -3.23861074 0.00758177973 0.010979872306042195 0 +9 0 -2.83651471 0.0135651343 0.019704301892903266 0 +10 0 -3.694624 0.003907208 0.0056479504685260657 0 +11 0 -3.5356257 0.00492454553 0.0071221685657312048 0 +12 1 -0.590930462 0.2681758 1.8987489826883688 0 +13 0 -3.03873348 0.010128649 0.01468705784147719 0 +14 1 4.52897453 0.9984695 0.0022097767487974694 1 +15 1 0.248248816 0.555483758 0.84818336499546876 1 +16 0 -3.01246476 0.0105209909 0.01525899281245123 0 +17 0 -2.90871668 0.0122229392 0.017742629524281441 0 +18 1 4.35306025 0.998021543 0.0028571373596076245 1 +19 0 -2.43676567 0.0240747333 0.035157419873519329 0 +20 1 3.200592 0.989425957 0.015336346354349288 1 +21 1 3.88487744 0.9960851 0.0056590808346631801 1 +22 0 -3.27404523 0.0072018 0.0104275952839934 0 +23 1 ? ? ? 0 +24 0 -3.64413142 0.00420525949 0.0060796993535018095 0 +25 1 0.6550276 0.6937055 0.52760477366555414 1 +26 0 -3.15779281 0.008524471 0.012350929666763313 0 +27 0 -2.76338673 0.0150725525 0.021910639322057369 0 +28 0 -3.5356257 0.00492454553 0.0071221685657312048 0 +29 0 -3.44263363 0.005637582 0.0081563241231015181 0 +30 0 -3.333743 0.006603924 0.0095590470040104909 0 +31 0 -3.413193 0.005884048 0.0085139598725839372 0 +32 1 3.61568475 0.994208336 0.0083798953714551323 1 +33 0 -3.26127243 0.00733653875 0.010623405411459631 0 +34 0 -3.044885 0.0100388862 0.014556238440078305 0 +35 0 -3.5356257 0.00492454553 0.0071221685657312048 0 +36 1 4.2966814 0.9978519 0.0031023746642231979 1 +37 0 -1.85091662 0.0548986979 0.08145911980851514 0 +38 1 2.82893562 0.9819327 0.026303947297194059 1 +39 1 0.750898838 0.7226509 0.46862924892958807 1 +40 0 ? ? ? 0 +41 1 1.0369091 0.7983079 0.3249828147671498 1 +42 1 4.974605 0.999201536 0.001152399931555637 1 +43 1 0.142294168 0.516984344 0.95180750458980801 1 +44 1 4.135688 0.9972835 0.0039243873362431953 1 +45 0 -3.644427 0.00420345 0.0060770776881953984 0 +46 1 2.83945584 0.9822035 0.025906155537906657 1 +47 0 -3.78055835 0.0034475422 0.0049823454013636472 0 +48 0 -2.64747477 0.0178061072 0.025920242852290207 0 +49 1 3.48782229 0.993026257 0.010096230396603824 1 +50 1 1.432833 0.8759399 0.19109619858154167 1 +51 1 -0.324052572 0.3512012 1.5096302940624107 0 +52 1 2.74381542 0.9795875 0.029753737221563312 1 +53 1 3.62717438 0.99430424 0.0082407360375667624 1 +54 1 3.44110823 0.9925371 0.010807092456410844 1 +55 1 2.52158785 0.971972 0.041013357576551591 1 +56 1 3.95000625 0.996439338 0.0051461170377164198 1 +57 1 -0.249369383 0.376463056 1.4094198026395468 0 +58 1 1.06590009 0.8050455 0.31285779632296751 1 +59 1 0.5570309 0.6624529 0.59411026245226617 1 +60 1 0.9123702 0.767401338 0.38194681524604779 1 +61 0 -3.46984386 0.00541892741 0.0078391187422378692 0 +62 1 4.122214 0.9972296 0.0040024235745129907 1 +63 1 0.004101038 0.4665367 1.0999375178358572 1 +64 0 -3.78055835 0.0034475422 0.0049823454013636472 0 +65 1 0.594244 0.6745077 0.56809323020513491 1 +66 0 -2.90871668 0.0122229392 0.017742629524281441 0 +67 1 2.32312632 0.962888241 0.054559735842220611 1 +68 1 4.43897533 0.998254657 0.0025201979367746328 1 +69 0 -3.569469 0.00468795327 0.0067791896481360289 0 +70 0 -2.35167551 0.027176911 0.039750624767311164 0 +71 1 3.024763 0.9863691 0.019800528687725676 1 +72 0 -2.17284846 0.0350122638 0.051417487265828753 0 +73 1 4.488894 0.998377264 0.0023430153976866925 1 +74 1 1.3723805 0.8660132 0.20753903169868643 1 +75 0 -2.93895149 0.0117006507 0.01698000411808669 0 +76 0 -3.10972548 0.009139335 0.013245894925929336 0 +77 0 -2.3095386 0.0288537573 0.042239530734669961 0 +78 0 -2.51557255 0.0215113889 0.031373035927598047 0 +79 0 -3.475928 0.00537120225 0.0077698924104884146 0 +80 0 -2.56806326 0.0199541952 0.029078916325822535 0 +81 0 -2.92301917 0.0119730635 0.017377720404537673 0 +82 0 -2.47118616 0.0229204036 0.033452000849850415 0 +83 0 -2.3600173 0.0268563628 0.039275330627440118 0 +84 1 4.301124 0.9978658 0.0030822956933520078 1 +85 1 2.99262 0.9857226 0.020746389329958963 1 +86 1 1.23290062 0.8405413 0.25060938143796063 1 +87 1 3.42107987 0.9923171 0.011126908380003894 1 +88 0 -2.90871668 0.0122229392 0.017742629524281441 0 +89 0 -3.34579515 0.00648934161 0.0093926500900978812 0 +90 0 -3.64413142 0.00420525949 0.0060796993535018095 0 +91 0 -3.28834772 0.007053839 0.010212600120393248 0 +92 0 -2.90871668 0.0122229392 0.017742629524281441 0 +93 0 -3.78055835 0.0034475422 0.0049823454013636472 0 +94 0 -3.413193 0.005884048 0.0085139598725839372 0 +95 0 -3.64413142 0.00420525949 0.0060796993535018095 0 +96 0 -3.51968956 0.00504003046 0.007289612355740928 0 +97 0 -2.66742277 0.0173032172 0.025181762183421101 0 +98 1 4.3791 0.9980953 0.0027505591586714463 1 +99 1 5.078471 0.999313951 0.00099009982328770638 1 +100 1 2.279173 0.9605226 0.058108547544452695 1 +101 1 -0.583294153 0.2703723 1.8869807345620795 0 +102 0 -2.69038534 0.01674159 0.024357474858508105 0 +103 1 -0.133133888 0.4171048 1.2615181439398047 0 +104 1 5.488136 0.999622941 0.00054408370695103255 1 +105 1 0.404060841 0.610788 0.71125640577650595 1 +106 1 5.49459267 0.9996265 0.00053892229049603603 1 +107 1 3.61195755 0.994176865 0.0084255640073111464 1 +108 0 -3.549697 0.00482476642 0.0069775125951322993 0 +109 1 3.09781528 0.987732768 0.01780732265733807 1 +110 0 -2.345357 0.0274221841 0.040114410252493599 0 +111 1 2.239045 0.9582372 0.061545316454678763 1 +112 1 3.94019938 0.996388137 0.0052202494445462676 1 +113 1 5.265396 0.9994779 0.00075339470162483256 1 +114 0 -1.95085716 0.0477931462 0.07065308175161876 0 +115 0 -2.61424422 0.0186758917 0.027198391432709686 0 +116 0 -0.770080566 0.219979241 0.35841557629657605 0 +117 1 4.15887451 0.99737376 0.0037938477510984885 1 +118 0 -3.158492 0.008515836 0.012338364617343436 0 +119 0 -2.6669848 0.0173141062 0.025197748424118319 0 +120 0 -3.29666519 0.006969186 0.010089609260225759 0 +121 0 -2.40540457 0.0251756981 0.036785877688029524 0 +122 1 5.704817 0.9997253 0.00039638846889223365 1 +123 1 1.93638277 0.9364723 0.094691778457870426 1 +124 1 3.240358 0.9900172 0.01447454035633826 1 +125 0 -3.78055835 0.0034475422 0.0049823454013636472 0 +126 1 3.616165 0.9942123 0.0083741004017039537 1 +127 0 -3.17029715 0.008371357 0.012128150713631205 0 +128 1 2.844878 0.9823415 0.025703405375544344 1 +129 0 -3.55365372 0.004797073 0.006937366407270217 0 +130 0 -2.35167551 0.027176911 0.039750624767311164 0 +131 0 -3.413193 0.005884048 0.0085139598725839372 0 +132 1 5.05268574 0.9992876 0.0010281345838752979 1 +133 0 -3.15709066 0.00853315 0.012363558519001954 0 +134 0 -3.21434736 0.007853377 0.011374751385524661 0 +135 0 -2.08802128 0.0394522436 0.058070751466098219 0 +136 0 -3.01246476 0.0105209909 0.01525899281245123 0 +137 0 -3.37779331 0.00619462132 0.0089647448212200921 0 +138 0 -2.78634953 0.0145822484 0.02119263388477512 0 +139 0 ? ? ? 0 +140 0 -3.37779331 0.00619462132 0.0089647448212200921 0 +141 0 -3.63937378 0.0042344844 0.0061220406501051013 0 +142 1 2.02801919 0.9439909 0.08315516377598145 1 +143 0 -2.61424422 0.0186758917 0.027198391432709686 0 +144 0 -3.5356257 0.00492454553 0.0071221685657312048 0 +145 0 ? ? ? 0 +146 1 0.609972 0.6795351 0.55738004176978684 1 +147 0 -3.41014624 0.005910158 0.008551852230919867 0 +148 0 -1.52164328 0.08592359 0.12961332560182665 0 +149 1 5.632679 0.999694765 0.00044042879818134223 1 +150 0 -3.694624 0.003907208 0.0056479504685260657 0 +151 1 2.22698474 0.9575259 0.06261657056137776 1 +152 1 5.008711 0.999240339 0.001096375932495456 1 +153 0 -2.471549 0.0229085274 0.033434465259579417 0 +154 0 -3.900954 0.00289278082 0.0041794485709644473 0 +155 1 2.02270126 0.9435784 0.083785667756784643 1 +156 0 -3.39209557 0.006067228 0.0087798214065122112 0 +157 0 -3.413193 0.005884048 0.0085139598725839372 0 +158 0 ? ? ? 0 +159 1 6.48426342 0.9999121 0.00012684278090196348 1 +160 1 5.01594257 0.9992483 0.0010848443808666457 1 +161 0 -2.77632332 0.0147943664 0.021503217462403655 0 +162 0 -3.17029715 0.008371357 0.012128150713631205 0 +163 0 -2.5987134 0.0190965626 0.027816974036179973 0 +164 0 ? ? ? 0 +165 0 -2.48856068 0.0223584343 0.032622470337482931 0 +166 1 4.40091038 0.998154938 0.0026643201529028002 1 +167 1 4.06925058 0.997007251 0.0043240981528398694 1 +168 0 -3.17029715 0.008371357 0.012128150713631205 0 +169 0 -3.80475187 0.003328142 0.0048095017423065989 0 +170 0 -3.37779331 0.00619462132 0.0089647448212200921 0 +171 0 -3.64413142 0.00420525949 0.0060796993535018095 0 +172 0 -3.78055835 0.0034475422 0.0049823454013636472 0 +173 1 7.189026 0.9999686 4.5318140264806784E-05 1 +174 1 3.10505152 0.987860262 0.017621114827569635 1 +175 1 4.39201736 0.998130858 0.0026991252850564256 1 +176 0 -3.413193 0.005884048 0.0085139598725839372 0 +177 1 2.66991019 0.9773115 0.033109638842967269 1 +178 0 -2.90871668 0.0122229392 0.017742629524281441 0 +179 1 1.19036865 0.8320304 0.26529182668995943 1 +180 0 -3.694624 0.003907208 0.0056479504685260657 0 +181 0 -3.900954 0.00289278082 0.0041794485709644473 0 +182 0 -2.43676567 0.0240747333 0.035157419873519329 0 +183 1 4.71948338 0.9988411 0.0016728988831946291 1 +184 1 3.41794968 0.9922821 0.011177776997423909 1 +185 0 -3.46368551 0.005467664 0.00790981558255059 0 +186 1 2.8064692 0.9813407 0.027173990020175707 1 +187 1 6.17948437 0.999862731 0.00019805161600238084 1 +188 1 4.15850639 0.9973724 0.0037958307608025798 1 +189 0 -3.05918741 0.009833215 0.014256539484967077 0 +190 1 6.36240864 0.9998949 0.00015161067272639314 1 +191 1 5.8389945 0.9997742 0.00032577191890018108 1 +192 0 -2.76338673 0.0150725525 0.021910639322057369 0 +193 0 -3.64413142 0.00420525949 0.0060796993535018095 0 +194 0 -3.17029715 0.008371357 0.012128150713631205 0 +195 0 -2.90871668 0.0122229392 0.017742629524281441 0 +196 0 2.93798423 0.9845539 6.0166111015557249 1 +197 0 -2.295424 0.0294375848 0.043107101731544854 0 +198 0 -3.900954 0.00289278082 0.0041794485709644473 0 +199 0 -3.27404523 0.0072018 0.0104275952839934 0 +200 1 5.18994141 0.999417067 0.00084124038011639598 1 +201 1 5.18347645 0.9994115 0.00084924226012300153 1 +202 0 -3.64413142 0.00420525949 0.0060796993535018095 0 +203 0 -2.66742277 0.0173032172 0.025181762183421101 0 +204 0 -3.64413142 0.00420525949 0.0060796993535018095 0 +205 1 6.259695 0.9998779 0.00017612098427402409 1 +206 1 3.81511474 0.9956666 0.0062653261769018264 1 +207 0 -3.694624 0.003907208 0.0056479504685260657 0 +208 0 -3.694624 0.003907208 0.0056479504685260657 0 +209 0 -2.69449 0.0166431032 0.024212975597467417 0 +210 1 7.252383 0.9999714 4.1276426673909545E-05 1 +211 1 5.04657269 0.9992812 0.0010373422445471184 1 +212 0 -3.64413142 0.00420525949 0.0060796993535018095 0 +213 1 7.57545853 0.9999822 2.571163541833834E-05 1 +214 1 7.06320667 0.9999623 5.4433535886118404E-05 1 +215 1 3.8888576 0.9961077 0.0056263624027311342 1 +216 0 -3.78055835 0.0034475422 0.0049823454013636472 0 +217 0 -3.64413142 0.00420525949 0.0060796993535018095 0 +218 1 3.78555369 0.995476067 0.0065414633614432382 1 +219 0 -2.02689314 0.0429815 0.063381283872070801 0 +220 0 -3.49221087 0.005245524 0.0075876093888766595 0 +221 1 5.005602 0.9992369 0.0011013672296929196 1 +222 1 -1.60298657 0.07703247 3.6983895193049099 0 +223 1 2.67806554 0.977574348 0.032721665551025854 1 +224 1 4.842024 0.999030948 0.0013987247971255085 1 +225 0 -3.78055835 0.0034475422 0.0049823454013636472 0 +226 1 5.113798 0.999348462 0.0009402775250079695 1 +227 1 3.89216161 0.9961264 0.0055992558733935997 1 +228 0 -3.694624 0.003907208 0.0056479504685260657 0 +229 1 6.92661858 0.9999539 6.6472825917701672E-05 1 +230 1 2.795224 0.9810373 0.027620077210660659 1 +231 1 4.326909 0.9979446 0.0029683765225967168 1 +232 0 0.241290808 0.552970767 1.1615589169584668 1 +233 1 3.27628088 0.99052304 0.013737561234255987 1 +234 0 -1.9102695 0.05056693 0.07486179377505417 0 +235 0 ? ? ? 0 +236 1 5.97012329 0.999813557 0.00026900594391058349 1 +237 1 3.35221267 0.9915102 0.012300462792434555 1 +238 1 6.64586735 0.999930561 0.00010018337250504059 1 +239 1 2.84136939 0.9822523 0.025834454361286535 1 +240 0 -1.56066775 0.08154757 0.12272309909228359 0 +241 0 -2.92890549 0.0118716964 0.017229714132355939 0 +242 0 -3.413193 0.005884048 0.0085139598725839372 0 +243 0 -2.176513 0.03483172 0.05114759434164428 0 +244 0 -3.64413142 0.00420525949 0.0060796993535018095 0 +245 0 -2.25364971 0.03123339 0.045778954836489617 0 +246 1 6.257004 0.999877453 0.00017680899902768678 1 +247 1 1.85674667 0.9291844 0.10596319723548779 1 +248 0 -2.23169374 0.0322193056 0.047247934780888547 0 +249 0 ? ? ? 0 +250 0 -3.49286652 0.005240525 0.0075803595270597822 0 +251 1 4.101593 0.997145057 0.0041247032079651811 1 +252 0 2.13269019 0.9515525 4.3674342740970227 1 +253 1 4.974605 0.999201536 0.001152399931555637 1 +254 1 4.122214 0.9972296 0.0040024235745129907 1 +255 1 2.7252965 0.979039133 0.030561568794320867 1 +256 0 -3.37779331 0.00619462132 0.0089647448212200921 0 +257 0 -3.27404523 0.0072018 0.0104275952839934 0 +258 0 -3.17029715 0.008371357 0.012128150713631205 0 +259 0 2.21123838 0.9565798 4.5254899654112384 1 +260 1 4.91683 0.999131262 0.0012538682883803704 1 +261 1 6.153361 0.999857366 0.00020579191855977315 1 +262 1 5.318161 0.999516666 0.00069747222754798889 1 +263 1 4.254122 0.997714341 0.0033012835999902709 1 +264 1 2.63360858 0.9761045 0.034892489572781897 1 +265 0 -1.89054751 0.0519691519 0.076994091023900021 0 +266 1 4.535683 0.9984844 0.0021882461246302489 1 +267 1 1.29838872 0.8529583 0.22945284658103121 1 +268 1 3.97027946 0.996543 0.0049960515675275766 1 +269 0 -3.64413142 0.00420525949 0.0060796993535018095 0 +270 1 3.61567473 0.9942082 0.0083800683559846004 1 +271 0 -2.66742277 0.0173032172 0.025181762183421101 0 +272 1 1.29838872 0.8529583 0.22945284658103121 1 +273 1 -0.23899436 0.380029857 1.395815326113772 0 +274 0 -3.0407052 0.0100997919 0.014645000570930024 0 +275 0 ? ? ? 0 +276 0 -3.27404523 0.0072018 0.0104275952839934 0 +277 0 -3.78055835 0.0034475422 0.0049823454013636472 0 +278 0 -3.64413142 0.00420525949 0.0060796993535018095 0 +279 1 3.451947 0.992653549 0.01063781193740174 1 +280 0 -3.17029715 0.008371357 0.012128150713631205 0 +281 0 -3.26127243 0.00733653875 0.010623405411459631 0 +282 1 2.054083 0.9459716 0.080131210877312078 1 +283 1 3.30453515 0.99090296 0.013184314517142018 1 +284 1 3.77685332 0.9954184 0.006624997282365771 1 +285 1 7.155258 0.999967039 4.7553986690113E-05 1 +286 1 8.021416 0.9999907 1.3414709134555599E-05 1 +287 0 -3.21434736 0.007853377 0.011374751385524661 0 +288 1 0.77104187 0.728513837 0.45697172083961368 1 +289 1 4.34308767 0.9979925 0.0028990987629432608 1 +290 0 -3.900954 0.00289278082 0.0041794485709644473 0 +291 0 -3.64413142 0.00420525949 0.0060796993535018095 0 +292 1 ? ? ? 0 +293 1 3.04578352 0.986776054 0.019205388705116795 1 +294 0 ? ? ? 0 +295 1 3.65415382 0.9945233 0.0079229426462178888 1 +296 0 0.7122183 0.7111764 1.7917394412208034 1 +297 0 ? ? ? 0 +298 0 -1.7920996 0.0595343821 0.088552893037769531 0 +299 1 3.16891837 0.9889304 0.016059099755649762 1 +300 1 3.4670527 0.9928128 0.010406361204663574 1 +301 0 -3.64413142 0.00420525949 0.0060796993535018095 0 +302 1 7.86794949 0.9999884 1.6768405908113185E-05 1 +303 0 -3.64413142 0.00420525949 0.0060796993535018095 0 +304 1 2.901587 0.9837235 0.02367519659306094 1 +305 1 4.79915047 0.998968363 0.0014891061013214593 1 +306 0 -3.64413142 0.00420525949 0.0060796993535018095 0 +307 0 -3.64413142 0.00420525949 0.0060796993535018095 0 +308 1 3.23013186 0.989868343 0.014691442129693146 1 +309 0 -1.69989109 0.06754502 0.10089402140794446 0 +310 0 -3.475928 0.00537120225 0.0077698924104884146 0 +311 0 -3.900954 0.00289278082 0.0041794485709644473 0 +312 1 1.50922155 0.8875743 0.17206017595056614 1 +313 0 -3.900954 0.00289278082 0.0041794485709644473 0 +314 0 -3.83105063 0.00320302276 0.0046284014518315104 0 +315 0 ? ? ? 0 +316 1 2.0967083 0.949069262 0.07541471823128014 1 +317 1 4.475775 0.9983459 0.0023883210642366183 1 +318 0 -3.48971343 0.00526460866 0.0076152881903264599 0 +319 0 0.5874169 0.672312737 1.6096084954234813 1 +320 1 3.21166778 0.989594 0.015091366966232641 1 +321 0 ? ? ? 0 +322 0 -3.17029715 0.008371357 0.012128150713631205 0 +323 1 2.76208162 0.9801147 0.028977503675241524 1 +324 0 -3.64413142 0.00420525949 0.0060796993535018095 0 +325 0 -2.638041 0.018048903 0.026276917429046594 0 +326 1 1.6740272 0.909465 0.13690995316367283 1 +327 0 -3.78055835 0.0034475422 0.0049823454013636472 0 +328 1 2.21737266 0.9569507 0.063483456373453803 1 +329 1 3.75630569 0.995279431 0.0068264660990948137 1 +330 1 2.96059084 0.9850485 0.021733375531701717 1 +331 0 -2.59062386 0.0193193424 0.02814467176775428 0 +332 0 -2.14993453 0.03616179 0.053137101370244522 0 +333 1 2.50216532 0.971188068 0.042177397532308675 1 +334 1 3.62696886 0.9943025 0.0082432440733423637 1 +335 0 -3.900954 0.00289278082 0.0041794485709644473 0 +336 1 2.90934372 0.983904064 0.023410443519403041 1 +337 0 -3.64413142 0.00420525949 0.0060796993535018095 0 +338 0 -3.83105063 0.00320302276 0.0046284014518315104 0 +339 1 3.03718233 0.986611 0.019446709817627899 1 +340 1 2.946356 0.9847389 0.022186865037433071 1 +341 0 -3.64413142 0.00420525949 0.0060796993535018095 0 +342 0 -3.63937378 0.0042344844 0.0061220406501051013 0 +343 0 -3.900954 0.00289278082 0.0041794485709644473 0 +344 1 4.56397438 0.9985457 0.0020996294585008315 1 +345 0 -3.900954 0.00289278082 0.0041794485709644473 0 +346 0 -2.08377171 0.0396883339 0.058425390918933999 0 +347 0 -3.71247339 0.00380695821 0.0055027607761118248 0 +348 1 -0.57234 0.273542851 1.8701612421797231 0 +349 1 1.28584218 0.8506431 0.23337414304276421 1 +350 0 -2.6757412 0.0170976557 0.024880009317771259 0 +351 0 -3.413193 0.005884048 0.0085139598725839372 0 +352 0 -0.08689451 0.4336253 0.82017130047407605 0 +353 1 4.61671543 0.9986535 0.0019439391114753155 1 +354 0 -3.78055835 0.0034475422 0.0049823454013636472 0 +355 0 -2.84631777 0.0133747067 0.019425821940625237 0 +356 1 -0.900367737 0.189042 2.4032212821815913 0 +357 1 6.616737 0.9999276 0.00010448324376444981 1 +358 1 3.179951 0.9891056 0.015803564964737706 1 +359 1 2.476605 0.9701239 0.043759100332545438 1 +360 1 7.376692 0.999976158 3.4396940217671718E-05 1 +361 1 3.43527412 0.992473662 0.010899278124026284 1 +362 0 -2.19320726 0.03402053 0.049935564463074671 0 +363 0 -1.203353 0.130203038 0.20124942583044297 0 +364 0 -3.413193 0.005884048 0.0085139598725839372 0 +365 0 -3.5356257 0.00492454553 0.0071221685657312048 0 +366 1 6.93450737 0.999954462 6.5698868536796025E-05 1 +367 1 5.53060627 0.99964565 0.00051130902611861177 1 +368 0 -3.44263363 0.005637582 0.0081563241231015181 0 +369 0 -3.39858317 0.00601030327 0.0086971973845605139 0 +370 0 -2.39949512 0.02538858 0.037100967219813395 0 +371 0 -3.44263363 0.005637582 0.0081563241231015181 0 +372 0 -2.78634953 0.0145822484 0.02119263388477512 0 +373 0 -2.661439 0.0174525864 0.025401067511833422 0 +374 0 -3.044885 0.0100388862 0.014556238440078305 0 +375 0 -3.900954 0.00289278082 0.0041794485709644473 0 +376 0 -3.78055835 0.0034475422 0.0049823454013636472 0 +377 0 -3.83105063 0.00320302276 0.0046284014518315104 0 +378 0 -2.71122646 0.0162473861 0.023632530938569745 0 +379 0 -1.563164 0.08127468 0.12229450633545844 0 +380 0 -3.900954 0.00289278082 0.0041794485709644473 0 +381 1 4.56174755 0.998540938 0.0021065188000870855 1 +382 0 -2.5623126 0.02011926 0.029321922221634263 0 +383 0 -3.63937378 0.0042344844 0.0061220406501051013 0 +384 0 -3.63937378 0.0042344844 0.0061220406501051013 0 +385 0 -2.34328151 0.0275032166 0.040234616651813743 0 +386 1 3.041533 0.986694753 0.019324257621276347 1 +387 0 -1.74042356 0.06390743 0.095276889557213348 0 +388 0 -3.42147255 0.00581367174 0.0084118309645924913 0 +389 0 -2.313829 0.02867853 0.041979244941978258 0 +390 0 -3.62863755 0.00430117967 0.0062186739991650945 0 +391 1 5.50033665 0.9996296 0.00053444907783827022 1 +392 0 -3.27404523 0.0072018 0.0104275952839934 0 +393 0 -3.96065187 0.00265167351 0.0038306372684312529 0 +394 0 -3.1629355 0.008461166 0.012258817872293441 0 +395 0 -3.27404523 0.0072018 0.0104275952839934 0 +396 0 -3.17029715 0.008371357 0.012128150713631205 0 +397 0 -3.11621284 0.009053852 0.013121437793081982 0 +398 0 -3.00056767 0.0107035935 0.015525258300279633 0 +399 0 -3.254984 0.007403792 0.010721151576072032 0 +400 1 4.860357 0.9990566 0.0013617131352557648 1 +401 0 -3.37779331 0.00619462132 0.0089647448212200921 0 +402 0 -1.9820329 0.04576134 0.067577957114250548 0 +403 0 -2.41954 0.0246735159 0.036042862585208599 0 +404 0 -3.3164773 0.006771585 0.009802559074870423 0 +405 0 -3.78055835 0.0034475422 0.0049823454013636472 0 +406 0 -2.6073544 0.0188613757 0.027471106872765348 0 +407 0 -3.78055835 0.0034475422 0.0049823454013636472 0 +408 0 -2.41573954 0.0248075649 0.036241160383883177 0 +409 0 -3.044885 0.0100388862 0.014556238440078305 0 +410 0 -3.78055835 0.0034475422 0.0049823454013636472 0 +411 0 ? ? ? 0 +412 1 4.788395 0.998952031 0.0015126922494384223 1 +413 0 -2.461567 0.02323744 0.033920194109643358 0 +414 1 3.47551942 0.9929006 0.010278784666155427 1 +415 0 -0.868057966 0.196389437 0.31543156679312212 0 +416 1 4.613462 0.998647034 0.0019532387267514424 1 +417 0 -3.78055835 0.0034475422 0.0049823454013636472 0 +418 0 -1.69347847 0.06813784 0.10181152517768946 0 +419 0 -3.14854884 0.008639444 0.012518236031716503 0 +420 0 -1.9289521 0.0492717065 0.072894999454249987 0 +421 1 6.371919 0.9998964 0.00014946066526327797 1 +422 0 -1.87438178 0.0531458557 0.078785888255212211 0 +423 0 -2.35167551 0.027176911 0.039750624767311164 0 +424 0 -3.37779331 0.00619462132 0.0089647448212200921 0 +425 1 7.394905 0.999976754 3.3537006717225477E-05 1 +426 0 -1.43597829 0.0962827951 0.14605670595847076 0 +427 1 2.44531751 0.9687694 0.045774841198830629 1 +428 0 -3.78055835 0.0034475422 0.0049823454013636472 0 +429 0 -3.5356257 0.00492454553 0.0071221685657312048 0 +430 0 -3.37280846 0.00623964425 0.0090301055312780459 0 +431 0 -2.228159 0.0323808156 0.047488721837585963 0 +432 0 -2.62832713 0.0183023047 0.026649266470900049 0 +433 0 -2.730778 0.01579682 0.022971916569033728 0 +434 0 3.01473856 0.9861706 6.176116573299149 1 +435 1 4.60573864 0.9986317 0.0019753686078578599 1 +436 1 2.671332 0.9773575 0.033041713988898533 1 +437 0 -3.11621284 0.009053852 0.013121437793081982 0 +438 0 -2.488044 0.02237495 0.032646843135181673 0 +439 0 -2.85935068 0.0131256152 0.019061633256597182 0 +440 1 4.24383926 0.9976798 0.0033511875018570926 1 +441 0 -1.32194912 0.111795194 0.17103571798824946 0 +442 0 -2.97080374 0.0111742709 0.016211812419461295 0 +443 0 -3.573592 0.00465991441 0.0067385481637882785 0 +444 0 -2.19359636 0.0340018421 0.049907657017243312 0 +445 0 -3.63937378 0.0042344844 0.0061220406501051013 0 +446 0 -3.900954 0.00289278082 0.0041794485709644473 0 +447 0 -2.85935068 0.0131256152 0.019061633256597182 0 +448 0 -3.96065187 0.00265167351 0.0038306372684312529 0 +449 1 5.19441032 0.9994209 0.00083573373578482452 1 +450 0 -2.834394 0.0136066815 0.019765067432605428 0 +451 0 -2.85935068 0.0131256152 0.019061633256597182 0 +452 0 -3.132863 0.008838066 0.012807313522219074 0 +453 1 3.69043326 0.9948047 0.0075147997388948663 1 +454 0 -3.2191658 0.007798683 0.011295222544037609 0 +455 1 0.34057498 0.5885124 0.76485523097918529 1 +456 1 4.915495 0.999129534 0.0012563642072738559 1 +457 1 4.114376 0.997197747 0.0040484712466912244 1 +458 0 -2.65349531 0.0176528357 0.025695127672574668 0 +459 0 -2.44763947 0.02370407 0.034609579098047183 0 +460 0 -2.6757412 0.0170976557 0.024880009317771259 0 +461 0 -2.12682748 0.03735775 0.054928352780122043 0 +462 0 -2.419607 0.02467116 0.036039377249238717 0 +463 0 -3.00909424 0.0105724074 0.015333961754446224 0 +464 0 -3.11621284 0.009053852 0.013121437793081982 0 +465 1 4.782152 0.998942435 0.001526551443335498 1 +466 1 4.636257 0.9986913 0.0018892620303051387 1 +467 1 3.73379683 0.995122254 0.0070543187253418226 1 +468 0 -3.11621284 0.009053852 0.013121437793081982 0 +469 0 -3.56804562 0.004697672 0.0067932770463530927 0 +470 0 -3.333743 0.006603924 0.0095590470040104909 0 +471 0 -2.419607 0.02467116 0.036039377249238717 0 +472 0 -2.669642 0.01724814 0.025100906677716882 0 +473 0 -3.11621284 0.009053852 0.013121437793081982 0 +474 0 -2.85935068 0.0131256152 0.019061633256597182 0 +475 0 -3.37779331 0.00619462132 0.0089647448212200921 0 +476 0 -2.88705254 0.01261127 0.018309916483475169 0 +477 0 -3.11621284 0.009053852 0.013121437793081982 0 +478 0 -2.67868233 0.01702555 0.02477417835308119 0 +479 1 4.285586 0.997816861 0.0031530473010327334 1 +480 0 -2.91433 0.0121242618 0.017598513790312083 0 +481 0 -1.97171152 0.0464247353 0.068581281498749549 0 +482 1 6.872484 0.9999502 7.1890539209731462E-05 1 +483 1 5.57829952 0.9996695 0.00047690071355122043 1 +484 0 -2.65349531 0.0176528357 0.025695127672574668 0 +485 0 -3.03981757 0.0101127727 0.014663919064912947 0 +486 0 -3.333743 0.006603924 0.0095590470040104909 0 +487 1 6.912789 0.999953032 6.7762755808549813E-05 1 +488 1 0.436611176 0.6220384 0.68492439487906187 1 +489 1 -0.7794001 0.217650488 2.1999148399267829 0 +490 0 -3.900954 0.00289278082 0.0041794485709644473 0 +491 1 3.64681482 0.9944645 0.008008199525875552 1 +492 0 -3.07216263 0.009650242 0.013989968059371619 0 +493 1 5.533239 0.999647 0.00050933052590907497 1 +494 0 -0.140151024 0.414612949 0.77253726152965863 0 +495 0 -3.333743 0.006603924 0.0095590470040104909 0 +496 0 -3.96065187 0.00265167351 0.0038306372684312529 0 +497 0 -2.94026279 0.0116785048 0.016947676412618715 0 +498 0 -3.01246476 0.0105209909 0.01525899281245123 0 +499 0 -3.01246476 0.0105209909 0.01525899281245123 0 +500 0 -2.43676567 0.0240747333 0.035157419873519329 0 +501 0 -3.01246476 0.0105209909 0.01525899281245123 0 +502 0 -2.92301917 0.0119730635 0.017377720404537673 0 +503 0 -2.90871668 0.0122229392 0.017742629524281441 0 +504 0 -3.900954 0.00289278082 0.0041794485709644473 0 +505 0 -3.025893 0.0103185875 0.014963911741922634 0 +506 1 5.4258585 0.999587 0.00059595696767469582 1 +507 0 -2.91886735 0.0120450743 0.017482872800599823 0 +508 0 -2.85935068 0.0131256152 0.019061633256597182 0 +509 0 -3.63937378 0.0042344844 0.0061220406501051013 0 +510 0 -3.900954 0.00289278082 0.0041794485709644473 0 +511 0 -2.76338673 0.0150725525 0.021910639322057369 0 +512 0 -2.85935068 0.0131256152 0.019061633256597182 0 +513 0 -3.333743 0.006603924 0.0095590470040104909 0 +514 1 5.11581326 0.999350369 0.00093752401120021536 1 +515 1 4.308322 0.9978881 0.0030500665132257082 1 +516 0 -3.96065187 0.00265167351 0.0038306372684312529 0 +517 0 -3.83105063 0.00320302276 0.0046284014518315104 0 +518 0 -3.05918741 0.009833215 0.014256539484967077 0 +519 1 3.18320179 0.989156663 0.015729060620257643 1 +520 0 -3.83750248 0.00317305047 0.004585022285101761 0 +521 0 -3.05651522 0.009871322 0.014312063192287916 0 +522 1 2.12751436 0.9512025 0.07217557015444305 1 +523 1 3.77960014 0.9954367 0.0065984766820997142 1 +524 0 -3.27404523 0.0072018 0.0104275952839934 0 +525 0 -3.28834772 0.007053839 0.010212600120393248 0 +526 0 -3.11621284 0.009053852 0.013121437793081982 0 +527 0 -2.90871668 0.0122229392 0.017742629524281441 0 +528 0 -2.09034586 0.03932367 0.057877651619643797 0 +529 0 -3.07216263 0.009650242 0.013989968059371619 0 +530 1 3.30377936 0.990893 0.013198806979119945 1 +531 0 -2.6073544 0.0188613757 0.027471106872765348 0 +532 0 -3.694624 0.003907208 0.0056479504685260657 0 +533 0 -3.27404523 0.0072018 0.0104275952839934 0 +534 0 -3.5356257 0.00492454553 0.0071221685657312048 0 +535 0 -3.101119 0.009253975 0.013412820965616291 0 +536 0 -2.66742277 0.0173032172 0.025181762183421101 0 +537 0 -2.461567 0.02323744 0.033920194109643358 0 +538 0 -3.01246476 0.0105209909 0.01525899281245123 0 +539 0 -2.53456831 0.0209345929 0.030522851708101831 0 +540 0 -2.45507669 0.0234537646 0.034239743578125648 0 +541 0 -3.37779331 0.00619462132 0.0089647448212200921 0 +542 0 -2.50273228 0.02191005 0.031960947525278408 0 +543 0 -3.01246476 0.0105209909 0.01525899281245123 0 +544 0 -2.98785949 0.0109021086 0.015814782787734814 0 +545 0 -2.76338673 0.0150725525 0.021910639322057369 0 +546 1 6.105034 0.999846935 0.00022084262574724909 1 +547 0 -3.69907141 0.00388198579 0.0056114204319575537 0 +548 0 -3.437491 0.005679882 0.0082176977167909161 0 +549 1 2.769906 0.980336368 0.028651250451064696 1 +550 0 -3.27404523 0.0072018 0.0104275952839934 0 +551 0 -3.64413142 0.00420525949 0.0060796993535018095 0 +552 0 -2.44592714 0.0237620678 0.034695285418990335 0 +553 0 -0.9822686 0.171366513 0.2711939703503336 0 +554 0 -3.37779331 0.00619462132 0.0089647448212200921 0 +555 0 -1.5299474 0.08497494 0.12811683502371124 0 +556 0 -2.31602621 0.02858919 0.04184655527656285 0 +557 0 -2.6757412 0.0170976557 0.024880009317771259 0 +558 0 -3.5356257 0.00492454553 0.0071221685657312048 0 +559 0 -2.76338673 0.0150725525 0.021910639322057369 0 +560 0 -2.66742277 0.0173032172 0.025181762183421101 0 +561 0 -2.66742277 0.0173032172 0.025181762183421101 0 +562 0 -3.64413142 0.00420525949 0.0060796993535018095 0 +563 0 -3.27404523 0.0072018 0.0104275952839934 0 +564 0 -2.77632332 0.0147943664 0.021503217462403655 0 +565 1 5.7963953 0.9997597 0.00034675869336092929 1 +566 0 -2.94113684 0.0116637666 0.016926162623438702 0 +567 0 -2.375585 0.02626795 0.038403265686245351 0 +568 1 2.51470876 0.971696734 0.041421974475898758 1 +569 1 5.134291 0.9993677 0.00091248448620147294 1 +570 1 4.18328238 0.9974656 0.0036609923096432846 1 +571 1 5.61406326 0.9996863 0.0004526433463961228 1 +572 0 -3.27404523 0.0072018 0.0104275952839934 0 +573 0 -3.78055835 0.0034475422 0.0049823454013636472 0 +574 1 2.96013474 0.985038638 0.021747779533044704 1 +575 0 -2.461567 0.02323744 0.033920194109643358 0 +576 0 -2.76338673 0.0150725525 0.021910639322057369 0 +577 0 -3.78055835 0.0034475422 0.0049823454013636472 0 +578 0 -3.78055835 0.0034475422 0.0049823454013636472 0 +579 0 -3.64413142 0.00420525949 0.0060796993535018095 0 +580 0 -2.557531 0.0202575214 0.029525502495468935 0 +581 1 4.54507256 0.998505 0.0021584482708646314 1 +582 1 4.56968832 0.9985578 0.0020821479018810676 1 +583 0 -3.37779331 0.00619462132 0.0089647448212200921 0 +584 0 -2.2929275 0.0295420326 0.043262366831461015 0 +585 0 -3.900954 0.00289278082 0.0041794485709644473 0 +586 1 7.016157 0.9999596 5.8303296725358504E-05 1 +587 0 -2.62832713 0.0183023047 0.026649266470900049 0 +588 1 2.970909 0.985269 0.021410414477781817 1 +589 0 -2.85935068 0.0131256152 0.019061633256597182 0 +590 1 1.59926319 0.900054157 0.15191628303657839 1 +591 1 2.84366536 0.9823108 0.025748575224610173 1 +592 1 2.80060673 0.9811831 0.027405692751976615 1 +593 0 -2.65349531 0.0176528357 0.025695127672574668 0 +594 1 2.65421653 0.976797163 0.033869083986625796 1 +595 0 -2.76338673 0.0150725525 0.021910639322057369 0 +596 0 -2.78634953 0.0145822484 0.02119263388477512 0 +597 0 -2.25571156 0.03114232 0.045643338572352289 0 +598 0 -3.27404523 0.0072018 0.0104275952839934 0 +599 0 -2.09163046 0.0392527878 0.057771209903846807 0 +600 0 -3.27404523 0.0072018 0.0104275952839934 0 +601 0 -3.83105063 0.00320302276 0.0046284014518315104 0 +602 0 -3.01246476 0.0105209909 0.01525899281245123 0 +603 1 1.48121214 0.8834232 0.1788233596974296 1 +604 1 2.39687657 0.9665543 0.04907733482899606 1 +605 1 4.92328548 0.9991394 0.0012420772816458751 1 +606 0 -3.02676725 0.0103055444 0.014944898354102855 0 +607 0 -3.900954 0.00289278082 0.0041794485709644473 0 +608 1 5.762363 0.999747455 0.0003643912603633563 1 +609 0 -2.85935068 0.0131256152 0.019061633256597182 0 +610 1 3.92145443 0.9962882 0.0053649869029975834 1 +611 1 3.3753655 0.9917905 0.011892727124567482 1 +612 1 7.771821 0.9999866 1.9348177961999343E-05 1 +613 0 -3.18372536 0.008209965 0.011893365205460048 0 +614 0 -3.59532332 0.00451486325 0.006528319168183817 0 +615 0 -2.580494 0.0196019113 0.028560423243733448 0 +616 0 -3.27404523 0.0072018 0.0104275952839934 0 +617 0 ? ? ? 0 +618 0 -3.01246476 0.0105209909 0.01525899281245123 0 +619 0 -2.76338673 0.0150725525 0.021910639322057369 0 +620 0 -3.27404523 0.0072018 0.0104275952839934 0 +621 0 -0.787310362 0.215687886 0.35050021110067653 0 +622 0 -1.87953925 0.05276774 0.078209877069736033 0 +623 0 -3.900954 0.00289278082 0.0041794485709644473 0 +624 0 -2.715375 0.016150739 0.0234908028289832 0 +625 0 -2.01026726 0.0439924225 0.064906041589407271 0 +626 1 2.45790267 0.969321251 0.044953214289875365 1 +627 0 -2.37053585 0.0264574066 0.038683995273164511 0 +628 0 -3.63937378 0.0042344844 0.0061220406501051013 0 +629 0 -3.11621284 0.009053852 0.013121437793081982 0 +630 0 -2.00752449 0.044161357 0.065161000290714727 0 +631 0 -2.76338673 0.0150725525 0.021910639322057369 0 +632 0 -3.900954 0.00289278082 0.0041794485709644473 0 +633 1 2.15529943 0.953053653 0.06937066030647869 1 +634 0 -3.37779331 0.00619462132 0.0089647448212200921 0 +635 0 -2.71465015 0.0161675848 0.023515505298539359 0 +636 1 4.35541344 0.998028338 0.0028473149486622643 1 +637 0 -1.71447229 0.06621483 0.098837417937638269 0 +638 0 -3.11621284 0.009053852 0.013121437793081982 0 +639 0 -2.6757412 0.0170976557 0.024880009317771259 0 +640 0 -2.91681433 0.0120808408 0.017535103021254658 0 +641 0 -3.27404523 0.0072018 0.0104275952839934 0 +642 0 -3.27404523 0.0072018 0.0104275952839934 0 +643 0 -3.900954 0.00289278082 0.0041794485709644473 0 +644 0 -3.63937378 0.0042344844 0.0061220406501051013 0 +645 0 -3.27404523 0.0072018 0.0104275952839934 0 +646 0 -3.49286652 0.005240525 0.0075803595270597822 0 +647 0 -3.56496 0.00471880939 0.0068239159755902152 0 +648 1 5.224649 0.999445856 0.00079968294291034563 1 +649 0 -3.27404523 0.0072018 0.0104275952839934 0 +650 0 -2.31234646 0.02873896 0.042069003567403215 0 +651 0 -3.271062 0.00723304972 0.010473006676902364 0 +652 0 -2.62832713 0.0183023047 0.026649266470900049 0 +653 0 -3.01246476 0.0105209909 0.01525899281245123 0 +654 0 -3.17029715 0.008371357 0.012128150713631205 0 +655 0 -3.27404523 0.0072018 0.0104275952839934 0 +656 0 -2.76338673 0.0150725525 0.021910639322057369 0 +657 0 -0.465034723 0.3057939 0.52656407599544219 0 +658 1 4.58259344 0.9985847 0.0020433103247990496 1 +659 0 -3.900954 0.00289278082 0.0041794485709644473 0 +660 0 -3.78055835 0.0034475422 0.0049823454013636472 0 +661 0 -2.90871668 0.0122229392 0.017742629524281441 0 +662 0 -3.33888555 0.00655478938 0.0094876911164833554 0 +663 0 -3.33888555 0.00655478938 0.0094876911164833554 0 +664 0 -3.01484013 0.0104849031 0.015206376469072776 0 +665 0 -3.900954 0.00289278082 0.0041794485709644473 0 +666 0 -2.27456117 0.0303215664 0.044421697224847234 0 +667 0 -3.17029715 0.008371357 0.012128150713631205 0 +668 1 1.0573945 0.8030866 0.31637256525368879 1 +669 1 4.245142 0.99768424 0.003344809359464131 1 +670 1 3.50679874 0.9932158 0.0098208838799085131 1 +671 0 -2.87596464 0.012814709 0.018607196285736434 0 +672 0 -3.413193 0.005884048 0.0085139598725839372 0 +673 0 -2.283297 0.02994833 0.043866499430623741 0 +674 0 -3.78055835 0.0034475422 0.0049823454013636472 0 +675 0 -2.46614432 0.0230860468 0.033696600000372452 0 +676 0 -3.56804562 0.004697672 0.0067932770463530927 0 +677 0 -2.85935068 0.0131256152 0.019061633256597182 0 +678 0 -3.900954 0.00289278082 0.0041794485709644473 0 +679 0 -3.63937378 0.0042344844 0.0061220406501051013 0 +680 1 7.992257 0.9999903 1.401665413523095E-05 1 +681 1 5.47343 0.9996148 0.0005558690104081642 1 +682 0 -2.51194167 0.0216233954 0.031538188985164167 0 +683 0 -3.900954 0.00289278082 0.0041794485709644473 0 +684 0 -3.900954 0.00289278082 0.0041794485709644473 0 +685 0 -3.900954 0.00289278082 0.0041794485709644473 0 +686 0 -3.900954 0.00289278082 0.0041794485709644473 0 +687 0 -3.02093315 0.010392894 0.015072235134239694 0 +688 0 -3.11621284 0.009053852 0.013121437793081982 0 +689 0 -3.19265366 0.00810437 0.011739770613803571 0 +690 0 -3.56496 0.00471880939 0.0068239159755902152 0 +691 1 2.9486227 0.9847886 0.022114038670657754 1 +692 0 -3.37779331 0.00619462132 0.0089647448212200921 0 +693 0 -3.14422464 0.008693753 0.01259727254303847 0 +694 0 -2.99073315 0.0108569032 0.015748847744767633 0 +695 0 -3.63937378 0.0042344844 0.0061220406501051013 0 +696 1 3.50632739 0.99321115 0.0098276370337411732 1 +697 1 2.00669169 0.942319453 0.085711868607651537 1 +698 1 2.57146478 0.9738914 0.038167223970429869 1 diff --git a/test/BaselineOutput/Common/WeightedEnsemble/WE-Default-CV-breast-cancer-out.txt b/test/BaselineOutput/Common/WeightedEnsemble/WE-Default-CV-breast-cancer-out.txt new file mode 100644 index 0000000000..1d14d97e5d --- /dev/null +++ b/test/BaselineOutput/Common/WeightedEnsemble/WE-Default-CV-breast-cancer-out.txt @@ -0,0 +1,216 @@ +maml.exe CV tr=WeightedEnsemble{nm=20 tp=-} threads=- dout=%Output% loader=Text{col=Label:BL:0 col=Features:R4:1-9} data=%Data% seed=1 +Automatically adding a MinMax normalization transform, use 'norm=Warn' or 'norm=No' to turn this behavior off. +Training 20 learners for the batch 1 +Beginning training model 1 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 9 instances with missing features during training (over 1 iterations; 9 inst/iter) +Trainer 1 of 20 finished in %Time% +Beginning training model 2 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 4 instances with missing features during training (over 1 iterations; 4 inst/iter) +Trainer 2 of 20 finished in %Time% +Beginning training model 3 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 12 instances with missing features during training (over 1 iterations; 12 inst/iter) +Trainer 3 of 20 finished in %Time% +Beginning training model 4 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 10 instances with missing features during training (over 1 iterations; 10 inst/iter) +Trainer 4 of 20 finished in %Time% +Beginning training model 5 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 11 instances with missing features during training (over 1 iterations; 11 inst/iter) +Trainer 5 of 20 finished in %Time% +Beginning training model 6 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 9 instances with missing features during training (over 1 iterations; 9 inst/iter) +Trainer 6 of 20 finished in %Time% +Beginning training model 7 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 9 instances with missing features during training (over 1 iterations; 9 inst/iter) +Trainer 7 of 20 finished in %Time% +Beginning training model 8 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 11 instances with missing features during training (over 1 iterations; 11 inst/iter) +Trainer 8 of 20 finished in %Time% +Beginning training model 9 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 9 instances with missing features during training (over 1 iterations; 9 inst/iter) +Trainer 9 of 20 finished in %Time% +Beginning training model 10 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 7 instances with missing features during training (over 1 iterations; 7 inst/iter) +Trainer 10 of 20 finished in %Time% +Beginning training model 11 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 8 instances with missing features during training (over 1 iterations; 8 inst/iter) +Trainer 11 of 20 finished in %Time% +Beginning training model 12 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 10 instances with missing features during training (over 1 iterations; 10 inst/iter) +Trainer 12 of 20 finished in %Time% +Beginning training model 13 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 11 instances with missing features during training (over 1 iterations; 11 inst/iter) +Trainer 13 of 20 finished in %Time% +Beginning training model 14 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 8 instances with missing features during training (over 1 iterations; 8 inst/iter) +Trainer 14 of 20 finished in %Time% +Beginning training model 15 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 10 instances with missing features during training (over 1 iterations; 10 inst/iter) +Trainer 15 of 20 finished in %Time% +Beginning training model 16 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 7 instances with missing features during training (over 1 iterations; 7 inst/iter) +Trainer 16 of 20 finished in %Time% +Beginning training model 17 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 6 instances with missing features during training (over 1 iterations; 6 inst/iter) +Trainer 17 of 20 finished in %Time% +Beginning training model 18 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 10 instances with missing features during training (over 1 iterations; 10 inst/iter) +Trainer 18 of 20 finished in %Time% +Beginning training model 19 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 11 instances with missing features during training (over 1 iterations; 11 inst/iter) +Trainer 19 of 20 finished in %Time% +Beginning training model 20 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 7 instances with missing features during training (over 1 iterations; 7 inst/iter) +Trainer 20 of 20 finished in %Time% +Training calibrator. +Automatically adding a MinMax normalization transform, use 'norm=Warn' or 'norm=No' to turn this behavior off. +Training 20 learners for the batch 1 +Beginning training model 1 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 13 instances with missing features during training (over 1 iterations; 13 inst/iter) +Trainer 1 of 20 finished in %Time% +Beginning training model 2 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 8 instances with missing features during training (over 1 iterations; 8 inst/iter) +Trainer 2 of 20 finished in %Time% +Beginning training model 3 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 7 instances with missing features during training (over 1 iterations; 7 inst/iter) +Trainer 3 of 20 finished in %Time% +Beginning training model 4 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 8 instances with missing features during training (over 1 iterations; 8 inst/iter) +Trainer 4 of 20 finished in %Time% +Beginning training model 5 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 6 instances with missing features during training (over 1 iterations; 6 inst/iter) +Trainer 5 of 20 finished in %Time% +Beginning training model 6 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 7 instances with missing features during training (over 1 iterations; 7 inst/iter) +Trainer 6 of 20 finished in %Time% +Beginning training model 7 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 8 instances with missing features during training (over 1 iterations; 8 inst/iter) +Trainer 7 of 20 finished in %Time% +Beginning training model 8 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 12 instances with missing features during training (over 1 iterations; 12 inst/iter) +Trainer 8 of 20 finished in %Time% +Beginning training model 9 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 7 instances with missing features during training (over 1 iterations; 7 inst/iter) +Trainer 9 of 20 finished in %Time% +Beginning training model 10 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 4 instances with missing features during training (over 1 iterations; 4 inst/iter) +Trainer 10 of 20 finished in %Time% +Beginning training model 11 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 6 instances with missing features during training (over 1 iterations; 6 inst/iter) +Trainer 11 of 20 finished in %Time% +Beginning training model 12 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 6 instances with missing features during training (over 1 iterations; 6 inst/iter) +Trainer 12 of 20 finished in %Time% +Beginning training model 13 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 11 instances with missing features during training (over 1 iterations; 11 inst/iter) +Trainer 13 of 20 finished in %Time% +Beginning training model 14 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 8 instances with missing features during training (over 1 iterations; 8 inst/iter) +Trainer 14 of 20 finished in %Time% +Beginning training model 15 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 9 instances with missing features during training (over 1 iterations; 9 inst/iter) +Trainer 15 of 20 finished in %Time% +Beginning training model 16 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 13 instances with missing features during training (over 1 iterations; 13 inst/iter) +Trainer 16 of 20 finished in %Time% +Beginning training model 17 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 7 instances with missing features during training (over 1 iterations; 7 inst/iter) +Trainer 17 of 20 finished in %Time% +Beginning training model 18 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 5 instances with missing features during training (over 1 iterations; 5 inst/iter) +Trainer 18 of 20 finished in %Time% +Beginning training model 19 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 8 instances with missing features during training (over 1 iterations; 8 inst/iter) +Trainer 19 of 20 finished in %Time% +Beginning training model 20 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 10 instances with missing features during training (over 1 iterations; 10 inst/iter) +Trainer 20 of 20 finished in %Time% +Training calibrator. +Warning: The predictor produced non-finite prediction values on 8 instances during testing. Possible causes: abnormal data or the predictor is numerically unstable. +TEST POSITIVE RATIO: 0.3785 (134.0/(134.0+220.0)) +Confusion table + ||====================== +PREDICTED || positive | negative | Recall +TRUTH ||====================== + positive || 129 | 5 | 0.9627 + negative || 7 | 213 | 0.9682 + ||====================== +Precision || 0.9485 | 0.9771 | +OVERALL 0/1 ACCURACY: 0.966102 +LOG LOSS/instance: 0.143167 +Test-set entropy (prior Log-Loss/instance): 0.956998 +LOG-LOSS REDUCTION (RIG): 0.850400 +AUC: 0.993996 +Warning: The predictor produced non-finite prediction values on 8 instances during testing. Possible causes: abnormal data or the predictor is numerically unstable. +TEST POSITIVE RATIO: 0.3191 (105.0/(105.0+224.0)) +Confusion table + ||====================== +PREDICTED || positive | negative | Recall +TRUTH ||====================== + positive || 103 | 2 | 0.9810 + negative || 4 | 220 | 0.9821 + ||====================== +Precision || 0.9626 | 0.9910 | +OVERALL 0/1 ACCURACY: 0.981763 +LOG LOSS/instance: 0.115437 +Test-set entropy (prior Log-Loss/instance): 0.903454 +LOG-LOSS REDUCTION (RIG): 0.872227 +AUC: 0.997832 + +OVERALL RESULTS +--------------------------------------- +AUC: 0.995914 (0.0019) +Accuracy: 0.973932 (0.0078) +Positive precision: 0.955573 (0.0070) +Positive recall: 0.971819 (0.0091) +Negative precision: 0.984028 (0.0070) +Negative recall: 0.975162 (0.0070) +Log-loss: 0.129302 (0.0139) +Log-loss reduction: 0.861313 (0.0109) +F1 Score: 0.963627 (0.0081) +AUPRC: 0.992159 (0.0031) + +--------------------------------------- +Physical memory usage(MB): %Number% +Virtual memory usage(MB): %Number% +%DateTime% Time elapsed(s): %Number% + diff --git a/test/BaselineOutput/Common/WeightedEnsemble/WE-Default-CV-breast-cancer-rp.txt b/test/BaselineOutput/Common/WeightedEnsemble/WE-Default-CV-breast-cancer-rp.txt new file mode 100644 index 0000000000..a1a6ed89f7 --- /dev/null +++ b/test/BaselineOutput/Common/WeightedEnsemble/WE-Default-CV-breast-cancer-rp.txt @@ -0,0 +1,4 @@ +WeightedEnsemble +AUC Accuracy Positive precision Positive recall Negative precision Negative recall Log-loss Log-loss reduction F1 Score AUPRC /bp /nm Learner Name Train Dataset Test Dataset Results File Run Time Physical Memory Virtual Memory Command Line Settings +0.995914 0.973932 0.955573 0.971819 0.984028 0.975162 0.129302 0.861313 0.963627 0.992159 Microsoft.ML.Runtime.ComponentFactoryUtils+SimpleComponentFactory`1[Microsoft.ML.Trainers.LinearSvmTrainer] 20 WeightedEnsemble %Data% %Output% 99 0 0 maml.exe CV tr=WeightedEnsemble{nm=20 tp=-} threads=- dout=%Output% loader=Text{col=Label:BL:0 col=Features:R4:1-9} data=%Data% seed=1 /bp:Microsoft.ML.Runtime.ComponentFactoryUtils+SimpleComponentFactory`1[Microsoft.ML.Trainers.LinearSvmTrainer];/nm:20 + diff --git a/test/BaselineOutput/Common/WeightedEnsemble/WE-Default-CV-breast-cancer.txt b/test/BaselineOutput/Common/WeightedEnsemble/WE-Default-CV-breast-cancer.txt new file mode 100644 index 0000000000..6c8ed143a6 --- /dev/null +++ b/test/BaselineOutput/Common/WeightedEnsemble/WE-Default-CV-breast-cancer.txt @@ -0,0 +1,700 @@ +Instance Label Score Probability Log-loss Assigned +5 1 7.057208 0.999964833 5.0735774115656486E-05 1 +6 0 -0.569736 0.326803476 0.57090036663055121 0 +8 0 -3.265584 0.009919913 0.014382865566317179 0 +9 0 -2.8438797 0.0180531759 0.026283195253444185 0 +10 0 -3.42607021 0.007889865 0.011427810517824316 0 +11 0 -3.27301645 0.009815384 0.014230559406451833 0 +18 1 4.12974262 0.9976289 0.0034247967532404057 1 +20 1 3.61101723 0.995010138 0.0072168704068302217 1 +21 1 3.70798635 0.9956574 0.0062787129039152982 1 +25 1 0.8509629 0.7895741 0.34085345267634748 1 +28 0 -3.27301645 0.009815384 0.014230559406451833 0 +31 0 -3.13378072 0.0119675556 0.017369677998532677 0 +32 1 3.22181535 0.9912949 0.012613756987139313 1 +35 0 -3.27301645 0.009815384 0.014230559406451833 0 +37 0 -2.10296226 0.050705 0.075071611636301286 0 +40 0 ? ? ? 0 +41 1 0.4294026 0.671624362 0.57427353141509008 1 +44 1 4.518715 0.9986441 0.0019574580164138182 1 +45 0 -3.41172981 0.008053098 0.011665198155882454 0 +46 1 3.47903919 0.9939724 0.0087222723575821337 1 +48 0 -2.66200113 0.02333 0.034056911171000702 0 +50 1 0.440663338 0.675189257 0.56663614622582026 1 +51 1 -0.21027565 0.448866934 1.1556402728846316 0 +52 1 2.10327148 0.957912564 0.062034118299456473 1 +54 1 3.51597667 0.9942828 0.0082718705836708154 1 +56 1 3.94411469 0.99690485 0.0044722823151831939 1 +60 1 0.9666352 0.815907955 0.2935216879238205 1 +63 1 -0.135355234 0.475668 1.0719730821990747 0 +64 0 -3.47882748 0.007317101 0.010595155854650925 0 +66 0 -2.900283 0.01666882 0.024250706613533433 0 +68 1 4.283942 0.9981 0.0027437528959210881 1 +69 0 -3.2245307 0.0105174417 0.015253817861314311 0 +70 0 -2.59506845 0.0256291423 0.037457110328873687 0 +71 1 3.56249046 0.9946511 0.0077375738770138498 1 +72 0 -1.587163 0.100903749 0.15345252671689891 0 +73 1 4.728806 0.9989976 0.0014468413771274271 1 +74 1 0.497558832 0.6928866 0.52930885804029171 1 +76 0 -2.85157967 0.017857736 0.025996079781682065 0 +77 0 -2.388582 0.0341966338 0.05019860308331358 0 +79 0 -3.1797595 0.0112097217 0.016263535993934659 0 +82 0 -2.620762 0.0247215685 0.03611394323422179 0 +88 0 -2.900283 0.01666882 0.024250706613533433 0 +90 0 -3.33281326 0.00901316 0.013062196086147755 0 +91 0 -3.11497951 0.0122918272 0.017843247309438984 0 +92 0 -2.900283 0.01666882 0.024250706613533433 0 +93 0 -3.47882748 0.007317101 0.010595155854650925 0 +95 0 -3.33281326 0.00901316 0.013062196086147755 0 +96 0 -3.31792665 0.00920658652 0.01334381690418124 0 +97 0 -2.78353429 0.0196592771 0.028644841704572911 0 +98 1 4.16591549 0.9977489 0.0032512952389382424 1 +99 1 5.3154254 0.999568939 0.00062202334009018689 1 +100 1 2.43109632 0.9733226 0.03901008486188097 1 +102 0 -2.75324774 0.0205173325 0.029908131443256804 0 +104 1 5.43333054 0.9996362 0.00052498655830247973 1 +105 1 -0.152683258 0.469451129 1.0909531158993548 0 +106 1 5.426038 0.999632359 0.00053049201666395714 1 +108 0 -3.2805295 0.009710831 0.014078234047398743 0 +109 1 3.34006834 0.9926474 0.010646734621373733 1 +111 1 2.133381 0.959625661 0.05945635759702371 1 +112 1 4.07894564 0.9974495 0.0036842691473833646 1 +113 1 5.289747 0.999552667 0.00064550928688674445 1 +115 0 -2.383574 0.03443552 0.050555492617516365 0 +117 1 4.46682835 0.9985391 0.0021091884287959238 1 +120 0 -2.94937181 0.0155493161 0.022609158778899734 0 +121 0 -2.41179633 0.0331100076 0.048576338080936747 0 +122 1 5.8156414 0.999790132 0.00030280723272274124 1 +123 1 2.32057929 0.968865633 0.045631495325782319 1 +125 0 -3.47882748 0.007317101 0.010595155854650925 0 +128 1 2.628265 0.979781151 0.029468557791152337 1 +129 0 -2.71849918 0.0215470158 0.031425565474393721 0 +131 0 -3.13378072 0.0119675556 0.017369677998532677 0 +132 1 5.71647358 0.999757946 0.00034925304341121586 1 +133 0 -3.039434 0.0136845605 0.019878977535115503 0 +137 0 -3.24181318 0.0102616735 0.0148809484151837 0 +138 0 -2.85885239 0.01767505 0.02572775183602494 0 +141 0 -3.43729782 0.00776435854 0.011245314601619131 0 +144 0 -3.27301645 0.009815384 0.014230559406451833 0 +145 0 ? ? ? 0 +147 0 -3.02075386 0.0140522793 0.020416944305599309 0 +150 0 -3.42607021 0.007889865 0.011427810517824316 0 +151 1 2.217757 0.964077353 0.052779188057458237 1 +152 1 5.177735 0.999474466 0.00075838481234789161 1 +154 0 -3.60546017 0.006105266 0.0088350346936084626 0 +156 0 -3.0233922 0.0139997583 0.02034009461654903 0 +161 0 -2.8170805 0.0187498946 0.027307190992271715 0 +164 0 ? ? ? 0 +167 1 2.929184 0.9867951 0.019177502987692791 1 +169 0 -3.60177875 0.006137506 0.0088818337422214635 0 +171 0 -3.33281326 0.00901316 0.013062196086147755 0 +173 1 7.9475913 0.9999902 1.4102646298686894E-05 1 +174 1 3.24989271 0.9916369 0.01211617836068587 1 +176 0 -3.13378072 0.0119675556 0.017369677998532677 0 +177 1 3.31922626 0.9924252 0.010969720954957895 1 +179 1 1.37836146 0.8890948 0.16959079187424589 1 +180 0 -3.42607021 0.007889865 0.011427810517824316 0 +181 0 -3.60546017 0.006105266 0.0088350346936084626 0 +183 1 5.419508 0.9996289 0.00053548135645133934 1 +187 1 5.826078 0.999793231 0.00029833475210104223 1 +188 1 4.99824429 0.9993196 0.00098192506222582999 1 +189 0 -2.68967485 0.022439355 0.032741889160371844 0 +191 1 6.31306839 0.9998974 0.00014799866201853666 1 +192 0 -2.889139 0.0169338081 0.024639535385338137 0 +196 0 2.904959 0.9863329 6.1931483378075782 1 +198 0 -3.60546017 0.006105266 0.0088350346936084626 0 +199 0 -3.12263656 0.0121587282 0.017648849424094316 0 +201 1 5.43551445 0.9996373 0.00052335212939580867 1 +202 0 -3.33281326 0.00901316 0.013062196086147755 0 +204 0 -3.33281326 0.00901316 0.013062196086147755 0 +205 1 7.059002 0.999964952 5.0563785426763426E-05 1 +206 1 3.804298 0.9962174 0.0054674425275320912 1 +207 0 -3.42607021 0.007889865 0.011427810517824316 0 +209 0 -2.88948035 0.0169256274 0.024627529825177852 0 +210 1 7.61664629 0.999984264 2.2701888528575377E-05 1 +211 1 5.476496 0.9996581 0.00049333058043002202 1 +212 0 -3.33281326 0.00901316 0.013062196086147755 0 +216 0 -3.47882748 0.007317101 0.010595155854650925 0 +218 1 4.51687765 0.998640537 0.0019626245103572994 1 +219 0 -2.36019087 0.0355723836 0.052255131689570362 0 +223 1 2.74526834 0.9828614 0.024940108774080157 1 +226 1 4.96402931 0.99928534 0.0010314045874782159 1 +228 0 -3.42607021 0.007889865 0.011427810517824316 0 +233 1 3.16380477 0.990543962 0.013707089821426618 1 +237 1 3.38094282 0.993064642 0.010040464153854149 1 +239 1 2.58121371 0.9783949 0.031511173482079449 1 +240 0 -1.73168266 0.08353538 0.12584890914690539 0 +241 0 -2.89645481 0.0167593881 0.024383588751320583 0 +242 0 -3.13378072 0.0119675556 0.017369677998532677 0 +244 0 -3.33281326 0.00901316 0.013062196086147755 0 +246 1 6.66489267 0.9999382 8.9175760478377495E-05 1 +247 1 1.53797054 0.9098069 0.13636770739767573 1 +248 0 -2.3839004 0.0344199 0.050532154021333195 0 +249 0 ? ? ? 0 +250 0 -3.19199657 0.01101615 0.015981132773143735 0 +252 0 2.36073565 0.970562756 5.0862135843602427 1 +254 1 4.49442673 0.998595953 0.0020270350213282213 1 +257 0 -3.12263656 0.0121587282 0.017648849424094316 0 +258 0 -3.01703215 0.0141266966 0.020525840105121977 0 +259 0 2.35941148 0.9705083 5.0835460984539447 1 +260 1 4.74125576 0.9990154 0.0014211904782310285 1 +262 1 5.535843 0.999686062 0.00045298741967439286 1 +267 1 1.78717756 0.9352332 0.096601987037004095 1 +268 1 3.39552546 0.993207753 0.0098325720507635179 1 +269 0 -3.33281326 0.00901316 0.013062196086147755 0 +271 0 -2.78353429 0.0196592771 0.028644841704572911 0 +272 1 1.78717756 0.9352332 0.096601987037004095 1 +275 0 ? ? ? 0 +276 0 -3.12263656 0.0121587282 0.017648849424094316 0 +277 0 -3.47882748 0.007317101 0.010595155854650925 0 +278 0 -3.33281326 0.00901316 0.013062196086147755 0 +279 1 2.500967 0.975813448 0.035322728606119444 1 +280 0 -3.01703215 0.0141266966 0.020525840105121977 0 +283 1 2.83906627 0.9849935 0.021813865188733895 1 +284 1 4.045544 0.9973242 0.0038654964837656733 1 +285 1 7.694996 0.999986 2.0208103005084558E-05 1 +288 1 0.650547743 0.7376656 0.43896114819175791 1 +290 0 -3.60546017 0.006105266 0.0088350346936084626 0 +291 0 -3.33281326 0.00901316 0.013062196086147755 0 +293 1 2.86597276 0.9855554 0.020991109422172988 1 +296 0 0.8463125 0.7884597 2.2409956855367694 1 +297 0 ? ? ? 0 +299 1 3.32911181 0.9925314 0.010815323080105525 1 +300 1 3.616962 0.995052457 0.0071555116925496928 1 +301 0 -3.33281326 0.00901316 0.013062196086147755 0 +303 0 -3.33281326 0.00901316 0.013062196086147755 0 +304 1 3.23745537 0.9914871 0.012334113503203046 1 +308 1 3.36895037 0.9929447 0.010214697519032298 1 +309 0 -1.57274163 0.102802709 0.15650283012399038 0 +311 0 -3.60546017 0.006105266 0.0088350346936084626 0 +312 1 1.59649658 0.9164854 0.12581615132950291 1 +314 0 -3.59035158 0.00623866124 0.0090286784423901038 0 +316 1 2.323201 0.968979239 0.045462338889384266 1 +317 1 5.18574762 0.9994805 0.00074969514792004902 1 +319 0 1.08797622 0.8407114 2.6502852096034966 1 +321 0 ? ? ? 0 +323 1 3.347725 0.992727458 0.010530397548729942 1 +327 0 -3.47882748 0.007317101 0.010595155854650925 0 +328 1 2.39244843 0.971839368 0.04121021896301344 1 +329 1 3.653625 0.9953056 0.0067885373582329865 1 +331 0 -2.764847 0.020184461 0.029417923269692232 0 +332 0 -2.330146 0.03708627 0.054521547034532859 0 +333 1 3.06502247 0.9891148 0.0157900895649082 1 +336 1 3.44965982 0.993713737 0.0090977867244981265 1 +338 0 -3.59035158 0.00623866124 0.0090286784423901038 0 +343 0 -3.60546017 0.006105266 0.0088350346936084626 0 +344 1 5.31238174 0.999567032 0.00062477625180167376 1 +346 0 -2.24155569 0.0419189371 0.061780367869687394 0 +347 0 -3.47015452 0.00740834232 0.010727765459185096 0 +348 1 0.009417534 0.527718961 0.92215827358141711 1 +349 1 1.92163181 0.9460146 0.08006567155579622 1 +350 0 -2.79726744 0.0192818828 0.028089565318717206 0 +352 0 0.486902952 0.6896129 1.6878596124189917 1 +353 1 5.684911 0.9997467 0.00036550943041529892 1 +354 0 -3.47882748 0.007317101 0.010595155854650925 0 +355 0 -2.8206 0.0186569113 0.027170487707885705 0 +358 1 3.42987156 0.993533254 0.0093598394390381444 1 +360 1 8.039464 0.9999915 1.2296811942483466E-05 1 +361 1 4.001849 0.997151 0.0041160794808710478 1 +366 1 7.635662 0.999984741 2.2013947263955502E-05 1 +368 0 -3.18594837 0.0111114066 0.016120096467324693 0 +370 0 -2.42889071 0.0323312432 0.047414812514884067 0 +371 0 -3.18594837 0.0111114066 0.016120096467324693 0 +373 0 -2.77587748 0.0198728237 0.028959136812696604 0 +376 0 -3.47882748 0.007317101 0.010595155854650925 0 +377 0 -3.59035158 0.00623866124 0.0090286784423901038 0 +378 0 -2.510999 0.0288310554 0.042205806088650363 0 +379 0 -1.69792247 0.0873318762 0.13183775070286191 0 +381 1 4.5339613 0.9986735 0.0019150073584088983 1 +383 0 -3.43729782 0.00776435854 0.011245314601619131 0 +384 0 -3.43729782 0.00776435854 0.011245314601619131 0 +387 0 -1.86985755 0.06951585 0.10394651430355639 0 +388 0 -3.16473365 0.0114520118 0.016617092807856053 0 +389 0 -2.38486671 0.0343737043 0.050463131205452497 0 +391 1 5.278063 0.9995451 0.00065643511405119592 1 +392 0 -3.12263656 0.0121587282 0.017648849424094316 0 +395 0 -3.12263656 0.0121587282 0.017648849424094316 0 +396 0 -3.01703215 0.0141266966 0.020525840105121977 0 +398 0 -2.87731075 0.0172195863 0.025058989055801298 0 +399 0 -2.983367 0.0148177361 0.021537439472888546 0 +404 0 -3.035563 0.0137599735 0.019989289097149296 0 +406 0 -2.67387533 0.0229436923 0.033486387837075506 0 +409 0 -2.97560143 0.0149818053 0.021777721492605467 0 +413 0 -2.636499 0.0241812486 0.035314888460076313 0 +414 1 3.51291919 0.9942577 0.0083082815579947292 1 +415 0 -0.4454751 0.367301762 0.66041051702463194 0 +416 1 5.09488773 0.999407947 0.00085440478690671634 1 +418 0 -1.76747322 0.07967491 0.11978453026995894 0 +419 0 -2.78455448 0.0196309946 0.028603221179789369 0 +422 0 -1.93198 0.06394966 0.09534197524455261 0 +423 0 -2.59506845 0.0256291423 0.037457110328873687 0 +428 0 -3.47882748 0.007317101 0.010595155854650925 0 +429 0 -3.27301645 0.009815384 0.014230559406451833 0 +430 0 -2.94501162 0.0156456828 0.022750389443224345 0 +434 0 3.337568 0.9926211 7.0823831717344676 1 +436 1 2.481813 0.9751541 0.036297871506884037 1 +439 0 -3.00913024 0.0142859891 0.020758962394393805 0 +440 1 4.70982456 0.998969853 0.0014869540997041347 1 +441 0 -1.24401617 0.155345351 0.24356650177804087 0 +442 0 -2.68195 0.02268457 0.033103826171634806 0 +449 1 5.942174 0.99982506 0.0002524066187454977 1 +450 0 -2.88531041 0.0170257948 0.024774536477376766 0 +451 0 -3.00913024 0.0142859891 0.020758962394393805 0 +452 0 -3.001718 0.014437018 0.020980025779911059 0 +453 1 4.84441471 0.9991512 0.0012251224992695688 1 +454 0 -2.97106838 0.015078404 0.021919210465799685 0 +455 1 -0.4145186 0.377717137 1.4046218533654737 0 +456 1 5.12116146 0.9994299 0.00082274158011057654 1 +457 1 4.60623741 0.9988044 0.0017259319729422537 1 +464 0 -3.12547159 0.0121098105 0.017577409199649934 0 +465 1 5.04898071 0.999367535 0.00091274262341741255 1 +466 1 4.630231 0.998845 0.0016673029728197354 1 +467 1 4.13757372 0.997655451 0.0033864401758712708 1 +474 0 -3.00913024 0.0142859891 0.020758962394393805 0 +480 0 -3.10068965 0.01254408 0.018211746670243046 0 +482 1 7.239645 0.999973 3.8954596327452357E-05 1 +483 1 5.804902 0.999786854 0.00030753775616068136 1 +484 0 -2.866447 0.017486237 0.025450478164537264 0 +487 1 6.835632 0.999951661 6.9740650547897242E-05 1 +489 1 -0.979645252 0.212031052 2.2376525348702296 0 +492 0 -3.11183357 0.0123469271 0.017923731081694132 0 +493 1 6.144757 0.9998693 0.00018859130260575588 1 +495 0 -3.22858262 0.0104569159 0.015165572253688843 0 +497 0 -2.979243 0.0149046443 0.021664712979310812 0 +501 0 -3.00588775 0.0143518643 0.020855380885325497 0 +502 0 -2.89262629 0.0168504436 0.024517199250348773 0 +504 0 -3.60546017 0.006105266 0.0088350346936084626 0 +507 0 -2.66730165 0.023156777 0.033801057345489688 0 +510 0 -3.60546017 0.006105266 0.0088350346936084626 0 +513 0 -3.22858262 0.0104569159 0.015165572253688843 0 +514 1 5.730318 0.9997627 0.00034237208821625534 1 +517 0 -3.59035158 0.00623866124 0.0090286784423901038 0 +519 1 3.836201 0.9963866 0.0052224933253354131 1 +520 0 -3.47504616 0.00735674333 0.0106527701878314 0 +521 0 -3.06941223 0.0131141776 0.01904491295884091 0 +522 1 1.80012441 0.9363529 0.094875715137687208 1 +523 1 4.25482464 0.9980188 0.0028611008075221564 1 +527 0 -2.900283 0.01666882 0.024250706613533433 0 +528 0 -2.47066259 0.0305020418 0.04469023492605069 0 +529 0 -3.11183357 0.0123469271 0.017923731081694132 0 +531 0 -2.67387533 0.0229436923 0.033486387837075506 0 +532 0 -3.42607021 0.007889865 0.011427810517824316 0 +533 0 -3.12263656 0.0121587282 0.017648849424094316 0 +534 0 -3.27301645 0.009815384 0.014230559406451833 0 +535 0 -2.7646904 0.02018892 0.029424489036836057 0 +538 0 -3.00588775 0.0143518643 0.020855380885325497 0 +539 0 -2.77239013 0.019970838 0.029103415787124165 0 +540 0 -2.60792017 0.025171198 0.03677921768778955 0 +541 0 -3.24181318 0.0102616735 0.0148809484151837 0 +544 0 -2.783347 0.0196644776 0.028652494924385759 0 +546 1 6.903044 0.999956131 6.3291003784773738E-05 1 +547 0 -3.53055477 0.006795632 0.009837488821763803 0 +548 0 -3.33418727 0.008995512 0.013036503245163452 0 +549 1 3.03490639 0.988638043 0.016485671543673565 1 +557 0 -2.79726744 0.0192818828 0.028089565318717206 0 +558 0 -3.27301645 0.009815384 0.014230559406451833 0 +559 0 -2.889139 0.0169338081 0.024639535385338137 0 +560 0 -2.78353429 0.0196592771 0.028644841704572911 0 +561 0 -2.78353429 0.0196592771 0.028644841704572911 0 +563 0 -3.12263656 0.0121587282 0.017648849424094316 0 +565 1 6.782516 0.9999478 7.533036773246141E-05 1 +566 0 -2.86999655 0.01739867 0.025321903500867275 0 +569 1 5.348246 0.9995888 0.00059337616434588982 1 +577 0 -3.47882748 0.007317101 0.010595155854650925 0 +578 0 -3.47882748 0.007317101 0.010595155854650925 0 +581 1 5.01246929 0.999333441 0.00096196162989513302 1 +582 1 5.00962543 0.9993307 0.0009659198746940407 1 +584 0 -2.24000168 0.04200887 0.061915795738465659 0 +586 1 7.861235 0.999989 1.5908482915272255E-05 1 +590 1 2.22713 0.964541733 0.052084433521439384 1 +593 0 -2.866447 0.017486237 0.025450478164537264 0 +594 1 3.25746822 0.9917268 0.011985329025336479 1 +600 0 -3.12263656 0.0121587282 0.017648849424094316 0 +602 0 -3.00588775 0.0143518643 0.020855380885325497 0 +604 1 2.95590162 0.987287 0.018458586152655318 1 +606 0 -2.998231 0.0145086134 0.021084832997094235 0 +607 0 -3.60546017 0.006105266 0.0088350346936084626 0 +609 0 -3.00913024 0.0142859891 0.020758962394393805 0 +612 1 8.09518051 0.999992132 1.135089961038231E-05 1 +613 0 -2.86519337 0.0175172668 0.025496042160873894 0 +614 0 -3.3662734 0.008592976 0.012450615095660842 0 +617 0 ? ? ? 0 +618 0 -3.00588775 0.0143518643 0.020855380885325497 0 +619 0 -2.889139 0.0169338081 0.024639535385338137 0 +621 0 -0.8592272 0.24243255 0.40055374949460493 0 +622 0 -2.25350475 0.0412335657 0.060748692634222383 0 +624 0 -2.8818233 0.017110005 0.024898135675659537 0 +627 0 -2.26393318 0.0406441875 0.059862103676784961 0 +629 0 -3.12547159 0.0121098105 0.017577409199649934 0 +633 1 2.22301912 0.9643388 0.052388029791135139 1 +634 0 -3.24181318 0.0102616735 0.0148809484151837 0 +638 0 -3.12547159 0.0121098105 0.017577409199649934 0 +639 0 -2.79726744 0.0192818828 0.028089565318717206 0 +641 0 -3.12263656 0.0121587282 0.017648849424094316 0 +642 0 -3.12263656 0.0121587282 0.017648849424094316 0 +644 0 -3.43729782 0.00776435854 0.011245314601619131 0 +645 0 -3.12263656 0.0121587282 0.017648849424094316 0 +649 0 -3.12263656 0.0121587282 0.017648849424094316 0 +652 0 -2.80593634 0.0190473236 0.027744555986751927 0 +653 0 -3.00588775 0.0143518643 0.020855380885325497 0 +654 0 -3.01703215 0.0141266966 0.020525840105121977 0 +656 0 -2.889139 0.0169338081 0.024639535385338137 0 +657 0 -0.182281256 0.458855 0.88591288668704316 0 +660 0 -3.47882748 0.007317101 0.010595155854650925 0 +661 0 -2.900283 0.01666882 0.024250706613533433 0 +665 0 -3.60546017 0.006105266 0.0088350346936084626 0 +668 1 1.15736818 0.8536372 0.22830501603144154 1 +670 1 4.26155233 0.9980379 0.002833529221533669 1 +678 0 -3.60546017 0.006105266 0.0088350346936084626 0 +679 0 -3.43729782 0.00776435854 0.011245314601619131 0 +680 1 8.287128 0.99999404 8.5991581706542968E-06 1 +681 1 5.99392557 0.999837637 0.00023425938852449468 1 +682 0 -2.70033169 0.022105312 0.032248988855708219 0 +683 0 -3.60546017 0.006105266 0.0088350346936084626 0 +685 0 -3.60546017 0.006105266 0.0088350346936084626 0 +688 0 -3.12547159 0.0121098105 0.017577409199649934 0 +689 0 -3.2345 0.0103691444 0.015037612200557401 0 +691 1 3.3198967 0.9924325 0.010959149978694181 1 +692 0 -3.24181318 0.0102616735 0.0148809484151837 0 +693 0 -3.103949 0.0124860965 0.018127034395988686 0 +694 0 -2.87412453 0.0172973759 0.025173186674684375 0 +696 1 4.16700649 0.9977525 0.0032461241280301052 1 +697 1 2.616522 0.97944355 0.02996574922798281 1 +698 1 3.27090669 0.991884 0.011756696347368025 1 +0 0 -2.18580484 0.0195222013 0.028443131501617723 0 +1 0 2.46434641 0.8795063 3.0529702573901116 1 +2 0 -2.35020447 0.0159028824 0.023127397101615162 0 +3 0 2.42789173 0.874514341 2.9944056022897438 1 +4 0 -2.082891 0.0221868474 0.032369283673121328 0 +7 0 -2.638155 0.0110870106 0.016084505454470342 0 +12 1 0.133486748 0.274540573 1.8649087192175371 1 +13 0 -2.49497175 0.0132681243 0.019269979978932417 0 +14 1 5.299982 0.996271968 0.0053884638761153361 1 +15 1 0.7744734 0.460618824 1.1183547250072077 1 +16 0 -2.49187756 0.0133196581 0.019345329226167315 0 +17 0 -2.4005723 0.0149324164 0.021705386349413373 0 +19 0 -1.959885 0.0258403122 0.037769811437677638 0 +22 0 -2.72705364 0.009915355 0.014376224049015385 0 +23 1 ? ? ? 0 +24 0 -3.05620337 0.006550616 0.0094816306677697523 0 +26 0 -2.56704378 0.0121219316 0.017595110812301896 0 +27 0 -2.280314 0.0173529927 0.025254839289623091 0 +29 0 -2.91682529 0.0078088576 0.011310016984425502 0 +30 0 -2.743445 0.009713114 0.014081560900358588 0 +33 0 -2.697853 0.010286021 0.014916439154444164 0 +34 0 -2.57251978 0.0120389527 0.017473933576498117 0 +36 1 5.37075663 0.99659127 0.0049261586606575156 1 +38 1 3.817384 0.976006866 0.035036798748799364 1 +39 1 1.74231339 0.744788 0.42509828380154596 1 +42 1 5.30029 0.9962734 0.0053863923631196743 1 +43 1 0.431302071 0.35581547 1.4907988571250606 1 +47 0 -3.16991329 0.00567496149 0.0082105581957195584 0 +49 1 4.063138 0.982324362 0.025728616279702864 1 +53 1 3.64990449 0.9704895 0.043215487885653277 1 +55 1 3.096236 0.9421367 0.085991683453461656 1 +57 1 0.313182354 0.3222287 1.6338430986952079 1 +58 1 1.48621321 0.678271 0.5600662936207903 1 +59 1 0.848643541 0.4840857 1.0466655911989673 1 +61 0 -2.90747046 0.00790142547 0.011444621537965636 0 +62 1 4.68961143 0.9919433 0.01167043745866826 1 +65 1 1.42205667 0.6602432 0.59893052828159399 1 +67 1 3.03390431 0.937668264 0.092850490546770872 1 +75 0 -2.26705265 0.0176424533 0.025679879972587048 0 +78 0 -2.0084548 0.0243325215 0.035538554511704831 0 +80 0 -2.03421974 0.0235678554 0.034408304733397191 0 +81 0 -2.310827 0.0167045724 0.024303160568656917 0 +83 0 -1.8550334 0.0294116829 0.043068600233709099 0 +84 1 5.27409458 0.9961479 0.0055681789505025291 1 +85 1 3.909911 0.978609443 0.031194891325044546 1 +86 1 1.96546316 0.794837654 0.33126787626487048 1 +87 1 4.488919 0.989629269 0.015039925710551912 1 +89 0 -2.795878 0.009093219 0.013178752422657699 0 +94 0 -2.8278563 0.008734566 0.012656671370376122 0 +101 1 0.2679627 0.3098195 1.6905001927406846 1 +103 1 0.231719017 0.3000664 1.7366462577282673 1 +107 1 4.46852541 0.9893601 0.015432385453245953 1 +110 0 -1.49092174 0.04590446 0.067794356627703653 0 +114 0 -1.42551076 0.0496821962 0.073518036969628622 0 +116 0 -0.30933404 0.1774163 0.28176560155499253 0 +118 0 -2.611174 0.0114689711 0.016641843684379737 0 +119 0 -2.10020518 0.0217148978 0.031673123011132898 0 +124 1 4.19224739 0.9849567 0.021867818470544111 1 +126 1 4.365718 0.9878945 0.017571150102842548 1 +127 0 -2.60425234 0.011569039 0.016787893374491106 0 +130 0 -1.920393 0.0271329954 0.039685499497402153 0 +134 0 -2.70864916 0.0101474058 0.014714395385468908 0 +135 0 -1.47098243 0.0470261 0.069491394938931556 0 +136 0 -2.49187756 0.0133196581 0.019345329226167315 0 +139 0 ? ? ? 0 +140 0 -2.86142683 0.008373138 0.012130741394297474 0 +142 1 2.57813931 0.893997967 0.16165654384957087 1 +143 0 -2.18314314 0.0195869952 0.028538473827737675 0 +146 1 1.11178446 0.567198157 0.81807524926928143 1 +148 0 -0.7938955 0.104406551 0.15908411878129727 0 +149 1 6.315874 0.998970866 0.0014854907404379277 1 +153 0 -2.10214 0.0216627736 0.031596256450828433 0 +155 1 2.581787 0.894436061 0.16094973974671178 1 +157 0 -2.8278563 0.008734566 0.012656671370376122 0 +158 0 ? ? ? 0 +159 1 7.29133129 0.9997015 0.00043070884426719654 1 +160 1 5.65635347 0.9976256 0.0034296237206113307 1 +162 0 -2.60425234 0.011569039 0.016787893374491106 0 +163 0 -2.21317625 0.018867977 0.027480813524908082 0 +165 0 -1.92896676 0.0268471129 0.039261617630834335 0 +166 1 5.371229 0.9965933 0.0049232249583664065 1 +168 0 -2.60425234 0.011569039 0.016787893374491106 0 +170 0 -2.86142683 0.008373138 0.012130741394297474 0 +172 0 -3.16991329 0.00567496149 0.0082105581957195584 0 +175 1 5.180046 0.995661438 0.0062728400019319355 1 +178 0 -2.4005723 0.0149324164 0.021705386349413373 0 +182 0 -1.959885 0.0258403122 0.037769811437677638 0 +184 1 3.955361 0.9797846 0.029463467380837841 1 +185 0 -2.87166452 0.008265896 0.011974726218235726 0 +186 1 3.985546 0.980529964 0.028366376544727477 1 +190 1 7.46572733 0.9997608 0.00034512446635598279 1 +193 0 -3.05620337 0.006550616 0.0094816306677697523 0 +194 0 -2.60425234 0.011569039 0.016787893374491106 0 +195 0 -2.4005723 0.0149324164 0.021705386349413373 0 +197 0 -1.83853054 0.0300157685 0.043966800471659635 0 +200 1 6.027786 0.998517036 0.0021410521205442053 1 +203 0 -2.18580484 0.0195222013 0.028443131501617723 0 +208 0 -3.09784317 0.00621537957 0.0089948796304217538 0 +213 1 8.651913 0.999946952 7.6534309653049576E-05 1 +214 1 8.021141 0.9998818 0.00017053087656366316 1 +215 1 4.766494 0.9926871 0.010589041348733017 1 +217 0 -3.05620337 0.006550616 0.0094816306677697523 0 +220 0 -2.91116858 0.00786470249 0.011391220512492049 0 +221 1 5.885932 0.9982249 0.0025631832715805789 1 +222 1 -1.09107912 0.0740194544 3.7559516878709833 0 +224 1 5.80837059 0.99804157 0.0028281872876838774 1 +225 0 -3.16991329 0.00567496149 0.0082105581957195584 0 +227 1 4.6679697 0.9917207 0.011994260047261323 1 +229 1 7.47627163 0.999763966 0.00034056584291574553 1 +230 1 3.55352354 0.966775537 0.048747127537939673 1 +231 1 5.227359 0.9959133 0.0059079033199315846 1 +232 0 0.676802635 0.429997027 0.81095865063432271 1 +234 0 -1.20080757 0.06501902 0.096991076219862818 0 +235 0 ? ? ? 0 +236 1 6.9204216 0.9995221 0.00068964325415644641 1 +238 1 7.6631403 0.9998138 0.0002686619145078823 1 +243 0 -1.72830355 0.0343697965 0.050457292720221654 0 +245 0 -1.72366273 0.0345658921 0.050750298070184989 0 +251 1 4.62756729 0.991288662 0.01262286539422033 1 +253 1 5.30029 0.9962734 0.0053863923631196743 1 +255 1 3.4353652 0.961604238 0.05648484121740667 1 +256 0 -2.86142683 0.008373138 0.012130741394297474 0 +261 1 6.75521374 0.9994106 0.00085053289008703693 1 +263 1 5.02655 0.994732738 0.007619137087775783 1 +264 1 3.01557517 0.936294138 0.094966268717484412 1 +265 0 -1.40027964 0.0512167327 0.075849528101537778 0 +266 1 5.25772238 0.9960673 0.00568489352489388 1 +270 1 4.08990955 0.982905 0.024876154420347706 1 +273 1 0.335526 0.3284555 1.606230137088891 1 +274 0 -2.41461515 0.0146724023 0.021324629190085015 0 +281 0 -2.697853 0.010286021 0.014916439154444164 0 +282 1 3.025592 0.937048554 0.093804289898579224 1 +286 1 8.930485 0.999962747 5.3745579490272704E-05 1 +287 0 -2.70864916 0.0101474058 0.014714395385468908 0 +289 1 5.335 0.9964335 0.0051545743258560924 1 +292 1 ? ? ? 0 +294 0 ? ? ? 0 +295 1 4.55094337 0.9904071 0.013906424460647474 1 +298 0 -1.56763256 0.041822426 0.061635047111115164 0 +302 1 8.872597 0.9999599 5.7873322786187624E-05 1 +305 1 5.7573247 0.9979107 0.003017407196179228 1 +306 0 -3.05620337 0.006550616 0.0094816306677697523 0 +307 0 -3.05620337 0.006550616 0.0094816306677697523 0 +310 0 -2.922761 0.007750682 0.011225429363134413 0 +313 0 -3.299302 0.00481935544 0.0069696683697878879 0 +315 0 ? ? ? 0 +318 0 -2.86914682 0.008292142 0.012012908069216954 0 +320 1 3.933642 0.9792311 0.030278688471899282 1 +322 0 -2.60425234 0.011569039 0.016787893374491106 0 +324 0 -3.05620337 0.006550616 0.0094816306677697523 0 +325 0 -2.07169771 0.022497274 0.032827369134600271 0 +326 1 2.32474756 0.859424233 0.218557637107754 1 +330 1 2.92094016 0.928740144 0.10665309970829556 1 +334 1 4.11280775 0.9833867 0.024169258370237343 1 +335 0 -3.299302 0.00481935544 0.0069696683697878879 0 +337 0 -3.05620337 0.006550616 0.0094816306677697523 0 +339 1 3.69971848 0.972248 0.040603706580857601 1 +340 1 3.82004952 0.976085961 0.034919887823900035 1 +341 0 -3.05620337 0.006550616 0.0094816306677697523 0 +342 0 -3.111528 0.006108972 0.0088404137863029985 0 +345 0 -3.299302 0.00481935544 0.0069696683697878879 0 +351 0 -2.8278563 0.008734566 0.012656671370376122 0 +356 1 -0.284852982 0.181998163 2.4580042025847506 0 +357 1 6.906702 0.9995137 0.00070177387936437943 1 +359 1 3.26117086 0.952550769 0.070132107772800226 1 +362 0 -1.70835483 0.0352204032 0.051728697539150721 0 +363 0 -0.7258245 0.112768881 0.1726181270358976 0 +364 0 -2.8278563 0.008734566 0.012656671370376122 0 +365 0 -2.977202 0.00723677361 0.010478418271730705 0 +367 1 6.78657341 0.9994336 0.0008174070864974929 1 +369 0 -2.85002947 0.008494151 0.012306811539543959 0 +372 0 -2.3130796 0.01665766 0.024234331881756287 0 +374 0 -2.57251978 0.0120389527 0.017473933576498117 0 +375 0 -3.299302 0.00481935544 0.0069696683697878879 0 +380 0 -3.299302 0.00481935544 0.0069696683697878879 0 +382 0 -1.90811241 0.027547637 0.040300515583319305 0 +385 0 -1.81237435 0.0309979189 0.045428330833790706 0 +386 1 3.58146334 0.967896342 0.047075545827350976 1 +390 0 -3.03970838 0.00668832846 0.0096816314790603344 0 +393 0 -3.34684086 0.004538348 0.0065623544866757642 0 +394 0 -2.53023 0.0126946224 0.018431709676809287 0 +397 0 -2.62625074 0.0112539595 0.016328082658637364 0 +400 1 5.613317 0.997492552 0.0036220259996406776 1 +401 0 -2.86142683 0.008373138 0.012130741394297474 0 +402 0 -1.60838628 0.03979731 0.058589116865399464 0 +403 0 -2.0118618 0.024230035 0.035387018309250291 0 +405 0 -3.16991329 0.00567496149 0.0082105581957195584 0 +407 0 -3.16991329 0.00567496149 0.0082105581957195584 0 +408 0 -1.79749548 0.03157042 0.046280948279897059 0 +410 0 -3.16991329 0.00567496149 0.0082105581957195584 0 +411 0 ? ? ? 0 +412 1 5.180189 0.9956622 0.0062717172439807865 1 +417 0 -3.16991329 0.00567496149 0.0082105581957195584 0 +420 0 -1.37035489 0.0530948527 0.078708178587860597 0 +421 1 6.74768066 0.999404967 0.00085870690667158795 1 +424 0 -2.86142683 0.008373138 0.012130741394297474 0 +425 1 8.440741 0.9999306 0.00010009737521057061 1 +426 0 -1.14377093 0.069562614 0.10401902777852406 0 +427 1 3.0290432 0.9373065 0.093407171483572096 1 +431 0 -1.6003871 0.0401872434 0.059175107212124864 0 +432 0 -2.06699181 0.0226290468 0.033021865599754556 0 +433 0 -2.19395256 0.0193251669 0.028153240295868898 0 +435 1 4.9303565 0.9940526 0.00860591734976527 1 +437 0 -2.62625074 0.0112539595 0.016328082658637364 0 +438 0 -1.95250154 0.02607735 0.038120899336029052 0 +443 0 -3.01080251 0.006936626 0.010042306261787228 0 +444 0 -1.73322284 0.0341631025 0.050148515604885764 0 +445 0 -3.111528 0.006108972 0.0088404137863029985 0 +446 0 -3.299302 0.00481935544 0.0069696683697878879 0 +447 0 -2.391075 0.0151108364 0.021966717597868491 0 +448 0 -3.34684086 0.004538348 0.0065623544866757642 0 +458 0 -2.212277 0.0188891273 0.027511914143950728 0 +459 0 -2.03347921 0.0235895012 0.034440287123152627 0 +460 0 -2.11832142 0.0212315954 0.030960564068005136 0 +461 0 -1.72729337 0.0344123878 0.050520927361522236 0 +462 0 -1.92701185 0.02691204 0.039357873691423587 0 +463 0 -2.435658 0.0142911058 0.020766451216628158 0 +468 0 -2.62625074 0.0112539595 0.016328082658637364 0 +469 0 -2.96984339 0.007304208 0.010576417869130071 0 +470 0 -2.743445 0.009713114 0.014081560900358588 0 +471 0 -1.92701185 0.02691204 0.039357873691423587 0 +472 0 -2.19599915 0.0192759819 0.028080884826884558 0 +473 0 -2.62625074 0.0112539595 0.016328082658637364 0 +475 0 -2.86142683 0.008373138 0.012130741394297474 0 +476 0 -2.447453 0.0140816672 0.020459946977315596 0 +477 0 -2.62625074 0.0112539595 0.016328082658637364 0 +478 0 -2.229387 0.0184907 0.026926157377814387 0 +479 1 5.24739265 0.996015549 0.0057598306401665739 1 +481 0 -1.45910347 0.04770666 0.070522051664368798 0 +485 0 -2.51326 0.0129675344 0.018830556058982564 0 +486 0 -2.743445 0.009713114 0.014081560900358588 0 +488 1 1.20174 0.5949928 0.74905584451233209 1 +490 0 -3.299302 0.00481935544 0.0069696683697878879 0 +491 1 4.41161442 0.988571942 0.01658213512685398 1 +494 0 0.1990602 0.291430175 0.49701806703908064 1 +496 0 -3.34684086 0.004538348 0.0065623544866757642 0 +498 0 -2.49187756 0.0133196581 0.019345329226167315 0 +499 0 -2.49187756 0.0133196581 0.019345329226167315 0 +500 0 -1.959885 0.0258403122 0.037769811437677638 0 +503 0 -2.4005723 0.0149324164 0.021705386349413373 0 +505 0 -2.50281715 0.0131383417 0.019080238066149724 0 +506 1 6.2301693 0.99885267 0.0016561973072822019 1 +508 0 -2.391075 0.0151108364 0.021966717597868491 0 +509 0 -3.111528 0.006108972 0.0088404137863029985 0 +511 0 -2.280314 0.0173529927 0.025254839289623091 0 +512 0 -2.391075 0.0151108364 0.021966717597868491 0 +515 1 5.60101461 0.997453153 0.0036790102734250547 1 +516 0 -3.34684086 0.004538348 0.0065623544866757642 0 +518 0 -2.53218627 0.0126635293 0.018386275703658014 0 +524 0 -2.72705364 0.009915355 0.014376224049015385 0 +525 0 -2.688909 0.0104022743 0.015085910206797618 0 +526 0 -2.62625074 0.0112539595 0.016328082658637364 0 +530 1 4.06117868 0.9822811 0.025792170722406128 1 +536 0 -2.18580484 0.0195222013 0.028443131501617723 0 +537 0 -1.987645 0.0249677356 0.036478135643832503 0 +542 0 -2.06806087 0.0225990452 0.032977580948398179 0 +543 0 -2.49187756 0.0133196581 0.019345329226167315 0 +545 0 -2.280314 0.0173529927 0.025254839289623091 0 +550 0 -2.72705364 0.009915355 0.014376224049015385 0 +551 0 -3.05620337 0.006550616 0.0094816306677697523 0 +552 0 -1.77373934 0.0325057581 0.047675019900819467 0 +553 0 -0.4828031 0.14751783 0.23025843519524158 0 +554 0 -2.86142683 0.008373138 0.012130741394297474 0 +555 0 -0.7290869 0.112355106 0.17194545831148736 0 +556 0 -1.85351372 0.0294668134 0.043150549355838286 0 +562 0 -3.05620337 0.006550616 0.0094816306677697523 0 +564 0 -2.168657 0.019943336 0.029062930891538222 0 +567 0 -1.976803 0.0253050625 0.036977343788428527 0 +568 1 3.12771726 0.944277644 0.082716980050333053 1 +570 1 4.79198074 0.9929183 0.010253062861046026 1 +571 1 6.33737 0.9989985 0.001445550213597871 1 +572 0 -2.72705364 0.009915355 0.014376224049015385 0 +573 0 -3.16991329 0.00567496149 0.0082105581957195584 0 +574 1 3.53734255 0.9661093 0.049741714558953425 1 +575 0 -1.987645 0.0249677356 0.036478135643832503 0 +576 0 -2.280314 0.0173529927 0.025254839289623091 0 +579 0 -3.05620337 0.006550616 0.0094816306677697523 0 +580 0 -2.08641338 0.022090029 0.032226441907387202 0 +583 0 -2.86142683 0.008373138 0.012130741394297474 0 +585 0 -3.299302 0.00481935544 0.0069696683697878879 0 +587 0 -2.06699181 0.0226290468 0.033021865599754556 0 +588 1 3.09949446 0.942361832 0.08564698778891873 1 +589 0 -2.391075 0.0151108364 0.021966717597868491 0 +591 1 3.93456817 0.979255 0.030243475027392927 1 +592 1 3.10678387 0.9428625 0.084880684135415621 1 +595 0 -2.280314 0.0173529927 0.025254839289623091 0 +596 0 -2.3130796 0.01665766 0.024234331881756287 0 +597 0 -1.79761052 0.0315659568 0.046274299798906078 0 +598 0 -2.72705364 0.009915355 0.014376224049015385 0 +599 0 -1.74648464 0.03361184 0.049325317974486131 0 +601 0 -3.21649551 0.005350793 0.0077402890679609355 0 +603 1 2.43052554 0.87488085 0.19284154464442543 1 +605 1 5.57525253 0.9973687 0.0038011762787992279 1 +608 1 6.1609683 0.998747468 0.0018081543347426264 1 +610 1 4.25627565 0.9861149 0.020172310679415974 1 +611 1 4.19180965 0.9849484 0.021879953871753063 1 +615 0 -2.12825346 0.020971112 0.030576665061562713 0 +616 0 -2.72705364 0.009915355 0.014376224049015385 0 +620 0 -2.72705364 0.009915355 0.014376224049015385 0 +623 0 -3.299302 0.00481935544 0.0069696683697878879 0 +625 0 -1.6549139 0.0376001969 0.055291747535232312 0 +626 1 3.28669167 0.953994036 0.067947848219348669 1 +628 0 -3.111528 0.006108972 0.0088404137863029985 0 +630 0 -1.72939181 0.034323968 0.050388824468901965 0 +631 0 -2.280314 0.0173529927 0.025254839289623091 0 +632 0 -3.299302 0.00481935544 0.0069696683697878879 0 +635 0 -2.32377982 0.016436575 0.023910007293990707 0 +636 1 5.42057037 0.9967995 0.0046247087970721114 1 +637 0 -1.24742484 0.06151211 0.091589964164117119 0 +640 0 -2.32399678 0.0164321214 0.023903474771901047 0 +643 0 -3.299302 0.00481935544 0.0069696683697878879 0 +646 0 -2.86926556 0.008290903 0.012011104766243683 0 +647 0 -3.05244 0.00658178562 0.0095268959776332787 0 +648 1 6.285733 0.998930752 0.0015434236851713337 1 +650 0 -1.821827 0.0306394361 0.044894703756425194 0 +651 0 -2.72723722 0.009913066 0.014372889731158276 0 +655 0 -2.72705364 0.009915355 0.014376224049015385 0 +658 1 5.1765275 0.995642066 0.006300909234690056 1 +659 0 -3.299302 0.00481935544 0.0069696683697878879 0 +662 0 -2.790698 0.009152672 0.013265314439549255 0 +663 0 -2.790698 0.009152672 0.013265314439549255 0 +664 0 -2.51144028 0.0129971383 0.018873827293542356 0 +666 0 -1.94549978 0.0263040867 0.03845680802952664 0 +667 0 -2.60425234 0.011569039 0.016787893374491106 0 +669 1 5.315997 0.9963467 0.0052802313043504865 1 +671 0 -2.322586 0.0164610967 0.023945976311179954 0 +672 0 -2.8278563 0.008734566 0.012656671370376122 0 +673 0 -1.82901454 0.03036955 0.0444930893633559 0 +674 0 -3.16991329 0.00567496149 0.0082105581957195584 0 +675 0 -2.04414058 0.0232797153 0.033982635761241255 0 +676 0 -2.96984339 0.007304208 0.010576417869130071 0 +677 0 -2.391075 0.0151108364 0.021966717597868491 0 +684 0 -3.299302 0.00481935544 0.0069696683697878879 0 +686 0 -3.299302 0.00481935544 0.0069696683697878879 0 +687 0 -2.43590355 0.0142867137 0.020760022876868865 0 +690 0 -3.05244 0.00658178562 0.0095268959776332787 0 +695 0 -3.111528 0.006108972 0.0088404137863029985 0 diff --git a/test/BaselineOutput/Common/WeightedEnsemble/WE-Default-TrainTest-breast-cancer-out.txt b/test/BaselineOutput/Common/WeightedEnsemble/WE-Default-TrainTest-breast-cancer-out.txt new file mode 100644 index 0000000000..981dca228a --- /dev/null +++ b/test/BaselineOutput/Common/WeightedEnsemble/WE-Default-TrainTest-breast-cancer-out.txt @@ -0,0 +1,118 @@ +maml.exe TrainTest test=%Data% tr=WeightedEnsemble{nm=20 tp=-} dout=%Output% loader=Text{col=Label:BL:0 col=Features:R4:1-9} data=%Data% out=%Output% seed=1 +Automatically adding a MinMax normalization transform, use 'norm=Warn' or 'norm=No' to turn this behavior off. +Training 20 learners for the batch 1 +Beginning training model 1 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 26 instances with missing features during training (over 1 iterations; 26 inst/iter) +Trainer 1 of 20 finished in %Time% +Beginning training model 2 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 15 instances with missing features during training (over 1 iterations; 15 inst/iter) +Trainer 2 of 20 finished in %Time% +Beginning training model 3 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 11 instances with missing features during training (over 1 iterations; 11 inst/iter) +Trainer 3 of 20 finished in %Time% +Beginning training model 4 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 12 instances with missing features during training (over 1 iterations; 12 inst/iter) +Trainer 4 of 20 finished in %Time% +Beginning training model 5 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 18 instances with missing features during training (over 1 iterations; 18 inst/iter) +Trainer 5 of 20 finished in %Time% +Beginning training model 6 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 15 instances with missing features during training (over 1 iterations; 15 inst/iter) +Trainer 6 of 20 finished in %Time% +Beginning training model 7 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 24 instances with missing features during training (over 1 iterations; 24 inst/iter) +Trainer 7 of 20 finished in %Time% +Beginning training model 8 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 18 instances with missing features during training (over 1 iterations; 18 inst/iter) +Trainer 8 of 20 finished in %Time% +Beginning training model 9 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 15 instances with missing features during training (over 1 iterations; 15 inst/iter) +Trainer 9 of 20 finished in %Time% +Beginning training model 10 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 18 instances with missing features during training (over 1 iterations; 18 inst/iter) +Trainer 10 of 20 finished in %Time% +Beginning training model 11 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 14 instances with missing features during training (over 1 iterations; 14 inst/iter) +Trainer 11 of 20 finished in %Time% +Beginning training model 12 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 13 instances with missing features during training (over 1 iterations; 13 inst/iter) +Trainer 12 of 20 finished in %Time% +Beginning training model 13 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 14 instances with missing features during training (over 1 iterations; 14 inst/iter) +Trainer 13 of 20 finished in %Time% +Beginning training model 14 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 17 instances with missing features during training (over 1 iterations; 17 inst/iter) +Trainer 14 of 20 finished in %Time% +Beginning training model 15 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 24 instances with missing features during training (over 1 iterations; 24 inst/iter) +Trainer 15 of 20 finished in %Time% +Beginning training model 16 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 12 instances with missing features during training (over 1 iterations; 12 inst/iter) +Trainer 16 of 20 finished in %Time% +Beginning training model 17 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 12 instances with missing features during training (over 1 iterations; 12 inst/iter) +Trainer 17 of 20 finished in %Time% +Beginning training model 18 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 19 instances with missing features during training (over 1 iterations; 19 inst/iter) +Trainer 18 of 20 finished in %Time% +Beginning training model 19 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 23 instances with missing features during training (over 1 iterations; 23 inst/iter) +Trainer 19 of 20 finished in %Time% +Beginning training model 20 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 18 instances with missing features during training (over 1 iterations; 18 inst/iter) +Trainer 20 of 20 finished in %Time% +Training calibrator. +Warning: The predictor produced non-finite prediction values on 16 instances during testing. Possible causes: abnormal data or the predictor is numerically unstable. +TEST POSITIVE RATIO: 0.3499 (239.0/(239.0+444.0)) +Confusion table + ||====================== +PREDICTED || positive | negative | Recall +TRUTH ||====================== + positive || 231 | 8 | 0.9665 + negative || 10 | 434 | 0.9775 + ||====================== +Precision || 0.9585 | 0.9819 | +OVERALL 0/1 ACCURACY: 0.973646 +LOG LOSS/instance: 0.115894 +Test-set entropy (prior Log-Loss/instance): 0.934003 +LOG-LOSS REDUCTION (RIG): 0.875917 +AUC: 0.995976 + +OVERALL RESULTS +--------------------------------------- +AUC: 0.995976 (0.0000) +Accuracy: 0.973646 (0.0000) +Positive precision: 0.958506 (0.0000) +Positive recall: 0.966527 (0.0000) +Negative precision: 0.981900 (0.0000) +Negative recall: 0.977477 (0.0000) +Log-loss: 0.115894 (0.0000) +Log-loss reduction: 0.875917 (0.0000) +F1 Score: 0.962500 (0.0000) +AUPRC: 0.991794 (0.0000) + +--------------------------------------- +Physical memory usage(MB): %Number% +Virtual memory usage(MB): %Number% +%DateTime% Time elapsed(s): %Number% + diff --git a/test/BaselineOutput/Common/WeightedEnsemble/WE-Default-TrainTest-breast-cancer-rp.txt b/test/BaselineOutput/Common/WeightedEnsemble/WE-Default-TrainTest-breast-cancer-rp.txt new file mode 100644 index 0000000000..d908097f1c --- /dev/null +++ b/test/BaselineOutput/Common/WeightedEnsemble/WE-Default-TrainTest-breast-cancer-rp.txt @@ -0,0 +1,4 @@ +WeightedEnsemble +AUC Accuracy Positive precision Positive recall Negative precision Negative recall Log-loss Log-loss reduction F1 Score AUPRC /bp /nm Learner Name Train Dataset Test Dataset Results File Run Time Physical Memory Virtual Memory Command Line Settings +0.995976 0.973646 0.958506 0.966527 0.9819 0.977477 0.115894 0.875917 0.9625 0.991794 Microsoft.ML.Runtime.ComponentFactoryUtils+SimpleComponentFactory`1[Microsoft.ML.Trainers.LinearSvmTrainer] 20 WeightedEnsemble %Data% %Data% %Output% 99 0 0 maml.exe TrainTest test=%Data% tr=WeightedEnsemble{nm=20 tp=-} dout=%Output% loader=Text{col=Label:BL:0 col=Features:R4:1-9} data=%Data% out=%Output% seed=1 /bp:Microsoft.ML.Runtime.ComponentFactoryUtils+SimpleComponentFactory`1[Microsoft.ML.Trainers.LinearSvmTrainer];/nm:20 + diff --git a/test/BaselineOutput/Common/WeightedEnsemble/WE-Default-TrainTest-breast-cancer.txt b/test/BaselineOutput/Common/WeightedEnsemble/WE-Default-TrainTest-breast-cancer.txt new file mode 100644 index 0000000000..05522d073a --- /dev/null +++ b/test/BaselineOutput/Common/WeightedEnsemble/WE-Default-TrainTest-breast-cancer.txt @@ -0,0 +1,700 @@ +Instance Label Score Probability Log-loss Assigned +0 0 -2.6328392 0.0173636749 0.02527052278685812 0 +1 0 2.11736512 0.9386242 4.0261863408227248 1 +2 0 -2.931671 0.0114151845 0.016563347762008285 0 +3 0 2.00851679 0.9290655 3.8173692357034033 1 +4 0 -2.64747477 0.017011689 0.024753833758538169 0 +5 1 6.8502965 0.9999226 0.00011170705633068974 1 +6 0 -1.16596484 0.124842755 0.1923858373893238 0 +7 0 -3.2027173 0.007788862 0.011280942836768386 0 +8 0 -3.28254628 0.00695786625 0.010073163779740036 0 +9 0 -2.83651471 0.01304978 0.018950775894263538 0 +10 0 -3.70307541 0.0038353866 0.0055439315923853155 0 +11 0 -3.54705429 0.00478486 0.0069196620982970164 0 +12 1 -0.590930462 0.244414985 2.0325953548126328 0 +13 0 -3.05513287 0.009592708 0.013906158561496101 0 +14 1 4.779662 0.998525441 0.0021289093874529001 1 +15 1 0.551175356 0.621868253 0.68531912744256229 1 +16 0 -3.04816484 0.009687425 0.014044135603633531 0 +17 0 -2.90669274 0.0118235666 0.017159444957377035 0 +18 1 4.50848 0.9978321 0.0031309855261114575 1 +19 0 -2.35898542 0.0254328828 0.037166549452166096 0 +20 1 3.16856718 0.985572457 0.020966155669068468 1 +21 1 3.99947834 0.9955355 0.0064553429659572857 1 +22 0 -3.29760957 0.0068112365 0.0098601552918805221 0 +23 1 ? ? ? 0 +24 0 -3.637394 0.00420978 0.0060862488150793693 0 +25 1 0.6473596 0.6534935 0.6137551565250392 1 +26 0 -3.19909048 0.007828871 0.011339117448128173 0 +27 0 -2.79872 0.0137613192 0.019991257709421052 0 +28 0 -3.54705429 0.00478486 0.0069196620982970164 0 +29 0 -3.44263363 0.00554758357 0.0080257537138657634 0 +30 0 -3.40901732 0.005817978 0.0084180801822107201 0 +31 0 -3.42188764 0.005712941 0.0082656645663824798 0 +32 1 3.89695716 0.994837463 0.0074672582961766206 1 +33 0 -3.26445127 0.00713815726 0.010335114964310919 0 +34 0 -3.044885 0.009732328 0.014109553171330081 0 +35 0 -3.54705429 0.00478486 0.0069196620982970164 0 +36 1 5.0909586 0.9990529 0.0013670496418724557 1 +37 0 -1.68722332 0.0635953546 0.094796003302770965 0 +38 1 2.768004 0.9747602 0.036880786032233069 1 +39 1 0.9743774 0.7502663 0.41452531121917446 1 +40 0 ? ? ? 0 +41 1 1.2467916 0.8157616 0.29378045196154795 1 +42 1 5.1614275 0.999143243 0.0012365691071442846 1 +43 1 0.365671158 0.558078647 0.84145964818964614 1 +44 1 4.497968 0.997799456 0.0031782119249289028 1 +45 0 -3.67325664 0.00400106 0.0057838880439724637 0 +46 1 2.76622725 0.9746978 0.036973064938771665 1 +47 0 -3.775422 0.00346128456 0.0050022401651684317 0 +48 0 -2.64747477 0.017011689 0.024753833758538169 0 +49 1 3.53555679 0.9913935 0.012470285475987138 1 +50 1 1.432833 0.8523024 0.23056274676973798 1 +51 1 -0.352403641 0.312380344 1.6786244191728434 0 +52 1 2.821775 0.9765768 0.034194582264557807 1 +53 1 3.737498 0.993530154 0.0093643400995652958 1 +54 1 3.85121417 0.994491935 0.0079684238840581657 1 +55 1 2.80059314 0.9758769 0.035228880972547991 1 +56 1 4.05116844 0.9958509 0.005998308515097508 1 +57 1 0.33322525 0.546656847 0.87129260095974759 1 +58 1 1.29503965 0.825863 0.27602561134210818 1 +59 1 0.6439309 0.652387261 0.61619948283742554 1 +60 1 0.9197371 0.7354082 0.44338285749168954 1 +61 0 -3.482952 0.005239716 0.0075791857754945683 0 +62 1 4.39021969 0.99743557 0.0037044427105447425 1 +63 1 0.0110094547 0.432516 1.2091745630102222 1 +64 0 -3.775422 0.00346128456 0.0050022401651684317 0 +65 1 0.8796387 0.7241509 0.46563774242381234 1 +66 0 -2.90669274 0.0118235666 0.017159444957377035 0 +67 1 2.36175466 0.9558647 0.065121630214399817 1 +68 1 4.79976654 0.998567 0.0020688861726438725 1 +69 0 -3.56391621 0.004671885 0.0067558992707747968 0 +70 0 -2.292564 0.0278851222 0.040801283315830265 0 +71 1 3.966124 0.995319366 0.0067685798106970722 1 +72 0 -1.92090356 0.04643221 0.068592587585505738 0 +73 1 4.63476 0.9981882 0.0026162490986497708 1 +74 1 1.555331 0.8729375 0.19604972966942677 1 +75 0 -2.9451952 0.0111999186 0.016249232843906697 0 +76 0 -3.10972548 0.008881699 0.012870825915637542 0 +77 0 -2.3095386 0.0272374023 0.039840336001333188 0 +78 0 -2.6525445 0.016891405 0.024577308194984122 0 +79 0 -3.475928 0.005292101 0.0076551614827592412 0 +80 0 -2.50084162 0.0208787657 0.030440590269376986 0 +81 0 -2.92301917 0.0115550337 0.016767451753478099 0 +82 0 -2.413708 0.0235714931 0.034413679574736851 0 +83 0 -2.3127532 0.0271163974 0.039660885965929024 0 +84 1 4.51044 0.99783814 0.0031222815592818324 1 +85 1 3.642143 0.9925963 0.010720976956973738 1 +86 1 1.28331256 0.8234487 0.28024928971577051 1 +87 1 3.45712233 0.9903864 0.013936552779816383 1 +88 0 -2.90669274 0.0118235666 0.017159444957377035 0 +89 0 -3.34579515 0.00636248849 0.0092084561084276196 0 +90 0 -3.637394 0.00420978 0.0060862488150793693 0 +91 0 -3.28834772 0.00690102624 0.0099905887747521443 0 +92 0 -2.90669274 0.0118235666 0.017159444957377035 0 +93 0 -3.775422 0.00346128456 0.0050022401651684317 0 +94 0 -3.42188764 0.005712941 0.0082656645663824798 0 +95 0 -3.637394 0.00420978 0.0060862488150793693 0 +96 0 -3.52607131 0.00492925057 0.0071289901150605548 0 +97 0 -2.6328392 0.0173636749 0.02527052278685812 0 +98 1 4.88085556 0.99872303 0.0018434554252592923 1 +99 1 5.09803343 0.999062359 0.0013533641241810506 1 +100 1 2.577565 0.967157066 0.04817789313903386 1 +101 1 -0.6653242 0.225386888 2.1495245078260394 0 +102 0 -2.677233 0.0163174886 0.023735341492643156 0 +103 1 0.309304237 0.538203835 0.89377542256587905 1 +104 1 6.073985 0.9997662 0.00033738341622184547 1 +105 1 0.546279669 0.6202278 0.68912987074723475 1 +106 1 5.49459267 0.999466658 0.00076965564318850046 1 +107 1 4.026117 0.9957009 0.0062156668230571805 1 +108 0 -3.55135226 0.004755808 0.0068775478178370996 0 +109 1 3.18584681 0.9859181 0.020460280864585768 1 +110 0 -2.26613474 0.0289234873 0.042343122458742256 0 +111 1 2.44200325 0.9604415 0.058230307447542838 1 +112 1 3.95095778 0.9952177 0.0069159784250622004 1 +113 1 5.340437 0.9993359 0.00095843363824960493 1 +114 0 -1.9839282 0.04261699 0.062831890283949463 0 +115 0 -2.750092 0.0147333136 0.021413816921171547 0 +116 0 -0.610472441 0.23931326 0.39462563759701119 0 +117 1 3.97634077 0.99538666 0.0066710423489583478 1 +118 0 -3.17290115 0.008123888 0.011768159089791598 0 +119 0 -2.69433546 0.0159312058 0.02316892002680418 0 +120 0 -3.24363184 0.00735133747 0.010644913398553347 0 +121 0 -2.444891 0.0225709658 0.032936134913504438 0 +122 1 5.87663937 0.999690354 0.00044679411322055547 1 +123 1 2.14594054 0.9409266 0.087845892086882885 1 +124 1 3.697999 0.99315846 0.0099041749865198519 1 +125 0 -3.775422 0.00346128456 0.0050022401651684317 0 +126 1 3.73013663 0.993462443 0.0094626657257874218 1 +127 0 -3.18054628 0.008036645 0.011641269358735686 0 +128 1 2.844878 0.9773175 0.033100752119347779 1 +129 0 -3.33790326 0.00643392 0.0093121735996220378 0 +130 0 -2.292564 0.0278851222 0.040801283315830265 0 +131 0 -3.42188764 0.005712941 0.0082656645663824798 0 +132 1 5.31207943 0.9993085 0.00099793042732034898 1 +133 0 -3.218942 0.007612353 0.011024318139484778 0 +134 0 -3.21434736 0.00766193075 0.01109639389260818 0 +135 0 -2.18386626 0.0324013 0.047519265316863665 0 +136 0 -3.04816484 0.009687425 0.014044135603633531 0 +137 0 -3.40409184 0.005858682 0.008477148383655193 0 +138 0 -2.7929306 0.0138736377 0.020155569329335588 0 +139 0 ? ? ? 0 +140 0 -3.40409184 0.005858682 0.008477148383655193 0 +141 0 -3.6535368 0.00411452027 0.0059482434828329106 0 +142 1 2.187746 0.9441495 0.08291278463946318 1 +143 0 -2.750092 0.0147333136 0.021413816921171547 0 +144 0 -3.54705429 0.00478486 0.0069196620982970164 0 +145 0 ? ? ? 0 +146 1 0.8966508 0.728962958 0.45608258879993824 1 +147 0 -3.39822674 0.00590752 0.0085480238215067898 0 +148 0 -1.33580148 0.100728244 0.15317093645555213 0 +149 1 6.271516 0.9998235 0.00025464278613373311 1 +150 0 -3.70307541 0.0038353866 0.0055439315923853155 0 +151 1 2.48424983 0.9626646 0.054894849423036525 1 +152 1 5.56014252 0.999514163 0.00070108561421183976 1 +153 0 -2.57273149 0.0188858025 0.027507025098657694 0 +154 0 -3.91345024 0.00284549058 0.0041110268765898331 0 +155 1 1.9855001 0.926875234 0.1095529436400368 1 +156 0 -3.39209557 0.005959007 0.0086227468831349217 0 +157 0 -3.42188764 0.005712941 0.0082656645663824798 0 +158 0 ? ? ? 0 +159 1 6.90917253 0.9999288 0.00010276329372282294 1 +160 1 5.14178038 0.9991189 0.0012716840799016607 1 +161 0 -2.82999587 0.0131698623 0.019126318848864912 0 +162 0 -3.18054628 0.008036645 0.011641269358735686 0 +163 0 -2.66463518 0.0166079029 0.024161333774042597 0 +164 0 ? ? ? 0 +165 0 -2.59714365 0.01825243 0.02657597362911987 0 +166 1 4.551165 0.9979597 0.0029465760728496285 1 +167 1 4.196242 0.9966226 0.0048807732282535217 1 +168 0 -3.18054628 0.008036645 0.011641269358735686 0 +169 0 -3.854062 0.0030957812 0.0044731957774744643 0 +170 0 -3.40409184 0.005858682 0.008477148383655193 0 +171 0 -3.637394 0.00420978 0.0060862488150793693 0 +172 0 -3.775422 0.00346128456 0.0050022401651684317 0 +173 1 7.839463 0.999981046 2.7345500645102764E-05 1 +174 1 3.22615862 0.986693144 0.019326610697212566 1 +175 1 4.71425867 0.9983818 0.0023364694494314537 1 +176 0 -3.42188764 0.005712941 0.0082656645663824798 0 +177 1 2.91240168 0.9793537 0.0300980640204347 1 +178 0 -2.90669274 0.0118235666 0.017159444957377035 0 +179 1 1.24605608 0.8156042 0.29405887276233217 1 +180 0 -3.70307541 0.0038353866 0.0055439315923853155 0 +181 0 -3.91345024 0.00284549058 0.0041110268765898331 0 +182 0 -2.35898542 0.0254328828 0.037166549452166096 0 +183 1 5.05071068 0.9989971 0.0014476160757998244 1 +184 1 3.64882565 0.992665946 0.01061979348169907 1 +185 0 -3.54578 0.004793508 0.0069321982683904198 0 +186 1 2.98114157 0.981242537 0.027318317873323002 1 +187 1 6.20997334 0.999807358 0.00027795073717637426 1 +188 1 4.302245 0.9970944 0.0041980069696079536 1 +189 0 -3.05918741 0.009538018 0.013826495483274739 0 +190 1 7.068928 0.999943256 8.1866064521166627E-05 1 +191 1 6.232257 0.9998134 0.00026926396601644696 1 +192 0 -2.79872 0.0137613192 0.019991257709421052 0 +193 0 -3.637394 0.00420978 0.0060862488150793693 0 +194 0 -3.18054628 0.008036645 0.011641269358735686 0 +195 0 -2.90669274 0.0118235666 0.017159444957377035 0 +196 0 3.0190897 0.9822116 5.8129186121745748 1 +197 0 -2.26370764 0.0290207043 0.042487561705564593 0 +198 0 -3.91345024 0.00284549058 0.0041110268765898331 0 +199 0 -3.29760957 0.0068112365 0.0098601552918805221 0 +200 1 5.44719076 0.999429464 0.00082334386288675145 1 +201 1 5.4106226 0.999399 0.00086731118468831418 1 +202 0 -3.637394 0.00420978 0.0060862488150793693 0 +203 0 -2.6328392 0.0173636749 0.02527052278685812 0 +204 0 -3.637394 0.00420978 0.0060862488150793693 0 +205 1 6.63203 0.9998944 0.00015238467619747771 1 +206 1 3.98650813 0.995452642 0.0065754119310075096 1 +207 0 -3.70307541 0.0038353866 0.0055439315923853155 0 +208 0 -3.70307541 0.0038353866 0.0055439315923853155 0 +209 0 -2.81870985 0.0133803412 0.01943406102667871 0 +210 1 7.619505 0.9999741 3.7406711505747454E-05 1 +211 1 5.429845 0.9994152 0.00084390766852057068 1 +212 0 -3.637394 0.00420978 0.0060862488150793693 0 +213 1 8.056784 0.9999861 2.0036117955462525E-05 1 +214 1 7.162195 0.999950349 7.1632552401153772E-05 1 +215 1 4.36069965 0.9973258 0.0038632547125735731 1 +216 0 -3.775422 0.00346128456 0.0050022401651684317 0 +217 0 -3.637394 0.00420978 0.0060862488150793693 0 +218 1 4.395052 0.9974531 0.0036790964843190005 1 +219 0 -1.9000659 0.047763627 0.070608357704899982 0 +220 0 -3.45075 0.00548419356 0.0079337940280517424 0 +221 1 5.3034296 0.999299943 0.0010103217996659561 1 +222 1 -1.65166473 0.06667773 3.9066512693549087 0 +223 1 2.80228877 0.9759337 0.035144996048568189 1 +224 1 4.982401 0.998894751 0.0015954189775320689 1 +225 0 -3.775422 0.00346128456 0.0050022401651684317 0 +226 1 5.298452 0.999295 0.0010174640973562138 1 +227 1 4.309857 0.997125566 0.0041529031554512121 1 +228 0 -3.70307541 0.0038353866 0.0055439315923853155 0 +229 1 7.17760324 0.9999514 7.0084632518499385E-05 1 +230 1 2.91424656 0.9794068 0.030019920439681005 1 +231 1 4.599201 0.9980943 0.0027519376461766941 1 +232 0 0.5308039 0.6150242 1.377160369430146 1 +233 1 3.43461132 0.9900764 0.01438820567238709 1 +234 0 -1.8945719 0.0481206626 0.071149389341804481 0 +235 0 ? ? ? 0 +236 1 6.31713772 0.9998346 0.00023864566495751231 1 +237 1 3.810334 0.994163752 0.0084445930322586821 1 +238 1 6.867334 0.9999244 0.00010904112129145599 1 +239 1 2.99594069 0.981626451 0.026753970341819432 1 +240 0 -1.62479627 0.06909822 0.10329913756170231 0 +241 0 -3.01180363 0.0101969074 0.014786545099445015 0 +242 0 -3.42188764 0.005712941 0.0082656645663824798 0 +243 0 -2.19949484 0.03171089 0.046490224601786277 0 +244 0 -3.637394 0.00420978 0.0060862488150793693 0 +245 0 -2.305316 0.02739715 0.040077276059953024 0 +246 1 6.578383 0.999886 0.00016451078479151539 1 +247 1 1.70538473 0.89480865 0.16034889204866712 1 +248 0 -2.29880738 0.0276451632 0.040445209226106546 0 +249 0 ? ? ? 0 +250 0 -3.49286652 0.005166651 0.0074732245262098523 0 +251 1 4.048608 0.995835841 0.0060201551284871507 1 +252 0 2.08435583 0.9358602 3.9626367083625218 1 +253 1 5.1614275 0.999143243 0.0012365691071442846 1 +254 1 4.39021969 0.99743557 0.0037044427105447425 1 +255 1 2.7252965 0.97322017 0.039161875104346891 1 +256 0 -3.40409184 0.005858682 0.008477148383655193 0 +257 0 -3.29760957 0.0068112365 0.0098601552918805221 0 +258 0 -3.18054628 0.008036645 0.011641269358735686 0 +259 0 2.230816 0.9472964 4.2459541588639542 1 +260 1 4.88506174 0.998730659 0.0018324345042610049 1 +261 1 6.57487 0.9998854 0.00016537079636382189 1 +262 1 5.525812 0.999489844 0.00073618755564433991 1 +263 1 4.36628 0.9973469 0.0038327324821600946 1 +264 1 3.10015345 0.984119833 0.0230940971580532 1 +265 0 -1.838001 0.05194722 0.076960717566259718 0 +266 1 4.64122438 0.998204768 0.0025923003181383287 1 +267 1 1.35275674 0.8373666 0.25606875316553179 1 +268 1 4.63762045 0.9981955 0.0026056530064384399 1 +269 0 -3.637394 0.00420978 0.0060862488150793693 0 +270 1 3.47361565 0.9906075 0.013614550957318554 1 +271 0 -2.6328392 0.0173636749 0.02527052278685812 0 +272 1 1.35275674 0.8373666 0.25606875316553179 1 +273 1 -0.183948278 0.366056859 1.4498603356193009 0 +274 0 -3.09982252 0.009006679 0.013052760862515228 0 +275 0 ? ? ? 0 +276 0 -3.29760957 0.0068112365 0.0098601552918805221 0 +277 0 -3.775422 0.00346128456 0.0050022401651684317 0 +278 0 -3.637394 0.00420978 0.0060862488150793693 0 +279 1 3.186985 0.9859406 0.020427399472005482 1 +280 0 -3.18054628 0.008036645 0.011641269358735686 0 +281 0 -3.26445127 0.00713815726 0.010335114964310919 0 +282 1 1.96301413 0.924675465 0.11298098625588758 1 +283 1 3.13104725 0.984792769 0.022107926313092451 1 +284 1 3.763544 0.9937642 0.0090244931794246086 1 +285 1 7.407968 0.999965 5.0477791090005655E-05 1 +286 1 8.45643 0.999992132 1.135089961038231E-05 1 +287 0 -3.21434736 0.00766193075 0.01109639389260818 0 +288 1 0.77104187 0.69222194 0.53069342729584723 1 +289 1 4.368184 0.997354031 0.0038223861099225243 1 +290 0 -3.91345024 0.00284549058 0.0041110268765898331 0 +291 0 -3.637394 0.00420978 0.0060862488150793693 0 +292 1 ? ? ? 0 +293 1 3.04578352 0.9828636 0.024936871618289391 1 +294 0 ? ? ? 0 +295 1 3.95846057 0.9952683 0.0068426228359784336 1 +296 0 0.7122183 0.6740973 1.6174867886149578 1 +297 0 ? ? ? 0 +298 0 -1.71484792 0.0612929426 0.09125308907791306 0 +299 1 3.28456354 0.9877419 0.017794002645609635 1 +300 1 3.791232 0.994003832 0.0086766808386321576 1 +301 0 -3.637394 0.00420978 0.0060862488150793693 0 +302 1 8.612249 0.9999937 9.115109290810302E-06 1 +303 0 -3.637394 0.00420978 0.0060862488150793693 0 +304 1 2.81464386 0.9763434 0.034539442266785979 1 +305 1 4.829171 0.9986257 0.0019840656579413166 1 +306 0 -3.637394 0.00420978 0.0060862488150793693 0 +307 0 -3.637394 0.00420978 0.0060862488150793693 0 +308 1 3.6297164 0.9924652 0.010911581544003367 1 +309 0 -1.76295733 0.0574683 0.085386951983802917 0 +310 0 -3.475928 0.005292101 0.0076551614827592412 0 +311 0 -3.91345024 0.00284549058 0.0041110268765898331 0 +312 1 1.88456845 0.916517735 0.12576529784564555 1 +313 0 -3.91345024 0.00284549058 0.0041110268765898331 0 +314 0 -3.85518837 0.003090836 0.0044660393836612727 0 +315 0 ? ? ? 0 +316 1 2.23254037 0.947418869 0.07792569035195504 1 +317 1 4.95329952 0.9988481 0.0016628262601476259 1 +318 0 -3.20374346 0.007777579 0.011264537259358637 0 +319 0 0.9583156 0.7459571 1.9768558175067299 1 +320 1 3.53467345 0.9913828 0.012485898370594568 1 +321 0 ? ? ? 0 +322 0 -3.18054628 0.008036645 0.011641269358735686 0 +323 1 2.5388124 0.965358555 0.050863204423697336 1 +324 0 -3.637394 0.00420978 0.0060862488150793693 0 +325 0 -2.6593976 0.0167301353 0.024340667108450266 0 +326 1 1.71892333 0.896609247 0.15744871690865739 1 +327 0 -3.775422 0.00346128456 0.0050022401651684317 0 +328 1 2.3329916 0.9541044 0.06778092189109225 1 +329 1 3.98380852 0.995435238 0.0066006363218682413 1 +330 1 2.96085882 0.9807036 0.028110932484191997 1 +331 0 -2.58576059 0.0185451265 0.02700615968261286 0 +332 0 -2.14993453 0.0339506343 0.049831181364778515 0 +333 1 2.43493152 0.9600572 0.058807732535601939 1 +334 1 3.37907267 0.989268661 0.015565720919030471 1 +335 0 -3.91345024 0.00284549058 0.0041110268765898331 0 +336 1 2.927155 0.979774237 0.029478738665664722 1 +337 0 -3.637394 0.00420978 0.0060862488150793693 0 +338 0 -3.85518837 0.003090836 0.0044660393836612727 0 +339 1 2.93235779 0.9799205 0.02926337582536782 1 +340 1 3.11057854 0.984350145 0.022756504523365872 1 +341 0 -3.637394 0.00420978 0.0060862488150793693 0 +342 0 -3.6535368 0.00411452027 0.0059482434828329106 0 +343 0 -3.91345024 0.00284549058 0.0041110268765898331 0 +344 1 4.89797735 0.9987538 0.0017990278518659488 1 +345 0 -3.91345024 0.00284549058 0.0041110268765898331 0 +346 0 -2.08377171 0.0371796265 0.054661425030536351 0 +347 0 -3.765569 0.00351001252 0.0050727856500071711 0 +348 1 -0.0578651428 0.408624232 1.2911533350569351 0 +349 1 1.28584218 0.8239717 0.27933332690290869 1 +350 0 -2.6757412 0.016351616 0.023785394494289468 0 +351 0 -3.42188764 0.005712941 0.0082656645663824798 0 +352 0 0.193401575 0.4970234 0.99143682230679375 1 +353 1 4.876693 0.99871546 0.0018543903285190208 1 +354 0 -3.775422 0.00346128456 0.0050022401651684317 0 +355 0 -2.83535242 0.0130711114 0.018981957282690046 0 +356 1 -0.8489101 0.1830331 2.4498235722842745 0 +357 1 7.00974035 0.9999383 8.900376720685931E-05 1 +358 1 3.40555716 0.9896617 0.01499265698459722 1 +359 1 2.476605 0.9622714 0.055484255806192428 1 +360 1 8.042091 0.9999858 2.0466080617959804E-05 1 +361 1 3.48360252 0.990738869 0.013423241770278925 1 +362 0 -2.09481478 0.0366208628 0.05382441339348297 0 +363 0 -1.19371545 0.120589517 0.18539136470225254 0 +364 0 -3.42188764 0.005712941 0.0082656645663824798 0 +365 0 -3.54705429 0.00478486 0.0069196620982970164 0 +366 1 7.425378 0.999965847 4.9273870913609948E-05 1 +367 1 5.725535 0.9996161 0.00055397647343730452 1 +368 0 -3.44263363 0.00554758357 0.0080257537138657634 0 +369 0 -3.42484713 0.00568905566 0.0082310079810035911 0 +370 0 -2.35329056 0.02563463 0.037465235160016659 0 +371 0 -3.44263363 0.00554758357 0.0080257537138657634 0 +372 0 -2.7929306 0.0138736377 0.020155569329335588 0 +373 0 -2.67125463 0.016454678 0.023936561166939072 0 +374 0 -3.044885 0.009732328 0.014109553171330081 0 +375 0 -3.91345024 0.00284549058 0.0041110268765898331 0 +376 0 -3.775422 0.00346128456 0.0050022401651684317 0 +377 0 -3.85518837 0.003090836 0.0044660393836612727 0 +378 0 -2.70431757 0.01570992 0.022844539980429755 0 +379 0 -1.38652766 0.09437218 0.14300982277131419 0 +380 0 -3.91345024 0.00284549058 0.0041110268765898331 0 +381 1 4.8580327 0.998680949 0.0019042442054887036 1 +382 0 -2.5623126 0.01916264 0.027914162661803264 0 +383 0 -3.6535368 0.00411452027 0.0059482434828329106 0 +384 0 -3.6535368 0.00411452027 0.0059482434828329106 0 +385 0 -2.33086634 0.0264442842 0.038664549385249568 0 +386 1 3.114027 0.9844256 0.022645912935580065 1 +387 0 -1.57609391 0.0736940056 0.11043924517288556 0 +388 0 -3.471188 0.00532774534 0.0077068601549602359 0 +389 0 -2.362008 0.02532643 0.037008972380008727 0 +390 0 -3.6039772 0.00441401 0.0063821659948234731 0 +391 1 5.60524464 0.9995444 0.00065746747995155833 1 +392 0 -3.29760957 0.0068112365 0.0098601552918805221 0 +393 0 -3.97553015 0.00260540447 0.0037637092354945701 0 +394 0 -3.1629355 0.00823902152 0.011935632258090027 0 +395 0 -3.29760957 0.0068112365 0.0098601552918805221 0 +396 0 -3.18054628 0.008036645 0.011641269358735686 0 +397 0 -3.15464687 0.008336013 0.012076731000788543 0 +398 0 -3.00103378 0.0103528388 0.015013841914863556 0 +399 0 -3.254984 0.00723432656 0.010474862194562938 0 +400 1 4.868566 0.998700559 0.0018759159707978693 1 +401 0 -3.40409184 0.005858682 0.008477148383655193 0 +402 0 -1.94066358 0.04520232 0.066733031825891048 0 +403 0 -2.54634523 0.01959464 0.028549722587625153 0 +404 0 -3.31751251 0.00662219245 0.0095855781252998078 0 +405 0 -3.775422 0.00346128456 0.0050022401651684317 0 +406 0 -2.6073544 0.0179937389 0.026195871910031258 0 +407 0 -3.775422 0.00346128456 0.0050022401651684317 0 +408 0 -2.41573954 0.0235050116 0.034315454964433549 0 +409 0 -3.044885 0.009732328 0.014109553171330081 0 +410 0 -3.775422 0.00346128456 0.0050022401651684317 0 +411 0 ? ? ? 0 +412 1 5.147028 0.9991255 0.0012622167236815698 1 +413 0 -2.40337944 0.0239123367 0.034917371523912978 0 +414 1 3.65053129 0.9926836 0.010594152221187944 1 +415 0 -0.868057966 0.17899166 0.28453121840291706 0 +416 1 4.86914349 0.998701632 0.0018743661138240317 1 +417 0 -3.775422 0.00346128456 0.0050022401651684317 0 +418 0 -1.72332454 0.0606022142 0.090191901761027982 0 +419 0 -2.982328 0.010629301 0.015416921259826712 0 +420 0 -1.95868087 0.0441079959 0.065080462026362723 0 +421 1 6.60801125 0.9998907 0.00015771671139298783 1 +422 0 -1.87438178 0.04945463 0.07317260494481162 0 +423 0 -2.292564 0.0278851222 0.040801283315830265 0 +424 0 -3.40409184 0.005858682 0.008477148383655193 0 +425 1 8.196907 0.9999886 1.6424436649469627E-05 1 +426 0 -1.43597829 0.0885242 0.13372374220553412 0 +427 1 2.59246826 0.9678244 0.047182783950707262 1 +428 0 -3.775422 0.00346128456 0.0050022401651684317 0 +429 0 -3.54705429 0.00478486 0.0069196620982970164 0 +430 0 -3.35573435 0.00627364637 0.0090794690735808582 0 +431 0 -2.04877186 0.03900521 0.057399483527574707 0 +432 0 -2.71445 0.0154883973 0.022519886205158234 0 +433 0 -2.730778 0.0151378689 0.022006316255241531 0 +434 0 3.277352 0.987616956 6.3354902155766029 1 +435 1 4.75078869 0.9984637 0.0022181307174977826 1 +436 1 2.99453473 0.981590331 0.026807057442240823 1 +437 0 -3.15464687 0.008336013 0.012076731000788543 0 +438 0 -2.47497177 0.0216451772 0.031570308319016145 0 +439 0 -2.90520215 0.0118483882 0.01719568389318981 0 +440 1 4.28276348 0.9970129 0.0043159044785095512 1 +441 0 -1.31492066 0.103453368 0.15754947099085548 0 +442 0 -3.001175 0.0103507806 0.015010841466840385 0 +443 0 -3.62845278 0.004263485 0.0061640579918126603 0 +444 0 -1.96413243 0.0437818952 0.064588373293324938 0 +445 0 -3.6535368 0.00411452027 0.0059482434828329106 0 +446 0 -3.91345024 0.00284549058 0.0041110268765898331 0 +447 0 -2.90520215 0.0118483882 0.01719568389318981 0 +448 0 -3.97553015 0.00260540447 0.0037637092354945701 0 +449 1 5.525615 0.9994897 0.00073635962608825482 1 +450 0 -2.85610628 0.0126953553 0.018432780697991182 0 +451 0 -2.90520215 0.0118483882 0.01719568389318981 0 +452 0 -3.14916134 0.008400825 0.012171023554296338 0 +453 1 3.83451128 0.994360149 0.0081596164039714775 1 +454 0 -3.2191658 0.00760994526 0.011020817581025999 0 +455 1 0.13920784 0.477748066 1.0656780613669095 1 +456 1 5.07329369 0.9990288 0.0014018234909620132 1 +457 1 4.68547058 0.9983142 0.0024341449727003877 1 +458 0 -2.64996815 0.0169524252 0.024666857154311061 0 +459 0 -2.39631176 0.0241483375 0.035266232039964998 0 +460 0 -2.6757412 0.016351616 0.023785394494289468 0 +461 0 -2.25858235 0.02922704 0.042794171852324328 0 +462 0 -2.404407 0.023878213 0.034866936327211301 0 +463 0 -3.08581543 0.009186436 0.013314476633591577 0 +464 0 -3.15464687 0.008336013 0.012076731000788543 0 +465 1 5.397848 0.999388 0.00088322923430280429 1 +466 1 4.881676 0.9987245 0.00184130289501091 1 +467 1 3.847084 0.9944596 0.0080152900815416792 1 +468 0 -3.15464687 0.008336013 0.012076731000788543 0 +469 0 -3.56804562 0.004644625 0.0067163869452357451 0 +470 0 -3.40901732 0.005817978 0.0084180801822107201 0 +471 0 -2.404407 0.023878213 0.034866936327211301 0 +472 0 -2.745381 0.0148309963 0.021556857718756046 0 +473 0 -3.15464687 0.008336013 0.012076731000788543 0 +474 0 -2.90520215 0.0118483882 0.01719568389318981 0 +475 0 -3.40409184 0.005858682 0.008477148383655193 0 +476 0 -2.89941263 0.0119452868 0.017337162058911885 0 +477 0 -3.15464687 0.008336013 0.012076731000788543 0 +478 0 -2.67868233 0.0162843987 0.023686811790945436 0 +479 1 4.35081673 0.997288 0.0039179204340691635 1 +480 0 -2.994873 0.010443097 0.01514542516512736 0 +481 0 -1.9328773 0.0456831977 0.067459821037105655 0 +482 1 7.87596941 0.999982 2.5969614015337708E-05 1 +483 1 5.70353746 0.999603868 0.00057161157324537237 1 +484 0 -2.64996815 0.0169524252 0.024666857154311061 0 +485 0 -3.09087944 0.009121042 0.013219260473139628 0 +486 0 -3.40901732 0.005817978 0.0084180801822107201 0 +487 1 7.05091667 0.9999418 8.3929974921495668E-05 1 +488 1 0.6096113 0.641225159 0.64109706257148902 1 +489 1 -0.658962965 0.226972073 2.1394132959007339 0 +490 0 -3.91345024 0.00284549058 0.0041110268765898331 0 +491 1 3.85680246 0.9945354 0.0079053903846417004 1 +492 0 -3.15952539 0.00827879 0.011993483728992817 0 +493 1 5.77308559 0.9996412 0.00051776067608620972 1 +494 0 -0.0895915 0.397755235 0.73157814800417431 0 +495 0 -3.40901732 0.005817978 0.0084180801822107201 0 +496 0 -3.97553015 0.00260540447 0.0037637092354945701 0 +497 0 -2.94026279 0.0112779588 0.016363100772292413 0 +498 0 -3.04816484 0.009687425 0.014044135603633531 0 +499 0 -3.04816484 0.009687425 0.014044135603633531 0 +500 0 -2.35898542 0.0254328828 0.037166549452166096 0 +501 0 -3.04816484 0.009687425 0.014044135603633531 0 +502 0 -2.92301917 0.0115550337 0.016767451753478099 0 +503 0 -2.90669274 0.0118235666 0.017159444957377035 0 +504 0 -3.91345024 0.00284549058 0.0041110268765898331 0 +505 0 -3.025893 0.009996418 0.014494349640235516 0 +506 1 5.61716747 0.999552 0.00064645561510042588 1 +507 0 -3.042879 0.00975989 0.014149707445597173 0 +508 0 -2.90520215 0.0118483882 0.01719568389318981 0 +509 0 -3.6535368 0.00411452027 0.0059482434828329106 0 +510 0 -3.91345024 0.00284549058 0.0041110268765898331 0 +511 0 -2.79872 0.0137613192 0.019991257709421052 0 +512 0 -2.90520215 0.0118483882 0.01719568389318981 0 +513 0 -3.40901732 0.005817978 0.0084180801822107201 0 +514 1 5.25185728 0.999246657 0.0010872539512012218 1 +515 1 4.453326 0.9976554 0.003386526369283925 1 +516 0 -3.97553015 0.00260540447 0.0037637092354945701 0 +517 0 -3.85518837 0.003090836 0.0044660393836612727 0 +518 0 -3.05918741 0.009538018 0.013826495483274739 0 +519 1 3.46562624 0.990501046 0.013769595977601232 1 +520 0 -3.82106638 0.00324420421 0.0046880058735476234 0 +521 0 -3.05651522 0.009574028 0.013878947569712329 0 +522 1 2.42715955 0.9596307 0.059448740831798338 1 +523 1 3.88349533 0.9947381 0.007611356909112005 1 +524 0 -3.29760957 0.0068112365 0.0098601552918805221 0 +525 0 -3.28834772 0.00690102624 0.0099905887747521443 0 +526 0 -3.15464687 0.008336013 0.012076731000788543 0 +527 0 -2.90669274 0.0118235666 0.017159444957377035 0 +528 0 -2.0452857 0.0391916856 0.0576794593857958 0 +529 0 -3.15952539 0.00827879 0.011993483728992817 0 +530 1 3.34140444 0.988684058 0.016418524867680265 1 +531 0 -2.6073544 0.0179937389 0.026195871910031258 0 +532 0 -3.70307541 0.0038353866 0.0055439315923853155 0 +533 0 -3.29760957 0.0068112365 0.0098601552918805221 0 +534 0 -3.54705429 0.00478486 0.0069196620982970164 0 +535 0 -3.073576 0.009346416 0.013547437409026714 0 +536 0 -2.6328392 0.0173636749 0.02527052278685812 0 +537 0 -2.40337944 0.0239123367 0.034917371523912978 0 +538 0 -3.04816484 0.009687425 0.014044135603633531 0 +539 0 -2.54927516 0.0195146613 0.028432037052712088 0 +540 0 -2.53926373 0.0197892729 0.028836159707279794 0 +541 0 -3.40409184 0.005858682 0.008477148383655193 0 +542 0 -2.50273228 0.0208238065 0.030359612400128837 0 +543 0 -3.04816484 0.009687425 0.014044135603633531 0 +544 0 -2.98785949 0.010546796 0.015296618021855357 0 +545 0 -2.79872 0.0137613192 0.019991257709421052 0 +546 1 6.58093071 0.9998864 0.00016390877699593663 1 +547 0 -3.7433486 0.00362242525 0.0052355433682667405 0 +548 0 -3.493857 0.00515940832 0.00746272100657087 0 +549 1 2.98113585 0.9812424 0.027318493143610147 1 +550 0 -3.29760957 0.0068112365 0.0098601552918805221 0 +551 0 -3.637394 0.00420978 0.0060862488150793693 0 +552 0 -2.36343 0.0252765026 0.036935071405760848 0 +553 0 -1.08910525 0.1372978 0.21306545018859047 0 +554 0 -3.40409184 0.005858682 0.008477148383655193 0 +555 0 -1.52276635 0.07904773 0.11880171010584918 0 +556 0 -2.31750154 0.0269386154 0.039397275993531607 0 +557 0 -2.6757412 0.016351616 0.023785394494289468 0 +558 0 -3.54705429 0.00478486 0.0069196620982970164 0 +559 0 -2.79872 0.0137613192 0.019991257709421052 0 +560 0 -2.6328392 0.0173636749 0.02527052278685812 0 +561 0 -2.6328392 0.0173636749 0.02527052278685812 0 +562 0 -3.637394 0.00420978 0.0060862488150793693 0 +563 0 -3.29760957 0.0068112365 0.0098601552918805221 0 +564 0 -2.82999587 0.0131698623 0.019126318848864912 0 +565 1 6.15782833 0.9997925 0.00029936686178311424 1 +566 0 -2.94113684 0.01126409 0.016342864930702094 0 +567 0 -2.48789215 0.0212590229 0.031000992426832553 0 +568 1 2.58049345 0.9672892 0.04798078994602336 1 +569 1 5.19968128 0.9991886 0.0011710750813801421 1 +570 1 4.62118435 0.9981529 0.0026672492653288605 1 +571 1 5.747953 0.9996281 0.00053659965911567373 1 +572 0 -3.29760957 0.0068112365 0.0098601552918805221 0 +573 0 -3.775422 0.00346128456 0.0050022401651684317 0 +574 1 3.35126233 0.988840044 0.016190927844793548 1 +575 0 -2.40337944 0.0239123367 0.034917371523912978 0 +576 0 -2.79872 0.0137613192 0.019991257709421052 0 +577 0 -3.775422 0.00346128456 0.0050022401651684317 0 +578 0 -3.775422 0.00346128456 0.0050022401651684317 0 +579 0 -3.637394 0.00420978 0.0060862488150793693 0 +580 0 -2.54348588 0.019673001 0.028665038398239539 0 +581 1 4.67283058 0.9982836 0.0024783336914946099 1 +582 1 4.617906 0.998144269 0.0026797411350850012 1 +583 0 -3.40409184 0.005858682 0.008477148383655193 0 +584 0 -2.07600737 0.0375773869 0.055257554453235401 0 +585 0 -3.91345024 0.00284549058 0.0041110268765898331 0 +586 1 7.6938014 0.9999767 3.3623000044204345E-05 1 +587 0 -2.71445 0.0154883973 0.022519886205158234 0 +588 1 3.18754458 0.985951662 0.020411177098160791 1 +589 0 -2.90520215 0.0118483882 0.01719568389318981 0 +590 1 1.71134186 0.8956043 0.15906661996637744 1 +591 1 2.864934 0.977942 0.032179205750400744 1 +592 1 2.68068171 0.9715139 0.04169341740394919 1 +593 0 -2.64996815 0.0169524252 0.024666857154311061 0 +594 1 2.90680337 0.979191959 0.030336384177642214 1 +595 0 -2.79872 0.0137613192 0.019991257709421052 0 +596 0 -2.7929306 0.0138736377 0.020155569329335588 0 +597 0 -2.180966 0.0325310156 0.047712683483938777 0 +598 0 -3.29760957 0.0068112365 0.0098601552918805221 0 +599 0 -2.077083 0.0375220329 0.055174579706508069 0 +600 0 -3.29760957 0.0068112365 0.0098601552918805221 0 +601 0 -3.85518837 0.003090836 0.0044660393836612727 0 +602 0 -3.04816484 0.009687425 0.014044135603633531 0 +603 1 1.88483047 0.9165463 0.12572035686765159 1 +604 1 2.34057426 0.9545749 0.067069718554773339 1 +605 1 5.311107 0.9993076 0.00099930724121406766 1 +606 0 -3.02718163 0.009978277 0.014467913345716541 0 +607 0 -3.91345024 0.00284549058 0.0041110268765898331 0 +608 1 6.112509 0.9997787 0.00031932112743631499 1 +609 0 -2.90520215 0.0118483882 0.01719568389318981 0 +610 1 3.88621426 0.9947583 0.0075820519461301259 1 +611 1 3.58875751 0.9920163 0.011564246412346192 1 +612 1 8.46260452 0.9999922 1.1264907610944685E-05 1 +613 0 -3.18372536 0.008000641 0.011588906689770289 0 +614 0 -3.65460062 0.00410831859 0.0059392594168813061 0 +615 0 -2.55414414 0.01938246 0.028237528059899108 0 +616 0 -3.29760957 0.0068112365 0.0098601552918805221 0 +617 0 ? ? ? 0 +618 0 -3.04816484 0.009687425 0.014044135603633531 0 +619 0 -2.79872 0.0137613192 0.019991257709421052 0 +620 0 -3.29760957 0.0068112365 0.0098601552918805221 0 +621 0 -0.4394667 0.2863918 0.48679589091812958 0 +622 0 -1.818351 0.0533424653 0.079085487951960826 0 +623 0 -3.91345024 0.00284549058 0.0041110268765898331 0 +624 0 -2.780283 0.01412217 0.02051921658538286 0 +625 0 -2.04546857 0.0391818844 0.057664742478955894 0 +626 1 2.82916069 0.9768161 0.033841089457974198 1 +627 0 -2.52296424 0.0202444866 0.029506308571251798 0 +628 0 -3.6535368 0.00411452027 0.0059482434828329106 0 +629 0 -3.15464687 0.008336013 0.012076731000788543 0 +630 0 -1.95878112 0.04410198 0.065071381793387784 0 +631 0 -2.79872 0.0137613192 0.019991257709421052 0 +632 0 -3.91345024 0.00284549058 0.0041110268765898331 0 +633 1 2.44950843 0.96084553 0.057623579567503906 1 +634 0 -3.40409184 0.005858682 0.008477148383655193 0 +635 0 -2.69031239 0.0160212554 0.02330094334398088 0 +636 1 5.0804615 0.999038637 0.0013876211988649851 1 +637 0 -1.64278042 0.06746925 0.100776791050083 0 +638 0 -3.15464687 0.008336013 0.012076731000788543 0 +639 0 -2.6757412 0.016351616 0.023785394494289468 0 +640 0 -2.91681433 0.0116563719 0.016915368463980788 0 +641 0 -3.29760957 0.0068112365 0.0098601552918805221 0 +642 0 -3.29760957 0.0068112365 0.0098601552918805221 0 +643 0 -3.91345024 0.00284549058 0.0041110268765898331 0 +644 0 -3.6535368 0.00411452027 0.0059482434828329106 0 +645 0 -3.29760957 0.0068112365 0.0098601552918805221 0 +646 0 -3.49286652 0.005166651 0.0074732245262098523 0 +647 0 -3.59211588 0.004488849 0.0064906189362599733 0 +648 1 6.042883 0.9997556 0.00035260752096818163 1 +649 0 -3.29760957 0.0068112365 0.0098601552918805221 0 +650 0 -2.40445375 0.0238766614 0.034864643109355768 0 +651 0 -3.23496485 0.007441935 0.01077659217791724 0 +652 0 -2.71445 0.0154883973 0.022519886205158234 0 +653 0 -3.04816484 0.009687425 0.014044135603633531 0 +654 0 -3.18054628 0.008036645 0.011641269358735686 0 +655 0 -3.29760957 0.0068112365 0.0098601552918805221 0 +656 0 -2.79872 0.0137613192 0.019991257709421052 0 +657 0 -0.440840483 0.285992235 0.48598833079405768 0 +658 1 4.78915453 0.9985452 0.0021003183911790101 1 +659 0 -3.91345024 0.00284549058 0.0041110268765898331 0 +660 0 -3.775422 0.00346128456 0.0050022401651684317 0 +661 0 -2.90669274 0.0118235666 0.017159444957377035 0 +662 0 -3.33888555 0.00642498629 0.0092992015748904477 0 +663 0 -3.33888555 0.00642498629 0.0092992015748904477 0 +664 0 -3.029537 0.009945203 0.014419717450438019 0 +665 0 -3.91345024 0.00284549058 0.0041110268765898331 0 +666 0 -2.223639 0.030672187 0.044943447584120944 0 +667 0 -3.18054628 0.008036645 0.011641269358735686 0 +668 1 1.17554736 0.800025761 0.32188163882176946 1 +669 1 4.966632 0.9988697 0.0016315757490214997 1 +670 1 3.652212 0.992700934 0.01056894453431385 1 +671 0 -2.87596464 0.0123458039 0.017922090426527201 0 +672 0 -3.42188764 0.005712941 0.0082656645663824798 0 +673 0 -2.21387434 0.0310882423 0.045562814733190911 0 +674 0 -3.775422 0.00346128456 0.0050022401651684317 0 +675 0 -2.428732 0.02308415 0.033693799757447823 0 +676 0 -3.56804562 0.004644625 0.0067163869452357451 0 +677 0 -2.90520215 0.0118483882 0.01719568389318981 0 +678 0 -3.91345024 0.00284549058 0.0041110268765898331 0 +679 0 -3.6535368 0.00411452027 0.0059482434828329106 0 +680 1 8.535917 0.999992967 1.0147012084681539E-05 1 +681 1 5.589761 0.999534249 0.00067209274290404005 1 +682 0 -2.55614233 0.0193284582 0.02815808220810033 0 +683 0 -3.91345024 0.00284549058 0.0041110268765898331 0 +684 0 -3.91345024 0.00284549058 0.0041110268765898331 0 +685 0 -3.91345024 0.00284549058 0.0041110268765898331 0 +686 0 -3.91345024 0.00284549058 0.0041110268765898331 0 +687 0 -3.1168623 0.00879269652 0.012741277208516274 0 +688 0 -3.15464687 0.008336013 0.012076731000788543 0 +689 0 -3.19265366 0.00790038 0.011443100642480211 0 +690 0 -3.59211588 0.004488849 0.0064906189362599733 0 +691 1 3.23820233 0.9869164 0.019000179865221012 1 +692 0 -3.40409184 0.005858682 0.008477148383655193 0 +693 0 -3.22007346 0.007600192 0.01100663874759376 0 +694 0 -3.02062726 0.0100708874 0.014602875304934926 0 +695 0 -3.6535368 0.00411452027 0.0059482434828329106 0 +696 1 3.89277768 0.9948068 0.0075116878874550473 1 +697 1 2.35095572 0.9552115 0.066107858757860072 1 +698 1 2.92370939 0.9796768 0.029622243979937089 1 diff --git a/test/BaselineOutput/Common/WeightedEnsemble/WE-Diverse-TrainTest-breast-cancer-out.txt b/test/BaselineOutput/Common/WeightedEnsemble/WE-Diverse-TrainTest-breast-cancer-out.txt new file mode 100644 index 0000000000..174f4eabce --- /dev/null +++ b/test/BaselineOutput/Common/WeightedEnsemble/WE-Diverse-TrainTest-breast-cancer-out.txt @@ -0,0 +1,119 @@ +maml.exe TrainTest test=%Data% tr=WeightedEnsemble{nm=20 pt=BestDiverseSelector tp=-} dout=%Output% loader=Text{col=Label:BL:0 col=Features:R4:1-9} data=%Data% out=%Output% seed=1 +Automatically adding a MinMax normalization transform, use 'norm=Warn' or 'norm=No' to turn this behavior off. +Training 20 learners for the batch 1 +Beginning training model 1 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 12 instances with missing features during training (over 1 iterations; 12 inst/iter) +Trainer 1 of 20 finished in %Time% +Beginning training model 2 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 18 instances with missing features during training (over 1 iterations; 18 inst/iter) +Trainer 2 of 20 finished in %Time% +Beginning training model 3 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 15 instances with missing features during training (over 1 iterations; 15 inst/iter) +Trainer 3 of 20 finished in %Time% +Beginning training model 4 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 24 instances with missing features during training (over 1 iterations; 24 inst/iter) +Trainer 4 of 20 finished in %Time% +Beginning training model 5 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 18 instances with missing features during training (over 1 iterations; 18 inst/iter) +Trainer 5 of 20 finished in %Time% +Beginning training model 6 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 15 instances with missing features during training (over 1 iterations; 15 inst/iter) +Trainer 6 of 20 finished in %Time% +Beginning training model 7 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 18 instances with missing features during training (over 1 iterations; 18 inst/iter) +Trainer 7 of 20 finished in %Time% +Beginning training model 8 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 14 instances with missing features during training (over 1 iterations; 14 inst/iter) +Trainer 8 of 20 finished in %Time% +Beginning training model 9 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 13 instances with missing features during training (over 1 iterations; 13 inst/iter) +Trainer 9 of 20 finished in %Time% +Beginning training model 10 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 14 instances with missing features during training (over 1 iterations; 14 inst/iter) +Trainer 10 of 20 finished in %Time% +Beginning training model 11 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 17 instances with missing features during training (over 1 iterations; 17 inst/iter) +Trainer 11 of 20 finished in %Time% +Beginning training model 12 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 24 instances with missing features during training (over 1 iterations; 24 inst/iter) +Trainer 12 of 20 finished in %Time% +Beginning training model 13 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 12 instances with missing features during training (over 1 iterations; 12 inst/iter) +Trainer 13 of 20 finished in %Time% +Beginning training model 14 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 12 instances with missing features during training (over 1 iterations; 12 inst/iter) +Trainer 14 of 20 finished in %Time% +Beginning training model 15 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 19 instances with missing features during training (over 1 iterations; 19 inst/iter) +Trainer 15 of 20 finished in %Time% +Beginning training model 16 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 23 instances with missing features during training (over 1 iterations; 23 inst/iter) +Trainer 16 of 20 finished in %Time% +Beginning training model 17 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 18 instances with missing features during training (over 1 iterations; 18 inst/iter) +Trainer 17 of 20 finished in %Time% +Beginning training model 18 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 20 instances with missing features during training (over 1 iterations; 20 inst/iter) +Trainer 18 of 20 finished in %Time% +Beginning training model 19 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 13 instances with missing features during training (over 1 iterations; 13 inst/iter) +Trainer 19 of 20 finished in %Time% +Beginning training model 20 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 15 instances with missing features during training (over 1 iterations; 15 inst/iter) +Trainer 20 of 20 finished in %Time% +Warning: 10 of 20 trainings failed. +Training calibrator. +Warning: The predictor produced non-finite prediction values on 16 instances during testing. Possible causes: abnormal data or the predictor is numerically unstable. +TEST POSITIVE RATIO: 0.3499 (239.0/(239.0+444.0)) +Confusion table + ||====================== +PREDICTED || positive | negative | Recall +TRUTH ||====================== + positive || 236 | 3 | 0.9874 + negative || 14 | 430 | 0.9685 + ||====================== +Precision || 0.9440 | 0.9931 | +OVERALL 0/1 ACCURACY: 0.975110 +LOG LOSS/instance: 0.114893 +Test-set entropy (prior Log-Loss/instance): 0.934003 +LOG-LOSS REDUCTION (RIG): 0.876989 +AUC: 0.996127 + +OVERALL RESULTS +--------------------------------------- +AUC: 0.996127 (0.0000) +Accuracy: 0.975110 (0.0000) +Positive precision: 0.944000 (0.0000) +Positive recall: 0.987448 (0.0000) +Negative precision: 0.993072 (0.0000) +Negative recall: 0.968468 (0.0000) +Log-loss: 0.114893 (0.0000) +Log-loss reduction: 0.876989 (0.0000) +F1 Score: 0.965235 (0.0000) +AUPRC: 0.992160 (0.0000) + +--------------------------------------- +Physical memory usage(MB): %Number% +Virtual memory usage(MB): %Number% +%DateTime% Time elapsed(s): %Number% + diff --git a/test/BaselineOutput/Common/WeightedEnsemble/WE-Diverse-TrainTest-breast-cancer-rp.txt b/test/BaselineOutput/Common/WeightedEnsemble/WE-Diverse-TrainTest-breast-cancer-rp.txt new file mode 100644 index 0000000000..a080a632bf --- /dev/null +++ b/test/BaselineOutput/Common/WeightedEnsemble/WE-Diverse-TrainTest-breast-cancer-rp.txt @@ -0,0 +1,4 @@ +WeightedEnsemble +AUC Accuracy Positive precision Positive recall Negative precision Negative recall Log-loss Log-loss reduction F1 Score AUPRC /pt /bp /nm Learner Name Train Dataset Test Dataset Results File Run Time Physical Memory Virtual Memory Command Line Settings +0.996127 0.97511 0.944 0.987448 0.993072 0.968468 0.114893 0.876989 0.965235 0.99216 BestDiverseSelector Microsoft.ML.Runtime.ComponentFactoryUtils+SimpleComponentFactory`1[Microsoft.ML.Trainers.LinearSvmTrainer] 20 WeightedEnsemble %Data% %Data% %Output% 99 0 0 maml.exe TrainTest test=%Data% tr=WeightedEnsemble{nm=20 pt=BestDiverseSelector tp=-} dout=%Output% loader=Text{col=Label:BL:0 col=Features:R4:1-9} data=%Data% out=%Output% seed=1 /pt:BestDiverseSelector;/bp:Microsoft.ML.Runtime.ComponentFactoryUtils+SimpleComponentFactory`1[Microsoft.ML.Trainers.LinearSvmTrainer];/nm:20 + diff --git a/test/BaselineOutput/Common/WeightedEnsemble/WE-Diverse-TrainTest-breast-cancer.txt b/test/BaselineOutput/Common/WeightedEnsemble/WE-Diverse-TrainTest-breast-cancer.txt new file mode 100644 index 0000000000..4e982f3a7a --- /dev/null +++ b/test/BaselineOutput/Common/WeightedEnsemble/WE-Diverse-TrainTest-breast-cancer.txt @@ -0,0 +1,700 @@ +Instance Label Score Probability Log-loss Assigned +0 0 -2.04737854 0.0177238826 0.025799472331098337 0 +1 0 2.47291875 0.9068278 3.4239567458634292 1 +2 0 -2.23222542 0.0137591986 0.019988155613655037 0 +3 0 2.197465 0.869003952 2.9324048019558933 1 +4 0 -1.9936024 0.0190748721 0.02778507238959756 0 +5 1 7.7100296 0.9999298 0.00010130133779963418 1 +6 0 -0.567204952 0.1239935 0.19098652192918114 0 +7 0 -2.607882 0.008203492 0.011883948460682877 0 +8 0 -2.59627414 0.008335969 0.01207666732006699 0 +9 0 -2.29782248 0.0125739044 0.0182553218399173 0 +10 0 -3.03867912 0.00452115247 0.0065374337765033447 0 +11 0 -2.94182587 0.00517012132 0.0074782568426424705 0 +12 1 0.11766386 0.268533 1.8968287492514777 1 +13 0 -2.418044 0.0106574772 0.015458008276805919 0 +14 1 5.68980837 0.998833 0.0016846073197354307 1 +15 1 1.06843853 0.5795736 0.78693628479695288 1 +16 0 -2.44690514 0.0102422182 0.01485258957583176 0 +17 0 -2.294839 0.0126255583 0.018330793653293341 0 +18 1 5.03585529 0.997105658 0.0041817073405913587 1 +19 0 -1.79991829 0.0248291865 0.036273147608133972 0 +20 1 4.504013 0.9939521 0.008751773519959434 1 +21 1 5.320127 0.998049438 0.0028168142040815242 1 +22 0 -2.6943655 0.00728006475 0.010541330820133778 0 +23 1 ? ? ? 0 +24 0 -3.03722 0.00453030039 0.0065506914300484933 0 +25 1 1.38843179 0.6827258 0.55062185009980469 1 +26 0 -2.640628 0.007840917 0.011356634344192673 0 +27 0 -2.20748162 0.0142343668 0.020683409794967259 0 +28 0 -2.94182587 0.00517012132 0.0074782568426424705 0 +29 0 -3.01946449 0.00464310637 0.0067141859607828848 0 +30 0 -2.77058172 0.006552259 0.0094840164338745887 0 +31 0 -2.78975964 0.00638080575 0.0092350517758326197 0 +32 1 4.49902153 0.9939102 0.0088125945357040453 1 +33 0 -2.6119566 0.008157486 0.011817029160998801 0 +34 0 -2.52093029 0.009248927 0.013405470573449966 0 +35 0 -2.94182587 0.00517012132 0.0074782568426424705 0 +36 1 5.722373 0.9988846 0.0016100537520699492 1 +37 0 -0.98325634 0.07350039 0.11013772182720209 0 +38 1 4.08761168 0.9892551 0.015585539758591687 1 +39 1 1.95095372 0.8247887 0.27790354582661531 1 +40 0 ? ? ? 0 +41 1 2.01599813 0.8374874 0.25586061034355817 1 +42 1 5.680566 0.9988179 0.0017063887081673129 1 +43 1 0.7670796 0.475432545 1.0726874309294687 1 +44 1 5.82098866 0.99902755 0.0014036310654402073 1 +45 0 -3.0401926 0.00451168325 0.0065237105968689729 0 +46 1 3.41673779 0.9731148 0.039318099666702321 1 +47 0 -3.18928623 0.00366943167 0.0053036074417511375 0 +48 0 -1.9936024 0.0190748721 0.02778507238959756 0 +49 1 3.8522532 0.985152841 0.021580527383708067 1 +50 1 1.77024317 0.7854404 0.34842631527596957 1 +51 1 0.3805077 0.3460825 1.5308120671247423 1 +52 1 3.47045159 0.9750027 0.036521871921773558 1 +53 1 4.73123932 0.9955844 0.0063844292279735017 1 +54 1 4.43732929 0.9933679 0.0095999519736853466 1 +55 1 3.00493288 0.953285336 0.069019989953097272 1 +56 1 4.181627 0.9905603 0.013683303467818451 1 +57 1 0.9078746 0.5243761 0.93132617851423405 1 +58 1 1.46517968 0.7053965 0.50349373178250445 1 +59 1 1.02548671 0.56494385 0.82382061018032626 1 +60 1 1.94173169 0.8229264 0.28116468499965946 1 +61 0 -2.87921214 0.00563816726 0.0081571733727525755 0 +62 1 4.94101429 0.9966986 0.0047707669354584798 1 +63 1 0.9907973 0.553043962 0.85453392988265275 1 +64 0 -3.18928623 0.00366943167 0.0053036074417511375 0 +65 1 1.86507654 0.8068433 0.30963961962567799 1 +66 0 -2.294839 0.0126255583 0.018330793653293341 0 +67 1 3.3161335 0.969199 0.045135175977866544 1 +68 1 5.92209053 0.999155045 0.0012195283254521378 1 +69 0 -2.882557 0.00561213167 0.0081193994748962241 0 +70 0 -1.91148376 0.02133485 0.031112767291121547 0 +71 1 4.87518072 0.9963831 0.0052275852216101789 1 +72 0 -1.33923519 0.04611062 0.068106122716522971 0 +73 1 5.40519857 0.9982668 0.0025026251426664169 1 +74 1 1.72873354 0.775545239 0.36671715524501497 1 +75 0 -2.23030519 0.0137955071 0.020041269536247113 0 +76 0 -2.690928 0.00731471833 0.010591692882261989 0 +77 0 -1.80208111 0.0247564167 0.036165493911918624 0 +78 0 -1.95079017 0.0202221628 0.029473437032121869 0 +79 0 -2.88521433 0.005591532 0.0080895133212544497 0 +80 0 -1.75376487 0.02643269 0.038647367111918011 0 +81 0 -2.29955626 0.0125439838 0.018211606519903376 0 +82 0 -1.89829588 0.0217214245 0.031682748100510146 0 +83 0 -1.62181365 0.03159224 0.046313453022438292 0 +84 1 5.6691103 0.9987989 0.0017338526666562308 1 +85 1 4.64856148 0.995048642 0.0071610425118938384 1 +86 1 1.82820678 0.798720956 0.32423652906546718 1 +87 1 4.39487934 0.9929672 0.010182048810300654 1 +88 0 -2.294839 0.0126255583 0.018330793653293341 0 +89 0 -2.72419119 0.00698615471 0.010114261933915616 0 +90 0 -3.03722 0.00453030039 0.0065506914300484933 0 +91 0 -2.71199274 0.00710491 0.010286805232323113 0 +92 0 -2.294839 0.0126255583 0.018330793653293341 0 +93 0 -3.18928623 0.00366943167 0.0053036074417511375 0 +94 0 -2.78975964 0.00638080575 0.0092350517758326197 0 +95 0 -3.03722 0.00453030039 0.0065506914300484933 0 +96 0 -2.94692755 0.00513373362 0.0074254886240401833 0 +97 0 -2.04737854 0.0177238826 0.025799472331098337 0 +98 1 5.306483 0.9980121 0.002870750987105551 1 +99 1 6.43594027 0.9995865 0.00059664518267542723 1 +100 1 4.097024 0.989393353 0.015383887082056685 1 +101 1 0.07914996 0.258136928 1.9537915522576559 1 +102 0 -2.11296129 0.0162029769 0.023567405352194089 0 +103 1 0.8716767 0.5118 0.96634797251099824 1 +104 1 6.93876362 0.999794543 0.00029644255293496616 1 +105 1 1.38289332 0.681053936 0.55415903701270686 1 +106 1 6.061452 0.9993039 0.0010046424074628471 1 +107 1 4.43647146 0.993360043 0.0096113786563774683 1 +108 0 -2.894959 0.00551663432 0.0079808550053923344 0 +109 1 4.17597961 0.990486562 0.013790692416560519 1 +110 0 -1.48958051 0.03773414 0.055492549820642605 0 +111 1 2.700789 0.930384338 0.10410128421251788 1 +112 1 5.263735 0.997890532 0.0030465334116011804 1 +113 1 5.8743515 0.999097049 0.0013032710723539808 1 +114 0 -1.49505317 0.03745858 0.055079470908099573 0 +115 0 -2.23229551 0.0137578752 0.019986219702198368 0 +116 0 -0.309020758 0.168561831 0.26631911551455556 0 +117 1 5.5910635 0.998661339 0.0019325729964333653 1 +118 0 -2.643249 0.007812594 0.011315450001444859 0 +119 0 -2.08314276 0.0168778878 0.024557472048392617 0 +120 0 -2.676042 0.007466682 0.010812562160887352 0 +121 0 -1.84630728 0.0233131647 0.034032044055551401 0 +122 1 6.49500275 0.9996191 0.00054958923818962543 1 +123 1 2.63005042 0.923732 0.11445377496427016 1 +124 1 5.20700741 0.9977177 0.0032964570619997535 1 +125 0 -3.18928623 0.00366943167 0.0053036074417511375 0 +126 1 5.048196 0.9971548 0.0041106465592691674 1 +127 0 -2.54229927 0.008980373 0.013014464711046104 0 +128 1 3.25244284 0.96644026 0.049247538522750517 1 +129 0 -2.44408369 0.0102820974 0.014910719687386845 0 +130 0 -1.91148376 0.02133485 0.031112767291121547 0 +131 0 -2.78975964 0.00638080575 0.0092350517758326197 0 +132 1 5.852737 0.9990695 0.0013430355175455993 1 +133 0 -2.542349 0.008979756 0.013013567178372232 0 +134 0 -2.637754 0.00787209347 0.011401968002373274 0 +135 0 -1.4709996 0.0386843272 0.056917839887372972 0 +136 0 -2.44690514 0.0102422182 0.01485258957583176 0 +137 0 -2.85225725 0.00585241336 0.0084680512460020736 0 +138 0 -2.291085 0.0126908468 0.018426192637472916 0 +139 0 ? ? ? 0 +140 0 -2.85225725 0.00585241336 0.0084680512460020736 0 +141 0 -3.09389186 0.0041881837 0.0060549603597100897 0 +142 1 2.70027018 0.930337548 0.10417384012856321 1 +143 0 -2.23229551 0.0137578752 0.019986219702198368 0 +144 0 -2.94182587 0.00517012132 0.0074782568426424705 0 +145 0 ? ? ? 0 +146 1 1.082717 0.584407449 0.77495352825680497 1 +147 0 -2.82260084 0.006097492 0.0088237500198337839 0 +148 0 -0.06651044 0.221253574 0.36077445768389027 0 +149 1 7.02314329 0.9998173 0.00026358749034727648 1 +150 0 -3.03867912 0.00452115247 0.0065374337765033447 0 +151 1 2.85223031 0.942856848 0.084889348389592542 1 +152 1 6.301532 0.9995015 0.00071932475169631535 1 +153 0 -2.09111547 0.016694773 0.024288782955882892 0 +154 0 -3.34135222 0.00297166174 0.0042935843640431078 0 +155 1 2.81801844 0.940237 0.088903662587376558 1 +156 0 -2.85832548 0.005803486 0.0083970499968287551 0 +157 0 -2.78975964 0.00638080575 0.0092350517758326197 0 +158 0 ? ? ? 0 +159 1 7.39132 0.999890566 0.00015788871285642869 1 +160 1 5.54496574 0.998572767 0.0020605330681013512 1 +161 0 -2.14282274 0.0155537138 0.022615603552370033 0 +162 0 -2.54229927 0.008980373 0.013014464711046104 0 +163 0 -2.07962 0.0169594269 0.024677132678855004 0 +164 0 ? ? ? 0 +165 0 -1.8417927 0.0234566443 0.034243997818103993 0 +166 1 5.265355 0.9978953 0.0030396395796851217 1 +167 1 4.63808346 0.9949763 0.0072659592572532905 1 +168 0 -2.54229927 0.008980373 0.013014464711046104 0 +169 0 -3.28407621 0.00321742147 0.0046492413037421175 0 +170 0 -2.85225725 0.00585241336 0.0084680512460020736 0 +171 0 -3.03722 0.00453030039 0.0065506914300484933 0 +172 0 -3.18928623 0.00366943167 0.0053036074417511375 0 +173 1 8.899683 0.9999866 1.9348177961999343E-05 1 +174 1 3.87216258 0.9855527 0.020995035751295892 1 +175 1 5.511031 0.9985038 0.002160170673398052 1 +176 0 -2.78975964 0.00638080575 0.0092350517758326197 0 +177 1 4.198026 0.990771353 0.013375939190817083 1 +178 0 -2.294839 0.0126255583 0.018330793653293341 0 +179 1 1.6457777 0.7548133 0.40580822530323118 1 +180 0 -3.03867912 0.00452115247 0.0065374337765033447 0 +181 0 -3.34135222 0.00297166174 0.0042935843640431078 0 +182 0 -1.79991829 0.0248291865 0.036273147608133972 0 +183 1 5.7833147 0.9989752 0.0014792069204608016 1 +184 1 4.073758 0.9890482 0.015887289140148555 1 +185 0 -2.801044 0.006282013 0.0090916156884600658 0 +186 1 3.94161868 0.9868663 0.019073459161910753 1 +187 1 7.94072533 0.999949038 7.3524456735390132E-05 1 +188 1 5.64296246 0.998754442 0.0017980807673267526 1 +189 0 -2.545905 0.008935825 0.012949614667469626 0 +190 1 7.577833 0.9999156 0.00012176885553963592 1 +191 1 6.715432 0.9997197 0.00040447389734465342 1 +192 0 -2.20748162 0.0142343668 0.020683409794967259 0 +193 0 -3.03722 0.00453030039 0.0065506914300484933 0 +194 0 -2.54229927 0.008980373 0.013014464711046104 0 +195 0 -2.294839 0.0126255583 0.018330793653293341 0 +196 0 4.070962 0.9890059 6.5071297365304561 1 +197 0 -1.58747911 0.0330872051 0.048542314904784174 0 +198 0 -3.34135222 0.00297166174 0.0042935843640431078 0 +199 0 -2.6943655 0.00728006475 0.010541330820133778 0 +200 1 6.135044 0.9993716 0.00090689152453107152 1 +201 1 5.993602 0.999235034 0.0011040349990695149 1 +202 0 -3.03722 0.00453030039 0.0065506914300484933 0 +203 0 -2.04737854 0.0177238826 0.025799472331098337 0 +204 0 -3.03722 0.00453030039 0.0065506914300484933 0 +205 1 7.526635 0.999909341 0.00013079873509009028 1 +206 1 4.440152 0.9933938 0.0095623830639942932 1 +207 0 -3.03867912 0.00452115247 0.0065374337765033447 0 +208 0 -3.03867912 0.00452115247 0.0065374337765033447 0 +209 0 -2.11447287 0.016169481 0.023518285855099855 0 +210 1 8.793967 0.999984443 2.244391051590061E-05 1 +211 1 6.09683561 0.9993373 0.00095636847641151471 1 +212 0 -3.03722 0.00453030039 0.0065506914300484933 0 +213 1 8.8057375 0.9999847 2.2099939904093242E-05 1 +214 1 8.300623 0.9999691 4.4544194232592452E-05 1 +215 1 4.877151 0.9963929 0.0052133452178541196 1 +216 0 -3.18928623 0.00366943167 0.0053036074417511375 0 +217 0 -3.03722 0.00453030039 0.0065506914300484933 0 +218 1 5.41219473 0.9982836 0.0024783336914946099 1 +219 0 -1.49211931 0.0376060642 0.055300543045032902 0 +220 0 -2.84831572 0.005884413 0.0085144896870024203 0 +221 1 7.255865 0.999867857 0.00019065536569229225 1 +222 1 -1.016855 0.0703792 3.8287071239494566 0 +223 1 3.44393635 0.9740875 0.037876757304410348 1 +224 1 5.86586666 0.9990863 0.0013187635830144183 1 +225 0 -3.18928623 0.00366943167 0.0053036074417511375 0 +226 1 6.03252363 0.9992753 0.0010458615343308805 1 +227 1 4.76296234 0.9957743 0.0061092720141472448 1 +228 0 -3.03867912 0.00452115247 0.0065374337765033447 0 +229 1 8.592083 0.999979436 2.9667312308287927E-05 1 +230 1 3.58230829 0.978528738 0.031313873475458186 1 +231 1 5.58637 0.9986526 0.0019452307211234221 1 +232 0 0.8061733 0.489015251 0.96864786330322716 1 +233 1 3.938344 0.9868071 0.019159987547040541 1 +234 0 -1.197134 0.0556320734 0.082579050533162085 0 +235 0 ? ? ? 0 +236 1 6.86996746 0.9997739 0.00032620197268999553 1 +237 1 4.499536 0.993914545 0.0088062787205957799 1 +238 1 7.84687328 0.999941945 8.3757982275361301E-05 1 +239 1 3.5478344 0.9774973 0.032835407454128211 1 +240 0 -0.9985498 0.07206418 0.10790307470622863 0 +241 0 -2.28697681 0.01276268 0.018531161535118423 0 +242 0 -2.78975964 0.00638080575 0.0092350517758326197 0 +243 0 -1.52390647 0.0360374525 0.05295099989787707 0 +244 0 -3.03722 0.00453030039 0.0065506914300484933 0 +245 0 -1.60443854 0.0323404223 0.047428497700885829 0 +246 1 7.20602226 0.9998584 0.00020432985822878773 1 +247 1 2.23633 0.875038862 0.1925810035433633 1 +248 0 -1.58226156 0.0333202854 0.048890127283682014 0 +249 0 ? ? ? 0 +250 0 -3.02542043 0.00460495846 0.006658894499375745 0 +251 1 5.27895737 0.997934639 0.0029827667232767618 1 +252 0 3.05429077 0.9562506 4.5145932212323601 1 +253 1 5.680566 0.9988179 0.0017063887081673129 1 +254 1 4.94101429 0.9966986 0.0047707669354584798 1 +255 1 3.004421 0.953253567 0.069068070143194535 1 +256 0 -2.85225725 0.00585241336 0.0084680512460020736 0 +257 0 -2.6943655 0.00728006475 0.010541330820133778 0 +258 0 -2.54229927 0.008980373 0.013014464711046104 0 +259 0 2.362196 0.8929678 3.2238834785886774 1 +260 1 5.836506 0.9990483 0.0013736772662807028 1 +261 1 7.400855 0.999892 0.00015582469664856058 1 +262 1 6.0569067 0.999299467 0.0010110102123613563 1 +263 1 5.328785 0.998072743 0.0027831262780034088 1 +264 1 3.61443543 0.9794482 0.029958901148311453 1 +265 0 -1.39853561 0.0426138639 0.062827180400112731 0 +266 1 5.025827 0.997065067 0.0042404386135787555 1 +267 1 2.07404375 0.8481838 0.23755114841845762 1 +268 1 5.246348 0.997839 0.0031210750729741259 1 +269 0 -3.03722 0.00453030039 0.0065506914300484933 0 +270 1 4.06228256 0.988873839 0.016141621337977419 1 +271 0 -2.04737854 0.0177238826 0.025799472331098337 0 +272 1 2.07404375 0.8481838 0.23755114841845762 1 +273 1 0.252223 0.3068632 1.7043325041573234 1 +274 0 -2.390283 0.0110726077 0.016063493650065177 0 +275 0 ? ? ? 0 +276 0 -2.6943655 0.00728006475 0.010541330820133778 0 +277 0 -3.18928623 0.00366943167 0.0053036074417511375 0 +278 0 -3.03722 0.00453030039 0.0065506914300484933 0 +279 1 4.30396175 0.9920262 0.011549857043328376 1 +280 0 -2.54229927 0.008980373 0.013014464711046104 0 +281 0 -2.6119566 0.008157486 0.011817029160998801 0 +282 1 3.124448 0.9601579 0.058656458492358454 1 +283 1 3.74873924 0.9828918 0.024895489158529745 1 +284 1 4.95330667 0.996754467 0.0046899284421530515 1 +285 1 8.533052 0.999977648 3.2247107427628552E-05 1 +286 1 9.164459 0.9999907 1.3414709134555599E-05 1 +287 0 -2.637754 0.00787209347 0.011401968002373274 0 +288 1 1.40058374 0.686377466 0.54292590600755375 1 +289 1 5.305091 0.9980083 0.0028762654044206908 1 +290 0 -3.34135222 0.00297166174 0.0042935843640431078 0 +291 0 -3.03722 0.00453030039 0.0065506914300484933 0 +292 1 ? ? ? 0 +293 1 3.817009 0.984418154 0.022656831948970259 1 +294 0 ? ? ? 0 +295 1 4.723201 0.995534956 0.0064561203587704171 1 +296 0 1.53064919 0.72396934 1.8570995738791416 1 +297 0 ? ? ? 0 +298 0 -1.29340613 0.04899838 0.072480293511325261 0 +299 1 4.35030842 0.9925205 0.010831177992832668 1 +300 1 4.514756 0.9940413 0.0086222670406965636 1 +301 0 -3.03722 0.00453030039 0.0065506914300484933 0 +302 1 9.238295 0.9999916 1.2124827835971334E-05 1 +303 0 -3.03722 0.00453030039 0.0065506914300484933 0 +304 1 3.92839932 0.9866257 0.019425269038353326 1 +305 1 5.529868 0.9985425 0.0021042797604629566 1 +306 0 -3.03722 0.00453030039 0.0065506914300484933 0 +307 0 -3.03722 0.00453030039 0.0065506914300484933 0 +308 1 4.37170029 0.9927383 0.010514632561486582 1 +309 0 -1.0508275 0.06734823 0.10058957651730065 0 +310 0 -2.88521433 0.005591532 0.0080895133212544497 0 +311 0 -3.34135222 0.00297166174 0.0042935843640431078 0 +312 1 2.77785635 0.9370181 0.093851184247058211 1 +313 0 -3.34135222 0.00297166174 0.0042935843640431078 0 +314 0 -3.22566366 0.00348892948 0.0050422624266321728 0 +315 0 ? ? ? 0 +316 1 2.5199604 0.9122135 0.13255656702535173 1 +317 1 5.817851 0.999023259 0.0014098284808447324 1 +318 0 -2.86738586 0.005731189 0.0082921428586188433 0 +319 0 1.67410088 0.7620343 2.0711744359007445 1 +320 1 4.265804 0.99159503 0.012177054661028959 1 +321 0 ? ? ? 0 +322 0 -2.54229927 0.008980373 0.013014464711046104 0 +323 1 3.6188314 0.979571 0.029778053372998929 1 +324 0 -3.03722 0.00453030039 0.0065506914300484933 0 +325 0 -2.05937243 0.0174356289 0.025376168641166517 0 +326 1 2.44324183 0.9032793 0.14675594005126763 1 +327 0 -3.18928623 0.00366943167 0.0053036074417511375 0 +328 1 3.577539 0.978388846 0.031520138310305965 1 +329 1 4.54132462 0.9942564 0.008310184294507467 1 +330 1 3.44708133 0.974197745 0.037713450668998637 1 +331 0 -1.902767 0.0215896033 0.03148836066247683 0 +332 0 -1.595554 0.0327295847 0.048008821276208868 0 +333 1 3.35587788 0.97080797 0.042742142957695302 1 +334 1 4.05640936 0.988783538 0.016273370001408004 1 +335 0 -3.34135222 0.00297166174 0.0042935843640431078 0 +336 1 3.87288427 0.985567033 0.020974095452784006 1 +337 0 -3.03722 0.00453030039 0.0065506914300484933 0 +338 0 -3.22566366 0.00348892948 0.0050422624266321728 0 +339 1 3.84908175 0.98508817 0.021675237206155249 1 +340 1 3.82445979 0.9845764 0.022424929866152531 1 +341 0 -3.03722 0.00453030039 0.0065506914300484933 0 +342 0 -3.09389186 0.0041881837 0.0060549603597100897 0 +343 0 -3.34135222 0.00297166174 0.0042935843640431078 0 +344 1 5.97869349 0.999219 0.0011271845598241779 1 +345 0 -3.34135222 0.00297166174 0.0042935843640431078 0 +346 0 -1.52680278 0.0358977 0.052741854664736895 0 +347 0 -3.14231443 0.00391633 0.0056611628559683864 0 +348 1 0.3466804 0.335507751 1.5755820006083268 1 +349 1 2.43482924 0.902251661 0.14839820111499399 1 +350 0 -2.08554125 0.0168225933 0.024476331736895146 0 +351 0 -2.78975964 0.00638080575 0.0092350517758326197 0 +352 0 1.07871032 0.583052635 1.2620628247057244 1 +353 1 5.733636 0.998901963 0.0015850025519741158 1 +354 0 -3.18928623 0.00366943167 0.0053036074417511375 0 +355 0 -2.23827744 0.01364538 0.019821668549198233 0 +356 1 -0.234364033 0.183627471 2.4451461876410403 0 +357 1 7.83929157 0.999941349 8.4617945711076365E-05 1 +358 1 4.713477 0.9954744 0.0065438820626155299 1 +359 1 3.27005482 0.967226148 0.048074848462549093 1 +360 1 8.971377 0.999987841 1.7542337039908934E-05 1 +361 1 4.22844028 0.9911504 0.012824132491149505 1 +362 0 -1.71194518 0.0279723145 0.040930689333086939 0 +363 0 -0.7628894 0.09731157 0.14769998076866139 0 +364 0 -2.78975964 0.00638080575 0.0092350517758326197 0 +365 0 -2.94182587 0.00517012132 0.0074782568426424705 0 +366 1 8.280231 0.999968231 4.5834104516944174E-05 1 +367 1 6.90556049 0.9997848 0.00031046208749865417 1 +368 0 -3.01946449 0.00464310637 0.0067141859607828848 0 +369 0 -2.92178 0.005315599 0.0076892429823570072 0 +370 0 -1.9418118 0.0204712059 0.029840192548272845 0 +371 0 -3.01946449 0.00464310637 0.0067141859607828848 0 +372 0 -2.291085 0.0126908468 0.018426192637472916 0 +373 0 -2.06721687 0.0172495954 0.025103042239526984 0 +374 0 -2.52093029 0.009248927 0.013405470573449966 0 +375 0 -3.34135222 0.00297166174 0.0042935843640431078 0 +376 0 -3.18928623 0.00366943167 0.0053036074417511375 0 +377 0 -3.22566366 0.00348892948 0.0050422624266321728 0 +378 0 -2.00266838 0.0188402347 0.027440020827541073 0 +379 0 -1.15734541 0.05861365 0.08714116139764444 0 +380 0 -3.34135222 0.00297166174 0.0042935843640431078 0 +381 1 5.85124 0.999067545 0.001345875876998962 1 +382 0 -1.89829934 0.0217213221 0.031682597021271006 0 +383 0 -3.09389186 0.0041881837 0.0060549603597100897 0 +384 0 -3.09389186 0.0041881837 0.0060549603597100897 0 +385 0 -1.78663445 0.0252807327 0.036941332371188601 0 +386 1 3.75633168 0.9830685 0.024636110291618839 1 +387 0 -1.2378608 0.0527284071 0.07814997375969425 0 +388 0 -2.7898097 0.0063803643 0.0092344108128599011 0 +389 0 -1.68799675 0.028892858 0.042297618222053281 0 +390 0 -3.028263 0.004586861 0.006632664834016591 0 +391 1 6.144244 0.9993796 0.00089536148735746776 1 +392 0 -2.6943655 0.00728006475 0.010541330820133778 0 +393 0 -3.42123985 0.00265982957 0.0038424353047165974 0 +394 0 -2.71722484 0.00705373054 0.01021244247732842 0 +395 0 -2.6943655 0.00728006475 0.010541330820133778 0 +396 0 -2.54229927 0.008980373 0.013014464711046104 0 +397 0 -2.6224122 0.008040601 0.01164702328661972 0 +398 0 -2.39326644 0.0110272393 0.015997309485299083 0 +399 0 -2.887312 0.005575325 0.0080659997844636094 0 +400 1 5.890465 0.9991171 0.0012743521642397035 1 +401 0 -2.85225725 0.00585241336 0.0084680512460020736 0 +402 0 -1.60995317 0.0321011245 0.047071769982367666 0 +403 0 -2.03545713 0.0180150326 0.026227155553833655 0 +404 0 -2.718207 0.007044165 0.010198544245299831 0 +405 0 -3.18928623 0.00366943167 0.0053036074417511375 0 +406 0 -2.15079927 0.015384661 0.02236787991526339 0 +407 0 -3.18928623 0.00366943167 0.0053036074417511375 0 +408 0 -1.88529754 0.0221091434 0.032254641448757788 0 +409 0 -2.52093029 0.009248927 0.013405470573449966 0 +410 0 -3.18928623 0.00366943167 0.0053036074417511375 0 +411 0 ? ? ? 0 +412 1 5.611967 0.998699665 0.0018772075195479271 1 +413 0 -1.87615442 0.02238591 0.032663016762892426 0 +414 1 4.108495 0.9895596 0.015141506576865124 1 +415 0 0.0145919323 0.241309479 0.39841658170323413 1 +416 1 5.51057148 0.998502851 0.0021615485969053547 1 +417 0 -3.18928623 0.00366943167 0.0053036074417511375 0 +418 0 -0.9928974 0.07259196 0.10872386160032099 0 +419 0 -2.62589216 0.008002068 0.011590981710096072 0 +420 0 -1.30970812 0.0479520224 0.070893816210781349 0 +421 1 7.29060745 0.9998741 0.00018162511149045948 1 +422 0 -1.32888556 0.0467482656 0.069070844049204452 0 +423 0 -1.91148376 0.02133485 0.031112767291121547 0 +424 0 -2.85225725 0.00585241336 0.0084680512460020736 0 +425 1 9.25426 0.999991834 1.1780859684453884E-05 1 +426 0 -1.24659336 0.0521247126 0.077230839675923188 0 +427 1 3.499035 0.975954056 0.035114862109105319 1 +428 0 -3.18928623 0.00366943167 0.0053036074417511375 0 +429 0 -2.94182587 0.00517012132 0.0074782568426424705 0 +430 0 -2.73910427 0.006843646 0.0099072338292299257 0 +431 0 -1.46009338 0.0392527021 0.057771081240944405 0 +432 0 -2.04742861 0.01772267 0.025797691380797363 0 +433 0 -2.25999451 0.0132445442 0.019235503979043885 0 +434 0 3.63925576 0.9801321 5.6534170147023035 1 +435 1 4.94144964 0.996700644 0.0047678335491371602 1 +436 1 3.716631 0.982124 0.026022951164747229 1 +437 0 -2.6224122 0.008040601 0.01164702328661972 0 +438 0 -1.97648025 0.01952585 0.028448500609675353 0 +439 0 -2.39256668 0.0110378647 0.016012809790595932 0 +440 1 5.45971966 0.9983933 0.0023198463195799358 1 +441 0 -0.6862271 0.107093848 0.16341954504456024 0 +442 0 -2.47437119 0.009861916 0.014298358813977983 0 +443 0 -3.0462997 0.004473673 0.006468626183367349 0 +444 0 -1.37764764 0.04381566 0.064639318726934153 0 +445 0 -3.09389186 0.0041881837 0.0060549603597100897 0 +446 0 -3.34135222 0.00297166174 0.0042935843640431078 0 +447 0 -2.39256668 0.0110378647 0.016012809790595932 0 +448 0 -3.42123985 0.00265982957 0.0038424353047165974 0 +449 1 6.328022 0.9995195 0.00069334265387105138 1 +450 0 -2.21643353 0.0140606193 0.020429147912468595 0 +451 0 -2.39256668 0.0110378647 0.016012809790595932 0 +452 0 -2.57585335 0.008574196 0.012423286533003624 0 +453 1 5.0711 0.9972438 0.0039818146992060266 1 +454 0 -2.73911476 0.006843547 0.00990708974825792 0 +455 1 0.7481482 0.468866825 1.0927498907944446 1 +456 1 6.10904026 0.999348462 0.0009402775250079695 1 +457 1 5.77922249 0.9989694 0.0014876427398724263 1 +458 0 -2.246325 0.0134954676 0.019602415604665184 0 +459 0 -2.09448934 0.0166178737 0.024175961519552487 0 +460 0 -2.08554125 0.0168225933 0.024476331736895146 0 +461 0 -1.734099 0.027146155 0.039705014395772592 0 +462 0 -1.8403852 0.0235015526 0.034310344671677304 0 +463 0 -2.44695473 0.0102415187 0.014851570079828461 0 +464 0 -2.6224122 0.008040601 0.01164702328661972 0 +465 1 6.2480855 0.999463 0.00077490392271068191 1 +466 1 5.70765543 0.998861551 0.0016433699395319865 1 +467 1 4.72297144 0.9955335 0.0064581934083200411 1 +468 0 -2.6224122 0.008040601 0.01164702328661972 0 +469 0 -3.00740838 0.00472129136 0.0068275136895167305 0 +470 0 -2.77058172 0.006552259 0.0094840164338745887 0 +471 0 -1.8403852 0.0235015526 0.034310344671677304 0 +472 0 -2.22161841 0.0139609445 0.020283304190196531 0 +473 0 -2.6224122 0.008040601 0.01164702328661972 0 +474 0 -2.39256668 0.0110378647 0.016012809790595932 0 +475 0 -2.85225725 0.00585241336 0.0084680512460020736 0 +476 0 -2.47617 0.009837503 0.014262786931821135 0 +477 0 -2.6224122 0.008040601 0.01164702328661972 0 +478 0 -2.25228071 0.0133855706 0.019441707757735537 0 +479 1 5.404051 0.9982641 0.0025065876167646023 1 +480 0 -2.46338 0.0100123938 0.014517630918032659 0 +481 0 -1.46888447 0.0387939364 0.057082345359724056 0 +482 1 8.837951 0.9999854 2.1068028560732374E-05 1 +483 1 6.73024035 0.9997254 0.00039621643899194752 1 +484 0 -2.246325 0.0134954676 0.019602415604665184 0 +485 0 -2.57588172 0.008573861 0.012422798648663837 0 +486 0 -2.77058172 0.006552259 0.0094840164338745887 0 +487 1 7.95542336 0.999950051 7.2062530441079897E-05 1 +488 1 1.30265594 0.6563253 0.60751709026481338 1 +489 1 -0.298506021 0.1706225 2.5511202060259284 0 +490 0 -3.34135222 0.00297166174 0.0042935843640431078 0 +491 1 4.290637 0.9918782 0.011765105781268249 1 +492 0 -2.53522563 0.009068404 0.013142623737580432 0 +493 1 6.52004862 0.9996322 0.00053075008554014005 1 +494 0 0.6909015 0.449087381 0.86010458671906043 1 +495 0 -2.77058172 0.006552259 0.0094840164338745887 0 +496 0 -3.42123985 0.00265982957 0.0038424353047165974 0 +497 0 -2.48212624 0.009757092 0.014145631452214714 0 +498 0 -2.44690514 0.0102422182 0.01485258957583176 0 +499 0 -2.44690514 0.0102422182 0.01485258957583176 0 +500 0 -1.79991829 0.0248291865 0.036273147608133972 0 +501 0 -2.44690514 0.0102422182 0.01485258957583176 0 +502 0 -2.29955626 0.0125439838 0.018211606519903376 0 +503 0 -2.294839 0.0126255583 0.018330793653293341 0 +504 0 -3.34135222 0.00297166174 0.0042935843640431078 0 +505 0 -2.57168531 0.008623643 0.012495241793024017 0 +506 1 6.499568 0.9996215 0.00054614827870340832 1 +507 0 -2.51519537 0.009322345 0.013512383139799845 0 +508 0 -2.39256668 0.0110378647 0.016012809790595932 0 +509 0 -3.09389186 0.0041881837 0.0060549603597100897 0 +510 0 -3.34135222 0.00297166174 0.0042935843640431078 0 +511 0 -2.20748162 0.0142343668 0.020683409794967259 0 +512 0 -2.39256668 0.0110378647 0.016012809790595932 0 +513 0 -2.77058172 0.006552259 0.0094840164338745887 0 +514 1 6.503404 0.999623537 0.00054322346959286591 1 +515 1 5.645215 0.9987584 0.0017923982731479003 1 +516 0 -3.42123985 0.00265982957 0.0038424353047165974 0 +517 0 -3.22566366 0.00348892948 0.0050422624266321728 0 +518 0 -2.56079245 0.008754208 0.012685258173403121 0 +519 1 4.021574 0.988233 0.017076905925459875 1 +520 0 -3.284741 0.00321445637 0.0046449497698947779 0 +521 0 -2.553711 0.008840137 0.012810328369317418 0 +522 1 3.0336647 0.955033958 0.066376063197684826 1 +523 1 4.866767 0.996340632 0.0052890346073051773 1 +524 0 -2.6943655 0.00728006475 0.010541330820133778 0 +525 0 -2.71199274 0.00710491 0.010286805232323113 0 +526 0 -2.6224122 0.008040601 0.01164702328661972 0 +527 0 -2.294839 0.0126255583 0.018330793653293341 0 +528 0 -1.66020823 0.0299978741 0.043940185636835417 0 +529 0 -2.53522563 0.009068404 0.013142623737580432 0 +530 1 4.240906 0.991301239 0.01260456189165356 1 +531 0 -2.15079927 0.015384661 0.02236787991526339 0 +532 0 -3.03867912 0.00452115247 0.0065374337765033447 0 +533 0 -2.6943655 0.00728006475 0.010541330820133778 0 +534 0 -2.94182587 0.00517012132 0.0074782568426424705 0 +535 0 -2.5345695 0.009076613 0.013154574781378141 0 +536 0 -2.04737854 0.0177238826 0.025799472331098337 0 +537 0 -1.87615442 0.02238591 0.032663016762892426 0 +538 0 -2.44690514 0.0102422182 0.01485258957583176 0 +539 0 -1.9776361 0.01949508 0.028403224224603112 0 +540 0 -1.94183314 0.0204706118 0.029839317407316712 0 +541 0 -2.85225725 0.00585241336 0.0084680512460020736 0 +542 0 -2.11199474 0.0162244327 0.023598869695700037 0 +543 0 -2.44690514 0.0102422182 0.01485258957583176 0 +544 0 -2.46083283 0.0100475922 0.014568925971967094 0 +545 0 -2.20748162 0.0142343668 0.020683409794967259 0 +546 1 7.304617 0.9998765 0.00017818502951934642 1 +547 0 -3.1825974 0.00370361982 0.0053531130244719007 0 +548 0 -2.943955 0.005154905 0.00745619027886459 0 +549 1 3.97891927 0.987522364 0.018114674730542287 1 +550 0 -2.6943655 0.00728006475 0.010541330820133778 0 +551 0 -3.03722 0.00453030039 0.0065506914300484933 0 +552 0 -1.74623334 0.026703747 0.039049093893288735 0 +553 0 -0.526294 0.130310923 0.20142838018335987 0 +554 0 -2.85225725 0.00585241336 0.0084680512460020736 0 +555 0 -0.7119185 0.103722759 0.15798303120040108 0 +556 0 -1.81746626 0.02424473 0.035408744442801621 0 +557 0 -2.08554125 0.0168225933 0.024476331736895146 0 +558 0 -2.94182587 0.00517012132 0.0074782568426424705 0 +559 0 -2.20748162 0.0142343668 0.020683409794967259 0 +560 0 -2.04737854 0.0177238826 0.025799472331098337 0 +561 0 -2.04737854 0.0177238826 0.025799472331098337 0 +562 0 -3.03722 0.00453030039 0.0065506914300484933 0 +563 0 -2.6943655 0.00728006475 0.010541330820133778 0 +564 0 -2.14282274 0.0155537138 0.022615603552370033 0 +565 1 7.16997337 0.9998511 0.00021482232403052668 1 +566 0 -2.36042166 0.0115370071 0.016741140967116246 0 +567 0 -1.90603018 0.0214938875 0.031347231869337799 0 +568 1 2.89409471 0.9459159 0.080216207358231165 1 +569 1 6.17985 0.999409556 0.00085208164756794401 1 +570 1 5.10812855 0.997381866 0.0037821221842090753 1 +571 1 6.491644 0.999617338 0.00055216996319014454 1 +572 0 -2.6943655 0.00728006475 0.010541330820133778 0 +573 0 -3.18928623 0.00366943167 0.0053036074417511375 0 +574 1 3.992125 0.9877468 0.017786863866411901 1 +575 0 -1.87615442 0.02238591 0.032663016762892426 0 +576 0 -2.20748162 0.0142343668 0.020683409794967259 0 +577 0 -3.18928623 0.00366943167 0.0053036074417511375 0 +578 0 -3.18928623 0.00366943167 0.0053036074417511375 0 +579 0 -3.03722 0.00453030039 0.0065506914300484933 0 +580 0 -2.06123972 0.0173911676 0.0253108877242369 0 +581 1 5.29941225 0.9979925 0.0028990987629432608 1 +582 1 5.42194557 0.9983067 0.0024449982168388198 1 +583 0 -2.85225725 0.00585241336 0.0084680512460020736 0 +584 0 -1.54565477 0.03500072 0.051400227621388218 0 +585 0 -3.34135222 0.00297166174 0.0042935843640431078 0 +586 1 8.606468 0.9999798 2.9151353837995199E-05 1 +587 0 -2.04742861 0.01772267 0.025797691380797363 0 +588 1 3.54602885 0.977441967 0.032917046768677412 1 +589 0 -2.39256668 0.0110378647 0.016012809790595932 0 +590 1 2.377914 0.895040452 0.15997520661786083 1 +591 1 3.8423605 0.9849501 0.021877422017030815 1 +592 1 3.5951488 0.978900969 0.030765178590673237 1 +593 0 -2.246325 0.0134954676 0.019602415604665184 0 +594 1 3.40585065 0.972715557 0.039910103905156133 1 +595 0 -2.20748162 0.0142343668 0.020683409794967259 0 +596 0 -2.291085 0.0126908468 0.018426192637472916 0 +597 0 -1.72847843 0.0273534842 0.040012506258519334 0 +598 0 -2.6943655 0.00728006475 0.010541330820133778 0 +599 0 -1.7933042 0.0250530224 0.036604334769657937 0 +600 0 -2.6943655 0.00728006475 0.010541330820133778 0 +601 0 -3.22566366 0.00348892948 0.0050422624266321728 0 +602 0 -2.44690514 0.0102422182 0.01485258957583176 0 +603 1 2.87082672 0.944235265 0.082781729231646312 1 +604 1 3.41939855 0.973211467 0.039174775360667879 1 +605 1 5.77895832 0.998969 0.0014881592202143593 1 +606 0 -2.48214746 0.009756806 0.014145214898279929 0 +607 0 -3.34135222 0.00297166174 0.0042935843640431078 0 +608 1 6.688491 0.999709 0.00041987074278691019 1 +609 0 -2.39256668 0.0110378647 0.016012809790595932 0 +610 1 5.00963974 0.9969984 0.00433686312767524 1 +611 1 4.31553841 0.992152631 0.011366015145907061 1 +612 1 9.825112 0.9999963 5.3314720279485219E-06 1 +613 0 -2.64366 0.00780816143 0.011309004728405991 0 +614 0 -2.99812269 0.004782402 0.0069160985813799571 0 +615 0 -2.144843 0.015510723 0.022552602417899182 0 +616 0 -2.6943655 0.00728006475 0.010541330820133778 0 +617 0 ? ? ? 0 +618 0 -2.44690514 0.0102422182 0.01485258957583176 0 +619 0 -2.20748162 0.0142343668 0.020683409794967259 0 +620 0 -2.6943655 0.00728006475 0.010541330820133778 0 +621 0 0.019670248 0.242605671 0.40088347636857019 1 +622 0 -1.25011337 0.05188322 0.076863328400486736 0 +623 0 -3.34135222 0.00297166174 0.0042935843640431078 0 +624 0 -2.133325 0.0157573912 0.022914121467846424 0 +625 0 -1.67196858 0.02952531 0.043237506678639272 0 +626 1 3.74166822 0.9827255 0.025139601579231716 1 +627 0 -1.97563648 0.0195483416 0.028481595476469272 0 +628 0 -3.09389186 0.0041881837 0.0060549603597100897 0 +629 0 -2.6224122 0.008040601 0.01164702328661972 0 +630 0 -1.7073648 0.0281461459 0.041188714973528419 0 +631 0 -2.20748162 0.0142343668 0.020683409794967259 0 +632 0 -3.34135222 0.00297166174 0.0042935843640431078 0 +633 1 3.45576382 0.9744997 0.037266349905748723 1 +634 0 -2.85225725 0.00585241336 0.0084680512460020736 0 +635 0 -2.38104582 0.0112142544 0.016270149516243941 0 +636 1 6.220779 0.9994422 0.00080493133166807901 1 +637 0 -1.18547845 0.0564903878 0.083890879181998765 0 +638 0 -2.6224122 0.008040601 0.01164702328661972 0 +639 0 -2.08554125 0.0168225933 0.024476331736895146 0 +640 0 -2.3306973 0.0120183816 0.017443894490136364 0 +641 0 -2.6943655 0.00728006475 0.010541330820133778 0 +642 0 -2.6943655 0.00728006475 0.010541330820133778 0 +643 0 -3.34135222 0.00297166174 0.0042935843640431078 0 +644 0 -3.09389186 0.0041881837 0.0060549603597100897 0 +645 0 -2.6943655 0.00728006475 0.010541330820133778 0 +646 0 -3.02542043 0.00460495846 0.006658894499375745 0 +647 0 -3.06616545 0.004352222 0.0062926328026999918 0 +648 1 6.96032953 0.9998006 0.00028766966195538439 1 +649 0 -2.6943655 0.00728006475 0.010541330820133778 0 +650 0 -1.84634662 0.0233119186 0.034030203388783403 0 +651 0 -2.74729848 0.006766576 0.0097952831934717526 0 +652 0 -2.04742861 0.01772267 0.025797691380797363 0 +653 0 -2.44690514 0.0102422182 0.01485258957583176 0 +654 0 -2.54229927 0.008980373 0.013014464711046104 0 +655 0 -2.6943655 0.00728006475 0.010541330820133778 0 +656 0 -2.20748162 0.0142343668 0.020683409794967259 0 +657 0 0.831835747 0.497941464 0.99407251476360448 1 +658 1 5.53885 0.998560548 0.0020781865933664928 1 +659 0 -3.34135222 0.00297166174 0.0042935843640431078 0 +660 0 -3.18928623 0.00366943167 0.0053036074417511375 0 +661 0 -2.294839 0.0126255583 0.018330793653293341 0 +662 0 -2.8454957 0.005907414 0.0085478697392223392 0 +663 0 -2.8454957 0.005907414 0.0085478697392223392 0 +664 0 -2.407846 0.0108081568 0.015677751467757343 0 +665 0 -3.34135222 0.00297166174 0.0042935843640431078 0 +666 0 -1.793142 0.0250585377 0.036612496144092219 0 +667 0 -2.54229927 0.008980373 0.013014464711046104 0 +668 1 2.523135 0.9125666 0.13199823934917135 1 +669 1 5.43146658 0.9983289 0.0024128694056195776 1 +670 1 4.380082 0.992821932 0.010393109348722045 1 +671 0 -2.23247933 0.0137544041 0.019981142204336683 0 +672 0 -2.78975964 0.00638080575 0.0092350517758326197 0 +673 0 -1.89705169 0.0217582472 0.031737052497711728 0 +674 0 -3.18928623 0.00366943167 0.0053036074417511375 0 +675 0 -2.1446867 0.0155140469 0.022557473337486086 0 +676 0 -3.00740838 0.00472129136 0.0068275136895167305 0 +677 0 -2.39256668 0.0110378647 0.016012809790595932 0 +678 0 -3.34135222 0.00297166174 0.0042935843640431078 0 +679 0 -3.09389186 0.0041881837 0.0060549603597100897 0 +680 1 9.80118752 0.9999962 5.5034553246245386E-06 1 +681 1 6.57088566 0.9996573 0.00049453487122621187 1 +682 0 -1.89536238 0.0218083411 0.031810932175038151 0 +683 0 -3.34135222 0.00297166174 0.0042935843640431078 0 +684 0 -3.34135222 0.00297166174 0.0042935843640431078 0 +685 0 -3.34135222 0.00297166174 0.0042935843640431078 0 +686 0 -3.34135222 0.00297166174 0.0042935843640431078 0 +687 0 -2.390944 0.011062541 0.016048807986696743 0 +688 0 -2.6224122 0.008040601 0.01164702328661972 0 +689 0 -2.569418 0.00865066051 0.012534559642738887 0 +690 0 -3.06616545 0.004352222 0.0062926328026999918 0 +691 1 3.44010854 0.9739527 0.038076369600267737 1 +692 0 -2.85225725 0.00585241336 0.0084680512460020736 0 +693 0 -2.55013871 0.008883801 0.012873885632171393 0 +694 0 -2.51790786 0.009287548 0.013461710163383293 0 +695 0 -3.09389186 0.0041881837 0.0060549603597100897 0 +696 1 5.02741051 0.9970715 0.004231124243362026 1 +697 1 3.06741762 0.9570085 0.063396384931027977 1 +698 1 3.59143019 0.9787938 0.030923132124325001 1 diff --git a/test/BaselineOutput/Common/WeightedEnsemble/WE-Hetero-TrainTest-breast-cancer-out.txt b/test/BaselineOutput/Common/WeightedEnsemble/WE-Hetero-TrainTest-breast-cancer-out.txt new file mode 100644 index 0000000000..4efbdc50df --- /dev/null +++ b/test/BaselineOutput/Common/WeightedEnsemble/WE-Hetero-TrainTest-breast-cancer-out.txt @@ -0,0 +1,118 @@ +maml.exe TrainTest test=%Data% tr=WeightedEnsemble{bp=svm bp=ap nm=20 tp=-} dout=%Output% loader=Text{col=Label:BL:0 col=Features:R4:1-9} data=%Data% out=%Output% seed=1 +Automatically adding a MinMax normalization transform, use 'norm=Warn' or 'norm=No' to turn this behavior off. +Training 20 learners for the batch 1 +Beginning training model 1 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 26 instances with missing features during training (over 1 iterations; 26 inst/iter) +Trainer 1 of 20 finished in %Time% +Beginning training model 2 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 15 instances with missing features during training (over 1 iterations; 15 inst/iter) +Trainer 2 of 20 finished in %Time% +Beginning training model 3 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 11 instances with missing features during training (over 1 iterations; 11 inst/iter) +Trainer 3 of 20 finished in %Time% +Beginning training model 4 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 12 instances with missing features during training (over 1 iterations; 12 inst/iter) +Trainer 4 of 20 finished in %Time% +Beginning training model 5 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 18 instances with missing features during training (over 1 iterations; 18 inst/iter) +Trainer 5 of 20 finished in %Time% +Beginning training model 6 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 15 instances with missing features during training (over 1 iterations; 15 inst/iter) +Trainer 6 of 20 finished in %Time% +Beginning training model 7 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 24 instances with missing features during training (over 1 iterations; 24 inst/iter) +Trainer 7 of 20 finished in %Time% +Beginning training model 8 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 18 instances with missing features during training (over 1 iterations; 18 inst/iter) +Trainer 8 of 20 finished in %Time% +Beginning training model 9 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 15 instances with missing features during training (over 1 iterations; 15 inst/iter) +Trainer 9 of 20 finished in %Time% +Beginning training model 10 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 18 instances with missing features during training (over 1 iterations; 18 inst/iter) +Trainer 10 of 20 finished in %Time% +Beginning training model 11 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 14 instances with missing features during training (over 1 iterations; 14 inst/iter) +Trainer 11 of 20 finished in %Time% +Beginning training model 12 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 13 instances with missing features during training (over 1 iterations; 13 inst/iter) +Trainer 12 of 20 finished in %Time% +Beginning training model 13 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 14 instances with missing features during training (over 1 iterations; 14 inst/iter) +Trainer 13 of 20 finished in %Time% +Beginning training model 14 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 17 instances with missing features during training (over 1 iterations; 17 inst/iter) +Trainer 14 of 20 finished in %Time% +Beginning training model 15 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 24 instances with missing features during training (over 1 iterations; 24 inst/iter) +Trainer 15 of 20 finished in %Time% +Beginning training model 16 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 12 instances with missing features during training (over 1 iterations; 12 inst/iter) +Trainer 16 of 20 finished in %Time% +Beginning training model 17 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 12 instances with missing features during training (over 1 iterations; 12 inst/iter) +Trainer 17 of 20 finished in %Time% +Beginning training model 18 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 19 instances with missing features during training (over 1 iterations; 19 inst/iter) +Trainer 18 of 20 finished in %Time% +Beginning training model 19 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 23 instances with missing features during training (over 1 iterations; 23 inst/iter) +Trainer 19 of 20 finished in %Time% +Beginning training model 20 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 18 instances with missing features during training (over 1 iterations; 18 inst/iter) +Trainer 20 of 20 finished in %Time% +Training calibrator. +Warning: The predictor produced non-finite prediction values on 16 instances during testing. Possible causes: abnormal data or the predictor is numerically unstable. +TEST POSITIVE RATIO: 0.3499 (239.0/(239.0+444.0)) +Confusion table + ||====================== +PREDICTED || positive | negative | Recall +TRUTH ||====================== + positive || 232 | 7 | 0.9707 + negative || 11 | 433 | 0.9752 + ||====================== +Precision || 0.9547 | 0.9841 | +OVERALL 0/1 ACCURACY: 0.973646 +LOG LOSS/instance: 0.119488 +Test-set entropy (prior Log-Loss/instance): 0.934003 +LOG-LOSS REDUCTION (RIG): 0.872069 +AUC: 0.995872 + +OVERALL RESULTS +--------------------------------------- +AUC: 0.995872 (0.0000) +Accuracy: 0.973646 (0.0000) +Positive precision: 0.954733 (0.0000) +Positive recall: 0.970711 (0.0000) +Negative precision: 0.984091 (0.0000) +Negative recall: 0.975225 (0.0000) +Log-loss: 0.119488 (0.0000) +Log-loss reduction: 0.872069 (0.0000) +F1 Score: 0.962656 (0.0000) +AUPRC: 0.991591 (0.0000) + +--------------------------------------- +Physical memory usage(MB): %Number% +Virtual memory usage(MB): %Number% +%DateTime% Time elapsed(s): %Number% + diff --git a/test/BaselineOutput/Common/WeightedEnsemble/WE-Hetero-TrainTest-breast-cancer-rp.txt b/test/BaselineOutput/Common/WeightedEnsemble/WE-Hetero-TrainTest-breast-cancer-rp.txt new file mode 100644 index 0000000000..35884c4374 --- /dev/null +++ b/test/BaselineOutput/Common/WeightedEnsemble/WE-Hetero-TrainTest-breast-cancer-rp.txt @@ -0,0 +1,4 @@ +WeightedEnsemble +AUC Accuracy Positive precision Positive recall Negative precision Negative recall Log-loss Log-loss reduction F1 Score AUPRC /bp /nm Learner Name Train Dataset Test Dataset Results File Run Time Physical Memory Virtual Memory Command Line Settings +0.995872 0.973646 0.954733 0.970711 0.984091 0.975225 0.119488 0.872069 0.962656 0.991591 svm,ap 20 WeightedEnsemble %Data% %Data% %Output% 99 0 0 maml.exe TrainTest test=%Data% tr=WeightedEnsemble{bp=svm bp=ap nm=20 tp=-} dout=%Output% loader=Text{col=Label:BL:0 col=Features:R4:1-9} data=%Data% out=%Output% seed=1 /bp:svm,ap;/nm:20 + diff --git a/test/BaselineOutput/Common/WeightedEnsemble/WE-Hetero-TrainTest-breast-cancer.txt b/test/BaselineOutput/Common/WeightedEnsemble/WE-Hetero-TrainTest-breast-cancer.txt new file mode 100644 index 0000000000..a480b459d5 --- /dev/null +++ b/test/BaselineOutput/Common/WeightedEnsemble/WE-Hetero-TrainTest-breast-cancer.txt @@ -0,0 +1,700 @@ +Instance Label Score Probability Log-loss Assigned +0 0 -2.64253736 0.018180633 0.026470469789084766 0 +1 0 2.413429 0.931832552 3.8747732152900811 1 +2 0 -2.90884018 0.0129081747 0.018743795610062966 0 +3 0 2.35082054 0.9264507 3.7651451550773323 1 +4 0 -2.645568 0.0181101058 0.026366840155240968 0 +5 1 7.87455273 0.999941647 8.4187963929142817E-05 1 +6 0 -0.963278532 0.1423926 0.22161074984231105 0 +7 0 -3.258512 0.008214119 0.011899407344248809 0 +8 0 -3.394763 0.006884138 0.0099660552425570367 0 +9 0 -2.84032249 0.0140996873 0.020486316132759173 0 +10 0 -3.75712681 0.00429946231 0.006216185673502031 0 +11 0 -3.59146929 0.005332595 0.0077138945216321175 0 +12 1 -0.241472483 0.298863351 1.7424421000942203 0 +13 0 -3.05513287 0.0106868846 0.01550089190887341 0 +14 1 5.44284058 0.998603344 0.0020163571441931556 1 +15 1 0.888972759 0.6511053 0.61903724670382432 1 +16 0 -3.041843 0.0108719906 0.015770853378051553 0 +17 0 -2.89621615 0.0131199779 0.01905339221272959 0 +18 1 4.947694 0.9973367 0.0038474761908028369 1 +19 0 -2.40246773 0.024711486 0.036099028626778586 0 +20 1 3.59601426 0.984630942 0.022345017445622391 1 +21 1 4.5730896 0.995662868 0.0062707672187045935 1 +22 0 -3.325173 0.00753429625 0.010910846422542917 0 +23 1 ? ? ? 0 +24 0 -3.67959213 0.004755537 0.0068771549577151874 0 +25 1 1.03312182 0.692576468 0.52995472567393553 1 +26 0 -3.19219923 0.00895069353 0.012971259043588729 0 +27 0 -2.80222845 0.0148083763 0.021523733114509613 0 +28 0 -3.59146929 0.005332595 0.0077138945216321175 0 +29 0 -3.46614027 0.006275154 0.0090816581203470217 0 +30 0 -3.38692 0.006954534 0.010068322651344731 0 +31 0 -3.460084 0.006324678 0.0091535591332347361 0 +32 1 4.225658 0.9931889 0.0098599313987153703 1 +33 0 -3.298637 0.007797957 0.011294166964821722 0 +34 0 -3.09268355 0.0101805525 0.014762707006017898 0 +35 0 -3.59146929 0.005332595 0.0077138945216321175 0 +36 1 5.75082064 0.9990655 0.0013488023137997387 1 +37 0 -1.726836 0.05770703 0.085752414897518253 0 +38 1 3.67166066 0.9860568 0.020257335265567656 1 +39 1 1.56846046 0.819278359 0.28757438720284217 1 +40 0 ? ? ? 0 +41 1 1.36777544 0.7771831 0.36367353657155882 1 +42 1 5.758877 0.9990753 0.0013346866145589923 1 +43 1 0.642315149 0.5748606 0.79871600891754191 1 +44 1 5.41969347 0.998560548 0.0020781865933664928 1 +45 0 -3.71896315 0.00451823138 0.006533200396175093 0 +46 1 3.58104253 0.984332144 0.022782887023401029 1 +47 0 -3.797206 0.00408106064 0.0058997728235235976 0 +48 0 -2.645568 0.0181101058 0.026366840155240968 0 +49 1 4.0441 0.9913817 0.012487459669348331 1 +50 1 1.34481716 0.771946847 0.37342658104217413 1 +51 1 -0.001224041 0.368444115 1.4404822848940235 0 +52 1 2.98387218 0.966441 0.049246470794546728 1 +53 1 4.080678 0.99178046 0.011907293321752332 1 +54 1 4.072842 0.9916966 0.012029290997083615 1 +55 1 3.0226624 0.968046 0.046852476975754585 1 +56 1 4.5250926 0.9953835 0.0066756210194013794 1 +57 1 0.122701883 0.406845331 1.2974476599339293 1 +58 1 1.32592559 0.767573535 0.38162312403184928 1 +59 1 0.675116539 0.5852969 0.77275938617906303 1 +60 1 1.54005671 0.8137198 0.29739598304124809 1 +61 0 -3.53577328 0.005732704 0.0082943408462856517 0 +62 1 5.356804 0.998437464 0.0022560256159071365 1 +63 1 0.260388136 0.4508659 1.1492297124697148 1 +64 0 -3.797206 0.00408106064 0.0058997728235235976 0 +65 1 0.825839043 0.632145941 0.66167042822194788 1 +66 0 -2.89621615 0.0131199779 0.01905339221272959 0 +67 1 2.92859936 0.9640188 0.052866780660655102 1 +68 1 5.571316 0.9988189 0.0017050112193161702 1 +69 0 -3.553704 0.00560073834 0.0081028696942604724 0 +70 0 -2.36935616 0.0257755835 0.037673953652738811 0 +71 1 4.137388 0.992362857 0.011060357258384971 1 +72 0 -1.851364 0.04947255 0.073199806971384104 0 +73 1 5.56853962 0.9988146 0.0017112099295037728 1 +74 1 1.43146157 0.7912562 0.33778321528564498 1 +75 0 -2.90632868 0.0129500413 0.018804987612508806 0 +76 0 -3.129921 0.009701905 0.014065230630468726 0 +77 0 -2.30234623 0.0280672312 0.041071572663400106 0 +78 0 -2.59991312 0.0192015953 0.02797146249966953 0 +79 0 -3.52868485 0.00578572135 0.0083712718486629329 0 +80 0 -2.48073435 0.0223636888 0.032630224399319396 0 +81 0 -2.92444015 0.0126511 0.018368113945997116 0 +82 0 -2.46059275 0.0229461938 0.033490081537254961 0 +83 0 -2.2203362 0.031142015 0.04564288370113366 0 +84 1 5.6200695 0.998891652 0.0015998954810425424 1 +85 1 4.16904354 0.99266994 0.010613989507829305 1 +86 1 1.75149965 0.852022946 0.23103581060666781 1 +87 1 4.41317654 0.994660735 0.0077235684359162789 1 +88 0 -2.89621615 0.0131199779 0.01905339221272959 0 +89 0 -3.36121845 0.00719026756 0.010410836710877457 0 +90 0 -3.67959213 0.004755537 0.0068771549577151874 0 +91 0 -3.28834772 0.007902635 0.01144638079485033 0 +92 0 -2.89621615 0.0131199779 0.01905339221272959 0 +93 0 -3.797206 0.00408106064 0.0058997728235235976 0 +94 0 -3.460084 0.006324678 0.0091535591332347361 0 +95 0 -3.67959213 0.004755537 0.0068771549577151874 0 +96 0 -3.52607131 0.00580539228 0.0083998164332397037 0 +97 0 -2.64253736 0.018180633 0.026470469789084766 0 +98 1 5.258258 0.998223245 0.0025655953122849296 1 +99 1 6.203556 0.999482453 0.0007468559619760001 1 +100 1 2.99701619 0.966993332 0.04842215353553226 1 +101 1 -0.180073977 0.3159351 1.6622998440364622 0 +102 0 -2.710878 0.0166538674 0.024228768013196206 0 +103 1 0.155122519 0.417103946 1.2615211332975855 1 +104 1 6.905554 0.999793053 0.00029859277945233756 1 +105 1 0.5579276 0.547724545 0.86847756204839288 1 +106 1 6.02159834 0.999343753 0.00094707528472289416 1 +107 1 4.866437 0.9970393 0.0042776966958144322 1 +108 0 -3.54592848 0.005657589 0.0081853522596192843 0 +109 1 3.91270041 0.9897844 0.01481376235531935 1 +110 0 -2.12017679 0.0353410542 0.051909125743970649 0 +111 1 2.61504078 0.946775138 0.078906272432797561 1 +112 1 4.16734171 0.9926537 0.010637552054229617 1 +113 1 6.017274 0.99934 0.00095249630593295686 1 +114 0 -1.9839282 0.041936487 0.061806794968352646 0 +115 0 -2.750092 0.0158354621 0.023028561728026701 0 +116 0 -0.726025343 0.18457523 0.29437631272131198 0 +117 1 4.60016727 0.995812953 0.0060533142568229429 1 +118 0 -3.17290115 0.009177091 0.013300868515968883 0 +119 0 -2.69433546 0.0170114543 0.024753489308050113 0 +120 0 -3.28179455 0.00797003 0.011544389283261123 0 +121 0 -2.444891 0.02341055 0.03417590121568876 0 +122 1 6.67553234 0.9997206 0.00040318366636157984 1 +123 1 2.3730526 0.928405166 0.10717354498708642 1 +124 1 4.42850161 0.994766 0.0075709006563936081 1 +125 0 -3.797206 0.00408106064 0.0058997728235235976 0 +126 1 4.30476665 0.99385345 0.0088949622177072005 1 +127 0 -3.172824 0.009178008 0.013302204234837588 0 +128 1 3.25639486 0.976253569 0.034672177214149419 1 +129 0 -3.19313335 0.008939876 0.012955512100747579 0 +130 0 -2.36935616 0.0257755835 0.037673953652738811 0 +131 0 -3.460084 0.006324678 0.0091535591332347361 0 +132 1 6.409749 0.999604642 0.00057049324344129223 1 +133 0 -3.20961618 0.008751121 0.012680766117625759 0 +134 0 -3.26757 0.008118288 0.011760013787328914 0 +135 0 -2.05151463 0.0385294519 0.056685429535903428 0 +136 0 -3.041843 0.0108719906 0.015770853378051553 0 +137 0 -3.42639732 0.00660731271 0.0095639682531163638 0 +138 0 -2.83381081 0.0142184142 0.020660062880562078 0 +139 0 ? ? ? 0 +140 0 -3.42639732 0.00660731271 0.0095639682531163638 0 +141 0 -3.686916 0.004710472 0.0068118309624495416 0 +142 1 2.28908873 0.920763731 0.11909708823403051 1 +143 0 -2.750092 0.0158354621 0.023028561728026701 0 +144 0 -3.59146929 0.005332595 0.0077138945216321175 0 +145 0 ? ? ? 0 +146 1 0.9915428 0.6808934 0.55449910102348887 1 +147 0 -3.41126871 0.00673829252 0.0097542015649444234 0 +148 0 -1.07589114 0.125356361 0.19323276406618903 0 +149 1 6.83185673 0.9997722 0.00032869628719892736 1 +150 0 -3.75712681 0.00429946231 0.006216185673502031 0 +151 1 2.67483783 0.9505766 0.073125201932120631 1 +152 1 6.1254425 0.9994269 0.00082704360545659173 1 +153 0 -2.58047557 0.0196856428 0.028683642741783484 0 +154 0 -3.91345024 0.00350815617 0.0050700980595032802 0 +155 1 2.59009027 0.9451087 0.081447809111222763 1 +156 0 -3.39209557 0.006908 0.010000719708995957 0 +157 0 -3.460084 0.006324678 0.0091535591332347361 0 +158 0 ? ? ? 0 +159 1 7.540417 0.9999097 0.00013028274045038382 1 +160 1 5.78493 0.999106169 0.0012901025690979656 1 +161 0 -2.75849843 0.0156652387 0.022779051375408362 0 +162 0 -3.172824 0.009178008 0.013302204234837588 0 +163 0 -2.72536421 0.0163468029 0.023778335282224099 0 +164 0 ? ? ? 0 +165 0 -2.47238731 0.0226033144 0.032983882499573201 0 +166 1 5.58161736 0.9988346 0.0016822828431565687 1 +167 1 4.408204 0.9946261 0.0077737984571249318 1 +168 0 -3.172824 0.009178008 0.013302204234837588 0 +169 0 -3.854062 0.00379006076 0.0054782899545107767 0 +170 0 -3.42639732 0.00660731271 0.0095639682531163638 0 +171 0 -3.67959213 0.004755537 0.0068771549577151874 0 +172 0 -3.797206 0.00408106064 0.0058997728235235976 0 +173 1 8.989049 0.9999864 1.9606155421105871E-05 1 +174 1 3.427541 0.9809199 0.027792764881826915 1 +175 1 5.63932371 0.9989192 0.0015601239555864484 1 +176 0 -3.460084 0.006324678 0.0091535591332347361 0 +177 1 4.036208 0.9912932 0.012616272636680408 1 +178 0 -2.89621615 0.0131199779 0.01905339221272959 0 +179 1 1.29458714 0.760190666 0.39556678423551223 1 +180 0 -3.75712681 0.00429946231 0.006216185673502031 0 +181 0 -3.91345024 0.00350815617 0.0050700980595032802 0 +182 0 -2.40246773 0.024711486 0.036099028626778586 0 +183 1 6.057196 0.9993735 0.00090413807444271715 1 +184 1 3.92090774 0.989892244 0.014656607087797329 1 +185 0 -3.53073215 0.005770359 0.0083489801256738608 0 +186 1 3.95252824 0.990297258 0.014066450374216536 1 +187 1 6.987734 0.999814153 0.00026814587055765224 1 +188 1 5.640044 0.9989202 0.0015586605221005036 1 +189 0 -3.05918741 0.0106310351 0.015419449950356186 0 +190 1 7.91984034 0.999945 7.9372176726815187E-05 1 +191 1 7.17181969 0.999853849 0.00021086613943841086 1 +192 0 -2.80222845 0.0148083763 0.021523733114509613 0 +193 0 -3.67959213 0.004755537 0.0068771549577151874 0 +194 0 -3.172824 0.009178008 0.013302204234837588 0 +195 0 -2.89621615 0.0131199779 0.01905339221272959 0 +196 0 3.61451125 0.9849923 6.0581557282874074 1 +197 0 -2.30915213 0.0278257318 0.040713145916813312 0 +198 0 -3.91345024 0.00350815617 0.0050700980595032802 0 +199 0 -3.325173 0.00753429625 0.010910846422542917 0 +200 1 6.17764568 0.9994647 0.00077249487400212185 1 +201 1 6.061877 0.9993773 0.00089863119003121797 1 +202 0 -3.67959213 0.004755537 0.0068771549577151874 0 +203 0 -2.64253736 0.018180633 0.026470469789084766 0 +204 0 -3.67959213 0.004755537 0.0068771549577151874 0 +205 1 7.81129837 0.9999366 9.1497671650749417E-05 1 +206 1 4.79876137 0.996766567 0.0046724154701213699 1 +207 0 -3.75712681 0.00429946231 0.006216185673502031 0 +208 0 -3.75712681 0.00429946231 0.006216185673502031 0 +209 0 -2.79424238 0.0149613367 0.021747742704448802 0 +210 1 8.792591 0.9999824 2.5367664027431829E-05 1 +211 1 6.36292171 0.9995797 0.00060645228210809876 1 +212 0 -3.67959213 0.004755537 0.0068771549577151874 0 +213 1 9.215306 0.999989867 1.4618599387059818E-05 1 +214 1 8.25581551 0.999964535 5.1165745927591405E-05 1 +215 1 5.048332 0.997664 0.0033741145708817495 1 +216 0 -3.797206 0.00408106064 0.0058997728235235976 0 +217 0 -3.67959213 0.004755537 0.0068771549577151874 0 +218 1 5.20468044 0.9980947 0.0027514207132078739 1 +219 0 -2.01349068 0.04041216 0.059513221380500345 0 +220 0 -3.54108477 0.00569329364 0.0082371570942808264 0 +221 1 5.83519745 0.999163 0.0012080818527807843 1 +222 1 -1.6515013 0.06329672 3.9817254390322585 0 +223 1 3.19611239 0.974358 0.03747611522450283 1 +224 1 5.713772 0.9990192 0.0014156816198384724 1 +225 0 -3.797206 0.00408106064 0.0058997728235235976 0 +226 1 5.899499 0.9992303 0.0011108335304303744 1 +227 1 4.90267 0.9971758 0.0040802053294082038 1 +228 0 -3.75712681 0.00429946231 0.006216185673502031 0 +229 1 8.03233051 0.9999525 6.8536714296662667E-05 1 +230 1 3.44485021 0.9813385 0.027177232199569145 1 +231 1 5.391059 0.9985058 0.0021573287103206188 1 +232 0 0.57379365 0.5528533 1.1611798217613218 1 +233 1 3.515334 0.982951939 0.024807216376976689 1 +234 0 -1.74194491 0.0566431843 0.084124535110112156 0 +235 0 ? ? ? 0 +236 1 6.91435337 0.999795437 0.00029515241856282071 1 +237 1 4.338744 0.99411875 0.0085098990948090161 1 +238 1 7.786206 0.9999345 9.4507562065679747E-05 1 +239 1 3.10871887 0.9713461 0.041942690714541979 1 +240 0 -1.53649414 0.07280983 0.1090628242847012 0 +241 0 -3.01180363 0.01130215 0.016398399684525101 0 +242 0 -3.460084 0.006324678 0.0091535591332347361 0 +243 0 -2.146171 0.0342015624 0.050205965271221897 0 +244 0 -3.67959213 0.004755537 0.0068771549577151874 0 +245 0 -2.2267797 0.0308890641 0.045266271816504271 0 +246 1 7.66384459 0.9999231 0.00011093307468276651 1 +247 1 1.94828129 0.8815925 0.18181612456010757 1 +248 0 -2.23169374 0.0306974947 0.044981114775220134 0 +249 0 ? ? ? 0 +250 0 -3.49286652 0.00606117 0.0087710278686882609 0 +251 1 4.547104 0.9955138 0.0064867845205387221 1 +252 0 2.58847547 0.9449992 4.1844040584434445 1 +253 1 5.758877 0.9990753 0.0013346866145589923 1 +254 1 5.356804 0.998437464 0.0022560256159071365 1 +255 1 3.2235527 0.975238442 0.036173098956949125 1 +256 0 -3.42639732 0.00660731271 0.0095639682531163638 0 +257 0 -3.325173 0.00753429625 0.010910846422542917 0 +258 0 -3.172824 0.009178008 0.013302204234837588 0 +259 0 2.6843257 0.9511556 4.3556631093779359 1 +260 1 5.308344 0.99833554 0.0024033084229980566 1 +261 1 7.316118 0.999878943 0.00017465895401192133 1 +262 1 6.191517 0.9994743 0.00075864292199269608 1 +263 1 5.008978 0.99754107 0.0035518548123979993 1 +264 1 3.3611052 0.9792261 0.030286064963170056 1 +265 0 -1.87710309 0.0479152575 0.070838105257584841 0 +266 1 5.109423 0.9978427 0.0031156458970751891 1 +267 1 1.501658 0.805997 0.31115360083563126 1 +268 1 4.882923 0.9971022 0.0041867093235616083 1 +269 0 -3.67959213 0.004755537 0.0068771549577151874 0 +270 1 3.960268 0.990393937 0.013925612741025259 1 +271 0 -2.64253736 0.018180633 0.026470469789084766 0 +272 1 1.501658 0.805997 0.31115360083563126 1 +273 1 -0.109868288 0.336080223 1.5731224452936896 0 +274 0 -3.04368734 0.0108461129 0.015733109847719653 0 +275 0 ? ? ? 0 +276 0 -3.325173 0.00753429625 0.010910846422542917 0 +277 0 -3.797206 0.00408106064 0.0058997728235235976 0 +278 0 -3.67959213 0.004755537 0.0068771549577151874 0 +279 1 3.727268 0.987020969 0.018847359633392936 1 +280 0 -3.172824 0.009178008 0.013302204234837588 0 +281 0 -3.298637 0.007797957 0.011294166964821722 0 +282 1 2.948954 0.96492976 0.051504167331237957 1 +283 1 3.47602415 0.9820698 0.026102542207520212 1 +284 1 4.49931335 0.9952262 0.0069036226288225599 1 +285 1 8.577406 0.9999767 3.3623000044204345E-05 1 +286 1 9.496314 0.999992967 1.0147012084681539E-05 1 +287 0 -3.26757 0.008118288 0.011760013787328914 0 +288 1 0.8100138 0.627326131 0.67271243645189849 1 +289 1 5.431204 0.998582 0.0020471854241160678 1 +290 0 -3.91345024 0.00350815617 0.0050700980595032802 0 +291 0 -3.67959213 0.004755537 0.0068771549577151874 0 +292 1 ? ? ? 0 +293 1 3.36491513 0.9793271 0.030137313011186571 1 +294 0 ? ? ? 0 +295 1 4.597705 0.9957995 0.0060728301417534106 1 +296 0 1.06758189 0.7020764 1.7469855650561286 1 +297 0 ? ? ? 0 +298 0 -1.76483035 0.05506684 0.081715809472052925 0 +299 1 3.765514 0.9876456 0.01793460897193587 1 +300 1 4.26839352 0.9935563 0.009326344579648576 1 +301 0 -3.67959213 0.004755537 0.0068771549577151874 0 +302 1 9.317425 0.9999911 1.2812764385033556E-05 1 +303 0 -3.67959213 0.004755537 0.0068771549577151874 0 +304 1 3.235935 0.975626051 0.035599813012792039 1 +305 1 5.61459446 0.9988837 0.0016113450628330982 1 +306 0 -3.67959213 0.004755537 0.0068771549577151874 0 +307 0 -3.67959213 0.004755537 0.0068771549577151874 0 +308 1 4.29657936 0.993787766 0.0089903138736592905 1 +309 0 -1.66301191 0.0624110959 0.092972598679328139 0 +310 0 -3.52868485 0.00578572135 0.0083712718486629329 0 +311 0 -3.91345024 0.00350815617 0.0050700980595032802 0 +312 1 2.63396358 0.948007047 0.077030311197794715 1 +313 0 -3.91345024 0.00350815617 0.0050700980595032802 0 +314 0 -3.901368 0.003563763 0.0051506065948178194 0 +315 0 ? ? ? 0 +316 1 2.564888 0.943375647 0.08409573623701791 1 +317 1 5.800872 0.9991246 0.0012635077231441711 1 +318 0 -3.48975 0.00608574459 0.0088066984672960889 0 +319 0 1.27006745 0.754303336 2.0250498264031993 1 +320 1 3.6912837 0.986404836 0.019748221836936301 1 +321 0 ? ? ? 0 +322 0 -3.172824 0.009178008 0.013302204234837588 0 +323 1 3.09693336 0.9709145 0.042583864416402155 1 +324 0 -3.67959213 0.004755537 0.0068771549577151874 0 +325 0 -2.619741 0.0187198557 0.027263026637813624 0 +326 1 1.86085272 0.8691423 0.20233570387554822 1 +327 0 -3.797206 0.00408106064 0.0058997728235235976 0 +328 1 3.00825977 0.9674589 0.04772771586111655 1 +329 1 4.11800575 0.9921686 0.011342787379363093 1 +330 1 3.08976483 0.9706489 0.042978574529766174 1 +331 0 -2.5752964 0.0198166221 0.028876413431907399 0 +332 0 -2.119359 0.0353774875 0.051963614628888079 0 +333 1 3.12981 0.972103 0.040818911441792879 1 +334 1 4.01187754 0.99101454 0.013021870054809568 1 +335 0 -3.91345024 0.00350815617 0.0050700980595032802 0 +336 1 3.73862481 0.9872097 0.018571557474666787 1 +337 0 -3.67959213 0.004755537 0.0068771549577151874 0 +338 0 -3.901368 0.003563763 0.0051506065948178194 0 +339 1 3.48462415 0.982266545 0.02581353124636715 1 +340 1 3.36833835 0.979417443 0.030004204432990587 1 +341 0 -3.67959213 0.004755537 0.0068771549577151874 0 +342 0 -3.686916 0.004710472 0.0068118309624495416 0 +343 0 -3.91345024 0.00350815617 0.0050700980595032802 0 +344 1 5.551338 0.9987877 0.0017500385672996087 1 +345 0 -3.91345024 0.00350815617 0.0050700980595032802 0 +346 0 -1.98204291 0.04203554 0.061955959241040477 0 +347 0 -3.71652269 0.004532592 0.0065540124424156862 0 +348 1 0.09750867 0.398928851 1.3257966307010955 1 +349 1 1.8328588 0.864927 0.20934973099353968 1 +350 0 -2.69035482 0.0170986224 0.024881428250692977 0 +351 0 -3.460084 0.006324678 0.0091535591332347361 0 +352 0 0.712132454 0.5969824 1.3110853212452638 1 +353 1 6.05897 0.999375 0.00090198694521767769 1 +354 0 -3.797206 0.00408106064 0.0058997728235235976 0 +355 0 -2.85113049 0.0139047792 0.020201129910050325 0 +356 1 -0.7471242 0.180463284 2.4702227481494954 0 +357 1 7.554069 0.9999113 0.00012796076685551788 1 +358 1 4.136027 0.9923494 0.011079940993624731 1 +359 1 2.82321215 0.9589267 0.060507588045536687 1 +360 1 9.16846752 0.9999892 1.5564513861652592E-05 1 +361 1 3.93849158 0.9901195 0.014325412160840816 1 +362 0 -2.16504478 0.0333965048 0.049003883413042612 0 +363 0 -1.092989 0.122928075 0.18923293879010264 0 +364 0 -3.460084 0.006324678 0.0091535591332347361 0 +365 0 -3.59146929 0.005332595 0.0077138945216321175 0 +366 1 8.610756 0.9999777 3.2161114182661135E-05 1 +367 1 7.041212 0.99982667 0.00025008444858781921 1 +368 0 -3.46614027 0.006275154 0.0090816581203470217 0 +369 0 -3.4399128 0.00649243966 0.0093971488243909228 0 +370 0 -2.36282372 0.0259907227 0.037992581050396608 0 +371 0 -3.46614027 0.006275154 0.0090816581203470217 0 +372 0 -2.83381081 0.0142184142 0.020660062880562078 0 +373 0 -2.708228 0.0167106446 0.024312069786661519 0 +374 0 -3.09268355 0.0101805525 0.014762707006017898 0 +375 0 -3.91345024 0.00350815617 0.0050700980595032802 0 +376 0 -3.797206 0.00408106064 0.0058997728235235976 0 +377 0 -3.901368 0.003563763 0.0051506065948178194 0 +378 0 -2.671918 0.0175081063 0.025482590799572408 0 +379 0 -1.4557817 0.08025562 0.12069513938124539 0 +380 0 -3.91345024 0.00350815617 0.0050700980595032802 0 +381 1 5.6021924 0.9988655 0.0016376880546950159 1 +382 0 -2.43940973 0.0235748 0.034418564560554241 0 +383 0 -3.686916 0.004710472 0.0068118309624495416 0 +384 0 -3.686916 0.004710472 0.0068118309624495416 0 +385 0 -2.27399874 0.0290951878 0.042598234540939722 0 +386 1 3.416553 0.9806494 0.028190638803940113 1 +387 0 -1.601608 0.06727252 0.10047247461195524 0 +388 0 -3.46226025 0.006306838 0.009127657264707852 0 +389 0 -2.3267045 0.0272121821 0.039802932622544092 0 +390 0 -3.64286041 0.004988104 0.0072143209435000834 0 +391 1 6.327367 0.9995597 0.00063535780509387549 1 +392 0 -3.325173 0.00753429625 0.010910846422542917 0 +393 0 -3.99284053 0.00316367578 0.0045714544741969074 0 +394 0 -3.1629355 0.009296215 0.013474331111907453 0 +395 0 -3.325173 0.00753429625 0.010910846422542917 0 +396 0 -3.172824 0.009178008 0.013302204234837588 0 +397 0 -3.15739 0.009363164 0.013571827794801039 0 +398 0 -2.98382688 0.0117178857 0.017005163646952998 0 +399 0 -3.25783968 0.008221276 0.01190981857774033 0 +400 1 5.43135548 0.998582244 0.0020468409704219605 1 +401 0 -3.42639732 0.00660731271 0.0095639682531163638 0 +402 0 -1.94403183 0.0440810062 0.065039727929642077 0 +403 0 -2.547016 0.02054703 0.029951874767018584 0 +404 0 -3.31751251 0.007609488 0.011020152807588732 0 +405 0 -3.797206 0.00408106064 0.0058997728235235976 0 +406 0 -2.62275124 0.018647762 0.027157037155194675 0 +407 0 -3.797206 0.00408106064 0.0058997728235235976 0 +408 0 -2.37650633 0.0255420879 0.037328219561200096 0 +409 0 -3.09268355 0.0101805525 0.014762707006017898 0 +410 0 -3.797206 0.00408106064 0.0058997728235235976 0 +411 0 ? ? ? 0 +412 1 5.91613 0.999246836 0.0010869957827585933 1 +413 0 -2.45412827 0.023136273 0.033770775407914295 0 +414 1 3.97791266 0.9906107 0.013609863405458561 1 +415 0 -0.2743504 0.2899427 0.49399266693442428 0 +416 1 5.29900551 0.998315156 0.0024327667888107778 1 +417 0 -3.797206 0.00408106064 0.0058997728235235976 0 +418 0 -1.57721782 0.0692993551 0.10361088804592049 0 +419 0 -3.08934164 0.0102246348 0.01482695985496498 0 +420 0 -1.89076006 0.04710798 0.069615354560390988 0 +421 1 7.349641 0.9998841 0.00016717682233439332 1 +422 0 -1.75981271 0.05540888 0.08223812213173505 0 +423 0 -2.36935616 0.0257755835 0.037673953652738811 0 +424 0 -3.42639732 0.00660731271 0.0095639682531163638 0 +425 1 9.223721 0.99999 1.4446615003766504E-05 1 +426 0 -1.44948137 0.08086519 0.12165161610150116 0 +427 1 3.04042983 0.968756139 0.045794546822462187 1 +428 0 -3.797206 0.00408106064 0.0058997728235235976 0 +429 0 -3.59146929 0.005332595 0.0077138945216321175 0 +430 0 -3.35573435 0.00724158576 0.010485411351808081 0 +431 0 -2.16767645 0.0332857147 0.04883853410737421 0 +432 0 -2.6769495 0.0173954088 0.025317114854763057 0 +433 0 -2.71571636 0.0165506825 0.024077390379495509 0 +434 0 3.788616 0.9880085 6.3818439522215424 1 +435 1 4.953781 0.9973577 0.0038170405133381266 1 +436 1 3.58638954 0.9844395 0.022625560115205764 1 +437 0 -3.15739 0.009363164 0.013571827794801039 0 +438 0 -2.461308 0.0229252577 0.033459168061186997 0 +439 0 -2.927829 0.0125959255 0.018287496519426308 0 +440 1 4.393746 0.99452424 0.0079215592089817138 1 +441 0 -1.31148458 0.0953154638 0.1445132833099102 0 +442 0 -3.0032897 0.0114271 0.016580736565403341 0 +443 0 -3.63688326 0.00502700452 0.0072707248213631295 0 +444 0 -1.99953663 0.04112494 0.060585248302459357 0 +445 0 -3.686916 0.004710472 0.0068118309624495416 0 +446 0 -3.91345024 0.00350815617 0.0050700980595032802 0 +447 0 -2.927829 0.0125959255 0.018287496519426308 0 +448 0 -3.99284053 0.00316367578 0.0045714544741969074 0 +449 1 6.515028 0.9996554 0.00049720151870953893 1 +450 0 -2.89364481 0.0131635377 0.019117072620756067 0 +451 0 -2.927829 0.0125959255 0.018287496519426308 0 +452 0 -3.14916134 0.009463391 0.013717798963013226 0 +453 1 4.92834663 0.9972687 0.0039458576593420008 1 +454 0 -3.2191658 0.008643576 0.01252424961551832 0 +455 1 0.119318724 0.4057793 1.3012328165105893 1 +456 1 5.988446 0.9993147 0.0009889811690383533 1 +457 1 5.342599 0.998408258 0.002298227924273756 1 +458 0 -2.73630357 0.0161186177 0.023443701603127542 0 +459 0 -2.49767923 0.0218848456 0.031923769847254882 0 +460 0 -2.69035482 0.0170986224 0.024881428250692977 0 +461 0 -2.260905 0.0295822453 0.043322148696416027 0 +462 0 -2.430673 0.0238389336 0.034808882973551468 0 +463 0 -3.07287383 0.0104446346 0.015147666884820206 0 +464 0 -3.15739 0.009363164 0.013571827794801039 0 +465 1 6.00358534 0.999328136 0.00096961998291377079 1 +466 1 5.27833652 0.998269141 0.0024992656622821003 1 +467 1 4.511291 0.995299935 0.00679674508777063 1 +468 0 -3.15739 0.009363164 0.013571827794801039 0 +469 0 -3.56804562 0.00549736572 0.007952902358798624 0 +470 0 -3.38692 0.006954534 0.010068322651344731 0 +471 0 -2.430673 0.0238389336 0.034808882973551468 0 +472 0 -2.75715351 0.0156923514 0.022818789729069301 0 +473 0 -3.15739 0.009363164 0.013571827794801039 0 +474 0 -2.927829 0.0125959255 0.018287496519426308 0 +475 0 -3.42639732 0.00660731271 0.0095639682531163638 0 +476 0 -2.952454 0.0122020785 0.01771216181684792 0 +477 0 -3.15739 0.009363164 0.013571827794801039 0 +478 0 -2.700109 0.0168858021 0.02456908615156167 0 +479 1 5.422683 0.998566151 0.0020700917793552748 1 +480 0 -3.01963663 0.0111883869 0.016232407868317561 0 +481 0 -1.95464623 0.0435004532 0.064163809931440968 0 +482 1 8.607498 0.9999776 3.2333100677721677E-05 1 +483 1 6.58452654 0.9996853 0.00045410565839549264 1 +484 0 -2.73630357 0.0161186177 0.023443701603127542 0 +485 0 -3.09087944 0.0102043264 0.014797358636986186 0 +486 0 -3.38692 0.006954534 0.010068322651344731 0 +487 1 7.696018 0.9999263 0.00010628919351499789 1 +488 1 1.0491457 0.6970149 0.52073854047967616 1 +489 1 -0.689220667 0.191920877 2.381416443337466 0 +490 0 -3.91345024 0.00350815617 0.0050700980595032802 0 +491 1 4.178467 0.992759 0.010484575618081977 1 +492 0 -3.147574 0.00948284753 0.013746136860986134 0 +493 1 6.74581242 0.9997451 0.00036783178636926771 1 +494 0 -0.066890955 0.348719329 0.61864868373531823 0 +495 0 -3.38692 0.006954534 0.010068322651344731 0 +496 0 -3.99284053 0.00316367578 0.0045714544741969074 0 +497 0 -2.977867 0.0118083851 0.017137280806999947 0 +498 0 -3.041843 0.0108719906 0.015770853378051553 0 +499 0 -3.041843 0.0108719906 0.015770853378051553 0 +500 0 -2.40246773 0.024711486 0.036099028626778586 0 +501 0 -3.041843 0.0108719906 0.015770853378051553 0 +502 0 -2.92444015 0.0126511 0.018368113945997116 0 +503 0 -2.89621615 0.0131199779 0.01905339221272959 0 +504 0 -3.91345024 0.00350815617 0.0050700980595032802 0 +505 0 -3.02394938 0.0111262342 0.016141728677803845 0 +506 1 6.724969 0.999738038 0.00037798138589097184 1 +507 0 -3.072129 0.0104546947 0.015162333873327377 0 +508 0 -2.927829 0.0125959255 0.018287496519426308 0 +509 0 -3.686916 0.004710472 0.0068118309624495416 0 +510 0 -3.91345024 0.00350815617 0.0050700980595032802 0 +511 0 -2.80222845 0.0148083763 0.021523733114509613 0 +512 0 -2.927829 0.0125959255 0.018287496519426308 0 +513 0 -3.38692 0.006954534 0.010068322651344731 0 +514 1 6.575938 0.9996818 0.00045918075271542201 1 +515 1 5.73308563 0.999043643 0.0013803909947758138 1 +516 0 -3.99284053 0.00316367578 0.0045714544741969074 0 +517 0 -3.901368 0.003563763 0.0051506065948178194 0 +518 0 -3.034531 0.01097518 0.015921368683516818 0 +519 1 3.72647738 0.987007737 0.018866700866053545 1 +520 0 -3.82106638 0.00395632256 0.0057190876873453424 0 +521 0 -3.096822 0.0101262229 0.0146835219159501 0 +522 1 2.5922153 0.945252538 0.081228277454268763 1 +523 1 4.4359293 0.994816244 0.0074980303966219915 1 +524 0 -3.325173 0.00753429625 0.010910846422542917 0 +525 0 -3.28834772 0.007902635 0.01144638079485033 0 +526 0 -3.15739 0.009363164 0.013571827794801039 0 +527 0 -2.89621615 0.0131199779 0.01905339221272959 0 +528 0 -2.15011382 0.0340318531 0.04995247832934497 0 +529 0 -3.147574 0.00948284753 0.013746136860986134 0 +530 1 3.79756474 0.9881462 0.017203605672845249 1 +531 0 -2.62275124 0.018647762 0.027157037155194675 0 +532 0 -3.75712681 0.00429946231 0.006216185673502031 0 +533 0 -3.325173 0.00753429625 0.010910846422542917 0 +534 0 -3.59146929 0.005332595 0.0077138945216321175 0 +535 0 -3.076754 0.010392379 0.015071484312425176 0 +536 0 -2.64253736 0.018180633 0.026470469789084766 0 +537 0 -2.45412827 0.023136273 0.033770775407914295 0 +538 0 -3.041843 0.0108719906 0.015770853378051553 0 +539 0 -2.575965 0.0197996683 0.028851459992936014 0 +540 0 -2.496859 0.02190779 0.031957612155008579 0 +541 0 -3.42639732 0.00660731271 0.0095639682531163638 0 +542 0 -2.51783967 0.0213281829 0.031102940069276851 0 +543 0 -3.041843 0.0108719906 0.015770853378051553 0 +544 0 -2.95253134 0.0122008622 0.017710385381236947 0 +545 0 -2.80222845 0.0148083763 0.021523733114509613 0 +546 1 7.705576 0.9999272 0.00010499922917678318 1 +547 0 -3.76333284 0.00426489767 0.0061661049837132458 0 +548 0 -3.50709486 0.005950225 0.0086100013759846713 0 +549 1 3.90045238 0.9896214 0.015051395561371757 1 +550 0 -3.325173 0.00753429625 0.010910846422542917 0 +551 0 -3.67959213 0.004755537 0.0068771549577151874 0 +552 0 -2.30335951 0.028031148 0.041018013369985402 0 +553 0 -0.9943423 0.137509078 0.21341882251242078 0 +554 0 -3.42639732 0.00660731271 0.0095639682531163638 0 +555 0 -1.37447739 0.08845257 0.13361037000741946 0 +556 0 -2.33357334 0.0269756746 0.039452222295553709 0 +557 0 -2.69035482 0.0170986224 0.024881428250692977 0 +558 0 -3.59146929 0.005332595 0.0077138945216321175 0 +559 0 -2.80222845 0.0148083763 0.021523733114509613 0 +560 0 -2.64253736 0.018180633 0.026470469789084766 0 +561 0 -2.64253736 0.018180633 0.026470469789084766 0 +562 0 -3.67959213 0.004755537 0.0068771549577151874 0 +563 0 -3.325173 0.00753429625 0.010910846422542917 0 +564 0 -2.75849843 0.0156652387 0.022779051375408362 0 +565 1 7.38857651 0.9998899 0.00015883472127190573 1 +566 0 -2.987708 0.01165932 0.016919672523423575 0 +567 0 -2.45049858 0.02324367 0.033929394014035712 0 +568 1 2.85860872 0.9607096 0.057827733186044755 1 +569 1 5.942483 0.999272346 0.0010501642250589274 1 +570 1 5.02962875 0.997606337 0.0034574652940616382 1 +571 1 6.535301 0.9996644 0.00048421241130769198 1 +572 0 -3.325173 0.00753429625 0.010910846422542917 0 +573 0 -3.797206 0.00408106064 0.0058997728235235976 0 +574 1 3.8598814 0.9890627 0.015866075078484695 1 +575 0 -2.45412827 0.023136273 0.033770775407914295 0 +576 0 -2.80222845 0.0148083763 0.021523733114509613 0 +577 0 -3.797206 0.00408106064 0.0058997728235235976 0 +578 0 -3.797206 0.00408106064 0.0058997728235235976 0 +579 0 -3.67959213 0.004755537 0.0068771549577151874 0 +580 0 -2.60076571 0.0191806331 0.027940628690246883 0 +581 1 5.41844845 0.9985582 0.0020815450933619833 1 +582 1 5.674245 0.9989673 0.0014906555444735932 1 +583 0 -3.42639732 0.00660731271 0.0095639682531163638 0 +584 0 -2.19329166 0.03222572 0.047257497739858927 0 +585 0 -3.91345024 0.00350815617 0.0050700980595032802 0 +586 1 8.814509 0.9999829 2.4679721491650575E-05 1 +587 0 -2.6769495 0.0173954088 0.025317114854763057 0 +588 1 3.39331174 0.980064869 0.029050852820670891 1 +589 0 -2.927829 0.0125959255 0.018287496519426308 0 +590 1 2.03315377 0.8926845 0.16377768759260961 1 +591 1 3.663708 0.9859132 0.020467432884748681 1 +592 1 2.949967 0.9649745 0.051437242265565231 1 +593 0 -2.73630357 0.0161186177 0.023443701603127542 0 +594 1 3.30674386 0.9777314 0.032489899094650831 1 +595 0 -2.80222845 0.0148083763 0.021523733114509613 0 +596 0 -2.83381081 0.0142184142 0.020660062880562078 0 +597 0 -2.27036071 0.0292297285 0.042798166274152998 0 +598 0 -3.325173 0.00753429625 0.010910846422542917 0 +599 0 -2.11023 0.03578669 0.052575747552166117 0 +600 0 -3.325173 0.00753429625 0.010910846422542917 0 +601 0 -3.901368 0.003563763 0.0051506065948178194 0 +602 0 -3.041843 0.0108719906 0.015770853378051553 0 +603 1 2.66343737 0.949872255 0.074194590918283093 1 +604 1 2.51864243 0.940061033 0.089173669140511128 1 +605 1 5.64123535 0.9989219 0.0015562501643008667 1 +606 0 -3.048391 0.0107803931 0.015637259899974859 0 +607 0 -3.91345024 0.00350815617 0.0050700980595032802 0 +608 1 6.661555 0.9997154 0.00041066702212545742 1 +609 0 -2.927829 0.0125959255 0.018287496519426308 0 +610 1 4.04960251 0.9914429 0.012398381603446116 1 +611 1 4.502653 0.9952469 0.0068736408219328702 1 +612 1 9.48448849 0.999992847 1.0318995955417813E-05 1 +613 0 -3.18372536 0.009049416 0.013114978345237449 0 +614 0 -3.64776087 0.00495643448 0.0071684030174950239 0 +615 0 -2.61884785 0.0187412985 0.02729455251501172 0 +616 0 -3.325173 0.00753429625 0.010910846422542917 0 +617 0 ? ? ? 0 +618 0 -3.041843 0.0108719906 0.015770853378051553 0 +619 0 -2.80222845 0.0148083763 0.021523733114509613 0 +620 0 -3.325173 0.00753429625 0.010910846422542917 0 +621 0 -0.4190638 0.252619356 0.42008489408954092 0 +622 0 -1.87680984 0.0479327366 0.070864591542403682 0 +623 0 -3.91345024 0.00350815617 0.0050700980595032802 0 +624 0 -2.766415 0.0155065786 0.022546529145288628 0 +625 0 -2.1035223 0.0360902548 0.053030027503309395 0 +626 1 3.40027428 0.9802418 0.028790462687017897 1 +627 0 -2.51565552 0.0213878155 0.031190849267758027 0 +628 0 -3.686916 0.004710472 0.0068118309624495416 0 +629 0 -3.15739 0.009363164 0.013571827794801039 0 +630 0 -2.07860565 0.0372397378 0.054751498902405868 0 +631 0 -2.80222845 0.0148083763 0.021523733114509613 0 +632 0 -3.91345024 0.00350815617 0.0050700980595032802 0 +633 1 2.96976161 0.965838 0.050146846237704622 1 +634 0 -3.42639732 0.00660731271 0.0095639682531163638 0 +635 0 -2.73052621 0.0162387341 0.023619842663828074 0 +636 1 5.976661 0.999304056 0.0010043842538030249 1 +637 0 -1.577994 0.069234 0.10350958138155769 0 +638 0 -3.15739 0.009363164 0.013571827794801039 0 +639 0 -2.69035482 0.0170986224 0.024881428250692977 0 +640 0 -2.93732166 0.0124426372 0.018063544610849989 0 +641 0 -3.325173 0.00753429625 0.010910846422542917 0 +642 0 -3.325173 0.00753429625 0.010910846422542917 0 +643 0 -3.91345024 0.00350815617 0.0050700980595032802 0 +644 0 -3.686916 0.004710472 0.0068118309624495416 0 +645 0 -3.325173 0.00753429625 0.010910846422542917 0 +646 0 -3.49286652 0.00606117 0.0087710278686882609 0 +647 0 -3.59211588 0.0053281174 0.0077073998041508052 0 +648 1 6.86652 0.999782264 0.00031416051503262935 1 +649 0 -3.325173 0.00753429625 0.010910846422542917 0 +650 0 -2.337792 0.0268314146 0.039238345046468712 0 +651 0 -3.23694253 0.008446856 0.01223799721807144 0 +652 0 -2.6769495 0.0173954088 0.025317114854763057 0 +653 0 -3.041843 0.0108719906 0.015770853378051553 0 +654 0 -3.172824 0.009178008 0.013302204234837588 0 +655 0 -3.325173 0.00753429625 0.010910846422542917 0 +656 0 -2.80222845 0.0148083763 0.021523733114509613 0 +657 0 0.130610943 0.4093408 0.75960213137490074 1 +658 1 5.160678 0.9979822 0.0029140052636661856 1 +659 0 -3.91345024 0.00350815617 0.0050700980595032802 0 +660 0 -3.797206 0.00408106064 0.0058997728235235976 0 +661 0 -2.89621615 0.0131199779 0.01905339221272959 0 +662 0 -3.33888555 0.007401535 0.010717871041737253 0 +663 0 -3.33888555 0.007401535 0.010717871041737253 0 +664 0 -3.05405974 0.010701715 0.015522518911179681 0 +665 0 -3.91345024 0.00350815617 0.0050700980595032802 0 +666 0 -2.263587 0.0294818431 0.043172891114315569 0 +667 0 -3.172824 0.009178008 0.013302204234837588 0 +668 1 1.63849282 0.8324294 0.2646001597179255 1 +669 1 5.74382639 0.999056935 0.0013611967001791454 1 +670 1 4.41910648 0.9947017 0.0076641765020727106 1 +671 0 -2.87245846 0.0135279177 0.019649872384720268 0 +672 0 -3.460084 0.006324678 0.0091535591332347361 0 +673 0 -2.23699141 0.0304922573 0.04467567487240142 0 +674 0 -3.797206 0.00408106064 0.0058997728235235976 0 +675 0 -2.456164 0.023076253 0.033682136721607067 0 +676 0 -3.56804562 0.00549736572 0.007952902358798624 0 +677 0 -2.927829 0.0125959255 0.018287496519426308 0 +678 0 -3.91345024 0.00350815617 0.0050700980595032802 0 +679 0 -3.686916 0.004710472 0.0068118309624495416 0 +680 1 9.640121 0.999994159 8.427174504939984E-06 1 +681 1 6.608304 0.999694943 0.00044017074546129814 1 +682 0 -2.505847 0.0216576289 0.031588670000511465 0 +683 0 -3.91345024 0.00350815617 0.0050700980595032802 0 +684 0 -3.91345024 0.00350815617 0.0050700980595032802 0 +685 0 -3.91345024 0.00350815617 0.0050700980595032802 0 +686 0 -3.91345024 0.00350815617 0.0050700980595032802 0 +687 0 -3.017318 0.0112219434 0.016281368250090737 0 +688 0 -3.15739 0.009363164 0.013571827794801039 0 +689 0 -3.33330846 0.007455248 0.010795942608265569 0 +690 0 -3.59211588 0.0053281174 0.0077073998041508052 0 +691 1 3.79695964 0.988136947 0.017217094282062421 1 +692 0 -3.42639732 0.00660731271 0.0095639682531163638 0 +693 0 -3.22202635 0.008611617 0.012477740883141681 0 +694 0 -3.06981564 0.010486003 0.015207980092467717 0 +695 0 -3.686916 0.004710472 0.0068118309624495416 0 +696 1 4.792446 0.996739864 0.0047110650708302544 1 +697 1 3.01194549 0.9676101 0.047502235539438789 1 +698 1 3.55301142 0.9837573 0.023625633638672462 1 diff --git a/test/BaselineOutput/Common/WeightedEnsemble/WE-RandomFeature-TrainTest-breast-cancer-out.txt b/test/BaselineOutput/Common/WeightedEnsemble/WE-RandomFeature-TrainTest-breast-cancer-out.txt new file mode 100644 index 0000000000..1aebbbfe11 --- /dev/null +++ b/test/BaselineOutput/Common/WeightedEnsemble/WE-RandomFeature-TrainTest-breast-cancer-out.txt @@ -0,0 +1,94 @@ +maml.exe TrainTest test=%Data% tr=WeightedEnsemble{nm=20 st=AllInstanceSelector{fs=RandomFeatureSelector} tp=-} dout=%Output% loader=Text{col=Label:BL:0 col=Features:R4:1-9} data=%Data% out=%Output% seed=1 +Automatically adding a MinMax normalization transform, use 'norm=Warn' or 'norm=No' to turn this behavior off. +Training 20 learners for the batch 1 +Beginning training model 1 of 20 +Warning: Skipped 16 instances with missing features during training (over 1 iterations; 16 inst/iter) +Trainer 1 of 20 finished in %Time% +Beginning training model 2 of 20 +Warning: Skipped 16 instances with missing features during training (over 1 iterations; 16 inst/iter) +Trainer 2 of 20 finished in %Time% +Beginning training model 3 of 20 +Warning: Skipped 16 instances with missing features during training (over 1 iterations; 16 inst/iter) +Trainer 3 of 20 finished in %Time% +Beginning training model 4 of 20 +Trainer 4 of 20 finished in %Time% +Beginning training model 5 of 20 +Warning: Skipped 16 instances with missing features during training (over 1 iterations; 16 inst/iter) +Trainer 5 of 20 finished in %Time% +Beginning training model 6 of 20 +Warning: Skipped 16 instances with missing features during training (over 1 iterations; 16 inst/iter) +Trainer 6 of 20 finished in %Time% +Beginning training model 7 of 20 +Warning: Skipped 16 instances with missing features during training (over 1 iterations; 16 inst/iter) +Trainer 7 of 20 finished in %Time% +Beginning training model 8 of 20 +Warning: Skipped 16 instances with missing features during training (over 1 iterations; 16 inst/iter) +Trainer 8 of 20 finished in %Time% +Beginning training model 9 of 20 +Warning: Skipped 16 instances with missing features during training (over 1 iterations; 16 inst/iter) +Trainer 9 of 20 finished in %Time% +Beginning training model 10 of 20 +Warning: Skipped 16 instances with missing features during training (over 1 iterations; 16 inst/iter) +Trainer 10 of 20 finished in %Time% +Beginning training model 11 of 20 +Trainer 11 of 20 finished in %Time% +Beginning training model 12 of 20 +Warning: Skipped 16 instances with missing features during training (over 1 iterations; 16 inst/iter) +Trainer 12 of 20 finished in %Time% +Beginning training model 13 of 20 +Warning: Skipped 16 instances with missing features during training (over 1 iterations; 16 inst/iter) +Trainer 13 of 20 finished in %Time% +Beginning training model 14 of 20 +Warning: Skipped 16 instances with missing features during training (over 1 iterations; 16 inst/iter) +Trainer 14 of 20 finished in %Time% +Beginning training model 15 of 20 +Warning: Skipped 16 instances with missing features during training (over 1 iterations; 16 inst/iter) +Trainer 15 of 20 finished in %Time% +Beginning training model 16 of 20 +Warning: Skipped 16 instances with missing features during training (over 1 iterations; 16 inst/iter) +Trainer 16 of 20 finished in %Time% +Beginning training model 17 of 20 +Trainer 17 of 20 finished in %Time% +Beginning training model 18 of 20 +Warning: Skipped 16 instances with missing features during training (over 1 iterations; 16 inst/iter) +Trainer 18 of 20 finished in %Time% +Beginning training model 19 of 20 +Warning: Skipped 16 instances with missing features during training (over 1 iterations; 16 inst/iter) +Trainer 19 of 20 finished in %Time% +Beginning training model 20 of 20 +Warning: Skipped 16 instances with missing features during training (over 1 iterations; 16 inst/iter) +Trainer 20 of 20 finished in %Time% +Training calibrator. +TEST POSITIVE RATIO: 0.3448 (241.0/(241.0+458.0)) +Confusion table + ||====================== +PREDICTED || positive | negative | Recall +TRUTH ||====================== + positive || 233 | 8 | 0.9668 + negative || 12 | 446 | 0.9738 + ||====================== +Precision || 0.9510 | 0.9824 | +OVERALL 0/1 ACCURACY: 0.971388 +LOG LOSS/instance: 0.130701 +Test-set entropy (prior Log-Loss/instance): 0.929318 +LOG-LOSS REDUCTION (RIG): 0.859358 +AUC: 0.994927 + +OVERALL RESULTS +--------------------------------------- +AUC: 0.994927 (0.0000) +Accuracy: 0.971388 (0.0000) +Positive precision: 0.951020 (0.0000) +Positive recall: 0.966805 (0.0000) +Negative precision: 0.982379 (0.0000) +Negative recall: 0.973799 (0.0000) +Log-loss: 0.130701 (0.0000) +Log-loss reduction: 0.859358 (0.0000) +F1 Score: 0.958848 (0.0000) +AUPRC: 0.989450 (0.0000) + +--------------------------------------- +Physical memory usage(MB): %Number% +Virtual memory usage(MB): %Number% +%DateTime% Time elapsed(s): %Number% + diff --git a/test/BaselineOutput/Common/WeightedEnsemble/WE-RandomFeature-TrainTest-breast-cancer-rp.txt b/test/BaselineOutput/Common/WeightedEnsemble/WE-RandomFeature-TrainTest-breast-cancer-rp.txt new file mode 100644 index 0000000000..2774ce294c --- /dev/null +++ b/test/BaselineOutput/Common/WeightedEnsemble/WE-RandomFeature-TrainTest-breast-cancer-rp.txt @@ -0,0 +1,4 @@ +WeightedEnsemble +AUC Accuracy Positive precision Positive recall Negative precision Negative recall Log-loss Log-loss reduction F1 Score AUPRC /bp /nm /st Learner Name Train Dataset Test Dataset Results File Run Time Physical Memory Virtual Memory Command Line Settings +0.994927 0.971388 0.95102 0.966805 0.982379 0.973799 0.130701 0.859358 0.958848 0.98945 Microsoft.ML.Runtime.ComponentFactoryUtils+SimpleComponentFactory`1[Microsoft.ML.Trainers.LinearSvmTrainer] 20 AllInstanceSelector{fs=RandomFeatureSelector} WeightedEnsemble %Data% %Data% %Output% 99 0 0 maml.exe TrainTest test=%Data% tr=WeightedEnsemble{nm=20 st=AllInstanceSelector{fs=RandomFeatureSelector} tp=-} dout=%Output% loader=Text{col=Label:BL:0 col=Features:R4:1-9} data=%Data% out=%Output% seed=1 /bp:Microsoft.ML.Runtime.ComponentFactoryUtils+SimpleComponentFactory`1[Microsoft.ML.Trainers.LinearSvmTrainer];/nm:20;/st:AllInstanceSelector{fs=RandomFeatureSelector} + diff --git a/test/BaselineOutput/Common/WeightedEnsemble/WE-RandomFeature-TrainTest-breast-cancer.txt b/test/BaselineOutput/Common/WeightedEnsemble/WE-RandomFeature-TrainTest-breast-cancer.txt new file mode 100644 index 0000000000..c3c65d1a01 --- /dev/null +++ b/test/BaselineOutput/Common/WeightedEnsemble/WE-RandomFeature-TrainTest-breast-cancer.txt @@ -0,0 +1,700 @@ +Instance Label Score Probability Log-loss Assigned +0 0 -2.35020351 0.02142645 0.031247807073335927 0 +1 0 2.20269275 0.9221814 3.6837416735536537 1 +2 0 -2.66031551 0.0140613765 0.020430255849727907 0 +3 0 1.976532 0.8965738 3.2733261468421255 1 +4 0 -2.42708087 0.0193079133 0.028127858200114327 0 +5 1 6.7246294 0.9998373 0.00023477542035385971 1 +6 0 -0.6417589 0.188505486 0.30134675261244487 0 +7 0 -2.94974327 0.009468592 0.013725373405768014 0 +8 0 -3.077827 0.007944322 0.011507002657738167 0 +9 0 -2.59433961 0.0153834717 0.02236613731126251 0 +10 0 -3.57871246 0.0039909156 0.0057691940062725823 0 +11 0 -3.36269617 0.005372264 0.0077714324034035831 0 +12 1 -0.441581964 0.234507844 2.0922919158687252 0 +13 0 -2.95115447 0.009450313 0.01369875044614853 0 +14 1 4.981148 0.9981911 0.0026120278818572685 1 +15 1 0.695231438 0.5959162 0.74681859909866888 1 +16 0 -2.76257753 0.01223037 0.017753482924837489 0 +17 0 -2.60325241 0.01519796 0.022094344146968328 0 +18 1 4.42420244 0.9961018 0.0056349088344622216 1 +19 0 -2.0173955 0.0335237458 0.049193808305993285 0 +20 1 3.3882792 0.9838776 0.023449248788516872 1 +21 1 4.056161 0.993533 0.0093601856431956044 1 +22 0 -3.02443838 0.008547617 0.012384610013862075 0 +23 1 1.14368916 0.7327102 0.44868542955696805 1 +24 0 -3.48153138 0.00456212368 0.006596812411908696 0 +25 1 0.597109556 0.5628745 0.82911481415763699 1 +26 0 -2.972652 0.00917614251 0.013299488048427441 0 +27 0 -2.50502563 0.01736992 0.025279692310691083 0 +28 0 -3.36269617 0.005372264 0.0077714324034035831 0 +29 0 -3.3805213 0.00524218846 0.0075827718664163695 0 +30 0 -3.14879227 0.007207334 0.010435636978038142 0 +31 0 -3.181472 0.006891175 0.0099762780042885069 0 +32 1 3.79490757 0.990746 0.013412826390273786 1 +33 0 -3.08061743 0.007913979 0.01146287647184982 0 +34 0 -2.83090782 0.0111402739 0.016162211708571711 0 +35 0 -3.36269617 0.005372264 0.0077714324034035831 0 +36 1 4.81860876 0.997736454 0.0032693080866665215 1 +37 0 -1.5955925 0.0585075356 0.086978547180957086 0 +38 1 3.15659857 0.9779258 0.032203123155154299 1 +39 1 1.33909583 0.782201648 0.35438751968266446 1 +40 0 3.11574125 0.976673 5.421855843878002 1 +41 1 1.38397646 0.792586148 0.33536034168221307 1 +42 1 4.97571 0.9981775 0.0026316695670057914 1 +43 1 0.5148642 0.5347295 0.90311887719353878 1 +44 1 4.734537 0.99745816 0.0036717685767250798 1 +45 0 -3.61522961 0.00379519816 0.0054857298846733476 0 +46 1 2.99987841 0.9727315 0.039886500400600128 1 +47 0 -3.66275549 0.00355473021 0.0051375283293989922 0 +48 0 -2.42708087 0.0193079133 0.028127858200114327 0 +49 1 3.55264664 0.9871123 0.018713894771840999 1 +50 1 1.48304844 0.814200759 0.29654352864176753 1 +51 1 -0.227678061 0.2916604 1.7776385850117988 0 +52 1 2.52921 0.9490073 0.075508951500451019 1 +53 1 3.744893 0.9900902 0.014368142718053877 1 +54 1 3.62345314 0.988299966 0.016979104114969619 1 +55 1 2.7223177 0.960482061 0.058169426211468332 1 +56 1 3.97636843 0.9927843 0.010447763212012117 1 +57 1 0.368098021 0.484069228 1.0467147085469759 1 +58 1 1.29036641 0.770507753 0.37611862158998971 1 +59 1 0.76894 0.620193839 0.68920890041492522 1 +60 1 0.9050958 0.663423955 0.59199698783940791 1 +61 0 -3.32607841 0.00564963045 0.0081738051192268811 0 +62 1 4.36777544 0.995786846 0.0060911373180374557 1 +63 1 0.04215908 0.374182284 1.4181868408675455 1 +64 0 -3.66275549 0.00355473021 0.0051375283293989922 0 +65 1 1.22161055 0.753270864 0.40875936599944873 1 +66 0 -2.60325241 0.01519796 0.022094344146968328 0 +67 1 2.41257882 0.940615654 0.088322752752844111 1 +68 1 4.88835955 0.997944057 0.0029691520387345742 1 +69 0 -3.36269617 0.005372264 0.0077714324034035831 0 +70 0 -2.07045174 0.03122702 0.045769468245393054 0 +71 1 3.80566263 0.9908814 0.013215729499207091 1 +72 0 -1.928551 0.0377392136 0.055500156898039839 0 +73 1 4.84930134 0.9978303 0.003133657051215251 1 +74 1 1.43255329 0.803408742 0.31579393534525346 1 +75 0 -2.71362352 0.0130754877 0.018988354560566929 0 +76 0 -2.9696815 0.009213552 0.013353959208047049 0 +77 0 -2.08397055 0.0306665953 0.044935125282995356 0 +78 0 -2.39090824 0.0202778634 0.029555456827693796 0 +79 0 -3.198965 0.00672763959 0.0097387284509941503 0 +80 0 -2.37172747 0.0208113864 0.030341313008337964 0 +81 0 -2.70613074 0.0132098272 0.019184746686686661 0 +82 0 -2.19789839 0.0263155755 0.038473830721658032 0 +83 0 -2.25335932 0.0244208425 0.035669158538303838 0 +84 1 4.690504 0.9972991 0.0039018826417654084 1 +85 1 3.50122118 0.9861759 0.020083105654149558 1 +86 1 1.483474 0.814289749 0.29638585495758274 1 +87 1 3.722689 0.9897844 0.01481376235531935 1 +88 0 -2.60325241 0.01519796 0.022094344146968328 0 +89 0 -3.01846623 0.008617866 0.012486834878585228 0 +90 0 -3.48153138 0.00456212368 0.006596812411908696 0 +91 0 -3.10732222 0.00762934 0.01104901319263845 0 +92 0 -2.60325241 0.01519796 0.022094344146968328 0 +93 0 -3.66275549 0.00355473021 0.0051375283293989922 0 +94 0 -3.181472 0.006891175 0.0099762780042885069 0 +95 0 -3.48153138 0.00456212368 0.006596812411908696 0 +96 0 -3.360118 0.005391342 0.0077991052755619584 0 +97 0 -2.35020351 0.02142645 0.031247807073335927 0 +98 1 4.73837566 0.997471631 0.0036522851446360209 1 +99 1 5.443738 0.9990449 0.0013785834494152035 1 +100 1 2.8794384 0.9679499 0.046995677706813399 1 +101 1 -0.564700842 0.2053433 2.2838901768608917 0 +102 0 -2.39008832 0.0203003939 0.029588634698803455 0 +103 1 0.116768122 0.398629665 1.3268790176844891 1 +104 1 6.0495553 0.9995864 0.00059681723647690731 1 +105 1 0.629760265 0.573946655 0.80101144237112865 1 +106 1 5.58598375 0.999215245 0.0011326062578501696 1 +107 1 3.780911 0.990566969 0.01367358069166719 1 +108 0 -3.30376959 0.00582554471 0.0084290603015252097 0 +109 1 3.49570513 0.9860716 0.020235708023850704 1 +110 0 -2.13783383 0.0285288915 0.041757004119532468 0 +111 1 2.50257373 0.9471956 0.078265731572671832 1 +112 1 3.91579556 0.992159 0.011356741328594802 1 +113 1 5.49076939 0.999105 0.001291823935243467 1 +114 0 -1.79101646 0.04528397 0.066856411266699695 0 +115 0 -2.619931 0.0148567092 0.021594512512594709 0 +116 0 -0.543985844 0.210055485 0.34017677232692572 0 +117 1 4.19383526 0.9946478 0.007742328841635851 1 +118 0 -3.05082059 0.008244023 0.011942907426198454 0 +119 0 -2.45069766 0.0186993368 0.027232859713050471 0 +120 0 -3.19335556 0.00677965628 0.009814282946090571 0 +121 0 -2.14901066 0.0281037837 0.041125830708455907 0 +122 1 5.876984 0.999475 0.0007576104836905449 1 +123 1 2.07765055 0.9088361 0.13790791163184479 1 +124 1 3.923765 0.992244244 0.011232807248884629 1 +125 0 -3.66275549 0.00355473021 0.0051375283293989922 0 +126 1 4.059747 0.9935648 0.0093140546712183192 1 +127 0 -2.881413 0.0103968307 0.015077974249878848 0 +128 1 2.93283916 0.97016263 0.043701485788462398 1 +129 0 -3.05638671 0.008181349 0.011851740162980797 0 +130 0 -2.07045174 0.03122702 0.045769468245393054 0 +131 0 -3.181472 0.006891175 0.0099762780042885069 0 +132 1 5.45903349 0.999064863 0.0013497491034473905 1 +133 0 -2.907415 0.0100334408 0.014548302681763527 0 +134 0 -2.89044762 0.0102691147 0.014891795241666813 0 +135 0 -1.911738 0.03859237 0.056779839280047775 0 +136 0 -2.76257753 0.01223037 0.017753482924837489 0 +137 0 -3.17440128 0.00695839245 0.010073928241131102 0 +138 0 -2.56165981 0.0160829369 0.023391382670451242 0 +139 0 -3.976965 0.00230521872 0.0033295667772822867 0 +140 0 -3.17440128 0.00695839245 0.010073928241131102 0 +141 0 -3.45500565 0.004731709 0.0068426147359647426 0 +142 1 2.0765543 0.9087105 0.13810737800447412 1 +143 0 -2.619931 0.0148567092 0.021594512512594709 0 +144 0 -3.36269617 0.005372264 0.0077714324034035831 0 +145 0 -3.459014 0.004705686 0.0068048934654552891 0 +146 1 0.8525505 0.647017 0.62812447149289763 1 +147 0 -3.23791885 0.006377194 0.0092298077875565075 0 +148 0 -1.11945617 0.107157588 0.16352253487811372 0 +149 1 5.98623657 0.999548554 0.00065144535594696034 1 +150 0 -3.57871246 0.0039909156 0.0057691940062725823 0 +151 1 2.42507768 0.9415735 0.086854376723176696 1 +152 1 5.16880751 0.9986039 0.0020155821400579415 1 +153 0 -2.48649859 0.0178125072 0.025929643591511155 0 +154 0 -3.818494 0.002868181 0.0041438561512361881 0 +155 1 2.38984656 0.938835859 0.091055147525217178 1 +156 0 -3.24683142 0.006299597 0.0091171450734618097 0 +157 0 -3.181472 0.006891175 0.0099762780042885069 0 +158 0 -3.55596852 0.00411786139 0.0059530836226468027 0 +159 1 6.889257 0.9998704 0.00018695725475655539 1 +160 1 5.16653728 0.99859947 0.0020219544086421295 1 +161 0 -2.50450349 0.0173822455 0.025297788139068099 0 +162 0 -2.881413 0.0103968307 0.015077974249878848 0 +163 0 -2.61993146 0.0148566989 0.021594497509945736 0 +164 0 -2.18737435 0.0266909264 0.039030090352712779 0 +165 0 -2.28251481 0.0234788079 0.034276741576575703 0 +166 1 4.652792 0.997154951 0.0041103878492270212 1 +167 1 4.16007471 0.9943935 0.0081111889462956461 1 +168 0 -2.881413 0.0103968307 0.015077974249878848 0 +169 0 -3.80500245 0.00292201829 0.0042217523254301217 0 +170 0 -3.17440128 0.00695839245 0.010073928241131102 0 +171 0 -3.48153138 0.00456212368 0.006596812411908696 0 +172 0 -3.66275549 0.00355473021 0.0051375283293989922 0 +173 1 7.749319 0.999960542 5.6927380571090205E-05 1 +174 1 3.28609467 0.9814768 0.026973952882474905 1 +175 1 4.754416 0.997526944 0.0035722851375678301 1 +176 0 -3.181472 0.006891175 0.0099762780042885069 0 +177 1 3.13484836 0.977267265 0.033174927144339776 1 +178 0 -2.60325241 0.01519796 0.022094344146968328 0 +179 1 1.35827088 0.7866837 0.34614444265252725 1 +180 0 -3.57871246 0.0039909156 0.0057691940062725823 0 +181 0 -3.818494 0.002868181 0.0041438561512361881 0 +182 0 -2.0173955 0.0335237458 0.049193808305993285 0 +183 1 5.10819435 0.998482049 0.0021916048808472235 1 +184 1 3.6821804 0.9892024 0.01566238379854416 1 +185 0 -3.27865314 0.00603012135 0.0087259619975374007 0 +186 1 2.848053 0.9665763 0.049044506421329993 1 +187 1 6.31880856 0.9997149 0.00041135514874218645 1 +188 1 4.466493 0.996322334 0.0053155311472171379 1 +189 0 -2.874613 0.0104939928 0.015219629159609047 0 +190 1 6.768491 0.9998469 0.00022092863023949783 1 +191 1 5.955166 0.999528766 0.0006800076529197925 1 +192 0 -2.50502563 0.01736992 0.025279692310691083 0 +193 0 -3.48153138 0.00456212368 0.006596812411908696 0 +194 0 -2.881413 0.0103968307 0.015077974249878848 0 +195 0 -2.60325241 0.01519796 0.022094344146968328 0 +196 0 3.08152938 0.9755709 5.3552564875711974 1 +197 0 -2.17913 0.0269885976 0.039471383294479342 0 +198 0 -3.818494 0.002868181 0.0041438561512361881 0 +199 0 -3.02443838 0.008547617 0.012384610013862075 0 +200 1 5.521848 0.9991425 0.0012376018882614337 1 +201 1 5.372613 0.9989463 0.0015209561005816261 1 +202 0 -3.48153138 0.00456212368 0.006596812411908696 0 +203 0 -2.35020351 0.02142645 0.031247807073335927 0 +204 0 -3.48153138 0.00456212368 0.006596812411908696 0 +205 1 6.60255861 0.9998074 0.00027786472928475564 1 +206 1 3.9141736 0.992141545 0.01138213612909901 1 +207 0 -3.57871246 0.0039909156 0.0057691940062725823 0 +208 0 -3.57871246 0.0039909156 0.0057691940062725823 0 +209 0 -2.41273022 0.0196871534 0.028685865849419321 0 +210 1 7.74634 0.999960363 5.7185364750060636E-05 1 +211 1 5.45536041 0.9990601 0.0013566348650311531 1 +212 0 -3.48153138 0.00456212368 0.006596812411908696 0 +213 1 7.91918373 0.999968767 4.5060158207936456E-05 1 +214 1 7.121114 0.999905944 0.00013570069337179254 1 +215 1 4.069581 0.993651152 0.0091886516605441688 1 +216 0 -3.66275549 0.00355473021 0.0051375283293989922 0 +217 0 -3.48153138 0.00456212368 0.006596812411908696 0 +218 1 4.409317 0.9960211 0.005751801477347003 1 +219 0 -1.67489111 0.0527533852 0.078188015874748204 0 +220 0 -3.41192532 0.00502060028 0.007261438805420674 0 +221 1 5.41395235 0.9990048 0.0014365121012461751 1 +222 1 -1.46128654 0.0696130246 3.8444989302969454 0 +223 1 2.88248 0.9680801 0.046801667229609267 1 +224 1 4.98893261 0.99821043 0.0025841164734903758 1 +225 0 -3.66275549 0.00355473021 0.0051375283293989922 0 +226 1 5.158442 0.998583734 0.0020446881366970571 1 +227 1 4.20624161 0.9947383 0.007611097570545804 1 +228 0 -3.57871246 0.0039909156 0.0057691940062725823 0 +229 1 7.42991924 0.9999386 8.857378411777087E-05 1 +230 1 2.90921783 0.969202757 0.045129586369336797 1 +231 1 4.741 0.99748075 0.0036390951828919097 1 +232 0 0.560968161 0.5505455 1.1537530650490297 1 +233 1 3.17858553 0.9785724 0.031249460206855914 1 +234 0 -1.63611937 0.05549677 0.082372365912587259 0 +235 0 -2.31716728 0.0224052239 0.032691518962483686 0 +236 1 6.32573366 0.9997176 0.00040748444079247441 1 +237 1 3.70228744 0.9894953 0.015235273194115648 1 +238 1 7.02103424 0.999892 0.00015582469664856058 1 +239 1 3.00211334 0.9728133 0.039765129675913551 1 +240 0 -1.44092715 0.07145809 0.10696106117327531 0 +241 0 -2.77039146 0.0121005615 0.017563902260437397 0 +242 0 -3.181472 0.006891175 0.0099762780042885069 0 +243 0 -1.96805692 0.0358052738 0.052603556158680231 0 +244 0 -3.48153138 0.00456212368 0.006596812411908696 0 +245 0 -2.12956548 0.0288473871 0.042230067388821964 0 +246 1 6.46913338 0.9997684 0.00033420099654801343 1 +247 1 1.89667988 0.88588053 0.17481594528378519 1 +248 0 -2.08970928 0.0304316487 0.044585487857956629 0 +249 0 -2.99093056 0.008949233 0.012969133230163358 0 +250 0 -3.42805529 0.0049104346 0.0071017101873699076 0 +251 1 3.86987066 0.9916494 0.012097968000748577 1 +252 0 2.35242 0.9357966 3.9612069048852279 1 +253 1 4.97571 0.9981775 0.0026316695670057914 1 +254 1 4.36777544 0.995786846 0.0060911373180374557 1 +255 1 2.844381 0.966411948 0.04928980339887587 1 +256 0 -3.17440128 0.00695839245 0.010073928241131102 0 +257 0 -3.02443838 0.008547617 0.012384610013862075 0 +258 0 -2.881413 0.0103968307 0.015077974249878848 0 +259 0 2.37593436 0.937722147 4.0051369694145214 1 +260 1 4.806753 0.997699142 0.0033232617896548763 1 +261 1 6.30249929 0.9997084 0.00042073090659653316 1 +262 1 5.574623 0.999202847 0.0011505066118902478 1 +263 1 4.556396 0.9967507 0.0046953635456650523 1 +264 1 3.23441172 0.9801322 0.028951709527155371 1 +265 0 -1.61290252 0.057203263 0.084981329496204919 0 +266 1 4.66580725 0.997205555 0.004037174771583614 1 +267 1 1.715203 0.857962132 0.22101412303954324 1 +268 1 4.57573748 0.9968362 0.0045716553081388391 1 +269 0 -3.48153138 0.00456212368 0.006596812411908696 0 +270 1 3.41119885 0.984372556 0.022723658111480655 1 +271 0 -2.35020351 0.02142645 0.031247807073335927 0 +272 1 1.715203 0.857962132 0.22101412303954324 1 +273 1 -0.154188156 0.313083559 1.6753803435990715 0 +274 0 -2.73666334 0.0126707917 0.018396887632218181 0 +275 0 -2.76448846 0.0121984985 0.017706933171887339 0 +276 0 -3.02443838 0.008547617 0.012384610013862075 0 +277 0 -3.66275549 0.00355473021 0.0051375283293989922 0 +278 0 -3.48153138 0.00456212368 0.006596812411908696 0 +279 1 3.41890669 0.984535635 0.022484670566986328 1 +280 0 -2.881413 0.0103968307 0.015077974249878848 0 +281 0 -3.08061743 0.007913979 0.01146287647184982 0 +282 1 2.14493966 0.916255534 0.12617808850525647 1 +283 1 3.06407046 0.974989057 0.036542068944039413 1 +284 1 3.71931362 0.989737153 0.014882658923080262 1 +285 1 7.45223141 0.999940455 8.5907891825789828E-05 1 +286 1 8.281744 0.9999811 2.72595076922999E-05 1 +287 0 -2.89044762 0.0102691147 0.014891795241666813 0 +288 1 0.737782 0.609996855 0.71312628977581283 1 +289 1 4.524617 0.996605337 0.0049057954383496085 1 +290 0 -3.818494 0.002868181 0.0041438561512361881 0 +291 0 -3.48153138 0.00456212368 0.006596812411908696 0 +292 1 3.0389185 0.974127054 0.037818141336623055 1 +293 1 2.96295929 0.9713446 0.041944903916230181 1 +294 0 -3.80317259 0.00292939739 0.0042324293565730474 0 +295 1 3.95550156 0.9925747 0.010752424979152048 1 +296 0 0.7664392 0.6193792 1.3935737762277538 1 +297 0 -1.35778 0.07947047 0.11946409269174511 0 +298 0 -1.60323417 0.05792835 0.086091305299457452 0 +299 1 3.47972465 0.985764861 0.020684539735782742 1 +300 1 3.47681069 0.985708237 0.020767413561728842 1 +301 0 -3.48153138 0.00456212368 0.006596812411908696 0 +302 1 8.260058 0.9999805 2.8119437450984077E-05 1 +303 0 -3.48153138 0.00456212368 0.006596812411908696 0 +304 1 2.989697 0.972355664 0.040443982174943882 1 +305 1 4.898939 0.997973859 0.0029260684405887673 1 +306 0 -3.48153138 0.00456212368 0.006596812411908696 0 +307 0 -3.48153138 0.00456212368 0.006596812411908696 0 +308 1 3.60909414 0.988068163 0.017317523118176806 1 +309 0 -1.406953 0.07463772 0.11190980017698328 0 +310 0 -3.198965 0.00672763959 0.0097387284509941503 0 +311 0 -3.818494 0.002868181 0.0041438561512361881 0 +312 1 1.96433091 0.894999266 0.16004159621277433 1 +313 0 -3.818494 0.002868181 0.0041438561512361881 0 +314 0 -3.748911 0.00315686315 0.0045615947751170118 0 +315 0 2.123838 0.9139899 3.5393501329947261 1 +316 1 2.34690475 0.935337 0.096441825295485159 1 +317 1 4.86161041 0.9978668 0.0030808307149993597 1 +318 0 -3.09892273 0.007717755 0.011177555607812048 0 +319 0 0.974621534 0.684537 1.6644572607518768 1 +320 1 3.65179253 0.9887444 0.016330508310653993 1 +321 0 -2.99093056 0.008949233 0.012969133230163358 0 +322 0 -2.881413 0.0103968307 0.015077974249878848 0 +323 1 2.84774518 0.966562569 0.049064968484752551 1 +324 0 -3.48153138 0.00456212368 0.006596812411908696 0 +325 0 -2.5032227 0.0174125116 0.025342226024504103 0 +326 1 1.97715855 0.896654069 0.15737659646080643 1 +327 0 -3.66275549 0.00355473021 0.0051375283293989922 0 +328 1 2.4004252 0.9396702 0.089773590147954746 1 +329 1 3.970356 0.992724538 0.010534641997806463 1 +330 1 2.88327217 0.968113959 0.046751214567035057 1 +331 0 -2.26780224 0.0239496622 0.034972541168988007 0 +332 0 -1.88953066 0.0397476666 0.05851453034886793 0 +333 1 2.743381 0.9615726 0.056532326600147891 1 +334 1 3.66461515 0.988939941 0.016045187203592843 1 +335 0 -3.818494 0.002868181 0.0041438561512361881 0 +336 1 3.31676149 0.982232 0.025864307531765385 1 +337 0 -3.48153138 0.00456212368 0.006596812411908696 0 +338 0 -3.748911 0.00315686315 0.0045615947751170118 0 +339 1 3.17993474 0.9786115 0.031191815845644115 1 +340 1 3.13580275 0.9772966 0.033131635919054034 1 +341 0 -3.48153138 0.00456212368 0.006596812411908696 0 +342 0 -3.45500565 0.004731709 0.0068426147359647426 0 +343 0 -3.818494 0.002868181 0.0041438561512361881 0 +344 1 5.06338835 0.9983852 0.0023315600077312105 1 +345 0 -3.818494 0.002868181 0.0041438561512361881 0 +346 0 -1.89094186 0.0396732762 0.058402769733496951 0 +347 0 -3.540265 0.0042078495 0.0060834517298904763 0 +348 1 -0.331160069 0.2630095 1.9268132463806442 0 +349 1 1.6181078 0.840806842 0.25015368597819598 1 +350 0 -2.51429367 0.01715259 0.024960644650324191 0 +351 0 -3.181472 0.006891175 0.0099762780042885069 0 +352 0 0.368098259 0.484069318 0.95475084870696658 1 +353 1 4.89962149 0.997975767 0.0029233111341136907 1 +354 0 -3.66275549 0.00355473021 0.0051375283293989922 0 +355 0 -2.57745838 0.0157409683 0.022890049049618721 0 +356 1 -0.5428343 0.210319757 2.2493437115180046 0 +357 1 6.7752533 0.9998483 0.00021886452384059331 1 +358 1 3.5845418 0.987661362 0.017911623471398467 1 +359 1 2.48263454 0.9457999 0.080393125183409062 1 +360 1 7.895276 0.999967754 4.6522057140172645E-05 1 +361 1 3.92873526 0.992297 0.011156112119962872 1 +362 0 -1.86892283 0.0408493578 0.060170675238705264 0 +363 0 -0.950819254 0.131588131 0.2035486511115758 0 +364 0 -3.181472 0.006891175 0.0099762780042885069 0 +365 0 -3.36269617 0.005372264 0.0077714324034035831 0 +366 1 7.38390064 0.9999346 9.4335568158520906E-05 1 +367 1 5.89548874 0.9994883 0.00073842447301602002 1 +368 0 -3.3805213 0.00524218846 0.0075827718664163695 0 +369 0 -3.31133318 0.00576530071 0.0083416399468317273 0 +370 0 -2.13917255 0.0284776464 0.041680903974576314 0 +371 0 -3.3805213 0.00524218846 0.0075827718664163695 0 +372 0 -2.56165981 0.0160829369 0.023391382670451242 0 +373 0 -2.42356062 0.0194002744 0.028263736943371753 0 +374 0 -2.83090782 0.0111402739 0.016162211708571711 0 +375 0 -3.818494 0.002868181 0.0041438561512361881 0 +376 0 -3.66275549 0.00355473021 0.0051375283293989922 0 +377 0 -3.748911 0.00315686315 0.0045615947751170118 0 +378 0 -2.492399 0.0176703669 0.025720874603191231 0 +379 0 -1.2230438 0.0942078 0.14274798032154373 0 +380 0 -3.818494 0.002868181 0.0041438561512361881 0 +381 1 4.99947739 0.9982363 0.0025467298157822579 1 +382 0 -2.30717421 0.0227098037 0.033141075517174851 0 +383 0 -3.45500565 0.004731709 0.0068426147359647426 0 +384 0 -3.45500565 0.004731709 0.0068426147359647426 0 +385 0 -2.19092846 0.0265635867 0.03884135237274277 0 +386 1 3.24195719 0.980334342 0.028654232803022463 1 +387 0 -1.38960135 0.0763113946 0.11452152368104433 0 +388 0 -3.17467213 0.0069558057 0.010070170206952719 0 +389 0 -2.03534627 0.03272899 0.048007932266294177 0 +390 0 -3.518002 0.004338793 0.0062731739956016523 0 +391 1 5.59244728 0.9992222 0.0011225374063043392 1 +392 0 -3.02443838 0.008547617 0.012384610013862075 0 +393 0 -3.93013477 0.00245900918 0.0035519693011514517 0 +394 0 -3.02315712 0.00856264 0.012406470881248672 0 +395 0 -3.02443838 0.008547617 0.012384610013862075 0 +396 0 -2.881413 0.0103968307 0.015077974249878848 0 +397 0 -2.9438014 0.009545941 0.013838035728266303 0 +398 0 -2.73963451 0.0126195122 0.018321959397515086 0 +399 0 -3.19632649 0.00675205747 0.0097741950461912828 0 +400 1 4.716372 0.9973937 0.0037649650926357748 1 +401 0 -3.17440128 0.00695839245 0.010073928241131102 0 +402 0 -1.71476543 0.0500658751 0.074100624504909354 0 +403 0 -2.423853 0.01939259 0.028252430124916475 0 +404 0 -3.226909 0.006474363 0.0093708997562864191 0 +405 0 -3.66275549 0.00355473021 0.0051375283293989922 0 +406 0 -2.38711739 0.0203822348 0.029709157760637532 0 +407 0 -3.66275549 0.00355473021 0.0051375283293989922 0 +408 0 -2.27986956 0.0235627927 0.034400824572784768 0 +409 0 -2.83090782 0.0111402739 0.016162211708571711 0 +410 0 -3.66275549 0.00355473021 0.0051375283293989922 0 +411 0 -3.976965 0.00230521872 0.0033295667772822867 0 +412 1 5.14380932 0.9985549 0.0020863675685671604 1 +413 0 -2.15964055 0.0277051888 0.040534272712979186 0 +414 1 3.69432354 0.98938024 0.015403008108993248 1 +415 0 -0.858623743 0.146847948 0.22912520856758853 0 +416 1 4.79923248 0.9976751 0.0033579966309372458 1 +417 0 -3.66275549 0.00355473021 0.0051375283293989922 0 +418 0 -1.47002935 0.06883432 0.10289020866533535 0 +419 0 -2.834514 0.0110854926 0.016082290810803899 0 +420 0 -1.89718175 0.03934594 0.057911095642695219 0 +421 1 6.608501 0.999808967 0.00027562852590240635 1 +422 0 -1.64955616 0.0545311421 0.080898155819311895 0 +423 0 -2.07045174 0.03122702 0.045769468245393054 0 +424 0 -3.17440128 0.00695839245 0.010073928241131102 0 +425 1 8.123915 0.999976456 3.3966973403377047E-05 1 +426 0 -1.20521867 0.09633162 0.14613465019460817 0 +427 1 2.66850567 0.9575598 0.062565471998703437 1 +428 0 -3.66275549 0.00355473021 0.0051375283293989922 0 +429 0 -3.36269617 0.005372264 0.0077714324034035831 0 +430 0 -3.155127 0.00714494335 0.010344975631523702 0 +431 0 -1.87725413 0.04040049 0.059495674186448806 0 +432 0 -2.43023157 0.019225616 0.028006795885563748 0 +433 0 -2.586056 0.0155578861 0.022621718061226741 0 +434 0 3.19198418 0.978957355 5.5705401326639343 1 +435 1 4.54872 0.9967162 0.0047453156939425917 1 +436 1 2.98793674 0.972290158 0.040541176703236259 1 +437 0 -2.9438014 0.009545941 0.013838035728266303 0 +438 0 -2.313085 0.02252916 0.032874431327864467 0 +439 0 -2.643742 0.0143825654 0.020900318768217858 0 +440 1 4.44634533 0.99621886 0.0054653709011308537 1 +441 0 -1.247864 0.09132043 0.1381564563530977 0 +442 0 -3.03585482 0.008414905 0.012191508552797105 0 +443 0 -3.50529838 0.00441532163 0.00638406686750772 0 +444 0 -1.96283352 0.0360553935 0.052977851179488201 0 +445 0 -3.45500565 0.004731709 0.0068426147359647426 0 +446 0 -3.818494 0.002868181 0.0041438561512361881 0 +447 0 -2.643742 0.0143825654 0.020900318768217858 0 +448 0 -3.93013477 0.00245900918 0.0035519693011514517 0 +449 1 5.54220772 0.9991663 0.0012032623125098148 1 +450 0 -2.6591115 0.0140844695 0.02046404766346872 0 +451 0 -2.643742 0.0143825654 0.020900318768217858 0 +452 0 -2.98848677 0.008979245 0.01301282285033965 0 +453 1 4.260419 0.9951161 0.0070632192737983259 1 +454 0 -3.079223 0.007929129 0.011484907568785944 0 +455 1 0.2693262 0.4500955 1.1516969394856902 1 +456 1 5.21796036 0.9986955 0.0018832347623362743 1 +457 1 4.641077 0.997108638 0.0041773953002259253 1 +458 0 -2.412013 0.0197062958 0.028714037388066598 0 +459 0 -2.20239353 0.0261568222 0.038238627358107499 0 +460 0 -2.51429367 0.01715259 0.024960644650324191 0 +461 0 -2.091114 0.030374404 0.044500311636638895 0 +462 0 -2.230789 0.02517532 0.036785318092455381 0 +463 0 -2.86242056 0.0106704608 0.015476941509396146 0 +464 0 -2.9438014 0.009545941 0.013838035728266303 0 +465 1 5.138484 0.9985442 0.0021017823742124804 1 +466 1 4.83243656 0.9977792 0.0032075137524430609 1 +467 1 4.171027 0.9944773 0.0079896950691364273 1 +468 0 -2.9438014 0.009545941 0.013838035728266303 0 +469 0 -3.43102646 0.004890406 0.0070726727911559172 0 +470 0 -3.14879227 0.007207334 0.010435636978038142 0 +471 0 -2.230789 0.02517532 0.036785318092455381 0 +472 0 -2.49261951 0.0176650751 0.025713102877142537 0 +473 0 -2.9438014 0.009545941 0.013838035728266303 0 +474 0 -2.643742 0.0143825654 0.020900318768217858 0 +475 0 -3.17440128 0.00695839245 0.010073928241131102 0 +476 0 -2.71207237 0.013103188 0.019028847714015511 0 +477 0 -2.9438014 0.009545941 0.013838035728266303 0 +478 0 -2.45954728 0.0184761714 0.026904802276599169 0 +479 1 4.53482342 0.9966527 0.0048372011051459173 1 +480 0 -2.74690437 0.0124949012 0.018139897594862873 0 +481 0 -1.818849 0.04364936 0.064388425356882634 0 +482 1 7.491995 0.9999437 8.1264091210582313E-05 1 +483 1 5.842266 0.999449253 0.00079477872804858344 1 +484 0 -2.412013 0.0197062958 0.028714037388066598 0 +485 0 -3.11580372 0.0075410814 0.010920709671793986 0 +486 0 -3.14879227 0.007207334 0.010435636978038142 0 +487 1 6.58712864 0.999803245 0.00028388529408090524 1 +488 1 0.4243822 0.5035153 0.9898924687479479 1 +489 1 -0.6615186 0.184362531 2.43938261837227 0 +490 0 -3.818494 0.002868181 0.0041438561512361881 0 +491 1 3.82046247 0.99106437 0.012949331320132883 1 +492 0 -2.84991145 0.0108545618 0.015745432826577192 0 +493 1 5.738032 0.9993639 0.00091799142349248603 1 +494 0 -0.120578527 0.323161334 0.56311610630170894 0 +495 0 -3.14879227 0.007207334 0.010435636978038142 0 +496 0 -3.93013477 0.00245900918 0.0035519693011514517 0 +497 0 -2.7596066 0.0122800851 0.017826096301800484 0 +498 0 -2.76257753 0.01223037 0.017753482924837489 0 +499 0 -2.76257753 0.01223037 0.017753482924837489 0 +500 0 -2.0173955 0.0335237458 0.049193808305993285 0 +501 0 -2.76257753 0.01223037 0.017753482924837489 0 +502 0 -2.70613074 0.0132098272 0.019184746686686661 0 +503 0 -2.60325241 0.01519796 0.022094344146968328 0 +504 0 -3.818494 0.002868181 0.0041438561512361881 0 +505 0 -2.86234832 0.010671515 0.015478478886294982 0 +506 1 5.72246838 0.9993501 0.0009379542473862564 1 +507 0 -2.95266986 0.009430723 0.013670219512809497 0 +508 0 -2.643742 0.0143825654 0.020900318768217858 0 +509 0 -3.45500565 0.004731709 0.0068426147359647426 0 +510 0 -3.818494 0.002868181 0.0041438561512361881 0 +511 0 -2.50502563 0.01736992 0.025279692310691083 0 +512 0 -2.643742 0.0143825654 0.020900318768217858 0 +513 0 -3.14879227 0.007207334 0.010435636978038142 0 +514 1 5.454713 0.99905926 0.0013578398766872189 1 +515 1 4.653659 0.997158349 0.0041054723672408746 1 +516 0 -3.93013477 0.00245900918 0.0035519693011514517 0 +517 0 -3.748911 0.00315686315 0.0045615947751170118 0 +518 0 -2.84912586 0.0108662285 0.015762449088241814 0 +519 1 3.57685065 0.987531066 0.01810196142028456 1 +520 0 -3.65329981 0.00360133173 0.0052050015309588298 0 +521 0 -2.85764551 0.01074037 0.015578889986020614 0 +522 1 2.19706678 0.9216215 0.1177537196139372 1 +523 1 4.184642 0.9945797 0.0078411491990904338 1 +524 0 -3.02443838 0.008547617 0.012384610013862075 0 +525 0 -3.10732222 0.00762934 0.01104901319263845 0 +526 0 -2.9438014 0.009545941 0.013838035728266303 0 +527 0 -2.60325241 0.01519796 0.022094344146968328 0 +528 0 -1.82377791 0.0433658138 0.063960746599059565 0 +529 0 -2.84991145 0.0108545618 0.015745432826577192 0 +530 1 3.43132186 0.9847948 0.022104957463047484 1 +531 0 -2.38711739 0.0203822348 0.029709157760637532 0 +532 0 -3.57871246 0.0039909156 0.0057691940062725823 0 +533 0 -3.02443838 0.008547617 0.012384610013862075 0 +534 0 -3.36269617 0.005372264 0.0077714324034035831 0 +535 0 -2.95907354 0.009348384 0.013550303254585653 0 +536 0 -2.35020351 0.02142645 0.031247807073335927 0 +537 0 -2.15964055 0.0277051888 0.040534272712979186 0 +538 0 -2.76257753 0.01223037 0.017753482924837489 0 +539 0 -2.230404 0.0251883864 0.036804655966841821 0 +540 0 -2.19188213 0.0265295189 0.038790862627098403 0 +541 0 -3.17440128 0.00695839245 0.010073928241131102 0 +542 0 -2.30772376 0.0226929486 0.033116193936559651 0 +543 0 -2.76257753 0.01223037 0.017753482924837489 0 +544 0 -2.84279156 0.0109607475 0.015900315782777862 0 +545 0 -2.50502563 0.01736992 0.025279692310691083 0 +546 1 6.397567 0.9997443 0.00036894995908781636 1 +547 0 -3.62020016 0.00376930786 0.0054482362641349244 0 +548 0 -3.27568221 0.00605478743 0.0087617639582233484 0 +549 1 3.01967335 0.973448 0.03882421216202165 1 +550 0 -3.02443838 0.008547617 0.012384610013862075 0 +551 0 -3.48153138 0.00456212368 0.006596812411908696 0 +552 0 -2.15863013 0.0277428385 0.04059013849484016 0 +553 0 -0.9062127 0.138796017 0.21557310230304613 0 +554 0 -3.17440128 0.00695839245 0.010073928241131102 0 +555 0 -1.33554757 0.081748046 0.12303803390524871 0 +556 0 -2.13086843 0.028796969 0.042155170793399907 0 +557 0 -2.51429367 0.01715259 0.024960644650324191 0 +558 0 -3.36269617 0.005372264 0.0077714324034035831 0 +559 0 -2.50502563 0.01736992 0.025279692310691083 0 +560 0 -2.35020351 0.02142645 0.031247807073335927 0 +561 0 -2.35020351 0.02142645 0.031247807073335927 0 +562 0 -3.48153138 0.00456212368 0.006596812411908696 0 +563 0 -3.02443838 0.008547617 0.012384610013862075 0 +564 0 -2.50450349 0.0173822455 0.025297788139068099 0 +565 1 6.09452724 0.9996113 0.00056085843795891524 1 +566 0 -2.649684 0.0142665906 0.020730570923947089 0 +567 0 -2.33630371 0.0218330622 0.031847392670864477 0 +568 1 2.62455773 0.9550211 0.066395511984124703 1 +569 1 5.197979 0.998658955 0.0019360172642625521 1 +570 1 4.760256 0.997546732 0.003543665522777626 1 +571 1 5.5805645 0.999209344 0.0011411261101997576 1 +572 0 -3.02443838 0.008547617 0.012384610013862075 0 +573 0 -3.66275549 0.00355473021 0.0051375283293989922 0 +574 1 3.3156774 0.9822058 0.025902741115356405 1 +575 0 -2.15964055 0.0277051888 0.040534272712979186 0 +576 0 -2.50502563 0.01736992 0.025279692310691083 0 +577 0 -3.66275549 0.00355473021 0.0051375283293989922 0 +578 0 -3.66275549 0.00355473021 0.0051375283293989922 0 +579 0 -3.48153138 0.00456212368 0.006596812411908696 0 +580 0 -2.267355 0.02396412 0.034993911404263681 0 +581 1 4.7087574 0.9973662 0.0038047974474049828 1 +582 1 4.716048 0.997392535 0.0037666894142244574 1 +583 0 -3.17440128 0.00695839245 0.010073928241131102 0 +584 0 -2.11449575 0.0294367652 0.043105883489207263 0 +585 0 -3.818494 0.002868181 0.0041438561512361881 0 +586 1 7.51488447 0.9999454 7.8770204456819672E-05 1 +587 0 -2.43023157 0.019225616 0.028006795885563748 0 +588 1 3.07839251 0.9754673 0.03583454795985614 1 +589 0 -2.643742 0.0143825654 0.020900318768217858 0 +590 1 1.799304 0.8715491 0.19834610309157608 1 +591 1 3.14444041 0.97756 0.03274286502472773 1 +592 1 2.7422967 0.961517155 0.056615496855699998 1 +593 0 -2.412013 0.0197062958 0.028714037388066598 0 +594 1 2.89192533 0.9684811 0.046204165185519709 1 +595 0 -2.50502563 0.01736992 0.025279692310691083 0 +596 0 -2.56165981 0.0160829369 0.023391382670451242 0 +597 0 -1.97792816 0.03533715 0.051903286975312858 0 +598 0 -3.02443838 0.008547617 0.012384610013862075 0 +599 0 -1.89719248 0.0393453762 0.057910250861073126 0 +600 0 -3.02443838 0.008547617 0.012384610013862075 0 +601 0 -3.748911 0.00315686315 0.0045615947751170118 0 +602 0 -2.76257753 0.01223037 0.017753482924837489 0 +603 1 1.86636734 0.8815752 0.18184441169788296 1 +604 1 2.44571733 0.94312346 0.084481455264119362 1 +605 1 4.958701 0.998134255 0.0026942146090977994 1 +606 0 -2.88735461 0.0103126625 0.014955274568890915 0 +607 0 -3.818494 0.002868181 0.0041438561512361881 0 +608 1 5.848234 0.9994537 0.00078832583915587986 1 +609 0 -2.643742 0.0143825654 0.020900318768217858 0 +610 1 3.929288 0.9923028 0.011147706235112258 1 +611 1 3.500236 0.9861573 0.020110311294284793 1 +612 1 8.31541 0.99998194 2.6055606891255495E-05 1 +613 0 -2.929119 0.009739763 0.014120384701839598 0 +614 0 -3.44885159 0.004771942 0.0069009353381546635 0 +615 0 -2.345525 0.0215624776 0.031448363568806384 0 +616 0 -3.02443838 0.008547617 0.012384610013862075 0 +617 0 -4.05037642 0.00208322145 0.0030085881224846938 0 +618 0 -2.76257753 0.01223037 0.017753482924837489 0 +619 0 -2.50502563 0.01736992 0.025279692310691083 0 +620 0 -3.02443838 0.008547617 0.012384610013862075 0 +621 0 -0.510834932 0.217760727 0.35431812489317488 0 +622 0 -1.5177772 0.06472213 0.096533040376405782 0 +623 0 -3.818494 0.002868181 0.0041438561512361881 0 +624 0 -2.492227 0.0176744927 0.025726933897705338 0 +625 0 -1.88094628 0.0402030833 0.059198916421936115 0 +626 1 2.84924865 0.9666296 0.04896488506679339 1 +627 0 -2.35552263 0.0212728176 0.03102132646627356 0 +628 0 -3.45500565 0.004731709 0.0068426147359647426 0 +629 0 -2.9438014 0.009545941 0.013838035728266303 0 +630 0 -1.75057673 0.04776322 0.070607742504938911 0 +631 0 -2.50502563 0.01736992 0.025279692310691083 0 +632 0 -3.818494 0.002868181 0.0041438561512361881 0 +633 1 2.32481384 0.933465242 0.099331790241031212 1 +634 0 -3.17440128 0.00695839245 0.010073928241131102 0 +635 0 -2.54867387 0.0163694881 0.023811607311942599 0 +636 1 5.209161 0.9986795 0.0019063107246208386 1 +637 0 -1.46641433 0.06915532 0.10338763477821405 0 +638 0 -2.9438014 0.009545941 0.013838035728266303 0 +639 0 -2.51429367 0.01715259 0.024960644650324191 0 +640 0 -2.81574464 0.0113735693 0.016502617712402454 0 +641 0 -3.02443838 0.008547617 0.012384610013862075 0 +642 0 -3.02443838 0.008547617 0.012384610013862075 0 +643 0 -3.818494 0.002868181 0.0041438561512361881 0 +644 0 -3.45500565 0.004731709 0.0068426147359647426 0 +645 0 -3.02443838 0.008547617 0.012384610013862075 0 +646 0 -3.42805529 0.0049104346 0.0071017101873699076 0 +647 0 -3.50710225 0.00440437347 0.0063682020462816225 0 +648 1 5.847275 0.999453 0.00078935829943915488 1 +649 0 -3.02443838 0.008547617 0.012384610013862075 0 +650 0 -2.21562624 0.0256948769 0.037554443078676135 0 +651 0 -3.184134 0.00686603738 0.0099397606460636164 0 +652 0 -2.43023157 0.019225616 0.028006795885563748 0 +653 0 -2.76257753 0.01223037 0.017753482924837489 0 +654 0 -2.881413 0.0103968307 0.015077974249878848 0 +655 0 -3.02443838 0.008547617 0.012384610013862075 0 +656 0 -2.50502563 0.01736992 0.025279692310691083 0 +657 0 -0.05579424 0.343051553 0.60614793263108213 0 +658 1 4.86108828 0.99786526 0.0030830712707291879 1 +659 0 -3.818494 0.002868181 0.0041438561512361881 0 +660 0 -3.66275549 0.00355473021 0.0051375283293989922 0 +661 0 -2.60325241 0.01519796 0.022094344146968328 0 +662 0 -3.19929743 0.00672457 0.0097342699091146449 0 +663 0 -3.19929743 0.00672457 0.0097342699091146449 0 +664 0 -2.79228544 0.0117440512 0.017043360582249374 0 +665 0 -3.818494 0.002868181 0.0041438561512361881 0 +666 0 -2.04716063 0.0322158858 0.047242836783368811 0 +667 0 -2.881413 0.0103968307 0.015077974249878848 0 +668 1 1.36945009 0.7892656 0.34141727320538723 1 +669 1 4.543991 0.996694744 0.0047763748964969885 1 +670 1 3.64351487 0.9886163 0.016517419441037965 1 +671 0 -2.600889 0.0152469371 0.022166095722467707 0 +672 0 -3.181472 0.006891175 0.0099762780042885069 0 +673 0 -2.03519154 0.03273576 0.048018033673701298 0 +674 0 -3.66275549 0.00355473021 0.0051375283293989922 0 +675 0 -2.298231 0.022985816 0.033548587928921486 0 +676 0 -3.43102646 0.004890406 0.0070726727911559172 0 +677 0 -2.643742 0.0143825654 0.020900318768217858 0 +678 0 -3.818494 0.002868181 0.0041438561512361881 0 +679 0 -3.45500565 0.004731709 0.0068426147359647426 0 +680 1 8.440186 0.9999848 2.1927954628943402E-05 1 +681 1 5.74694443 0.9993717 0.00090671943374659176 1 +682 0 -2.2894578 0.0232597552 0.033953153368665862 0 +683 0 -3.818494 0.002868181 0.0041438561512361881 0 +684 0 -3.818494 0.002868181 0.0041438561512361881 0 +685 0 -3.818494 0.002868181 0.0041438561512361881 0 +686 0 -3.818494 0.002868181 0.0041438561512361881 0 +687 0 -2.865121 0.0106311264 0.015419583039454196 0 +688 0 -2.9438014 0.009545941 0.013838035728266303 0 +689 0 -3.00721264 0.00875179749 0.012681750193826559 0 +690 0 -3.50710225 0.00440437347 0.0063682020462816225 0 +691 1 3.1353004 0.977281153 0.033154425243608135 1 +692 0 -3.17440128 0.00695839245 0.010073928241131102 0 +693 0 -2.92984581 0.009730079 0.01410627645602513 0 +694 0 -2.74522161 0.0125236362 0.018181878627427504 0 +695 0 -3.45500565 0.004731709 0.0068426147359647426 0 +696 1 4.02798843 0.9932779 0.0097306717027177518 1 +697 1 2.32177043 0.9332035 0.099736348264798103 1 +698 1 2.84598112 0.966483653 0.049182764441884248 1 diff --git a/test/BaselineOutput/Common/WeightedEnsemble/WE-RandomPartition-TrainTest-breast-cancer-out.txt b/test/BaselineOutput/Common/WeightedEnsemble/WE-RandomPartition-TrainTest-breast-cancer-out.txt new file mode 100644 index 0000000000..1708f624e7 --- /dev/null +++ b/test/BaselineOutput/Common/WeightedEnsemble/WE-RandomPartition-TrainTest-breast-cancer-out.txt @@ -0,0 +1,58 @@ +maml.exe TrainTest test=%Data% tr=WeightedEnsemble{nm=5 st=RandomPartitionSelector tp=-} dout=%Output% loader=Text{col=Label:BL:0 col=Features:R4:1-9} data=%Data% out=%Output% seed=1 +Automatically adding a MinMax normalization transform, use 'norm=Warn' or 'norm=No' to turn this behavior off. +Training 5 learners for the batch 1 +Beginning training model 1 of 5 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 4 instances with missing features during training (over 1 iterations; 4 inst/iter) +Trainer 1 of 5 finished in %Time% +Beginning training model 2 of 5 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 5 instances with missing features during training (over 1 iterations; 5 inst/iter) +Trainer 2 of 5 finished in %Time% +Beginning training model 3 of 5 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 3 instances with missing features during training (over 1 iterations; 3 inst/iter) +Trainer 3 of 5 finished in %Time% +Beginning training model 4 of 5 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 1 instances with missing features during training (over 1 iterations; 1 inst/iter) +Trainer 4 of 5 finished in %Time% +Beginning training model 5 of 5 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 3 instances with missing features during training (over 1 iterations; 3 inst/iter) +Trainer 5 of 5 finished in %Time% +Training calibrator. +Warning: The predictor produced non-finite prediction values on 16 instances during testing. Possible causes: abnormal data or the predictor is numerically unstable. +TEST POSITIVE RATIO: 0.3499 (239.0/(239.0+444.0)) +Confusion table + ||====================== +PREDICTED || positive | negative | Recall +TRUTH ||====================== + positive || 235 | 4 | 0.9833 + negative || 13 | 431 | 0.9707 + ||====================== +Precision || 0.9476 | 0.9908 | +OVERALL 0/1 ACCURACY: 0.975110 +LOG LOSS/instance: 0.126392 +Test-set entropy (prior Log-Loss/instance): 0.934003 +LOG-LOSS REDUCTION (RIG): 0.864677 +AUC: 0.995448 + +OVERALL RESULTS +--------------------------------------- +AUC: 0.995448 (0.0000) +Accuracy: 0.975110 (0.0000) +Positive precision: 0.947581 (0.0000) +Positive recall: 0.983264 (0.0000) +Negative precision: 0.990805 (0.0000) +Negative recall: 0.970721 (0.0000) +Log-loss: 0.126392 (0.0000) +Log-loss reduction: 0.864677 (0.0000) +F1 Score: 0.965092 (0.0000) +AUPRC: 0.990701 (0.0000) + +--------------------------------------- +Physical memory usage(MB): %Number% +Virtual memory usage(MB): %Number% +%DateTime% Time elapsed(s): %Number% + diff --git a/test/BaselineOutput/Common/WeightedEnsemble/WE-RandomPartition-TrainTest-breast-cancer-rp.txt b/test/BaselineOutput/Common/WeightedEnsemble/WE-RandomPartition-TrainTest-breast-cancer-rp.txt new file mode 100644 index 0000000000..41f59ab412 --- /dev/null +++ b/test/BaselineOutput/Common/WeightedEnsemble/WE-RandomPartition-TrainTest-breast-cancer-rp.txt @@ -0,0 +1,4 @@ +WeightedEnsemble +AUC Accuracy Positive precision Positive recall Negative precision Negative recall Log-loss Log-loss reduction F1 Score AUPRC /bp /nm /st Learner Name Train Dataset Test Dataset Results File Run Time Physical Memory Virtual Memory Command Line Settings +0.995448 0.97511 0.947581 0.983264 0.990805 0.970721 0.126392 0.864677 0.965092 0.990701 Microsoft.ML.Runtime.ComponentFactoryUtils+SimpleComponentFactory`1[Microsoft.ML.Trainers.LinearSvmTrainer] 5 RandomPartitionSelector WeightedEnsemble %Data% %Data% %Output% 99 0 0 maml.exe TrainTest test=%Data% tr=WeightedEnsemble{nm=5 st=RandomPartitionSelector tp=-} dout=%Output% loader=Text{col=Label:BL:0 col=Features:R4:1-9} data=%Data% out=%Output% seed=1 /bp:Microsoft.ML.Runtime.ComponentFactoryUtils+SimpleComponentFactory`1[Microsoft.ML.Trainers.LinearSvmTrainer];/nm:5;/st:RandomPartitionSelector + diff --git a/test/BaselineOutput/Common/WeightedEnsemble/WE-RandomPartition-TrainTest-breast-cancer.txt b/test/BaselineOutput/Common/WeightedEnsemble/WE-RandomPartition-TrainTest-breast-cancer.txt new file mode 100644 index 0000000000..9b8991888c --- /dev/null +++ b/test/BaselineOutput/Common/WeightedEnsemble/WE-RandomPartition-TrainTest-breast-cancer.txt @@ -0,0 +1,700 @@ +Instance Label Score Probability Log-loss Assigned +0 0 -1.88945413 0.0146355424 0.021270660765179913 0 +1 0 2.388715 0.9462658 4.218016030721552 1 +2 0 -1.95107377 0.0132358046 0.019222726322552175 0 +3 0 2.181253 0.925892234 3.7542314515478239 1 +4 0 -1.67818642 0.0206327718 0.030078173772772502 0 +5 1 6.49601841 0.999936461 9.1669665219582821E-05 1 +6 0 -0.472200632 0.134147048 0.20780606234723856 0 +7 0 -2.16003823 0.00940348 0.013630542459781892 0 +8 0 -2.301506 0.00745579926 0.010796744002841982 0 +9 0 -1.88065124 0.0148470653 0.021580389633454124 0 +10 0 -2.67285872 0.004047238 0.0058507776534977608 0 +11 0 -2.35600328 0.006817358 0.0098690474636270696 0 +12 1 0.0327115059 0.263195634 1.9257925382611054 1 +13 0 -2.02741218 0.011683749 0.016955331728356806 0 +14 1 5.0282917 0.99928 0.0010391493624186305 1 +15 1 1.02427888 0.6481865 0.62551911064298549 1 +16 0 -2.07431316 0.0108208638 0.015696284165498346 0 +17 0 -2.03029919 0.0116287228 0.016875009355058094 0 +18 1 4.16688442 0.9970126 0.0043163357233667772 1 +19 0 -1.74860907 0.01840535 0.026800708897476874 0 +20 1 3.57824659 0.992127657 0.011402330948237581 1 +21 1 3.85234022 0.9949834 0.0072556746590812663 1 +22 0 -2.21515846 0.008590921 0.012447624039547705 0 +23 1 ? ? ? 0 +24 0 -2.54023266 0.005035263 0.0072826995691439942 0 +25 1 1.14677 0.6929045 0.52927162674574246 1 +26 0 -2.08930445 0.010558581 0.015313801450601652 0 +27 0 -1.93346822 0.0136216721 0.019786992746044799 0 +28 0 -2.35600328 0.006817358 0.0098690474636270696 0 +29 0 -2.27498078 0.00778767653 0.011279218989776345 0 +30 0 -2.27677822 0.00776473153 0.011245856930144569 0 +31 0 -2.31918931 0.007242388 0.010486577319588677 0 +32 1 3.55068755 0.9917635 0.011932004171204935 1 +33 0 -2.1777215 0.009134833 0.013239340000729637 0 +34 0 -2.039102 0.0114625171 0.016632424437498333 0 +35 0 -2.35600328 0.006817358 0.0098690474636270696 0 +36 1 4.612729 0.9985691 0.0020658721602730876 1 +37 0 -1.44051051 0.03027171 0.044347523645436177 0 +38 1 3.171527 0.9846867 0.022263362877881662 1 +39 1 1.41202211 0.77774626 0.36262854238607223 1 +40 0 ? ? ? 0 +41 1 1.1915319 0.7084338 0.49729503486169196 1 +42 1 4.844702 0.9990247 0.0014077626727518485 1 +43 1 1.15632081 0.6962565 0.52230916583888853 1 +44 1 4.844701 0.9990247 0.0014077626727518485 1 +45 0 -2.54907465 0.004962503 0.0071772016458376666 0 +46 1 3.12731886 0.9835438 0.02393877423942881 1 +47 0 -2.566758 0.004820112 0.0069707653436585594 0 +48 0 -1.67818642 0.0206327718 0.030078173772772502 0 +49 1 3.631297 0.9927844 0.010447676595693744 1 +50 1 1.49963069 0.801790833 0.31870217101098014 1 +51 1 0.404064655 0.397704661 1.3302306250070643 1 +52 1 2.56144714 0.959075868 0.060283150086936022 1 +53 1 3.85233974 0.9949834 0.0072556746590812663 1 +54 1 3.260194 0.9867484 0.019245823952805156 1 +55 1 2.89927888 0.9761813 0.034778937738260109 1 +56 1 3.93191576 0.9955995 0.0063625770977561854 1 +57 1 0.368697166 0.38377583 1.381664239657491 1 +58 1 0.971461535 0.628011048 0.67113815595524395 1 +59 1 0.636954546 0.49256736 1.0216070629650555 1 +60 1 1.19097853 0.7082447 0.49768023232106878 1 +61 0 -2.3103478 0.00734832324 0.01064053259217026 0 +62 1 4.38695431 0.9979224 0.0030004315371227495 1 +63 1 0.70214653 0.5195212 0.94474553716625886 1 +64 0 -2.566758 0.004820112 0.0069707653436585594 0 +65 1 0.828467369 0.57129 0.80770477718615641 1 +66 0 -2.03029919 0.0116287228 0.016875009355058094 0 +67 1 2.82885575 0.97331655 0.03901900806061543 1 +68 1 3.87886572 0.9951978 0.0069448378299051591 1 +69 0 -2.46949887 0.005656847 0.0081842753063353438 0 +70 0 -1.58135557 0.0241311677 0.035240848409215604 0 +71 1 3.57186413 0.992044747 0.011522899033465331 1 +72 0 -1.567642 0.0246712659 0.036039534295816682 0 +73 1 5.00176525 0.9992477 0.0010857049412384345 1 +74 1 1.42040563 0.780134559 0.35820511143251743 1 +75 0 -2.02741241 0.0116837444 0.016955324930864327 0 +76 0 -2.08930445 0.010558581 0.015313801450601652 0 +77 0 -1.37008786 0.033885628 0.049734104406440646 0 +78 0 -1.73100352 0.018939035 0.027585303854584502 0 +79 0 -2.29438353 0.00754351029 0.010924240448467558 0 +80 0 -2.01857066 0.0118538728 0.017203691335993775 0 +81 0 -1.8982569 0.01442699 0.020965347242305369 0 +82 0 -1.6957922 0.0200523268 0.0292233800397287 0 +83 0 -1.7750175 0.017632382 0.02566508925576803 0 +84 1 4.21485138 0.997239947 0.0039874195943072985 1 +85 1 4.23207569 0.997317255 0.0038755844972378898 1 +86 1 1.88947582 0.885188162 0.17594393686479112 1 +87 1 3.56056356 0.991895854 0.011739444157028895 1 +88 0 -2.03029919 0.0116287228 0.016875009355058094 0 +89 0 -2.11832714 0.0100685563 0.014599478028311253 0 +90 0 -2.54023266 0.005035263 0.0072826995691439942 0 +91 0 -2.083116 0.01066608 0.015470552992552487 0 +92 0 -2.03029919 0.0116287228 0.016875009355058094 0 +93 0 -2.566758 0.004820112 0.0069707653436585594 0 +94 0 -2.31918931 0.007242388 0.010486577319588677 0 +95 0 -2.54023266 0.005035263 0.0072826995691439942 0 +96 0 -2.230772 0.008373677 0.01213152591619099 0 +97 0 -1.88945413 0.0146355424 0.021270660765179913 0 +98 1 4.524313 0.998344064 0.0023909912144566342 1 +99 1 4.95755672 0.9991907 0.0011680629440896843 1 +100 1 2.969701 0.978745162 0.030994823082120003 1 +101 1 0.200704575 0.320497125 1.6416166778506516 1 +102 0 -1.8542428 0.0154998591 0.022536682309307329 0 +103 1 -0.058467865 0.235002413 2.0892525220507543 0 +104 1 5.47921944 0.9996584 0.00049290047681786698 1 +105 1 0.8461504 0.578439832 0.789761194282116 1 +106 1 5.39927864 0.9996101 0.00056257893421877482 1 +107 1 4.26790142 0.9974713 0.0036528024004950014 1 +108 0 -2.42529035 0.0060834894 0.0088034249870077297 0 +109 1 2.95932579 0.97838515 0.031525587546841906 1 +110 0 -1.59416723 0.0236370228 0.03451050445484477 0 +111 1 2.32709527 0.940839946 0.087978779332385779 1 +112 1 3.85878587 0.995036364 0.0071788449806203707 1 +113 1 4.74787045 0.9988554 0.001652237168155676 1 +114 0 -1.167623 0.04673843 0.069055959737964523 0 +115 0 -1.66058087 0.0212296452 0.030957689510691219 0 +116 0 -0.331355572 0.16358982 0.25771747418091318 0 +117 1 3.79044771 0.9944455 0.0080357837127376201 1 +118 0 -1.9774822 0.0126771368 0.018406159184860077 0 +119 0 -1.757412 0.0181440674 0.026416740956279775 0 +120 0 -2.23961377 0.00825308 0.011956082754189343 0 +121 0 -1.6165669 0.02279669 0.033269345237308644 0 +122 1 5.44329262 0.9996375 0.0005230940618429665 1 +123 1 1.99258828 0.901419759 0.14972902113142872 1 +124 1 3.84475231 0.9949204 0.0073470286565809955 1 +125 0 -2.566758 0.004820112 0.0069707653436585594 0 +126 1 3.65782213 0.99309206 0.010000632443032644 1 +127 0 -2.17114425 0.009233856 0.013383524044177808 0 +128 1 3.21618 0.985761642 0.020689250330912869 1 +129 0 -1.75384021 0.01824964 0.026571873325410108 0 +130 0 -1.58135557 0.0241311677 0.035240848409215604 0 +131 0 -2.31918931 0.007242388 0.010486577319588677 0 +132 1 5.735629 0.9997765 0.00032250351428699664 1 +133 0 -2.05393744 0.0111877108 0.016231421367196953 0 +134 0 -2.1535387 0.009504177 0.013777203248472718 0 +135 0 -1.317271 0.0368658081 0.054191274445953133 0 +136 0 -2.07431316 0.0108208638 0.015696284165498346 0 +137 0 -2.25917244 0.007992394 0.011576913076588022 0 +138 0 -1.8982569 0.01442699 0.020965347242305369 0 +139 0 ? ? ? 0 +140 0 -2.25917244 0.007992394 0.011576913076588022 0 +141 0 -2.4000175 0.006341602 0.0091781309117178744 0 +142 1 2.29188466 0.937513 0.093089408680511784 1 +143 0 -1.66058087 0.0212296452 0.030957689510691219 0 +144 0 -2.35600328 0.006817358 0.0098690474636270696 0 +145 0 ? ? ? 0 +146 1 1.12110925 0.6837974 0.54835911742200871 1 +147 0 -2.21515846 0.008590921 0.012447624039547705 0 +148 0 -0.128890514 0.214706108 0.34869541894771983 0 +149 1 5.63837051 0.999737442 0.00037884152472567251 1 +150 0 -2.67285872 0.004047238 0.0058507776534977608 0 +151 1 2.16864514 0.9244482 0.11333562443741119 1 +152 1 4.842614 0.999021351 0.0014125828962366625 1 +153 0 -1.67818666 0.0206327643 0.030078162797404599 0 +154 0 -2.59328318 0.0046141115 0.0066721606932722503 0 +155 1 2.51195455 0.9557384 0.065312271895014304 1 +156 0 -2.239614 0.008253077 0.011956078689802259 0 +157 0 -2.31918931 0.007242388 0.010486577319588677 0 +158 0 ? ? ? 0 +159 1 5.95667267 0.9998449 0.0002237667813599861 1 +160 1 4.9767437 0.999215961 0.0011315735518934089 1 +161 0 -1.8542428 0.0154998591 0.022536682309307329 0 +162 0 -2.17114425 0.009233856 0.013383524044177808 0 +163 0 -1.96552014 0.0129272491 0.018771674306787799 0 +164 0 ? ? ? 0 +165 0 -1.63417244 0.0221567564 0.032324887138755184 0 +166 1 4.90450668 0.9991165 0.0012752128376593949 1 +167 1 4.020334 0.9961961 0.0054983446411586423 1 +168 0 -2.17114425 0.009233856 0.013383524044177808 0 +169 0 -2.58444142 0.004681786 0.0067702503537536854 0 +170 0 -2.25917244 0.007992394 0.011576913076588022 0 +171 0 -2.54023266 0.005035263 0.0072826995691439942 0 +172 0 -2.566758 0.004820112 0.0069707653436585594 0 +173 1 6.81432056 0.999962449 5.4175552199232265E-05 1 +174 1 2.90808129 0.976517558 0.034282110424092752 1 +175 1 5.169758 0.9994302 0.0008223113782815344 1 +176 0 -2.31918931 0.007242388 0.010486577319588677 0 +177 1 4.043646 0.99633944 0.005290760751438515 1 +178 0 -2.03029919 0.0116287228 0.016875009355058094 0 +179 1 1.42920852 0.782622457 0.35361158739671822 1 +180 0 -2.67285872 0.004047238 0.0058507776534977608 0 +181 0 -2.59328318 0.0046141115 0.0066721606932722503 0 +182 0 -1.74860907 0.01840535 0.026800708897476874 0 +183 1 4.931031 0.9991544 0.0012204750302656237 1 +184 1 3.29540563 0.9874889 0.018163526232138641 1 +185 0 -2.45181561 0.005823811 0.0084265445095301882 0 +186 1 3.2725687 0.987013459 0.018858337057947899 1 +187 1 5.717947 0.999769866 0.00033205071695870494 1 +188 1 4.259059 0.9974341 0.0037065980224333375 1 +189 0 -2.05393744 0.0111877108 0.016231421367196953 0 +190 1 6.45181 0.999931633 9.8635421988888176E-05 1 +191 1 6.2307663 0.9999015 0.00014215066385522485 1 +192 0 -1.93346822 0.0136216721 0.019786992746044799 0 +193 0 -2.54023266 0.005035263 0.0072826995691439942 0 +194 0 -2.17114425 0.009233856 0.013383524044177808 0 +195 0 -2.03029919 0.0116287228 0.016875009355058094 0 +196 0 2.97700834 0.978995264 5.5731415073973878 1 +197 0 -1.361285 0.0343656652 0.050451120314913403 0 +198 0 -2.59328318 0.0046141115 0.0066721606932722503 0 +199 0 -2.21515846 0.008590921 0.012447624039547705 0 +200 1 5.099024 0.9993594 0.00092444489238100217 1 +201 1 4.62463 0.9985969 0.0020256572262892943 1 +202 0 -2.54023266 0.005035263 0.0072826995691439942 0 +203 0 -1.88945413 0.0146355424 0.021270660765179913 0 +204 0 -2.54023266 0.005035263 0.0072826995691439942 0 +205 1 6.071615 0.9998718 0.00018497919931008834 1 +206 1 4.2108984 0.9972219 0.00401354731573734 1 +207 0 -2.67285872 0.004047238 0.0058507776534977608 0 +208 0 -2.67285872 0.004047238 0.0058507776534977608 0 +209 0 -1.95107388 0.0132358018 0.019222722237641723 0 +210 1 6.787796 0.9999608 5.658340173755871E-05 1 +211 1 5.21396732 0.999470353 0.00076432134586143009 1 +212 0 -2.54023266 0.005035263 0.0072826995691439942 0 +213 1 7.433243 0.9999865 1.9434170443242565E-05 1 +214 1 6.646327 0.999950469 7.1460561221065264E-05 1 +215 1 4.10875034 0.996712 0.004751354930960263 1 +216 0 -2.566758 0.004820112 0.0069707653436585594 0 +217 0 -2.54023266 0.005035263 0.0072826995691439942 0 +218 1 4.33770943 0.997746468 0.0032548288420523788 1 +219 0 -1.3964963 0.0324840471 0.047642645559460212 0 +220 0 -2.398765 0.00635467237 0.00919710766407697 0 +221 1 4.985546 0.9992273 0.0011152224727564242 1 +222 1 -0.903538465 0.0705411062 3.8253919907406178 0 +223 1 2.70561671 0.9674779 0.047699362239944788 1 +224 1 4.595046 0.998526633 0.0021271870222412284 1 +225 0 -2.566758 0.004820112 0.0069707653436585594 0 +226 1 4.77188 0.998899937 0.0015879294738724288 1 +227 1 3.87886572 0.9951978 0.0069448378299051591 1 +228 0 -2.67285872 0.004047238 0.0058507776534977608 0 +229 1 6.816532 0.9999626 5.3917568558478928E-05 1 +230 1 2.80017424 0.9720561 0.04088853040811892 1 +231 1 4.30326843 0.9976146 0.0034454838699692343 1 +232 0 0.80420804 0.561433136 1.1891312833707244 1 +233 1 3.013715 0.9802084 0.028839589303877214 1 +234 0 -1.09362841 0.0525055639 0.077810623475737736 0 +235 0 ? ? ? 0 +236 1 5.664896 0.999748766 0.0003624989745574528 1 +237 1 3.58708858 0.9922411 0.011237400419902939 1 +238 1 6.3191843 0.9999149 0.00012280083891345824 1 +239 1 2.51195455 0.9557384 0.065312271895014304 1 +240 0 -0.6834688 0.09847262 0.14955678304012165 0 +241 0 -1.94783664 0.013305936 0.019325265303788433 0 +242 0 -2.31918931 0.007242388 0.010486577319588677 0 +243 0 -1.48452449 0.0282056648 0.041277072232213781 0 +244 0 -2.54023266 0.005035263 0.0072826995691439942 0 +245 0 -1.49332738 0.0278091971 0.040688608826042451 0 +246 1 6.24845028 0.9999043 0.00013810867897905818 1 +247 1 1.75491261 0.860551357 0.21666680074302541 1 +248 0 -1.52343369 0.0264937654 0.038737876519248086 0 +249 0 ? ? ? 0 +250 0 -2.266139 0.007901527 0.011444769158360808 0 +251 1 4.14047527 0.996879637 0.004508770040804572 1 +252 0 2.54376364 0.957912 4.5704481887485322 1 +253 1 4.844702 0.9990247 0.0014077626727518485 1 +254 1 4.38695431 0.9979224 0.0030004315371227495 1 +255 1 3.04774284 0.9812715 0.027275727824793296 1 +256 0 -2.25917244 0.007992394 0.011576913076588022 0 +257 0 -2.21515846 0.008590921 0.012447624039547705 0 +258 0 -2.17114425 0.009233856 0.013383524044177808 0 +259 0 2.5823772 0.960413635 4.658852583017687 1 +260 1 4.74787045 0.9988554 0.001652237168155676 1 +261 1 5.97435665 0.9998494 0.0002173164459794357 1 +262 1 5.187442 0.9994466 0.00079865047523824744 1 +263 1 4.52779961 0.998353541 0.0023772959801739752 1 +264 1 3.63871551 0.992871761 0.010320702664525649 1 +265 0 -1.18522859 0.04545768 0.067118933067010111 0 +266 1 4.334138 0.9977331 0.0032741345338308117 1 +267 1 1.31477189 0.7486988 0.4175427086425138 1 +268 1 3.46330357 0.9904945 0.013779145767720701 1 +269 0 -2.54023266 0.005035263 0.0072826995691439942 0 +270 1 3.489829 0.9908989 0.013190215621813762 1 +271 0 -1.88945413 0.0146355424 0.021270660765179913 0 +272 1 1.31477189 0.7486988 0.4175427086425138 1 +273 1 0.188010931 0.3159409 1.6622734427847135 1 +274 0 -2.02741241 0.0116837444 0.016955324930864327 0 +275 0 ? ? ? 0 +276 0 -2.21515846 0.008590921 0.012447624039547705 0 +277 0 -2.566758 0.004820112 0.0069707653436585594 0 +278 0 -2.54023266 0.005035263 0.0072826995691439942 0 +279 1 3.295311 0.987487 0.018166312820423702 1 +280 0 -2.17114425 0.009233856 0.013383524044177808 0 +281 0 -2.1777215 0.009134833 0.013239340000729637 0 +282 1 2.53492212 0.9573183 0.062929397876851242 1 +283 1 2.69681382 0.9670165 0.048387561544603171 1 +284 1 4.02033329 0.9961961 0.0054983446411586423 1 +285 1 6.9734726 0.999971151 4.1620401857849153E-05 1 +286 1 7.857645 0.9999933 9.6310605954860058E-06 1 +287 0 -2.1535387 0.009504177 0.013777203248472718 0 +288 1 0.8638344 0.585557342 0.77211763918033938 1 +289 1 4.524312 0.998344064 0.0023909912144566342 1 +290 0 -2.59328318 0.0046141115 0.0066721606932722503 0 +291 0 -2.54023266 0.005035263 0.0072826995691439942 0 +292 1 ? ? ? 0 +293 1 3.09294057 0.9825974 0.025327657554674653 1 +294 0 ? ? ? 0 +295 1 4.10526466 0.9966931 0.0047787906402774333 1 +296 0 1.182137 0.7052129 1.7622546584304035 1 +297 0 ? ? ? 0 +298 0 -1.17642593 0.0460938141 0.068080706807467542 0 +299 1 3.172166 0.9847026 0.022240046325527927 1 +300 1 3.286469 0.9873049 0.018432369708828639 1 +301 0 -2.54023266 0.005035263 0.0072826995691439942 0 +302 1 7.38019276 0.9999853 2.1240013733369503E-05 1 +303 0 -2.54023266 0.005035263 0.0072826995691439942 0 +304 1 3.25139165 0.9865566 0.019526287528318573 1 +305 1 4.789564 0.998931646 0.0015421324352010058 1 +306 0 -2.54023266 0.005035263 0.0072826995691439942 0 +307 0 -2.54023266 0.005035263 0.0072826995691439942 0 +308 1 3.08310986 0.98231715 0.025739208492395773 1 +309 0 -1.20283437 0.0442103855 0.065235003423241969 0 +310 0 -2.29438353 0.00754351029 0.010924240448467558 0 +311 0 -2.59328318 0.0046141115 0.0066721606932722503 0 +312 1 2.811251 0.972549558 0.040156328278529572 1 +313 0 -2.59328318 0.0046141115 0.0066721606932722503 0 +314 0 -2.699384 0.00387413986 0.005600057084163724 0 +315 0 ? ? ? 0 +316 1 2.32709527 0.940839946 0.087978779332385779 1 +317 1 4.70998859 0.9987815 0.0017589925477972826 1 +318 0 -2.26797533 0.007877747 0.011410188477730752 0 +319 0 1.26195455 0.7319033 1.8991747120934499 1 +320 1 3.19805241 0.9853345 0.021314500240895127 1 +321 0 ? ? ? 0 +322 0 -2.17114425 0.009233856 0.013383524044177808 0 +323 1 3.18096948 0.984920442 0.021920900690988874 1 +324 0 -2.54023266 0.005035263 0.0072826995691439942 0 +325 0 -1.6560595 0.021385638 0.031187639245277173 0 +326 1 1.41160226 0.777626157 0.36285134757304588 1 +327 0 -2.566758 0.004820112 0.0069707653436585594 0 +328 1 3.02609015 0.9806017 0.028260791023580156 1 +329 1 3.14575744 0.9840303 0.023225346261805203 1 +330 1 2.300687 0.938360751 0.091785424648036962 1 +331 0 -1.78382039 0.0173818767 0.025297246655644021 0 +332 0 -1.21163726 0.04359904 0.064312515681515831 0 +333 1 2.95209622 0.978130758 0.03190075544347238 1 +334 1 3.58589816 0.992225945 0.011259413178325755 1 +335 0 -2.59328318 0.0046141115 0.0066721606932722503 0 +336 1 3.59470153 0.992337465 0.011097271954499971 1 +337 0 -2.54023266 0.005035263 0.0072826995691439942 0 +338 0 -2.699384 0.00387413986 0.005600057084163724 0 +339 1 3.09294057 0.9825974 0.025327657554674653 1 +340 1 2.59118 0.9609636 0.057446299927579179 1 +341 0 -2.54023266 0.005035263 0.0072826995691439942 0 +342 0 -2.4000175 0.006341602 0.0091781309117178744 0 +343 0 -2.59328318 0.0046141115 0.0066721606932722503 0 +344 1 4.77427864 0.9989043 0.0015816452076970466 1 +345 0 -2.59328318 0.0046141115 0.0066721606932722503 0 +346 0 -1.01797509 0.05909271 0.087875519901343818 0 +347 0 -2.58444142 0.004681786 0.0067702503537536854 0 +348 1 0.6958413 0.5169167 0.95199630439003224 1 +349 1 1.56233168 0.817762434 0.29024630352035607 1 +350 0 -1.62536955 0.0224745013 0.032793759327479917 0 +351 0 -2.31918931 0.007242388 0.010486577319588677 0 +352 0 0.966230869 0.6259871 1.4188401125282242 1 +353 1 5.01060772 0.999258637 0.0010699567676901072 1 +354 0 -2.566758 0.004820112 0.0069707653436585594 0 +355 0 -1.83663726 0.0159506649 0.023197448274278189 0 +356 1 -0.179489851 0.20092836 2.3152468858855864 0 +357 1 6.540228 0.999940932 8.521992042107883E-05 1 +358 1 3.94158268 0.995669 0.0062618715578363921 1 +359 1 2.73202538 0.9688249 0.045692116021997138 1 +360 1 7.07957268 0.9999758 3.4912900563974312E-05 1 +361 1 3.94960022 0.99572587 0.0061794813444101416 1 +362 0 -1.28205991 0.0389909931 0.057378142330293917 0 +363 0 -0.3489611 0.1596433 0.25092627947035062 0 +364 0 -2.31918931 0.007242388 0.010486577319588677 0 +365 0 -2.35600328 0.006817358 0.0098690474636270696 0 +366 1 6.734745 0.9999572 6.1743092852039014E-05 1 +367 1 5.417326 0.99962157 0.00054606225482140512 1 +368 0 -2.27498078 0.00778767653 0.011279218989776345 0 +369 0 -2.38108158 0.0065421 0.0094692636985393205 0 +370 0 -1.4493134 0.0298470929 0.043715944606827255 0 +371 0 -2.27498078 0.00778767653 0.011279218989776345 0 +372 0 -1.8982569 0.01442699 0.020965347242305369 0 +373 0 -1.757412 0.0181440674 0.026416740956279775 0 +374 0 -2.039102 0.0114625171 0.016632424437498333 0 +375 0 -2.59328318 0.0046141115 0.0066721606932722503 0 +376 0 -2.566758 0.004820112 0.0069707653436585594 0 +377 0 -2.699384 0.00387413986 0.005600057084163724 0 +378 0 -1.49690843 0.0276494641 0.040451590461704337 0 +379 0 -0.8507216 0.076489836 0.11480025549439182 0 +380 0 -2.59328318 0.0046141115 0.0066721606932722503 0 +381 1 4.541996 0.9983917 0.0023221718236317128 1 +382 0 -1.55880046 0.0250257086 0.036563917214848314 0 +383 0 -2.4000175 0.006341602 0.0091781309117178744 0 +384 0 -2.4000175 0.006341602 0.0091781309117178744 0 +385 0 -1.27586484 0.0393768661 0.057957542609429841 0 +386 1 2.74963117 0.969692767 0.044400372553701516 1 +387 0 -0.912341356 0.06959217 0.10406485743500918 0 +388 0 -2.27498078 0.00778767653 0.011279218989776345 0 +389 0 -1.4757216 0.02860762 0.041873925463189056 0 +390 0 -2.42529035 0.0060834894 0.0088034249870077297 0 +391 1 5.116708 0.9993779 0.00089777074124071778 1 +392 0 -2.21515846 0.008590921 0.012447624039547705 0 +393 0 -2.72590923 0.00370841729 0.0053600600643168772 0 +394 0 -2.08046269 0.0107125025 0.015538250436622488 0 +395 0 -2.21515846 0.008590921 0.012447624039547705 0 +396 0 -2.17114425 0.009233856 0.013383524044177808 0 +397 0 -2.11832738 0.0100685516 0.014599471241909705 0 +398 0 -1.8859446 0.0147195132 0.021393609661647217 0 +399 0 -2.106988 0.0102572711 0.014874531313555506 0 +400 1 4.382845 0.9979083 0.0030208540549018889 1 +401 0 -2.25917244 0.007992394 0.011576913076588022 0 +402 0 -1.19403148 0.0448298976 0.066170415078085548 0 +403 0 -1.59896111 0.0234546736 0.034241086441653891 0 +404 0 -2.071621 0.0108686443 0.015765972716961778 0 +405 0 -2.566758 0.004820112 0.0069707653436585594 0 +406 0 -1.70459485 0.0197681412 0.028805057942489269 0 +407 0 -2.566758 0.004820112 0.0069707653436585594 0 +408 0 -1.42617416 0.0309757963 0.045395394007152833 0 +409 0 -2.039102 0.0114625171 0.016632424437498333 0 +410 0 -2.566758 0.004820112 0.0069707653436585594 0 +411 0 ? ? ? 0 +412 1 4.9767437 0.999215961 0.0011315735518934089 1 +413 0 -1.71339774 0.0194878951 0.028392653544381613 0 +414 1 3.41864467 0.989773 0.014830443189656227 1 +415 0 0.0327117443 0.2631957 0.44064660110429327 1 +416 1 4.45737743 0.998150468 0.0026707814382228513 1 +417 0 -2.566758 0.004820112 0.0069707653436585594 0 +418 0 -0.842620134 0.07744204 0.11628854704371126 0 +419 0 -1.82783437 0.0161808822 0.023535004815539062 0 +420 0 -1.0091722 0.0599076971 0.089125680289544901 0 +421 1 6.06829262 0.9998711 0.00018601122790031077 1 +422 0 -1.08134675 0.05352567 0.07936471885819861 0 +423 0 -1.58135557 0.0241311677 0.035240848409215604 0 +424 0 -2.25917244 0.007992394 0.011576913076588022 0 +425 1 7.63660145 0.999990344 1.3930661976900586E-05 1 +426 0 -0.709876537 0.09466144 0.14347068738103194 0 +427 1 2.69158268 0.966739357 0.048801119089523212 1 +428 0 -2.566758 0.004820112 0.0069707653436585594 0 +429 0 -2.35600328 0.006817358 0.0098690474636270696 0 +430 0 -2.16003823 0.00940348 0.013630542459781892 0 +431 0 -1.23804569 0.041813083 0.061620979703453538 0 +432 0 -1.757412 0.0181440674 0.026416740956279775 0 +433 0 -1.541117 0.0257495567 0.037635412072888889 0 +434 0 3.506673 0.9911468 6.8195857442483767 1 +435 1 3.88519454 0.995247543 0.0068726904002030147 1 +436 1 3.39257 0.989326954 0.015480711616793305 1 +437 0 -2.11832738 0.0100685516 0.014599471241909705 0 +438 0 -1.41410184 0.031580966 0.046296659461484334 0 +439 0 -1.97748232 0.012677134 0.018406155102261037 0 +440 1 3.74434924 0.994007945 0.0086707116572397481 1 +441 0 -0.419383526 0.144625083 0.22537119196415259 0 +442 0 -1.79752731 0.01699876 0.024734859033136738 0 +443 0 -2.336873 0.00703503657 0.010185281443340742 0 +444 0 -0.9915664 0.06156933 0.091677929272589873 0 +445 0 -2.4000175 0.006341602 0.0091781309117178744 0 +446 0 -2.59328318 0.0046141115 0.0066721606932722503 0 +447 0 -1.97748232 0.012677134 0.018406155102261037 0 +448 0 -2.72590923 0.00370841729 0.0053600600643168772 0 +449 1 5.222808 0.999478042 0.00075322262914912983 1 +450 0 -1.78382039 0.0173818767 0.025297246655644021 0 +451 0 -1.97748232 0.012677134 0.018406155102261037 0 +452 0 -1.90705955 0.0142213712 0.020664390393237964 0 +453 1 3.87002373 0.9951273 0.0070469736538930012 1 +454 0 -1.95107377 0.0132358046 0.019222726322552175 0 +455 1 0.302447081 0.358205825 1.4811392964783026 1 +456 1 4.985546 0.9992273 0.0011152224727564242 1 +457 1 4.3817234 0.9979044 0.0030264552178903403 1 +458 0 -1.80142593 0.0168913156 0.024577176991795652 0 +459 0 -1.62536955 0.0224745013 0.032793759327479917 0 +460 0 -1.62536955 0.0224745013 0.032793759327479917 0 +461 0 -1.37889075 0.0334120654 0.049027108335364937 0 +462 0 -1.48452449 0.0282056648 0.041277072232213781 0 +463 0 -1.942271 0.0134273712 0.01950283267496988 0 +464 0 -2.11832738 0.0100685516 0.014599471241909705 0 +465 1 4.83377266 0.9990069 0.001433413332902892 1 +466 1 4.263715 0.997453749 0.0036781481647689394 1 +467 1 3.81697369 0.9946827 0.0076916676609583888 1 +468 0 -2.11832738 0.0100685516 0.014599471241909705 0 +469 0 -2.40760684 0.006262969 0.0090639680403581304 0 +470 0 -2.27677822 0.00776473153 0.011245856930144569 0 +471 0 -1.48452449 0.0282056648 0.041277072232213781 0 +472 0 -1.8982569 0.01442699 0.020965347242305369 0 +473 0 -2.11832738 0.0100685516 0.014599471241909705 0 +474 0 -1.97748232 0.012677134 0.018406155102261037 0 +475 0 -2.25917244 0.007992394 0.011576913076588022 0 +476 0 -1.942271 0.0134273712 0.01950283267496988 0 +477 0 -2.11832738 0.0100685516 0.014599471241909705 0 +478 0 -1.78382039 0.0173818767 0.025297246655644021 0 +479 1 4.488944 0.9982444 0.0025350143804774346 1 +480 0 -2.039102 0.0114625171 0.016632424437498333 0 +481 0 -0.8859329 0.07247491 0.10854178984472743 0 +482 1 6.51370239 0.9999383 8.900376720685931E-05 1 +483 1 5.19628239 0.9994546 0.00078703526484083107 1 +484 0 -1.80142593 0.0168913156 0.024577176991795652 0 +485 0 -1.93015325 0.0136955595 0.019895065872186559 0 +486 0 -2.27677822 0.00776473153 0.011245856930144569 0 +487 1 6.59646225 0.9999462 7.7652256621784743E-05 1 +488 1 1.35013008 0.7595444 0.39679384549866076 1 +489 1 -0.419383764 0.144625038 2.7896107574009181 0 +490 0 -2.59328318 0.0046141115 0.0066721606932722503 0 +491 1 3.62110972 0.992662668 0.010624557955311836 1 +492 0 -2.135933 0.009782338 0.014182412146275174 0 +493 1 5.77983856 0.9997922 0.00029979690770190096 1 +494 0 0.8306167 0.5721607 1.2248591549993364 1 +495 0 -2.27677822 0.00776473153 0.011245856930144569 0 +496 0 -2.72590923 0.00370841729 0.0053600600643168772 0 +497 0 -1.92466533 0.0138187483 0.020075268868213394 0 +498 0 -2.07431316 0.0108208638 0.015696284165498346 0 +499 0 -2.07431316 0.0108208638 0.015696284165498346 0 +500 0 -1.74860907 0.01840535 0.026800708897476874 0 +501 0 -2.07431316 0.0108208638 0.015696284165498346 0 +502 0 -1.8982569 0.01442699 0.020965347242305369 0 +503 0 -2.03029919 0.0116287228 0.016875009355058094 0 +504 0 -2.59328318 0.0046141115 0.0066721606932722503 0 +505 0 -1.87184858 0.0150615936 0.021894587151481272 0 +506 1 5.541111 0.999691665 0.00044490171932937031 1 +507 0 -1.88065147 0.01484706 0.021580381450271136 0 +508 0 -1.97748232 0.012677134 0.018406155102261037 0 +509 0 -2.4000175 0.006341602 0.0091781309117178744 0 +510 0 -2.59328318 0.0046141115 0.0066721606932722503 0 +511 0 -1.93346822 0.0136216721 0.019786992746044799 0 +512 0 -1.97748232 0.012677134 0.018406155102261037 0 +513 0 -2.27677822 0.00776473153 0.011245856930144569 0 +514 1 4.860298 0.999049544 0.0013718697293316642 1 +515 1 4.1971674 0.9971582 0.0041056448400095093 1 +516 0 -2.72590923 0.00370841729 0.0053600600643168772 0 +517 0 -2.699384 0.00387413986 0.005600057084163724 0 +518 0 -1.90705967 0.0142213684 0.020664386304243484 0 +519 1 3.08413815 0.982346654 0.025695877204860228 1 +520 0 -2.4792428 0.005566888 0.008053759790763778 0 +521 0 -2.05670762 0.0111371223 0.01615761370142613 0 +522 1 2.098222 0.915897369 0.12674214828458047 1 +523 1 4.175688 0.9970557 0.0042539790553121669 1 +524 0 -2.21515846 0.008590921 0.012447624039547705 0 +525 0 -2.083116 0.01066608 0.015470552992552487 0 +526 0 -2.11832738 0.0100685516 0.014599471241909705 0 +527 0 -2.03029919 0.0116287228 0.016875009355058094 0 +528 0 -1.3084681 0.0373864323 0.054971337146725134 0 +529 0 -2.135933 0.009782338 0.014182412146275174 0 +530 1 3.268786 0.986933 0.018975957563074363 1 +531 0 -1.70459485 0.0197681412 0.028805057942489269 0 +532 0 -2.67285872 0.004047238 0.0058507776534977608 0 +533 0 -2.21515846 0.008590921 0.012447624039547705 0 +534 0 -2.35600328 0.006817358 0.0098690474636270696 0 +535 0 -2.08046269 0.0107125025 0.015538250436622488 0 +536 0 -1.88945413 0.0146355424 0.021270660765179913 0 +537 0 -1.71339774 0.0194878951 0.028392653544381613 0 +538 0 -2.07431316 0.0108208638 0.015696284165498346 0 +539 0 -1.792623 0.0171348732 0.02493463787566573 0 +540 0 -1.71339774 0.0194878951 0.028392653544381613 0 +541 0 -2.25917244 0.007992394 0.011576913076588022 0 +542 0 -1.59015846 0.02379057 0.034737406889435905 0 +543 0 -2.07431316 0.0108208638 0.015696284165498346 0 +544 0 -1.88594437 0.01471952 0.021393619207458154 0 +545 0 -1.93346822 0.0136216721 0.019786992746044799 0 +546 1 6.09814 0.9998773 0.00017706700464490148 1 +547 0 -2.504866 0.005337061 0.0077203717076407804 0 +548 0 -2.3207922 0.007223346 0.010458904956010278 0 +549 1 3.03890085 0.9810008 0.027673809788536528 1 +550 0 -2.21515846 0.008590921 0.012447624039547705 0 +551 0 -2.54023266 0.005035263 0.0072826995691439942 0 +552 0 -1.5322752 0.0261190869 0.038182725630602758 0 +553 0 -0.331355572 0.16358982 0.25771747418091318 0 +554 0 -2.25917244 0.007992394 0.011576913076588022 0 +555 0 -0.838346958 0.07794865 0.11708100026054763 0 +556 0 -1.4493134 0.0298470929 0.043715944606827255 0 +557 0 -1.62536955 0.0224745013 0.032793759327479917 0 +558 0 -2.35600328 0.006817358 0.0098690474636270696 0 +559 0 -1.93346822 0.0136216721 0.019786992746044799 0 +560 0 -1.88945413 0.0146355424 0.021270660765179913 0 +561 0 -1.88945413 0.0146355424 0.021270660765179913 0 +562 0 -2.54023266 0.005035263 0.0072826995691439942 0 +563 0 -2.21515846 0.008590921 0.012447624039547705 0 +564 0 -1.8542428 0.0154998591 0.022536682309307329 0 +565 1 5.58531952 0.9997134 0.00041350554653493378 1 +566 0 -1.99508786 0.0123176891 0.017881022959617951 0 +567 0 -1.63417268 0.0221567471 0.032324873398163714 0 +568 1 2.635194 0.9636054 0.053485610652944007 1 +569 1 5.161602 0.999422431 0.00083349666752889878 1 +570 1 4.07885647 0.9965459 0.0049918233818767137 1 +571 1 4.877981 0.9990769 0.0013323627017050837 1 +572 0 -2.21515846 0.008590921 0.012447624039547705 0 +573 0 -2.566758 0.004820112 0.0069707653436585594 0 +574 1 3.08310986 0.98231715 0.025739208492395773 1 +575 0 -1.71339774 0.0194878951 0.028392653544381613 0 +576 0 -1.93346822 0.0136216721 0.019786992746044799 0 +577 0 -2.566758 0.004820112 0.0069707653436585594 0 +578 0 -2.566758 0.004820112 0.0069707653436585594 0 +579 0 -2.54023266 0.005035263 0.0072826995691439942 0 +580 0 -1.757412 0.0181440674 0.026416740956279775 0 +581 1 4.58620358 0.998505 0.0021584482708646314 1 +582 1 4.56852055 0.99846065 0.002222523029722704 1 +583 0 -2.25917244 0.007992394 0.011576913076588022 0 +584 0 -1.03558064 0.0574938431 0.085426052161287722 0 +585 0 -2.59328318 0.0046141115 0.0066721606932722503 0 +586 1 6.88505459 0.9999666 4.8155945935111852E-05 1 +587 0 -1.757412 0.0181440674 0.026416740956279775 0 +588 1 3.40103984 0.989473939 0.015266385244016521 1 +589 0 -1.97748232 0.012677134 0.018406155102261037 0 +590 1 1.76569128 0.862677634 0.21310654250125011 1 +591 1 3.04774284 0.9812715 0.027275727824793296 1 +592 1 2.520757 0.956350446 0.064388717584256586 1 +593 0 -1.80142593 0.0168913156 0.024577176991795652 0 +594 1 2.83765912 0.973692238 0.038462253171681907 1 +595 0 -1.93346822 0.0136216721 0.019786992746044799 0 +596 0 -1.8982569 0.01442699 0.020965347242305369 0 +597 0 -1.53734136 0.0259067267 0.037868172211473594 0 +598 0 -2.21515846 0.008590921 0.012447624039547705 0 +599 0 -1.317271 0.0368658081 0.054191274445953133 0 +600 0 -2.21515846 0.008590921 0.012447624039547705 0 +601 0 -2.699384 0.00387413986 0.005600057084163724 0 +602 0 -2.07431316 0.0108208638 0.015696284165498346 0 +603 1 1.93454123 0.8925507 0.16399396220239409 1 +604 1 2.3975184 0.9470016 0.078561267613256366 1 +605 1 4.665779 0.9986892 0.0018923617779227511 1 +606 0 -1.942271 0.0134273712 0.01950283267496988 0 +607 0 -2.59328318 0.0046141115 0.0066721606932722503 0 +608 1 5.80420876 0.999800444 0.00028792768739922532 1 +609 0 -1.97748232 0.012677134 0.018406155102261037 0 +610 1 3.65632081 0.993075 0.010025397246966026 1 +611 1 3.631297 0.9927844 0.010447676595693744 1 +612 1 7.04420662 0.99997437 3.6976743794448992E-05 1 +613 0 -2.00972867 0.0120264534 0.017455681302641062 0 +614 0 -2.47834063 0.00557515724 0.0080657565779257579 0 +615 0 -1.72220063 0.0192115437 0.027986096049624622 0 +616 0 -2.21515846 0.008590921 0.012447624039547705 0 +617 0 ? ? ? 0 +618 0 -2.07431316 0.0108208638 0.015696284165498346 0 +619 0 -1.93346822 0.0136216721 0.019786992746044799 0 +620 0 -2.21515846 0.008590921 0.012447624039547705 0 +621 0 -0.243327141 0.18450512 0.29425227559098643 0 +622 0 -1.12360883 0.050092034 0.07414035353621333 0 +623 0 -2.59328318 0.0046141115 0.0066721606932722503 0 +624 0 -1.81903172 0.0164143629 0.023877426941553922 0 +625 0 -1.18522859 0.04545768 0.067118933067010111 0 +626 1 2.83554125 0.9736023 0.038595526190172355 1 +627 0 -1.51973581 0.0266520157 0.038972415877474062 0 +628 0 -2.4000175 0.006341602 0.0091781309117178744 0 +629 0 -2.11832738 0.0100685516 0.014599471241909705 0 +630 0 -1.29086256 0.0384489335 0.056564615988171016 0 +631 0 -1.93346822 0.0136216721 0.019786992746044799 0 +632 0 -2.59328318 0.0046141115 0.0066721606932722503 0 +633 1 2.517239 0.956106842 0.0647562515953643 1 +634 0 -2.25917244 0.007992394 0.011576913076588022 0 +635 0 -1.73100352 0.018939035 0.027585303854584502 0 +636 1 4.65461159 0.998664737 0.0019276649289841729 1 +637 0 -0.780298948 0.08513698 0.12837234591520091 0 +638 0 -2.11832738 0.0100685516 0.014599471241909705 0 +639 0 -1.62536955 0.0224745013 0.032793759327479917 0 +640 0 -1.76621461 0.0178864319 0.02603823258893153 0 +641 0 -2.21515846 0.008590921 0.012447624039547705 0 +642 0 -2.21515846 0.008590921 0.012447624039547705 0 +643 0 -2.59328318 0.0046141115 0.0066721606932722503 0 +644 0 -2.4000175 0.006341602 0.0091781309117178744 0 +645 0 -2.21515846 0.008590921 0.012447624039547705 0 +646 0 -2.266139 0.007901527 0.011444769158360808 0 +647 0 -2.3103478 0.00734832324 0.01064053259217026 0 +648 1 5.693343 0.99976027 0.0003458985736538843 1 +649 0 -2.21515846 0.008590921 0.012447624039547705 0 +650 0 -1.361285 0.0343656652 0.050451120314913403 0 +651 0 -2.11582971 0.0101098223 0.014659619015222487 0 +652 0 -1.757412 0.0181440674 0.026416740956279775 0 +653 0 -2.07431316 0.0108208638 0.015696284165498346 0 +654 0 -2.17114425 0.009233856 0.013383524044177808 0 +655 0 -2.21515846 0.008590921 0.012447624039547705 0 +656 0 -1.93346822 0.0136216721 0.019786992746044799 0 +657 0 0.64575696 0.496207684 0.98909897788669376 1 +658 1 4.577362 0.998483 0.0021902269286327644 1 +659 0 -2.59328318 0.0046141115 0.0066721606932722503 0 +660 0 -2.566758 0.004820112 0.0069707653436585594 0 +661 0 -2.03029919 0.0116287228 0.016875009355058094 0 +662 0 -2.24845552 0.008134205 0.011783165639717253 0 +663 0 -2.24845552 0.008134205 0.011783165639717253 0 +664 0 -1.86304545 0.015279185 0.022213340821590894 0 +665 0 -2.59328318 0.0046141115 0.0066721606932722503 0 +666 0 -1.32607365 0.0363521762 0.053422101664556951 0 +667 0 -2.17114425 0.009233856 0.013383524044177808 0 +668 1 2.01899743 0.9052349 0.14363593078587411 1 +669 1 4.65461063 0.998664737 0.0019276649289841729 1 +670 1 3.8700242 0.9951273 0.0070469736538930012 1 +671 0 -1.80142593 0.0168913156 0.024577176991795652 0 +672 0 -2.31918931 0.007242388 0.010486577319588677 0 +673 0 -1.35248232 0.034852244 0.05117827106784497 0 +674 0 -2.566758 0.004820112 0.0069707653436585594 0 +675 0 -1.59015822 0.02379058 0.034737420653024181 0 +676 0 -2.40760684 0.006262969 0.0090639680403581304 0 +677 0 -1.97748232 0.012677134 0.018406155102261037 0 +678 0 -2.59328318 0.0046141115 0.0066721606932722503 0 +679 0 -2.4000175 0.006341602 0.0091781309117178744 0 +680 1 7.30061626 0.9999832 2.4249757573371186E-05 1 +681 1 5.576478 0.9997092 0.00041961269374402835 1 +682 0 -1.71339774 0.0194878951 0.028392653544381613 0 +683 0 -2.59328318 0.0046141115 0.0066721606932722503 0 +684 0 -2.59328318 0.0046141115 0.0066721606932722503 0 +685 0 -2.59328318 0.0046141115 0.0066721606932722503 0 +686 0 -2.59328318 0.0046141115 0.0066721606932722503 0 +687 0 -1.98320365 0.0125592025 0.018233841609591416 0 +688 0 -2.11832738 0.0100685516 0.014599471241909705 0 +689 0 -2.46949887 0.005656847 0.0081842753063353438 0 +690 0 -2.3103478 0.00734832324 0.01064053259217026 0 +691 1 3.3837285 0.9891714 0.015707588087033722 1 +692 0 -2.25917244 0.007992394 0.011576913076588022 0 +693 0 -2.10952473 0.0102147507 0.014812552839430559 0 +694 0 -1.9774822 0.0126771368 0.018406159184860077 0 +695 0 -2.4000175 0.006341602 0.0091781309117178744 0 +696 1 4.448575 0.9981234 0.0027098943698025662 1 +697 1 2.12820244 0.9196401 0.12085868275888806 1 +698 1 2.66754818 0.965436757 0.050746340022452187 1 diff --git a/test/BaselineOutput/Common/WeightedEnsemble/WE-StackingAP-TrainTest-breast-cancer-out.txt b/test/BaselineOutput/Common/WeightedEnsemble/WE-StackingAP-TrainTest-breast-cancer-out.txt new file mode 100644 index 0000000000..1d42306786 --- /dev/null +++ b/test/BaselineOutput/Common/WeightedEnsemble/WE-StackingAP-TrainTest-breast-cancer-out.txt @@ -0,0 +1,61 @@ +maml.exe TrainTest test=%Data% tr=WeightedEnsemble{nm=5 oc=Stacking{bp=ap} tp=-} dout=%Output% loader=Text{col=Label:BL:0 col=Features:R4:1-9} data=%Data% out=%Output% seed=1 +Automatically adding a MinMax normalization transform, use 'norm=Warn' or 'norm=No' to turn this behavior off. +Training 5 learners for the batch 1 +Beginning training model 1 of 5 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 12 instances with missing features during training (over 1 iterations; 12 inst/iter) +Trainer 1 of 5 finished in %Time% +Beginning training model 2 of 5 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 18 instances with missing features during training (over 1 iterations; 18 inst/iter) +Trainer 2 of 5 finished in %Time% +Beginning training model 3 of 5 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 15 instances with missing features during training (over 1 iterations; 15 inst/iter) +Trainer 3 of 5 finished in %Time% +Beginning training model 4 of 5 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 24 instances with missing features during training (over 1 iterations; 24 inst/iter) +Trainer 4 of 5 finished in %Time% +Beginning training model 5 of 5 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 18 instances with missing features during training (over 1 iterations; 18 inst/iter) +Trainer 5 of 5 finished in %Time% +The number of instances used for stacking trainer is 213 +Warning: The trainer specified for stacking wants normalization, but we do not currently allow this. +Warning: Skipped 4 instances with missing features during training (over 1 iterations; 4 inst/iter) +Training calibrator. +Warning: The predictor produced non-finite prediction values on 16 instances during testing. Possible causes: abnormal data or the predictor is numerically unstable. +TEST POSITIVE RATIO: 0.3499 (239.0/(239.0+444.0)) +Confusion table + ||====================== +PREDICTED || positive | negative | Recall +TRUTH ||====================== + positive || 226 | 13 | 0.9456 + negative || 9 | 435 | 0.9797 + ||====================== +Precision || 0.9617 | 0.9710 | +OVERALL 0/1 ACCURACY: 0.967789 +LOG LOSS/instance: 0.117470 +Test-set entropy (prior Log-Loss/instance): 0.934003 +LOG-LOSS REDUCTION (RIG): 0.874229 +AUC: 0.995929 + +OVERALL RESULTS +--------------------------------------- +AUC: 0.995929 (0.0000) +Accuracy: 0.967789 (0.0000) +Positive precision: 0.961702 (0.0000) +Positive recall: 0.945607 (0.0000) +Negative precision: 0.970982 (0.0000) +Negative recall: 0.979730 (0.0000) +Log-loss: 0.117470 (0.0000) +Log-loss reduction: 0.874229 (0.0000) +F1 Score: 0.953586 (0.0000) +AUPRC: 0.991703 (0.0000) + +--------------------------------------- +Physical memory usage(MB): %Number% +Virtual memory usage(MB): %Number% +%DateTime% Time elapsed(s): %Number% + diff --git a/test/BaselineOutput/Common/WeightedEnsemble/WE-StackingAP-TrainTest-breast-cancer-rp.txt b/test/BaselineOutput/Common/WeightedEnsemble/WE-StackingAP-TrainTest-breast-cancer-rp.txt new file mode 100644 index 0000000000..7f7342f834 --- /dev/null +++ b/test/BaselineOutput/Common/WeightedEnsemble/WE-StackingAP-TrainTest-breast-cancer-rp.txt @@ -0,0 +1,4 @@ +WeightedEnsemble +AUC Accuracy Positive precision Positive recall Negative precision Negative recall Log-loss Log-loss reduction F1 Score AUPRC /oc /bp /nm Learner Name Train Dataset Test Dataset Results File Run Time Physical Memory Virtual Memory Command Line Settings +0.995929 0.967789 0.961702 0.945607 0.970982 0.97973 0.11747 0.874229 0.953586 0.991703 Stacking{bp=ap} Microsoft.ML.Runtime.ComponentFactoryUtils+SimpleComponentFactory`1[Microsoft.ML.Trainers.LinearSvmTrainer] 5 WeightedEnsemble %Data% %Data% %Output% 99 0 0 maml.exe TrainTest test=%Data% tr=WeightedEnsemble{nm=5 oc=Stacking{bp=ap} tp=-} dout=%Output% loader=Text{col=Label:BL:0 col=Features:R4:1-9} data=%Data% out=%Output% seed=1 /oc:Stacking{bp=ap};/bp:Microsoft.ML.Runtime.ComponentFactoryUtils+SimpleComponentFactory`1[Microsoft.ML.Trainers.LinearSvmTrainer];/nm:5 + diff --git a/test/BaselineOutput/Common/WeightedEnsemble/WE-StackingAP-TrainTest-breast-cancer.txt b/test/BaselineOutput/Common/WeightedEnsemble/WE-StackingAP-TrainTest-breast-cancer.txt new file mode 100644 index 0000000000..e0c19d881d --- /dev/null +++ b/test/BaselineOutput/Common/WeightedEnsemble/WE-StackingAP-TrainTest-breast-cancer.txt @@ -0,0 +1,700 @@ +Instance Label Score Probability Log-loss Assigned +0 0 -37.39005 0.0169453844 0.024656524298402895 0 +1 0 21.7282028 0.926378369 3.7637264849928962 1 +2 0 -39.4836464 0.0134643717 0.019556940662900459 0 +3 0 22.48525 0.9319332 3.8769054163038046 1 +4 0 -36.54084 0.018597357 0.027082938140064027 0 +5 1 85.79065 0.9999373 9.0465710668344019E-05 1 +6 0 -16.9599152 0.14402549 0.22436025900146123 0 +7 0 -43.14179 0.008994405 0.013034892542986109 0 +8 0 -44.09461 0.00809502 0.011726171922990143 0 +9 0 -39.66323 0.0132009182 0.0191717216705709 0 +10 0 -49.2633553 0.00456478028 0.0066006626424978598 0 +11 0 -47.6558533 0.00545617 0.0078931422965895035 0 +12 1 -9.487345 0.279107958 1.841104834988788 0 +13 0 -41.5772438 0.0106906714 0.01550641407042622 0 +14 1 58.6051254 0.9987011 0.0018751410421028283 1 +15 1 2.14325047 0.586176634 0.77059263440421444 1 +16 0 -42.0454941 0.0101522487 0.014721453822577368 0 +17 0 -40.19523 0.01244996 0.018074242601738368 0 +18 1 52.20739 0.9973524 0.0038247140372067072 1 +19 0 -34.58487 0.0230261646 0.033608169412830541 0 +20 1 38.733593 0.988212168 0.017107274562557628 1 +21 1 47.40369 0.9954846 0.0065291107722648964 1 +22 0 -44.8506775 0.00744531676 0.010781507429313841 0 +23 1 ? ? ? 0 +24 0 -48.6107635 0.00490769558 0.0070977391230301334 0 +25 1 5.689286 0.6777923 0.56108482477637756 1 +26 0 -43.121 0.009015095 0.013065013513646834 0 +27 0 -39.24032 0.0138296289 0.020091186362389955 0 +28 0 -47.6558533 0.00545617 0.0078931422965895035 0 +29 0 -46.9837151 0.005878379 0.0085057329068239373 0 +30 0 -45.503273 0.00692632142 0.010027336074133397 0 +31 0 -45.8055878 0.006698232 0.0096960157532054586 0 +32 1 43.92846 0.993361235 0.0096096473350005523 1 +33 0 -43.9783936 0.008199753 0.011878510592902751 0 +34 0 -42.18688 0.009995015 0.014492305726476216 0 +35 0 -47.6558533 0.00545617 0.0078931422965895035 0 +36 1 60.76223 0.998978555 0.0014743864743720723 1 +37 0 -25.4541187 0.06125233 0.091190672297892086 0 +38 1 36.5044136 0.984935343 0.021899073883648028 1 +39 1 10.8473415 0.7889959 0.34191025204490028 1 +40 0 ? ? ? 0 +41 1 10.7299051 0.7868073 0.34591775433732741 1 +42 1 61.84508 0.999094665 0.0013067138381233159 1 +43 1 -0.466801137 0.514271736 0.95939722807994665 0 +44 1 56.516098 0.998360932 0.0023666155103331901 1 +45 0 -49.10816 0.004644105 0.0067156337100680525 0 +46 1 36.3493576 0.9846766 0.02227812148966574 1 +47 0 -50.4610367 0.003996316 0.0057770161935282848 0 +48 0 -36.54084 0.018597357 0.027082938140064027 0 +49 1 42.2838058 0.992035449 0.011536421316282644 1 +50 1 12.6010923 0.8197222 0.28679306784249498 1 +51 1 -5.993655 0.363717318 1.4591104767823224 0 +52 1 30.2865238 0.970309556 0.043483014615141605 1 +53 1 44.455925 0.9937381 0.0090623942171737353 1 +54 1 41.8900528 0.991680861 0.01205218296839052 1 +55 1 29.6801128 0.968298 0.046476953493553429 1 +56 1 48.73619 0.996105731 0.0056292112076827775 1 +57 1 -3.61150646 0.427122056 1.2272796945453688 0 +58 1 8.281571 0.737447262 0.43938821548521279 1 +59 1 3.059425 0.610723853 0.71140790110083618 1 +60 1 10.6920137 0.7860976 0.34721967512259827 1 +61 0 -46.9442749 0.00590413855 0.0085431161736722952 0 +62 1 52.5982475 0.9974651 0.0036617681981840448 1 +63 1 -2.294691 0.4633786 1.1097366508518682 0 +64 0 -50.4610367 0.003996316 0.0057770161935282848 0 +65 1 8.156918 0.734746754 0.44468101427235501 1 +66 0 -40.19523 0.01244996 0.018074242601738368 0 +67 1 26.8422623 0.957003 0.06340465155014588 1 +68 1 60.3114777 0.9989259 0.0015503964549817143 1 +69 0 -46.8533058 0.00596398255 0.0086299681931887653 0 +70 0 -33.9127274 0.0247741155 0.036191676376825327 0 +71 1 42.92646 0.9925822 0.010741509059066708 1 +72 0 -27.5329018 0.049201604 0.072788625404150417 0 +73 1 56.4911575 0.998356342 0.0023732477282679193 1 +74 1 12.2046833 0.8130965 0.29850146903393066 1 +75 0 -38.8224754 0.0144797731 0.021042613348364884 0 +76 0 -42.46965 0.009687742 0.014044598258168754 0 +77 0 -32.6485863 0.0284183677 0.041592878772611537 0 +78 0 -35.96689 0.0198023263 0.028855372132877033 0 +79 0 -47.00326 0.005865654 0.0084872659926257012 0 +80 0 -34.01104 0.0245105959 0.035801892790058028 0 +81 0 -40.2183037 0.0124183595 0.018028078406513017 0 +82 0 -35.007782 0.0219886918 0.032076948478322785 0 +83 0 -32.08257 0.03021412 0.044261846074937876 0 +84 1 58.686985 0.9987129 0.0018580927161210405 1 +85 1 42.9837379 0.992629051 0.010673416374528644 1 +86 1 14.2624664 0.845501661 0.24212050554143819 1 +87 1 45.53334 0.9944432 0.0080391561103222361 1 +88 0 -40.19523 0.01244996 0.018074242601738368 0 +89 0 -45.1760674 0.007181888 0.010398660037251109 0 +90 0 -48.6107635 0.00490769558 0.0070977391230301334 0 +91 0 -44.87375 0.00742632663 0.010753905213514032 0 +92 0 -40.19523 0.01244996 0.018074242601738368 0 +93 0 -50.4610367 0.003996316 0.0057770161935282848 0 +94 0 -45.8055878 0.006698232 0.0096960157532054586 0 +95 0 -48.6107635 0.00490769558 0.0070977391230301334 0 +96 0 -47.6789322 0.00544222165 0.0078729087261354518 0 +97 0 -37.39005 0.0169453844 0.024656524298402895 0 +98 1 56.06948 0.998277366 0.0024873783329675079 1 +99 1 66.6411743 0.9994695 0.00076552586288871098 1 +100 1 28.51341 0.9640511 0.052818434594778474 1 +101 1 -8.321441 0.306004375 1.7083758143154262 0 +102 0 -37.5314331 0.0166847166 0.02427402835644624 0 +103 1 -4.21967936 0.410613984 1.2841453341828799 0 +104 1 74.73968 0.999784946 0.00031029006784409933 1 +105 1 4.01867533 0.6358354 0.65327473364351807 1 +106 1 68.34192 0.999561131 0.00063329310567416478 1 +107 1 48.2036362 0.9958684 0.005973008305090728 1 +108 0 -46.94611 0.00590293761 0.0085413732936678154 0 +109 1 39.6129532 0.9893015 0.015517826514073683 1 +110 0 -29.9339256 0.0380835943 0.056016571278007285 0 +111 1 24.6744843 0.9458805 0.080270207723861731 1 +112 1 48.4453468 0.9959778 0.0058144819355854978 1 +113 1 64.7942352 0.9993482 0.00094062171460345431 1 +114 0 -28.5385532 0.04421248 0.065238163583771158 0 +115 0 -38.2442627 0.0154293636 0.022433381245535376 0 +116 0 -13.6081543 0.196478069 0.3155906938115699 0 +117 1 53.29091 0.997653067 0.0033898879163941245 1 +118 0 -43.9726 0.008205011 0.011886158023676456 0 +119 0 -37.10138 0.0174900889 0.025456134268816662 0 +120 0 -44.1197777 0.008072513 0.011693436164096523 0 +121 0 -34.296196 0.02376162 0.034694624786196025 0 +122 1 70.67043 0.999661446 0.00048851342729806411 1 +123 1 20.8115444 0.919095159 0.12171385611522345 1 +124 1 44.7761459 0.9939565 0.0087453714569306826 1 +125 0 -50.4610367 0.003996316 0.0057770161935282848 0 +126 1 46.83864 0.995192349 0.006952700821715692 1 +127 0 -43.00041 0.009136039 0.013241096023488342 0 +128 1 31.9750824 0.975279748 0.036111995206353928 1 +129 0 -47.0905647 0.00580915157 0.0084052716127175891 0 +130 0 -33.9127274 0.0247741155 0.036191676376825327 0 +131 0 -45.8055878 0.006698232 0.0096960157532054586 0 +132 1 66.20119 0.9994428 0.00080407093875999697 1 +133 0 -43.0932159 0.009042818 0.013105373291969572 0 +134 0 -44.1980858 0.008002884 0.011592168211310211 0 +135 0 -30.35836 0.0363870077 0.053474249438625443 0 +136 0 -42.0454941 0.0101522487 0.014721453822577368 0 +137 0 -46.7009468 0.006065565 0.008777407740639679 0 +138 0 -39.3817024 0.0136162257 0.019779026800883639 0 +139 0 ? ? ? 0 +140 0 -46.7009468 0.006065565 0.008777407740639679 0 +141 0 -49.5061264 0.004443391 0.0064247429041880541 0 +142 1 22.9397011 0.9350786 0.096840423759425198 1 +143 0 -38.2442627 0.0154293636 0.022433381245535376 0 +144 0 -47.6558533 0.00545617 0.0078931422965895035 0 +145 0 ? ? ? 0 +146 1 5.706744 0.678217351 0.56018040030382477 1 +147 0 -46.2916832 0.00634702761 0.009186008113200559 0 +148 0 -19.4276772 0.113300636 0.17348305482358511 0 +149 1 73.85792 0.9997627 0.00034237208821625534 1 +150 0 -49.2633553 0.00456478028 0.0066006626424978598 0 +151 1 24.5338783 0.9450721 0.08150367537871421 1 +152 1 66.29957 0.9994489 0.00079529496040686156 1 +153 0 -36.8622437 0.01795432 0.02613796119551615 0 +154 0 -52.311306 0.00325363129 0.004701650605626397 0 +155 1 24.2336311 0.943307757 0.084199563004643957 1 +156 0 -45.41497 0.0069943876 0.010126223097517803 0 +157 0 -45.8055878 0.006698232 0.0096960157532054586 0 +158 0 ? ? ? 0 +159 1 84.76125 0.999929667 0.00010147333253723388 1 +160 1 61.60065 0.999069631 0.0013428633747281523 1 +161 0 -38.437767 0.0151049392 0.021958079325052126 0 +162 0 -43.00041 0.009136039 0.013241096023488342 0 +163 0 -35.9859047 0.0197612047 0.028794848924082038 0 +164 0 ? ? ? 0 +165 0 -34.921 0.0221977849 0.03238542121067306 0 +166 1 57.38378 0.998511851 0.002148544496203096 1 +167 1 48.5169373 0.9960097 0.0057682915267605129 1 +168 0 -43.00041 0.009136039 0.013241096023488342 0 +169 0 -51.1367035 0.00370732648 0.0053584804990551294 0 +170 0 -46.7009468 0.006065565 0.008777407740639679 0 +171 0 -48.6107635 0.00490769558 0.0070977391230301334 0 +172 0 -50.4610367 0.003996316 0.0057770161935282848 0 +173 1 99.7272 0.999986768 1.9090200549023394E-05 1 +174 1 36.9278564 0.9856202 0.020896270007302924 1 +175 1 57.34869 0.998506 0.0021569842303280186 1 +176 0 -45.8055878 0.006698232 0.0096960157532054586 0 +177 1 37.817955 0.986961365 0.0189344843497295 1 +178 0 -40.19523 0.01244996 0.018074242601738368 0 +179 1 10.617692 0.7847006 0.34978584214508907 1 +180 0 -49.2633553 0.00456478028 0.0066006626424978598 0 +181 0 -52.311306 0.00325363129 0.004701650605626397 0 +182 0 -34.58487 0.0230261646 0.033608169412830541 0 +183 1 64.03474 0.999290645 0.0010237459064781438 1 +184 1 40.7762451 0.99059093 0.013638683409785465 1 +185 0 -46.45818 0.0062309904 0.009017542330171352 0 +186 1 38.4942131 0.9878971 0.017567320125663085 1 +187 1 80.31165 0.9998844 0.0001667468159458573 1 +188 1 57.2158737 0.9984838 0.0021891073434276084 1 +189 0 -40.9009056 0.011518578 0.016714243455246691 0 +190 1 85.28676 0.99993366 9.5711519989921217E-05 1 +191 1 75.41309 0.9998005 0.00028784167891281751 1 +192 0 -39.24032 0.0138296289 0.020091186362389955 0 +193 0 -48.6107635 0.00490769558 0.0070977391230301334 0 +194 0 -43.00041 0.009136039 0.013241096023488342 0 +195 0 -40.19523 0.01244996 0.018074242601738368 0 +196 0 36.9344177 0.9856306 6.1208535399705886 1 +197 0 -31.9084625 0.0307882745 0.045116236275880434 0 +198 0 -52.311306 0.00325363129 0.004701650605626397 0 +199 0 -44.8506775 0.00744531676 0.010781507429313841 0 +200 1 67.4335556 0.999514341 0.00070082751486429009 1 +201 1 63.56283 0.9992524 0.0010789925839516878 1 +202 0 -48.6107635 0.00490769558 0.0070977391230301334 0 +203 0 -37.39005 0.0169453844 0.024656524298402895 0 +204 0 -48.6107635 0.00490769558 0.0070977391230301334 0 +205 1 85.53309 0.999935448 9.3131611382537777E-05 1 +206 1 47.4008942 0.9954832 0.0065310975452050075 1 +207 0 -49.2633553 0.00456478028 0.0066006626424978598 0 +208 0 -49.2633553 0.00456478028 0.0066006626424978598 0 +209 0 -38.04264 0.0157746933 0.022939482988082908 0 +210 1 98.32638 0.9999845 2.2357917855260308E-05 1 +211 1 67.771225 0.9995323 0.00067493178171970517 1 +212 0 -48.6107635 0.00490769558 0.0070977391230301334 0 +213 1 100.685883 0.9999881 1.7198367596743667E-05 1 +214 1 92.5378342 0.999970436 4.2652327901742838E-05 1 +215 1 51.8242645 0.997237146 0.0039914723781763316 1 +216 0 -50.4610367 0.003996316 0.0057770161935282848 0 +217 0 -48.6107635 0.00490769558 0.0070977391230301334 0 +218 1 53.3835526 0.997677147 0.0033550661157482876 1 +219 0 -29.25728 0.04094641 0.060316661128362585 0 +220 0 -46.78357 0.00601026649 0.0086971439908781079 0 +221 1 63.9833527 0.9992866 0.0010295974793077662 1 +222 1 -24.976656 0.06438659 3.9570959153651635 0 +223 1 32.66614 0.9770713 0.033464272589512595 1 +224 1 61.0731964 0.999013364 0.0014241170678079575 1 +225 0 -50.4610367 0.003996316 0.0057770161935282848 0 +226 1 63.8963776 0.9992796 0.0010396656822262537 1 +227 1 51.3682938 0.997093439 0.0041993868408366393 1 +228 0 -49.2633553 0.00456478028 0.0066006626424978598 0 +229 1 89.4952 0.9999585 5.9851203967434847E-05 1 +230 1 34.773037 0.9817852 0.026520708098979398 1 +231 1 57.2742081 0.9984936 0.0021748972990120281 1 +232 0 1.16346037 0.5594458 1.182608569332374 1 +233 1 37.2055168 0.9860525 0.020263614202921942 1 +234 0 -26.13216 0.0570458844 0.084740524252915186 0 +235 0 ? ? ? 0 +236 1 75.680336 0.999806345 0.00027941287211839295 1 +237 1 44.3151169 0.993639648 0.0092053541238289963 1 +238 1 87.968 0.9999508 7.0944587803823482E-05 1 +239 1 31.809227 0.974829853 0.036777662957060531 1 +240 0 -23.2266464 0.077191554 0.11589688656190421 0 +241 0 -39.99861 0.0127224568 0.018472383183038902 0 +242 0 -45.8055878 0.006698232 0.0096960157532054586 0 +243 0 -30.8431854 0.03453799 0.050708602725894621 0 +244 0 -48.6107635 0.00490769558 0.0070977391230301334 0 +245 0 -32.046093 0.0303335413 0.04443951375123608 0 +246 1 82.5424 0.9999099 0.00013002474319973714 1 +247 1 17.8127022 0.890482366 0.16734105246955061 1 +248 0 -31.40791 0.0324984 0.04766404876175364 0 +249 0 ? ? ? 0 +250 0 -47.26524 0.00569772162 0.0082435819017703388 0 +251 1 49.7908745 0.9965364 0.0050056297482000498 1 +252 0 24.8551 0.946902335 4.2352077622645048 1 +253 1 61.84508 0.999094665 0.0013067138381233159 1 +254 1 52.5982475 0.9974651 0.0036617681981840448 1 +255 1 31.4108257 0.9737164 0.038426486166744517 1 +256 0 -46.7009468 0.006065565 0.008777407740639679 0 +257 0 -44.8506775 0.00744531676 0.010781507429313841 0 +258 0 -43.00041 0.009136039 0.013241096023488342 0 +259 0 24.73384 0.9462183 4.2167411474907004 1 +260 1 59.966 0.9988838 0.0016112589754129296 1 +261 1 81.18282 0.999895155 0.00015126667131697943 1 +262 1 67.80528 0.9995341 0.00067235083711096578 1 +263 1 53.21041 0.997631967 0.0034204007791504643 1 +264 1 33.8901 0.979938149 0.029237401062084386 1 +265 0 -27.6412 0.04863967 0.071936226120769997 0 +266 1 54.07116 0.9978482 0.0031077176134176689 1 +267 1 13.9510279 0.840909958 0.24997676559159329 1 +268 1 54.25466 0.9978917 0.0030448099505338728 1 +269 0 -48.6107635 0.00490769558 0.0070977391230301334 0 +270 1 41.87111 0.9916634 0.012077590013349739 1 +271 0 -37.39005 0.0169453844 0.024656524298402895 0 +272 1 13.9510279 0.840909958 0.24997676559159329 1 +273 1 -6.688231 0.345986843 1.531210917551691 0 +274 0 -41.2429428 0.0110922605 0.016092164343468305 0 +275 0 ? ? ? 0 +276 0 -44.8506775 0.00744531676 0.010781507429313841 0 +277 0 -50.4610367 0.003996316 0.0057770161935282848 0 +278 0 -48.6107635 0.00490769558 0.0070977391230301334 0 +279 1 40.5403366 0.990342557 0.014000458156112654 1 +280 0 -43.00041 0.009136039 0.013241096023488342 0 +281 0 -43.9783936 0.008199753 0.011878510592902751 0 +282 1 25.7522831 0.95171386 0.071400213328933884 1 +283 1 35.96795 0.984021366 0.02323845435139292 1 +284 1 47.7730331 0.995666 0.0062661898329607318 1 +285 1 94.65366 0.999976635 3.3708993376308942E-05 1 +286 1 105.210136 0.9999928 1.0404987898474284E-05 1 +287 0 -44.1980858 0.008002884 0.011592168211310211 0 +288 1 5.63266563 0.676411748 0.56402637816625512 1 +289 1 56.61795 0.9983794 0.0023399146816529266 1 +290 0 -52.311306 0.00325363129 0.004701650605626397 0 +291 0 -48.6107635 0.00490769558 0.0070977391230301334 0 +292 1 ? ? ? 0 +293 1 34.2255249 0.9806605 0.028174328901613796 1 +294 0 ? ? ? 0 +295 1 47.8468552 0.995701432 0.0062148895598010357 1 +296 0 6.787552 0.7039389 1.7560331571550576 1 +297 0 ? ? ? 0 +298 0 -28.3350048 0.0451817252 0.066701915618634283 0 +299 1 37.8852158 0.9870575 0.01879395478335797 1 +300 1 44.52896 0.9937886 0.0089891024700856168 1 +301 0 -48.6107635 0.00490769558 0.0070977391230301334 0 +302 1 104.62677 0.9999923 1.1092923627446125E-05 1 +303 0 -48.6107635 0.00490769558 0.0070977391230301334 0 +304 1 34.2173462 0.9806432 0.028199758399591782 1 +305 1 59.6165543 0.9988395 0.0016752233446540509 1 +306 0 -48.6107635 0.00490769558 0.0070977391230301334 0 +307 0 -48.6107635 0.00490769558 0.0070977391230301334 0 +308 1 43.9658775 0.9933887 0.0095697409531382074 1 +309 0 -25.2328262 0.0626868457 0.093396965109374425 0 +310 0 -47.00326 0.005865654 0.0084872659926257012 0 +311 0 -52.311306 0.00325363129 0.004701650605626397 0 +312 1 19.6235771 0.9086807 0.13815469380938536 1 +313 0 -52.311306 0.00325363129 0.004701650605626397 0 +314 0 -51.11363 0.00371684367 0.0053722620535232623 0 +315 0 ? ? ? 0 +316 1 23.59523 0.939377964 0.090222344447095926 1 +317 1 61.4029121 0.999048948 0.0013727304609297203 1 +318 0 -46.6529922 0.0060978923 0.008824331318632599 0 +319 0 8.017976 0.731715858 1.8981663138970053 1 +320 1 40.33294 0.9901188 0.014326454354495559 1 +321 0 ? ? ? 0 +322 0 -43.00041 0.009136039 0.013241096023488342 0 +323 1 33.0216141 0.9779428 0.03217797471823846 1 +324 0 -48.6107635 0.00490769558 0.0070977391230301334 0 +325 0 -36.2896843 0.0191155262 0.027844865611489817 0 +326 1 17.2596283 0.8843207 0.17735847385183745 1 +327 0 -50.4610367 0.003996316 0.0057770161935282848 0 +328 1 28.5985279 0.964378655 0.052328375436696724 1 +329 1 43.592926 0.9931099 0.0099747424207127424 1 +330 1 31.7607574 0.9746969 0.036974476516637191 1 +331 0 -36.03717 0.01965076 0.028632306629597197 0 +332 0 -30.6095943 0.0354172662 0.052023109035643271 0 +333 1 30.5329475 0.9710911 0.042321463215010555 1 +334 1 41.5027237 0.991316855 0.012581834642933836 1 +335 0 -52.311306 0.00325363129 0.004701650605626397 0 +336 1 36.6181831 0.985122442 0.021625044590369503 1 +337 0 -48.6107635 0.00490769558 0.0070977391230301334 0 +338 0 -51.11363 0.00371684367 0.0053722620535232623 0 +339 1 35.53458 0.9832434 0.024379489207827343 1 +340 1 34.5672226 0.981370151 0.02713070324198693 1 +341 0 -48.6107635 0.00490769558 0.0070977391230301334 0 +342 0 -49.5061264 0.004443391 0.0064247429041880541 0 +343 0 -52.311306 0.00325363129 0.004701650605626397 0 +344 1 58.62223 0.9987036 0.0018715247136963089 1 +345 0 -52.311306 0.00325363129 0.004701650605626397 0 +346 0 -30.0498085 0.03761298 0.055310907854297799 0 +347 0 -49.448967 0.00447167968 0.0064657372553758655 0 +348 1 -5.944748 0.364980519 1.4541086340587266 0 +349 1 16.2817326 0.872688353 0.19646155187292519 1 +350 0 -37.4361954 0.0168598667 0.024531027010496011 0 +351 0 -45.8055878 0.006698232 0.0096960157532054586 0 +352 0 -0.609951 0.510283053 1.0299799722665508 0 +353 1 63.7893829 0.999271 0.0010521434671029357 1 +354 0 -50.4610367 0.003996316 0.0057770161935282848 0 +355 0 -39.63544 0.0132413516 0.019230836254239207 0 +356 1 -13.3911152 0.200327471 2.3195678234535699 0 +357 1 84.6609955 0.999928832 0.00010267729627456687 1 +358 1 41.61453 0.9914235 0.012426657005157981 1 +359 1 29.0259323 0.965980351 0.04993425142122445 1 +360 1 101.390495 0.999989 1.5908482915272255E-05 1 +361 1 42.2327538 0.9919903 0.011602040860280062 1 +362 0 -31.4659042 0.03229565 0.047361750322308924 0 +363 0 -18.0465851 0.129719809 0.20044813585328966 0 +364 0 -45.8055878 0.006698232 0.0096960157532054586 0 +365 0 -47.6558533 0.00545617 0.0078931422965895035 0 +366 1 94.1764 0.9999754 3.5514854534548089E-05 1 +367 1 75.76007 0.9998081 0.00027691864281536248 1 +368 0 -46.9837151 0.005878379 0.0085057329068239373 0 +369 0 -45.7860374 0.00671275426 0.0097171081685260337 0 +370 0 -33.9645729 0.0246348046 0.035985602199845562 0 +371 0 -46.9837151 0.005878379 0.0085057329068239373 0 +372 0 -39.3817024 0.0136162257 0.019779026800883639 0 +373 0 -37.4131241 0.01690257 0.024593692286923138 0 +374 0 -42.18688 0.009995015 0.014492305726476216 0 +375 0 -52.311306 0.00325363129 0.004701650605626397 0 +376 0 -50.4610367 0.003996316 0.0057770161935282848 0 +377 0 -51.11363 0.00371684367 0.0053722620535232623 0 +378 0 -36.959034 0.0177649818 0.025859837177138897 0 +379 0 -23.32894 0.07638284 0.11463311468489933 0 +380 0 -52.311306 0.00325363129 0.004701650605626397 0 +381 1 59.72876 0.9988539 0.0016543894163324336 1 +382 0 -34.3696823 0.0235722531 0.034414802431984637 0 +383 0 -49.5061264 0.004443391 0.0064247429041880541 0 +384 0 -49.5061264 0.004443391 0.0064247429041880541 0 +385 0 -32.3296 0.0294172447 0.043076867475755232 0 +386 1 36.15695 0.9843494 0.022757552825421256 1 +387 0 -24.8181324 0.0654598251 0.097671409637848428 0 +388 0 -45.89839 0.006629722 0.0095965137069048724 0 +389 0 -33.3135 0.026441006 0.038659691404489611 0 +390 0 -48.6338425 0.00489514228 0.0070795393389592324 0 +391 1 68.54033 0.9995707 0.00061944249013147902 1 +392 0 -44.8506775 0.00744531676 0.010781507429313841 0 +393 0 -52.9638977 0.00302593922 0.0043721257447136798 0 +394 0 -42.7511749 0.00939112 0.013612540842068902 0 +395 0 -44.8506775 0.00744531676 0.010781507429313841 0 +396 0 -43.00041 0.009136039 0.013241096023488342 0 +397 0 -43.8957672 0.008275032 0.011988016996993103 0 +398 0 -40.7109451 0.0117623 0.017070000467217603 0 +399 0 -44.6014481 0.0076535535 0.011084214816316017 0 +400 1 62.880928 0.9991933 0.0011642762661327733 1 +401 0 -46.7009468 0.006065565 0.008777407740639679 0 +402 0 -29.1481857 0.0414268635 0.061039585153797843 0 +403 0 -36.1506653 0.0194084 0.028275690588475389 0 +404 0 -44.51718 0.00772525929 0.011188466026196834 0 +405 0 -50.4610367 0.003996316 0.0057770161935282848 0 +406 0 -36.99943 0.0176865384 0.025744624994847313 0 +407 0 -50.4610367 0.003996316 0.0057770161935282848 0 +408 0 -33.0333 0.027257422 0.039870027350182274 0 +409 0 -42.18688 0.009995015 0.014492305726476216 0 +410 0 -50.4610367 0.003996316 0.0057770161935282848 0 +411 0 ? ? ? 0 +412 1 61.9334526 0.9991035 0.0012939756458135957 1 +413 0 -34.7262535 0.0226741154 0.033088392735518585 0 +414 1 42.342308 0.9920868 0.011461703620278103 1 +415 0 -11.7201233 0.23184976 0.38053958398183885 0 +416 1 57.3045845 0.9984987 0.0021675770277238384 1 +417 0 -50.4610367 0.003996316 0.0057770161935282848 0 +418 0 -23.8896847 0.07208626 0.10793739757151098 0 +419 0 -42.7838135 0.009357318 0.013563314237889956 0 +420 0 -27.2664318 0.05061058 0.074928121747093207 0 +421 1 80.894 0.9998917 0.00015625469978168647 1 +422 0 -26.588171 0.0543711148 0.080653990136382359 0 +423 0 -33.9127274 0.0247741155 0.036191676376825327 0 +424 0 -46.7009468 0.006065565 0.008777407740639679 0 +425 1 101.330666 0.9999889 1.5994475191491157E-05 1 +426 0 -23.00707 0.07895404 0.11865494816786017 0 +427 1 30.132534 0.9698108 0.044224798938981562 1 +428 0 -50.4610367 0.003996316 0.0057770161935282848 0 +429 0 -47.6558533 0.00545617 0.0078931422965895035 0 +430 0 -43.9390335 0.008235528 0.011930550500416491 0 +431 0 -30.67301 0.03517645 0.051662970834552173 0 +432 0 -37.4828568 0.0167738274 0.024404775377762047 0 +433 0 -38.00049 0.0158478469 0.02304671671819598 0 +434 0 37.9476776 0.9871462 6.281661119522326 1 +435 1 53.28988 0.997652769 0.0033903188845388131 1 +436 1 35.1127853 0.982450545 0.025543308908793576 1 +437 0 -43.8957672 0.008275032 0.011988016996993103 0 +438 0 -34.913784 0.0222152621 0.032411208117171582 0 +439 0 -41.0905876 0.0112801949 0.016366363592206966 0 +440 1 51.46292 0.9971239 0.0041553178554741069 1 +441 0 -20.0126553 0.106910057 0.16312261914002651 0 +442 0 -41.62339 0.0106363781 0.015427241115476263 0 +443 0 -48.794548 0.00480860937 0.006954090080733142 0 +444 0 -29.4517517 0.04010316 0.059048726755231408 0 +445 0 -49.5061264 0.004443391 0.0064247429041880541 0 +446 0 -52.311306 0.00325363129 0.004701650605626397 0 +447 0 -41.0905876 0.0112801949 0.016366363592206966 0 +448 0 -52.9638977 0.00302593922 0.0043721257447136798 0 +449 1 70.3340454 0.9996485 0.00050717998528051805 1 +450 0 -39.0436974 0.0141318962 0.020533449013168576 0 +451 0 -41.0905876 0.0112801949 0.016366363592206966 0 +452 0 -43.0465546 0.009089569 0.013173437130560219 0 +453 1 50.53729 0.9968121 0.0046065064858908855 1 +454 0 -44.0889969 0.008100047 0.011733482608753679 0 +455 1 -3.08194256 0.4416312 1.1790860035951234 0 +456 1 63.64004 0.999258757 0.0010697846574534151 1 +457 1 56.56067 0.998369038 0.0023549015375531231 1 +458 0 -38.4267921 0.0151231578 0.021984766461795726 0 +459 0 -35.7629929 0.0202485472 0.029512287788622443 0 +460 0 -37.4361954 0.0168598667 0.024531027010496011 0 +461 0 -32.6339 0.0284636263 0.041660084457600437 0 +462 0 -34.6310158 0.0229106769 0.033437639031607264 0 +463 0 -42.1383 0.0100487638 0.014570633395452752 0 +464 0 -43.8957672 0.008275032 0.011988016996993103 0 +465 1 64.3884048 0.999318063 0.00098416236064944847 1 +466 1 58.2958069 0.9986556 0.0019408392530377457 1 +467 1 48.4515457 0.9959806 0.0058104240273325544 1 +468 0 -43.8957672 0.008275032 0.011988016996993103 0 +469 0 -47.7972374 0.005371273 0.0077699950766316169 0 +470 0 -45.503273 0.00692632142 0.010027336074133397 0 +471 0 -34.6310158 0.0229106769 0.033437639031607264 0 +472 0 -38.938 0.0142970681 0.020775177774578404 0 +473 0 -43.8957672 0.008275032 0.011988016996993103 0 +474 0 -41.0905876 0.0112801949 0.016366363592206966 0 +475 0 -46.7009468 0.006065565 0.008777407740639679 0 +476 0 -41.23197 0.0111056892 0.016111755360571522 0 +477 0 -43.8957672 0.008275032 0.011988016996993103 0 +478 0 -38.7083168 0.0146625768 0.021310243048958448 0 +479 1 56.29516 0.9983201 0.0024256174810125537 1 +480 0 -41.7431831 0.0104967076 0.015223587338230466 0 +481 0 -28.18067 0.0459300876 0.067833106826705333 0 +482 1 95.92277 0.999979734 2.9237346903563094E-05 1 +483 1 71.80187 0.99970156 0.00043062282726824926 1 +484 0 -38.4267921 0.0151231578 0.021984766461795726 0 +485 0 -42.2759857 0.00989716 0.01434971256736268 0 +486 0 -45.503273 0.00692632142 0.010027336074133397 0 +487 1 87.41927 0.999947667 7.550235937389033E-05 1 +488 1 6.305593 0.6926157 0.52987302972740491 1 +489 1 -14.1033888 0.1879046 2.4119277379773418 0 +490 0 -52.311306 0.00325363129 0.004701650605626397 0 +491 1 45.78231 0.9945945 0.0078196208279997792 1 +492 0 -42.69809 0.00944635551 0.013692986973089462 0 +493 1 72.6640549 0.9997289 0.0003911415661618596 1 +494 0 -4.75953341 0.3961259 0.7276803276484245 0 +495 0 -45.503273 0.00692632142 0.010027336074133397 0 +496 0 -52.9638977 0.00302593922 0.0043721257447136798 0 +497 0 -41.5134964 0.0107661244 0.015616450194493247 0 +498 0 -42.0454941 0.0101522487 0.014721453822577368 0 +499 0 -42.0454941 0.0101522487 0.014721453822577368 0 +500 0 -34.58487 0.0230261646 0.033608169412830541 0 +501 0 -42.0454941 0.0101522487 0.014721453822577368 0 +502 0 -40.2183037 0.0124183595 0.018028078406513017 0 +503 0 -40.19523 0.01244996 0.018074242601738368 0 +504 0 -52.311306 0.00325363129 0.004701650605626397 0 +505 0 -41.9364052 0.0102752363 0.014900718480588115 0 +506 1 72.48463 0.999723434 0.00039905493496979655 1 +507 0 -41.7610245 0.0104760611 0.015193485085475284 0 +508 0 -41.0905876 0.0112801949 0.016366363592206966 0 +509 0 -49.5061264 0.004443391 0.0064247429041880541 0 +510 0 -52.311306 0.00325363129 0.004701650605626397 0 +511 0 -39.24032 0.0138296289 0.020091186362389955 0 +512 0 -41.0905876 0.0112801949 0.016366363592206966 0 +513 0 -45.503273 0.00692632142 0.010027336074133397 0 +514 1 68.47886 0.999567747 0.00062374390929427644 1 +515 1 59.42199 0.9988141 0.0017118986767242189 1 +516 0 -52.9638977 0.00302593922 0.0043721257447136798 0 +517 0 -51.11363 0.00371684367 0.0053722620535232623 0 +518 0 -42.2099533 0.009969584 0.014455245702155462 0 +519 1 40.457 0.99025327 0.014130535180229123 1 +520 0 -51.65871 0.00349839823 0.0050559708474600847 0 +521 0 -43.2431755 0.008894179 0.012888991806734753 0 +522 1 25.5550652 0.950693 0.07294853991903294 1 +523 1 47.6846237 0.9956233 0.0063281153199289935 1 +524 0 -44.8506775 0.00744531676 0.010781507429313841 0 +525 0 -44.87375 0.00742632663 0.010753905213514032 0 +526 0 -43.8957672 0.008275032 0.011988016996993103 0 +527 0 -40.19523 0.01244996 0.018074242601738368 0 +528 0 -31.130621 0.0334849 0.049135820621575121 0 +529 0 -42.69809 0.00944635551 0.013692986973089462 0 +530 1 39.57945 0.989261866 0.015575630304778732 1 +531 0 -36.99943 0.0176865384 0.025744624994847313 0 +532 0 -49.2633553 0.00456478028 0.0066006626424978598 0 +533 0 -44.8506775 0.00744531676 0.010781507429313841 0 +534 0 -47.6558533 0.00545617 0.0078931422965895035 0 +535 0 -41.4559822 0.01083465 0.015716391406668435 0 +536 0 -37.39005 0.0169453844 0.024656524298402895 0 +537 0 -34.7262535 0.0226741154 0.033088392735518585 0 +538 0 -42.0454941 0.0101522487 0.014721453822577368 0 +539 0 -36.43514 0.018813733 0.02740105329333082 0 +540 0 -35.72356 0.0203359723 0.029641027994990906 0 +541 0 -46.7009468 0.006065565 0.008777407740639679 0 +542 0 -36.32605 0.01903963 0.027733242278976725 0 +543 0 -42.0454941 0.0101522487 0.014721453822577368 0 +544 0 -40.50107 0.0120375026 0.017471816077853402 0 +545 0 -39.24032 0.0138296289 0.020091186362389955 0 +546 1 83.44877 0.9999186 0.00011746893276120316 1 +547 0 -50.15872 0.004132784 0.0059747015380142137 0 +548 0 -47.35354 0.005642205 0.0081630316550459058 0 +549 1 37.615387 0.9866674 0.01936426043416032 1 +550 0 -44.8506775 0.00744531676 0.010781507429313841 0 +551 0 -48.6107635 0.00490769558 0.0070977391230301334 0 +552 0 -32.519413 0.0288188346 0.042187651885809731 0 +553 0 -16.8149719 0.14602977 0.22774231815634144 0 +554 0 -46.7009468 0.006065565 0.008777407740639679 0 +555 0 -21.2781715 0.0941629261 0.14267650795121675 0 +556 0 -33.0196533 0.0272977985 0.039929911919424306 0 +557 0 -37.4361954 0.0168598667 0.024531027010496011 0 +558 0 -47.6558533 0.00545617 0.0078931422965895035 0 +559 0 -39.24032 0.0138296289 0.020091186362389955 0 +560 0 -37.39005 0.0169453844 0.024656524298402895 0 +561 0 -37.39005 0.0169453844 0.024656524298402895 0 +562 0 -48.6107635 0.00490769558 0.0070977391230301334 0 +563 0 -44.8506775 0.00744531676 0.010781507429313841 0 +564 0 -38.437767 0.0151049392 0.021958079325052126 0 +565 1 79.70017 0.9998763 0.00017844303538264487 1 +566 0 -40.33661 0.0122575872 0.017793235442797432 0 +567 0 -35.0119743 0.0219786391 0.032062119508737059 0 +568 1 26.95737 0.9575282 0.062613157947190384 1 +569 1 66.31184 0.9994497 0.0007941764571973922 1 +570 1 52.5141029 0.997441232 0.0036962525547336697 1 +571 1 71.98971 0.999707758 0.00042167708737945704 1 +572 0 -44.8506775 0.00744531676 0.010781507429313841 0 +573 0 -50.4610367 0.003996316 0.0057770161935282848 0 +574 1 38.0989723 0.98735857 0.018353984512501881 1 +575 0 -34.7262535 0.0226741154 0.033088392735518585 0 +576 0 -39.24032 0.0138296289 0.020091186362389955 0 +577 0 -50.4610367 0.003996316 0.0057770161935282848 0 +578 0 -50.4610367 0.003996316 0.0057770161935282848 0 +579 0 -48.6107635 0.00490769558 0.0070977391230301334 0 +580 0 -36.5765228 0.0185248647 0.026976375965354182 0 +581 1 58.0532646 0.9986188 0.0019940544138133142 1 +582 1 59.14354 0.998776734 0.0017658802629188005 1 +583 0 -46.7009468 0.006065565 0.008777407740639679 0 +584 0 -31.302021 0.0328717567 0.048220888016893872 0 +585 0 -52.311306 0.00325363129 0.004701650605626397 0 +586 1 95.91511 0.999979734 2.9237346903563094E-05 1 +587 0 -37.4828568 0.0167738274 0.024404775377762047 0 +588 1 35.2648544 0.9827405 0.025117551018437319 1 +589 0 -41.0905876 0.0112801949 0.016366363592206966 0 +590 1 21.4576168 0.924293637 0.11357684309089512 1 +591 1 37.98667 0.9872013 0.01858383939285501 1 +592 1 30.8542652 0.9720803 0.040852614740406024 1 +593 0 -38.4267921 0.0151231578 0.021984766461795726 0 +594 1 33.81114 0.9797643 0.029493395740486289 1 +595 0 -39.24032 0.0138296289 0.020091186362389955 0 +596 0 -39.3817024 0.0136162257 0.019779026800883639 0 +597 0 -32.062458 0.0302799065 0.044359716604374738 0 +598 0 -44.8506775 0.00744531676 0.010781507429313841 0 +599 0 -31.369524 0.0326332673 0.047865169938072862 0 +600 0 -44.8506775 0.00744531676 0.010781507429313841 0 +601 0 -51.11363 0.00371684367 0.0053722620535232623 0 +602 0 -42.0454941 0.0101522487 0.014721453822577368 0 +603 1 22.2398129 0.930176258 0.10442397804896666 1 +604 1 27.0415039 0.9579081 0.062040851027156986 1 +605 1 62.777668 0.999184 0.0011777018055561753 1 +606 0 -42.068573 0.0101264166 0.014683804246729489 0 +607 0 -52.311306 0.00325363129 0.004701650605626397 0 +608 1 74.29277 0.9997739 0.00032620197268999553 1 +609 0 -41.0905876 0.0112801949 0.016366363592206966 0 +610 1 46.6124 0.9950701 0.0071299319288885615 1 +611 1 46.22503 0.994853556 0.0074439203430526468 1 +612 1 108.091026 0.999994755 7.5672564839008544E-06 1 +613 0 -42.8913231 0.009246828 0.013402413797769982 0 +614 0 -48.30845 0.005075125 0.0073405004782658937 0 +615 0 -36.7179031 0.0182403587 0.026558234014482464 0 +616 0 -44.8506775 0.00744531676 0.010781507429313841 0 +617 0 ? ? ? 0 +618 0 -42.0454941 0.0101522487 0.014721453822577368 0 +619 0 -39.24032 0.0138296289 0.020091186362389955 0 +620 0 -44.8506775 0.00744531676 0.010781507429313841 0 +621 0 -11.118187 0.2440197 0.40357945893181052 0 +622 0 -27.3117752 0.0503681526 0.074559775982699278 0 +623 0 -52.311306 0.00325363129 0.004701650605626397 0 +624 0 -38.0657158 0.01573479 0.022880992988090584 0 +625 0 -30.1704788 0.037128862 0.054585361199378922 0 +626 1 33.84227 0.979833 0.029392203522277817 1 +627 0 -35.4390831 0.0209777355 0.03058642556861205 0 +628 0 -49.5061264 0.004443391 0.0064247429041880541 0 +629 0 -43.8957672 0.008275032 0.011988016996993103 0 +630 0 -30.5755463 0.0355472155 0.052217483075226535 0 +631 0 -39.24032 0.0138296289 0.020091186362389955 0 +632 0 -52.311306 0.00325363129 0.004701650605626397 0 +633 1 27.30109 0.95906 0.060307000005793997 1 +634 0 -46.7009468 0.006065565 0.008777407740639679 0 +635 0 -38.7095528 0.0146605857 0.021307327656986973 0 +636 1 61.58719 0.99906826 0.0013448430183689054 1 +637 0 -24.50069 0.06765913 0.10107058807378756 0 +638 0 -43.8957672 0.008275032 0.011988016996993103 0 +639 0 -37.4361954 0.0168598667 0.024531027010496011 0 +640 0 -40.241375 0.0123868445 0.017982040851327478 0 +641 0 -44.8506775 0.00744531676 0.010781507429313841 0 +642 0 -44.8506775 0.00744531676 0.010781507429313841 0 +643 0 -52.311306 0.00325363129 0.004701650605626397 0 +644 0 -49.5061264 0.004443391 0.0064247429041880541 0 +645 0 -44.8506775 0.00744531676 0.010781507429313841 0 +646 0 -47.26524 0.00569772162 0.0082435819017703388 0 +647 0 -48.6569138 0.004882625 0.0070613917674471676 0 +648 1 74.15177 0.999770343 0.00033136262816700184 1 +649 0 -44.8506775 0.00744531676 0.010781507429313841 0 +650 0 -33.6708755 0.0254341867 0.037168479602967396 0 +651 0 -44.4243736 0.007805005 0.011304415391964244 0 +652 0 -37.4828568 0.0167738274 0.024404775377762047 0 +653 0 -42.0454941 0.0101522487 0.014721453822577368 0 +654 0 -43.00041 0.009136039 0.013241096023488342 0 +655 0 -44.8506775 0.00744531676 0.010781507429313841 0 +656 0 -39.24032 0.0138296289 0.020091186362389955 0 +657 0 -6.885182 0.341033667 0.60172333667223199 0 +658 1 58.6736832 0.998711 0.0018608479874796808 1 +659 0 -52.311306 0.00325363129 0.004701650605626397 0 +660 0 -50.4610367 0.003996316 0.0057770161935282848 0 +661 0 -40.19523 0.01244996 0.018074242601738368 0 +662 0 -45.1334457 0.00721586 0.010448026444449088 0 +663 0 -45.1334457 0.00721586 0.010448026444449088 0 +664 0 -41.19629 0.0111494781 0.016175640310380872 0 +665 0 -52.311306 0.00325363129 0.004701650605626397 0 +666 0 -32.86445 0.0277611725 0.040617343893470138 0 +667 0 -43.00041 0.009136039 0.013241096023488342 0 +668 1 13.4293337 0.832971334 0.26366124677282643 1 +669 1 60.0579834 0.998895168 0.0015948163723510319 1 +670 1 46.46173 0.994986951 0.0072504891752000443 1 +671 0 -38.7909431 0.01453004 0.021116200976925518 0 +672 0 -45.8055878 0.006698232 0.0096960157532054586 0 +673 0 -33.00966 0.02732741 0.039973833086261748 0 +674 0 -50.4610367 0.003996316 0.0057770161935282848 0 +675 0 -35.904377 0.0199380964 0.02905521791429666 0 +676 0 -47.7972374 0.005371273 0.0077699950766316169 0 +677 0 -41.0905876 0.0112801949 0.016366363592206966 0 +678 0 -52.311306 0.00325363129 0.004701650605626397 0 +679 0 -49.5061264 0.004443391 0.0064247429041880541 0 +680 1 108.990143 0.9999953 6.7933307031989985E-06 1 +681 1 72.61894 0.999727547 0.00039311990429454906 1 +682 0 -35.6325836 0.0205390919 0.029940181583579653 0 +683 0 -52.311306 0.00325363129 0.004701650605626397 0 +684 0 -52.311306 0.00325363129 0.004701650605626397 0 +685 0 -52.311306 0.00325363129 0.004701650605626397 0 +686 0 -52.311306 0.00325363129 0.004701650605626397 0 +687 0 -41.3357468 0.0109793041 0.015927384242593722 0 +688 0 -43.8957672 0.008275032 0.011988016996993103 0 +689 0 -42.8411522 0.009298228 0.013477261911043771 0 +690 0 -48.6569138 0.004882625 0.0070613917674471676 0 +691 1 37.82821 0.9869761 0.018912964055495095 1 +692 0 -46.7009468 0.006065565 0.008777407740639679 0 +693 0 -43.4978 0.008647323 0.012529702115805152 0 +694 0 -42.5315933 0.009621691 0.013948377448766133 0 +695 0 -49.5061264 0.004443391 0.0064247429041880541 0 +696 1 48.3968163 0.9959561 0.0058459095264315449 1 +697 1 28.2233028 0.9629129 0.054522763791826356 1 +698 1 36.0325165 0.9841342 0.023073038992678114 1 diff --git a/test/BaselineOutput/Common/WeightedEnsemble/WE-Voting-TrainTest-breast-cancer-out.txt b/test/BaselineOutput/Common/WeightedEnsemble/WE-Voting-TrainTest-breast-cancer-out.txt new file mode 100644 index 0000000000..bc323f4a00 --- /dev/null +++ b/test/BaselineOutput/Common/WeightedEnsemble/WE-Voting-TrainTest-breast-cancer-out.txt @@ -0,0 +1,118 @@ +maml.exe TrainTest test=%Data% tr=WeightedEnsemble{nm=20 oc=Voting tp=-} dout=%Output% loader=Text{col=Label:BL:0 col=Features:R4:1-9} data=%Data% out=%Output% seed=1 +Automatically adding a MinMax normalization transform, use 'norm=Warn' or 'norm=No' to turn this behavior off. +Training 20 learners for the batch 1 +Beginning training model 1 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 26 instances with missing features during training (over 1 iterations; 26 inst/iter) +Trainer 1 of 20 finished in %Time% +Beginning training model 2 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 15 instances with missing features during training (over 1 iterations; 15 inst/iter) +Trainer 2 of 20 finished in %Time% +Beginning training model 3 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 11 instances with missing features during training (over 1 iterations; 11 inst/iter) +Trainer 3 of 20 finished in %Time% +Beginning training model 4 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 12 instances with missing features during training (over 1 iterations; 12 inst/iter) +Trainer 4 of 20 finished in %Time% +Beginning training model 5 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 18 instances with missing features during training (over 1 iterations; 18 inst/iter) +Trainer 5 of 20 finished in %Time% +Beginning training model 6 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 15 instances with missing features during training (over 1 iterations; 15 inst/iter) +Trainer 6 of 20 finished in %Time% +Beginning training model 7 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 24 instances with missing features during training (over 1 iterations; 24 inst/iter) +Trainer 7 of 20 finished in %Time% +Beginning training model 8 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 18 instances with missing features during training (over 1 iterations; 18 inst/iter) +Trainer 8 of 20 finished in %Time% +Beginning training model 9 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 15 instances with missing features during training (over 1 iterations; 15 inst/iter) +Trainer 9 of 20 finished in %Time% +Beginning training model 10 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 18 instances with missing features during training (over 1 iterations; 18 inst/iter) +Trainer 10 of 20 finished in %Time% +Beginning training model 11 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 14 instances with missing features during training (over 1 iterations; 14 inst/iter) +Trainer 11 of 20 finished in %Time% +Beginning training model 12 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 13 instances with missing features during training (over 1 iterations; 13 inst/iter) +Trainer 12 of 20 finished in %Time% +Beginning training model 13 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 14 instances with missing features during training (over 1 iterations; 14 inst/iter) +Trainer 13 of 20 finished in %Time% +Beginning training model 14 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 17 instances with missing features during training (over 1 iterations; 17 inst/iter) +Trainer 14 of 20 finished in %Time% +Beginning training model 15 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 24 instances with missing features during training (over 1 iterations; 24 inst/iter) +Trainer 15 of 20 finished in %Time% +Beginning training model 16 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 12 instances with missing features during training (over 1 iterations; 12 inst/iter) +Trainer 16 of 20 finished in %Time% +Beginning training model 17 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 12 instances with missing features during training (over 1 iterations; 12 inst/iter) +Trainer 17 of 20 finished in %Time% +Beginning training model 18 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 19 instances with missing features during training (over 1 iterations; 19 inst/iter) +Trainer 18 of 20 finished in %Time% +Beginning training model 19 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 23 instances with missing features during training (over 1 iterations; 23 inst/iter) +Trainer 19 of 20 finished in %Time% +Beginning training model 20 of 20 +Warning: Training data does not support shuffling, so ignoring request to shuffle +Warning: Skipped 18 instances with missing features during training (over 1 iterations; 18 inst/iter) +Trainer 20 of 20 finished in %Time% +Training calibrator. +Warning: The predictor produced non-finite prediction values on 16 instances during testing. Possible causes: abnormal data or the predictor is numerically unstable. +TEST POSITIVE RATIO: 0.3499 (239.0/(239.0+444.0)) +Confusion table + ||====================== +PREDICTED || positive | negative | Recall +TRUTH ||====================== + positive || 231 | 8 | 0.9665 + negative || 10 | 434 | 0.9775 + ||====================== +Precision || 0.9585 | 0.9819 | +OVERALL 0/1 ACCURACY: 0.973646 +LOG LOSS/instance: 0.129466 +Test-set entropy (prior Log-Loss/instance): 0.934003 +LOG-LOSS REDUCTION (RIG): 0.861385 +AUC: 0.992339 + +OVERALL RESULTS +--------------------------------------- +AUC: 0.992339 (0.0000) +Accuracy: 0.973646 (0.0000) +Positive precision: 0.958506 (0.0000) +Positive recall: 0.966527 (0.0000) +Negative precision: 0.981900 (0.0000) +Negative recall: 0.977477 (0.0000) +Log-loss: 0.129466 (0.0000) +Log-loss reduction: 0.861385 (0.0000) +F1 Score: 0.962500 (0.0000) +AUPRC: 0.963675 (0.0000) + +--------------------------------------- +Physical memory usage(MB): %Number% +Virtual memory usage(MB): %Number% +%DateTime% Time elapsed(s): %Number% + diff --git a/test/BaselineOutput/Common/WeightedEnsemble/WE-Voting-TrainTest-breast-cancer-rp.txt b/test/BaselineOutput/Common/WeightedEnsemble/WE-Voting-TrainTest-breast-cancer-rp.txt new file mode 100644 index 0000000000..1396d347ab --- /dev/null +++ b/test/BaselineOutput/Common/WeightedEnsemble/WE-Voting-TrainTest-breast-cancer-rp.txt @@ -0,0 +1,4 @@ +WeightedEnsemble +AUC Accuracy Positive precision Positive recall Negative precision Negative recall Log-loss Log-loss reduction F1 Score AUPRC /oc /bp /nm Learner Name Train Dataset Test Dataset Results File Run Time Physical Memory Virtual Memory Command Line Settings +0.992339 0.973646 0.958506 0.966527 0.9819 0.977477 0.129466 0.861385 0.9625 0.963675 Voting Microsoft.ML.Runtime.ComponentFactoryUtils+SimpleComponentFactory`1[Microsoft.ML.Trainers.LinearSvmTrainer] 20 WeightedEnsemble %Data% %Data% %Output% 99 0 0 maml.exe TrainTest test=%Data% tr=WeightedEnsemble{nm=20 oc=Voting tp=-} dout=%Output% loader=Text{col=Label:BL:0 col=Features:R4:1-9} data=%Data% out=%Output% seed=1 /oc:Voting;/bp:Microsoft.ML.Runtime.ComponentFactoryUtils+SimpleComponentFactory`1[Microsoft.ML.Trainers.LinearSvmTrainer];/nm:20 + diff --git a/test/BaselineOutput/Common/WeightedEnsemble/WE-Voting-TrainTest-breast-cancer.txt b/test/BaselineOutput/Common/WeightedEnsemble/WE-Voting-TrainTest-breast-cancer.txt new file mode 100644 index 0000000000..61b4a9aff0 --- /dev/null +++ b/test/BaselineOutput/Common/WeightedEnsemble/WE-Voting-TrainTest-breast-cancer.txt @@ -0,0 +1,700 @@ +Instance Label Score Probability Log-loss Assigned +0 0 -1 0.0124691306 0.018102248502573935 0 +1 0 0.9 0.969497263 5.0349175091769718 1 +2 0 -1 0.0124691306 0.018102248502573935 0 +3 0 1 0.9795901 5.6145883434887764 1 +4 0 -1 0.0124691306 0.018102248502573935 0 +5 1 1 0.9795901 0.029749874765915209 1 +6 0 -0.9 0.0187102128 0.027248849528611944 0 +7 0 -1 0.0124691306 0.018102248502573935 0 +8 0 -1 0.0124691306 0.018102248502573935 0 +9 0 -1 0.0124691306 0.018102248502573935 0 +10 0 -1 0.0124691306 0.018102248502573935 0 +11 0 -1 0.0124691306 0.018102248502573935 0 +12 1 -0.2 0.254505575 1.974230837373812 0 +13 0 -1 0.0124691306 0.018102248502573935 0 +14 1 1 0.9795901 0.029749874765915209 1 +15 1 0.4 0.801897347 0.31851052964248283 1 +16 0 -1 0.0124691306 0.018102248502573935 0 +17 0 -1 0.0124691306 0.018102248502573935 0 +18 1 1 0.9795901 0.029749874765915209 1 +19 0 -1 0.0124691306 0.018102248502573935 0 +20 1 0.9 0.969497263 0.044691268798002393 1 +21 1 1 0.9795901 0.029749874765915209 1 +22 0 -1 0.0124691306 0.018102248502573935 0 +23 1 ? ? ? 0 +24 0 -1 0.0124691306 0.018102248502573935 0 +25 1 0.4 0.801897347 0.31851052964248283 1 +26 0 -1 0.0124691306 0.018102248502573935 0 +27 0 -1 0.0124691306 0.018102248502573935 0 +28 0 -1 0.0124691306 0.018102248502573935 0 +29 0 -1 0.0124691306 0.018102248502573935 0 +30 0 -1 0.0124691306 0.018102248502573935 0 +31 0 -1 0.0124691306 0.018102248502573935 0 +32 1 1 0.9795901 0.029749874765915209 1 +33 0 -1 0.0124691306 0.018102248502573935 0 +34 0 -1 0.0124691306 0.018102248502573935 0 +35 0 -1 0.0124691306 0.018102248502573935 0 +36 1 1 0.9795901 0.029749874765915209 1 +37 0 -1 0.0124691306 0.018102248502573935 0 +38 1 0.9 0.969497263 0.044691268798002393 1 +39 1 0.8 0.954644561 0.066964414934699193 1 +40 0 ? ? ? 0 +41 1 0.8 0.954644561 0.066964414934699193 1 +42 1 1 0.9795901 0.029749874765915209 1 +43 1 0.3 0.7283054 0.45738455403071537 1 +44 1 1 0.9795901 0.029749874765915209 1 +45 0 -1 0.0124691306 0.018102248502573935 0 +46 1 1 0.9795901 0.029749874765915209 1 +47 0 -1 0.0124691306 0.018102248502573935 0 +48 0 -1 0.0124691306 0.018102248502573935 0 +49 1 1 0.9795901 0.029749874765915209 1 +50 1 0.7 0.9330589 0.099959912702360154 1 +51 1 -0.2 0.254505575 1.974230837373812 0 +52 1 1 0.9795901 0.029749874765915209 1 +53 1 0.9 0.969497263 0.044691268798002393 1 +54 1 1 0.9795901 0.029749874765915209 1 +55 1 1 0.9795901 0.029749874765915209 1 +56 1 1 0.9795901 0.029749874765915209 1 +57 1 0.1 0.540345848 0.88804499702609785 1 +58 1 0.7 0.9330589 0.099959912702360154 1 +59 1 0.4 0.801897347 0.31851052964248283 1 +60 1 0.6 0.9022521 0.14839753396284014 1 +61 0 -1 0.0124691306 0.018102248502573935 0 +62 1 1 0.9795901 0.029749874765915209 1 +63 1 0.1 0.540345848 0.88804499702609785 1 +64 0 -1 0.0124691306 0.018102248502573935 0 +65 1 0.8 0.954644561 0.066964414934699193 1 +66 0 -1 0.0124691306 0.018102248502573935 0 +67 1 1 0.9795901 0.029749874765915209 1 +68 1 1 0.9795901 0.029749874765915209 1 +69 0 -1 0.0124691306 0.018102248502573935 0 +70 0 -1 0.0124691306 0.018102248502573935 0 +71 1 0.9 0.969497263 0.044691268798002393 1 +72 0 -1 0.0124691306 0.018102248502573935 0 +73 1 1 0.9795901 0.029749874765915209 1 +74 1 0.8 0.954644561 0.066964414934699193 1 +75 0 -1 0.0124691306 0.018102248502573935 0 +76 0 -1 0.0124691306 0.018102248502573935 0 +77 0 -1 0.0124691306 0.018102248502573935 0 +78 0 -1 0.0124691306 0.018102248502573935 0 +79 0 -1 0.0124691306 0.018102248502573935 0 +80 0 -1 0.0124691306 0.018102248502573935 0 +81 0 -1 0.0124691306 0.018102248502573935 0 +82 0 -1 0.0124691306 0.018102248502573935 0 +83 0 -1 0.0124691306 0.018102248502573935 0 +84 1 1 0.9795901 0.029749874765915209 1 +85 1 1 0.9795901 0.029749874765915209 1 +86 1 0.8 0.954644561 0.066964414934699193 1 +87 1 1 0.9795901 0.029749874765915209 1 +88 0 -1 0.0124691306 0.018102248502573935 0 +89 0 -1 0.0124691306 0.018102248502573935 0 +90 0 -1 0.0124691306 0.018102248502573935 0 +91 0 -1 0.0124691306 0.018102248502573935 0 +92 0 -1 0.0124691306 0.018102248502573935 0 +93 0 -1 0.0124691306 0.018102248502573935 0 +94 0 -1 0.0124691306 0.018102248502573935 0 +95 0 -1 0.0124691306 0.018102248502573935 0 +96 0 -1 0.0124691306 0.018102248502573935 0 +97 0 -1 0.0124691306 0.018102248502573935 0 +98 1 1 0.9795901 0.029749874765915209 1 +99 1 1 0.9795901 0.029749874765915209 1 +100 1 1 0.9795901 0.029749874765915209 1 +101 1 -0.3 0.184390724 2.4391620161320371 0 +102 0 -1 0.0124691306 0.018102248502573935 0 +103 1 0.1 0.540345848 0.88804499702609785 1 +104 1 1 0.9795901 0.029749874765915209 1 +105 1 0.4 0.801897347 0.31851052964248283 1 +106 1 1 0.9795901 0.029749874765915209 1 +107 1 1 0.9795901 0.029749874765915209 1 +108 0 -1 0.0124691306 0.018102248502573935 0 +109 1 1 0.9795901 0.029749874765915209 1 +110 0 -1 0.0124691306 0.018102248502573935 0 +111 1 0.9 0.969497263 0.044691268798002393 1 +112 1 1 0.9795901 0.029749874765915209 1 +113 1 1 0.9795901 0.029749874765915209 1 +114 0 -1 0.0124691306 0.018102248502573935 0 +115 0 -1 0.0124691306 0.018102248502573935 0 +116 0 -0.6 0.0616101734 0.091740722231051938 0 +117 1 1 0.9795901 0.029749874765915209 1 +118 0 -1 0.0124691306 0.018102248502573935 0 +119 0 -1 0.0124691306 0.018102248502573935 0 +120 0 -1 0.0124691306 0.018102248502573935 0 +121 0 -1 0.0124691306 0.018102248502573935 0 +122 1 1 0.9795901 0.029749874765915209 1 +123 1 0.9 0.969497263 0.044691268798002393 1 +124 1 1 0.9795901 0.029749874765915209 1 +125 0 -1 0.0124691306 0.018102248502573935 0 +126 1 1 0.9795901 0.029749874765915209 1 +127 0 -1 0.0124691306 0.018102248502573935 0 +128 1 1 0.9795901 0.029749874765915209 1 +129 0 -1 0.0124691306 0.018102248502573935 0 +130 0 -1 0.0124691306 0.018102248502573935 0 +131 0 -1 0.0124691306 0.018102248502573935 0 +132 1 1 0.9795901 0.029749874765915209 1 +133 0 -1 0.0124691306 0.018102248502573935 0 +134 0 -1 0.0124691306 0.018102248502573935 0 +135 0 -1 0.0124691306 0.018102248502573935 0 +136 0 -1 0.0124691306 0.018102248502573935 0 +137 0 -1 0.0124691306 0.018102248502573935 0 +138 0 -1 0.0124691306 0.018102248502573935 0 +139 0 ? ? ? 0 +140 0 -1 0.0124691306 0.018102248502573935 0 +141 0 -1 0.0124691306 0.018102248502573935 0 +142 1 0.9 0.969497263 0.044691268798002393 1 +143 0 -1 0.0124691306 0.018102248502573935 0 +144 0 -1 0.0124691306 0.018102248502573935 0 +145 0 ? ? ? 0 +146 1 0.5 0.859403968 0.21859165685519127 1 +147 0 -1 0.0124691306 0.018102248502573935 0 +148 0 -0.5 0.09020085 0.13638000634685848 0 +149 1 1 0.9795901 0.029749874765915209 1 +150 0 -1 0.0124691306 0.018102248502573935 0 +151 1 0.9 0.969497263 0.044691268798002393 1 +152 1 1 0.9795901 0.029749874765915209 1 +153 0 -1 0.0124691306 0.018102248502573935 0 +154 0 -1 0.0124691306 0.018102248502573935 0 +155 1 0.9 0.969497263 0.044691268798002393 1 +156 0 -1 0.0124691306 0.018102248502573935 0 +157 0 -1 0.0124691306 0.018102248502573935 0 +158 0 ? ? ? 0 +159 1 1 0.9795901 0.029749874765915209 1 +160 1 1 0.9795901 0.029749874765915209 1 +161 0 -1 0.0124691306 0.018102248502573935 0 +162 0 -1 0.0124691306 0.018102248502573935 0 +163 0 -1 0.0124691306 0.018102248502573935 0 +164 0 ? ? ? 0 +165 0 -1 0.0124691306 0.018102248502573935 0 +166 1 1 0.9795901 0.029749874765915209 1 +167 1 0.9 0.969497263 0.044691268798002393 1 +168 0 -1 0.0124691306 0.018102248502573935 0 +169 0 -1 0.0124691306 0.018102248502573935 0 +170 0 -1 0.0124691306 0.018102248502573935 0 +171 0 -1 0.0124691306 0.018102248502573935 0 +172 0 -1 0.0124691306 0.018102248502573935 0 +173 1 1 0.9795901 0.029749874765915209 1 +174 1 1 0.9795901 0.029749874765915209 1 +175 1 1 0.9795901 0.029749874765915209 1 +176 0 -1 0.0124691306 0.018102248502573935 0 +177 1 1 0.9795901 0.029749874765915209 1 +178 0 -1 0.0124691306 0.018102248502573935 0 +179 1 0.7 0.9330589 0.099959912702360154 1 +180 0 -1 0.0124691306 0.018102248502573935 0 +181 0 -1 0.0124691306 0.018102248502573935 0 +182 0 -1 0.0124691306 0.018102248502573935 0 +183 1 1 0.9795901 0.029749874765915209 1 +184 1 1 0.9795901 0.029749874765915209 1 +185 0 -1 0.0124691306 0.018102248502573935 0 +186 1 1 0.9795901 0.029749874765915209 1 +187 1 1 0.9795901 0.029749874765915209 1 +188 1 1 0.9795901 0.029749874765915209 1 +189 0 -1 0.0124691306 0.018102248502573935 0 +190 1 1 0.9795901 0.029749874765915209 1 +191 1 1 0.9795901 0.029749874765915209 1 +192 0 -1 0.0124691306 0.018102248502573935 0 +193 0 -1 0.0124691306 0.018102248502573935 0 +194 0 -1 0.0124691306 0.018102248502573935 0 +195 0 -1 0.0124691306 0.018102248502573935 0 +196 0 1 0.9795901 5.6145883434887764 1 +197 0 -1 0.0124691306 0.018102248502573935 0 +198 0 -1 0.0124691306 0.018102248502573935 0 +199 0 -1 0.0124691306 0.018102248502573935 0 +200 1 1 0.9795901 0.029749874765915209 1 +201 1 1 0.9795901 0.029749874765915209 1 +202 0 -1 0.0124691306 0.018102248502573935 0 +203 0 -1 0.0124691306 0.018102248502573935 0 +204 0 -1 0.0124691306 0.018102248502573935 0 +205 1 1 0.9795901 0.029749874765915209 1 +206 1 1 0.9795901 0.029749874765915209 1 +207 0 -1 0.0124691306 0.018102248502573935 0 +208 0 -1 0.0124691306 0.018102248502573935 0 +209 0 -1 0.0124691306 0.018102248502573935 0 +210 1 1 0.9795901 0.029749874765915209 1 +211 1 1 0.9795901 0.029749874765915209 1 +212 0 -1 0.0124691306 0.018102248502573935 0 +213 1 1 0.9795901 0.029749874765915209 1 +214 1 1 0.9795901 0.029749874765915209 1 +215 1 1 0.9795901 0.029749874765915209 1 +216 0 -1 0.0124691306 0.018102248502573935 0 +217 0 -1 0.0124691306 0.018102248502573935 0 +218 1 1 0.9795901 0.029749874765915209 1 +219 0 -1 0.0124691306 0.018102248502573935 0 +220 0 -1 0.0124691306 0.018102248502573935 0 +221 1 1 0.9795901 0.029749874765915209 1 +222 1 -1 0.0124691306 6.3254953155186193 0 +223 1 1 0.9795901 0.029749874765915209 1 +224 1 1 0.9795901 0.029749874765915209 1 +225 0 -1 0.0124691306 0.018102248502573935 0 +226 1 1 0.9795901 0.029749874765915209 1 +227 1 1 0.9795901 0.029749874765915209 1 +228 0 -1 0.0124691306 0.018102248502573935 0 +229 1 1 0.9795901 0.029749874765915209 1 +230 1 1 0.9795901 0.029749874765915209 1 +231 1 1 0.9795901 0.029749874765915209 1 +232 0 0.3 0.7283054 1.8799421998861698 1 +233 1 0.9 0.969497263 0.044691268798002393 1 +234 0 -1 0.0124691306 0.018102248502573935 0 +235 0 ? ? ? 0 +236 1 1 0.9795901 0.029749874765915209 1 +237 1 1 0.9795901 0.029749874765915209 1 +238 1 1 0.9795901 0.029749874765915209 1 +239 1 1 0.9795901 0.029749874765915209 1 +240 0 -1 0.0124691306 0.018102248502573935 0 +241 0 -1 0.0124691306 0.018102248502573935 0 +242 0 -1 0.0124691306 0.018102248502573935 0 +243 0 -1 0.0124691306 0.018102248502573935 0 +244 0 -1 0.0124691306 0.018102248502573935 0 +245 0 -1 0.0124691306 0.018102248502573935 0 +246 1 1 0.9795901 0.029749874765915209 1 +247 1 0.9 0.969497263 0.044691268798002393 1 +248 0 -1 0.0124691306 0.018102248502573935 0 +249 0 ? ? ? 0 +250 0 -1 0.0124691306 0.018102248502573935 0 +251 1 1 0.9795901 0.029749874765915209 1 +252 0 0.9 0.969497263 5.0349175091769718 1 +253 1 1 0.9795901 0.029749874765915209 1 +254 1 1 0.9795901 0.029749874765915209 1 +255 1 1 0.9795901 0.029749874765915209 1 +256 0 -1 0.0124691306 0.018102248502573935 0 +257 0 -1 0.0124691306 0.018102248502573935 0 +258 0 -1 0.0124691306 0.018102248502573935 0 +259 0 0.9 0.969497263 5.0349175091769718 1 +260 1 1 0.9795901 0.029749874765915209 1 +261 1 1 0.9795901 0.029749874765915209 1 +262 1 1 0.9795901 0.029749874765915209 1 +263 1 1 0.9795901 0.029749874765915209 1 +264 1 0.9 0.969497263 0.044691268798002393 1 +265 0 -1 0.0124691306 0.018102248502573935 0 +266 1 1 0.9795901 0.029749874765915209 1 +267 1 0.7 0.9330589 0.099959912702360154 1 +268 1 1 0.9795901 0.029749874765915209 1 +269 0 -1 0.0124691306 0.018102248502573935 0 +270 1 1 0.9795901 0.029749874765915209 1 +271 0 -1 0.0124691306 0.018102248502573935 0 +272 1 0.7 0.9330589 0.099959912702360154 1 +273 1 -0.1 0.340162158 1.5557054403189898 0 +274 0 -1 0.0124691306 0.018102248502573935 0 +275 0 ? ? ? 0 +276 0 -1 0.0124691306 0.018102248502573935 0 +277 0 -1 0.0124691306 0.018102248502573935 0 +278 0 -1 0.0124691306 0.018102248502573935 0 +279 1 1 0.9795901 0.029749874765915209 1 +280 0 -1 0.0124691306 0.018102248502573935 0 +281 0 -1 0.0124691306 0.018102248502573935 0 +282 1 0.9 0.969497263 0.044691268798002393 1 +283 1 1 0.9795901 0.029749874765915209 1 +284 1 1 0.9795901 0.029749874765915209 1 +285 1 1 0.9795901 0.029749874765915209 1 +286 1 1 0.9795901 0.029749874765915209 1 +287 0 -1 0.0124691306 0.018102248502573935 0 +288 1 0.2 0.639659941 0.64462295779685108 1 +289 1 1 0.9795901 0.029749874765915209 1 +290 0 -1 0.0124691306 0.018102248502573935 0 +291 0 -1 0.0124691306 0.018102248502573935 0 +292 1 ? ? ? 0 +293 1 0.9 0.969497263 0.044691268798002393 1 +294 0 ? ? ? 0 +295 1 1 0.9795901 0.029749874765915209 1 +296 0 0.4 0.801897347 2.3356798936975096 1 +297 0 ? ? ? 0 +298 0 -1 0.0124691306 0.018102248502573935 0 +299 1 1 0.9795901 0.029749874765915209 1 +300 1 1 0.9795901 0.029749874765915209 1 +301 0 -1 0.0124691306 0.018102248502573935 0 +302 1 1 0.9795901 0.029749874765915209 1 +303 0 -1 0.0124691306 0.018102248502573935 0 +304 1 0.9 0.969497263 0.044691268798002393 1 +305 1 1 0.9795901 0.029749874765915209 1 +306 0 -1 0.0124691306 0.018102248502573935 0 +307 0 -1 0.0124691306 0.018102248502573935 0 +308 1 1 0.9795901 0.029749874765915209 1 +309 0 -1 0.0124691306 0.018102248502573935 0 +310 0 -1 0.0124691306 0.018102248502573935 0 +311 0 -1 0.0124691306 0.018102248502573935 0 +312 1 0.9 0.969497263 0.044691268798002393 1 +313 0 -1 0.0124691306 0.018102248502573935 0 +314 0 -1 0.0124691306 0.018102248502573935 0 +315 0 ? ? ? 0 +316 1 0.9 0.969497263 0.044691268798002393 1 +317 1 1 0.9795901 0.029749874765915209 1 +318 0 -1 0.0124691306 0.018102248502573935 0 +319 0 0.8 0.954644561 4.462580615509057 1 +320 1 1 0.9795901 0.029749874765915209 1 +321 0 ? ? ? 0 +322 0 -1 0.0124691306 0.018102248502573935 0 +323 1 0.9 0.969497263 0.044691268798002393 1 +324 0 -1 0.0124691306 0.018102248502573935 0 +325 0 -1 0.0124691306 0.018102248502573935 0 +326 1 0.9 0.969497263 0.044691268798002393 1 +327 0 -1 0.0124691306 0.018102248502573935 0 +328 1 1 0.9795901 0.029749874765915209 1 +329 1 1 0.9795901 0.029749874765915209 1 +330 1 1 0.9795901 0.029749874765915209 1 +331 0 -1 0.0124691306 0.018102248502573935 0 +332 0 -1 0.0124691306 0.018102248502573935 0 +333 1 0.9 0.969497263 0.044691268798002393 1 +334 1 1 0.9795901 0.029749874765915209 1 +335 0 -1 0.0124691306 0.018102248502573935 0 +336 1 0.9 0.969497263 0.044691268798002393 1 +337 0 -1 0.0124691306 0.018102248502573935 0 +338 0 -1 0.0124691306 0.018102248502573935 0 +339 1 1 0.9795901 0.029749874765915209 1 +340 1 1 0.9795901 0.029749874765915209 1 +341 0 -1 0.0124691306 0.018102248502573935 0 +342 0 -1 0.0124691306 0.018102248502573935 0 +343 0 -1 0.0124691306 0.018102248502573935 0 +344 1 1 0.9795901 0.029749874765915209 1 +345 0 -1 0.0124691306 0.018102248502573935 0 +346 0 -1 0.0124691306 0.018102248502573935 0 +347 0 -1 0.0124691306 0.018102248502573935 0 +348 1 0 0.437720537 1.1919180202527921 0 +349 1 0.9 0.969497263 0.044691268798002393 1 +350 0 -1 0.0124691306 0.018102248502573935 0 +351 0 -1 0.0124691306 0.018102248502573935 0 +352 0 0.1 0.540345848 1.1213793213003707 1 +353 1 1 0.9795901 0.029749874765915209 1 +354 0 -1 0.0124691306 0.018102248502573935 0 +355 0 -1 0.0124691306 0.018102248502573935 0 +356 1 -0.6 0.0616101734 4.0206875928975396 0 +357 1 1 0.9795901 0.029749874765915209 1 +358 1 1 0.9795901 0.029749874765915209 1 +359 1 0.9 0.969497263 0.044691268798002393 1 +360 1 1 0.9795901 0.029749874765915209 1 +361 1 0.9 0.969497263 0.044691268798002393 1 +362 0 -1 0.0124691306 0.018102248502573935 0 +363 0 -0.8 0.0279865637 0.040951838372586473 0 +364 0 -1 0.0124691306 0.018102248502573935 0 +365 0 -1 0.0124691306 0.018102248502573935 0 +366 1 1 0.9795901 0.029749874765915209 1 +367 1 1 0.9795901 0.029749874765915209 1 +368 0 -1 0.0124691306 0.018102248502573935 0 +369 0 -1 0.0124691306 0.018102248502573935 0 +370 0 -1 0.0124691306 0.018102248502573935 0 +371 0 -1 0.0124691306 0.018102248502573935 0 +372 0 -1 0.0124691306 0.018102248502573935 0 +373 0 -1 0.0124691306 0.018102248502573935 0 +374 0 -1 0.0124691306 0.018102248502573935 0 +375 0 -1 0.0124691306 0.018102248502573935 0 +376 0 -1 0.0124691306 0.018102248502573935 0 +377 0 -1 0.0124691306 0.018102248502573935 0 +378 0 -1 0.0124691306 0.018102248502573935 0 +379 0 -1 0.0124691306 0.018102248502573935 0 +380 0 -1 0.0124691306 0.018102248502573935 0 +381 1 1 0.9795901 0.029749874765915209 1 +382 0 -1 0.0124691306 0.018102248502573935 0 +383 0 -1 0.0124691306 0.018102248502573935 0 +384 0 -1 0.0124691306 0.018102248502573935 0 +385 0 -1 0.0124691306 0.018102248502573935 0 +386 1 1 0.9795901 0.029749874765915209 1 +387 0 -1 0.0124691306 0.018102248502573935 0 +388 0 -1 0.0124691306 0.018102248502573935 0 +389 0 -1 0.0124691306 0.018102248502573935 0 +390 0 -1 0.0124691306 0.018102248502573935 0 +391 1 1 0.9795901 0.029749874765915209 1 +392 0 -1 0.0124691306 0.018102248502573935 0 +393 0 -1 0.0124691306 0.018102248502573935 0 +394 0 -1 0.0124691306 0.018102248502573935 0 +395 0 -1 0.0124691306 0.018102248502573935 0 +396 0 -1 0.0124691306 0.018102248502573935 0 +397 0 -1 0.0124691306 0.018102248502573935 0 +398 0 -1 0.0124691306 0.018102248502573935 0 +399 0 -1 0.0124691306 0.018102248502573935 0 +400 1 1 0.9795901 0.029749874765915209 1 +401 0 -1 0.0124691306 0.018102248502573935 0 +402 0 -1 0.0124691306 0.018102248502573935 0 +403 0 -1 0.0124691306 0.018102248502573935 0 +404 0 -1 0.0124691306 0.018102248502573935 0 +405 0 -1 0.0124691306 0.018102248502573935 0 +406 0 -1 0.0124691306 0.018102248502573935 0 +407 0 -1 0.0124691306 0.018102248502573935 0 +408 0 -1 0.0124691306 0.018102248502573935 0 +409 0 -1 0.0124691306 0.018102248502573935 0 +410 0 -1 0.0124691306 0.018102248502573935 0 +411 0 ? ? ? 0 +412 1 1 0.9795901 0.029749874765915209 1 +413 0 -1 0.0124691306 0.018102248502573935 0 +414 1 1 0.9795901 0.029749874765915209 1 +415 0 -0.5 0.09020085 0.13638000634685848 0 +416 1 1 0.9795901 0.029749874765915209 1 +417 0 -1 0.0124691306 0.018102248502573935 0 +418 0 -1 0.0124691306 0.018102248502573935 0 +419 0 -1 0.0124691306 0.018102248502573935 0 +420 0 -1 0.0124691306 0.018102248502573935 0 +421 1 1 0.9795901 0.029749874765915209 1 +422 0 -1 0.0124691306 0.018102248502573935 0 +423 0 -1 0.0124691306 0.018102248502573935 0 +424 0 -1 0.0124691306 0.018102248502573935 0 +425 1 1 0.9795901 0.029749874765915209 1 +426 0 -1 0.0124691306 0.018102248502573935 0 +427 1 1 0.9795901 0.029749874765915209 1 +428 0 -1 0.0124691306 0.018102248502573935 0 +429 0 -1 0.0124691306 0.018102248502573935 0 +430 0 -1 0.0124691306 0.018102248502573935 0 +431 0 -1 0.0124691306 0.018102248502573935 0 +432 0 -1 0.0124691306 0.018102248502573935 0 +433 0 -1 0.0124691306 0.018102248502573935 0 +434 0 0.9 0.969497263 5.0349175091769718 1 +435 1 1 0.9795901 0.029749874765915209 1 +436 1 1 0.9795901 0.029749874765915209 1 +437 0 -1 0.0124691306 0.018102248502573935 0 +438 0 -1 0.0124691306 0.018102248502573935 0 +439 0 -1 0.0124691306 0.018102248502573935 0 +440 1 1 0.9795901 0.029749874765915209 1 +441 0 -1 0.0124691306 0.018102248502573935 0 +442 0 -1 0.0124691306 0.018102248502573935 0 +443 0 -1 0.0124691306 0.018102248502573935 0 +444 0 -1 0.0124691306 0.018102248502573935 0 +445 0 -1 0.0124691306 0.018102248502573935 0 +446 0 -1 0.0124691306 0.018102248502573935 0 +447 0 -1 0.0124691306 0.018102248502573935 0 +448 0 -1 0.0124691306 0.018102248502573935 0 +449 1 1 0.9795901 0.029749874765915209 1 +450 0 -1 0.0124691306 0.018102248502573935 0 +451 0 -1 0.0124691306 0.018102248502573935 0 +452 0 -1 0.0124691306 0.018102248502573935 0 +453 1 1 0.9795901 0.029749874765915209 1 +454 0 -1 0.0124691306 0.018102248502573935 0 +455 1 0.2 0.639659941 0.64462295779685108 1 +456 1 1 0.9795901 0.029749874765915209 1 +457 1 1 0.9795901 0.029749874765915209 1 +458 0 -1 0.0124691306 0.018102248502573935 0 +459 0 -1 0.0124691306 0.018102248502573935 0 +460 0 -1 0.0124691306 0.018102248502573935 0 +461 0 -1 0.0124691306 0.018102248502573935 0 +462 0 -1 0.0124691306 0.018102248502573935 0 +463 0 -1 0.0124691306 0.018102248502573935 0 +464 0 -1 0.0124691306 0.018102248502573935 0 +465 1 1 0.9795901 0.029749874765915209 1 +466 1 1 0.9795901 0.029749874765915209 1 +467 1 1 0.9795901 0.029749874765915209 1 +468 0 -1 0.0124691306 0.018102248502573935 0 +469 0 -1 0.0124691306 0.018102248502573935 0 +470 0 -1 0.0124691306 0.018102248502573935 0 +471 0 -1 0.0124691306 0.018102248502573935 0 +472 0 -1 0.0124691306 0.018102248502573935 0 +473 0 -1 0.0124691306 0.018102248502573935 0 +474 0 -1 0.0124691306 0.018102248502573935 0 +475 0 -1 0.0124691306 0.018102248502573935 0 +476 0 -1 0.0124691306 0.018102248502573935 0 +477 0 -1 0.0124691306 0.018102248502573935 0 +478 0 -1 0.0124691306 0.018102248502573935 0 +479 1 1 0.9795901 0.029749874765915209 1 +480 0 -1 0.0124691306 0.018102248502573935 0 +481 0 -1 0.0124691306 0.018102248502573935 0 +482 1 1 0.9795901 0.029749874765915209 1 +483 1 1 0.9795901 0.029749874765915209 1 +484 0 -1 0.0124691306 0.018102248502573935 0 +485 0 -1 0.0124691306 0.018102248502573935 0 +486 0 -1 0.0124691306 0.018102248502573935 0 +487 1 1 0.9795901 0.029749874765915209 1 +488 1 0.6 0.9022521 0.14839753396284014 1 +489 1 -0.7 0.04166677 4.5849589750811219 0 +490 0 -1 0.0124691306 0.018102248502573935 0 +491 1 1 0.9795901 0.029749874765915209 1 +492 0 -1 0.0124691306 0.018102248502573935 0 +493 1 1 0.9795901 0.029749874765915209 1 +494 0 -0.2 0.254505575 0.42373053061037591 0 +495 0 -1 0.0124691306 0.018102248502573935 0 +496 0 -1 0.0124691306 0.018102248502573935 0 +497 0 -1 0.0124691306 0.018102248502573935 0 +498 0 -1 0.0124691306 0.018102248502573935 0 +499 0 -1 0.0124691306 0.018102248502573935 0 +500 0 -1 0.0124691306 0.018102248502573935 0 +501 0 -1 0.0124691306 0.018102248502573935 0 +502 0 -1 0.0124691306 0.018102248502573935 0 +503 0 -1 0.0124691306 0.018102248502573935 0 +504 0 -1 0.0124691306 0.018102248502573935 0 +505 0 -1 0.0124691306 0.018102248502573935 0 +506 1 1 0.9795901 0.029749874765915209 1 +507 0 -1 0.0124691306 0.018102248502573935 0 +508 0 -1 0.0124691306 0.018102248502573935 0 +509 0 -1 0.0124691306 0.018102248502573935 0 +510 0 -1 0.0124691306 0.018102248502573935 0 +511 0 -1 0.0124691306 0.018102248502573935 0 +512 0 -1 0.0124691306 0.018102248502573935 0 +513 0 -1 0.0124691306 0.018102248502573935 0 +514 1 1 0.9795901 0.029749874765915209 1 +515 1 1 0.9795901 0.029749874765915209 1 +516 0 -1 0.0124691306 0.018102248502573935 0 +517 0 -1 0.0124691306 0.018102248502573935 0 +518 0 -1 0.0124691306 0.018102248502573935 0 +519 1 1 0.9795901 0.029749874765915209 1 +520 0 -1 0.0124691306 0.018102248502573935 0 +521 0 -1 0.0124691306 0.018102248502573935 0 +522 1 0.9 0.969497263 0.044691268798002393 1 +523 1 1 0.9795901 0.029749874765915209 1 +524 0 -1 0.0124691306 0.018102248502573935 0 +525 0 -1 0.0124691306 0.018102248502573935 0 +526 0 -1 0.0124691306 0.018102248502573935 0 +527 0 -1 0.0124691306 0.018102248502573935 0 +528 0 -1 0.0124691306 0.018102248502573935 0 +529 0 -1 0.0124691306 0.018102248502573935 0 +530 1 1 0.9795901 0.029749874765915209 1 +531 0 -1 0.0124691306 0.018102248502573935 0 +532 0 -1 0.0124691306 0.018102248502573935 0 +533 0 -1 0.0124691306 0.018102248502573935 0 +534 0 -1 0.0124691306 0.018102248502573935 0 +535 0 -1 0.0124691306 0.018102248502573935 0 +536 0 -1 0.0124691306 0.018102248502573935 0 +537 0 -1 0.0124691306 0.018102248502573935 0 +538 0 -1 0.0124691306 0.018102248502573935 0 +539 0 -1 0.0124691306 0.018102248502573935 0 +540 0 -1 0.0124691306 0.018102248502573935 0 +541 0 -1 0.0124691306 0.018102248502573935 0 +542 0 -1 0.0124691306 0.018102248502573935 0 +543 0 -1 0.0124691306 0.018102248502573935 0 +544 0 -1 0.0124691306 0.018102248502573935 0 +545 0 -1 0.0124691306 0.018102248502573935 0 +546 1 1 0.9795901 0.029749874765915209 1 +547 0 -1 0.0124691306 0.018102248502573935 0 +548 0 -1 0.0124691306 0.018102248502573935 0 +549 1 1 0.9795901 0.029749874765915209 1 +550 0 -1 0.0124691306 0.018102248502573935 0 +551 0 -1 0.0124691306 0.018102248502573935 0 +552 0 -1 0.0124691306 0.018102248502573935 0 +553 0 -0.9 0.0187102128 0.027248849528611944 0 +554 0 -1 0.0124691306 0.018102248502573935 0 +555 0 -0.9 0.0187102128 0.027248849528611944 0 +556 0 -1 0.0124691306 0.018102248502573935 0 +557 0 -1 0.0124691306 0.018102248502573935 0 +558 0 -1 0.0124691306 0.018102248502573935 0 +559 0 -1 0.0124691306 0.018102248502573935 0 +560 0 -1 0.0124691306 0.018102248502573935 0 +561 0 -1 0.0124691306 0.018102248502573935 0 +562 0 -1 0.0124691306 0.018102248502573935 0 +563 0 -1 0.0124691306 0.018102248502573935 0 +564 0 -1 0.0124691306 0.018102248502573935 0 +565 1 1 0.9795901 0.029749874765915209 1 +566 0 -1 0.0124691306 0.018102248502573935 0 +567 0 -1 0.0124691306 0.018102248502573935 0 +568 1 0.9 0.969497263 0.044691268798002393 1 +569 1 1 0.9795901 0.029749874765915209 1 +570 1 1 0.9795901 0.029749874765915209 1 +571 1 1 0.9795901 0.029749874765915209 1 +572 0 -1 0.0124691306 0.018102248502573935 0 +573 0 -1 0.0124691306 0.018102248502573935 0 +574 1 1 0.9795901 0.029749874765915209 1 +575 0 -1 0.0124691306 0.018102248502573935 0 +576 0 -1 0.0124691306 0.018102248502573935 0 +577 0 -1 0.0124691306 0.018102248502573935 0 +578 0 -1 0.0124691306 0.018102248502573935 0 +579 0 -1 0.0124691306 0.018102248502573935 0 +580 0 -1 0.0124691306 0.018102248502573935 0 +581 1 1 0.9795901 0.029749874765915209 1 +582 1 1 0.9795901 0.029749874765915209 1 +583 0 -1 0.0124691306 0.018102248502573935 0 +584 0 -1 0.0124691306 0.018102248502573935 0 +585 0 -1 0.0124691306 0.018102248502573935 0 +586 1 1 0.9795901 0.029749874765915209 1 +587 0 -1 0.0124691306 0.018102248502573935 0 +588 1 0.9 0.969497263 0.044691268798002393 1 +589 0 -1 0.0124691306 0.018102248502573935 0 +590 1 0.9 0.969497263 0.044691268798002393 1 +591 1 1 0.9795901 0.029749874765915209 1 +592 1 0.9 0.969497263 0.044691268798002393 1 +593 0 -1 0.0124691306 0.018102248502573935 0 +594 1 0.9 0.969497263 0.044691268798002393 1 +595 0 -1 0.0124691306 0.018102248502573935 0 +596 0 -1 0.0124691306 0.018102248502573935 0 +597 0 -1 0.0124691306 0.018102248502573935 0 +598 0 -1 0.0124691306 0.018102248502573935 0 +599 0 -1 0.0124691306 0.018102248502573935 0 +600 0 -1 0.0124691306 0.018102248502573935 0 +601 0 -1 0.0124691306 0.018102248502573935 0 +602 0 -1 0.0124691306 0.018102248502573935 0 +603 1 0.8 0.954644561 0.066964414934699193 1 +604 1 0.9 0.969497263 0.044691268798002393 1 +605 1 1 0.9795901 0.029749874765915209 1 +606 0 -1 0.0124691306 0.018102248502573935 0 +607 0 -1 0.0124691306 0.018102248502573935 0 +608 1 1 0.9795901 0.029749874765915209 1 +609 0 -1 0.0124691306 0.018102248502573935 0 +610 1 1 0.9795901 0.029749874765915209 1 +611 1 1 0.9795901 0.029749874765915209 1 +612 1 1 0.9795901 0.029749874765915209 1 +613 0 -1 0.0124691306 0.018102248502573935 0 +614 0 -1 0.0124691306 0.018102248502573935 0 +615 0 -1 0.0124691306 0.018102248502573935 0 +616 0 -1 0.0124691306 0.018102248502573935 0 +617 0 ? ? ? 0 +618 0 -1 0.0124691306 0.018102248502573935 0 +619 0 -1 0.0124691306 0.018102248502573935 0 +620 0 -1 0.0124691306 0.018102248502573935 0 +621 0 -0.3 0.184390724 0.29404991101364197 0 +622 0 -1 0.0124691306 0.018102248502573935 0 +623 0 -1 0.0124691306 0.018102248502573935 0 +624 0 -1 0.0124691306 0.018102248502573935 0 +625 0 -1 0.0124691306 0.018102248502573935 0 +626 1 0.9 0.969497263 0.044691268798002393 1 +627 0 -1 0.0124691306 0.018102248502573935 0 +628 0 -1 0.0124691306 0.018102248502573935 0 +629 0 -1 0.0124691306 0.018102248502573935 0 +630 0 -1 0.0124691306 0.018102248502573935 0 +631 0 -1 0.0124691306 0.018102248502573935 0 +632 0 -1 0.0124691306 0.018102248502573935 0 +633 1 1 0.9795901 0.029749874765915209 1 +634 0 -1 0.0124691306 0.018102248502573935 0 +635 0 -1 0.0124691306 0.018102248502573935 0 +636 1 1 0.9795901 0.029749874765915209 1 +637 0 -1 0.0124691306 0.018102248502573935 0 +638 0 -1 0.0124691306 0.018102248502573935 0 +639 0 -1 0.0124691306 0.018102248502573935 0 +640 0 -1 0.0124691306 0.018102248502573935 0 +641 0 -1 0.0124691306 0.018102248502573935 0 +642 0 -1 0.0124691306 0.018102248502573935 0 +643 0 -1 0.0124691306 0.018102248502573935 0 +644 0 -1 0.0124691306 0.018102248502573935 0 +645 0 -1 0.0124691306 0.018102248502573935 0 +646 0 -1 0.0124691306 0.018102248502573935 0 +647 0 -1 0.0124691306 0.018102248502573935 0 +648 1 1 0.9795901 0.029749874765915209 1 +649 0 -1 0.0124691306 0.018102248502573935 0 +650 0 -1 0.0124691306 0.018102248502573935 0 +651 0 -1 0.0124691306 0.018102248502573935 0 +652 0 -1 0.0124691306 0.018102248502573935 0 +653 0 -1 0.0124691306 0.018102248502573935 0 +654 0 -1 0.0124691306 0.018102248502573935 0 +655 0 -1 0.0124691306 0.018102248502573935 0 +656 0 -1 0.0124691306 0.018102248502573935 0 +657 0 -0.2 0.254505575 0.42373053061037591 0 +658 1 1 0.9795901 0.029749874765915209 1 +659 0 -1 0.0124691306 0.018102248502573935 0 +660 0 -1 0.0124691306 0.018102248502573935 0 +661 0 -1 0.0124691306 0.018102248502573935 0 +662 0 -1 0.0124691306 0.018102248502573935 0 +663 0 -1 0.0124691306 0.018102248502573935 0 +664 0 -1 0.0124691306 0.018102248502573935 0 +665 0 -1 0.0124691306 0.018102248502573935 0 +666 0 -1 0.0124691306 0.018102248502573935 0 +667 0 -1 0.0124691306 0.018102248502573935 0 +668 1 0.6 0.9022521 0.14839753396284014 1 +669 1 1 0.9795901 0.029749874765915209 1 +670 1 1 0.9795901 0.029749874765915209 1 +671 0 -1 0.0124691306 0.018102248502573935 0 +672 0 -1 0.0124691306 0.018102248502573935 0 +673 0 -1 0.0124691306 0.018102248502573935 0 +674 0 -1 0.0124691306 0.018102248502573935 0 +675 0 -1 0.0124691306 0.018102248502573935 0 +676 0 -1 0.0124691306 0.018102248502573935 0 +677 0 -1 0.0124691306 0.018102248502573935 0 +678 0 -1 0.0124691306 0.018102248502573935 0 +679 0 -1 0.0124691306 0.018102248502573935 0 +680 1 1 0.9795901 0.029749874765915209 1 +681 1 1 0.9795901 0.029749874765915209 1 +682 0 -1 0.0124691306 0.018102248502573935 0 +683 0 -1 0.0124691306 0.018102248502573935 0 +684 0 -1 0.0124691306 0.018102248502573935 0 +685 0 -1 0.0124691306 0.018102248502573935 0 +686 0 -1 0.0124691306 0.018102248502573935 0 +687 0 -1 0.0124691306 0.018102248502573935 0 +688 0 -1 0.0124691306 0.018102248502573935 0 +689 0 -1 0.0124691306 0.018102248502573935 0 +690 0 -1 0.0124691306 0.018102248502573935 0 +691 1 1 0.9795901 0.029749874765915209 1 +692 0 -1 0.0124691306 0.018102248502573935 0 +693 0 -1 0.0124691306 0.018102248502573935 0 +694 0 -1 0.0124691306 0.018102248502573935 0 +695 0 -1 0.0124691306 0.018102248502573935 0 +696 1 1 0.9795901 0.029749874765915209 1 +697 1 0.9 0.969497263 0.044691268798002393 1 +698 1 1 0.9795901 0.029749874765915209 1 diff --git a/test/BaselineOutput/Common/WeightedEnsembleMulticlass/WE-Average-TrainTest-iris-out.txt b/test/BaselineOutput/Common/WeightedEnsembleMulticlass/WE-Average-TrainTest-iris-out.txt new file mode 100644 index 0000000000..5e299a5d5c --- /dev/null +++ b/test/BaselineOutput/Common/WeightedEnsembleMulticlass/WE-Average-TrainTest-iris-out.txt @@ -0,0 +1,61 @@ +maml.exe TrainTest test=%Data% tr=WeightedEnsembleMulticlass{bp=mlr{t-} nm=5 oc=MultiAverage tp=-} dout=%Output% data=%Data% out=%Output% seed=1 xf=Term{col=Label} +Automatically adding a MinMax normalization transform, use 'norm=Warn' or 'norm=No' to turn this behavior off. +Training 5 learners for the batch 1 +Beginning training model 1 of 5 +Beginning optimization +num vars: 15 +improvement criterion: Mean Improvement +L1 regularization selected 12 of 15 weights. +Trainer 1 of 5 finished in %Time% +Beginning training model 2 of 5 +Beginning optimization +num vars: 15 +improvement criterion: Mean Improvement +L1 regularization selected 11 of 15 weights. +Trainer 2 of 5 finished in %Time% +Beginning training model 3 of 5 +Beginning optimization +num vars: 15 +improvement criterion: Mean Improvement +L1 regularization selected 11 of 15 weights. +Trainer 3 of 5 finished in %Time% +Beginning training model 4 of 5 +Beginning optimization +num vars: 15 +improvement criterion: Mean Improvement +L1 regularization selected 10 of 15 weights. +Trainer 4 of 5 finished in %Time% +Beginning training model 5 of 5 +Beginning optimization +num vars: 15 +improvement criterion: Mean Improvement +L1 regularization selected 11 of 15 weights. +Trainer 5 of 5 finished in %Time% +Not training a calibrator because it is not needed. + +Confusion table + ||======================== +PREDICTED || 0 | 1 | 2 | Recall +TRUTH ||======================== + 0 || 50 | 0 | 0 | 1.0000 + 1 || 0 | 45 | 5 | 0.9000 + 2 || 0 | 3 | 47 | 0.9400 + ||======================== +Precision ||1.0000 |0.9375 |0.9038 | +Accuracy(micro-avg): 0.946667 +Accuracy(macro-avg): 0.946667 +Log-loss: 0.433374 +Log-loss reduction: 0.605526 + +OVERALL RESULTS +--------------------------------------- +Accuracy(micro-avg): 0.946667 (0.0000) +Accuracy(macro-avg): 0.946667 (0.0000) +Log-loss: 0.433374 (0.0000) +Log-loss reduction: 0.605526 (0.0000) + +--------------------------------------- +Physical memory usage(MB): %Number% +Virtual memory usage(MB): %Number% +%DateTime% Time elapsed(s): %Number% + diff --git a/test/BaselineOutput/Common/WeightedEnsembleMulticlass/WE-Average-TrainTest-iris-rp.txt b/test/BaselineOutput/Common/WeightedEnsembleMulticlass/WE-Average-TrainTest-iris-rp.txt new file mode 100644 index 0000000000..aee52944d9 --- /dev/null +++ b/test/BaselineOutput/Common/WeightedEnsembleMulticlass/WE-Average-TrainTest-iris-rp.txt @@ -0,0 +1,4 @@ +WeightedEnsembleMulticlass +Accuracy(micro-avg) Accuracy(macro-avg) Log-loss Log-loss reduction /oc /bp /nm Learner Name Train Dataset Test Dataset Results File Run Time Physical Memory Virtual Memory Command Line Settings +0.946667 0.946667 0.433374 0.605526 MultiAverage mlr{t-} 5 WeightedEnsembleMulticlass %Data% %Data% %Output% 99 0 0 maml.exe TrainTest test=%Data% tr=WeightedEnsembleMulticlass{bp=mlr{t-} nm=5 oc=MultiAverage tp=-} dout=%Output% data=%Data% out=%Output% seed=1 xf=Term{col=Label} /oc:MultiAverage;/bp:mlr{t-};/nm:5 + diff --git a/test/BaselineOutput/Common/WeightedEnsembleMulticlass/WE-Average-TrainTest-iris.txt b/test/BaselineOutput/Common/WeightedEnsembleMulticlass/WE-Average-TrainTest-iris.txt new file mode 100644 index 0000000000..e0958b0827 --- /dev/null +++ b/test/BaselineOutput/Common/WeightedEnsembleMulticlass/WE-Average-TrainTest-iris.txt @@ -0,0 +1,151 @@ +Instance Label Assigned Log-loss #1 Score #2 Score #3 Score #1 Class #2 Class #3 Class +0 0 0 0.19964178652788431 0.8190241 0.169545561 0.0114304228 0 1 2 +1 0 0 0.22951878783682567 0.794916034 0.193235487 0.0118485028 0 1 2 +2 0 0 0.2084796113210233 0.8118176 0.177210674 0.0109718461 0 1 2 +3 0 0 0.23157376368569668 0.7932842 0.194256365 0.0124595007 0 1 2 +4 0 0 0.19400474503683612 0.823654 0.165016592 0.0113294413 0 1 2 +5 0 0 0.25624980316652674 0.7739486 0.204800844 0.0212505851 0 1 2 +6 0 0 0.23077132670987072 0.793921 0.19157438 0.0145046478 0 1 2 +7 0 0 0.21337880346374763 0.807850063 0.17991963 0.0122303637 0 1 2 +8 0 0 0.2354409214974969 0.790222347 0.197895989 0.0118817212 0 1 2 +9 0 0 0.20605934828580402 0.8137848 0.176337421 0.009877759 0 1 2 +10 0 0 0.19663610735576834 0.8214895 0.166520938 0.0119895991 0 1 2 +11 0 0 0.22159895795533788 0.8012366 0.185794353 0.0129691036 0 1 2 +12 0 0 0.20383082880816983 0.815600336 0.175041348 0.009358378 0 1 2 +13 0 0 0.1808468099213901 0.8345632 0.157684714 0.00775217 0 1 2 +14 0 0 0.16087145456021751 0.8514015 0.138852566 0.009745969 0 1 2 +15 0 0 0.20751656840989915 0.8125998 0.169173434 0.01822683 0 1 2 +16 0 0 0.21917957335463989 0.8031775 0.1800779 0.0167446025 0 1 2 +17 0 0 0.22508323789680376 0.798449755 0.187088668 0.0144615844 0 1 2 +18 0 0 0.23374900200093426 0.7915605 0.191414639 0.01702489 0 1 2 +19 0 0 0.21546694161236146 0.8061649 0.178815886 0.0150192995 0 1 2 +20 0 0 0.23110269328463856 0.793657959 0.192498714 0.0138433492 0 1 2 +21 0 0 0.24978411806025416 0.77896893 0.201938823 0.01909236 0 1 2 +22 0 0 0.16530709303176674 0.847633362 0.143547833 0.00881886 0 1 2 +23 0 0 0.33750806982820597 0.7135462 0.258942366 0.0275114775 0 1 2 +24 0 0 0.24884263271547108 0.779702663 0.2047818 0.0155155761 0 1 2 +25 0 0 0.24797642346965068 0.780378342 0.206260011 0.0133616524 0 1 2 +26 0 0 0.28127653581346135 0.7548196 0.224557638 0.0206227489 0 1 2 +27 0 0 0.20771705557310186 0.8124369 0.1754048 0.0121583557 0 1 2 +28 0 0 0.20544311883623737 0.8142864 0.174183115 0.0115304766 0 1 2 +29 0 0 0.23417370988320155 0.79122436 0.195629567 0.0131461071 0 1 2 +30 0 0 0.24091548654658776 0.785908043 0.200842962 0.013249062 0 1 2 +31 0 0 0.27128537128965724 0.7623989 0.218081042 0.0195200685 0 1 2 +32 0 0 0.15541422202837735 0.8560605 0.134833977 0.009105596 0 1 2 +33 0 0 0.16443896124544094 0.848369539 0.140814647 0.0108158756 0 1 2 +34 0 0 0.20605934828580402 0.8137848 0.176337421 0.009877759 0 1 2 +35 0 0 0.20083761564339722 0.818045259 0.171603829 0.0103509827 0 1 2 +36 0 0 0.19238816304164053 0.8249866 0.1642232 0.0107903061 0 1 2 +37 0 0 0.20605934828580402 0.8137848 0.176337421 0.009877759 0 1 2 +38 0 0 0.22024514366862238 0.8023221 0.186568588 0.01110938 0 1 2 +39 0 0 0.21349435246329268 0.8077567 0.180002347 0.0122409752 0 1 2 +40 0 0 0.21636426223018745 0.805441856 0.18095386 0.01360445 0 1 2 +41 0 0 0.300935140554702 0.7401258 0.245229438 0.0146448705 0 1 2 +42 0 0 0.20814442617103573 0.812089741 0.176966742 0.0109434845 0 1 2 +43 0 0 0.34684869556561498 0.7069123 0.260975718 0.03211199 0 1 2 +44 0 0 0.28382412296815501 0.752899051 0.223123714 0.0239772853 0 1 2 +45 0 0 0.25802469477335477 0.772576153 0.212492138 0.0149317207 0 1 2 +46 0 0 0.19856340302091133 0.8199078 0.1674826 0.01260967 0 1 2 +47 0 0 0.21662360032740066 0.805233 0.183120176 0.0116468724 0 1 2 +48 0 0 0.19652742318576463 0.8215788 0.166442171 0.0119790332 0 1 2 +49 0 0 0.21107032843614537 0.8097171 0.178683609 0.0115993088 0 1 2 +50 1 1 0.72253030579451949 0.485522181 0.3959627 0.118515171 1 2 0 +51 1 1 0.75069376608937632 0.472038954 0.417035669 0.110925436 1 2 0 +52 1 1 0.76992466339057941 0.463047951 0.448236853 0.08871519 1 2 0 +53 1 1 0.57114309662714047 0.564879358 0.2779409 0.157179758 1 2 0 +54 1 1 0.71212027535206868 0.490602881 0.4126532 0.09674395 1 2 0 +55 1 1 0.63447748116277813 0.530212462 0.329003 0.1407846 1 2 0 +56 1 2 0.82721263714924198 0.476524562 0.4372664 0.08620906 2 1 0 +57 1 1 0.60860888553171666 0.544107258 0.306743264 0.149149552 1 0 2 +58 1 1 0.64934687567342519 0.522386849 0.340838641 0.13677451 1 2 0 +59 1 1 0.63354104040495318 0.5307092 0.3140783 0.155212566 1 2 0 +60 1 1 0.544000291794008 0.580421746 0.263815016 0.155763283 1 0 2 +61 1 1 0.70913928400159099 0.492067546 0.38484475 0.123087779 1 2 0 +62 1 1 0.53772245447215139 0.584077 0.228479132 0.187443942 1 0 2 +63 1 1 0.6876554148173839 0.502753437 0.385602355 0.111644246 1 2 0 +64 1 1 0.63969776714863891 0.5274518 0.2611303 0.211417943 1 2 0 +65 1 1 0.69557776962683771 0.498786181 0.367391735 0.133822158 1 2 0 +66 1 1 0.72665495461553098 0.4835237 0.409676 0.106800377 1 2 0 +67 1 1 0.59049921667944671 0.5540506 0.245183617 0.200765833 1 0 2 +68 1 1 0.64084881934070614 0.526845038 0.384088039 0.08906691 1 2 0 +69 1 1 0.57461759069095975 0.5629201 0.224040136 0.2130398 1 0 2 +70 1 2 0.94947276807720926 0.557861745 0.386944979 0.0551933162 2 1 0 +71 1 1 0.62570932315008765 0.5348819 0.290267438 0.1748507 1 2 0 +72 1 1 0.70221282931715068 0.49548766 0.4264362 0.07807618 1 2 0 +73 1 1 0.61438280971727099 0.5409747 0.309653163 0.149372146 1 2 0 +74 1 1 0.64093503197233559 0.5267996 0.316553921 0.15664652 1 2 0 +75 1 1 0.68409274636431439 0.5045478 0.364429623 0.131022632 1 2 0 +76 1 1 0.68343601449981528 0.504879236 0.391535372 0.103585385 1 2 0 +77 1 2 0.88369901028835107 0.52954185 0.41325146 0.05720675 2 1 0 +78 1 1 0.71580603916625718 0.488797963 0.406992733 0.104209363 1 2 0 +79 1 1 0.61677282561025504 0.5396833 0.29744342 0.1628734 1 0 2 +80 1 1 0.56705401365812369 0.5671939 0.227945253 0.204860836 1 0 2 +81 1 1 0.57748726284013074 0.561307 0.266655535 0.172037512 1 0 2 +82 1 1 0.60202423566927699 0.547701836 0.2471485 0.205149725 1 2 0 +83 1 2 0.79633524316154058 0.487356961 0.450978667 0.06166442 2 1 0 +84 1 1 0.72641836314623376 0.4836381 0.409415454 0.106946483 1 2 0 +85 1 2 0.82110668345240334 0.462358445 0.4399445 0.09769705 2 1 0 +86 1 1 0.75367750482391882 0.4706326 0.431179047 0.0981883556 1 2 0 +87 1 1 0.5791870940103041 0.5603537 0.308470815 0.1311755 1 2 0 +88 1 1 0.64913273180683329 0.5224987 0.302041262 0.175460026 1 2 0 +89 1 1 0.59236617660626123 0.5530172 0.282663852 0.164318964 1 2 0 +90 1 1 0.58820663203562762 0.5553223 0.281281233 0.163396522 1 2 0 +91 1 1 0.6932948917784636 0.49992615 0.380387276 0.119686626 1 2 0 +92 1 1 0.58914215832554684 0.554803 0.25239563 0.192801386 1 2 0 +93 1 1 0.59569935036516897 0.551176965 0.3005272 0.148295969 1 0 2 +94 1 1 0.61653948519540414 0.5398092 0.302781016 0.157409832 1 2 0 +95 1 1 0.6328785109024514 0.531060934 0.275205731 0.193733349 1 2 0 +96 1 1 0.6388684290940182 0.527889431 0.307758659 0.16435194 1 2 0 +97 1 1 0.64083105729409251 0.5268544 0.316307634 0.156838045 1 2 0 +98 1 1 0.63013315167809769 0.5325209 0.310088456 0.157390758 1 0 2 +99 1 1 0.62635886668941432 0.5345346 0.2975428 0.167922691 1 2 0 +100 2 2 0.17859283178890564 0.8364464 0.157708928 0.005844734 2 1 0 +101 2 2 0.51434815688955982 0.5978902 0.3675669 0.03454289 2 1 0 +102 2 2 0.31784208398474723 0.7277177 0.257523477 0.0147588132 2 1 0 +103 2 2 0.4981384961186156 0.6076608 0.35929355 0.0330457352 2 1 0 +104 2 2 0.28869117148345358 0.749243557 0.238149866 0.0126066161 2 1 0 +105 2 2 0.26633537690849279 0.7661821 0.224274546 0.009543408 2 1 0 +106 2 2 0.75901868200630807 0.468125582 0.464001715 0.0678726956 2 1 0 +107 2 2 0.4193688539414191 0.657461643 0.320701718 0.02183666 2 1 0 +108 2 2 0.5002123761989884 0.606401861 0.366542 0.02705622 2 1 0 +109 2 2 0.16479698922555039 0.848065853 0.146216467 0.00571767054 2 1 0 +110 2 2 0.42694692042576293 0.6524982 0.316612929 0.0308889244 2 1 0 +111 2 2 0.48923841699985121 0.613093138 0.356241226 0.0306656044 2 1 0 +112 2 2 0.35160322709748859 0.7035592 0.277582854 0.018857969 2 1 0 +113 2 2 0.48202166334124041 0.6175337 0.353819072 0.0286472384 2 1 0 +114 2 2 0.27781122705951888 0.7574398 0.230573371 0.011986834 2 1 0 +115 2 2 0.28058827322063545 0.755339265 0.230445966 0.0142147811 2 1 0 +116 2 2 0.5033410937064291 0.604507565 0.359842926 0.03564959 2 1 0 +117 2 2 0.20074428341502251 0.8181216 0.173690245 0.008188161 2 1 0 +118 2 2 0.20552693477312187 0.814218163 0.181163326 0.00461854925 2 1 0 +119 2 1 0.86016511836866272 0.5073447 0.423092216 0.0695631057 1 2 0 +120 2 2 0.25277774509739498 0.7766405 0.212321281 0.0110382736 2 1 0 +121 2 2 0.47463402676516708 0.6221127 0.345621347 0.03226599 2 1 0 +122 2 2 0.30329327119364452 0.7383825 0.250785321 0.0108321486 2 1 0 +123 2 2 0.60322989264566518 0.5470419 0.4060424 0.0469157621 2 1 0 +124 2 2 0.32000100495981476 0.7261483 0.256151229 0.01770047 2 1 0 +125 2 2 0.43285107863415828 0.6486571 0.323673338 0.027669657 2 1 0 +126 2 2 0.61063117193670835 0.543008 0.406340837 0.05065116 2 1 0 +127 2 2 0.58260569566962017 0.558441341 0.391519129 0.05003956 2 1 0 +128 2 2 0.35379102298130893 0.702021658 0.2808303 0.0171480719 2 1 0 +129 2 2 0.5876438266123245 0.5556349 0.3996999 0.0446652435 2 1 0 +130 2 2 0.39649659112841007 0.67267257 0.307880431 0.0194469839 2 1 0 +131 2 2 0.27998391801888911 0.7557959 0.22853224 0.01567193 2 1 0 +132 2 2 0.31316717053358906 0.7311277 0.255125284 0.0137470756 2 1 0 +133 2 1 0.79097307844554376 0.4712194 0.453403383 0.07537719 1 2 0 +134 2 1 0.80256455301821128 0.4850471 0.4481781 0.0667748 1 2 0 +135 2 2 0.23504417514416331 0.7905359 0.20123589 0.008228176 2 1 0 +136 2 2 0.22171940393592623 0.8011401 0.189145848 0.009714014 2 1 0 +137 2 2 0.49710850703430304 0.608287 0.355297565 0.0364154875 2 1 0 +138 2 2 0.59703036598034886 0.5504438 0.396652877 0.05290335 2 1 0 +139 2 2 0.35515806685565859 0.7010626 0.278541237 0.0203961879 2 1 0 +140 2 2 0.23235270716193504 0.7926665 0.198152646 0.0091809025 2 1 0 +141 2 2 0.29924923237144857 0.7413746 0.242910311 0.0157151017 2 1 0 +142 2 2 0.51434815688955982 0.5978902 0.3675669 0.03454289 2 1 0 +143 2 2 0.24025451662084005 0.7864277 0.203824714 0.009747634 2 1 0 +144 2 2 0.19277744540305095 0.8246655 0.168277055 0.00705749542 2 1 0 +145 2 2 0.29643526389506997 0.743463755 0.242001578 0.0145347184 2 1 0 +146 2 2 0.53942056698930985 0.583086 0.381982148 0.0349318273 2 1 0 +147 2 2 0.42786767900756778 0.651897669 0.320094556 0.028007796 2 1 0 +148 2 2 0.26539295353718467 0.766904533 0.219215557 0.0138799325 2 1 0 +149 2 2 0.5556033513335692 0.573726 0.381408066 0.0448659733 2 1 0 diff --git a/test/BaselineOutput/Common/WeightedEnsembleMulticlass/WE-Bootstrap-TrainTest-iris-out.txt b/test/BaselineOutput/Common/WeightedEnsembleMulticlass/WE-Bootstrap-TrainTest-iris-out.txt new file mode 100644 index 0000000000..1446808e2b --- /dev/null +++ b/test/BaselineOutput/Common/WeightedEnsembleMulticlass/WE-Bootstrap-TrainTest-iris-out.txt @@ -0,0 +1,151 @@ +maml.exe TrainTest test=%Data% tr=WeightedEnsembleMulticlass{bp=mlr{t-} nm=20 st=BootstrapSelector{} tp=-} dout=%Output% data=%Data% out=%Output% seed=1 xf=Term{col=Label} +Automatically adding a MinMax normalization transform, use 'norm=Warn' or 'norm=No' to turn this behavior off. +Training 20 learners for the batch 1 +Beginning training model 1 of 20 +Beginning optimization +num vars: 15 +improvement criterion: Mean Improvement +L1 regularization selected 12 of 15 weights. +Trainer 1 of 20 finished in %Time% +Beginning training model 2 of 20 +Beginning optimization +num vars: 15 +improvement criterion: Mean Improvement +L1 regularization selected 11 of 15 weights. +Trainer 2 of 20 finished in %Time% +Beginning training model 3 of 20 +Beginning optimization +num vars: 15 +improvement criterion: Mean Improvement +L1 regularization selected 11 of 15 weights. +Trainer 3 of 20 finished in %Time% +Beginning training model 4 of 20 +Beginning optimization +num vars: 15 +improvement criterion: Mean Improvement +L1 regularization selected 10 of 15 weights. +Trainer 4 of 20 finished in %Time% +Beginning training model 5 of 20 +Beginning optimization +num vars: 15 +improvement criterion: Mean Improvement +L1 regularization selected 11 of 15 weights. +Trainer 5 of 20 finished in %Time% +Beginning training model 6 of 20 +Beginning optimization +num vars: 15 +improvement criterion: Mean Improvement +L1 regularization selected 11 of 15 weights. +Trainer 6 of 20 finished in %Time% +Beginning training model 7 of 20 +Beginning optimization +num vars: 15 +improvement criterion: Mean Improvement +L1 regularization selected 11 of 15 weights. +Trainer 7 of 20 finished in %Time% +Beginning training model 8 of 20 +Beginning optimization +num vars: 15 +improvement criterion: Mean Improvement +L1 regularization selected 11 of 15 weights. +Trainer 8 of 20 finished in %Time% +Beginning training model 9 of 20 +Beginning optimization +num vars: 15 +improvement criterion: Mean Improvement +L1 regularization selected 11 of 15 weights. +Trainer 9 of 20 finished in %Time% +Beginning training model 10 of 20 +Beginning optimization +num vars: 15 +improvement criterion: Mean Improvement +L1 regularization selected 11 of 15 weights. +Trainer 10 of 20 finished in %Time% +Beginning training model 11 of 20 +Beginning optimization +num vars: 15 +improvement criterion: Mean Improvement +L1 regularization selected 12 of 15 weights. +Trainer 11 of 20 finished in %Time% +Beginning training model 12 of 20 +Beginning optimization +num vars: 15 +improvement criterion: Mean Improvement +L1 regularization selected 11 of 15 weights. +Trainer 12 of 20 finished in %Time% +Beginning training model 13 of 20 +Beginning optimization +num vars: 15 +improvement criterion: Mean Improvement +L1 regularization selected 11 of 15 weights. +Trainer 13 of 20 finished in %Time% +Beginning training model 14 of 20 +Beginning optimization +num vars: 15 +improvement criterion: Mean Improvement +L1 regularization selected 12 of 15 weights. +Trainer 14 of 20 finished in %Time% +Beginning training model 15 of 20 +Beginning optimization +num vars: 15 +improvement criterion: Mean Improvement +L1 regularization selected 11 of 15 weights. +Trainer 15 of 20 finished in %Time% +Beginning training model 16 of 20 +Beginning optimization +num vars: 15 +improvement criterion: Mean Improvement +L1 regularization selected 11 of 15 weights. +Trainer 16 of 20 finished in %Time% +Beginning training model 17 of 20 +Beginning optimization +num vars: 15 +improvement criterion: Mean Improvement +L1 regularization selected 11 of 15 weights. +Trainer 17 of 20 finished in %Time% +Beginning training model 18 of 20 +Beginning optimization +num vars: 15 +improvement criterion: Mean Improvement +L1 regularization selected 11 of 15 weights. +Trainer 18 of 20 finished in %Time% +Beginning training model 19 of 20 +Beginning optimization +num vars: 15 +improvement criterion: Mean Improvement +L1 regularization selected 12 of 15 weights. +Trainer 19 of 20 finished in %Time% +Beginning training model 20 of 20 +Beginning optimization +num vars: 15 +improvement criterion: Mean Improvement +L1 regularization selected 12 of 15 weights. +Trainer 20 of 20 finished in %Time% +Not training a calibrator because it is not needed. + +Confusion table + ||======================== +PREDICTED || 0 | 1 | 2 | Recall +TRUTH ||======================== + 0 || 50 | 0 | 0 | 1.0000 + 1 || 0 | 45 | 5 | 0.9000 + 2 || 0 | 3 | 47 | 0.9400 + ||======================== +Precision ||1.0000 |0.9375 |0.9038 | +Accuracy(micro-avg): 0.946667 +Accuracy(macro-avg): 0.946667 +Log-loss: 0.434962 +Log-loss reduction: 0.604081 + +OVERALL RESULTS +--------------------------------------- +Accuracy(micro-avg): 0.946667 (0.0000) +Accuracy(macro-avg): 0.946667 (0.0000) +Log-loss: 0.434962 (0.0000) +Log-loss reduction: 0.604081 (0.0000) + +--------------------------------------- +Physical memory usage(MB): %Number% +Virtual memory usage(MB): %Number% +%DateTime% Time elapsed(s): %Number% + diff --git a/test/BaselineOutput/Common/WeightedEnsembleMulticlass/WE-Bootstrap-TrainTest-iris-rp.txt b/test/BaselineOutput/Common/WeightedEnsembleMulticlass/WE-Bootstrap-TrainTest-iris-rp.txt new file mode 100644 index 0000000000..c071d09bbd --- /dev/null +++ b/test/BaselineOutput/Common/WeightedEnsembleMulticlass/WE-Bootstrap-TrainTest-iris-rp.txt @@ -0,0 +1,4 @@ +WeightedEnsembleMulticlass +Accuracy(micro-avg) Accuracy(macro-avg) Log-loss Log-loss reduction /bp /nm Learner Name Train Dataset Test Dataset Results File Run Time Physical Memory Virtual Memory Command Line Settings +0.946667 0.946667 0.434962 0.604081 mlr{t-} 20 WeightedEnsembleMulticlass %Data% %Data% %Output% 99 0 0 maml.exe TrainTest test=%Data% tr=WeightedEnsembleMulticlass{bp=mlr{t-} nm=20 st=BootstrapSelector{} tp=-} dout=%Output% data=%Data% out=%Output% seed=1 xf=Term{col=Label} /bp:mlr{t-};/nm:20 + diff --git a/test/BaselineOutput/Common/WeightedEnsembleMulticlass/WE-Bootstrap-TrainTest-iris.txt b/test/BaselineOutput/Common/WeightedEnsembleMulticlass/WE-Bootstrap-TrainTest-iris.txt new file mode 100644 index 0000000000..65fd773a6c --- /dev/null +++ b/test/BaselineOutput/Common/WeightedEnsembleMulticlass/WE-Bootstrap-TrainTest-iris.txt @@ -0,0 +1,151 @@ +Instance Label Assigned Log-loss #1 Score #2 Score #3 Score #1 Class #2 Class #3 Class +0 0 0 0.19138086602936072 0.825818 0.163233042 0.0112717841 0 1 2 +1 0 0 0.21081139603990082 0.8099268 0.177834481 0.0116552487 0 1 2 +2 0 0 0.19393455236298313 0.8237118 0.165597647 0.0107214814 0 1 2 +3 0 0 0.21423806664589068 0.8071562 0.180384234 0.0122111794 0 1 2 +4 0 0 0.18771144144498667 0.828853846 0.160079092 0.0111219641 0 1 2 +5 0 0 0.25527505392676209 0.7747034 0.202383041 0.0211011246 0 1 2 +6 0 0 0.22015310238339808 0.80239594 0.1834884 0.0140191894 0 1 2 +7 0 0 0.20277266873254787 0.8164638 0.17210038 0.0120522631 0 1 2 +8 0 0 0.21475895892869667 0.8067359 0.180851191 0.011660195 0 1 2 +9 0 0 0.18917001968647357 0.8276458 0.16313988 0.009780681 0 1 2 +10 0 0 0.1923006008088228 0.8250588 0.162721261 0.0118958335 0 1 2 +11 0 0 0.21070785656836516 0.8100107 0.177851424 0.0127292387 0 1 2 +12 0 0 0.18598646367878061 0.830284834 0.160191745 0.00923373 0 1 2 +13 0 0 0.16375354954827126 0.8489512 0.1434604 0.00762989931 0 1 2 +14 0 0 0.1596759482253457 0.85242 0.136936992 0.009566072 0 1 2 +15 0 0 0.20845163822409238 0.8118403 0.169246256 0.0179626718 0 1 2 +16 0 0 0.21744498529134865 0.804571867 0.177335054 0.0164115764 0 1 2 +17 0 0 0.21691780548735229 0.804996133 0.181036428 0.0141520379 0 1 2 +18 0 0 0.23189900760647247 0.7930262 0.188312337 0.01704574 0 1 2 +19 0 0 0.21253030374720744 0.8085358 0.175618231 0.0147444345 0 1 2 +20 0 0 0.22016261069522264 0.8023883 0.184409738 0.0138513763 0 1 2 +21 0 0 0.24602820967441841 0.781900167 0.198100865 0.0185912009 0 1 2 +22 0 0 0.15894935190694653 0.853039563 0.137772977 0.008559331 0 1 2 +23 0 0 0.32356854476605679 0.72356236 0.2498953 0.026824316 0 1 2 +24 0 0 0.23726801509885151 0.788779855 0.196431652 0.01535066 0 1 2 +25 0 0 0.22838375008659054 0.7958188 0.190765321 0.0132612176 0 1 2 +26 0 0 0.27040443187140795 0.7630708 0.2173248 0.0200844 0 1 2 +27 0 0 0.19932606476395923 0.8192827 0.169289112 0.0120774228 0 1 2 +28 0 0 0.1951161099660853 0.8227391 0.166436434 0.0114225345 0 1 2 +29 0 0 0.21846443459392462 0.803752065 0.183887184 0.012927331 0 1 2 +30 0 0 0.22307894196650208 0.8000517 0.187162071 0.0130768539 0 1 2 +31 0 0 0.26075994599575425 0.770465851 0.2110865 0.0192779079 0 1 2 +32 0 0 0.1544188472754566 0.85691303 0.133014292 0.00901475549 0 1 2 +33 0 0 0.16400149087641372 0.848740757 0.1394019 0.01065959 0 1 2 +34 0 0 0.18917001968647357 0.8276458 0.16313988 0.009780681 0 1 2 +35 0 0 0.1867339144240599 0.829664469 0.160415888 0.0101680746 0 1 2 +36 0 0 0.18437912639336113 0.831620455 0.158098668 0.010674349 0 1 2 +37 0 0 0.18917001968647357 0.8276458 0.16313988 0.009780681 0 1 2 +38 0 0 0.20168448813751255 0.8173528 0.171194747 0.0108815078 0 1 2 +39 0 0 0.20291445140827119 0.8163481 0.172192335 0.0121164536 0 1 2 +40 0 0 0.20830326907859717 0.811960757 0.174680948 0.01324258 0 1 2 +41 0 0 0.27346553912976834 0.760738552 0.2233804 0.0145715475 0 1 2 +42 0 0 0.19325957932886312 0.824268 0.164984852 0.0106696784 0 1 2 +43 0 0 0.33929133391732746 0.7122749 0.2542582 0.0308294818 0 1 2 +44 0 0 0.28214916847960514 0.7541612 0.219979763 0.0237289015 0 1 2 +45 0 0 0.2386627953540944 0.787680447 0.19647029 0.0145918755 0 1 2 +46 0 0 0.19526536108078277 0.822616339 0.164269522 0.0124966912 0 1 2 +47 0 0 0.20167019513296086 0.817364454 0.171325445 0.011365016 0 1 2 +48 0 0 0.19209581314708737 0.8252278 0.162629187 0.0118732676 0 1 2 +49 0 0 0.19850422960374481 0.8199563 0.169211149 0.0113908406 0 1 2 +50 1 1 0.69338234853340464 0.49988243 0.389050663 0.119555935 1 2 0 +51 1 1 0.72199108756686314 0.485784054 0.4118773 0.112099662 1 2 0 +52 1 1 0.74430930313900279 0.4750623 0.441785038 0.08971132 1 2 0 +53 1 1 0.59005823682368141 0.554295 0.2805785 0.162364 1 2 0 +54 1 1 0.69733709261676402 0.497909427 0.408558458 0.09854807 1 2 0 +55 1 1 0.63344007763671317 0.5307628 0.321192145 0.143899232 1 2 0 +56 1 2 0.7989761924891351 0.4711786 0.449789226 0.0879787 2 1 0 +57 1 1 0.64427237002427129 0.525044441 0.31777966 0.149036676 1 0 2 +58 1 1 0.64040192259339201 0.527080536 0.333352447 0.139775947 1 2 0 +59 1 1 0.64377548959005892 0.5253054 0.3095969 0.159397885 1 2 0 +60 1 1 0.57999104003971469 0.5599034 0.278461277 0.158175588 1 0 2 +61 1 1 0.6948248484831383 0.499161869 0.3813758 0.12565276 1 2 0 +62 1 1 0.56028646073174748 0.571045458 0.237074688 0.18956089 1 0 2 +63 1 1 0.67210112759574603 0.510634542 0.377848744 0.1133353 1 2 0 +64 1 1 0.64373555012557371 0.5253264 0.2573457 0.216509834 1 2 0 +65 1 1 0.67477166014025181 0.5092727 0.3622924 0.135824978 1 2 0 +66 1 1 0.70602374663596845 0.493603 0.4035221 0.108713821 1 2 0 +67 1 1 0.59228502094529401 0.5530621 0.250644982 0.197752431 1 0 2 +68 1 1 0.65512757096894336 0.5193758 0.386527926 0.09174858 1 2 0 +69 1 1 0.58735455307568774 0.55579567 0.230519652 0.213071108 1 0 2 +70 1 2 0.93058674434711564 0.550455153 0.394322276 0.05753904 2 1 0 +71 1 1 0.63125026933312522 0.531926334 0.286607325 0.179237887 1 2 0 +72 1 1 0.69638812449595444 0.498382151 0.421951115 0.07993452 1 2 0 +73 1 1 0.61700920316728458 0.5395557 0.300539166 0.152734786 1 2 0 +74 1 1 0.63753910583806361 0.528591633 0.310740083 0.160509318 1 2 0 +75 1 1 0.66803145869993386 0.5127169 0.359449774 0.133419067 1 2 0 +76 1 1 0.67038997945667722 0.511509061 0.38495332 0.105532221 1 2 0 +77 1 2 0.86646706354671721 0.522770643 0.4204343 0.0589993745 2 1 0 +78 1 1 0.69894902677508997 0.497107476 0.402091861 0.106100589 1 2 0 +79 1 1 0.63938535737701618 0.5276166 0.304748774 0.163344815 1 0 2 +80 1 1 0.58540475782173829 0.5568804 0.235509977 0.206337124 1 0 2 +81 1 1 0.60117857370459193 0.5481652 0.2755529 0.173453778 1 0 2 +82 1 1 0.60782079529998934 0.544536233 0.245322734 0.2100347 1 2 0 +83 1 2 0.78062180375864654 0.4824288 0.458121061 0.0627425238 2 1 0 +84 1 1 0.70652252556196582 0.493356854 0.40282768 0.109011084 1 2 0 +85 1 2 0.79257614337943871 0.457337052 0.452677131 0.09964131 2 1 0 +86 1 1 0.72583855327706692 0.4839186 0.425667822 0.0992677361 1 2 0 +87 1 1 0.59537573771641028 0.551355362 0.308920562 0.134899408 1 2 0 +88 1 1 0.64943188422197029 0.522342443 0.29575032 0.179883242 1 2 0 +89 1 1 0.60563378034104909 0.545728445 0.2827467 0.1685501 1 2 0 +90 1 1 0.59600878868517615 0.551006436 0.277129084 0.166962817 1 2 0 +91 1 1 0.67441662757334309 0.509453535 0.372967064 0.121301547 1 2 0 +92 1 1 0.5972358031569589 0.550330758 0.2510595 0.197839946 1 2 0 +93 1 1 0.63248138330015757 0.5312719 0.3124712 0.148757339 1 0 2 +94 1 1 0.62468564680212546 0.5354297 0.298784733 0.1609534 1 2 0 +95 1 1 0.63119345940551341 0.531956553 0.26764822 0.197635174 1 2 0 +96 1 1 0.64067924426500766 0.5269344 0.301267326 0.169217929 1 2 0 +97 1 1 0.63826250522164152 0.5282094 0.310103238 0.160978049 1 2 0 +98 1 1 0.66502086899234869 0.5142628 0.320909441 0.156654671 1 0 2 +99 1 1 0.63226846487201493 0.531385 0.292993963 0.172219425 1 2 0 +100 2 2 0.19734066429015004 0.820910931 0.173172742 0.00623135036 2 1 0 +101 2 2 0.52497321090002491 0.5915712 0.372569382 0.035970252 2 1 0 +102 2 2 0.33682542074088945 0.7140335 0.270837575 0.0150250122 2 1 0 +103 2 2 0.51402730103197281 0.598082066 0.368052781 0.0337526053 2 1 0 +104 2 2 0.30625106310246369 0.736201763 0.250829279 0.0130713228 2 1 0 +105 2 2 0.28796328696649681 0.7497891 0.24044463 0.009776946 2 1 0 +106 2 2 0.76584105622308773 0.464942724 0.4625973 0.07062295 2 1 0 +107 2 2 0.44083626191557751 0.643498063 0.3342227 0.0220780186 2 1 0 +108 2 2 0.50692967491726626 0.6023421 0.3694929 0.0276594125 2 1 0 +109 2 2 0.18505322209501723 0.831060052 0.162578642 0.00599237438 2 1 0 +110 2 2 0.44600422919445565 0.640181065 0.3287335 0.0320880935 2 1 0 +111 2 2 0.5006392525442519 0.606143057 0.3610608 0.0317034647 2 1 0 +112 2 2 0.37022654689019097 0.690577865 0.2890131 0.0193106532 2 1 0 +113 2 2 0.48950833492155066 0.6129277 0.356904268 0.0301504768 2 1 0 +114 2 2 0.29002847774143492 0.748242259 0.238432437 0.0130701875 2 1 0 +115 2 2 0.30012884587440908 0.7407228 0.24480629 0.0149193872 2 1 0 +116 2 2 0.52116422112851724 0.5938288 0.369788945 0.0365619436 2 1 0 +117 2 2 0.22348013577680714 0.7997308 0.191225588 0.008526973 2 1 0 +118 2 2 0.22451993487847671 0.798899651 0.196335942 0.004770579 2 1 0 +119 2 1 0.86318800698872344 0.5035064 0.4218152 0.07180249 1 2 0 +120 2 2 0.27320051273701751 0.7609402 0.227141678 0.0113828648 2 1 0 +121 2 2 0.48522470399661904 0.615558863 0.351651281 0.0339448825 2 1 0 +122 2 2 0.32339079174986224 0.723691 0.2639643 0.01099197 2 1 0 +123 2 2 0.61061327999099768 0.543017745 0.4098112 0.04818538 2 1 0 +124 2 2 0.34376291022147898 0.709097 0.272929937 0.01831156 2 1 0 +125 2 2 0.45958937265042671 0.6315429 0.340537727 0.0280004926 2 1 0 +126 2 2 0.61809427424133345 0.5389706 0.410561621 0.05199326 2 1 0 +127 2 2 0.59453697331128474 0.551818 0.397813559 0.0517787151 2 1 0 +128 2 2 0.36829194004763732 0.691915154 0.289500117 0.0178627353 2 1 0 +129 2 2 0.60866420767228135 0.544077158 0.4106283 0.0451274961 2 1 0 +130 2 2 0.41374040089214881 0.661172569 0.3188539 0.0195883736 2 1 0 +131 2 2 0.30889684824646108 0.7342565 0.248563647 0.0160106979 2 1 0 +132 2 2 0.32850642798103019 0.7199983 0.2649906 0.0144322384 2 1 0 +133 2 1 0.80518215694201356 0.480504066 0.4470065 0.0766250044 1 2 0 +134 2 1 0.81224569131726831 0.487922341 0.443860173 0.0677389 1 2 0 +135 2 2 0.25395058596336034 0.775730133 0.215902731 0.008439373 2 1 0 +136 2 2 0.24196454887411839 0.785084 0.204565436 0.0102195553 2 1 0 +137 2 2 0.51757109151285696 0.595966339 0.367023945 0.03743413 2 1 0 +138 2 2 0.60743557219085698 0.544746041 0.402551055 0.0547405928 2 1 0 +139 2 2 0.37583538927032423 0.686715364 0.291458547 0.0209568329 2 1 0 +140 2 2 0.25110997119230455 0.7779368 0.212525666 0.009618061 2 1 0 +141 2 2 0.3173512582560189 0.728074968 0.255932242 0.0163358264 2 1 0 +142 2 2 0.52497321090002491 0.5915712 0.372569382 0.035970252 2 1 0 +143 2 2 0.26038009422309916 0.770758569 0.219255418 0.0100839045 2 1 0 +144 2 2 0.21253738081917511 0.8085301 0.184108466 0.00744177 2 1 0 +145 2 2 0.31378914947167852 0.7306731 0.2543345 0.0152301081 2 1 0 +146 2 2 0.54643963123708827 0.5790076 0.384474248 0.03620721 2 1 0 +147 2 2 0.44537223976850404 0.6405858 0.33086136 0.0288730673 2 1 0 +148 2 2 0.28679030786903981 0.7506691 0.234608829 0.0145600671 2 1 0 +149 2 2 0.57030806349714436 0.565351248 0.388658673 0.0463798642 2 1 0 diff --git a/test/BaselineOutput/Common/WeightedEnsembleMulticlass/WE-SDCA-Average-TrainTest-iris-out.txt b/test/BaselineOutput/Common/WeightedEnsembleMulticlass/WE-SDCA-Average-TrainTest-iris-out.txt new file mode 100644 index 0000000000..a6c93db8f3 --- /dev/null +++ b/test/BaselineOutput/Common/WeightedEnsembleMulticlass/WE-SDCA-Average-TrainTest-iris-out.txt @@ -0,0 +1,71 @@ +maml.exe TrainTest test=%Data% tr=WeightedEnsembleMulticlass{bp=SDCAMC{nt=1} nm=5 oc=MultiAverage tp=-} dout=%Output% data=%Data% out=%Output% seed=1 xf=Term{col=Label} +Automatically adding a MinMax normalization transform, use 'norm=Warn' or 'norm=No' to turn this behavior off. +Training 5 learners for the batch 1 +Beginning training model 1 of 5 +Using 1 thread to train. +Automatically choosing a check frequency of 1. +Auto-tuning parameters: maxIterations = 10563. +Auto-tuning parameters: L2 = 2.667015E-05. +Auto-tuning parameters: L1Threshold (L1/L2) = 0. +Using best model from iteration 958. +Trainer 1 of 5 finished in %Time% +Beginning training model 2 of 5 +Using 1 thread to train. +Automatically choosing a check frequency of 1. +Auto-tuning parameters: maxIterations = 8928. +Auto-tuning parameters: L2 = 2.666837E-05. +Auto-tuning parameters: L1Threshold (L1/L2) = 0. +Using best model from iteration 874. +Trainer 2 of 5 finished in %Time% +Beginning training model 3 of 5 +Using 1 thread to train. +Automatically choosing a check frequency of 1. +Auto-tuning parameters: maxIterations = 9201. +Auto-tuning parameters: L2 = 2.667378E-05. +Auto-tuning parameters: L1Threshold (L1/L2) = 0. +Using best model from iteration 754. +Trainer 3 of 5 finished in %Time% +Beginning training model 4 of 5 +Using 1 thread to train. +Automatically choosing a check frequency of 1. +Auto-tuning parameters: maxIterations = 10344. +Auto-tuning parameters: L2 = 2.66688E-05. +Auto-tuning parameters: L1Threshold (L1/L2) = 0. +Using best model from iteration 976. +Trainer 4 of 5 finished in %Time% +Beginning training model 5 of 5 +Using 1 thread to train. +Automatically choosing a check frequency of 1. +Auto-tuning parameters: maxIterations = 9315. +Auto-tuning parameters: L2 = 2.66746E-05. +Auto-tuning parameters: L1Threshold (L1/L2) = 0. +Using best model from iteration 1058. +Trainer 5 of 5 finished in %Time% +Not training a calibrator because it is not needed. + +Confusion table + ||======================== +PREDICTED || 0 | 1 | 2 | Recall +TRUTH ||======================== + 0 || 50 | 0 | 0 | 1.0000 + 1 || 0 | 48 | 2 | 0.9600 + 2 || 0 | 1 | 49 | 0.9800 + ||======================== +Precision ||1.0000 |0.9796 |0.9608 | +Accuracy(micro-avg): 0.980000 +Accuracy(macro-avg): 0.980000 +Log-loss: 0.061647 +Log-loss reduction: 0.943887 + +OVERALL RESULTS +--------------------------------------- +Accuracy(micro-avg): 0.980000 (0.0000) +Accuracy(macro-avg): 0.980000 (0.0000) +Log-loss: 0.061647 (0.0000) +Log-loss reduction: 0.943887 (0.0000) + +--------------------------------------- +Physical memory usage(MB): %Number% +Virtual memory usage(MB): %Number% +%DateTime% Time elapsed(s): %Number% + diff --git a/test/BaselineOutput/Common/WeightedEnsembleMulticlass/WE-SDCA-Average-TrainTest-iris-rp.txt b/test/BaselineOutput/Common/WeightedEnsembleMulticlass/WE-SDCA-Average-TrainTest-iris-rp.txt new file mode 100644 index 0000000000..0992b489d5 --- /dev/null +++ b/test/BaselineOutput/Common/WeightedEnsembleMulticlass/WE-SDCA-Average-TrainTest-iris-rp.txt @@ -0,0 +1,4 @@ +WeightedEnsembleMulticlass +Accuracy(micro-avg) Accuracy(macro-avg) Log-loss Log-loss reduction /oc /bp /nm Learner Name Train Dataset Test Dataset Results File Run Time Physical Memory Virtual Memory Command Line Settings +0.98 0.98 0.061647 0.943887 MultiAverage SDCAMC{nt=1} 5 WeightedEnsembleMulticlass %Data% %Data% %Output% 99 0 0 maml.exe TrainTest test=%Data% tr=WeightedEnsembleMulticlass{bp=SDCAMC{nt=1} nm=5 oc=MultiAverage tp=-} dout=%Output% data=%Data% out=%Output% seed=1 xf=Term{col=Label} /oc:MultiAverage;/bp:SDCAMC{nt=1};/nm:5 + diff --git a/test/BaselineOutput/Common/WeightedEnsembleMulticlass/WE-SDCA-Average-TrainTest-iris.txt b/test/BaselineOutput/Common/WeightedEnsembleMulticlass/WE-SDCA-Average-TrainTest-iris.txt new file mode 100644 index 0000000000..ba4318c575 --- /dev/null +++ b/test/BaselineOutput/Common/WeightedEnsembleMulticlass/WE-SDCA-Average-TrainTest-iris.txt @@ -0,0 +1,151 @@ +Instance Label Assigned Log-loss #1 Score #2 Score #3 Score #1 Class #2 Class #3 Class +0 0 0 0.0021632449196932351 0.9978391 0.002160947 6.04765346E-15 0 1 2 +1 0 0 0.0079895884500299184 0.992042243 0.007957775 9.68857156E-14 0 1 2 +2 0 0 0.0034284286585717972 0.996577442 0.003422501 2.36189361E-14 0 1 2 +3 0 0 0.005962289210830086 0.99405545 0.005944559 1.11923078E-13 0 1 2 +4 0 0 0.0015346853281102198 0.9984665 0.00153351948 3.82847542E-15 0 1 2 +5 0 0 0.0035573859226362497 0.996448934 0.00355116441 5.743688E-14 0 1 2 +6 0 0 0.0034875220381507346 0.996518552 0.00348154618 6.53355747E-14 0 1 2 +7 0 0 0.00316841143066658 0.9968366 0.00316347065 1.73878618E-14 0 1 2 +8 0 0 0.0080876483364587699 0.991944969 0.008055024 2.330561E-13 0 1 2 +9 0 0 0.0044032577016354458 0.9956064 0.004393641 2.00896513E-14 0 1 2 +10 0 0 0.0017300303383470188 0.998271465 0.00172853586 2.84920941E-15 0 1 2 +11 0 0 0.0033182659118411637 0.996687233 0.00331283966 3.18519252E-14 0 1 2 +12 0 0 0.0047188101464650705 0.9952923 0.00470773876 2.27861671E-14 0 1 2 +13 0 0 0.0022097785731952629 0.997792661 0.00220742333 7.561946E-15 0 1 2 +14 0 0 0.00061662130811644331 0.999383569 0.000616459 1.30595712E-16 0 1 2 +15 0 0 0.00080075129044645252 0.999199569 0.0008004363 1.77890027E-15 0 1 2 +16 0 0 0.0019398060233437617 0.9980621 0.00193793536 9.545111E-15 0 1 2 +17 0 0 0.0035064829009267837 0.996499658 0.00350032561 2.81692919E-14 0 1 2 +18 0 0 0.0035088754643611184 0.9964973 0.003502793 1.68672178E-14 0 1 2 +19 0 0 0.0017625716470783322 0.998239 0.00176108012 9.636584E-15 0 1 2 +20 0 0 0.0055482836599041047 0.9944671 0.00553299161 3.46845457E-14 0 1 2 +21 0 0 0.0038022471215707378 0.996205 0.00379505358 7.55707549E-14 0 1 2 +22 0 0 0.00065187000241988691 0.999348342 0.0006516743 7.705539E-16 0 1 2 +23 0 0 0.025671561453119392 0.974655151 0.0253448915 6.881716E-12 0 1 2 +24 0 0 0.0054063646497523711 0.9946082 0.00539183 1.33688159E-13 0 1 2 +25 0 0 0.011664505897147102 0.988403261 0.0115968315 2.35799282E-13 0 1 2 +26 0 0 0.0096848780603521201 0.990361869 0.009638118 5.94649438E-13 0 1 2 +27 0 0 0.0026899363724517031 0.9973137 0.00268630125 9.070273E-15 0 1 2 +28 0 0 0.0030583968972770301 0.9969463 0.00305377436 9.592416E-15 0 1 2 +29 0 0 0.0055600911742843106 0.994455338 0.005544726 9.881408E-14 0 1 2 +30 0 0 0.0078139218642764643 0.9922165 0.00778349629 1.575795E-13 0 1 2 +31 0 0 0.010642996060550395 0.98941344 0.0105865467 2.91241E-13 0 1 2 +32 0 0 0.00030426839186627181 0.9996958 0.0003043345 8.896724E-17 0 1 2 +33 0 0 0.00038851893370880301 0.999611557 0.000388382963 1.3963084E-16 0 1 2 +34 0 0 0.0044032577016354458 0.9956064 0.004393641 2.00896513E-14 0 1 2 +35 0 0 0.0034938622235497398 0.996512234 0.00348785683 1.21640614E-14 0 1 2 +36 0 0 0.0023991612209818066 0.9976037 0.00239626318 3.001639E-15 0 1 2 +37 0 0 0.0044032577016354458 0.9956064 0.004393641 2.00896513E-14 0 1 2 +38 0 0 0.0051597923110851384 0.9948535 0.00514655141 8.41418542E-14 0 1 2 +39 0 0 0.0033633582075250873 0.9966423 0.00335767167 1.631569E-14 0 1 2 +40 0 0 0.0028259114314285834 0.9971781 0.002821923 1.88525885E-14 0 1 2 +41 0 0 0.063285225115162325 0.9386757 0.0613243356 1.506976E-11 0 1 2 +42 0 0 0.002919819727088034 0.997084439 0.00291556516 2.894814E-14 0 1 2 +43 0 0 0.019364377670876787 0.9808219 0.0191781074 7.75738848E-12 0 1 2 +44 0 0 0.0053491951991256319 0.9946651 0.00533493049 2.87338268E-13 0 1 2 +45 0 0 0.012085757567287364 0.987987 0.0120130256 4.707762E-13 0 1 2 +46 0 0 0.0012669837540169471 0.9987338 0.00126628845 3.24697375E-15 0 1 2 +47 0 0 0.0038111620857659971 0.9961961 0.003803882 4.06568869E-14 0 1 2 +48 0 0 0.001623397800359935 0.9983779 0.001622087 3.02381826E-15 0 1 2 +49 0 0 0.0035964472251083443 0.99641 0.00359009276 1.83884127E-14 0 1 2 +50 1 1 0.0074352983065214123 0.9925923 0.00625397358 0.00115381461 1 2 0 +51 1 1 0.019926841553419904 0.9802704 0.0183767434 0.00135288027 1 2 0 +52 1 1 0.047162589813160904 0.9539323 0.0456770733 0.00039064986 1 2 0 +53 1 1 0.017346598963771529 0.982803 0.0161944963 0.00100264873 1 2 0 +54 1 1 0.061816310629283465 0.940055549 0.05961532 0.0003291377 1 2 0 +55 1 1 0.020161695297233093 0.9800402 0.0183108151 0.00164904189 1 2 0 +56 1 1 0.088502278834805548 0.915301 0.08389361 0.0008053317 1 2 0 +57 1 1 0.025073464908208595 0.975238264 0.0246359184 0.000125828636 1 0 2 +58 1 1 0.0068805350198553328 0.9931431 0.005733231 0.001123723 1 2 0 +59 1 1 0.020703796500439534 0.979509056 0.0177517869 0.00273919129 1 2 0 +60 1 1 0.0063936823423325385 0.9936267 0.005712928 0.0006604138 1 0 2 +61 1 1 0.023987437333748099 0.976298 0.0220667683 0.00163530058 1 2 0 +62 1 1 0.0030460808061124503 0.996958554 0.002483546 0.000557865 1 0 2 +63 1 1 0.04295391208143521 0.957955539 0.04128821 0.000756256166 1 2 0 +64 1 1 0.010794697941462671 0.989263356 0.009997835 0.00073887076 1 0 2 +65 1 1 0.0059882526983131412 0.994029641 0.004309889 0.00166049378 1 2 0 +66 1 1 0.095863410267416085 0.9085881 0.09029834 0.00111357961 1 2 0 +67 1 1 0.0097239387018826836 0.9903232 0.009428238 0.0002486164 1 0 2 +68 1 1 0.27279218212922079 0.761251 0.238682672 6.630487E-05 1 2 0 +69 1 1 0.0059322491577234893 0.9940853 0.00514177745 0.000772929343 1 0 2 +70 1 2 1.1896056525031624 0.695575953 0.304341257 8.282701E-05 2 1 0 +71 1 1 0.0048213412146064158 0.995190263 0.002954562 0.00185527839 1 0 2 +72 1 1 0.38634745701996759 0.6795344 0.320397973 6.767444E-05 1 2 0 +73 1 1 0.0091309108768176956 0.990910649 0.0075464 0.00154302432 1 2 0 +74 1 1 0.0047250383693937907 0.9952861 0.00266963476 0.0020443208 1 2 0 +75 1 1 0.0075564854894616769 0.992472 0.00622363 0.00130430458 1 2 0 +76 1 1 0.031580373583293263 0.9689131 0.0307525937 0.0003343052 1 2 0 +77 1 1 0.59788249639030699 0.549975 0.449962437 6.25779649E-05 1 2 0 +78 1 1 0.067534809015276351 0.9346952 0.064628914 0.0006759624 1 2 0 +79 1 1 0.019504705675815134 0.9806843 0.0192696638 4.6089157E-05 1 0 2 +80 1 1 0.005679972309884455 0.9943361 0.004838101 0.0008257646 1 0 2 +81 1 1 0.0094255757542048031 0.9906187 0.009168978 0.000212341853 1 0 2 +82 1 1 0.005895015134779971 0.9941223 0.00495022 0.0009275159 1 0 2 +83 1 2 1.090327178267041 0.6638631 0.3361065 3.0397423E-05 2 1 0 +84 1 1 0.13002584293861474 0.878072739 0.120709874 0.00121744443 1 2 0 +85 1 1 0.057945427629684881 0.943701446 0.0544599853 0.00183855835 1 2 0 +86 1 1 0.031861115147701145 0.9686411 0.0307453778 0.000613590644 1 2 0 +87 1 1 0.02230187829020458 0.97794497 0.021722788 0.000332317257 1 2 0 +88 1 1 0.0090997529464692684 0.9909415 0.005918171 0.00314031658 1 0 2 +89 1 1 0.011372496388815115 0.9886919 0.009535785 0.00177237613 1 2 0 +90 1 1 0.013179765508661938 0.9869067 0.011053375 0.00203985069 1 2 0 +91 1 1 0.024377450545832371 0.9759173 0.0228784 0.00120444165 1 2 0 +92 1 1 0.0049148980085899439 0.99509716 0.00318418047 0.001718707 1 0 2 +93 1 1 0.017874250993087683 0.982284546 0.0175741836 0.00014131474 1 0 2 +94 1 1 0.011933980359046322 0.988136947 0.0097251 0.00213795877 1 2 0 +95 1 1 0.0090239072937195672 0.9910167 0.00761275832 0.001370523 1 0 2 +96 1 1 0.0085386553356135136 0.9914977 0.004940027 0.00356230582 1 2 0 +97 1 1 0.0057169585410897228 0.994299352 0.00342920539 0.002271554 1 2 0 +98 1 1 0.028929001566074013 0.971485436 0.0284460969 6.846197E-05 1 0 2 +99 1 1 0.0077067587745058546 0.992322862 0.00453239447 0.00314473221 1 2 0 +100 2 2 1.3530345898730908E-05 0.99998647 1.36084482E-05 1.91686372E-11 2 1 0 +101 2 2 0.018773745222579363 0.9814014 0.01859814 4.897308E-07 2 1 0 +102 2 2 0.0017586905179432194 0.998242855 0.001757161 5.81769166E-09 2 1 0 +103 2 2 0.037805422958546393 0.9629003 0.037098866 8.773631E-07 2 1 0 +104 2 2 0.0003672916597854794 0.9996328 0.0003671986 1.28443267E-09 2 1 0 +105 2 2 0.00044087351856321493 0.9995592 0.0004407741 3.434822E-10 2 1 0 +106 2 2 0.19962941481976826 0.8190342 0.180940539 2.53322069E-05 2 1 0 +107 2 2 0.018685077196642596 0.9814884 0.0185114164 7.66340449E-08 2 1 0 +108 2 2 0.018027053928997613 0.982134461 0.0178654175 7.345061E-08 2 1 0 +109 2 2 6.8249647222011489E-05 0.999931753 6.83225E-05 1.07742329E-10 2 1 0 +110 2 2 0.044449695375517594 0.9565237 0.04347425 1.96782162E-06 2 1 0 +111 2 2 0.020430427852599752 0.979776859 0.0202228948 2.9133534E-07 2 1 0 +112 2 2 0.0040348400172863768 0.9959733 0.00402667141 2.98350074E-08 2 1 0 +113 2 2 0.0048606316061821486 0.995151162 0.004848757 5.78924144E-08 2 1 0 +114 2 2 0.0001059230634154853 0.9998941 0.000105975509 3.334062E-10 2 1 0 +115 2 2 0.00090628194074056107 0.9990941 0.000905906 7.23405069E-09 2 1 0 +116 2 2 0.071237455712918507 0.931240737 0.06875683 2.42950887E-06 2 1 0 +117 2 2 0.00072457808662163502 0.9992757 0.0007242736 2.55426613E-09 2 1 0 +118 2 2 1.0311656711091902E-05 0.9999897 1.03620578E-05 6.121642E-13 2 1 0 +119 2 2 0.45803878925846125 0.632522941 0.367462963 1.41449873E-05 2 1 0 +120 2 2 0.00047653344972066648 0.9995236 0.000476537127 1.56395541E-09 2 1 0 +121 2 2 0.01087598057789437 0.989182949 0.0108167455 3.65963444E-07 2 1 0 +122 2 2 0.00085074124474289903 0.9991496 0.0008504184 4.51549742E-10 2 1 0 +123 2 2 0.15909036606310079 0.8529193 0.1470739 6.811021E-06 2 1 0 +124 2 2 0.0037817848868678329 0.996225357 0.00377455354 4.96917E-08 2 1 0 +125 2 2 0.061988978574891308 0.939893246 0.0601057075 1.08795427E-06 2 1 0 +126 2 2 0.22758963028939758 0.796451032 0.203533515 1.55026974E-05 2 1 0 +127 2 2 0.23148908836684481 0.793351352 0.206623986 2.46640611E-05 2 1 0 +128 2 2 0.001149420649647469 0.99885124 0.00114880712 5.37582645E-09 2 1 0 +129 2 2 0.46118055335464953 0.6305388 0.3694485 1.26931409E-05 2 1 0 +130 2 2 0.0092526646852335252 0.99079 0.00920999 2.54842725E-08 2 1 0 +131 2 2 0.022876730419094909 0.977382958 0.0226167273 2.672297E-07 2 1 0 +132 2 2 0.00037820337633663733 0.999621868 0.000378114724 1.11644249E-09 2 1 0 +133 2 1 1.1971732881189494 0.6978348 0.3020468 0.000118445467 1 2 0 +134 2 2 0.50242571294052596 0.6050612 0.394913048 2.58076016E-05 2 1 0 +135 2 2 0.00027213222801581404 0.9997279 0.0002721111 1.560498E-10 2 1 0 +136 2 2 0.00018091646128941553 0.9998191 0.000180994539 9.867085E-10 2 1 0 +137 2 2 0.077497686512126934 0.925429165 0.074567236 3.68499036E-06 2 1 0 +138 2 2 0.27141367354335955 0.7623011 0.237663791 3.51991E-05 2 1 0 +139 2 2 0.0082592762789907258 0.991774738 0.008225129 8.48062243E-08 2 1 0 +140 2 2 0.00013715969358890206 0.99986285 0.000137181021 2.77440931E-10 2 1 0 +141 2 2 0.0029276507980937399 0.997076631 0.00292336266 1.64982072E-08 2 1 0 +142 2 2 0.018773745222579363 0.9814014 0.01859814 4.897308E-07 2 1 0 +143 2 2 0.00021412280668525778 0.9997859 0.000214120941 5.683524E-10 2 1 0 +144 2 2 5.9427596625705856E-05 0.9999406 5.93945733E-05 1.04773613E-10 2 1 0 +145 2 2 0.0011606392732310788 0.998840034 0.00115994 5.18025045E-09 2 1 0 +146 2 2 0.027995750014108291 0.9723925 0.0276071317 3.96819644E-07 2 1 0 +147 2 2 0.020005038927645435 0.980193734 0.0198057871 4.547713E-07 2 1 0 +148 2 2 0.00086076339787730691 0.9991396 0.0008603865 1.10151221E-08 2 1 0 +149 2 2 0.10889557677394049 0.896824062 0.103166319 9.609822E-06 2 1 0 diff --git a/test/BaselineOutput/Common/WeightedEnsembleMulticlass/WE-Stacking-TrainTest-iris-out.txt b/test/BaselineOutput/Common/WeightedEnsembleMulticlass/WE-Stacking-TrainTest-iris-out.txt new file mode 100644 index 0000000000..99e556e559 --- /dev/null +++ b/test/BaselineOutput/Common/WeightedEnsembleMulticlass/WE-Stacking-TrainTest-iris-out.txt @@ -0,0 +1,67 @@ +maml.exe TrainTest test=%Data% tr=WeightedEnsembleMulticlass{bp=mlr{t-} nm=5 oc=MultiStacking{bp=mlr{t-}} tp=-} dout=%Output% data=%Data% out=%Output% seed=1 xf=Term{col=Label} +Automatically adding a MinMax normalization transform, use 'norm=Warn' or 'norm=No' to turn this behavior off. +Training 5 learners for the batch 1 +Beginning training model 1 of 5 +Beginning optimization +num vars: 15 +improvement criterion: Mean Improvement +L1 regularization selected 10 of 15 weights. +Trainer 1 of 5 finished in %Time% +Beginning training model 2 of 5 +Beginning optimization +num vars: 15 +improvement criterion: Mean Improvement +L1 regularization selected 11 of 15 weights. +Trainer 2 of 5 finished in %Time% +Beginning training model 3 of 5 +Beginning optimization +num vars: 15 +improvement criterion: Mean Improvement +L1 regularization selected 11 of 15 weights. +Trainer 3 of 5 finished in %Time% +Beginning training model 4 of 5 +Beginning optimization +num vars: 15 +improvement criterion: Mean Improvement +L1 regularization selected 11 of 15 weights. +Trainer 4 of 5 finished in %Time% +Beginning training model 5 of 5 +Beginning optimization +num vars: 15 +improvement criterion: Mean Improvement +L1 regularization selected 11 of 15 weights. +Trainer 5 of 5 finished in %Time% +The number of instances used for stacking trainer is 43 +Warning: The trainer specified for stacking wants normalization, but we do not currently allow this. +Beginning optimization +num vars: 48 +improvement criterion: Mean Improvement +L1 regularization selected 26 of 48 weights. +Not training a calibrator because it is not needed. + +Confusion table + ||======================== +PREDICTED || 0 | 1 | 2 | Recall +TRUTH ||======================== + 0 || 50 | 0 | 0 | 1.0000 + 1 || 0 | 35 | 15 | 0.7000 + 2 || 0 | 0 | 50 | 1.0000 + ||======================== +Precision ||1.0000 |1.0000 |0.7692 | +Accuracy(micro-avg): 0.900000 +Accuracy(macro-avg): 0.900000 +Log-loss: 0.431088 +Log-loss reduction: 0.607607 + +OVERALL RESULTS +--------------------------------------- +Accuracy(micro-avg): 0.900000 (0.0000) +Accuracy(macro-avg): 0.900000 (0.0000) +Log-loss: 0.431088 (0.0000) +Log-loss reduction: 0.607607 (0.0000) + +--------------------------------------- +Physical memory usage(MB): %Number% +Virtual memory usage(MB): %Number% +%DateTime% Time elapsed(s): %Number% + diff --git a/test/BaselineOutput/Common/WeightedEnsembleMulticlass/WE-Stacking-TrainTest-iris-rp.txt b/test/BaselineOutput/Common/WeightedEnsembleMulticlass/WE-Stacking-TrainTest-iris-rp.txt new file mode 100644 index 0000000000..4613357145 --- /dev/null +++ b/test/BaselineOutput/Common/WeightedEnsembleMulticlass/WE-Stacking-TrainTest-iris-rp.txt @@ -0,0 +1,4 @@ +WeightedEnsembleMulticlass +Accuracy(micro-avg) Accuracy(macro-avg) Log-loss Log-loss reduction /oc /bp /nm Learner Name Train Dataset Test Dataset Results File Run Time Physical Memory Virtual Memory Command Line Settings +0.9 0.9 0.431088 0.607607 MultiStacking{bp=mlr{t-}} mlr{t-} 5 WeightedEnsembleMulticlass %Data% %Data% %Output% 99 0 0 maml.exe TrainTest test=%Data% tr=WeightedEnsembleMulticlass{bp=mlr{t-} nm=5 oc=MultiStacking{bp=mlr{t-}} tp=-} dout=%Output% data=%Data% out=%Output% seed=1 xf=Term{col=Label} /oc:MultiStacking{bp=mlr{t-}};/bp:mlr{t-};/nm:5 + diff --git a/test/BaselineOutput/Common/WeightedEnsembleMulticlass/WE-Stacking-TrainTest-iris.txt b/test/BaselineOutput/Common/WeightedEnsembleMulticlass/WE-Stacking-TrainTest-iris.txt new file mode 100644 index 0000000000..f835cbe99a --- /dev/null +++ b/test/BaselineOutput/Common/WeightedEnsembleMulticlass/WE-Stacking-TrainTest-iris.txt @@ -0,0 +1,151 @@ +Instance Label Assigned Log-loss #1 Score #2 Score #3 Score #1 Class #2 Class #3 Class +0 0 0 0.18415280900555114 0.8318087 0.122028828 0.04616249 0 1 2 +1 0 0 0.20102714840054076 0.8178902 0.133012265 0.0490975529 0 1 2 +2 0 0 0.18796467611284964 0.828644 0.124647491 0.04670851 0 1 2 +3 0 0 0.20338126432029827 0.8159671 0.1344232 0.049609866 0 1 2 +4 0 0 0.18098708897265092 0.834446132 0.119963579 0.04559037 0 1 2 +5 0 0 0.23036577174987058 0.794243038 0.149871811 0.05588504 0 1 2 +6 0 0 0.20561720031530076 0.8141447 0.135494858 0.0503605045 0 1 2 +7 0 0 0.19297622790066812 0.8245016 0.1276637 0.0478347428 0 1 2 +8 0 0 0.2043132777987306 0.815206945 0.135164246 0.0496288426 0 1 2 +9 0 0 0.18542315686855285 0.8307527 0.123143822 0.04610339 0 1 2 +10 0 0 0.18352959017339793 0.832327247 0.121492989 0.04617967 0 1 2 +11 0 0 0.19883728924380173 0.819683254 0.131343767 0.0489730351 0 1 2 +12 0 0 0.18316006876170685 0.832634866 0.121760875 0.04560431 0 1 2 +13 0 0 0.16823361360534603 0.8451564 0.112175122 0.042668514 0 1 2 +14 0 0 0.16139433859086047 0.85095644 0.10722363 0.04181991 0 1 2 +15 0 0 0.19664742627957968 0.8214802 0.1288998 0.04962003 0 1 2 +16 0 0 0.20144036833910339 0.8175523 0.132341027 0.0501066744 0 1 2 +17 0 0 0.20235203854964523 0.8168073 0.13336353 0.04982926 0 1 2 +18 0 0 0.21159288710344162 0.8092941 0.1387468 0.0519590341 0 1 2 +19 0 0 0.19780183097739662 0.820532441 0.130283177 0.04918442 0 1 2 +20 0 0 0.20582082185802125 0.8139789 0.135669008 0.05035215 0 1 2 +21 0 0 0.22309712037386253 0.800037146 0.145771846 0.0541910455 0 1 2 +22 0 0 0.16204596280473024 0.8504021 0.107862428 0.041735407 0 1 2 +23 0 0 0.29268506479820222 0.7462571 0.186943442 0.0667995 0 1 2 +24 0 0 0.21913719974218848 0.8032115 0.143906251 0.05288234 0 1 2 +25 0 0 0.21445342255707586 0.8069824 0.14138031 0.0516373 0 1 2 +26 0 0 0.24519634734267021 0.7825509 0.159402087 0.0580471344 0 1 2 +27 0 0 0.1897826406773841 0.8271389 0.125577465 0.04728355 0 1 2 +28 0 0 0.18742635049526854 0.8290902 0.124160558 0.0467491522 0 1 2 +29 0 0 0.20619031667785392 0.8136782 0.1360931 0.05022871 0 1 2 +30 0 0 0.21020171979646474 0.810420752 0.138666078 0.05091307 0 1 2 +31 0 0 0.23710110455779138 0.7889115 0.154546723 0.0565416254 0 1 2 +32 0 0 0.15831901442292842 0.853577435 0.105257191 0.0411653221 0 1 2 +33 0 0 0.16479228027307619 0.848069847 0.109299093 0.0426310822 0 1 2 +34 0 0 0.18542315686855285 0.8307527 0.123143822 0.04610339 0 1 2 +35 0 0 0.18277143462328696 0.8329585 0.121341564 0.0456998944 0 1 2 +36 0 0 0.17921590402125057 0.8359254 0.118886366 0.0451882444 0 1 2 +37 0 0 0.18542315686855285 0.8307527 0.123143822 0.04610339 0 1 2 +38 0 0 0.19447661109122485 0.823265433 0.128905743 0.0478287227 0 1 2 +39 0 0 0.19306399391162335 0.8244292 0.127715334 0.0478554368 0 1 2 +40 0 0 0.19601224102179418 0.8220022 0.1294205 0.04857734 0 1 2 +41 0 0 0.24689959897614386 0.7812191 0.1618791 0.0569017567 0 1 2 +42 0 0 0.18772100580389944 0.8288459 0.124502547 0.046651572 0 1 2 +43 0 0 0.30445789780903659 0.7375231 0.192890182 0.06958673 0 1 2 +44 0 0 0.25219586694807866 0.7770925 0.162933454 0.05997394 0 1 2 +45 0 0 0.22212980289299991 0.8008114 0.146040261 0.053148333 0 1 2 +46 0 0 0.185578072285153 0.830624 0.122723959 0.0466520041 0 1 2 +47 0 0 0.19361498470917726 0.8239751 0.128220156 0.0478047132 0 1 2 +48 0 0 0.18344831382097285 0.8323949 0.121445 0.04616007 0 1 2 +49 0 0 0.19055630097242918 0.8264992 0.126210138 0.04729072 0 1 2 +50 1 2 0.83967341638932402 0.453740239 0.431851536 0.114408143 2 1 0 +51 1 2 0.87007186866802178 0.47155267 0.418921441 0.109525941 2 1 0 +52 1 2 0.93595458385506314 0.512233853 0.3922113 0.09555494 2 1 0 +53 1 1 0.64753283943206463 0.523335338 0.325773716 0.150891036 1 2 0 +54 1 2 0.85554302352325062 0.471195281 0.425052315 0.103752449 2 1 0 +55 1 1 0.72188042025636456 0.485837817 0.379738 0.13442409 1 2 0 +56 1 2 0.989554144296801 0.5361127 0.3717424 0.0921449438 2 1 0 +57 1 1 0.66696877774171925 0.513262033 0.284242749 0.202495322 1 0 2 +58 1 1 0.74505962631018863 0.474706 0.395164371 0.1301296 1 2 0 +59 1 1 0.69966576355033383 0.4967513 0.357132971 0.146115839 1 2 0 +60 1 1 0.6123037804599436 0.542100549 0.245147958 0.212751508 1 0 2 +61 1 1 0.80614127753024245 0.446577966 0.4334349 0.119987205 1 2 0 +62 1 1 0.60309599210698117 0.547115147 0.243904024 0.208980843 1 2 0 +63 1 1 0.80956937760990522 0.445049673 0.4417687 0.113181517 1 2 0 +64 1 1 0.6741400843986759 0.50959444 0.302481949 0.18792364 1 2 0 +65 1 1 0.78863909355727335 0.454462856 0.4195242 0.126012921 1 2 0 +66 1 2 0.84764720461917853 0.462445617 0.428421736 0.109132729 2 1 0 +67 1 1 0.63812653884947024 0.5282812 0.253122 0.218596786 1 2 0 +68 1 1 0.79090347252250515 0.453434944 0.442657977 0.103906989 1 2 0 +69 1 1 0.62701005979117386 0.5341866 0.263382256 0.202431172 1 2 0 +70 1 2 1.1798051256107349 0.62038517 0.307338625 0.07227611 2 1 0 +71 1 1 0.68460203697015498 0.5042909 0.336482972 0.1592262 1 2 0 +72 1 2 0.88051846358244901 0.4912943 0.414567918 0.09413785 2 1 0 +73 1 1 0.70055713830919164 0.4963087 0.363413721 0.14027743 1 2 0 +74 1 1 0.71465673271688934 0.489360064 0.366220564 0.144419387 1 2 0 +75 1 1 0.78054263693188641 0.458157331 0.416729122 0.125113443 1 2 0 +76 1 2 0.82228410689806763 0.452240318 0.4394268 0.108332776 2 1 0 +77 1 2 1.1197906877396573 0.598423362 0.3263481 0.07522855 2 1 0 +78 1 2 0.8431873213947364 0.461574554 0.4303367 0.108088814 2 1 0 +79 1 1 0.66610410620409988 0.513706 0.2710436 0.215250343 1 0 2 +80 1 1 0.62111694388966021 0.5373439 0.255637079 0.207019046 1 2 0 +81 1 1 0.63259829457181638 0.531209767 0.242757887 0.226032332 1 0 2 +82 1 1 0.65074261480868734 0.521658242 0.293950737 0.184390992 1 2 0 +83 1 2 1.0149089438958341 0.556178451 0.36243543 0.08138615 2 1 0 +84 1 2 0.84598172251401038 0.461517155 0.429135859 0.109347083 2 1 0 +85 1 2 0.95653150852469715 0.5168891 0.384223253 0.09888767 2 1 0 +86 1 2 0.89816130109549719 0.490823537 0.4073179 0.101858482 2 1 0 +87 1 1 0.6813039323183222 0.5059568 0.362298757 0.131744385 1 2 0 +88 1 1 0.7025473051049741 0.495321959 0.346677929 0.158 1 2 0 +89 1 1 0.66108051966920456 0.516293168 0.329245716 0.154461071 1 2 0 +90 1 1 0.66192375572424977 0.515858 0.331417829 0.1527241 1 2 0 +91 1 1 0.80305495270444083 0.44795838 0.434375823 0.117665656 1 2 0 +92 1 1 0.6457812073341398 0.524252832 0.300425649 0.175321534 1 2 0 +93 1 1 0.65719553986028489 0.518302858 0.2789565 0.202740625 1 0 2 +94 1 1 0.68844697352715922 0.502355635 0.3502431 0.147401363 1 2 0 +95 1 1 0.68057686373859072 0.5063248 0.321670741 0.1720045 1 2 0 +96 1 1 0.70294262405003094 0.4951262 0.354318172 0.150555566 1 2 0 +97 1 1 0.71354414976745772 0.489904821 0.3654074 0.144687757 1 2 0 +98 1 1 0.67999890703300703 0.506617546 0.2867733 0.206609145 1 0 2 +99 1 1 0.68892228090302166 0.5021169 0.3436669 0.154216141 1 2 0 +100 2 2 0.16401483412401432 0.848729432 0.119427159 0.0318435058 2 1 0 +101 2 2 0.40604648196440402 0.6662792 0.272454619 0.0612662435 2 1 0 +102 2 2 0.24548026249257032 0.7823287 0.175214678 0.04245658 2 1 0 +103 2 2 0.38447310103010285 0.680809259 0.2602182 0.0589724928 2 1 0 +104 2 2 0.22861670768509171 0.795633435 0.164019048 0.04034753 2 1 0 +105 2 2 0.20682027400821598 0.8131658 0.149455428 0.0373786725 2 1 0 +106 2 2 0.64271581882801387 0.525862336 0.386756778 0.08738086 2 1 0 +107 2 2 0.31022976744503572 0.733278453 0.216761291 0.04996032 2 1 0 +108 2 2 0.37690521018236767 0.6859811 0.257114917 0.0569039136 2 1 0 +109 2 2 0.15634514606062988 0.855263948 0.11390055 0.03083543 2 1 0 +110 2 2 0.33718827317238759 0.713774443 0.231632337 0.05459329 2 1 0 +111 2 2 0.37944264976304698 0.684242666 0.257651418 0.05810583 2 1 0 +112 2 2 0.27324249863884714 0.760908246 0.193030283 0.04606139 2 1 0 +113 2 2 0.37884472862275104 0.6846519 0.257622838 0.05772521 2 1 0 +114 2 2 0.22750424400264527 0.796519041 0.163256958 0.0402238965 2 1 0 +115 2 2 0.22895193226059671 0.795366764 0.16392526 0.04070787 2 1 0 +116 2 2 0.39065734627057058 0.67661196 0.263285518 0.0601025522 2 1 0 +117 2 2 0.17291209607975502 0.841211557 0.1255971 0.0331913531 2 1 0 +118 2 2 0.16959072963129782 0.8440102 0.123667806 0.0323220268 2 1 0 +119 2 2 0.71281825117767161 0.490260571 0.418660969 0.09107839 2 1 0 +120 2 2 0.20727468753483022 0.812796354 0.149459183 0.0377443731 2 1 0 +121 2 2 0.37806864376202554 0.685183465 0.256260544 0.05855613 2 1 0 +122 2 2 0.22676757566096883 0.797106 0.163146824 0.0397470556 2 1 0 +123 2 2 0.48465511775599091 0.6159096 0.313702852 0.07038761 2 1 0 +124 2 2 0.2517602171038133 0.77743113 0.178920835 0.0436479524 2 1 0 +125 2 2 0.32715654603030991 0.720970869 0.226308525 0.0527204871 2 1 0 +126 2 2 0.4951267110043599 0.6094937 0.3182633 0.07224296 2 1 0 +127 2 2 0.47111250874852117 0.624307334 0.3053932 0.07029938 2 1 0 +128 2 2 0.27314999099426979 0.760978639 0.193284348 0.04573701 2 1 0 +129 2 2 0.45285059997385629 0.6358131 0.297234565 0.06695242 2 1 0 +130 2 2 0.29503916938347752 0.7445024 0.207360968 0.04813649 2 1 0 +131 2 2 0.22044277504974089 0.802163541 0.158022642 0.0398138724 2 1 0 +132 2 2 0.24528051555040742 0.782485 0.17519024 0.04232483 2 1 0 +133 2 2 0.65295816831312881 0.520503759 0.3896718 0.08982452 2 1 0 +134 2 2 0.64944637633309854 0.5223349 0.391364247 0.0863008052 2 1 0 +135 2 2 0.19184347996657866 0.825436056 0.139052927 0.0355111249 2 1 0 +136 2 2 0.19107437944816186 0.826071143 0.138234288 0.03569463 2 1 0 +137 2 2 0.38702141493447267 0.679076552 0.260932773 0.05999077 2 1 0 +138 2 2 0.48647798676067955 0.6147879 0.312924117 0.07228809 2 1 0 +139 2 2 0.27737520965534401 0.7577701 0.195463046 0.0467668772 2 1 0 +140 2 2 0.19539260437810943 0.8225117 0.141371578 0.0361168236 2 1 0 +141 2 2 0.24146746346253872 0.78547436 0.172211587 0.04231404 2 1 0 +142 2 2 0.40604648196440402 0.6662792 0.272454619 0.0612662435 2 1 0 +143 2 2 0.19849826883938582 0.8199612 0.143505931 0.0365328938 2 1 0 +144 2 2 0.17281651629066277 0.841291964 0.125598609 0.0331094638 2 1 0 +145 2 2 0.23844745859411229 0.7878501 0.170364022 0.0417857766 2 1 0 +146 2 2 0.4245241018940587 0.654081 0.283138216 0.06278075 2 1 0 +147 2 2 0.33463382486357185 0.7156001 0.23068 0.05371997 2 1 0 +148 2 2 0.219788289204249 0.8026887 0.157672763 0.0396385565 2 1 0 +149 2 2 0.44450485886110414 0.641141653 0.29202044 0.06683803 2 1 0 diff --git a/test/BaselineOutput/Common/WeightedEnsembleMulticlass/WE-Voting-TrainTest-iris-out.txt b/test/BaselineOutput/Common/WeightedEnsembleMulticlass/WE-Voting-TrainTest-iris-out.txt new file mode 100644 index 0000000000..94766eed5c --- /dev/null +++ b/test/BaselineOutput/Common/WeightedEnsembleMulticlass/WE-Voting-TrainTest-iris-out.txt @@ -0,0 +1,61 @@ +maml.exe TrainTest test=%Data% tr=WeightedEnsembleMulticlass{bp=mlr{t-} nm=5 oc=MultiVoting tp=-} dout=%Output% data=%Data% out=%Output% seed=1 xf=Term{col=Label} +Automatically adding a MinMax normalization transform, use 'norm=Warn' or 'norm=No' to turn this behavior off. +Training 5 learners for the batch 1 +Beginning training model 1 of 5 +Beginning optimization +num vars: 15 +improvement criterion: Mean Improvement +L1 regularization selected 12 of 15 weights. +Trainer 1 of 5 finished in %Time% +Beginning training model 2 of 5 +Beginning optimization +num vars: 15 +improvement criterion: Mean Improvement +L1 regularization selected 11 of 15 weights. +Trainer 2 of 5 finished in %Time% +Beginning training model 3 of 5 +Beginning optimization +num vars: 15 +improvement criterion: Mean Improvement +L1 regularization selected 11 of 15 weights. +Trainer 3 of 5 finished in %Time% +Beginning training model 4 of 5 +Beginning optimization +num vars: 15 +improvement criterion: Mean Improvement +L1 regularization selected 10 of 15 weights. +Trainer 4 of 5 finished in %Time% +Beginning training model 5 of 5 +Beginning optimization +num vars: 15 +improvement criterion: Mean Improvement +L1 regularization selected 11 of 15 weights. +Trainer 5 of 5 finished in %Time% +Not training a calibrator because it is not needed. + +Confusion table + ||======================== +PREDICTED || 0 | 1 | 2 | Recall +TRUTH ||======================== + 0 || 50 | 0 | 0 | 1.0000 + 1 || 0 | 44 | 6 | 0.8800 + 2 || 0 | 2 | 48 | 0.9600 + ||======================== +Precision ||1.0000 |0.9565 |0.8889 | +Accuracy(micro-avg): 0.946667 +Accuracy(macro-avg): 0.946667 +Log-loss: 0.511576 +Log-loss reduction: 0.534344 + +OVERALL RESULTS +--------------------------------------- +Accuracy(micro-avg): 0.946667 (0.0000) +Accuracy(macro-avg): 0.946667 (0.0000) +Log-loss: 0.511576 (0.0000) +Log-loss reduction: 0.534344 (0.0000) + +--------------------------------------- +Physical memory usage(MB): %Number% +Virtual memory usage(MB): %Number% +%DateTime% Time elapsed(s): %Number% + diff --git a/test/BaselineOutput/Common/WeightedEnsembleMulticlass/WE-Voting-TrainTest-iris-rp.txt b/test/BaselineOutput/Common/WeightedEnsembleMulticlass/WE-Voting-TrainTest-iris-rp.txt new file mode 100644 index 0000000000..704a2be9d5 --- /dev/null +++ b/test/BaselineOutput/Common/WeightedEnsembleMulticlass/WE-Voting-TrainTest-iris-rp.txt @@ -0,0 +1,4 @@ +WeightedEnsembleMulticlass +Accuracy(micro-avg) Accuracy(macro-avg) Log-loss Log-loss reduction /oc /bp /nm Learner Name Train Dataset Test Dataset Results File Run Time Physical Memory Virtual Memory Command Line Settings +0.946667 0.946667 0.511576 0.534344 MultiVoting mlr{t-} 5 WeightedEnsembleMulticlass %Data% %Data% %Output% 99 0 0 maml.exe TrainTest test=%Data% tr=WeightedEnsembleMulticlass{bp=mlr{t-} nm=5 oc=MultiVoting tp=-} dout=%Output% data=%Data% out=%Output% seed=1 xf=Term{col=Label} /oc:MultiVoting;/bp:mlr{t-};/nm:5 + diff --git a/test/BaselineOutput/Common/WeightedEnsembleMulticlass/WE-Voting-TrainTest-iris.txt b/test/BaselineOutput/Common/WeightedEnsembleMulticlass/WE-Voting-TrainTest-iris.txt new file mode 100644 index 0000000000..780cf4a618 --- /dev/null +++ b/test/BaselineOutput/Common/WeightedEnsembleMulticlass/WE-Voting-TrainTest-iris.txt @@ -0,0 +1,151 @@ +Instance Label Assigned Log-loss #1 Score #2 Score #3 Score #1 Class #2 Class #3 Class +0 0 0 0 1 0 0 0 1 2 +1 0 0 0 1 0 0 0 1 2 +2 0 0 0 1 0 0 0 1 2 +3 0 0 0 1 0 0 0 1 2 +4 0 0 0 1 0 0 0 1 2 +5 0 0 0 1 0 0 0 1 2 +6 0 0 0 1 0 0 0 1 2 +7 0 0 0 1 0 0 0 1 2 +8 0 0 0 1 0 0 0 1 2 +9 0 0 0 1 0 0 0 1 2 +10 0 0 0 1 0 0 0 1 2 +11 0 0 0 1 0 0 0 1 2 +12 0 0 0 1 0 0 0 1 2 +13 0 0 0 1 0 0 0 1 2 +14 0 0 0 1 0 0 0 1 2 +15 0 0 0 1 0 0 0 1 2 +16 0 0 0 1 0 0 0 1 2 +17 0 0 0 1 0 0 0 1 2 +18 0 0 0 1 0 0 0 1 2 +19 0 0 0 1 0 0 0 1 2 +20 0 0 0 1 0 0 0 1 2 +21 0 0 0 1 0 0 0 1 2 +22 0 0 0 1 0 0 0 1 2 +23 0 0 0 1 0 0 0 1 2 +24 0 0 0 1 0 0 0 1 2 +25 0 0 0 1 0 0 0 1 2 +26 0 0 0 1 0 0 0 1 2 +27 0 0 0 1 0 0 0 1 2 +28 0 0 0 1 0 0 0 1 2 +29 0 0 0 1 0 0 0 1 2 +30 0 0 0 1 0 0 0 1 2 +31 0 0 0 1 0 0 0 1 2 +32 0 0 0 1 0 0 0 1 2 +33 0 0 0 1 0 0 0 1 2 +34 0 0 0 1 0 0 0 1 2 +35 0 0 0 1 0 0 0 1 2 +36 0 0 0 1 0 0 0 1 2 +37 0 0 0 1 0 0 0 1 2 +38 0 0 0 1 0 0 0 1 2 +39 0 0 0 1 0 0 0 1 2 +40 0 0 0 1 0 0 0 1 2 +41 0 0 0 1 0 0 0 1 2 +42 0 0 0 1 0 0 0 1 2 +43 0 0 0 1 0 0 0 1 2 +44 0 0 0 1 0 0 0 1 2 +45 0 0 0 1 0 0 0 1 2 +46 0 0 0 1 0 0 0 1 2 +47 0 0 0 1 0 0 0 1 2 +48 0 0 0 1 0 0 0 1 2 +49 0 0 0 1 0 0 0 1 2 +50 1 1 0 1 0 0 1 0 2 +51 1 1 0 1 0 0 1 0 2 +52 1 2 0.91629071697299402 0.6 0.4 0 2 1 0 +53 1 1 0 1 0 0 1 0 2 +54 1 1 0 1 0 0 1 0 2 +55 1 1 0 1 0 0 1 0 2 +56 1 2 0.91629071697299402 0.6 0.4 0 2 1 0 +57 1 1 0 1 0 0 1 0 2 +58 1 1 0 1 0 0 1 0 2 +59 1 1 0 1 0 0 1 0 2 +60 1 1 0 1 0 0 1 0 2 +61 1 1 0 1 0 0 1 0 2 +62 1 1 0 1 0 0 1 0 2 +63 1 1 0 1 0 0 1 0 2 +64 1 1 0 1 0 0 1 0 2 +65 1 1 0 1 0 0 1 0 2 +66 1 1 0 1 0 0 1 0 2 +67 1 1 0 1 0 0 1 0 2 +68 1 1 0 1 0 0 1 0 2 +69 1 1 0 1 0 0 1 0 2 +70 1 2 34.538776391283193 1 0 0 2 0 1 +71 1 1 0 1 0 0 1 0 2 +72 1 1 0.22314353641304868 0.8 0.2 0 1 2 0 +73 1 1 0 1 0 0 1 0 2 +74 1 1 0 1 0 0 1 0 2 +75 1 1 0 1 0 0 1 0 2 +76 1 1 0 1 0 0 1 0 2 +77 1 2 34.538776391283193 1 0 0 2 0 1 +78 1 1 0 1 0 0 1 0 2 +79 1 1 0 1 0 0 1 0 2 +80 1 1 0 1 0 0 1 0 2 +81 1 1 0 1 0 0 1 0 2 +82 1 1 0 1 0 0 1 0 2 +83 1 2 0.91629071697299402 0.6 0.4 0 2 1 0 +84 1 1 0 1 0 0 1 0 2 +85 1 2 0.91629071697299402 0.6 0.4 0 2 1 0 +86 1 1 0.22314353641304868 0.8 0.2 0 1 2 0 +87 1 1 0 1 0 0 1 0 2 +88 1 1 0 1 0 0 1 0 2 +89 1 1 0 1 0 0 1 0 2 +90 1 1 0 1 0 0 1 0 2 +91 1 1 0 1 0 0 1 0 2 +92 1 1 0 1 0 0 1 0 2 +93 1 1 0 1 0 0 1 0 2 +94 1 1 0 1 0 0 1 0 2 +95 1 1 0 1 0 0 1 0 2 +96 1 1 0 1 0 0 1 0 2 +97 1 1 0 1 0 0 1 0 2 +98 1 1 0 1 0 0 1 0 2 +99 1 1 0 1 0 0 1 0 2 +100 2 2 0 1 0 0 2 0 1 +101 2 2 0 1 0 0 2 0 1 +102 2 2 0 1 0 0 2 0 1 +103 2 2 0 1 0 0 2 0 1 +104 2 2 0 1 0 0 2 0 1 +105 2 2 0 1 0 0 2 0 1 +106 2 2 0.51082558402956157 0.6 0.4 0 2 1 0 +107 2 2 0 1 0 0 2 0 1 +108 2 2 0 1 0 0 2 0 1 +109 2 2 0 1 0 0 2 0 1 +110 2 2 0 1 0 0 2 0 1 +111 2 2 0 1 0 0 2 0 1 +112 2 2 0 1 0 0 2 0 1 +113 2 2 0 1 0 0 2 0 1 +114 2 2 0 1 0 0 2 0 1 +115 2 2 0 1 0 0 2 0 1 +116 2 2 0 1 0 0 2 0 1 +117 2 2 0 1 0 0 2 0 1 +118 2 2 0 1 0 0 2 0 1 +119 2 1 1.6094378975329393 0.8 0.2 0 1 2 0 +120 2 2 0 1 0 0 2 0 1 +121 2 2 0 1 0 0 2 0 1 +122 2 2 0 1 0 0 2 0 1 +123 2 2 0 1 0 0 2 0 1 +124 2 2 0 1 0 0 2 0 1 +125 2 2 0 1 0 0 2 0 1 +126 2 2 0 1 0 0 2 0 1 +127 2 2 0 1 0 0 2 0 1 +128 2 2 0 1 0 0 2 0 1 +129 2 2 0 1 0 0 2 0 1 +130 2 2 0 1 0 0 2 0 1 +131 2 2 0 1 0 0 2 0 1 +132 2 2 0 1 0 0 2 0 1 +133 2 2 0.51082558402956157 0.6 0.4 0 2 1 0 +134 2 1 0.91629071697299402 0.6 0.4 0 1 2 0 +135 2 2 0 1 0 0 2 0 1 +136 2 2 0 1 0 0 2 0 1 +137 2 2 0 1 0 0 2 0 1 +138 2 2 0 1 0 0 2 0 1 +139 2 2 0 1 0 0 2 0 1 +140 2 2 0 1 0 0 2 0 1 +141 2 2 0 1 0 0 2 0 1 +142 2 2 0 1 0 0 2 0 1 +143 2 2 0 1 0 0 2 0 1 +144 2 2 0 1 0 0 2 0 1 +145 2 2 0 1 0 0 2 0 1 +146 2 2 0 1 0 0 2 0 1 +147 2 2 0 1 0 0 2 0 1 +148 2 2 0 1 0 0 2 0 1 +149 2 2 0 1 0 0 2 0 1 diff --git a/test/Microsoft.ML.Core.Tests/Microsoft.ML.Core.Tests.csproj b/test/Microsoft.ML.Core.Tests/Microsoft.ML.Core.Tests.csproj index 889cca84c0..e514000470 100644 --- a/test/Microsoft.ML.Core.Tests/Microsoft.ML.Core.Tests.csproj +++ b/test/Microsoft.ML.Core.Tests/Microsoft.ML.Core.Tests.csproj @@ -39,7 +39,9 @@ - + + $(TensorFlowMajorVersion) + diff --git a/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs b/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs index ea0cfb4b87..26c19a35f3 100644 --- a/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs +++ b/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs @@ -970,8 +970,8 @@ public void EntryPointPipelineEnsembleText() var data = splitOutput.TrainData[i]; if (i % 2 == 0) { - data = new TextFeaturizingEstimator(Env, "Features", new List { "Text" }, - new TextFeaturizingEstimator.Options { + data = new TextFeaturizingEstimator(Env, "Features", new List { "Text" }, + new TextFeaturizingEstimator.Options { StopWordsRemoverOptions = new StopWordsRemovingEstimator.Options(), }).Fit(data).Transform(data); } @@ -1339,7 +1339,14 @@ public void EntryPointPipelineEnsembleGetSummary() } else if (i % 2 == 1) { - var trainer = new FastTreeBinaryTrainer(Env, "Label", "Features"); + var ftInput = new FastTreeBinaryTrainer.Options + { + NumberOfThreads = 1, + NumberOfLeaves = 5, + NumberOfTrees = 2 + }; + + var trainer = new FastTreeBinaryTrainer(Env, ftInput); var rmd = new RoleMappedData(data, false, RoleMappedSchema.CreatePair(RoleMappedSchema.ColumnRole.Feature, "Features"), RoleMappedSchema.CreatePair(RoleMappedSchema.ColumnRole.Label, "Label")); @@ -5653,5 +5660,82 @@ public void LoadEntryPointModel() } } } + + [Fact] + public void SummarizeEntryPointTest() + { + var dataPath = GetDataPath(@"breast-cancer.txt"); + var outputPath = GetOutputPath("EntryPoints", "Summarize.txt"); + + string inputGraph = @" + { + 'Nodes': + [ + { + 'Name': 'Data.TextLoader', + 'Inputs': + { + 'InputFile': '$inputFile', + 'Arguments': + { + 'UseThreads': true, + 'HeaderFile': null, + 'MaxRows': null, + 'AllowQuoting': true, + 'AllowSparse': true, + 'InputSize': null, + 'Separator': + [ + '\t' + ], + 'Column': + [ + {'Name':'Label','Type':null,'Source':[{'Min':0,'Max':0,'AutoEnd':false,'VariableEnd':false,'AllOther':false,'ForceVector':false}],'KeyCount':null}, + {'Name':'Strat','Type':null,'Source':[{'Min':1,'Max':1,'AutoEnd':false,'VariableEnd':false,'AllOther':false,'ForceVector':false}],'KeyCount':null}, + {'Name':'Features','Type':null,'Source':[{'Min':2,'Max':9,'AutoEnd':false,'VariableEnd':false,'AllOther':false,'ForceVector':false}],'KeyCount':null} + ], + 'TrimWhitespace': false, + 'HasHeader': false + } + }, + 'Outputs': + { + 'Data': '$data' + } + }, + { + 'Name': 'Trainers.FastTreeBinaryClassifier', + 'Inputs': {'TrainingData':'$data','NumberOfThreads':1}, + 'Outputs': {'PredictorModel':'$model'} + }, + { + 'Inputs': + { + 'PredictorModel': '$model' + }, + 'Name': 'Models.Summarizer', + 'Outputs': + { + 'Summary': '$output_data' + } + } + ] + } + "; + + JObject graph = JObject.Parse(inputGraph); + var runner = new GraphRunner(Env, graph[FieldNames.Nodes] as JArray); + var inputFile = new SimpleFileHandle(Env, dataPath, false, false); + runner.SetInput("inputFile", inputFile); + runner.RunAll(); + var data = runner.GetOutput("output_data"); + + using (var f = File.Open(outputPath, FileMode.Create, FileAccess.Write, FileShare.None)) + ML.Data.SaveAsText(data, f); + + CheckEquality("EntryPoints", "Summarize.txt"); + + Done(); + } } } diff --git a/test/Microsoft.ML.Functional.Tests/Microsoft.ML.Functional.Tests.csproj b/test/Microsoft.ML.Functional.Tests/Microsoft.ML.Functional.Tests.csproj index c1c7af9263..ef3f8517fa 100644 --- a/test/Microsoft.ML.Functional.Tests/Microsoft.ML.Functional.Tests.csproj +++ b/test/Microsoft.ML.Functional.Tests/Microsoft.ML.Functional.Tests.csproj @@ -45,7 +45,9 @@ - + + $(TensorFlowMajorVersion) + diff --git a/test/Microsoft.ML.OnnxTransformerTest/OnnxTransformTests.cs b/test/Microsoft.ML.OnnxTransformerTest/OnnxTransformTests.cs index 7cae32ce02..2be146218c 100644 --- a/test/Microsoft.ML.OnnxTransformerTest/OnnxTransformTests.cs +++ b/test/Microsoft.ML.OnnxTransformerTest/OnnxTransformTests.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; +using System.Drawing; using System.IO; using System.Linq; using Microsoft.ML; @@ -14,9 +15,11 @@ using Microsoft.ML.StaticPipe; using Microsoft.ML.TestFramework.Attributes; using Microsoft.ML.Tools; +using Microsoft.ML.Transforms.Image; using Microsoft.ML.Transforms.StaticPipe; using Xunit; using Xunit.Abstractions; +using Microsoft.ML.Transforms.Onnx; namespace Microsoft.ML.Tests { @@ -118,7 +121,7 @@ void TestSimpleCase() catch (ArgumentOutOfRangeException) { } catch (InvalidOperationException) { } } - + [OnnxTheory] [InlineData(null, false)] [InlineData(null, true)] @@ -316,5 +319,293 @@ public void TestUnknownDimensions() Assert.Equal(0, predictions[1].argmax[0]); Assert.Equal(2, predictions[2].argmax[0]); } + + /// + /// This class is used in to describe data points which will be consumed by ML.NET pipeline. + /// + private class ImageDataPoint + { + /// + /// Height of . + /// + private const int height = 224; + + /// + /// Width of . + /// + private const int width = 224; + + /// + /// Image will be consumed by ONNX image multiclass classification model. + /// + [ImageType(height, width)] + public Bitmap Image { get; set; } + + /// + /// Output of ONNX model. It contains probabilities of all classes. + /// + [ColumnName("softmaxout_1")] + public float[] Scores { get; set; } + + public ImageDataPoint() + { + Image = null; + } + + public ImageDataPoint(Color color) + { + Image = new Bitmap(width, height); + for (int i = 0; i < width; ++i) + for (int j = 0; j < height; ++j) + Image.SetPixel(i, j, color); + } + } + + /// + /// Test applying ONNX transform on in-memory image. + /// + [OnnxFact] + public void OnnxModelInMemoryImage() + { + // Path of ONNX model. It's a multiclass classifier. It consumes an input "data_0" and produces an output "softmaxout_1". + var modelFile = "squeezenet/00000001/model.onnx"; + + // Create in-memory data points. Its Image/Scores field is the input/output of the used ONNX model. + var dataPoints = new ImageDataPoint[] + { + new ImageDataPoint(Color.Red), + new ImageDataPoint(Color.Green) + }; + + // Convert training data to IDataView, the general data type used in ML.NET. + var dataView = ML.Data.LoadFromEnumerable(dataPoints); + + // Create a ML.NET pipeline which contains two steps. First, ExtractPixle is used to convert the 224x224 image to a 3x224x224 float tensor. + // Then the float tensor is fed into a ONNX model with an input called "data_0" and an output called "softmaxout_1". Note that "data_0" and + // "softmaxout_1" are model input and output names stored in the used ONNX model file. Users may need to inspect their own models to + // get the right input and output column names. + var pipeline = ML.Transforms.ExtractPixels("data_0", "Image") // Map column "Image" to column "data_0" + .Append(ML.Transforms.ApplyOnnxModel("softmaxout_1", "data_0", modelFile)); // Map column "data_0" to column "softmaxout_1" + var model = pipeline.Fit(dataView); + var onnx = model.Transform(dataView); + + // Convert IDataView back to IEnumerable so that user can inspect the output, column "softmaxout_1", of the ONNX transform. + // Note that Column "softmaxout_1" would be stored in ImageDataPont.Scores because the added attributed [ColumnName("softmaxout_1")] + // tells that ImageDataPont.Scores is equivalent to column "softmaxout_1". + var transformedDataPoints = ML.Data.CreateEnumerable(onnx, false).ToList(); + + // The scores are probabilities of all possible classes, so they should all be positive. + foreach (var dataPoint in transformedDataPoints) + foreach (var score in dataPoint.Scores) + Assert.True(score > 0); + } + + private class ZipMapInput + { + [ColumnName("input")] + [VectorType(3)] + public float[] Input { get; set; } + } + + private class ZipMapStringOutput + { + [OnnxSequenceType(typeof(IDictionary))] + public IEnumerable> output { get; set; } + } + + private class ZipMapInt64Output + { + [OnnxSequenceType(typeof(IDictionary))] + public IEnumerable> output { get; set; } + } + + /// + /// A test to check if sequence output works. + /// + [OnnxFact] + public void TestOnnxZipMapWithInt64Keys() + { + var modelFile = Path.Combine(Directory.GetCurrentDirectory(), "zipmap", "TestZipMapInt64.onnx"); + + var dataPoints = new ZipMapInput[] { + new ZipMapInput() { Input = new float[] {1,2,3}, }, + new ZipMapInput() { Input = new float[] {8,7,6}, }, + }; + + var dataView = ML.Data.LoadFromEnumerable(dataPoints); + var transformedDataView = ML.Transforms.ApplyOnnxModel(new[] { "output" }, new[] { "input" }, modelFile).Fit(dataView).Transform(dataView); + + // Verify output column carried by an IDataView. + var outputColumn = transformedDataView.Schema["output"]; + using (var curs = transformedDataView.GetRowCursor(outputColumn, transformedDataView.Schema["output"])) + { + IEnumerable> buffer = null; + var getMapSequence = curs.GetGetter>>(outputColumn); + int i = 0; + while (curs.MoveNext()) + { + getMapSequence(ref buffer); + Assert.Single(buffer); + var dictionary = buffer.First(); + Assert.Equal(3, dictionary.Count()); + Assert.Equal(dataPoints[i].Input[0], dictionary[94]); + Assert.Equal(dataPoints[i].Input[1], dictionary[17]); + Assert.Equal(dataPoints[i].Input[2], dictionary[36]); + ++i; + } + } + + // Convert IDataView to IEnumerable and then inspect the values. + var transformedDataPoints = ML.Data.CreateEnumerable(transformedDataView, false).ToList(); + + for (int i = 0; i < transformedDataPoints.Count; ++i) + { + Assert.Single(transformedDataPoints[i].output); + var dictionary = transformedDataPoints[i].output.First(); + Assert.Equal(3, dictionary.Count()); + Assert.Equal(dataPoints[i].Input[0], dictionary[94]); + Assert.Equal(dataPoints[i].Input[1], dictionary[17]); + Assert.Equal(dataPoints[i].Input[2], dictionary[36]); + } + } + + /// + /// A test to check if sequence output works. + /// + [OnnxFact] + public void TestOnnxZipMapWithStringKeys() + { + var modelFile = Path.Combine(Directory.GetCurrentDirectory(), "zipmap", "TestZipMapString.onnx"); + + var dataPoints = new ZipMapInput[] { + new ZipMapInput() { Input = new float[] {1,2,3}, }, + new ZipMapInput() { Input = new float[] {8,7,6}, }, + }; + + var dataView = ML.Data.LoadFromEnumerable(dataPoints); + var transformedDataView = ML.Transforms.ApplyOnnxModel(new[] { "output" }, new[] { "input" }, modelFile).Fit(dataView).Transform(dataView); + + // Verify output column carried by an IDataView. + var outputColumn = transformedDataView.Schema["output"]; + using (var curs = transformedDataView.GetRowCursor(outputColumn, transformedDataView.Schema["output"])) + { + IEnumerable> buffer = null; + var getMapSequence = curs.GetGetter>>(outputColumn); + int i = 0; + while (curs.MoveNext()) + { + getMapSequence(ref buffer); + Assert.Single(buffer); + var dictionary = buffer.First(); + Assert.Equal(3, dictionary.Count()); + Assert.Equal(dataPoints[i].Input[0], dictionary["A"]); + Assert.Equal(dataPoints[i].Input[1], dictionary["B"]); + Assert.Equal(dataPoints[i].Input[2], dictionary["C"]); + ++i; + } + } + + // Convert IDataView to IEnumerable and then inspect the values. + var transformedDataPoints = ML.Data.CreateEnumerable(transformedDataView, false).ToList(); + + for (int i = 0; i < transformedDataPoints.Count; ++i) + { + Assert.Single(transformedDataPoints[i].output); + var dictionary = transformedDataPoints[i].output.First(); + Assert.Equal(3, dictionary.Count()); + Assert.Equal(dataPoints[i].Input[0], dictionary["A"]); + Assert.Equal(dataPoints[i].Input[1], dictionary["B"]); + Assert.Equal(dataPoints[i].Input[2], dictionary["C"]); + } + } + + [OnnxFact] + public void TestOnnxModelDisposal() + { + // Create a ONNX model as a byte[]. + var modelFile = Path.Combine(Directory.GetCurrentDirectory(), "zipmap", "TestZipMapInt64.onnx"); + var modelInBytes = File.ReadAllBytes(modelFile); + + // Create ONNX model from the byte[]. + var onnxModel = OnnxModel.CreateFromBytes(modelInBytes); + + // Check if a temporal file is crated for storing the byte[]. + Assert.True(File.Exists(onnxModel.ModelFile)); + + // Delete the temporal file. + onnxModel.Dispose(); + + // Make sure the temporal file is deleted. + Assert.False(File.Exists(onnxModel.ModelFile)); + } + + [OnnxFact] + public void TestOnnxModelNotDisposal() + { + // Declare the path the tested ONNX model file. + var modelFile = Path.Combine(Directory.GetCurrentDirectory(), "zipmap", "TestZipMapInt64.onnx"); + + // Create ONNX model from the model file. + var onnxModel = new OnnxModel(modelFile); + + // Check if a temporal file is crated for storing the byte[]. + Assert.True(File.Exists(onnxModel.ModelFile)); + + // Don't delete the temporal file! + onnxModel.Dispose(); + + // Make sure the temporal file still exists. + Assert.True(File.Exists(onnxModel.ModelFile)); + } + + private class OnnxMapInput + { + [OnnxMapType(typeof(int),typeof(float))] + public IDictionary Input { get; set; } + } + + private class OnnxMapOutput + { + [OnnxMapType(typeof(int),typeof(float))] + public IDictionary Output { get; set; } + } + + /// + /// Use + /// to test if ML.NET can manipulate properly. ONNXRuntime's C# API doesn't support map yet. + /// + [OnnxFact] + public void SmokeInMemoryOnnxMapTypeTest() + { + var inputDict0 = new Dictionary { { 0, 94.17f }, { 1, 17.36f } }; + var inputDict1 = new Dictionary { { 0, 12.28f }, { 1, 75.12f } }; + + var dataPoints = new[] { + new OnnxMapInput() { Input = inputDict0 }, + new OnnxMapInput() { Input = inputDict1 } + }; + + Action action = (input, output) => + { + output.Output = new Dictionary(); + foreach (var pair in input.Input) + { + output.Output.Add(pair.Key + 1, pair.Value); + } + }; + + var dataView = ML.Data.LoadFromEnumerable(dataPoints); + var pipeline = ML.Transforms.CustomMapping(action, contractName: null); + var model = pipeline.Fit(dataView); + var transformedDataView = model.Transform(dataView); + var transformedDataPoints = ML.Data.CreateEnumerable(transformedDataView, false).ToList(); + + for(int i = 0; i < dataPoints.Count(); ++i) + { + Assert.Equal(dataPoints[i].Input.Count(), transformedDataPoints[i].Output.Count()); + foreach(var pair in dataPoints[i].Input) + Assert.Equal(pair.Value, transformedDataPoints[i].Output[pair.Key + 1]); + } + } } } diff --git a/test/Microsoft.ML.Predictor.Tests/TestPredictors.cs b/test/Microsoft.ML.Predictor.Tests/TestPredictors.cs index 134e35716c..1657c34c96 100644 --- a/test/Microsoft.ML.Predictor.Tests/TestPredictors.cs +++ b/test/Microsoft.ML.Predictor.Tests/TestPredictors.cs @@ -2181,6 +2181,135 @@ public void MulticlassNaiveBayes() RunOneAllTests(TestLearners.MulticlassNaiveBayesClassifier, TestDatasets.breastCancerPipe); Done(); } + + [LessThanNetCore30OrNotNetCoreFact("output on .NetCore 3.0 differs. Tracked on issue 3856 in GitHub.")] + public void EnsemblesMultiClassBootstrapSelectorTest() + { + var pa = new PredictorAndArgs(new SubComponent("WeightedEnsembleMulticlass", "bp=mlr{t-} nm=20 st=BootstrapSelector{} tp=-"), "WE-Bootstrap"); + Run_TrainTest(pa, TestDatasets.iris, digitsOfPrecision: 6, parseOption: NumberParseOption.UseSingle); + Done(); + } + + [Fact] + public void EnsemblesDefaultTest() + { + // This one does CV as well as TrainTest. + var pa = new PredictorAndArgs(new SubComponent("WeightedEnsemble", "nm=20 tp=-"), "WE-Default"); + RunOneAllTests(pa, TestDatasets.breastCancer, new[] { "loader=Text{col=Label:BL:0 col=Features:R4:1-9}" }, digitsOfPrecision: 5, parseOption: NumberParseOption.UseSingle); + Done(); + } + + [Fact] + public void EnsemblesBaseLearnerTest() + { + var pa = new PredictorAndArgs(new SubComponent("WeightedEnsemble", "bp=AvgPer nm=3 tp=-"), "WE-AvgPer"); + Run_TrainTest(pa, TestDatasets.breastCancer, new[] { "loader=Text{col=Label:BL:0 col=Features:R4:1-9}" }, digitsOfPrecision: 5, parseOption: NumberParseOption.UseSingle); + Done(); + } + + [Fact] + public void EnsemblesHeterogeneousTest() + { + var pa = new PredictorAndArgs(new SubComponent("WeightedEnsemble", "bp=svm bp=ap nm=20 tp=-"), "WE-Hetero"); + Run_TrainTest(pa, TestDatasets.breastCancer, new[] { "loader=Text{col=Label:BL:0 col=Features:R4:1-9}" }, digitsOfPrecision: 5, parseOption: NumberParseOption.UseSingle); + Done(); + } + + [Fact] + public void EnsemblesVotingCombinerTest() + { + var pa = new PredictorAndArgs(new SubComponent("WeightedEnsemble", "nm=20 oc=Voting tp=-"), "WE-Voting"); + Run_TrainTest(pa, TestDatasets.breastCancer, new[] { "loader=Text{col=Label:BL:0 col=Features:R4:1-9}" }, digitsOfPrecision: 6, parseOption: NumberParseOption.UseSingle); + Done(); + } + + [Fact] + public void EnsemblesStackingCombinerTest() + { + var pa = new PredictorAndArgs(new SubComponent("WeightedEnsemble", "nm=5 oc=Stacking{bp=ap} tp=-"), "WE-StackingAP"); + Run_TrainTest(pa, TestDatasets.breastCancer, new[] { "loader=Text{col=Label:BL:0 col=Features:R4:1-9}" }, digitsOfPrecision: 5, parseOption: NumberParseOption.UseSingle); + Done(); + } + + [Fact] + public void EnsemblesAveragerCombinerTest() + { + var pa = new PredictorAndArgs(new SubComponent("WeightedEnsemble", "nm=20 oc=Average tp=-"), "WE-Average"); + Run_TrainTest(pa, TestDatasets.breastCancer, new[] { "loader=Text{col=Label:BL:0 col=Features:R4:1-9}" }, digitsOfPrecision: 6, parseOption: NumberParseOption.UseSingle); + Done(); + } + + [Fact] + public void EnsemblesBestPerformanceSelectorTest() + { + var pa = new PredictorAndArgs(new SubComponent("WeightedEnsemble", "nm=20 pt=BestPerformanceSelector tp=-"), "WE-BestPerf"); + Run_TrainTest(pa, TestDatasets.breastCancer, new[] { "loader=Text{col=Label:BL:0 col=Features:R4:1-9}" }, digitsOfPrecision: 4, parseOption: NumberParseOption.UseSingle); + Done(); + } + + [Fact] + public void EnsemblesBestDiverseSelectorTest() + { + var pa = new PredictorAndArgs(new SubComponent("WeightedEnsemble", "nm=20 pt=BestDiverseSelector tp=-"), "WE-Diverse"); + Run_TrainTest(pa, TestDatasets.breastCancer, new[] { "loader=Text{col=Label:BL:0 col=Features:R4:1-9}" }, digitsOfPrecision: 6, parseOption: NumberParseOption.UseSingle); + Done(); + } + + [Fact] + public void EnsemblesRandomPartitionInstanceSelectorTest() + { + var pa = new PredictorAndArgs(new SubComponent("WeightedEnsemble", "nm=5 st=RandomPartitionSelector tp=-"), "WE-RandomPartition"); + Run_TrainTest(pa, TestDatasets.breastCancer, new[] { "loader=Text{col=Label:BL:0 col=Features:R4:1-9}" }, digitsOfPrecision: 4, parseOption: NumberParseOption.UseSingle); + Done(); + } + + [Fact] + public void EnsemblesAllDataSetSelectorTest() + { + var pa = new PredictorAndArgs(new SubComponent("WeightedEnsemble", "nm=20 st=AllInstanceSelector tp=-"), "WE-All"); + Run_TrainTest(pa, TestDatasets.breastCancer, new[] { "loader=Text{col=Label:BL:0 col=Features:R4:1-9}" }, digitsOfPrecision: 6, parseOption: NumberParseOption.UseSingle); + Done(); + } + + [Fact] + public void EnsemblesRandomSubSpaceSelectorTest() + { + var pa = new PredictorAndArgs(new SubComponent("WeightedEnsemble", "nm=20 st=AllInstanceSelector{fs=RandomFeatureSelector} tp=-"), "WE-RandomFeature"); + Run_TrainTest(pa, TestDatasets.breastCancer, new[] { "loader=Text{col=Label:BL:0 col=Features:R4:1-9}" }, digitsOfPrecision: 5, parseOption: NumberParseOption.UseSingle); + Done(); + } + + [LessThanNetCore30OrNotNetCoreFact("output on .NetCore 3.0 differs. Tracked on issue 3856 in GitHub.")] + public void EnsemblesMultiAveragerTest() + { + var pa = new PredictorAndArgs(new SubComponent("WeightedEnsembleMulticlass", "bp=mlr{t-} nm=5 oc=MultiAverage tp=-"), "WE-Average"); + Run_TrainTest(pa, TestDatasets.iris, digitsOfPrecision: 6, parseOption: NumberParseOption.UseSingle); + Done(); + } + + [LessThanNetCore30OrNotNetCoreFact("output on .NetCore 3.0 differs. Tracked on issue 3856 in GitHub.")] + public void EnsemblesMultiVotingCombinerTest() + { + var pa = new PredictorAndArgs(new SubComponent("WeightedEnsembleMulticlass", "bp=mlr{t-} nm=5 oc=MultiVoting tp=-"), "WE-Voting"); + Run_TrainTest(pa, TestDatasets.iris, digitsOfPrecision: 6, parseOption: NumberParseOption.UseSingle); + Done(); + } + + [LessThanNetCore30OrNotNetCoreFact("output on .NetCore 3.0 differs. Tracked on issue 3856 in GitHub.")] + public void EnsemblesMultiStackCombinerTest() + { + var pa = new PredictorAndArgs(new SubComponent("WeightedEnsembleMulticlass", "bp=mlr{t-} nm=5 oc=MultiStacking{bp=mlr{t-}} tp=-"), "WE-Stacking"); + Run_TrainTest(pa, TestDatasets.iris, digitsOfPrecision: 6, parseOption: NumberParseOption.UseSingle); + Done(); + } + + [Fact] + public void EnsemblesMultiAveragerSDCATest() + { + var pa = new PredictorAndArgs(new SubComponent("WeightedEnsembleMulticlass", "bp=SDCAMC{nt=1} nm=5 oc=MultiAverage tp=-"), "WE-SDCA-Average"); + Run_TrainTest(pa, TestDatasets.iris, digitsOfPrecision: 6, parseOption: NumberParseOption.UseSingle); + Done(); + } } #if OLD_TESTS // REVIEW: We should have some tests that verify we can't deserialize old models. diff --git a/test/Microsoft.ML.Tests/FeatureContributionTests.cs b/test/Microsoft.ML.Tests/FeatureContributionTests.cs index 02cf7b3d1b..7e9c4b67c2 100644 --- a/test/Microsoft.ML.Tests/FeatureContributionTests.cs +++ b/test/Microsoft.ML.Tests/FeatureContributionTests.cs @@ -124,7 +124,7 @@ public void TestSVMBinary() [Fact] public void TestLogisticRegressionBinary() { - TestFeatureContribution(ML.BinaryClassification.Trainers.LbfgsLogisticRegression(), GetSparseDataset(TaskType.BinaryClassification, 100), "LogisticRegressionBinary"); + TestFeatureContribution(ML.BinaryClassification.Trainers.LbfgsLogisticRegression(), GetSparseDataset(TaskType.BinaryClassification, 100), "LogisticRegressionBinary", 3); } [Fact] diff --git a/test/Microsoft.ML.Tests/Microsoft.ML.Tests.csproj b/test/Microsoft.ML.Tests/Microsoft.ML.Tests.csproj index 8f6dd9fb9b..7f257530c1 100644 --- a/test/Microsoft.ML.Tests/Microsoft.ML.Tests.csproj +++ b/test/Microsoft.ML.Tests/Microsoft.ML.Tests.csproj @@ -46,7 +46,9 @@ - + + $(TensorFlowMajorVersion) + diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/TrainerEstimators.cs b/test/Microsoft.ML.Tests/TrainerEstimators/TrainerEstimators.cs index 489d557ebe..1322827ac2 100644 --- a/test/Microsoft.ML.Tests/TrainerEstimators/TrainerEstimators.cs +++ b/test/Microsoft.ML.Tests/TrainerEstimators/TrainerEstimators.cs @@ -177,6 +177,18 @@ public void TestEstimatorMulticlassNaiveBayesTrainer() return (pipeline, data); } + /// + /// Same data as , but with additional + /// OneHotEncoding to obtain categorical splits in tree models. + /// + private (IEstimator, IDataView) GetOneHotBinaryClassificationPipeline() + { + var (pipeline, data) = GetBinaryClassificationPipeline(); + var oneHotPipeline = pipeline.Append(ML.Transforms.Categorical.OneHotEncoding("Features")); + + return (oneHotPipeline, data); + } + private (IEstimator, IDataView) GetRankingPipeline() { diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/TreeEnsembleFeaturizerTest.cs b/test/Microsoft.ML.Tests/TrainerEstimators/TreeEnsembleFeaturizerTest.cs index 188a9ced46..71253a32b3 100644 --- a/test/Microsoft.ML.Tests/TrainerEstimators/TreeEnsembleFeaturizerTest.cs +++ b/test/Microsoft.ML.Tests/TrainerEstimators/TreeEnsembleFeaturizerTest.cs @@ -3,6 +3,8 @@ // See the LICENSE file in the project root for more information. using System; +using System.Collections.Generic; +using System.IO; using System.Linq; using Microsoft.ML.Data; using Microsoft.ML.Trainers.FastTree; @@ -32,7 +34,12 @@ public void TreeEnsembleFeaturizerOutputSchemaTest() var model = trainer.Fit(dataView); // From the trained tree model, a mapper of tree featurizer is created. - var treeFeaturizer = new TreeEnsembleFeaturizerBindableMapper(Env, new TreeEnsembleFeaturizerBindableMapper.Arguments(), model.Model); + const string treesColumnName = "MyTrees"; + const string leavesColumnName = "MyLeaves"; + const string pathsColumnName = "MyPaths"; + var args = new TreeEnsembleFeaturizerBindableMapper.Arguments() { + TreesColumnName = treesColumnName, LeavesColumnName = leavesColumnName, PathsColumnName = pathsColumnName }; + var treeFeaturizer = new TreeEnsembleFeaturizerBindableMapper(Env, args, model.Model); // To get output schema, we need to create RoleMappedSchema for calling Bind(...). var roleMappedSchema = new RoleMappedSchema(dataView.Schema, @@ -46,7 +53,7 @@ public void TreeEnsembleFeaturizerOutputSchemaTest() { // Check if output schema is correct. var treeValuesColumn = outputSchema[0]; - Assert.Equal("Trees", treeValuesColumn.Name); + Assert.Equal(treesColumnName, treeValuesColumn.Name); VectorDataViewType treeValuesType = treeValuesColumn.Type as VectorDataViewType; Assert.NotNull(treeValuesType); Assert.Equal(NumberDataViewType.Single, treeValuesType.ItemType); @@ -64,7 +71,7 @@ public void TreeEnsembleFeaturizerOutputSchemaTest() { var treeLeafIdsColumn = outputSchema[1]; // Check column of tree leaf IDs. - Assert.Equal("Leaves", treeLeafIdsColumn.Name); + Assert.Equal(leavesColumnName, treeLeafIdsColumn.Name); VectorDataViewType treeLeafIdsType = treeLeafIdsColumn.Type as VectorDataViewType; Assert.NotNull(treeLeafIdsType); Assert.Equal(NumberDataViewType.Single, treeLeafIdsType.ItemType); @@ -87,7 +94,7 @@ public void TreeEnsembleFeaturizerOutputSchemaTest() { var treePathIdsColumn = outputSchema[2]; // Check column of path IDs. - Assert.Equal("Paths", treePathIdsColumn.Name); + Assert.Equal(pathsColumnName, treePathIdsColumn.Name); VectorDataViewType treePathIdsType = treePathIdsColumn.Type as VectorDataViewType; Assert.NotNull(treePathIdsType); Assert.Equal(NumberDataViewType.Single, treePathIdsType.ItemType); @@ -108,5 +115,710 @@ public void TreeEnsembleFeaturizerOutputSchemaTest() } } + + [Fact] + public void TreeEnsembleFeaturizerTransformerFastTreeBinary() + { + // Create data set + int dataPointCount = 20; + var data = SamplesUtils.DatasetUtils.GenerateBinaryLabelFloatFeatureVectorFloatWeightSamples(dataPointCount).ToList(); + var dataView = ML.Data.LoadFromEnumerable(data); + + // Define a tree model whose trees will be extracted to construct a tree featurizer. + var trainer = ML.BinaryClassification.Trainers.FastTree( + new FastTreeBinaryTrainer.Options + { + NumberOfThreads = 1, + NumberOfTrees = 1, + NumberOfLeaves = 4, + MinimumExampleCountPerLeaf = 1 + }); + + // Train the defined tree model. + var model = trainer.Fit(dataView); + var predicted = model.Transform(dataView); + + // From the trained tree model, a mapper of tree featurizer is created. + const string treesColumnName = "MyTrees"; + const string leavesColumnName = "MyLeaves"; + const string pathsColumnName = "MyPaths"; + var treeFeaturizer = new TreeEnsembleFeaturizationTransformer(ML, dataView.Schema, dataView.Schema["Features"], model.Model.SubModel, + treesColumnName: treesColumnName, leavesColumnName: leavesColumnName, pathsColumnName: pathsColumnName); + + // Apply TreeEnsembleFeaturizer to the input data. + var transformed = treeFeaturizer.Transform(dataView); + + // Extract the outputs of TreeEnsembleFeaturizer. + var features = transformed.GetColumn("Features").ToArray(); + var leafValues = transformed.GetColumn(treesColumnName).ToArray(); + var leafIds = transformed.GetColumn(leavesColumnName).ToArray(); + var paths = transformed.GetColumn(pathsColumnName).ToArray(); + + // Check if the TreeEnsembleFeaturizer produce expected values. + List path = null; + for (int dataPointIndex = 0; dataPointIndex < dataPointCount; ++dataPointIndex) + { + int treeIndex = 0; + var leafId = model.Model.SubModel.GetLeaf(treeIndex, new VBuffer(10, features[dataPointIndex]), ref path); + var leafValue = model.Model.SubModel.GetLeafValue(0, leafId); + Assert.Equal(leafValues[dataPointIndex][treeIndex], leafValue); + Assert.Equal(1.0, leafIds[dataPointIndex][leafId]); + foreach (var nodeId in path) + Assert.Equal(1.0, paths[dataPointIndex][nodeId]); + } + } + + [Fact] + public void TreeEnsembleFeaturizerTransformerFastForestBinary() + { + // Create data set + int dataPointCount = 20; + var data = SamplesUtils.DatasetUtils.GenerateBinaryLabelFloatFeatureVectorFloatWeightSamples(dataPointCount).ToList(); + var dataView = ML.Data.LoadFromEnumerable(data); + + // Define a tree model whose trees will be extracted to construct a tree featurizer. + var trainer = ML.BinaryClassification.Trainers.FastForest( + new FastForestBinaryTrainer.Options + { + NumberOfThreads = 1, + NumberOfTrees = 1, + NumberOfLeaves = 4, + MinimumExampleCountPerLeaf = 1 + }); + + // Train the defined tree model. + var model = trainer.Fit(dataView); + + // From the trained tree model, a mapper of tree featurizer is created. + const string treesColumnName = "MyTrees"; + const string leavesColumnName = "MyLeaves"; + const string pathsColumnName = "MyPaths"; + var treeFeaturizer = new TreeEnsembleFeaturizationTransformer(ML, dataView.Schema, dataView.Schema["Features"], model.Model, + treesColumnName: treesColumnName, leavesColumnName: leavesColumnName, pathsColumnName: pathsColumnName); + + // Apply TreeEnsembleFeaturizer to the input data. + var transformed = treeFeaturizer.Transform(dataView); + + // Extract the outputs of TreeEnsembleFeaturizer. + var features = transformed.GetColumn("Features").ToArray(); + var leafValues = transformed.GetColumn(treesColumnName).ToArray(); + var leafIds = transformed.GetColumn(leavesColumnName).ToArray(); + var paths = transformed.GetColumn(pathsColumnName).ToArray(); + + // Check if the TreeEnsembleFeaturizer produce expected values. + List path = null; + for (int dataPointIndex = 0; dataPointIndex < dataPointCount; ++dataPointIndex) + { + int treeIndex = 0; + var leafId = model.Model.GetLeaf(treeIndex, new VBuffer(10, features[dataPointIndex]), ref path); + var leafValue = model.Model.GetLeafValue(0, leafId); + Assert.Equal(leafValues[dataPointIndex][treeIndex], leafValue); + Assert.Equal(1.0, leafIds[dataPointIndex][leafId]); + foreach (var nodeId in path) + Assert.Equal(1.0, paths[dataPointIndex][nodeId]); + } + } + + /// + /// A test of . + /// + [Fact] + public void TestPretrainedTreeFeaturizationEstimator() + { + // Create data set + int dataPointCount = 20; + var data = SamplesUtils.DatasetUtils.GenerateBinaryLabelFloatFeatureVectorFloatWeightSamples(dataPointCount).ToList(); + var dataView = ML.Data.LoadFromEnumerable(data); + dataView = ML.Data.Cache(dataView); + + // Define a tree model whose trees will be extracted to construct a tree featurizer. + var trainer = ML.BinaryClassification.Trainers.FastTree( + new FastTreeBinaryTrainer.Options + { + NumberOfThreads = 1, + NumberOfTrees = 1, + NumberOfLeaves = 4, + MinimumExampleCountPerLeaf = 1 + }); + + // Train the defined tree model. + var model = trainer.Fit(dataView); + var predicted = model.Transform(dataView); + + // From the trained tree model, a mapper of tree featurizer is created. + string featureColumnName = "Features"; + string treesColumnName = "MyTrees"; // a tree-based feature column. + string leavesColumnName = "MyLeaves"; // a tree-based feature column. + string pathsColumnName = "MyPaths"; // a tree-based feature column. + var options = new PretrainedTreeFeaturizationEstimator.Options() + { + InputColumnName = featureColumnName, + ModelParameters = model.Model.SubModel, + TreesColumnName = treesColumnName, + LeavesColumnName = leavesColumnName, + PathsColumnName = pathsColumnName + }; + var treeFeaturizer = ML.Transforms.FeaturizeByPretrainTreeEnsemble(options).Fit(dataView); + + // Apply TreeEnsembleFeaturizer to the input data. + var transformed = treeFeaturizer.Transform(dataView); + + // Extract the outputs of TreeEnsembleFeaturizer. + var features = transformed.GetColumn(featureColumnName).ToArray(); + var leafValues = transformed.GetColumn(treesColumnName).ToArray(); + var leafIds = transformed.GetColumn(leavesColumnName).ToArray(); + var paths = transformed.GetColumn(pathsColumnName).ToArray(); + + // Check if the TreeEnsembleFeaturizer produce expected values. + List path = null; + for (int dataPointIndex = 0; dataPointIndex < dataPointCount; ++dataPointIndex) + { + int treeIndex = 0; + var leafId = model.Model.SubModel.GetLeaf(treeIndex, new VBuffer(10, features[dataPointIndex]), ref path); + var leafValue = model.Model.SubModel.GetLeafValue(0, leafId); + Assert.Equal(leafValues[dataPointIndex][treeIndex], leafValue); + Assert.Equal(1.0, leafIds[dataPointIndex][leafId]); + foreach (var nodeId in path) + Assert.Equal(1.0, paths[dataPointIndex][nodeId]); + } + } + + /// + /// This test contains several steps. + /// 1. It first trains a using . + /// 2. Then, it creates the a from the trained . + /// 3. The feature produced in step 2 would be fed into to enhance the training accuracy of that linear model. + /// 4. We train another without features from trees and finally compare their scores. + /// + [Fact] + public void TreeEnsembleFeaturizingPipeline() + { + // Create data set + int dataPointCount = 200; + var data = SamplesUtils.DatasetUtils.GenerateBinaryLabelFloatFeatureVectorFloatWeightSamples(dataPointCount).ToList(); + var dataView = ML.Data.LoadFromEnumerable(data); + dataView = ML.Data.Cache(dataView); + + // Define a tree model whose trees will be extracted to construct a tree featurizer. + var trainer = ML.BinaryClassification.Trainers.FastTree( + new FastTreeBinaryTrainer.Options + { + NumberOfThreads = 1, + NumberOfTrees = 10, + NumberOfLeaves = 4, + MinimumExampleCountPerLeaf = 10 + }); + + // Train the defined tree model. This trained model will be used to construct TreeEnsembleFeaturizationEstimator. + var treeModel = trainer.Fit(dataView); + var predicted = treeModel.Transform(dataView); + + // Combine the output of TreeEnsembleFeaturizationTransformer and the original features as the final training features. + // Then train a linear model. + var options = new PretrainedTreeFeaturizationEstimator.Options() + { + InputColumnName = "Features", + TreesColumnName = "Trees", + LeavesColumnName = "Leaves", + PathsColumnName = "Paths", + ModelParameters = treeModel.Model.SubModel + }; + var pipeline = ML.Transforms.FeaturizeByPretrainTreeEnsemble(options) + .Append(ML.Transforms.Concatenate("CombinedFeatures", "Features", "Trees", "Leaves", "Paths")) + .Append(ML.BinaryClassification.Trainers.SdcaLogisticRegression("Label", "CombinedFeatures")); + var model = pipeline.Fit(dataView); + var prediction = model.Transform(dataView); + var metrics = ML.BinaryClassification.Evaluate(prediction); + + // Then train the same linear model without tree features. + var naivePipeline = ML.BinaryClassification.Trainers.SdcaLogisticRegression("Label", "Features"); + var naiveModel = naivePipeline.Fit(dataView); + var naivePrediction = naiveModel.Transform(dataView); + var naiveMetrics = ML.BinaryClassification.Evaluate(naivePrediction); + + // The linear model trained with tree features should perform better than that without tree features. + Assert.True(metrics.Accuracy > naiveMetrics.Accuracy); + Assert.True(metrics.LogLoss < naiveMetrics.LogLoss); + Assert.True(metrics.AreaUnderPrecisionRecallCurve > naiveMetrics.AreaUnderPrecisionRecallCurve); + } + + [Fact] + public void TestFastTreeBinaryFeaturizationInPipeline() + { + int dataPointCount = 200; + var data = SamplesUtils.DatasetUtils.GenerateBinaryLabelFloatFeatureVectorFloatWeightSamples(dataPointCount).ToList(); + var dataView = ML.Data.LoadFromEnumerable(data); + dataView = ML.Data.Cache(dataView); + + var trainerOptions = new FastTreeBinaryTrainer.Options + { + NumberOfThreads = 1, + NumberOfTrees = 10, + NumberOfLeaves = 4, + MinimumExampleCountPerLeaf = 10, + FeatureColumnName = "Features", + LabelColumnName = "Label" + }; + + var options = new FastTreeBinaryFeaturizationEstimator.Options() + { + InputColumnName = "Features", + TreesColumnName = "Trees", + LeavesColumnName = "Leaves", + PathsColumnName = "Paths", + TrainerOptions = trainerOptions + }; + + var pipeline = ML.Transforms.FeaturizeByFastTreeBinary(options) + .Append(ML.Transforms.Concatenate("CombinedFeatures", "Features", "Trees", "Leaves", "Paths")) + .Append(ML.BinaryClassification.Trainers.SdcaLogisticRegression("Label", "CombinedFeatures")); + var model = pipeline.Fit(dataView); + var prediction = model.Transform(dataView); + var metrics = ML.BinaryClassification.Evaluate(prediction); + + Assert.True(metrics.Accuracy > 0.98); + Assert.True(metrics.LogLoss < 0.05); + Assert.True(metrics.AreaUnderPrecisionRecallCurve > 0.98); + } + + [Fact] + public void TestFastForestBinaryFeaturizationInPipeline() + { + int dataPointCount = 200; + var data = SamplesUtils.DatasetUtils.GenerateBinaryLabelFloatFeatureVectorFloatWeightSamples(dataPointCount).ToList(); + var dataView = ML.Data.LoadFromEnumerable(data); + dataView = ML.Data.Cache(dataView); + + var trainerOptions = new FastForestBinaryTrainer.Options + { + NumberOfThreads = 1, + NumberOfTrees = 10, + NumberOfLeaves = 4, + MinimumExampleCountPerLeaf = 10, + FeatureColumnName = "Features", + LabelColumnName = "Label" + }; + + var options = new FastForestBinaryFeaturizationEstimator.Options() + { + InputColumnName = "Features", + TreesColumnName = "Trees", + LeavesColumnName = "Leaves", + PathsColumnName = "Paths", + TrainerOptions = trainerOptions + }; + + var pipeline = ML.Transforms.FeaturizeByFastForestBinary(options) + .Append(ML.Transforms.Concatenate("CombinedFeatures", "Features", "Trees", "Leaves", "Paths")) + .Append(ML.BinaryClassification.Trainers.SdcaLogisticRegression("Label", "CombinedFeatures")); + var model = pipeline.Fit(dataView); + var prediction = model.Transform(dataView); + var metrics = ML.BinaryClassification.Evaluate(prediction); + + Assert.True(metrics.Accuracy > 0.97); + Assert.True(metrics.LogLoss < 0.07); + Assert.True(metrics.AreaUnderPrecisionRecallCurve > 0.98); + } + + [Fact] + public void TestFastTreeRegressionFeaturizationInPipeline() + { + int dataPointCount = 200; + var data = SamplesUtils.DatasetUtils.GenerateFloatLabelFloatFeatureVectorSamples(dataPointCount).ToList(); + var dataView = ML.Data.LoadFromEnumerable(data); + dataView = ML.Data.Cache(dataView); + + var trainerOptions = new FastTreeRegressionTrainer.Options + { + NumberOfThreads = 1, + NumberOfTrees = 10, + NumberOfLeaves = 4, + MinimumExampleCountPerLeaf = 10, + FeatureColumnName = "Features", + LabelColumnName = "Label" + }; + + var options = new FastTreeRegressionFeaturizationEstimator.Options() + { + InputColumnName = "Features", + TreesColumnName = "Trees", + LeavesColumnName = "Leaves", + PathsColumnName = "Paths", + TrainerOptions = trainerOptions + }; + + var pipeline = ML.Transforms.FeaturizeByFastTreeRegression(options) + .Append(ML.Transforms.Concatenate("CombinedFeatures", "Features", "Trees", "Leaves", "Paths")) + .Append(ML.Regression.Trainers.Sdca("Label", "CombinedFeatures")); + var model = pipeline.Fit(dataView); + var prediction = model.Transform(dataView); + var metrics = ML.Regression.Evaluate(prediction); + + Assert.True(metrics.MeanAbsoluteError < 0.2); + Assert.True(metrics.MeanSquaredError < 0.05); + } + + [Fact] + public void TestFastForestRegressionFeaturizationInPipeline() + { + int dataPointCount = 200; + var data = SamplesUtils.DatasetUtils.GenerateFloatLabelFloatFeatureVectorSamples(dataPointCount).ToList(); + var dataView = ML.Data.LoadFromEnumerable(data); + dataView = ML.Data.Cache(dataView); + + var trainerOptions = new FastForestRegressionTrainer.Options + { + NumberOfThreads = 1, + NumberOfTrees = 10, + NumberOfLeaves = 4, + MinimumExampleCountPerLeaf = 10, + FeatureColumnName = "Features", + LabelColumnName = "Label" + }; + + var options = new FastForestRegressionFeaturizationEstimator.Options() + { + InputColumnName = "Features", + TreesColumnName = "Trees", + LeavesColumnName = "Leaves", + PathsColumnName = "Paths", + TrainerOptions = trainerOptions + }; + + var pipeline = ML.Transforms.FeaturizeByFastForestRegression(options) + .Append(ML.Transforms.Concatenate("CombinedFeatures", "Features", "Trees", "Leaves", "Paths")) + .Append(ML.Regression.Trainers.Sdca("Label", "CombinedFeatures")); + var model = pipeline.Fit(dataView); + var prediction = model.Transform(dataView); + var metrics = ML.Regression.Evaluate(prediction); + + Assert.True(metrics.MeanAbsoluteError < 0.25); + Assert.True(metrics.MeanSquaredError < 0.1); + } + + [Fact] + public void TestFastTreeTweedieFeaturizationInPipeline() + { + int dataPointCount = 200; + var data = SamplesUtils.DatasetUtils.GenerateFloatLabelFloatFeatureVectorSamples(dataPointCount).ToList(); + var dataView = ML.Data.LoadFromEnumerable(data); + dataView = ML.Data.Cache(dataView); + + var trainerOptions = new FastTreeTweedieTrainer.Options + { + NumberOfThreads = 1, + NumberOfTrees = 10, + NumberOfLeaves = 4, + MinimumExampleCountPerLeaf = 10, + FeatureColumnName = "Features", + LabelColumnName = "Label" + }; + + var options = new FastTreeTweedieFeaturizationEstimator.Options() + { + InputColumnName = "Features", + TreesColumnName = "Trees", + LeavesColumnName = "Leaves", + PathsColumnName = "Paths", + TrainerOptions = trainerOptions + }; + + var pipeline = ML.Transforms.FeaturizeByFastTreeTweedie(options) + .Append(ML.Transforms.Concatenate("CombinedFeatures", "Features", "Trees", "Leaves", "Paths")) + .Append(ML.Regression.Trainers.Sdca("Label", "CombinedFeatures")); + var model = pipeline.Fit(dataView); + var prediction = model.Transform(dataView); + var metrics = ML.Regression.Evaluate(prediction); + + Assert.True(metrics.MeanAbsoluteError < 0.25); + Assert.True(metrics.MeanSquaredError < 0.1); + } + + [Fact] + public void TestFastTreeRankingFeaturizationInPipeline() + { + int dataPointCount = 200; + var data = SamplesUtils.DatasetUtils.GenerateFloatLabelFloatFeatureVectorSamples(dataPointCount).ToList(); + var dataView = ML.Data.LoadFromEnumerable(data); + dataView = ML.Data.Cache(dataView); + + var trainerOptions = new FastTreeRankingTrainer.Options + { + NumberOfThreads = 1, + NumberOfTrees = 10, + NumberOfLeaves = 4, + MinimumExampleCountPerLeaf = 10, + FeatureColumnName = "Features", + LabelColumnName = "Label" + }; + + var options = new FastTreeRankingFeaturizationEstimator.Options() + { + InputColumnName = "Features", + TreesColumnName = "Trees", + LeavesColumnName = "Leaves", + PathsColumnName = "Paths", + TrainerOptions = trainerOptions + }; + + var pipeline = ML.Transforms.FeaturizeByFastTreeRanking(options) + .Append(ML.Transforms.Concatenate("CombinedFeatures", "Features", "Trees", "Leaves", "Paths")) + .Append(ML.Regression.Trainers.Sdca("Label", "CombinedFeatures")); + var model = pipeline.Fit(dataView); + var prediction = model.Transform(dataView); + var metrics = ML.Regression.Evaluate(prediction); + + Assert.True(metrics.MeanAbsoluteError < 0.25); + Assert.True(metrics.MeanSquaredError < 0.1); + } + + [Fact] + public void TestSaveAndLoadTreeFeaturizer() + { + int dataPointCount = 200; + var data = SamplesUtils.DatasetUtils.GenerateFloatLabelFloatFeatureVectorSamples(dataPointCount).ToList(); + var dataView = ML.Data.LoadFromEnumerable(data); + dataView = ML.Data.Cache(dataView); + + var trainerOptions = new FastForestRegressionTrainer.Options + { + NumberOfThreads = 1, + NumberOfTrees = 10, + NumberOfLeaves = 4, + MinimumExampleCountPerLeaf = 10, + FeatureColumnName = "Features", + LabelColumnName = "Label" + }; + + var options = new FastForestRegressionFeaturizationEstimator.Options() + { + InputColumnName = "Features", + TreesColumnName = "Trees", + LeavesColumnName = "Leaves", + PathsColumnName = "Paths", + TrainerOptions = trainerOptions + }; + + var pipeline = ML.Transforms.FeaturizeByFastForestRegression(options) + .Append(ML.Transforms.Concatenate("CombinedFeatures", "Features", "Trees", "Leaves", "Paths")) + .Append(ML.Regression.Trainers.Sdca("Label", "CombinedFeatures")); + var model = pipeline.Fit(dataView); + var prediction = model.Transform(dataView); + var metrics = ML.Regression.Evaluate(prediction); + + Assert.True(metrics.MeanAbsoluteError < 0.25); + Assert.True(metrics.MeanSquaredError < 0.1); + + // Save the trained model into file. + ITransformer loadedModel = null; + var tempPath = Path.GetTempFileName(); + using (var file = new SimpleFileHandle(Env, tempPath, true, true)) + { + using (var fs = file.CreateWriteStream()) + ML.Model.Save(model, null, fs); + + using (var fs = file.OpenReadStream()) + loadedModel = ML.Model.Load(fs, out var schema); + } + var loadedPrediction = loadedModel.Transform(dataView); + var loadedMetrics = ML.Regression.Evaluate(loadedPrediction); + + Assert.Equal(metrics.MeanAbsoluteError, loadedMetrics.MeanAbsoluteError); + Assert.Equal(metrics.MeanSquaredError, loadedMetrics.MeanSquaredError); + } + + [Fact] + public void TestSaveAndLoadDoubleTreeFeaturizer() + { + int dataPointCount = 200; + var data = SamplesUtils.DatasetUtils.GenerateFloatLabelFloatFeatureVectorSamples(dataPointCount).ToList(); + var dataView = ML.Data.LoadFromEnumerable(data); + dataView = ML.Data.Cache(dataView); + + var trainerOptions = new FastForestRegressionTrainer.Options + { + NumberOfThreads = 1, + NumberOfTrees = 10, + NumberOfLeaves = 4, + MinimumExampleCountPerLeaf = 10, + FeatureColumnName = "Features", + LabelColumnName = "Label" + }; + + // Trains tree featurization on "Features" and applies on "CopiedFeatures". + var options = new FastForestRegressionFeaturizationEstimator.Options() + { + InputColumnName = "CopiedFeatures", + TrainerOptions = trainerOptions, + TreesColumnName = "OhMyTrees", + LeavesColumnName = "OhMyLeaves", + PathsColumnName = "OhMyPaths" + }; + + var pipeline = ML.Transforms.CopyColumns("CopiedFeatures", "Features") + .Append(ML.Transforms.FeaturizeByFastForestRegression(options)) + .Append(ML.Transforms.Concatenate("CombinedFeatures", "Features", "OhMyTrees", "OhMyLeaves", "OhMyPaths")) + .Append(ML.Regression.Trainers.Sdca("Label", "CombinedFeatures")); + var model = pipeline.Fit(dataView); + var prediction = model.Transform(dataView); + var metrics = ML.Regression.Evaluate(prediction); + + Assert.True(metrics.MeanAbsoluteError < 0.25); + Assert.True(metrics.MeanSquaredError < 0.1); + + // Save the trained model into file and then load it back. + ITransformer loadedModel = null; + var tempPath = Path.GetTempFileName(); + using (var file = new SimpleFileHandle(Env, tempPath, true, true)) + { + using (var fs = file.CreateWriteStream()) + ML.Model.Save(model, null, fs); + + using (var fs = file.OpenReadStream()) + loadedModel = ML.Model.Load(fs, out var schema); + } + + // Compute prediction using the loaded model. + var loadedPrediction = loadedModel.Transform(dataView); + var loadedMetrics = ML.Regression.Evaluate(loadedPrediction); + + // Check if the loaded model produces the same result as the trained model. + Assert.Equal(metrics.MeanAbsoluteError, loadedMetrics.MeanAbsoluteError); + Assert.Equal(metrics.MeanSquaredError, loadedMetrics.MeanSquaredError); + + var secondPipeline = ML.Transforms.CopyColumns("CopiedFeatures", "Features") + .Append(ML.Transforms.NormalizeBinning("CopiedFeatures")) + .Append(ML.Transforms.FeaturizeByFastForestRegression(options)) + .Append(ML.Transforms.Concatenate("CombinedFeatures", "Features", "OhMyTrees", "OhMyLeaves", "OhMyPaths")) + .Append(ML.Regression.Trainers.Sdca("Label", "CombinedFeatures")); + var secondModel = secondPipeline.Fit(dataView); + var secondPrediction = secondModel.Transform(dataView); + var secondMetrics = ML.Regression.Evaluate(secondPrediction); + + // The second pipeline trains a tree featurizer on a bin-based normalized feature, so the second pipeline + // is different from the first pipeline. + Assert.NotEqual(metrics.MeanAbsoluteError, secondMetrics.MeanAbsoluteError); + Assert.NotEqual(metrics.MeanSquaredError, secondMetrics.MeanSquaredError); + } + + [Fact] + public void TestFastTreeBinaryFeaturizationInPipelineWithOptionalOutputs() + { + int dataPointCount = 200; + var data = SamplesUtils.DatasetUtils.GenerateBinaryLabelFloatFeatureVectorFloatWeightSamples(dataPointCount).ToList(); + var dataView = ML.Data.LoadFromEnumerable(data); + dataView = ML.Data.Cache(dataView); + + var trainerOptions = new FastTreeBinaryTrainer.Options + { + NumberOfThreads = 1, + NumberOfTrees = 10, + NumberOfLeaves = 4, + MinimumExampleCountPerLeaf = 10, + FeatureColumnName = "Features", + LabelColumnName = "Label" + }; + + var options = new FastTreeBinaryFeaturizationEstimator.Options() + { + InputColumnName = "Features", + TrainerOptions = trainerOptions, + TreesColumnName = null, + PathsColumnName = null, + LeavesColumnName = "Leaves" + }; + + + bool isWrong = false; + try + { + var wrongPipeline = ML.Transforms.FeaturizeByFastTreeBinary(options) + .Append(ML.Transforms.Concatenate("CombinedFeatures", "Features", "Trees", "Leaves", "Paths")) + .Append(ML.BinaryClassification.Trainers.SdcaLogisticRegression("Label", "CombinedFeatures")); + var wrongModel = wrongPipeline.Fit(dataView); + } + catch + { + isWrong = true; // Only "Leaves" is produced by the tree featurizer, so accessing "Trees" and "Paths" will lead to an error. + } + Assert.True(isWrong); + + var pipeline = ML.Transforms.FeaturizeByFastTreeBinary(options) + .Append(ML.Transforms.Concatenate("CombinedFeatures", "Features", "Leaves")) + .Append(ML.BinaryClassification.Trainers.SdcaLogisticRegression("Label", "CombinedFeatures")); + var model = pipeline.Fit(dataView); + var prediction = model.Transform(dataView); + var metrics = ML.BinaryClassification.Evaluate(prediction); + + Assert.True(metrics.Accuracy > 0.98); + Assert.True(metrics.LogLoss < 0.05); + Assert.True(metrics.AreaUnderPrecisionRecallCurve > 0.98); + } + + /// + /// Apply tree-based featurization on multiclass classification by converting key-typed labels to floats and training + /// a regression tree model for featurization. + /// + [Fact] + public void TreeEnsembleFeaturizingPipelineMulticlass() + { + int dataPointCount = 1000; + var data = SamplesUtils.DatasetUtils.GenerateRandomMulticlassClassificationExamples(dataPointCount).ToList(); + var dataView = ML.Data.LoadFromEnumerable(data); + dataView = ML.Data.Cache(dataView); + + var trainerOptions = new FastForestRegressionTrainer.Options + { + NumberOfThreads = 1, + NumberOfTrees = 10, + NumberOfLeaves = 4, + MinimumExampleCountPerLeaf = 10, + FeatureColumnName = "Features", + LabelColumnName = "FloatLabel", + ShuffleLabels = true + }; + + var options = new FastForestRegressionFeaturizationEstimator.Options() + { + InputColumnName = "Features", + TreesColumnName = "Trees", + LeavesColumnName = "Leaves", + PathsColumnName = "Paths", + TrainerOptions = trainerOptions + }; + + Action actionConvertKeyToFloat = (RowWithKey rowWithKey, RowWithFloat rowWithFloat) => + { + rowWithFloat.FloatLabel = rowWithKey.KeyLabel == 0 ? float.NaN : rowWithKey.KeyLabel - 1; + }; + + var split = ML.Data.TrainTestSplit(dataView, 0.5); + var trainData = split.TrainSet; + var testData = split.TestSet; + + var pipeline = ML.Transforms.Conversion.MapValueToKey("KeyLabel", "Label") + .Append(ML.Transforms.CustomMapping(actionConvertKeyToFloat, "KeyLabel")) + .Append(ML.Transforms.FeaturizeByFastForestRegression(options)) + .Append(ML.Transforms.Concatenate("CombinedFeatures", "Trees", "Leaves", "Paths")) + .Append(ML.MulticlassClassification.Trainers.SdcaMaximumEntropy("KeyLabel", "CombinedFeatures")); + + var model = pipeline.Fit(trainData); + var prediction = model.Transform(testData); + var metrics = ML.MulticlassClassification.Evaluate(prediction, labelColumnName: "KeyLabel"); + + Assert.True(metrics.MacroAccuracy > 0.6); + Assert.True(metrics.MicroAccuracy > 0.6); + } + + private class RowWithKey + { + [KeyType(4)] + public uint KeyLabel { get; set; } + } + + private class RowWithFloat + { + public float FloatLabel { get; set; } + } } } diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs b/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs index 73a5fa4149..1c719a6f85 100644 --- a/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs +++ b/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs @@ -9,6 +9,7 @@ using Microsoft.ML.Calibrators; using Microsoft.ML.Data; using Microsoft.ML.Internal.Utilities; +using Microsoft.ML.Model; using Microsoft.ML.RunTests; using Microsoft.ML.Runtime; using Microsoft.ML.TestFramework.Attributes; @@ -22,7 +23,7 @@ namespace Microsoft.ML.Tests.TrainerEstimators public partial class TrainerEstimators : TestDataPipeBase { /// - /// FastTreeBinaryClassification TrainerEstimator test + /// FastTreeBinaryClassification TrainerEstimator test /// [Fact] public void FastTreeBinaryEstimator() @@ -55,6 +56,28 @@ public void LightGBMBinaryEstimator() NumberOfLeaves = 10, NumberOfThreads = 1, MinimumExampleCountPerLeaf = 2, + UnbalancedSets = false, // default value + }); + + var pipeWithTrainer = pipe.Append(trainer); + TestEstimatorCore(pipeWithTrainer, dataView); + + var transformedDataView = pipe.Fit(dataView).Transform(dataView); + var model = trainer.Fit(transformedDataView, transformedDataView); + Done(); + } + + [LightGBMFact] + public void LightGBMBinaryEstimatorUnbalanced() + { + var (pipe, dataView) = GetBinaryClassificationPipeline(); + + var trainer = ML.BinaryClassification.Trainers.LightGbm(new LightGbmBinaryTrainer.Options + { + NumberOfLeaves = 10, + NumberOfThreads = 1, + MinimumExampleCountPerLeaf = 2, + UnbalancedSets = true, }); var pipeWithTrainer = pipe.Append(trainer); @@ -66,7 +89,7 @@ public void LightGBMBinaryEstimator() } /// - /// LightGBMBinaryTrainer CorrectSigmoid test + /// LightGBMBinaryTrainer CorrectSigmoid test /// [LightGBMFact] public void LightGBMBinaryEstimatorCorrectSigmoid() @@ -131,7 +154,7 @@ public void FastForestClassificationEstimator() } /// - /// FastTreeRankingTrainer TrainerEstimator test + /// FastTreeRankingTrainer TrainerEstimator test /// [Fact] public void FastTreeRankerEstimator() @@ -155,7 +178,7 @@ public void FastTreeRankerEstimator() } /// - /// LightGbmRankingTrainer TrainerEstimator test + /// LightGbmRankingTrainer TrainerEstimator test /// [LightGBMFact] public void LightGBMRankerEstimator() @@ -173,7 +196,7 @@ public void LightGBMRankerEstimator() } /// - /// FastTreeRegressor TrainerEstimator test + /// FastTreeRegressor TrainerEstimator test /// [Fact] public void FastTreeRegressorEstimator() @@ -188,7 +211,7 @@ public void FastTreeRegressorEstimator() } /// - /// LightGbmRegressionTrainer TrainerEstimator test + /// LightGbmRegressionTrainer TrainerEstimator test /// [LightGBMFact] public void LightGBMRegressorEstimator() @@ -208,7 +231,7 @@ public void LightGBMRegressorEstimator() /// - /// RegressionGamTrainer TrainerEstimator test + /// RegressionGamTrainer TrainerEstimator test /// [Fact] public void GAMRegressorEstimator() @@ -226,7 +249,7 @@ public void GAMRegressorEstimator() } /// - /// FastTreeTweedieTrainer TrainerEstimator test + /// FastTreeTweedieTrainer TrainerEstimator test /// [Fact] public void TweedieRegressorEstimator() @@ -245,7 +268,7 @@ public void TweedieRegressorEstimator() } /// - /// FastForestRegression TrainerEstimator test + /// FastForestRegression TrainerEstimator test /// [Fact] public void FastForestRegressorEstimator() @@ -264,7 +287,7 @@ public void FastForestRegressorEstimator() } /// - /// LightGbmMulticlass TrainerEstimator test + /// LightGbmMulticlass TrainerEstimator test /// [LightGBMFact] public void LightGbmMulticlassEstimator() @@ -297,7 +320,7 @@ public void LightGbmMulticlassEstimatorWithOptions() } /// - /// LightGbmMulticlass CorrectSigmoid test + /// LightGbmMulticlass CorrectSigmoid test /// [LightGBMFact] public void LightGbmMulticlassEstimatorCorrectSigmoid() @@ -322,6 +345,44 @@ public void LightGbmMulticlassEstimatorCorrectSigmoid() Done(); } + /// + /// LightGbmMulticlass Test of Balanced Data + /// + [LightGBMFact] + public void LightGbmMulticlassEstimatorBalanced() + { + var (pipeline, dataView) = GetMulticlassPipeline(); + + var trainer = ML.MulticlassClassification.Trainers.LightGbm(new LightGbmMulticlassTrainer.Options + { + UnbalancedSets = false + }); + + var pipe = pipeline.Append(trainer) + .Append(new KeyToValueMappingEstimator(Env, "PredictedLabel")); + TestEstimatorCore(pipe, dataView); + Done(); + } + + /// + /// LightGbmMulticlass Test of Unbalanced Data + /// + [LightGBMFact] + public void LightGbmMulticlassEstimatorUnbalanced() + { + var (pipeline, dataView) = GetMulticlassPipeline(); + + var trainer = ML.MulticlassClassification.Trainers.LightGbm(new LightGbmMulticlassTrainer.Options + { + UnbalancedSets = true + }); + + var pipe = pipeline.Append(trainer) + .Append(new KeyToValueMappingEstimator(Env, "PredictedLabel")); + TestEstimatorCore(pipe, dataView); + Done(); + } + // Number of examples private const int _rowNumber = 1000; // Number of features @@ -338,7 +399,7 @@ private class GbmExample public float[] Score; } - private void LightGbmHelper(bool useSoftmax, double sigmoid, out string modelString, out List mlnetPredictions, out double[] lgbmRawScores, out double[] lgbmProbabilities) + private void LightGbmHelper(bool useSoftmax, double sigmoid, out string modelString, out List mlnetPredictions, out double[] lgbmRawScores, out double[] lgbmProbabilities, bool unbalancedSets = false) { // Prepare data and train LightGBM model via ML.NET // Training matrix. It contains all feature vectors. @@ -365,14 +426,15 @@ private void LightGbmHelper(bool useSoftmax, double sigmoid, out string modelStr var mlContext = new MLContext(seed: 0); var dataView = mlContext.Data.LoadFromEnumerable(dataList); int numberOfTrainingIterations = 3; - var gbmTrainer = new LightGbmMulticlassTrainer(mlContext, + var gbmTrainer = new LightGbmMulticlassTrainer(mlContext, new LightGbmMulticlassTrainer.Options { NumberOfIterations = numberOfTrainingIterations, MinimumExampleCountPerGroup = 1, MinimumExampleCountPerLeaf = 1, UseSoftmax = useSoftmax, - Sigmoid = sigmoid // Custom sigmoid value. + Sigmoid = sigmoid, // Custom sigmoid value. + UnbalancedSets = unbalancedSets // false by default }); var gbm = gbmTrainer.Fit(dataView); @@ -583,6 +645,35 @@ public void LightGbmMulticlassEstimatorCompareSoftMax() Done(); } + [LightGBMFact] + public void LightGbmMulticlassEstimatorCompareUnbalanced() + { + // Train ML.NET LightGBM and native LightGBM and apply the trained models to the training set. + LightGbmHelper(useSoftmax: true, sigmoid: .5, out string modelString, out List mlnetPredictions, out double[] nativeResult1, out double[] nativeResult0, unbalancedSets:true); + + // The i-th predictor returned by LightGBM produces the raw score, denoted by z_i, of the i-th class. + // Assume that we have n classes in total. The i-th class probability can be computed via + // p_i = exp(z_i) / (exp(z_1) + ... + exp(z_n)). + Assert.True(modelString != null); + // Compare native LightGBM's and ML.NET's LightGBM results example by example + for (int i = 0; i < _rowNumber; ++i) + { + double sum = 0; + for (int j = 0; j < _classNumber; ++j) + { + Assert.Equal(nativeResult0[j + i * _classNumber], mlnetPredictions[i].Score[j], 6); + sum += Math.Exp((float)nativeResult1[j + i * _classNumber]); + } + for (int j = 0; j < _classNumber; ++j) + { + double prob = Math.Exp(nativeResult1[j + i * _classNumber]); + Assert.Equal(prob / sum, mlnetPredictions[i].Score[j], 6); + } + } + + Done(); + } + [LightGBMFact] public void LightGbmInDifferentCulture() { @@ -597,5 +688,247 @@ public void LightGbmInDifferentCulture() Assert.True(metrics.MacroAccuracy > 0.8); Thread.CurrentThread.CurrentCulture = currentCulture; } + + private class SummaryDataRow + { + public double Bias { get; set; } + public double TreeWeights { get; set; } + public int TreeID { get; set; } + public string IsLeaf { get; set; } + public int LeftChild { get; set; } + public int RightChild { get; set; } + public int NumericalSplitFeatureIndexes { get; set; } + public float NumericalSplitThresholds { get; set; } + public bool CategoricalSplitFlags { get; set; } + public double LeafValues { get; set; } + public double SplitGains { get; set; } + [VectorType(0)] + public int[] CategoricalSplitFeatures { get; set; } + [VectorType(0)] + public int[] CategoricalCategoricalSplitFeatureRange { get; set; } + } + + private class QuantileTestSummaryDataRow : SummaryDataRow + { + [VectorType(0)] + public double[] LeafSamples { get; set; } + [VectorType(0)] + public double[] LeafSampleWeights { get; set; } + } + + private static void CheckSummaryRowTreeNode(SummaryDataRow row, int treeIndex, double bias, double treeWeight, RegressionTreeBase tree, int nodeId) + { + Assert.Equal(row.TreeID, treeIndex); + Assert.Equal(row.Bias, bias); + Assert.Equal(row.TreeWeights, treeWeight); + Assert.Equal("Tree node", row.IsLeaf); + Assert.Equal(row.LeftChild, tree.LeftChild[nodeId]); + Assert.Equal(row.RightChild, tree.RightChild[nodeId]); + Assert.Equal(row.NumericalSplitFeatureIndexes, tree.NumericalSplitFeatureIndexes[nodeId]); + Assert.Equal(row.NumericalSplitThresholds, tree.NumericalSplitThresholds[nodeId]); + Assert.Equal(row.CategoricalSplitFlags, tree.CategoricalSplitFlags[nodeId]); + Assert.Equal(0, row.LeafValues); + Assert.Equal(row.SplitGains, tree.SplitGains[nodeId]); + if(tree.GetCategoricalSplitFeaturesAt(nodeId).Count() > 0) + Assert.Equal(row.CategoricalSplitFeatures, tree.GetCategoricalSplitFeaturesAt(nodeId).ToArray()); + else + Assert.Null(row.CategoricalSplitFeatures); + if (tree.GetCategoricalCategoricalSplitFeatureRangeAt(nodeId).Count() > 0) + Assert.Equal(row.CategoricalCategoricalSplitFeatureRange, tree.GetCategoricalCategoricalSplitFeatureRangeAt(nodeId).ToArray()); + else + Assert.Null(row.CategoricalCategoricalSplitFeatureRange); + } + + private static void CheckSummaryRowLeafNode(SummaryDataRow row, int treeIndex, double bias, double treeWeight, RegressionTreeBase tree, int nodeId) + { + Assert.Equal(row.TreeID, treeIndex); + Assert.Equal(row.Bias, bias); + Assert.Equal(row.TreeWeights, treeWeight); + Assert.Equal("Leaf node", row.IsLeaf); + Assert.Equal(0, row.LeftChild); + Assert.Equal(0, row.RightChild); + Assert.Equal(0, row.NumericalSplitFeatureIndexes); + Assert.Equal(0, row.NumericalSplitThresholds); + Assert.False(row.CategoricalSplitFlags); + Assert.Equal(tree.LeafValues[nodeId], row.LeafValues); + Assert.Equal(0d, row.SplitGains); + Assert.Null(row.CategoricalSplitFeatures); + Assert.Null(row.CategoricalCategoricalSplitFeatureRange); + } + + private static void CheckSummaryRowLeafNodeQuantileTree(QuantileTestSummaryDataRow row, int treeIndex, double bias, double treeWeight, QuantileRegressionTree tree, int nodeId) + { + if (tree.GetLeafSamplesAt(nodeId).Count() > 0) + Assert.Equal(row.LeafSamples, tree.GetLeafSamplesAt(nodeId).ToArray()); + else + Assert.Null(row.LeafSamples); + if (tree.GetLeafSampleWeightsAt(nodeId).Count() > 0) + Assert.Equal(row.LeafSampleWeights, tree.GetLeafSampleWeightsAt(nodeId).ToArray()); + else + Assert.Null(row.LeafSampleWeights); + } + + private void CheckSummary(ICanGetSummaryAsIDataView modelParameters, double bias, IReadOnlyList treeWeights, IReadOnlyList trees) + { + var quantileTrees = trees as IReadOnlyList; + var summaryDataView = modelParameters.GetSummaryDataView(null); + IEnumerable summaryDataEnumerable; + + if (quantileTrees == null) + summaryDataEnumerable = ML.Data.CreateEnumerable(summaryDataView, false); + else + summaryDataEnumerable = ML.Data.CreateEnumerable(summaryDataView, false); + + var summaryDataEnumerator = summaryDataEnumerable.GetEnumerator(); + + for (int i = 0; i < trees.Count(); i++) + { + for (int j = 0; j < trees[i].NumberOfNodes; j++) + { + Assert.True(summaryDataEnumerator.MoveNext()); + var row = summaryDataEnumerator.Current; + CheckSummaryRowTreeNode(row, i, bias, treeWeights[i], trees[i], j); + } + + for (int j = 0; j < trees[i].NumberOfLeaves; j++) + { + Assert.True(summaryDataEnumerator.MoveNext()); + var row = summaryDataEnumerator.Current; + CheckSummaryRowLeafNode(row, i, bias, treeWeights[i], trees[i], j); + if (quantileTrees != null) + { + var quantileRow = row as QuantileTestSummaryDataRow; + Assert.NotNull(quantileRow); + CheckSummaryRowLeafNodeQuantileTree(quantileRow, i, bias, treeWeights[i], quantileTrees[i], j); + } + } + } + } + + [Fact] + public void FastTreeRegressorTestSummary() + { + var dataView = GetRegressionPipeline(); + var trainer = ML.Regression.Trainers.FastTree( + new FastTreeRegressionTrainer.Options { NumberOfTrees = 10, NumberOfThreads = 1, NumberOfLeaves = 5}); + + var transformer = trainer.Fit(dataView); + + var trainedTreeEnsemble = transformer.Model.TrainedTreeEnsemble; + + var modelParameters = transformer.Model as ICanGetSummaryAsIDataView; + Assert.NotNull(modelParameters); + + CheckSummary(modelParameters, trainedTreeEnsemble.Bias, trainedTreeEnsemble.TreeWeights, trainedTreeEnsemble.Trees); + Done(); + } + + [Fact] + public void FastForestRegressorTestSummary() + { + var dataView = GetRegressionPipeline(); + var trainer = ML.Regression.Trainers.FastForest( + new FastForestRegressionTrainer.Options { NumberOfTrees = 10, NumberOfThreads = 1, NumberOfLeaves = 5}); + + var transformer = trainer.Fit(dataView); + + var trainedTreeEnsemble = transformer.Model.TrainedTreeEnsemble; + + var modelParameters = transformer.Model as ICanGetSummaryAsIDataView; + Assert.NotNull(modelParameters); + + CheckSummary(modelParameters, trainedTreeEnsemble.Bias, trainedTreeEnsemble.TreeWeights, trainedTreeEnsemble.Trees); + Done(); + } + + [Fact] + public void FastTreeTweedieRegressorTestSummary() + { + var dataView = GetRegressionPipeline(); + var trainer = ML.Regression.Trainers.FastTreeTweedie( + new FastTreeTweedieTrainer.Options { NumberOfTrees = 10, NumberOfThreads = 1, NumberOfLeaves = 5}); + + var transformer = trainer.Fit(dataView); + + var trainedTreeEnsemble = transformer.Model.TrainedTreeEnsemble; + + var modelParameters = transformer.Model as ICanGetSummaryAsIDataView; + Assert.NotNull(modelParameters); + + CheckSummary(modelParameters, trainedTreeEnsemble.Bias, trainedTreeEnsemble.TreeWeights, trainedTreeEnsemble.Trees); + Done(); + } + + [LightGBMFact] + public void LightGbmRegressorTestSummary() + { + var dataView = GetRegressionPipeline(); + var trainer = ML.Regression.Trainers.LightGbm( + new LightGbmRegressionTrainer.Options { NumberOfIterations = 10, NumberOfThreads = 1, NumberOfLeaves = 5}); + + var transformer = trainer.Fit(dataView); + + var trainedTreeEnsemble = transformer.Model.TrainedTreeEnsemble; + + var modelParameters = transformer.Model as ICanGetSummaryAsIDataView; + Assert.NotNull(modelParameters); + + CheckSummary(modelParameters, trainedTreeEnsemble.Bias, trainedTreeEnsemble.TreeWeights, trainedTreeEnsemble.Trees); + Done(); + } + + [Fact] + public void FastTreeBinaryClassificationTestSummary() + { + var (pipeline, dataView) = GetBinaryClassificationPipeline(); + var estimator = pipeline.Append(ML.BinaryClassification.Trainers.FastTree( + new FastTreeBinaryTrainer.Options { NumberOfTrees = 2, NumberOfThreads = 1, NumberOfLeaves = 5})); + + var transformer = estimator.Fit(dataView); + + var trainedTreeEnsemble = transformer.LastTransformer.Model.SubModel.TrainedTreeEnsemble; + + var modelParameters = transformer.LastTransformer.Model.SubModel as ICanGetSummaryAsIDataView; + Assert.NotNull(modelParameters); + + CheckSummary(modelParameters, trainedTreeEnsemble.Bias, trainedTreeEnsemble.TreeWeights, trainedTreeEnsemble.Trees); + Done(); + } + + [Fact] + public void FastForestBinaryClassificationTestSummary() + { + var (pipeline, dataView) = GetOneHotBinaryClassificationPipeline(); + var estimator = pipeline.Append(ML.BinaryClassification.Trainers.FastForest( + new FastForestBinaryTrainer.Options { NumberOfTrees = 2, NumberOfThreads = 1, NumberOfLeaves = 4, CategoricalSplit = true })); + + var transformer = estimator.Fit(dataView); + + var trainedTreeEnsemble = transformer.LastTransformer.Model.TrainedTreeEnsemble; + + var modelParameters = transformer.LastTransformer.Model as ICanGetSummaryAsIDataView; + Assert.NotNull(modelParameters); + + CheckSummary(modelParameters, trainedTreeEnsemble.Bias, trainedTreeEnsemble.TreeWeights, trainedTreeEnsemble.Trees); + Done(); + } + + [LightGBMFact] + public void LightGbmBinaryClassificationTestSummary() + { + var (pipeline, dataView) = GetOneHotBinaryClassificationPipeline(); + var trainer = pipeline.Append(ML.BinaryClassification.Trainers.LightGbm( + new LightGbmBinaryTrainer.Options { NumberOfIterations = 10, NumberOfThreads = 1, NumberOfLeaves = 5, UseCategoricalSplit = true })); + + var transformer = trainer.Fit(dataView); + + var trainedTreeEnsemble = transformer.LastTransformer.Model.SubModel.TrainedTreeEnsemble; + + var modelParameters = transformer.LastTransformer.Model.SubModel as ICanGetSummaryAsIDataView; + Assert.NotNull(modelParameters); + + CheckSummary(modelParameters, trainedTreeEnsemble.Bias, trainedTreeEnsemble.TreeWeights, trainedTreeEnsemble.Trees); + Done(); + } } } diff --git a/test/Microsoft.ML.Tests/Transformers/NormalizerTests.cs b/test/Microsoft.ML.Tests/Transformers/NormalizerTests.cs index ed26cc0b06..b5c2ce13e6 100644 --- a/test/Microsoft.ML.Tests/Transformers/NormalizerTests.cs +++ b/test/Microsoft.ML.Tests/Transformers/NormalizerTests.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information. using System; +using System.Collections.Generic; using System.Collections.Immutable; using System.IO; using Microsoft.ML; @@ -17,6 +18,7 @@ using Microsoft.ML.Transforms; using Xunit; using Xunit.Abstractions; +using static Microsoft.ML.Transforms.NormalizingTransformer; namespace Microsoft.ML.Tests.Transformers { @@ -492,8 +494,8 @@ public void NormalizerExperimentalExtensions() // Normalizer Extensions var est1 = ML.Transforms.NormalizeMinMax("float4", "float4"); var est2 = ML.Transforms.NormalizeMeanVariance("float4", "float4"); - var est3 = ML.Transforms.NormalizeLogMeanVariance("float4", "float4"); - var est4 = ML.Transforms.NormalizeBinning("float4", "float4"); + var est3 = ML.Transforms.NormalizeLogMeanVariance("float4", "float4"); + var est4 = ML.Transforms.NormalizeBinning("float4", "float4"); var est5 = ML.Transforms.NormalizeSupervisedBinning("float4", "float4"); // Normalizer Extensions (Experimental) @@ -503,7 +505,7 @@ public void NormalizerExperimentalExtensions() var est9 = ML.Transforms.NormalizeBinning("float4", "float4"); var est10 = ML.Transforms.NormalizeSupervisedBinning("float4", "float4"); - // Fit and Transpose + // Fit and Transpose var data1 = est1.Fit(data).Transform(data); var data2 = est2.Fit(data).Transform(data); var data3 = est3.Fit(data).Transform(data); @@ -782,5 +784,105 @@ void TestNormalizeBackCompatibility() Assert.Equal(3, result.Schema.Count); } } + + private sealed class DataPointVec + { + [VectorType(5)] + public float[] Features { get; set; } + } + + private sealed class DataPointOne + { + public float Features { get; set; } + } + + [Fact] + void TestNormalizeLogMeanVarianceFixZeroOne() + { + var samples = new List() + { + new DataPointOne(){ Features = 1f }, + new DataPointOne(){ Features = 2f }, + new DataPointOne(){ Features = 0f }, + new DataPointOne(){ Features = -1 } + }; + // Convert training data to IDataView, the general data type used in ML.NET. + var data = ML.Data.LoadFromEnumerable(samples); + // NormalizeLogMeanVariance normalizes the data based on the computed mean and variance of the logarithm of the data. + // Uses Cumulative distribution function as output. + var normalize = ML.Transforms.NormalizeLogMeanVariance("Features", true, useCdf: true); + + // NormalizeLogMeanVariance normalizes the data based on the computed mean and variance of the logarithm of the data. + var normalizeNoCdf = ML.Transforms.NormalizeLogMeanVariance("Features", true, useCdf: false); + + // Now we can transform the data and look at the output to confirm the behavior of the estimator. + var normalizeTransform = normalize.Fit(data); + var transformedData = normalizeTransform.Transform(data); + var normalizeNoCdfTransform = normalizeNoCdf.Fit(data); + var noCdfData = normalizeNoCdfTransform.Transform(data); + + var transformParams = normalizeTransform.GetNormalizerModelParameters(0) as CdfNormalizerModelParameters; + var noCdfParams = normalizeNoCdfTransform.GetNormalizerModelParameters(0) as AffineNormalizerModelParameters; + + // Standard deviation and offset should not be zero for the given data even when FixZero is set to true. + Assert.NotEqual(0f, transformParams.Mean); + Assert.NotEqual(0f, transformParams.StandardDeviation); + + // Offset should be zero when FixZero is set to true but not the scale (on this data). + Assert.Equal(0f, noCdfParams.Offset); + Assert.NotEqual(0f, noCdfParams.Scale); + + var transformedDataArray = ML.Data.CreateEnumerable(noCdfData, false).ToImmutableArray(); + // Without the Cdf and fixing zero, any 0 should stay 0. + Assert.Equal(0f, transformedDataArray[2].Features); + } + + [Fact] + void TestNormalizeLogMeanVarianceFixZeroVec() + { + var samples = new List() + { + new DataPointVec(){ Features = new float[5] { 1, 1, 3, 0, float.MaxValue } }, + new DataPointVec(){ Features = new float[5] { 2, 2, 2, 0, float.MinValue } }, + new DataPointVec(){ Features = new float[5] { 0, 0, 1, 0.5f, 0} }, + new DataPointVec(){ Features = new float[5] {-1,-1,-1, 1, 1} } + }; + // Convert training data to IDataView, the general data type used in ML.NET. + var data = ML.Data.LoadFromEnumerable(samples); + // NormalizeLogMeanVariance normalizes the data based on the computed mean and variance of the logarithm of the data. + // Uses Cumulative distribution function as output. + var normalize = ML.Transforms.NormalizeLogMeanVariance("Features", true, useCdf: true); + + // NormalizeLogMeanVariance normalizes the data based on the computed mean and variance of the logarithm of the data. + var normalizeNoCdf = ML.Transforms.NormalizeLogMeanVariance("Features", true, useCdf: false); + + // Now we can transform the data and look at the output to confirm the behavior of the estimator. + var normalizeTransform = normalize.Fit(data); + var transformedData = normalizeTransform.Transform(data); + var normalizeNoCdfTransform = normalizeNoCdf.Fit(data); + var noCdfData = normalizeNoCdfTransform.Transform(data); + + var transformParams = normalizeTransform.GetNormalizerModelParameters(0) as CdfNormalizerModelParameters>; + var noCdfParams = normalizeNoCdfTransform.GetNormalizerModelParameters(0) as AffineNormalizerModelParameters>; + + for (int i = 0; i < 5; i++) + { + // Standard deviation and offset should not be zero for the given data even when FixZero is set to true. + Assert.NotEqual(0f, transformParams.Mean[i]); + Assert.NotEqual(0f, transformParams.StandardDeviation[i]); + + // Offset should be zero when FixZero is set to true but not the scale (on this data). + Assert.Empty(noCdfParams.Offset); + Assert.NotEqual(0f, noCdfParams.Scale[i]); + } + + var transformedDataArray = ML.Data.CreateEnumerable(noCdfData, false).ToImmutableArray(); + // Without the Cdf and fixing zero, any 0 should stay 0. + Assert.Equal(0f, transformedDataArray[0].Features[3]); + Assert.Equal(0f, transformedDataArray[1].Features[3]); + Assert.Equal(0f, transformedDataArray[2].Features[0]); + Assert.Equal(0f, transformedDataArray[2].Features[1]); + Assert.Equal(0f, transformedDataArray[2].Features[4]); + } } } diff --git a/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs b/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs index dd25deb016..85bac94fff 100644 --- a/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs +++ b/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs @@ -2,12 +2,10 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -using System; using System.Collections.Generic; using System.IO; using Microsoft.ML.Data; using Microsoft.ML.TestFramework.Attributes; -using Microsoft.ML.TimeSeries; using Microsoft.ML.Transforms.TimeSeries; using Xunit; @@ -24,6 +22,18 @@ private sealed class Prediction #pragma warning restore CS0649 } + private sealed class ForecastPrediction + { +#pragma warning disable CS0649 + [VectorType(4)] + public float[] Forecast; + [VectorType(4)] + public float[] MinCnf; + [VectorType(4)] + public float[] MaxCnf; +#pragma warning restore CS0649 + } + public class Prediction1 { public float Random; @@ -43,6 +53,22 @@ public Data(float value) } } + class ForecastResult + { +#pragma warning disable CS0649 + public float Forecast; +#pragma warning restore CS0649 + } + + class ForecastResultArray + { +#pragma warning disable CS0649 + public float[] Forecast; + public float[] ConfidenceLowerBound; + public float[] ConfidenceUpperBound; +#pragma warning restore CS0649 + } + private sealed class TimeSeriesData { public float Value; @@ -187,7 +213,7 @@ public void ChangePointDetectionWithSeasonalityPredictionEngineNoColumn() var model = pipeline.Fit(dataView); //Create prediction function. - var engine = model.CreateTimeSeriesPredictionFunction(ml); + var engine = model.CreateTimeSeriesEngine(ml); //Checkpoint with no inputs passed at prediction. var modelPath = "temp.zip"; @@ -200,7 +226,7 @@ public void ChangePointDetectionWithSeasonalityPredictionEngineNoColumn() model2 = ml.Model.Load(file, out var schema); //Raw score after state gets updated with two inputs. - var engine2 = model2.CreateTimeSeriesPredictionFunction(ml); + var engine2 = model2.CreateTimeSeriesEngine(ml); var prediction2 = engine2.Predict(new Data(1)); //Raw score after first input. Assert.Equal(1.1661833524703979, prediction2.Change[1], precision: 5); // Raw score @@ -221,7 +247,7 @@ public void ChangePointDetectionWithSeasonalityPredictionEngineNoColumn() //Load the model with state updated with just one input, then pass in the second input //and raw score should match the raw score obtained by passing the two input in the first model. - var engine3 = model3.CreateTimeSeriesPredictionFunction(ml); + var engine3 = model3.CreateTimeSeriesEngine(ml); var prediction3 = engine3.Predict(new Data(1)); Assert.Equal(0.12216401100158691, prediction2.Change[1], precision: 5); // Raw score } @@ -262,9 +288,9 @@ public void ChangePointDetectionWithSeasonalityPredictionEngine() // Train. var model = pipeline.Fit(dataView); - + //Model 1: Prediction #1. - var engine = model.CreateTimeSeriesPredictionFunction(ml); + var engine = model.CreateTimeSeriesEngine(ml); var prediction = engine.Predict(new Data(1)); Assert.Equal(0, prediction.Change[0], precision: 7); // Alert Assert.Equal(1.1661833524703979, prediction.Change[1], precision: 5); // Raw score @@ -288,7 +314,7 @@ public void ChangePointDetectionWithSeasonalityPredictionEngine() model2 = ml.Model.Load(file, out var schema); //Predict and expect the same result after checkpointing(Prediction #2). - engine = model2.CreateTimeSeriesPredictionFunction(ml); + engine = model2.CreateTimeSeriesEngine(ml); prediction = engine.Predict(new Data(1)); Assert.Equal(0, prediction.Change[0], precision: 7); // Alert Assert.Equal(0.12216401100158691, prediction.Change[1], precision: 5); // Raw score @@ -296,6 +322,150 @@ public void ChangePointDetectionWithSeasonalityPredictionEngine() Assert.Equal(1.5292508189989167E-07, prediction.Change[3], precision: 5); // Martingale score } + [LessThanNetCore30OrNotNetCoreFact("netcoreapp3.0 output differs from Baseline")] + public void SsaForecast() + { + var env = new MLContext(); + const int ChangeHistorySize = 10; + const int SeasonalitySize = 10; + const int NumberOfSeasonsInTraining = 5; + + List data = new List(); + var dataView = env.Data.LoadFromEnumerable(data); + + var args = new SsaForecastingTransformer.Options() + { + ConfidenceLevel = 0.95f, + Source = "Value", + Name = "Forecast", + ConfidenceLowerBoundColumn = "MinCnf", + ConfidenceUpperBoundColumn = "MaxCnf", + WindowSize = 10, + SeriesLength = 11, + TrainSize = 22, + Horizon = 4, + IsAdaptive = true + }; + + for (int j = 0; j < NumberOfSeasonsInTraining; j++) + for (int i = 0; i < SeasonalitySize; i++) + data.Add(new Data(i)); + + for (int i = 0; i < ChangeHistorySize; i++) + data.Add(new Data(i * 100)); + + // Train + var detector = new SsaForecastingEstimator(env, args).Fit(dataView); + // Transform + var output = detector.Transform(dataView); + // Get predictions + var enumerator = env.Data.CreateEnumerable(output, true).GetEnumerator(); + ForecastPrediction row = null; + List expectedForecast = new List() { 0.191491723f, 2.53994083f, 5.26454258f, 7.37313938f }; + List minCnf = new List() { -3.9741993f, -2.36872721f, 0.09407653f, 2.18899345f }; + List maxCnf = new List() { 4.3571825f, 7.448609f, 10.435009f, 12.5572853f }; + enumerator.MoveNext(); + row = enumerator.Current; + + for (int localIndex = 0; localIndex < 4; localIndex++) + { + Assert.Equal(expectedForecast[localIndex], row.Forecast[localIndex], precision: 7); + Assert.Equal(minCnf[localIndex], row.MinCnf[localIndex], precision: 7); + Assert.Equal(maxCnf[localIndex], row.MaxCnf[localIndex], precision: 7); + } + + } + + [LessThanNetCore30OrNotNetCoreFact("netcoreapp3.0 output differs from Baseline")] + public void SsaForecastPredictionEngine() + { + const int ChangeHistorySize = 10; + const int SeasonalitySize = 10; + const int NumberOfSeasonsInTraining = 5; + + List data = new List(); + + var ml = new MLContext(seed: 1); + var dataView = ml.Data.LoadFromEnumerable(data); + + var args = new SsaForecastingTransformer.Options() + { + ConfidenceLevel = 0.95f, + Source = "Value", + Name = "Forecast", + WindowSize = 10, + SeriesLength = 11, + TrainSize = 22, + Horizon = 4, + ConfidenceLowerBoundColumn = "ConfidenceLowerBound", + ConfidenceUpperBoundColumn = "ConfidenceUpperBound", + VariableHorizon = true + }; + + for (int j = 0; j < NumberOfSeasonsInTraining; j++) + for (int i = 0; i < SeasonalitySize; i++) + data.Add(new Data(i)); + + for (int i = 0; i < ChangeHistorySize; i++) + data.Add(new Data(i * 100)); + + // Train + var model = ml.Transforms.Text.FeaturizeText("Text_Featurized", "Text") + .Append(ml.Transforms.Conversion.ConvertType("Value", "Value", DataKind.Single)) + .Append(ml.Forecasting.ForecastBySsa("Forecast", "Value", 10, 11, 22, 4, + confidenceLowerBoundColumn: "ConfidenceLowerBound", + confidenceUpperBoundColumn: "ConfidenceUpperBound", variableHorizon: true)) + .Append(ml.Transforms.Concatenate("Forecast", "Forecast", "ConfidenceLowerBound", "ConfidenceUpperBound")) + .Fit(dataView); + + //Prediction engine. + var engine = model.CreateTimeSeriesEngine(ml); + ForecastResultArray result = new ForecastResultArray(); + + // Forecast and change the horizon to 5. + engine.Predict(null, ref result, horizon: 5); + // [Forecast, ConfidenceLowerBound, ConfidenceUpperBound] + Assert.Equal(result.Forecast, new float[] { -1.02245092f, 0.08333081f, 2.60737085f, 5.397319f, 7.500832f, -5.188142f, -4.82533741f, + -2.563095f, 0.213172823f, 2.29317045f, 3.14324f, 4.991999f, 7.777837f, 10.5814648f, 12.7084932f }); + + // Update the forecasting model. + engine.Predict(new Data(2)); + + // Update the model and then forecast. + engine.Predict(new Data(2), ref result); + + engine.CheckPoint(ml, "model.zip"); + // [Forecast, ConfidenceLowerBound, ConfidenceUpperBound] + Assert.Equal(result.Forecast, new float[] { 4.310587f, 6.39716768f, 7.73934f, 8.029469f, 0.144895911f, + 1.48849952f, 2.568874f, 2.84532261f, 8.476278f, 11.3058357f, 12.9098063f, 13.2136145f }); + + // Checkpoint the model. + ITransformer modelCopy; + using (var file = File.OpenRead("model.zip")) + modelCopy = ml.Model.Load(file, out DataViewSchema schema); + + // We must create a new prediction engine from the persisted model. + var forecastEngineCopy = modelCopy.CreateTimeSeriesEngine(ml); + ForecastResultArray resultCopy = new ForecastResultArray(); + + // Update both the models. + engine.Predict(new Data(3)); + forecastEngineCopy.Predict(new Data(3)); + + // Forecast values with the original and check-pointed model. + forecastEngineCopy.Predict(null, ref resultCopy, horizon: 5); + engine.Predict(null, ref result, horizon: 5); + // [Forecast, ConfidenceLowerBound, ConfidenceUpperBound] + Assert.Equal(result.Forecast, new float[] { 6.00658846f, 7.506871f, 7.96424866f, 7.17514229f, + 5.02655172f, 1.84089744f, 2.59820318f, 2.79378271f, 1.99099624f, + -0.181109816f, 10.1722794f, 12.41554f, 13.1347151f, 12.3592882f, 10.2342129f}); + + // The forecasted results should be the same because the state of the models + // is the same. + Assert.Equal(result.Forecast, resultCopy.Forecast); + + } + [Fact] public void AnomalyDetectionWithSrCnn() { @@ -336,103 +506,5 @@ public void AnomalyDetectionWithSrCnn() k += 1; } } - - [Fact] - public void Forecasting() - { - const int SeasonalitySize = 10; - const int NumberOfSeasonsInTraining = 5; - - List data = new List(); - - var ml = new MLContext(seed: 1); - var dataView = ml.Data.LoadFromEnumerable(data); - - for (int j = 0; j < NumberOfSeasonsInTraining; j++) - for (int i = 0; i < SeasonalitySize; i++) - data.Add(new Data(i)); - - // Create forecasting model. - var model = ml.Forecasting.AdaptiveSingularSpectrumSequenceModeler("Value", data.Count, SeasonalitySize + 1, SeasonalitySize, - 1, AdaptiveSingularSpectrumSequenceModeler.RankSelectionMethod.Exact, null, SeasonalitySize / 2, false, false); - - // Train. - model.Train(dataView); - - // Forecast. - var forecast = model.Forecast(5); - - // Update with new observations. - model.Update(dataView); - - // Checkpoint. - ml.Model.SaveForecastingModel(model, "model.zip"); - - // Load the checkpointed model from disk. - var modelCopy = ml.Model.LoadForecastingModel("model.zip"); - - // Forecast with the checkpointed model loaded from disk. - var forecastCopy = modelCopy.Forecast(5); - - // Forecast with the original model(that was checkpointed to disk). - forecast = model.Forecast(5); - - // Both the forecasted values from model loaded from disk and - // already in memory model should be the same. - Assert.Equal(forecast, forecastCopy); - } - - [Fact] - public void ForecastingWithConfidenceInterval() - { - const int SeasonalitySize = 10; - const int NumberOfSeasonsInTraining = 5; - - List data = new List(); - - var ml = new MLContext(seed: 1); - var dataView = ml.Data.LoadFromEnumerable(data); - - for (int j = 0; j < NumberOfSeasonsInTraining; j++) - for (int i = 0; i < SeasonalitySize; i++) - data.Add(new Data(i)); - - // Create forecasting model. - var model = ml.Forecasting.AdaptiveSingularSpectrumSequenceModeler("Value", data.Count, SeasonalitySize + 1, SeasonalitySize, - 1, AdaptiveSingularSpectrumSequenceModeler.RankSelectionMethod.Exact, null, SeasonalitySize / 2, shouldComputeForecastIntervals: true, false); - - // Train. - model.Train(dataView); - - // Forecast. - float[] forecast; - float[] lowConfInterval; - float[] upperConfInterval; - model.ForecastWithConfidenceIntervals(5, out forecast, out lowConfInterval, out upperConfInterval); - - // Update with new observations. - model.Update(dataView); - - // Checkpoint. - ml.Model.SaveForecastingModel(model, "model.zip"); - - // Load the checkpointed model from disk. - var modelCopy = ml.Model.LoadForecastingModel("model.zip"); - - // Forecast with the checkpointed model loaded from disk. - float[] forecastCopy; - float[] lowConfIntervalCopy; - float[] upperConfIntervalCopy; - modelCopy.ForecastWithConfidenceIntervals(5, out forecastCopy, out lowConfIntervalCopy, out upperConfIntervalCopy); - - // Forecast with the original model(that was checkpointed to disk). - model.ForecastWithConfidenceIntervals(5, out forecast, out lowConfInterval, out upperConfInterval); - - // Both the forecasted values from model loaded from disk and - // already in memory model should be the same. - Assert.Equal(forecast, forecastCopy); - Assert.Equal(lowConfInterval, lowConfIntervalCopy); - Assert.Equal(upperConfInterval, upperConfIntervalCopy); - } } } diff --git a/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesEstimatorTests.cs b/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesEstimatorTests.cs index 399b276194..d9fe04bccd 100644 --- a/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesEstimatorTests.cs +++ b/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesEstimatorTests.cs @@ -74,6 +74,42 @@ void TestSsaChangePointEstimator() Done(); } + [Fact] + void TestSsaForecastingEstimator() + { + const int ChangeHistorySize = 10; + const int SeasonalitySize = 10; + const int NumberOfSeasonsInTraining = 5; + + List data = new List(); + + var ml = new MLContext(seed: 1); + var dataView = ml.Data.LoadFromEnumerable(data); + + for (int j = 0; j < NumberOfSeasonsInTraining; j++) + for (int i = 0; i < SeasonalitySize; i++) + data.Add(new Data(i)); + + for (int i = 0; i < ChangeHistorySize; i++) + data.Add(new Data(i * 100)); + + // Train + var pipe = new SsaForecastingEstimator(Env, "Forecast", "Value", 10, 11, 22, 4, + confidenceLowerBoundColumn: "ConfidenceLowerBound", + confidenceUpperBoundColumn: "ConfidenceUpperBound"); + + var xyData = new List { new TestDataXY() { A = new float[inputSize] } }; + var stringData = new List { new TestDataDifferntType() { data_0 = new string[inputSize] } }; + + var invalidDataWrongNames = ML.Data.LoadFromEnumerable(xyData); + var invalidDataWrongTypes = ML.Data.LoadFromEnumerable(stringData); + + TestEstimatorCore(pipe, dataView, invalidInput: invalidDataWrongTypes); + TestEstimatorCore(pipe, dataView, invalidInput: invalidDataWrongNames); + + Done(); + } + [Fact] void TestSsaSpikeEstimator() { diff --git a/tools-local/Microsoft.ML.StableApi/Microsoft.ML.StableApi.csproj b/tools-local/Microsoft.ML.StableApi/Microsoft.ML.StableApi.csproj index 428b87b97b..121698cfea 100644 --- a/tools-local/Microsoft.ML.StableApi/Microsoft.ML.StableApi.csproj +++ b/tools-local/Microsoft.ML.StableApi/Microsoft.ML.StableApi.csproj @@ -7,13 +7,16 @@ - - - - - - - + + + + + + + + + +