From 2683243d57e2b33354dd0ca9ce367fc487b1a626 Mon Sep 17 00:00:00 2001 From: Gani Nazirov Date: Mon, 3 Jun 2019 12:31:53 -0700 Subject: [PATCH 01/17] package System.Drawings.Common.dll as its missing in dotnetcore2 --- build/libs_linux.txt | 1 + src/python/tests/test_docs_example.py | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/build/libs_linux.txt b/build/libs_linux.txt index c5e38f5a..a83aa1e3 100644 --- a/build/libs_linux.txt +++ b/build/libs_linux.txt @@ -8,4 +8,5 @@ libSymSgdNative.so lib_lightgbm.so libtensorflow.so libtensorflow_framework.so +System.Drawings.Common.dll Microsoft.ML.* diff --git a/src/python/tests/test_docs_example.py b/src/python/tests/test_docs_example.py index 310f83ce..69ad6360 100644 --- a/src/python/tests/test_docs_example.py +++ b/src/python/tests/test_docs_example.py @@ -59,8 +59,6 @@ def test_examples(self): 'CharTokenizer_df.py', # Bug todo: CustomStopWordsRemover fails on ML.NET side 'NGramFeaturizer2.py', - # System.Drawings.Common.dll 4.0.0 is needed - 'Image.py', 'Image_df.py', ]: continue if os.name != "nt": From 056ac1f6face45fd7ffbc10d407d7297c8a25d0f Mon Sep 17 00:00:00 2001 From: Gani Nazirov Date: Mon, 3 Jun 2019 12:49:21 -0700 Subject: [PATCH 02/17] typo --- build/libs_linux.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/libs_linux.txt b/build/libs_linux.txt index a83aa1e3..15c3395e 100644 --- a/build/libs_linux.txt +++ b/build/libs_linux.txt @@ -8,5 +8,5 @@ libSymSgdNative.so lib_lightgbm.so libtensorflow.so libtensorflow_framework.so -System.Drawings.Common.dll +System.Drawing.Common.dll Microsoft.ML.* From f4ec950183a8ade3f4fb021c708083ca6f4c1c92 Mon Sep 17 00:00:00 2001 From: Gani Nazirov Date: Mon, 3 Jun 2019 13:24:21 -0700 Subject: [PATCH 03/17] Add png for Image examples --- build/libs_mac.txt | 1 + build/libs_win.txt | 1 + src/python/MANIFEST.in | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/build/libs_mac.txt b/build/libs_mac.txt index efb3e632..2be6a809 100644 --- a/build/libs_mac.txt +++ b/build/libs_mac.txt @@ -8,4 +8,5 @@ libSymSgdNative.dylib lib_lightgbm.dylib libtensorflow.dylib libtensorflow_framework.dylib +System.Drawing.Common.dll Microsoft.ML.* diff --git a/build/libs_win.txt b/build/libs_win.txt index 3359f7cd..dda6dcd6 100644 --- a/build/libs_win.txt +++ b/build/libs_win.txt @@ -9,4 +9,5 @@ libiomp5md.dll MklImports.dll SymSgdNative.dll tensorflow.dll +System.Drawing.Common.dll Microsoft.ML.* diff --git a/src/python/MANIFEST.in b/src/python/MANIFEST.in index fd270a1a..1205728f 100644 --- a/src/python/MANIFEST.in +++ b/src/python/MANIFEST.in @@ -1,3 +1,3 @@ recursive-include nimbusml/internal/libs * -recursive-include nimbusml/datasets *.csv *.tsv +recursive-include nimbusml/datasets *.csv *.tsv *.png recursive-include nimbusml/examples *.py From 0a98a66aff191769899591145b82548bd02885c1 Mon Sep 17 00:00:00 2001 From: Gani Nazirov Date: Mon, 3 Jun 2019 14:33:00 -0700 Subject: [PATCH 04/17] try linux fix --- build.cmd | 1 - build.sh | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/build.cmd b/build.cmd index 8669ceff..2d66de5f 100644 --- a/build.cmd +++ b/build.cmd @@ -321,7 +321,6 @@ echo "#################################" call "%PythonExe%" -m pip install --upgrade nose pytest graphviz imageio pytest-cov "jupyter_client>=4.4.0" "nbconvert>=4.2.0" if %PythonVersion% == 2.7 ( call "%PythonExe%" -m pip install --upgrade pyzmq ) call "%PythonExe%" -m pip install --upgrade "%__currentScriptDir%target\%WheelFile%" -call "%PythonExe%" -m pip install "scikit-learn==0.19.2" set PackagePath=%PythonRoot%\Lib\site-packages\nimbusml set TestsPath1=%PackagePath%\tests diff --git a/build.sh b/build.sh index a4a57545..1a9a894e 100755 --- a/build.sh +++ b/build.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash set -e +export FONTCONFIG_PATH=/etc/fonts ProductVersion=$( Date: Mon, 3 Jun 2019 14:45:09 -0700 Subject: [PATCH 05/17] rollback scikit learn version --- build.cmd | 2 +- build.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/build.cmd b/build.cmd index 2d66de5f..120e656f 100644 --- a/build.cmd +++ b/build.cmd @@ -321,7 +321,7 @@ echo "#################################" call "%PythonExe%" -m pip install --upgrade nose pytest graphviz imageio pytest-cov "jupyter_client>=4.4.0" "nbconvert>=4.2.0" if %PythonVersion% == 2.7 ( call "%PythonExe%" -m pip install --upgrade pyzmq ) call "%PythonExe%" -m pip install --upgrade "%__currentScriptDir%target\%WheelFile%" - +call "%PythonExe%" -m pip install "scikit-learn==0.19.2" set PackagePath=%PythonRoot%\Lib\site-packages\nimbusml set TestsPath1=%PackagePath%\tests set TestsPath2=%__currentScriptDir%src\python\tests diff --git a/build.sh b/build.sh index 1a9a894e..e67793cb 100755 --- a/build.sh +++ b/build.sh @@ -261,6 +261,7 @@ then "${PythonExe}" -m pip install --upgrade pyzmq fi "${PythonExe}" -m pip install --upgrade "${Wheel}" + "${PythonExe}" -m pip install "scikit-learn==0.19.2" PackagePath=${PythonRoot}/lib/python${PythonVersion}/site-packages/nimbusml # TestsPath1=${PackagePath}/tests From 92278a44e7dedf5b636b397ec5de3f930fcfc413 Mon Sep 17 00:00:00 2001 From: Gani Nazirov Date: Mon, 3 Jun 2019 15:07:01 -0700 Subject: [PATCH 06/17] test --- src/python/tests/test_copyright.py | 1 + src/python/tests/test_docs_example.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/src/python/tests/test_copyright.py b/src/python/tests/test_copyright.py index 87dc0830..a695a306 100644 --- a/src/python/tests/test_copyright.py +++ b/src/python/tests/test_copyright.py @@ -12,6 +12,7 @@ class TestCopyright(unittest.TestCase): """ def test_copyrigth_present(self): + return root = os.path.join(os.path.abspath(os.path.dirname(__file__)), '..') root = os.path.normpath(root) diff --git a/src/python/tests/test_docs_example.py b/src/python/tests/test_docs_example.py index 69ad6360..0b51e6a1 100644 --- a/src/python/tests/test_docs_example.py +++ b/src/python/tests/test_docs_example.py @@ -53,6 +53,8 @@ def test_examples(self): for i, (fold, name) in enumerate(fold_files): if i <= start: continue + if name != 'Image.py': + continue if name in [ # Bug 294481: CharTokenizer_df fails # with error about variable length vector From d5a7ecd24e66f9657f6386e0ea5cdab7b4a420d4 Mon Sep 17 00:00:00 2001 From: Gani Nazirov Date: Mon, 3 Jun 2019 15:28:40 -0700 Subject: [PATCH 07/17] debug --- src/DotNetBridge/RunGraph.cs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/DotNetBridge/RunGraph.cs b/src/DotNetBridge/RunGraph.cs index 09617aa6..4b46849d 100644 --- a/src/DotNetBridge/RunGraph.cs +++ b/src/DotNetBridge/RunGraph.cs @@ -5,6 +5,7 @@ using System; using System.Collections.Generic; +using System.Drawing; using System.Globalization; using System.IO; using System.Linq; @@ -85,10 +86,25 @@ private static void SavePredictorModelToFile(PredictorModel model, string path, model.Save(host, fs); } + private static void OpenBitmap() + { + try + { + string path = "/home/ganik/sources/NimbusML/dependencies/Python3.7/bin/a.png"; + var dst = new Bitmap(path); + } + catch (Exception) + { + } + throw new Exception("Good"); + } + private static void RunGraphCore(EnvironmentBlock* penv, IHostEnvironment env, string graphStr, int cdata, DataSourceBlock** ppdata) { Contracts.AssertValue(env); + OpenBitmap(); + var args = new RunGraphArgs(); string err = null; if (!CmdParser.ParseArguments(env, graphStr, args, e => err = err ?? e)) From 3532fd5e193fc1549eb9805ce4617e3d50bf4bdd Mon Sep 17 00:00:00 2001 From: Gani Nazirov Date: Mon, 3 Jun 2019 15:48:49 -0700 Subject: [PATCH 08/17] rollback test --- src/DotNetBridge/RunGraph.cs | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/src/DotNetBridge/RunGraph.cs b/src/DotNetBridge/RunGraph.cs index 4b46849d..19a8d0ff 100644 --- a/src/DotNetBridge/RunGraph.cs +++ b/src/DotNetBridge/RunGraph.cs @@ -86,25 +86,10 @@ private static void SavePredictorModelToFile(PredictorModel model, string path, model.Save(host, fs); } - private static void OpenBitmap() - { - try - { - string path = "/home/ganik/sources/NimbusML/dependencies/Python3.7/bin/a.png"; - var dst = new Bitmap(path); - } - catch (Exception) - { - } - throw new Exception("Good"); - } - private static void RunGraphCore(EnvironmentBlock* penv, IHostEnvironment env, string graphStr, int cdata, DataSourceBlock** ppdata) { Contracts.AssertValue(env); - OpenBitmap(); - var args = new RunGraphArgs(); string err = null; if (!CmdParser.ParseArguments(env, graphStr, args, e => err = err ?? e)) From eafdab543f888de5f83972306f73c55767736739 Mon Sep 17 00:00:00 2001 From: Gani Nazirov Date: Mon, 3 Jun 2019 17:06:18 -0700 Subject: [PATCH 09/17] rollback --- build.cmd | 1 + src/DotNetBridge/RunGraph.cs | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/build.cmd b/build.cmd index 120e656f..8669ceff 100644 --- a/build.cmd +++ b/build.cmd @@ -322,6 +322,7 @@ call "%PythonExe%" -m pip install --upgrade nose pytest graphviz imageio pytest- if %PythonVersion% == 2.7 ( call "%PythonExe%" -m pip install --upgrade pyzmq ) call "%PythonExe%" -m pip install --upgrade "%__currentScriptDir%target\%WheelFile%" call "%PythonExe%" -m pip install "scikit-learn==0.19.2" + set PackagePath=%PythonRoot%\Lib\site-packages\nimbusml set TestsPath1=%PackagePath%\tests set TestsPath2=%__currentScriptDir%src\python\tests diff --git a/src/DotNetBridge/RunGraph.cs b/src/DotNetBridge/RunGraph.cs index 19a8d0ff..09617aa6 100644 --- a/src/DotNetBridge/RunGraph.cs +++ b/src/DotNetBridge/RunGraph.cs @@ -5,7 +5,6 @@ using System; using System.Collections.Generic; -using System.Drawing; using System.Globalization; using System.IO; using System.Linq; From e8dbe443e6d1232d24c785bc22156ac359c866f0 Mon Sep 17 00:00:00 2001 From: Gani Nazirov Date: Mon, 3 Jun 2019 19:34:57 -0700 Subject: [PATCH 10/17] fix fontconfig err --- src/python/nimbusml/examples/Image.py | 8 +++----- .../examples/examples_from_dataframe/Image_df.py | 9 +++------ 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/src/python/nimbusml/examples/Image.py b/src/python/nimbusml/examples/Image.py index 9d120568..08c6aa35 100644 --- a/src/python/nimbusml/examples/Image.py +++ b/src/python/nimbusml/examples/Image.py @@ -31,10 +31,8 @@ pipeline.fit(X, y) # predict -nimbusml_pred = pipeline.predict(X) -print("Predicted Labels : {0}".format(nimbusml_pred.PredictedLabel.values)) +scores = pipeline.predict(X) +print("Predicted Labels:", scores.PredictedLabel.values) # Predicted Labels : [True False] -print( - "Accuracy : {0}".format(np.mean( - y.Label.values == nimbusml_pred.PredictedLabel.values))) +print("Accuracy:", np.mean(y.Label.values == scores.PredictedLabel.values)) # Accuracy : 1 diff --git a/src/python/nimbusml/examples/examples_from_dataframe/Image_df.py b/src/python/nimbusml/examples/examples_from_dataframe/Image_df.py index c45ceaea..8dd050a0 100644 --- a/src/python/nimbusml/examples/examples_from_dataframe/Image_df.py +++ b/src/python/nimbusml/examples/examples_from_dataframe/Image_df.py @@ -29,9 +29,6 @@ pipeline.fit(X, y) # scoring -nimbusml_pred = pipeline.predict(X) -print("Predicted Labels : {0}".format(nimbusml_pred.PredictedLabel.values)) -print( - "Accuracy : {0}".format( - np.mean( - y.Label.values == nimbusml_pred.PredictedLabel.values))) +scores = pipeline.predict(X) +print("Predicted Labels:", scores.PredictedLabel.values) +print("Accuracy:", np.mean(y.Label.values == scores.PredictedLabel.values)) From 210c00fb6b1046eaa1eddae95039b07ceffcf64a Mon Sep 17 00:00:00 2001 From: Gani Nazirov Date: Mon, 3 Jun 2019 19:57:58 -0700 Subject: [PATCH 11/17] fix tests --- src/python/tests/test_docs_example.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/python/tests/test_docs_example.py b/src/python/tests/test_docs_example.py index 0b51e6a1..fa91a480 100644 --- a/src/python/tests/test_docs_example.py +++ b/src/python/tests/test_docs_example.py @@ -63,11 +63,20 @@ def test_examples(self): 'NGramFeaturizer2.py', ]: continue + # skip for all linux tests if os.name != "nt": if name in [ # SymSgdNative fails to load on linux 'SymSgdBinaryClassifier.py', - 'SymSgdBinaryClassifier_infert_df.py', + 'SymSgdBinaryClassifier_infert_df.py' + ]: + continue + # skip for centos7 tests + if platform.linux_distribution()[0] != "Ubuntu": + if name in [ + # libgdiplus needs to be setup + 'Image.py', + 'Image_df.py' # MICROSOFTML_RESOURCE_PATH needs to be setup on linux 'WordEmbedding.py', 'WordEmbedding_df.py', From bf9ebd0f54ea4c9e34917aceb08bc78cabffee4f Mon Sep 17 00:00:00 2001 From: Gani Nazirov Date: Mon, 3 Jun 2019 20:25:00 -0700 Subject: [PATCH 12/17] print platform --- src/python/tests/test_docs_example.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/python/tests/test_docs_example.py b/src/python/tests/test_docs_example.py index fa91a480..892b09b6 100644 --- a/src/python/tests/test_docs_example.py +++ b/src/python/tests/test_docs_example.py @@ -50,6 +50,10 @@ def test_examples(self): ran = 0 excs = [] + print("os:", os.name) + print("linux:", platform.linux_distribution()[0]) + print("version:", platform.linux_distribution()[1]) + for i, (fold, name) in enumerate(fold_files): if i <= start: continue From 2fb5dea7cc15e6aa60872918aa5addee166250c6 Mon Sep 17 00:00:00 2001 From: Gani Nazirov Date: Mon, 3 Jun 2019 20:31:57 -0700 Subject: [PATCH 13/17] get os names --- src/python/tests/test_docs_example.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python/tests/test_docs_example.py b/src/python/tests/test_docs_example.py index 892b09b6..892eb4eb 100644 --- a/src/python/tests/test_docs_example.py +++ b/src/python/tests/test_docs_example.py @@ -76,7 +76,7 @@ def test_examples(self): ]: continue # skip for centos7 tests - if platform.linux_distribution()[0] != "Ubuntu": + if platform.linux_distribution()[0] == "centos": if name in [ # libgdiplus needs to be setup 'Image.py', From 3c999d5d88c194a04295b92a07a0f4101e9fe5f6 Mon Sep 17 00:00:00 2001 From: Gani Nazirov Date: Mon, 3 Jun 2019 21:04:36 -0700 Subject: [PATCH 14/17] test --- build.sh | 2 +- src/python/tests/test_docs_example.py | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/build.sh b/build.sh index e67793cb..1853dca4 100755 --- a/build.sh +++ b/build.sh @@ -265,7 +265,7 @@ then PackagePath=${PythonRoot}/lib/python${PythonVersion}/site-packages/nimbusml # TestsPath1=${PackagePath}/tests - TestsPath2=${__currentScriptDir}/src/python/tests + TestsPath2=${__currentScriptDir}/src/python/tests/test_docs_example.py ReportPath=${__currentScriptDir}/build/TestCoverageReport #"${PythonExe}" -m pytest --verbose --maxfail=1000 --capture=sys "${TestsPath1}" --cov="${PackagePath}" --cov-report term-missing --cov-report html:"${ReportPath}" "${PythonExe}" -m pytest --verbose --maxfail=1000 --capture=sys "${TestsPath2}" --cov="${PackagePath}" --cov-report term-missing --cov-report html:"${ReportPath}" diff --git a/src/python/tests/test_docs_example.py b/src/python/tests/test_docs_example.py index 892eb4eb..d27dff85 100644 --- a/src/python/tests/test_docs_example.py +++ b/src/python/tests/test_docs_example.py @@ -57,7 +57,7 @@ def test_examples(self): for i, (fold, name) in enumerate(fold_files): if i <= start: continue - if name != 'Image.py': + if name not in ['Image.py', 'WordEmbedding.py']: continue if name in [ # Bug 294481: CharTokenizer_df fails @@ -76,7 +76,7 @@ def test_examples(self): ]: continue # skip for centos7 tests - if platform.linux_distribution()[0] == "centos": + if platform.linux_distribution()[0] == 'CentOS Linux': if name in [ # libgdiplus needs to be setup 'Image.py', @@ -141,7 +141,10 @@ def test_examples(self): # FastLinearClassifier_iris_df.py "FutureWarning: elementwise comparison failed", # PcaAnomalyDetector_df.py - "FutureWarning: Sorting because non-concatenation axis" + "FutureWarning: Sorting because non-concatenation axis", + # Image.py + "Unable to revert mtime: /Library/Fonts", + "Fontconfig error: Cannot load default config file", ] if sys.version_info[:2] <= (3, 6): # This warning is new but it does not break any From 41c045bf5b94a51e9a2fde666e4ed0255da640ce Mon Sep 17 00:00:00 2001 From: Gani Nazirov Date: Mon, 3 Jun 2019 21:31:47 -0700 Subject: [PATCH 15/17] test --- build.sh | 1 - src/python/tests/test_copyright.py | 1 - src/python/tests/test_docs_example.py | 17 +++++++---------- 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/build.sh b/build.sh index 1853dca4..c539e6d4 100755 --- a/build.sh +++ b/build.sh @@ -1,6 +1,5 @@ #!/usr/bin/env bash set -e -export FONTCONFIG_PATH=/etc/fonts ProductVersion=$( Date: Mon, 3 Jun 2019 21:56:40 -0700 Subject: [PATCH 16/17] fix linux --- build.sh | 8 ++++---- src/python/tests/test_docs_example.py | 8 +------- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/build.sh b/build.sh index c539e6d4..cbadd3d4 100755 --- a/build.sh +++ b/build.sh @@ -263,11 +263,11 @@ then "${PythonExe}" -m pip install "scikit-learn==0.19.2" PackagePath=${PythonRoot}/lib/python${PythonVersion}/site-packages/nimbusml - # TestsPath1=${PackagePath}/tests - TestsPath2=${__currentScriptDir}/src/python/tests/test_docs_example.py + TestsPath1=${PackagePath}/tests + TestsPath2=${__currentScriptDir}/src/python/tests ReportPath=${__currentScriptDir}/build/TestCoverageReport - #"${PythonExe}" -m pytest --verbose --maxfail=1000 --capture=sys "${TestsPath1}" --cov="${PackagePath}" --cov-report term-missing --cov-report html:"${ReportPath}" - "${PythonExe}" -m pytest --verbose --maxfail=1000 --capture=sys "${TestsPath2}" --cov="${PackagePath}" --cov-report term-missing --cov-report html:"${ReportPath}" + "${PythonExe}" -m pytest --verbose --maxfail=1000 --capture=sys "${TestsPath1}" + "${PythonExe}" -m pytest --verbose --maxfail=1000 --capture=sys "${TestsPath2}" fi exit $? diff --git a/src/python/tests/test_docs_example.py b/src/python/tests/test_docs_example.py index 66160be1..f8c8eeac 100644 --- a/src/python/tests/test_docs_example.py +++ b/src/python/tests/test_docs_example.py @@ -49,13 +49,7 @@ def test_examples(self): ran = 0 excs = [] - print("os:", os.name) - print("linux:", platform.linux_distribution()[0]) - print("version:", platform.linux_distribution()[1]) - for i, (fold, name) in enumerate(fold_files): - if name not in ['Image.py', 'SymSgdBinaryClassifier.py']: - continue if name in [ # Bug 294481: CharTokenizer_df fails # with error about variable length vector @@ -64,7 +58,7 @@ def test_examples(self): 'NGramFeaturizer2.py', ]: continue - # skip for all linux tests + # skip for all linux tests, mac is ok if os.name == "posix" and platform.linux_distribution()[0] != '': if name in [ # SymSgdNative fails to load on linux From a1d646ddedefb2c01756880beef19b5a2886c7a1 Mon Sep 17 00:00:00 2001 From: Gani Nazirov Date: Mon, 3 Jun 2019 22:35:47 -0700 Subject: [PATCH 17/17] Upgrade version --- src/python/nimbusml/__init__.py | 2 +- src/python/setup.py | 2 +- version.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/python/nimbusml/__init__.py b/src/python/nimbusml/__init__.py index aa21ec31..f8407a64 100644 --- a/src/python/nimbusml/__init__.py +++ b/src/python/nimbusml/__init__.py @@ -2,7 +2,7 @@ Microsoft Machine Learning for Python """ -__version__ = '1.0.0' +__version__ = '1.0.1' # CoreCLR version of MicrosoftML is built on Windows. # But file permissions are not preserved when it's copied to Linux. diff --git a/src/python/setup.py b/src/python/setup.py index ee6dc3b3..99d69118 100644 --- a/src/python/setup.py +++ b/src/python/setup.py @@ -45,7 +45,7 @@ # Versions should comply with PEP440. For a discussion on # single-sourcing the version across setup.py and the project code, see # https://packaging.python.org/en/latest/single_source_version.html - version='1.0.0', + version='1.0.1', description='NimbusML', long_description=long_description, diff --git a/version.txt b/version.txt index afaf360d..7f207341 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.0.0 \ No newline at end of file +1.0.1 \ No newline at end of file