Merge branch 'documentation' into temp/docs

.gitignore

@@ -348,3 +348,9 @@ data.csv
 data.txt
 
 /build/TestCoverageReport
+
+# The folder generated by make_yaml.bat
+*_build
+*mymodeluci.zip
+build/sphinxmdoutput-0.2.4.1-py3-none-any.whl
+*build

build.cmd

@@ -21,6 +21,7 @@ set BoostRoot=%DependenciesDir%BoostDbg3.7
 set PythonVersion=3.7
 set PythonTag=cp37
 set RunTests=False
+set RunExtendedTests=False
 set BuildDotNetBridgeOnly=False
 set SkipDotNetBridge=False
 
@@ -33,6 +34,10 @@ if /i [%1] == [--runTests]     (
     set RunTests=True
     shift && goto :Arg_Loop
 )
+if /i [%1] == [--includeExtendedTests]     (
+    set RunExtendedTests=True
+    shift && goto :Arg_Loop
+)
 if /i [%1] == [--buildDotNetBridgeOnly]     (
     set BuildDotNetBridgeOnly=True
     shift && goto :Arg_Loop
@@ -43,11 +48,12 @@ if /i [%1] == [--skipDotNetBridge]     (
 ) else goto :Usage
 
 :Usage
-echo "Usage: build.cmd [--configuration <Configuration>] [--runTests] [--buildDotNetBridgeOnly] [--skipDotNetBridge]"
+echo "Usage: build.cmd [--configuration <Configuration>] [--runTests] [--includeExtendedTests] [--buildDotNetBridgeOnly] [--skipDotNetBridge]"
 echo ""
 echo "Options:"
 echo "  --configuration <Configuration>   Build Configuration (DbgWinPy3.7,DbgWinPy3.6,DbgWinPy3.5,DbgWinPy2.7,RlsWinPy3.7,RlsWinPy3.6,RlsWinPy3.5,RlsWinPy2.7)"
 echo "  --runTests                        Run tests after build"
+echo "  --includeExtendedTests            Include the extended tests if the tests are run"
 echo "  --buildDotNetBridgeOnly           Build only DotNetBridge"
 echo "  --skipDotNetBridge                Build everything except DotNetBridge"
 goto :Exit_Success
@@ -326,6 +332,7 @@ call "%PythonExe%" -m pip install "scikit-learn==0.19.2"
 set PackagePath=%PythonRoot%\Lib\site-packages\nimbusml
 set TestsPath1=%PackagePath%\tests
 set TestsPath2=%__currentScriptDir%src\python\tests
+set TestsPath3=%__currentScriptDir%src\python\tests_extended
 set ReportPath=%__currentScriptDir%build\TestCoverageReport
 call "%PythonExe%" -m pytest --verbose --maxfail=1000 --capture=sys "%TestsPath1%" --cov="%PackagePath%" --cov-report term-missing --cov-report html:"%ReportPath%"
 if errorlevel 1 (
@@ -336,6 +343,13 @@ if errorlevel 1 (
     goto :Exit_Error
 )
 
+if "%RunExtendedTests%" == "True" (
+    call "%PythonExe%" -m pytest --verbose --maxfail=1000 --capture=sys "%TestsPath3%" --cov="%PackagePath%" --cov-report term-missing --cov-report html:"%ReportPath%"
+    if errorlevel 1 (
+        goto :Exit_Error
+    )
+)
+
 :Exit_Success
 endlocal
 exit /b %ERRORLEVEL%

build.sh

@@ -11,12 +11,13 @@ mkdir -p "${DependenciesDir}"
 
 usage()
 {
-    echo "Usage: $0 --configuration <Configuration> [--runTests]"
+    echo "Usage: $0 --configuration <Configuration> [--runTests] [--includeExtendedTests]"
     echo ""
     echo "Options:"
     echo "  --configuration <Configuration>   Build Configuration (DbgLinPy3.7,DbgLinPy3.6,DbgLinPy3.5,DbgLinPy2.7,RlsLinPy3.7,RlsLinPy3.6,RlsLinPy3.5,RlsLinPy2.7,DbgMacPy3.7,DbgMacPy3.6,DbgMacPy3.5,DbgMacPy2.7,RlsMacPy3.7,RlsMacPy3.6,RlsMacPy3.5,RlsMacPy2.7)"
     echo "  --runTests                        Run tests after build"
     echo "  --runTestsOnly                    Run tests on a wheel file in default build location (<repo>/target/)"
+    echo "  --includeExtendedTests            Include the extended tests if the tests are run"
     echo "  --buildNativeBridgeOnly           Build only the native bridge code"
     echo "  --skipNativeBridge                Build the DotNet bridge and python wheel but use existing native bridge binaries (e.g. <repo>/x64/DbgLinPy3.7/pybridge.so)"
     exit 1
@@ -30,6 +31,7 @@ else
     __configuration=DbgLinPy3.7
 fi
 __runTests=false
+__runExtendedTests=false
 __buildNativeBridge=true
 __buildDotNetBridge=true
 
@@ -47,6 +49,9 @@ while [ "$1" != "" ]; do
         --runtests)
             __runTests=true
             ;;
+        --includeextendedtests)
+            __runExtendedTests=true
+            ;;
         --runtestsonly)
             __buildNativeBridge=false
             __buildDotNetBridge=false
@@ -268,9 +273,15 @@ then
     PackagePath=${PythonRoot}/lib/python${PythonVersion}/site-packages/nimbusml
     TestsPath1=${PackagePath}/tests
     TestsPath2=${__currentScriptDir}/src/python/tests
+    TestsPath3=${__currentScriptDir}/src/python/tests_extended
     ReportPath=${__currentScriptDir}/build/TestCoverageReport
     "${PythonExe}" -m pytest --verbose --maxfail=1000 --capture=sys "${TestsPath1}"
     "${PythonExe}" -m pytest --verbose --maxfail=1000 --capture=sys "${TestsPath2}"
+
+    if [ ${__runExtendedTests} = true ]
+    then 
+        "${PythonExe}" -m pytest --verbose --maxfail=1000 --capture=sys "${TestsPath3}"
+    fi
 fi
 
 exit $?

build/ci/phase-template.yml

@@ -4,6 +4,7 @@ parameters:
   buildMatrix: {}
   buildQueue: {}
   testDistro: ''
+  testOptions: ''
 
 phases:
 
@@ -12,6 +13,7 @@ phases:
     _buildScript: ${{ parameters.buildScript }}
     _dockerRun: docker run -e SYSTEM_TEAMFOUNDATIONCOLLECTIONURI="$(System.TeamFoundationCollectionUri)" -e BUILD_BUILDNUMBER="$(Build.BuildNumber)" -i -v $(Build.SourcesDirectory):/builddir -w="/builddir"
     _distro: ${{ parameters.testDistro }}
+    _testOptions: ${{ parameters.testOptions }}
   queue:
    parallel: 99
    matrix:
@@ -21,28 +23,28 @@ phases:
 
   # Windows phases
   - ${{ if eq(parameters.name, 'Windows') }}:
-    - script: $(_buildScript) --configuration $(_configuration) --runTests
+    - script: $(_buildScript) --configuration $(_configuration) --runTests $(_testOptions)
   # Mac phases
   - ${{ if eq(parameters.name, 'Mac') }}:
     - script: brew update && brew install https://raw.githubusercontent.com/Homebrew/homebrew-core/f5b1ac99a7fba27c19cee0bc4f036775c889b359/Formula/libomp.rb mono-libgdiplus gettext && brew link gettext --force
     - ${{ if eq(parameters.testDistro, 'noTests') }}:
       - script: chmod 777 $(_buildScript) && $(_buildScript) --configuration $(_configuration)
     - ${{ if eq(parameters.testDistro, '') }}:
-      - script: chmod 777 $(_buildScript) && $(_buildScript) --configuration $(_configuration) --runTests
+      - script: chmod 777 $(_buildScript) && $(_buildScript) --configuration $(_configuration) --runTests $(_testOptions)
   # Linux phases
   - ${{ if or(eq(parameters.name, 'Linux_Ubuntu16'), eq(parameters.name, 'Linux_Ubuntu14'), eq(parameters.name, 'Linux_CentOS7')) }}:
     - script: $(_dockerRun) mlnet/ubuntu16-nativebuild:0.1 bash -c "source /etc/profile && chmod 777 $(_buildScript) && $(_buildScript) --configuration $(_configuration) --buildNativeBridgeOnly"
       displayName: Build native bridge
     - script: $(_dockerRun) mlnet/ubuntu16-general:0.1 bash -c "source /etc/profile && chmod 777 $(_buildScript) && $(_buildScript) --configuration $(_configuration) --skipNativeBridge"
       displayName: Build python wheel
     - ${{ if eq(parameters.testDistro, 'ubuntu16') }}:
-      - script: $(_dockerRun) mlnet/ubuntu16-general:0.1 bash -c "source /etc/profile && chmod 777 $(_buildScript) && $(_buildScript) --configuration $(_configuration) --runTestsOnly"
+      - script: $(_dockerRun) mlnet/ubuntu16-general:0.1 bash -c "source /etc/profile && chmod 777 $(_buildScript) && $(_buildScript) --configuration $(_configuration) --runTestsOnly $(_testOptions)"
         displayName: Run tests Ubuntu16
     - ${{ if eq(parameters.testDistro, 'ubuntu14') }}:
-      - script: $(_dockerRun) mlnet/ubuntu14-general:0.1 bash -c "source /etc/profile && chmod 777 $(_buildScript) && $(_buildScript) --configuration $(_configuration) --runTestsOnly"
+      - script: $(_dockerRun) mlnet/ubuntu14-general:0.1 bash -c "source /etc/profile && chmod 777 $(_buildScript) && $(_buildScript) --configuration $(_configuration) --runTestsOnly $(_testOptions)"
         displayName: Run tests Ubuntu14
     - ${{ if eq(parameters.testDistro, 'centos7') }}:
-      - script: $(_dockerRun) mlnet/centos7-general:0.1 bash -c "source /root/.bash_profile && source /etc/profile && chmod 777 $(_buildScript) && $(_buildScript) --configuration $(_configuration) --runTestsOnly"
+      - script: $(_dockerRun) mlnet/centos7-general:0.1 bash -c "source /root/.bash_profile && source /etc/profile && chmod 777 $(_buildScript) && $(_buildScript) --configuration $(_configuration) --runTestsOnly $(_testOptions)"
         displayName: Run tests CentOS7
 
   # Publish build artifacts

build/vsts-ci-nightly.yml

@@ -16,6 +16,7 @@ phases:
         _configuration: RlsWinPy2.7
     buildQueue:
       name: Hosted VS2017
+    testOptions: --includeExtendedTests
 
 # Build all configurations for Mac
 - template: /build/ci/phase-template.yml
@@ -33,6 +34,7 @@ phases:
         _configuration: RlsMacPy2.7
     buildQueue:
       name: Hosted macOS
+    testOptions: --includeExtendedTests
 
 # Build all configurations for Linux
   # Run tests on Ubuntu16
@@ -52,6 +54,8 @@ phases:
         _configuration: RlsLinPy2.7
     buildQueue:
       name: Hosted Ubuntu 1604
+    testOptions: --includeExtendedTests
+
   # Run tests on Ubuntu14
 - template: /build/ci/phase-template.yml
   parameters:
@@ -69,6 +73,8 @@ phases:
         _configuration: RlsLinPy2.7
     buildQueue:
       name: Hosted Ubuntu 1604
+    testOptions: --includeExtendedTests
+
   # Run tests on CentOS7
 - template: /build/ci/phase-template.yml
   parameters:
@@ -85,4 +91,5 @@ phases:
       Py27:
         _configuration: RlsLinPy2.7
     buildQueue:
-      name: Hosted Ubuntu 1604
+      name: Hosted Ubuntu 1604
+    testOptions: --includeExtendedTests

src/python/docs/sphinx/apiguide.rst

@@ -50,7 +50,7 @@ Multiclass Classifiers
 ,, :py:class:`OneVsRestClassifier<nimbusml.multiclass.OneVsRestClassifier>` ,, Yes ,, Yes ,,
 
 
-Regressors Classifiers
+Regressors
 """"""""""""""""""""""
 
 ,, Trainer ,,

src/python/docs/sphinx/installationguide.rst

@@ -8,7 +8,7 @@ Installation Guide
 Supported Platforms 
 -------------------
 
-Release 0.6:
+Release 1.1:
    * Windows 10, Ubuntu 14.04, Ubuntu 16.04, CentOS 7, RHEL 7, Mac OS 10.11, 10.12, 10.13
 
 

src/python/docs/sphinx/make.bat

@@ -1,17 +1,16 @@
 @ECHO OFF
 
 pushd %~dp0
-set PYTHONINTERPRETER=%~dp0..\..\..\..\dependencies\Python3.7\python.exe
-set PYTHONPATH=%~dp0..\..\..\..\Python\
+set PYTHONINTERPRETER=%~dp0..\..\..\..\dependencies\Python3.6\python.exe
 set SPHINXOPTS=-j 4
 
 REM Command file for Sphinx documentation
 
 if "%SPHINXBUILD%" == "" (
-	set SPHINXBUILD=%PYTHONINTERPRETER% -msphinx
+	set SPHINXBUILD=%PYTHONINTERPRETER% -m sphinx
 )
 set SOURCEDIR=.
-set BUILDDIR=_build
+set BUILDDIR=%~dp0_build
 set SPHINXPROJ=microsoftml
 
 if "%1" == "" goto html: 
@@ -24,8 +23,8 @@ set format=html
 :next:
 
 @echo remove %BUILDDIR%\%format%
-rmdir /s /q %BUILDDIR%\doctrees
-rmdir /s /q %BUILDDIR%\%format%
+call rmdir /s /q %BUILDDIR%\doctrees
+call rmdir /s /q %BUILDDIR%\%format%
 if exist %BUILDDIR%\_static rmdir /S /Q %BUILDDIR%\_static
 if exist %BUILDDIR%\%format% goto issue:
 
@@ -42,11 +41,11 @@ if errorlevel 9009 (
 	exit /b 1
 )
 
-%SPHINXBUILD% -M %format% %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
+call %SPHINXBUILD% -M %format% %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
 goto end
 
 :help
-%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
+call %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
 goto end:
 
 :issue:

src/python/docs/sphinx/make_yaml.bat

@@ -1,34 +1,87 @@
-@ECHO ON
-set PY=%~dp0..\..\..\..\dependencies\Python3.7\python.exe
-set PYS=%~dp0..\..\..\..\dependencies\Python3.7\Scripts
+@if not defined _echo @echo off
+if exist %~dp0build (rmdir /S /Q %~dp0build)
+if exist %~dp0..\..\..\..\dependencies\Python3.6 (
+echo "Python3.6 exists"
+) else (
+echo "Please run build.cmd under NimbusML with Python3.6's configuration first"
+call exit /b
+)
+echo "###Downloading Dependencies######"
+echo "Downloading Dependencies "
+set PY=%~dp0..\..\..\..\dependencies\Python3.6\python.exe
 set PYTHONPATH=%~dp0..\..\..\..\python
-%PYS%\pip install sphinx==1.5.5
-%PYS%\pip install sphinx-docfx-yaml
-%PYS%\pip install sphinx_rtd_theme
-%PYS%\sphinx-build -c ci_script . _build
+call %PY% -m pip -q install pip==9.0.3
+echo "Installing sphinx-docfx-yaml "
+call %PY% -m pip -q install sphinx-docfx-yaml
+echo "Installing sphinx "
+call %PY% -m pip -q install sphinx==2.1.1
+echo "Installing sphinx_rtd_theme "
+call %PY% -m pip -q install sphinx_rtd_theme
+echo "Installing NimbusML "
+call %PY% -m pip -q install nimbusml
+echo "#################################"
+echo.
 
-if exist _build del /Q _build
+echo.
+echo "#################################"
+echo "Running sphinx-build "
+echo "#################################"
+call %PY% -m sphinx -c %~dp0ci_script %~dp0 %~dp0_build
 
-mkdir _build\ms_doc_ref\
-xcopy /S /I /Q /Y /F _build\docfx_yaml\* _build\ms_doc_ref\nimbusml\docs-ref-autogen
-del _build\ms_doc_ref\nimbusml\docs-ref-autogen\toc.yml
+echo.
+echo "#################################"
+echo "Copying files "
+echo "#################################"
+call mkdir %~dp0_build\ms_doc_ref\
+call xcopy /S /I /Q /Y /F %~dp0_build\docfx_yaml\* %~dp0_build\ms_doc_ref\nimbusml\docs-ref-autogen
 
-%PYS%\pip install sphinx==1.6.2
-CALL make_md.bat
+echo.
+echo "#################################"
+echo "Running make_md.bat"
+echo "Fixing API guide
+echo "#################################"
+call make md
+call %py% %~dp0ci_script\fix_apiguide.py
 
-copy /Y toc.yml _build\ms_doc_ref\nimbusml\toc.yml
-xcopy /Y /S _build\md\* _build\ms_doc_ref\nimbusml
+call copy /Y %~dp0toc.yml %~dp0_build\ms_doc_ref\nimbusml\toc.yml
+call xcopy /Y /S %~dp0_build\md\* %~dp0_build\ms_doc_ref\nimbusml
+:: Append the text in index.md under tutorial.md
 
-del _build\ms_doc_ref\nimbusml\doc-warnings-rx.log
-del _build\ms_doc_ref\nimbusml\doc-warnings-rx-all.log
-del _build\ms_doc_ref\nimbusml\tutorial.md
+echo.
+echo "#################################"
+echo "updating yml......."
+echo "#################################"
+call %PY% %~dp0ci_script\gen_toc_yml.py -input %~dp0_build\ms_doc_ref\nimbusml\index.md -temp %~dp0_build\ms_doc_ref\nimbusml\toc_ref.yml -output %~dp0_build\ms_doc_ref\nimbusml\toc.yml
 
-echo updating yml...
-%PY% ci_script\gen_toc_yml.py -input _build\ms_doc_ref\nimbusml\index.md -temp _build\ms_doc_ref\nimbusml\toc_ref.yml -output _build\ms_doc_ref\nimbusml\toc.yml
+echo.
+echo "#################################"
+echo "updating reference links...."
+echo "#################################"
+call %PY% %~dp0ci_script\update_all_toc_yml.py
 
-echo updating reference links....
-%PY% ci_script\update_all_toc_yml.py
+echo.
+echo "#################################"
+echo "updating ms-scikit.md to modules.md"
+echo "#################################"
+call move %~dp0_build\ms_doc_ref\nimbusml\modules.md %~dp0_build\ms_doc_ref\nimbusml\ms-scikit.md
 
-echo updating ms-scikit.md to modules.md
-del _build\ms_doc_ref\nimbusml\ms-scikit.md
-mv _build\ms_doc_ref\nimbusml\modules.md _build\ms_doc_ref\nimbusml\ms-scikit.md
+echo.
+echo "#################################"
+echo "Cleaning files"
+echo "#################################"
+call mkdir %~dp0_build\ms_doc_ref\nimbusml\_images\_static
+call xcopy /S /I /Q /Y /F %~dp0ci_script\_static %~dp0_build\ms_doc_ref\nimbusml\_images\_static
+call mkdir %~dp0build
+call move %~dp0_build\ms_doc_ref %~dp0\build\
+call more +29 %~dp0build\ms_doc_ref\nimbusml\index.md >> %~dp0build\ms_doc_ref\nimbusml\overview.md
+call del /Q %~dp0build\ms_doc_ref\nimbusml\*log
+call del /Q %~dp0build\ms_doc_ref\nimbusml\concepts.md
+call del /Q %~dp0build\ms_doc_ref\nimbusml\index.md
+call del /Q %~dp0build\ms_doc_ref\nimbusml\toc.yml
+call rmdir /Q %~dp0build\ms_doc_ref\nimbusml\_static
+:: call rmdir /S /Q %~dp0_build
+
+echo.
+echo "#################################"
+echo "#########Built Finished##########"
+echo "#################################"