Merge remote-tracking branch 'upstream/development' into development

Author: NeoZhangJianyu

.repo-tools/Docs_Automation/guids.json

@@ -1127,11 +1127,36 @@
         "removed": "False",
         "ver": "2021.1.Gold"
     },
+    "81B93386-B5EB-4998-B91B-3453263EC03E": {
+        "guid": "81B93386-B5EB-4998-B91B-3453263EC03E",
+        "ver": "2021.4.0",
+        "name": "Intel® Python Scikit-learn Extension Getting Started"
+    },
+    "87FA013D-0B19-45AE-BCD5-43412BE0A99F" : {
+        "guid": "87FA013D-0B19-45AE-BCD5-43412BE0A99F",
+        "ver": "2021.4.0",
+        "name": "Intel oneAPI Rendering Toolkit Embree Getting Started"
+    },
+    "41F82C06-A9FA-4521-B87A-8F7304CB6631" : {
+        "guid": "41F82C06-A9FA-4521-B87A-8F7304CB6631",
+        "ver": "2021.4.0",
+        "name": "Intel oneAPI Rendering Toolkit Open Image Denoise Getting Started"
+    },
+    "80BB7E55-6FCA-491F-8541-E271324CF169" : {
+        "guid": "80BB7E55-6FCA-491F-8541-E271324CF169",
+        "ver": "2021.4.0",
+        "name": "Intel oneAPI Rendering Toolkit Open VKL Getting Started"
+    },
+    "40C26D31-D553-4BD9-9E1D-63DDF352CB85" : {
+        "guid": "40C26D31-D553-4BD9-9E1D-63DDF352CB85",
+        "ver": "2021.4.0",
+        "name": "Intel oneAPI Rendering Toolkit OSPRay Getting Started"
+    },
     "f92e64c7-7a7c-4749-b421-9f7a9e120099": {
         "guid": "f92e64c7-7a7c-4749-b421-9f7a9e120099",
         "name": "Vectorize VecMatMult",
         "notes": "-",
         "removed": "False",
         "ver": "2021.1.Gold"
     }
-}
+}

AI-and-Analytics/End-to-end-Workloads/Census/README.md

@@ -39,24 +39,23 @@ You can refer to the oneAPI [main page](https://software.intel.com/en-us/oneapi)
 To install the Intel® Distribution of Modin* python environment, use the following command:
 #### Linux
 ```
-conda create -n aikit-modin --override-channels intel-aikit-modin omniscidbe4py python=3.7 -c intel -c conda-forge
+conda create -y -n modin-conda-forge -c conda-forge --strict-channel-priority modin-all scikit-learn-intelex
 ```
 Then activate your conda environment with the following command:
 ```
-conda activate aikit-modin
+conda activate modin-conda-forge
 ```
 
 Additionally, install the following in the conda environment
 
 ### Install Jupyter Notebook
 Needed to launch Jupyter Notebook in the directory housing the code example
 ```
-conda install jupyter nb_conda_kernels
+conda install jupyter nb_conda_kernels -c conda-forge
 ```
 
-### ray-dashboard and opencensus
+### opencensus
 ```
-conda install ray-dashboard
 pip install opencensus
 ```
 

AI-and-Analytics/End-to-end-Workloads/Census/census_modin.ipynb

@@ -91,7 +91,7 @@
    "outputs": [],
    "source": [
     "#import modin.pandas as pd\n",
-    "os.environ[\"MODIN_ENGINE\"] = \"ray\"\n",
+    "os.environ[\"MODIN_ENGINE\"] = \"native\"\n",
     "os.environ[\"MODIN_BACKEND\"] = \"omnisci\"\n",
     "os.environ[\"MODIN_EXPERIMENTAL\"] = \"True\"\n",
     "import modin.pandas as pd"
@@ -148,7 +148,7 @@
    },
    "outputs": [],
    "source": [
-    "df = pd.read_csv('ipums_education2income_1970-2010.csv.gz', compression=\"gzip\")"
+    "df = pd.read_csv('ipums_education2income_1970-2010.csv.gz', compression=\"gzip\", nrows=10000)"
    ]
   },
   {

AI-and-Analytics/End-to-end-Workloads/Census/sample.json

@@ -13,8 +13,9 @@
     {
   		"env": ["source activate base"],
   		"steps": [
-         "conda create -y -n aikit-modin --override-channels intel-aikit-modin omniscidbe4py python=3.7 runipy ray-dashboard -c intel -c conda-forge",
-         "conda activate aikit-modin",
+         "conda create -y -n modin-conda-forge -c conda-forge --strict-channel-priority modin-all",
+         "conda activate modin-conda-forge",
+	 "conda install -y -c conda-forge scikit-learn-intelex runipy",
          "pip install opencensus",
          "runipy census_modin.ipynb" 
   		 ]

AI-and-Analytics/Features-and-Functionality/IntelPython_XGBoost_Performance/IntelPython_XGBoost_Performance.ipynb

@@ -126,13 +126,14 @@
    "outputs": [],
    "source": [
     "def load_higgs(nrows_train, nrows_test, dtype=np.float32):\n",
-    "    if not os.path.isfile(\"./data/batch/HIGGS.csv.gz\"):\n",
+    "    if not os.path.isfile(\"./HIGGS.csv.gz\"):\n",
     "        print(\"Loading data set...\")\n",
     "        url = \"https://archive.ics.uci.edu/ml/machine-learning-databases/00280/HIGGS.csv.gz\"\n",
     "        myfile = requests.get(url)\n",
-    "        open('./data/batch/HIGGS.csv.gz', 'wb').write(myfile.content)\n",
+    "        with open('./HIGGS.csv.gz', 'wb') as f:\n",
+    "            f.write(myfile.content)\n",
     "    print(\"Reading data set...\")\n",
-    "    data = pd.read_csv(\"./data/batch/HIGGS.csv.gz\", delimiter=\",\", header=None, compression=\"gzip\", dtype=dtype, nrows=nrows_train+nrows_test)\n",
+    "    data = pd.read_csv(\"./HIGGS.csv.gz\", delimiter=\",\", header=None, compression=\"gzip\", dtype=dtype, nrows=nrows_train+nrows_test)\n",
     "    print(\"Pre-processing data set...\")\n",
     "    data = data[list(data.columns[1:])+list(data.columns[0:1])]\n",
     "    n_features = data.shape[1]-1\n",

AI-and-Analytics/Getting-Started-Samples/IntelModin_GettingStarted/README.md

@@ -1,21 +1,21 @@
 # `Intel Modin Getting Started` Sample
-This Getting Started sample code shows how to use distributed Pandas using the Intel® Distribution of Modin* package. It demonstrates how to use software products that can be found in the [Intel® oneAPI AI Analytics Toolkit](https://software.intel.com/content/www/us/en/develop/tools/oneapi/ai-analytics-toolkit.html). 
+This Getting Started sample code shows how to use distributed Pandas using the Intel® Distribution of Modin* package. It demonstrates how to use software products that can be found in the [Intel® oneAPI AI Analytics Toolkit](https://software.intel.com/content/www/us/en/develop/tools/oneapi/ai-analytics-toolkit.html).
 
 | Optimized for                     | Description
 | :---                              | :---
 | OS                                | 64-bit Linux: Ubuntu 18.04 or higher
-| Hardware                          | Intel Atom® Processors; Intel® Core™ Processor Family; Intel® Xeon® Processor Family; Intel® Xeon® Scalable processor family
+| Hardware                          | Intel Atom® Processors; Intel® Core™ Processor Family; Intel® Xeon® Processor Family; Intel® Xeon® Scalable Performance Processor Family
 | Software                          | Intel Distribution of Modin*, Intel® oneAPI AI Analytics Toolkit
 | What you will learn               | Basic Intel Distribution of Modin* programming model for Intel CPU
 | Time to complete                  | 5-8 minutes
 
 ## Purpose
-Intel Distribution of Modin* uses Ray or Dask to provide an effortless way to speed up your Pandas notebooks, scripts, and libraries. Unlike other distributed DataFrame libraries, Intel Distribution of Modin* provides seamless integration and compatibility with existing Pandas code. 
+Intel Distribution of Modin* uses Ray or Dask to provide an effortless way to speed up your Pandas notebooks, scripts, and libraries. Unlike other distributed DataFrame libraries, Intel Distribution of Modin* provides seamless integration and compatibility with existing Pandas code.
 
 In this sample, you will run Intel Distribution of Modin*-accelerated Pandas functions and note the performance gain when compared to "stock" (aka standard) Pandas functions.
 
 ## Key Implementation Details
-This Getting Started sample code is implemented for CPU using the Python language. The example assumes you have Pandas and MoIntel Distribution of Modin* installed inside a conda environment, similar to what is directed by the [Intel® oneAPI AI Analytics Toolkit](https://software.intel.com/content/www/us/en/develop/articles/installing-ai-kit-with-conda.html). 
+This Getting Started sample code is implemented for CPU using the Python language. The example assumes you have Pandas and MoIntel Distribution of Modin* installed inside a conda environment, similar to what is directed by the [Intel® oneAPI AI Analytics Toolkit](https://www.intel.com/content/www/us/en/develop/documentation/installation-guide-for-intel-oneapi-toolkits-linux/top/installation/install-using-package-managers/conda/install-intel-ai-analytics-toolkit-via-conda.html).
 
 ## License
 
@@ -26,11 +26,11 @@ Third party program Licenses can be found here: [third-party-programs.txt](https
 
 ## Building Intel Distribution of Modin* for CPU
 
-Intel® Distribution of Modin* is ready for use once you finish the Intel® Distribution of Modin* installation and have run the post installation script.
+Intel Distribution of Modin* is ready for use once you finish the Intel Distribution of Modin installation and have run the post installation script.
 
-For this sample, you will also have to install the matplotlib module. 
+For this sample, you will also have to install the matplotlib module.
 
-Please install matplotlib with the command: 
+Please install matplotlib with the command:
 
 ```
 conda install -c intel matplotlib
@@ -41,7 +41,7 @@ You can refer to the oneAPI [main page](https://software.intel.com/en-us/oneapi)
 
 ### Activate conda environment With Root Access
 
-Please follow the Getting Started Guide steps (above) to set up your oneAPI environment with the `setvars.sh` script and [Intel® Distribution of Modin* environment installation] (https://software.intel.com/content/www/us/en/develop/articles/installing-ai-kit-with-conda.html). Then navigate in Linux shell to your oneapi installation path, typically `/opt/intel/oneapi/` when installed as root or sudo, and `~/intel/oneapi/` when not installed as a superuser. If you customized the installation folder, the `setvars.sh` file is in your custom folder.
+Please follow the Getting Started Guide steps (above) to set up your oneAPI environment with the `setvars.sh` script and [Intel Distribution of Modin environment installation](https://www.intel.com/content/www/us/en/develop/documentation/installation-guide-for-intel-oneapi-toolkits-linux/top/installation/install-using-package-managers/conda/install-intel-ai-analytics-toolkit-via-conda.html). Then navigate in Linux shell to your oneapi installation path, typically `/opt/intel/oneapi/` when installed as root or sudo, and `~/intel/oneapi/` when not installed as a superuser. If you customized the installation folder, the `setvars.sh` file is in your custom folder.
 
 Activate the conda environment with the following command:
 
@@ -91,7 +91,7 @@ Open .ipynb file and run cells in Jupyter Notebook using the "Run" button (see t
 
 ![Click the Run Button in the Jupyter Notebook](Jupyter_Run.jpg "Run Button on Jupyter Notebook")
 
-#### Intel® DevCloud for oneAPI JupyterLab 
+#### Intel® DevCloud for oneAPI JupyterLab
 
 Please note that as of right now, this sample cannot be run on Intel® DevCloud for oneAPI JupyterLab due to conflicts between the Intel® DevCloud for oneAPI JupyterLab platform and Modin dependencies. This is a known issue that Intel is currently working on resolving. Thank you for your patience.
 

AI-and-Analytics/Jupyter/Numba_DPPY_Essentials_training/00_DPPY_Prerequisites/License.txt

@@ -0,0 +1,8 @@
+Copyright 2020 Intel Corporation
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+

AI-and-Analytics/Jupyter/Numba_DPPY_Essentials_training/00_DPPY_Prerequisites/Setup_Instructions_Numba.ipynb

@@ -0,0 +1,121 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Introduction to JupyterLab and Notebooks"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "If you are familiar with Jupyter skip below and head to the first exercise.\n",
+    "\n",
+    "<video controls src=\"src/Introduction_to_Jupyter.mp4\" width=\"720\"/>"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "__JupyterLab__ is a sequence of boxes referred to as \"cells\". Each cell will contain text, like this one, or C++ or Python code that may be executed as part of this tutorial. As you proceed, please note the following:\n",
+    " \n",
+    "* The active cell is indicated by the blue bar on the left. Click on a cell to select it.\n",
+    " \n",
+    "* Use the __\"run\"__ ▶ button at the top or __Shift+Enter__ to execute a selected cell, starting with this one.\n",
+    "  * Note: If you mistakenly press just Enter, you will enter the editing mode for the cell. To exit editing mode and continue, press Shift+Enter.\n",
+    "\n",
+    "\n",
+    "* Unless stated otherwise, the cells containing code within this tutorial MUST be executed in sequence.\n",
+    " \n",
+    "* You may save the tutorial at any time, which will save the output, but not the state. Saved Jupyter Notebooks will save sequence numbers which may make a cell appear to have been executed when it has not been executed for the new session. Because state is not saved, re-opening or __restarting a Jupyter Notebook__ will required re-executing all the executable steps, starting in order from the beginning.\n",
+    " \n",
+    "* If for any reason you need to restart the tutorial from the beginning, you may reset the state of the Jupyter Notebook and clear all output. Use the menu at the top to select __Kernel -> \"Restart Kernel and Clear All Outputs\"__\n",
+    " \n",
+    "* Cells containing Markdown can be executed and will render. However, there is no indication of execution, and it is not necessary to explicitly execute Markdown cells.\n",
+    " \n",
+    "* Cells containing executable code will have \"a [ ]:\" to the left of the cell:\n",
+    "  * __[ ]__ blank indicates that the cell has not yet been executed.\n",
+    "  * __[\\*]__ indicates that the cell is currently executing.\n",
+    "  * Once a cell is done executing, a number will appear in the small brackets with each cell execution to indicate where in the sequence the cell has been executed. Any output (e.g. print()'s) from the code will appear below the cell."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Code editing, Compiling and Running in Jupyter Notebooks\n",
+    "This code shows a simple python Hello world. Inspect code, there are no modifications necessary:\n",
+    "1. Inspect the code cell below and click run ▶ to run the file"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def main():\n",
+    "    print(\"Hello World\")\n",
+    "    \n",
+    "if __name__ == \"__main__\":\n",
+    "    main()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.9"
+  },
+  "toc": {
+   "base_numbering": 1,
+   "nav_menu": {},
+   "number_sections": true,
+   "sideBar": true,
+   "skip_h1_title": false,
+   "title_cell": "Table of Contents",
+   "title_sidebar": "Contents",
+   "toc_cell": false,
+   "toc_position": {
+    "height": "calc(100% - 180px)",
+    "left": "10px",
+    "top": "150px",
+    "width": "384.391px"
+   },
+   "toc_section_display": true,
+   "toc_window_display": true
+  },
+  "widgets": {
+   "application/vnd.jupyter.widget-state+json": {
+    "state": {},
+    "version_major": 2,
+    "version_minor": 0
+   }
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

AI-and-Analytics/Jupyter/Numba_DPPY_Essentials_training/00_DPPY_Prerequisites/q

@@ -0,0 +1,52 @@
+#!/bin/bash
+#==========================================
+# Copyright © 2020 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+#==========================================
+# Script to submit job in Intel(R) DevCloud
+# Version: 0.7
+#==========================================
+if [ -z "$1" ]; then
+    echo "Missing script argument, Usage: ./q run.sh"
+elif [ ! -f "$1" ]; then
+    echo "File $1 does not exist"
+else
+    echo "Job has been submitted to Intel(R) DevCloud and will execute soon."
+    echo ""
+    echo " If you do not see result in 60 seconds, please restart the Jupyter kernel:"
+    echo " Kernel -> 'Restart Kernel and Clear All Outputs...' and then try again"
+    echo ""
+    script=$1
+    # Remove old output files
+    rm *.sh.* > /dev/null 2>&1
+    # Submit job using qsub
+    qsub_id=`qsub -l nodes=1:gpu:ppn=2 -d . $script`
+    job_id="$(cut -d'.' -f1 <<<"$qsub_id")"
+    # Print qstat output
+    qstat 
+    # Wait for output file to be generated and display
+    echo ""
+    echo -ne "Waiting for Output "
+    until [ -f $script.o$job_id ]; do
+        sleep 1
+        echo -ne "█"
+        ((timeout++))
+        # Timeout if no output file generated within 60 seconds
+        if [ $timeout == 60 ]; then
+            echo ""
+            echo ""
+            echo "TimeOut 60 seconds: Job is still queued for execution, check for output file later ($script.o$job_id)"
+            echo ""
+            break
+        fi
+    done
+    # Print output and error file content if exist
+    if [ -n "$(find -name '*.sh.o'$job_id)" ]; then
+        echo " Done⬇"
+        cat $script.o$job_id
+        cat $script.e$job_id
+        echo "Job Completed in $timeout seconds."
+        rm *.sh.*$job_id > /dev/null 2>&1
+    fi
+fi

AI-and-Analytics/Jupyter/Numba_DPPY_Essentials_training/00_DPPY_Prerequisites/run_hello.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+source /opt/intel/oneapi/setvars.sh > /dev/null 2>&1
+/bin/echo "##" $(whoami) is compiling AI Essentials Module 0 -- DPPY Prerequisites - 2 of 5 hello_world.py
+python src/hello_world.py