Merge branch 'master' into ci/migrate-tpu

Author: carmocca

.azure/gpu-tests.yml renamed to .azure/gpu-tests-pytorch.yml

@@ -14,7 +14,7 @@ trigger:
       - "refs/tags/*"
   paths:
     include:
-      - ".azure/gpu-tests.yml"
+      - ".azure/gpu-tests-pytorch.yml"
       - "examples/run_ddp_examples.sh"
       - "examples/convert_from_pt_to_pl/**"
       - "examples/run_pl_examples.sh"
@@ -39,7 +39,7 @@ pr:
       - "release/*"
   paths:
     include:
-      - ".azure/gpu-tests.yml"
+      - ".azure/gpu-tests-pytorch.yml"
       - "examples/run_ddp_examples.sh"
       - "examples/convert_from_pt_to_pl/**"
       - "examples/run_pl_examples.sh"
@@ -97,6 +97,7 @@ jobs:
         set -e
         python -c "fname = 'requirements/pytorch/strategies.txt' ; lines = [line for line in open(fname).readlines() if 'horovod' not in line] ; open(fname, 'w').writelines(lines)"
         python -c "fname = 'requirements/pytorch/strategies.txt' ; lines = [line for line in open(fname).readlines() if 'bagua' not in line] ; open(fname, 'w').writelines(lines)"
+        python -c "fname = 'requirements/pytorch/strategies.txt' ; lines = [line for line in open(fname).readlines() if 'colossalai' not in line] ; open(fname, 'w').writelines(lines)"
 
         PYTORCH_VERSION=$(python -c "import torch; print(torch.__version__.split('+')[0])")
         python ./requirements/pytorch/adjust-versions.py requirements/pytorch/base.txt ${PYTORCH_VERSION}
@@ -110,6 +111,11 @@ jobs:
         CUDA_VERSION_BAGUA=$(python -c "print([ver for ver in [116,113,111,102] if $CUDA_VERSION_MM >= ver][0])")
         pip install "bagua-cuda$CUDA_VERSION_BAGUA"
 
+        PYTORCH_VERSION_COLOSSALAI=$(python -c "import torch; print(torch.__version__.split('+')[0][:4])")
+        CUDA_VERSION_MM_COLOSSALAI=$(python -c "import torch ; print(''.join(map(str, torch.version.cuda)))")
+        CUDA_VERSION_COLOSSALAI=$(python -c "print([ver for ver in [11.3, 11.1] if $CUDA_VERSION_MM_COLOSSALAI >= ver][0])")
+        pip install "colossalai==0.1.10+torch${PYTORCH_VERSION_COLOSSALAI}cu${CUDA_VERSION_COLOSSALAI}" --find-links https://release.colossalai.org
+
         pip list
       env:
         PACKAGE_NAME: pytorch

.github/checkgroup.yml

@@ -85,7 +85,7 @@ subprojects:
 
   - id: "pytorch_lightning: Azure GPU"
     paths:
-      - ".azure/gpu-tests.yml"
+      - ".azure/gpu-tests-pytorch.yml"
       - "tests/tests_pytorch/run_standalone_*.sh"
     checks:
       - "pytorch-lightning (GPUs)"

.github/workflows/README.md

@@ -10,7 +10,7 @@
 | Test PyTorch slow          | .github/workflows/ci-pytorch-test-slow.yml | Run only slow tests. Slow tests usually need to spawn threads and cannot be speed up or simplified.                                                                         | CPU           |
 | pytorch-lightning (IPUs)   | .azure-pipelines/ipu-tests.yml             | Run only IPU-specific tests.                                                                                                                                                | IPU           |
 | pytorch-lightning (HPUs)   | .azure-pipelines/hpu-tests.yml             | Run only HPU-specific tests.                                                                                                                                                | HPU           |
-| pytorch-lightning (GPUs)   | .azure-pipelines/gpu-tests.yml             | Run all CPU and GPU-specific tests, standalone, and examples. Each standalone test needs to be run in separate processes to avoid unwanted interactions between test cases. | GPU           |
+| pytorch-lightning (GPUs)   | .azure-pipelines/gpu-tests-pytorch.yml     | Run all CPU and GPU-specific tests, standalone, and examples. Each standalone test needs to be run in separate processes to avoid unwanted interactions between test cases. | GPU           |
 | PyTorchLightning.Benchmark | .azure-pipelines/gpu-benchmark.yml         | Run speed/memory benchmarks for parity with pure PyTorch.                                                                                                                   | GPU           |
 | test-on-tpus               | .github/workflows/ci-pytorch-test-tpu.yml  | Run only TPU-specific tests.                                                                                                                                                | TPU           |
 

.github/workflows/ci-pytorch-test-slow.yml

@@ -29,7 +29,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-20.04, windows-2022, macOS-11]
-        # same config as '.azure-pipelines/gpu-tests.yml'
+        # same config as '.azure-pipelines/gpu-tests-pytorch.yml'
         python-version: ["3.7"]
         pytorch-version: ["1.11"]
 

.github/workflows/code-checks.yml

@@ -18,7 +18,7 @@ jobs:
 
     - uses: actions/setup-python@v4
       with:
-        python-version: 3.9
+        python-version: '3.10.6'
 
     # Note: This uses an internal pip API and may not always work
     # https://github.com/actions/cache/blob/master/examples.md#multiple-oss-in-a-workflow
@@ -35,8 +35,10 @@ jobs:
         pip install torch==1.12 --find-links https://download.pytorch.org/whl/cpu/torch_stable.html
         python ./requirements/pytorch/adjust-versions.py requirements/pytorch/extra.txt
         # todo: adjust requirements for both code-bases
-        pip install -r requirements/pytorch/devel.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html
+        pip install -r requirements/pytorch/devel.txt -r requirements/app/devel.txt -r requirements/lite/devel.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html
         pip list
 
     - name: Check typing
-      run: mypy
+      run: |
+        mkdir .mypy_cache
+        mypy

dockers/base-cuda/Dockerfile

@@ -54,9 +54,10 @@ RUN \
         libopenmpi-dev \
         openmpi-bin \
         ssh \
+        ninja-build \
         libnccl2=$TO_INSTALL_NCCL \
         libnccl-dev=$TO_INSTALL_NCCL && \
-# Install python
+    # Install python
     add-apt-repository ppa:deadsnakes/ppa && \
     apt-get install -y \
         python${PYTHON_VERSION} \
@@ -65,7 +66,7 @@ RUN \
     && \
     update-alternatives --install /usr/bin/python${PYTHON_VERSION%%.*} python${PYTHON_VERSION%%.*} /usr/bin/python${PYTHON_VERSION} 1 && \
     update-alternatives --install /usr/bin/python python /usr/bin/python${PYTHON_VERSION} 1 && \
-# Cleaning
+    # Cleaning
     apt-get autoremove -y && \
     apt-get clean && \
     rm -rf /root/.cache && \
@@ -82,14 +83,15 @@ RUN \
     rm get-pip.py && \
     pip install -q fire && \
     # Disable cache \
-    CUDA_VERSION_MM=$(python -c "print(''.join('$CUDA_VERSION'.split('.')[:2]))") && \
+    export CUDA_VERSION_MM=$(python -c "print(''.join('$CUDA_VERSION'.split('.')[:2]))") && \
     pip config set global.cache-dir false && \
     # set particular PyTorch version
     python ./requirements/pytorch/adjust-versions.py requirements/pytorch/base.txt ${PYTORCH_VERSION} && \
     python ./requirements/pytorch/adjust-versions.py requirements/pytorch/extra.txt ${PYTORCH_VERSION} && \
     python ./requirements/pytorch/adjust-versions.py requirements/pytorch/examples.txt ${PYTORCH_VERSION} && \
-    # Install all requirements \
-    pip install -r requirements/pytorch/devel.txt --no-cache-dir --find-links https://download.pytorch.org/whl/cu${CUDA_VERSION_MM}/torch_stable.html && \
+
+    # Install base requirements \
+    pip install -r requirements/pytorch/base.txt --no-cache-dir --find-links https://download.pytorch.org/whl/cu${CUDA_VERSION_MM}/torch_stable.html && \
     rm assistant.py
 
 ENV \
@@ -108,7 +110,7 @@ RUN \
     export HOROVOD_BUILD_CUDA_CC_LIST=${HOROVOD_BUILD_CUDA_CC_LIST//"."/""} && \
     echo $HOROVOD_BUILD_CUDA_CC_LIST && \
     cmake --version && \
-    pip install --no-cache-dir -r ./requirements/pytorch/strategies.txt && \
+    pip install --no-cache-dir horovod && \
     horovodrun --check-build
 
 RUN \
@@ -136,6 +138,28 @@ RUN \
     if [[ "$CUDA_VERSION_MM" = "$CUDA_VERSION_BAGUA" ]]; then python -c "import bagua_core; bagua_core.install_deps()"; fi && \
     python -c "import bagua; print(bagua.__version__)"
 
+RUN \
+    # install ColossalAI
+    SHOULD_INSTALL_COLOSSAL=$(python -c "import torch; print(1 if int(torch.__version__.split('.')[1]) > 9 else 0)") && \
+    if [[ "$SHOULD_INSTALL_COLOSSAL" = "1" ]]; then \
+        PYTORCH_VERSION_COLOSSALAI=$(python -c "import torch; print(torch.__version__.split('+')[0][:4])") ; \
+        CUDA_VERSION_MM_COLOSSALAI=$(python -c "import torch ; print(''.join(map(str, torch.version.cuda)))") ; \
+        CUDA_VERSION_COLOSSALAI=$(python -c "print([ver for ver in [11.3, 11.1] if $CUDA_VERSION_MM_COLOSSALAI >= ver][0])") ; \
+        pip install "colossalai==0.1.10+torch${PYTORCH_VERSION_COLOSSALAI}cu${CUDA_VERSION_COLOSSALAI}" --find-links https://release.colossalai.org ; \
+        python -c "import colossalai; print(colossalai.__version__)" ; \
+    fi
+
+RUN \
+    # install rest of strategies
+    # remove colossalai from requirements since they are installed separately
+    SHOULD_INSTALL_COLOSSAL=$(python -c "import torch; print(1 if int(torch.__version__.split('.')[1]) > 9 else 0)") && \
+    if [[ "$SHOULD_INSTALL_COLOSSAL" = "0" ]]; then \
+        python -c "fname = 'requirements/pytorch/strategies.txt' ; lines = [line for line in open(fname).readlines() if 'colossalai' not in line] ; open(fname, 'w').writelines(lines)" ; \
+    fi && \
+    echo "$SHOULD_INSTALL_COLOSSAL" && \
+    cat requirements/pytorch/strategies.txt && \
+    pip install -r requirements/pytorch/devel.txt -r requirements/pytorch/strategies.txt --no-cache-dir --find-links https://download.pytorch.org/whl/cu${CUDA_VERSION_MM}/torch_stable.html
+
 COPY requirements/pytorch/check-avail-extras.py check-avail-extras.py
 COPY requirements/pytorch/check-avail-strategies.py check-avail-strategies.py
 

dockers/release/Dockerfile

@@ -41,7 +41,11 @@ RUN \
     fi && \
     # otherwise there is collision with folder name ans pkg name on Pypi
     cd lightning && \
-    pip install .["extra","loggers","strategies"] --no-cache-dir && \
+    SHOULD_INSTALL_COLOSSAL=$(python -c "import torch; print(1 if int(torch.__version__.split('.')[1]) > 9 else 0)") && \
+    if [[ "$SHOULD_INSTALL_COLOSSAL" = "0" ]]; then \
+        python -c "fname = 'requirements/pytorch/strategies.txt' ; lines = [line for line in open(fname).readlines() if 'colossalai' not in line] ; open(fname, 'w').writelines(lines)" ; \
+    fi && \
+    pip install .["extra","loggers","strategies"] --no-cache-dir --find-links https://release.colossalai.org && \
     cd .. && \
     rm -rf lightning
 

docs/source-app/glossary/secrets.rst

@@ -4,55 +4,56 @@
 Encrypted Secrets
 #################
 
-Is your App using data or values (for example: API keys or access credentials) that you don't want to expose in your App code? If the answer is yes, you'll want to use Secrets. Secrets are encrypted values that are stored in the Lightning.ai database and are decrypted at runtime.
+Encrypted Secrets allow you to pass private data to your apps, like API keys, access tokens, database passwords, or other credentials, in a secure way without exposing them in your code.
+Secrets provide you with a secure way to store this data in a way that is accessible to Apps so that they can authenticate third-party services/solutions.
 
 .. tip::
 	For non-sensitive configuration values, we recommend using :ref:`plain-text Environment Variables <environment_variables>`.
 
-***************
-What did we do?
-***************
+************
+Add a secret
+************
 
-When a Lightning App (App) **runs in the cloud**, a Secret can be exposed to the App using environment variables.
-The value of the Secret is encrypted in the Lightning.ai database, and is only decrypted and accessible to
-LightningFlow (Flow) or LightningWork (Work) processes in the cloud (when you use the ``--cloud`` option running your App).
+Add the secret to your profile on lightning.ai.
+Log in to your lightning.ai account > **Profile** > **Secrets** tab > click the **+New** button.
+Provide a name and value to your secret, for example, name could be "github_api_token".
 
-----
-
-**********************
-What were we thinking?
-**********************
+.. note::
+    Secret names must start with a letter and can only contain letters, numbers, dashes, and periods. The Secret names must comply with `RFC1123 naming conventions <https://www.rfc-editor.org/rfc/rfc1123>`_. The Secret value has no restrictions.
 
-Many Apps require access to private data like API keys, access tokens, database passwords, or other credentials. You need to protect this data.
-We developed this feature to provide you with a secure way to store this data in a way that is accessible to Apps so that they can authenticate third-party services/solutions.
+.. raw:: html
 
-----
+    <br />
+    <video id="background-video" autoplay loop muted controls poster="https://pl-flash-data.s3.amazonaws.com/assets_lightning/docs/images/storage/encrypted_secrets_login.png" width="100%">
+        <source src="https://pl-flash-data.s3.amazonaws.com/assets_lightning/docs/images/storage/encrypted_secrets_login.mp4" type="video/mp4" width="100%">
+    </video>
+    <br />
+    <br />
 
-*********************
-Use Encrypted Secrets
-*********************
+************
+Use a secret
+************
 
-To use Encrypted Secrets:
+1. Add an environment variable to your app to read the secret. For example, add an "api_token" environment variable:
 
-#. Log in to your lightning.ai account, go to **Secrets**, and create the Secret (provide a name and value for the secret).
+.. code:: python
 
-    .. note:: Once you create a Secret, you can bind it to any of your Apps. You do not need to create a new Secret for each App if the Secret value is the same.
+    import os
 
-#. Prepare an environment variable to use with the Secret in your App.
+    component.connect(api_token=os.environ["api_token"])
 
-#. Use the following command to add the Secret to your App:
+2. Pass the secret to your app run with the following command:
 
 .. code:: bash
 
     lightning run app app.py --cloud --secret <environment-variable>=<secret-name>
 
-The environment variables are available in all Flows and Works, and can be accessed as follows:
+In this example, the command would be:
 
-.. code:: python
+.. code:: bash
 
-    import os
+    lightning run app app.py --cloud --secret api_token=github_api_token
 
-    print(os.environ["<environment-variable>"])
 
 The ``--secret`` option can be used for multiple Secrets, and alongside the ``--env`` option.
 
@@ -62,41 +63,13 @@ Here's an example:
 
     lightning run app app.py --cloud --env FOO=bar --secret MY_APP_SECRET=my-secret --secret ANOTHER_SECRET=another-secret
 
-----
-
-Example
-^^^^^^^
 
-The best way to show you how to use Encrypted Secrets is with an example.
-
-First, log in to your `lightning.ai account <https://lightning.ai/>`_ and create a Secret.
-
-.. raw:: html
-
-    <br />
-    <video id="background-video" autoplay loop muted controls poster="https://pl-flash-data.s3.amazonaws.com/assets_lightning/docs/images/storage/encrypted_secrets_login.png" width="100%">
-        <source src="https://pl-flash-data.s3.amazonaws.com/assets_lightning/docs/images/storage/encrypted_secrets_login.mp4" type="video/mp4" width="100%">
-    </video>
-    <br />
-    <br />
-
-.. note::
-    Secret names must start with a letter and can only contain letters, numbers, dashes, and periods. The Secret names must comply with `RFC1123 naming conventions <https://www.rfc-editor.org/rfc/rfc1123>`_. The Secret value has no restrictions.
-
-After creating a Secret named ``my-secret`` with the value ``some-secret-value`` we'll bind it to the environment variable ``MY_APP_SECRET`` within our App. The binding is accomplished by using the ``--secret`` option when running the App from the Lightning CLI.
-
-The ``--secret``` option works similar to ``--env``, but instead of providing a value, you provide the name of the Secret that is replaced with with the value that you want to bind to the environment variable:
-
-.. code:: bash
-
-    lightning run app app.py --cloud --secret MY_APP_SECRET=my-secret
-
-The environment variables are available in all Flows and Works, and can be accessed as follows:
-
-.. code:: python
-
-    import os
+----
 
-    print(os.environ["MY_APP_SECRET"])
+******************
+How does this work
+******************
 
-This code prints out ``some-secret-value``.
+When a Lightning App (App) **runs in the cloud**, a Secret can be exposed to the App using environment variables.
+The value of the Secret is encrypted in the Lightning.ai database, and is only decrypted and accessible to
+LightningFlow (Flow) or LightningWork (Work) processes in the cloud (when you use the ``--cloud`` option running your App).

docs/source-app/glossary/storage/drive.rst

@@ -10,4 +10,4 @@ Drive Storage
 
 ----
 
-.. include:: ../../glossary/storage/drive_content.rst
+.. include:: ../../glossary/storage/drive_content_old.rst

docs/source-app/glossary/storage/drive_content.rst

@@ -1,3 +1,4 @@
+:orphan:
 
 **************************
 What are Lightning Drives?