Merge branch 'master' into codeq/mlflow-logger

Author: awaelchli

.azure/hpu-tests.yml

@@ -20,23 +20,44 @@ jobs:
     timeoutInMinutes: "10"
     # how much time to give 'run always even if cancelled tasks' before stopping them
     cancelTimeoutInMinutes: "2"
-    pool: intel-hpus
+    pool: habana-gaudi-hpus
+    container:
+      image: "vault.habana.ai/gaudi-docker/1.5.0/ubuntu20.04/habanalabs/pytorch-installer-1.11.0:latest"
+      options: "--runtime=habana -e HABANA_VISIBLE_DEVICES=all -e OMPI_MCA_btl_vader_single_copy_mechanism=none --cap-add=sys_nice --ipc=host --shm-size=4g --name cd-container -v /usr/bin/docker:/tmp/docker:ro"
     workspace:
       clean: all
 
     steps:
+    - script: |
+        /tmp/docker exec -t -u 0 cd-container \
+        sh -c "apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confold" -y install sudo"
+      displayName: 'Install Sudo in container (thanks Microsoft!)'
+
     - bash: |
-        apt-get install -y hwinfo
+        sudo apt-get install -y hwinfo
         hwinfo --short
+        python --version
+        sudo pip install pip -U
       displayName: 'Instance HW info'
 
     - bash: |
-        pip install -e .[extra] -r requirements/pytorch/test.txt
+        set -e
+        pip --version
+        sudo pip uninstall -y lightning pytorch-lightning
+        pip install fire
+        python .actions/assistant.py requirements-prune-pkgs torch,torchvision,torchtext
+        pip install ".[extra,test]"
+        pip list
       env:
         PACKAGE_NAME: pytorch
         FREEZE_REQUIREMENTS: 1
       displayName: 'Install dependencies'
 
+    - bash: |
+        hl-smi -L
+        lsmod | grep habanalabs
+      displayName: 'Check the driver status'
+
     - bash: |
         python -m pytest -sv accelerators/test_hpu.py --forked --junitxml=hpu1_test-results.xml
       workingDirectory: tests/tests_pytorch

.github/workflows/ci_pr-gatekeeper.yml

@@ -26,7 +26,7 @@ jobs:
         run: |
           patterns = ('docs/source-${{ matrix.pkg }}', 'src/lightning_${{ matrix.pkg }}', 'tests/tests_${{ matrix.pkg }}')
           changed = any(p in "${{steps.changed-files.outputs.all_changed_and_modified_files}}" for p in patterns)
-          print('::set-output name=files::' + int(changed))
+          print(f'::set-output name=files::{int(changed)}')
         shell: python
       - uses: octodemo/pr-gatekeeper@main
         if: steps.touched.outputs.files == 1

.github/workflows/cicd-pytorch_dockers.yml

@@ -225,7 +225,7 @@ jobs:
           build-args: |
             DIST=latest
             GAUDI_VERSION=${{ matrix.gaudi_version }}
-            PYTORCH_VERSION=${{ matrix.pytorch_version }}
+            PYTORCH_INSTALLER_VERSION=${{ matrix.pytorch_version }}
           file: dockers/ci-runner-hpu/Dockerfile
           push: ${{ env.PUSH_TO_HUB }}
           tags: pytorchlightning/pytorch_lightning:hpu-ci-runner-gaudi${{ matrix.gaudi_version }}

.pre-commit-config.yaml

@@ -73,7 +73,7 @@ repos:
         name: Format imports
 
   - repo: https://github.com/psf/black
-    rev: 22.3.0
+    rev: 22.6.0
     hooks:
       - id: black
         name: Format code

dockers/ci-runner-hpu/Dockerfile

@@ -1,24 +1,65 @@
+# Run command to build:
+#    gaudi_ver=$(curl -s "https://vault.habana.ai/artifactory/gaudi-docker/" |  sed -n 's/.*href="\([^"]*\).*/\1/p' | tail -2 | head -1 | sed "s/\///1")
+#    pytorch_install_ver=$(curl -s "https://vault.habana.ai/artifactory/gaudi-docker/$gaudi_ver/ubuntu20.04/habanalabs/" | sed -n 's/.*href="\([^"]*\).*/\1/p'| sed "s/\///1" | grep pytorch-installer)
+#    pytorch_install_ver=${pytorch_install_ver/"pytorch-installer-"/""}
+#    docker build -t gaudi-docker-agent:latest \
+#       --build-arg GAUDI_VERSION=$gaudi_ver \
+#       --build-arg PYTORCH_INSTALLER_VERSION=$pytorch_install_ver \
+#       -f Dockerfile .
+# Run command:
+#    docker run --privileged \
+#        -v /dev:/dev \
+#        -e AZP_URL="https://dev.azure.com/ORGANIZATION/" \
+#        -e AZP_TOKEN="XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" \
+#        -e AZP_AGENT_NAME="hpu1" \
+#        -e AZP_POOL="intel-hpus" \
+#        gaudi-docker-agent:latest
+
 ARG DIST="latest"
 ARG GAUDI_VERSION="1.5.0"
-ARG PYTORCH_VERSION="1.11.0"
-
-FROM vault.habana.ai/gaudi-docker/${GAUDI_VERSION}/ubuntu20.04/habanalabs/pytorch-installer-${PYTORCH_VERSION}:${DIST}
+ARG PYTORCH_INSTALLER_VERSION="1.11.0"
+FROM vault.habana.ai/gaudi-docker/${GAUDI_VERSION}/ubuntu20.04/habanalabs/pytorch-installer-${PYTORCH_INSTALLER_VERSION}:${DIST}
 
 LABEL maintainer="https://vault.habana.ai/"
+# update the base packages and add a non-sudo user
+RUN \
+    apt-get update -y && \
+    apt-get upgrade -y && \
+    useradd -m docker
 
-RUN echo "ALL ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
-
-WORKDIR /azp
-
-COPY ./dockers/ci-runner-hpu/start.sh /usr/local/bin/
+# To make it easier for build and release pipelines to run apt-get,
+# configure apt to not require confirmation (assume the -y argument by default)
+ENV DEBIAN_FRONTEND=noninteractive
+RUN echo "APT::Get::Assume-Yes \"true\";" > /etc/apt/apt.conf.d/90assumeyes
 
-RUN chmod +x /usr/local/bin/start.sh
+RUN apt-get update --fix-missing && \
+    apt-get install -y --no-install-recommends \
+        ca-certificates \
+        build-essential \
+        curl \
+        jq \
+        git \
+        iputils-ping \
+        libcurl4 \
+        libunwind8 \
+        netcat \
+        libssl1.0 \
+        libssl-dev \
+        libffi-dev \
+        python3 \
+        python3-venv \
+        python3-dev \
+        python3-pip
 
 RUN curl -fsSL https://get.docker.com -o get-docker.sh && \
     sh get-docker.sh && \
     rm get-docker.sh
 
-#RUN docker --help
+RUN pip uninstall pytorch-lightning -y
+
+WORKDIR /azp
+
+COPY ./dockers/ci-runner-hpu/start.sh /usr/local/bin/
+RUN chmod +x /usr/local/bin/start.sh
 
 ENTRYPOINT ["/usr/local/bin/start.sh"]
-CMD ["bash"]

dockers/ci-runner-hpu/start.sh

@@ -93,4 +93,4 @@ trap 'cleanup; exit 143' TERM
 
 # To be aware of TERM and INT signals call run.sh
 # Running it with the --once flag at the end will shut down the agent after the build is executed
-./run.sh --once & wait $!
+./run.sh & wait $!

requirements/pytorch/base.txt

@@ -7,4 +7,4 @@ tensorboard>=2.9.1, <2.10.0
 torchmetrics>=0.7.0, <0.9.2  # needed for using fixed compare_version
 pyDeprecate>=0.3.1, <=0.3.2
 packaging>=17.0, <=21.3
-typing-extensions>=4.0.0, <4.2.1
+typing-extensions>=4.0.0, <4.3.1

src/lightning/__setup__.py

@@ -39,6 +39,7 @@ def _adjust_manifest(**kwargs: Any) -> None:
         lines += [
             "recursive-include src *.md" + os.linesep,
             "recursive-include requirements *.txt" + os.linesep,
+            "recursive-include src/lightning_app/cli/*-template *" + os.linesep,  # Add templates
         ]
     with open(manifest_path, "w") as fp:
         fp.writelines(lines)

src/lightning_app/__setup__.py

@@ -50,6 +50,7 @@ def _adjust_manifest(**__: Any) -> None:
         "recursive-exclude requirements *.txt" + os.linesep,
         "recursive-include src/lightning_app *.md" + os.linesep,
         "recursive-include requirements/app *.txt" + os.linesep,
+        "recursive-include src/lightning_app/cli/*-template *" + os.linesep,  # Add templates
     ]
 
     # TODO: remove this once lightning-ui package is ready as a dependency

src/pytorch_lightning/plugins/precision/fully_sharded_native_amp.py

@@ -11,10 +11,19 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from typing import Any
+from typing import Any, Optional
+
+import torch
 
 from pytorch_lightning.plugins.precision.sharded_native_amp import ShardedNativeMixedPrecisionPlugin
+from pytorch_lightning.utilities.enums import PrecisionType
 from pytorch_lightning.utilities.exceptions import MisconfigurationException
+from pytorch_lightning.utilities.imports import _TORCH_GREATER_EQUAL_1_12
+
+if _TORCH_GREATER_EQUAL_1_12:
+    from torch.distributed.fsdp.fully_sharded_data_parallel import MixedPrecision
+else:
+    MixedPrecision = None
 
 
 class FullyShardedNativeMixedPrecisionPlugin(ShardedNativeMixedPrecisionPlugin):
@@ -29,3 +38,18 @@ def clip_grad_by_norm(self, *_: Any, **__: Any) -> None:
         raise MisconfigurationException(
             f"`gradient_clip_algorithm='norm'` is currently not supported for `{self.__class__.__name__}`"
         )
+
+    @property
+    def mixed_precision_config(self) -> Optional[MixedPrecision]:
+        assert MixedPrecision is not None
+        if self.precision == PrecisionType.HALF:
+            dtype = torch.float16
+        elif self.precision == PrecisionType.BFLOAT:
+            dtype = torch.bfloat16
+        else:
+            raise MisconfigurationException(f"Was unable to infer precision type, received {self.precision!r}.")
+        return MixedPrecision(
+            param_dtype=dtype,
+            reduce_dtype=dtype,
+            buffer_dtype=dtype,
+        )