Some minor CI cleanup (#10088)

Author: carmocca

.github/workflows/ci_dockers.yml

@@ -46,7 +46,7 @@ jobs:
       fail-fast: false
       matrix:
         python_version: ["3.7"]
-        xla_version: ["1.6", "1.8", "nightly"]
+        xla_version: ["1.6", "1.8", "1.10"]
     steps:
       - name: Checkout
         uses: actions/checkout@v2

.github/workflows/ci_test-base.yml

@@ -1,3 +1,5 @@
+# this jobs runs `pytest` over the source directory. It does not install any extra dependencies.
+# this is useful to catch errors where an import has been added which is not part of the basic dependencies.
 name: CI basic testing
 
 # see: https://help.github.com/en/actions/reference/events-that-trigger-workflows
@@ -13,10 +15,10 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
-      # max-parallel: 6
       matrix:
-        os: [ubuntu-20.04, windows-2019, macOS-10.15]
-        python-version: [3.8]
+        os: [ubuntu-20.04]
+        # this will install stable torch
+        python-version: [3.9]
 
     # Timeout: https://stackoverflow.com/a/59076067/4521646
     timeout-minutes: 20
@@ -27,12 +29,6 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
 
-    # Github Actions: Run step on specific OS: https://stackoverflow.com/a/57948488/4521646
-    - name: Setup macOS
-      if: runner.os == 'macOS'
-      run: |
-        brew install libomp  # https://github.com/pytorch/pytorch/issues/20030
-
     - name: Weekly reset caching
       run: echo "::set-output name=period::$(python -c 'import time ; days = time.time() / 60 / 60 / 24 ; print(int(days / 7))' 2>&1)"
       id: times
@@ -54,21 +50,14 @@ jobs:
 
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade --user pip
-        pip install --requirement ./requirements.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --upgrade
-        pip install "pytest>6.0" "pytest-cov>2.10" --upgrade-strategy only-if-needed
         python --version
+        python -m pip install --upgrade --user pip
         pip --version
+        pip install --requirement requirements.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --upgrade
+        pip install --requirement requirements/test.txt
         pip list
       shell: bash
 
-    - name: Cache datasets
-      uses: actions/cache@v2
-      with:
-        path: Datasets # This path is specific to Ubuntu
-        # Look to see if there is a cache hit for the corresponding requirements file
-        key: PL-dataset
-
     - name: Test Package [only]
       run: |
         # NOTE: run coverage on tests does not propagate failure status for Win, https://github.com/nedbat/coveragepy/issues/1003

.github/workflows/ci_test-conda.yml

@@ -3,7 +3,7 @@ name: PyTorch & Conda
 # see: https://help.github.com/en/actions/reference/events-that-trigger-workflows
 on:  # Trigger the workflow on push or pull request, but only for the master branch
   push:
-    branches: [ master, "release/*" ]
+    branches: [master, "release/*"]
   pull_request:
     branches: [master, "release/*"]
 

.github/workflows/ci_test-full.yml

@@ -45,36 +45,24 @@ jobs:
       run: echo "::set-output name=period::$(python -c 'import time ; days = time.time() / 60 / 60 / 24 ; print(int(days / 7))' 2>&1)"
       id: times
 
-    - name: Update pip
+    - name: Upgrade pip
       run: |
+        python --version
         # needed for `pip cache` command
-        pip install --quiet "pip==20.2" --upgrade --user
+        pip install --upgrade pip --user
+        pip --version
 
     # Github Actions: Run step on specific OS: https://stackoverflow.com/a/57948488/4521646
     - name: Setup macOS
       if: runner.os == 'macOS'
       run: |
-        brew install libomp  # https://github.com/pytorch/pytorch/issues/20030
         brew install openmpi libuv  # Horovod on macOS requires OpenMPI, Gloo not currently supported
 
     - name: Setup Windows
       if: runner.os == 'windows'
       run: |
         python .github/prune-packages.py requirements/extra.txt "horovod"
 
-    # todo: re-enable when allow testing py 3.9 with min config, atm some Hydra issues
-    #- name: Adjust minimal for Python 3.9
-    #  if: matrix.requires == 'minimal' && matrix.python-version == 3.9
-    #  run: |
-    #    import re
-    #    def _req(fname, ptn, ver):
-    #        req = re.sub(ptn, ver, open(fname).read())
-    #        open(fname, 'w').write(req)
-    #
-    #    _req('requirements.txt', r'torch>=[\d\.]+', 'torch>=1.8.0')
-    #    _req('requirements/extra.txt', r'onnxruntime>=[\d\.]+', 'onnxruntime>=1.7.0')
-    #  shell: python
-
     - name: Set min. dependencies
       if: matrix.requires == 'minimal'
       run: |
@@ -104,37 +92,35 @@ jobs:
         unzip -o checkpoints.zip
         ls -l checkpoints/
 
-    # todo: re-enable testing with Horovod
-    - name: py3.9 - temp skip Horovod
-      if: matrix.python-version == 3.9
+    - name: Install dependencies
       run: |
-        # pip uninstall -y horovod
-        python .github/prune-packages.py requirements/extra.txt "horovod"
+        flag=$(python -c "print('--pre' if '${{matrix.release}}' == 'pre' else '')" 2>&1)
+        pip install --requirement requirements.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --upgrade $flag
+        # adjust versions according installed Torch version
+        python ./requirements/adjust_versions.py requirements/examples.txt
+        pip install --requirement requirements/examples.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --upgrade
+        pip install --requirement requirements/test.txt --upgrade
+        pip list
+      shell: bash
 
-    - name: Install dependencies
+    - name: Install extra dependencies
       env:
-        # MAKEFLAGS: "-j2"
         HOROVOD_BUILD_ARCH_FLAGS: "-mfma"
         HOROVOD_WITHOUT_MXNET: 1
         HOROVOD_WITHOUT_TENSORFLOW: 1
       run: |
-        python --version
-        pip --version
-        # python -m pip install --upgrade --user pip
-        flag=$(python -c "print('--pre' if '${{matrix.release}}' == 'pre' else '')" 2>&1)
-        pip install --requirement requirements.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --upgrade $flag
         # adjust versions according installed Torch version
         python ./requirements/adjust_versions.py requirements/extra.txt
-        python ./requirements/adjust_versions.py requirements/examples.txt
-        pip install --requirement ./requirements/devel.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --upgrade
+        pip install --requirement ./requirements/extra.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --upgrade
         pip list
       shell: bash
 
     - name: Reinstall Horovod if necessary
-      # todo: re-enable horovod on py3.9 when it will be supported
-      if: runner.os != 'windows' && matrix.python-version != 3.9
+      if: runner.os != 'windows'
       env:
         HOROVOD_BUILD_ARCH_FLAGS: "-mfma"
+        HOROVOD_WITHOUT_MXNET: 1
+        HOROVOD_WITHOUT_TENSORFLOW: 1
       run: |
         HOROVOD_BUILT=$(python -c "import horovod.torch; horovod.torch.nccl_built(); print('SUCCESS')" || true)
         if [[ $HOROVOD_BUILT != "SUCCESS" ]]; then

.github/workflows/code-checks.yml

@@ -14,7 +14,7 @@ jobs:
       - uses: actions/checkout@master
       - uses: actions/setup-python@v2
         with:
-          python-version: 3.8
+          python-version: 3.9
       - name: Install mypy
         run: |
           grep mypy requirements/test.txt | xargs -0 pip install

.github/workflows/docs-checks.yml

@@ -14,10 +14,9 @@ jobs:
       - uses: actions/checkout@v2
         with:
           submodules: true
-          # lfs: true
       - uses: actions/setup-python@v2
         with:
-          python-version: 3.7
+          python-version: 3.9
 
       - name: Weekly reset caching
         run: echo "::set-output name=period::$(python -c 'import time ; days = time.time() / 60 / 60 / 24 ; print(int(days / 7))' 2>&1)"
@@ -66,7 +65,7 @@ jobs:
           # lfs: true
       - uses: actions/setup-python@v2
         with:
-          python-version: 3.7
+          python-version: 3.9
 
       # Note: This uses an internal pip API and may not always work
       # https://github.com/actions/cache/blob/master/examples.md#multiple-oss-in-a-workflow
@@ -83,7 +82,6 @@ jobs:
           sudo apt-get update
           sudo apt-get install -y cmake pandoc
           pip --version
-          # pip install --requirement requirements.txt --upgrade-strategy only-if-needed --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --quiet
           pip install --requirement requirements/docs.txt
           # install Texlive, see https://linuxconfig.org/how-to-install-latex-on-ubuntu-20-04-focal-fossa-linux
           sudo apt-get update && sudo apt-get install -y texlive-latex-extra dvipng texlive-pictures

.github/workflows/events-nightly.yml

@@ -21,7 +21,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2
         with:
-          python-version: 3.8
+          python-version: 3.9
 
       - name: Install dependencies
         run: >-
@@ -54,7 +54,7 @@ jobs:
       fail-fast: false
       matrix:
         python_version: ["3.7"]
-        xla_version: ["1.6", "1.7", "1.8", "1.9"] # todo: , "nightly"
+        xla_version: ["1.6", "1.7", "1.8", "1.9"]
 
     steps:
       - name: Checkout

.github/workflows/legacy-checkpoints.yml

@@ -12,7 +12,7 @@ jobs:
 
     - uses: actions/setup-python@v2
       with:
-        python-version: 3.8
+        python-version: 3.9
 
     - name: Install dependencies
       run: |

.github/workflows/release-docker.yml

@@ -38,7 +38,7 @@ jobs:
 
       - name: Publish Latest to Docker
         uses: docker/[email protected]
-        # only on releases AND latest PT and py
+        # only on releases and latest Python and PyTorch
         if: matrix.python_version == 3.9 && matrix.pytorch_version == 1.9
         with:
           repository: pytorchlightning/pytorch_lightning
@@ -48,25 +48,3 @@ jobs:
           build_args: PYTHON_VERSION=${{ matrix.python_version }},PYTORCH_VERSION=${{ matrix.pytorch_version }},LIGHTNING_VERSION=${{ steps.get_version.outputs.RELEASE_VERSION }}
           tags: "latest"
         timeout-minutes: 55
-
-#  nvidia-PL:
-#    runs-on: ubuntu-20.04
-#    if: startsWith(github.ref, 'refs/tags/') || github.event_name == 'release'
-#    steps:
-#      - name: Checkout
-#        uses: actions/checkout@v2
-#
-#      - name: Get release version
-#        id: get_version
-#        run: echo "::set-output name=RELEASE_VERSION::$(echo ${GITHUB_REF##*/})"
-#
-#      - name: Publish Releases to Docker
-#        uses: docker/[email protected]
-#        with:
-#          repository: nvcr.io/pytorchlightning/pytorch_lightning
-#          username: ${{ secrets.DOCKER_USERNAME }}
-#          password: ${{ secrets.DOCKER_PASSWORD }}
-#          dockerfile: dockers/nvidia/Dockerfile
-#          build_args: LIGHTNING_VERSION=${{ steps.get_version.outputs.RELEASE_VERSION }}
-#          tags: "${{ steps.get_version.outputs.RELEASE_VERSION }}-nvidia"
-#        timeout-minutes: 55

.github/workflows/release-pypi.yml

@@ -16,7 +16,7 @@ jobs:
     - uses: actions/checkout@v2
     - uses: actions/setup-python@v2
       with:
-        python-version: 3.8
+        python-version: 3.9
 
     - name: Install dependencies
       run: >-
@@ -90,7 +90,7 @@ jobs:
 
     - uses: actions/setup-python@v2
       with:
-        python-version: 3.8
+        python-version: 3.9
 
     - name: Weekly reset caching
       run: echo "::set-output name=period::$(python -c 'import time ; days = time.time() / 60 / 60 / 24 ; print(int(days / 7))' 2>&1)"