From da197b72ff1f0aebcdfe8686b26c792f27c762e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mochol=C3=AD?= Date: Thu, 11 Aug 2022 20:21:05 +0200 Subject: [PATCH] Remove skipping logic in favor of path filtering --- .azure/gpu-tests.yml | 53 ++++++++------------ .github/checkgroup.yml | 1 - .github/file-filters.yml | 9 ---- .github/workflows/ci-app-cloud-e2e-test.yml | 28 ++--------- .github/workflows/ci-app-examples.yml | 7 +++ .github/workflows/ci-app-tests.yml | 4 +- .github/workflows/ci-pytorch-test-conda.yml | 35 +++---------- .github/workflows/ci-pytorch-test-full.yml | 54 +++++---------------- .github/workflows/ci-pytorch-test-slow.yml | 38 ++++----------- 9 files changed, 65 insertions(+), 164 deletions(-) delete mode 100644 .github/file-filters.yml diff --git a/.azure/gpu-tests.yml b/.azure/gpu-tests.yml index 8ae670d265ced..8444468c0c58a 100644 --- a/.azure/gpu-tests.yml +++ b/.azure/gpu-tests.yml @@ -12,15 +12,31 @@ trigger: - "master" - "release/*" - "refs/tags/*" + paths: + include: + - ".azure/**" + - "examples/run_ddp_examples.sh" + - "examples/convert_from_pt_to_pl/**" + - "examples/run_pl_examples.sh" + - "examples/pl_basics/backbone_image_classifier.py" + - "examples/pl_basics/autoencoder.py" + - "examples/pl_loops/mnist_lite.py" + - "examples/pl_fault_tolerant/automatic.py" + - "examples/test_pl_examples.py" + - "examples/pl_integrations/dali_image_classifier.py" + - "requirements/pytorch/**" + - "src/pytorch_lightning/**" + - "tests/tests_pytorch/**" + - "setup.cfg" + - "pyproject.toml" + - ".github/workflows/ci-pytorch*.yml" + - ".github/workflows/docs-*.yml" + pr: - "master" - "release/*" -variables: - - name: continue - value: '1' - jobs: - job: testing strategy: @@ -41,22 +57,6 @@ jobs: clean: all steps: - - - bash: | - CHANGED_FILES=$(git diff --name-status origin/master -- . | awk '{print $2}') - FILTER='src/pytorch_lightning|requirements/pytorch|tests/tests_pytorch|examples/pl_*|.azure/gpu-tests.yml' - echo $CHANGED_FILES > changed_files.txt - MATCHES=$(cat changed_files.txt | grep -E $FILTER) - echo $MATCHES - if [ -z "$MATCHES" ]; then - echo "Skip" - echo "##vso[task.setvariable variable=continue]0" - else - echo "Continue" - echo "##vso[task.setvariable variable=continue]1" - fi - displayName: Skipper - - bash: | lspci | egrep 'VGA|3D' whereis nvidia @@ -66,7 +66,6 @@ jobs: pip --version pip list displayName: 'Image info & NVIDIA' - condition: eq(variables['continue'], '1') - bash: | set -e @@ -82,7 +81,6 @@ jobs: PACKAGE_NAME: pytorch FREEZE_REQUIREMENTS: 1 displayName: 'Install dependencies' - condition: eq(variables['continue'], '1') - bash: | set -e @@ -91,16 +89,13 @@ jobs: python requirements/pytorch/check-avail-strategies.py python requirements/pytorch/check-avail-extras.py displayName: 'Env details' - condition: eq(variables['continue'], '1') - bash: bash .actions/pull_legacy_checkpoints.sh displayName: 'Get legacy checkpoints' - condition: eq(variables['continue'], '1') - bash: python -m coverage run --source pytorch_lightning -m pytest workingDirectory: src/pytorch_lightning displayName: 'Testing: PyTorch doctests' - condition: eq(variables['continue'], '1') - bash: python -m coverage run --source pytorch_lightning -m pytest --ignore benchmarks -v --junitxml=$(Build.StagingDirectory)/test-results.xml --durations=50 env: @@ -108,7 +103,6 @@ jobs: workingDirectory: tests/tests_pytorch displayName: 'Testing: PyTorch standard' timeoutInMinutes: "35" - condition: eq(variables['continue'], '1') - bash: bash run_standalone_tests.sh workingDirectory: tests/tests_pytorch @@ -117,7 +111,6 @@ jobs: PL_RUN_CUDA_TESTS: "1" displayName: 'Testing: PyTorch standalone tests' timeoutInMinutes: "35" - condition: eq(variables['continue'], '1') - bash: bash run_standalone_tasks.sh workingDirectory: tests/tests_pytorch @@ -126,7 +119,6 @@ jobs: PL_RUN_CUDA_TESTS: "1" displayName: 'Testing: PyTorch standalone tasks' timeoutInMinutes: "10" - condition: eq(variables['continue'], '1') - bash: | python -m coverage report @@ -136,14 +128,13 @@ jobs: ls -l workingDirectory: tests/tests_pytorch displayName: 'Statistics' - condition: eq(variables['continue'], '1') - task: PublishTestResults@2 displayName: 'Publish test results' inputs: testResultsFiles: '$(Build.StagingDirectory)/test-results.xml' testRunTitle: '$(Agent.OS) - $(Build.DefinitionName) - Python $(python.version)' - condition: and(succeededOrFailed(), eq(variables['continue'], '1')) + condition: succeededOrFailed() - script: | set -e @@ -155,11 +146,9 @@ jobs: env: PL_USE_MOCKED_MNIST: "1" displayName: 'Testing: PyTorch examples' - condition: eq(variables['continue'], '1') - bash: python -m pytest benchmarks -v --maxfail=2 --durations=0 workingDirectory: tests/tests_pytorch env: PL_RUN_CUDA_TESTS: "1" displayName: 'Testing: PyTorch benchmarks' - condition: eq(variables['continue'], '1') diff --git a/.github/checkgroup.yml b/.github/checkgroup.yml index 8f1d3c6fb5e86..0cb80d6e34bd8 100644 --- a/.github/checkgroup.yml +++ b/.github/checkgroup.yml @@ -101,7 +101,6 @@ subprojects: - id: "lightning_app" paths: - ".github/workflows/ci-app*.yml" - - "examples/app_**" - "requirements/app/**" - "src/lightning_app/**" - "tests/tests_app/**" diff --git a/.github/file-filters.yml b/.github/file-filters.yml deleted file mode 100644 index e621cd83881e4..0000000000000 --- a/.github/file-filters.yml +++ /dev/null @@ -1,9 +0,0 @@ -# This file contains filters to be used in the CI to detect file changes and run the required CI jobs. - -app_examples: - - "src/lightning_app/**" - - "tests/tests_app_examples/**" - - "requirements/app/**" - - "examples/app_*" - - "setup.py" - - "src/pytorch_lightning/__version__.py" diff --git a/.github/workflows/ci-app-cloud-e2e-test.yml b/.github/workflows/ci-app-cloud-e2e-test.yml index 9a5a10a95cd33..707c506e89e5a 100644 --- a/.github/workflows/ci-app-cloud-e2e-test.yml +++ b/.github/workflows/ci-app-cloud-e2e-test.yml @@ -7,37 +7,19 @@ on: # Trigger the workflow on push or pull request, but only for the master bran branches: [master, "release/*"] pull_request: branches: [master, "release/*"] + paths: + - ".github/workflows/ci-app-cloud-e2e-test.yml" + - "requirements/app/**" + - "src/lightning_app/**" + - "examples/app_*" concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} jobs: - # This is job should once only once per PR to detect file changes so run required jobs. - # see .github/file-filters.yml to define file filters and run the jobs based on the output of each filter. - # More info: https://github.com/marketplace/actions/paths-changes-filter - - changes: - runs-on: ubuntu-latest - # Set job outputs to the values from filter step - outputs: - app_examples: ${{ steps.filter.outputs.app_examples }} - steps: - - uses: actions/checkout@v2 - - name: Set up Python 3.8 - uses: actions/setup-python@v4 - with: - python-version: "3.8" - - - uses: dorny/paths-filter@v2 - id: filter - with: - filters: .github/file-filters.yml - cloud-test: name: Cloud Test - needs: changes - if: ${{ needs.changes.outputs.app_examples == 'true' }} runs-on: ubuntu-20.04 strategy: fail-fast: false diff --git a/.github/workflows/ci-app-examples.yml b/.github/workflows/ci-app-examples.yml index 01570f59c2c77..8114f59b01aaa 100644 --- a/.github/workflows/ci-app-examples.yml +++ b/.github/workflows/ci-app-examples.yml @@ -6,6 +6,13 @@ on: # Trigger the workflow on push or pull request, but only for the master bran branches: [master, "release/*"] pull_request: branches: [master, "release/*"] + paths: + - ".github/workflows/ci-app-examples.yml" + - "requirements/app/**" + - "src/lightning_app/**" + - "tests/tests_app_examples/**" + # the examples are used in the app CI + - "examples/app_*" concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }} diff --git a/.github/workflows/ci-app-tests.yml b/.github/workflows/ci-app-tests.yml index fe3cc36dc16d3..fb2cdbda69079 100644 --- a/.github/workflows/ci-app-tests.yml +++ b/.github/workflows/ci-app-tests.yml @@ -6,10 +6,10 @@ on: # Trigger the workflow on push or pull request, but only for the master bran branches: [master, "release/*"] pull_request: paths: + - ".github/workflows/ci-app-tests.yml" + - "requirements/app/**" - "src/lightning_app/**" - "tests/tests_app/**" - - "requirements/app/**" - - "setup.py" concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }} diff --git a/.github/workflows/ci-pytorch-test-conda.yml b/.github/workflows/ci-pytorch-test-conda.yml index 2bbdb699c2c1e..d314a742bbdcb 100644 --- a/.github/workflows/ci-pytorch-test-conda.yml +++ b/.github/workflows/ci-pytorch-test-conda.yml @@ -6,6 +6,12 @@ on: # Trigger the workflow on push or pull request, but only for the master bra branches: [master, "release/*"] pull_request: branches: [master, "release/*"] + paths: + - "requirements/pytorch/**" + - "src/pytorch_lightning/**" + - "tests/tests_pytorch/**" + - "setup.cfg" # includes pytest config + - ".github/workflows/ci-pytorch-test-conda.yml" concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }} @@ -35,28 +41,7 @@ jobs: - uses: actions/checkout@v2 - - name: Get changed files - id: changed-files - uses: tj-actions/changed-files@v24 - - - name: Decide if the test should be skipped - id: skip - shell: bash -l {0} - run: | - FILTER='src/pytorch_lightning|requirements/pytorch|tests/tests_pytorch|examples/pl_*|.github/workflows/ci-pytorch-test-conda.yml' - echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr " " "\n" > changed_files.txt - MATCHES=$(cat changed_files.txt | grep -E $FILTER) - echo $MATCHES - if [ -z "$MATCHES" ]; then - echo "Skip" - echo "::set-output name=continue::0" - else - echo "Continue" - echo "::set-output name=continue::1" - fi - - name: Update base dependencies - if: ${{ (steps.skip.outputs.continue == '1') }} env: PACKAGE_NAME: pytorch FREEZE_REQUIREMENTS: 1 @@ -70,12 +55,10 @@ jobs: run: pip install "Pillow<9.0" # It messes with torchvision - name: DocTests - if: ${{ (steps.skip.outputs.continue == '1') }} working-directory: ./src run: pytest pytorch_lightning --cov=pytorch_lightning - name: Update all dependencies - if: ${{ (steps.skip.outputs.continue == '1') }} env: HOROVOD_BUILD_ARCH_FLAGS: "-mfma" HOROVOD_WITHOUT_MXNET: 1 @@ -95,11 +78,9 @@ jobs: python requirements/pytorch/check-avail-extras.py - name: Pull legacy checkpoints - if: ${{ (steps.skip.outputs.continue == '1') }} run: bash .actions/pull_legacy_checkpoints.sh - name: Testing PyTorch - if: ${{ (steps.skip.outputs.continue == '1') }} working-directory: tests/tests_pytorch run: coverage run --source pytorch_lightning -m pytest -v --timeout 150 --durations=50 --junitxml=results-${{ runner.os }}-torch${{ matrix.pytorch-version }}.xml @@ -111,7 +92,7 @@ jobs: if: failure() - name: Statistics - if: ${{ success() && (steps.skip.outputs.continue == '1') }} + if: success() working-directory: tests/tests_pytorch run: | coverage report @@ -119,7 +100,7 @@ jobs: - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 - if: ${{ success() && (steps.skip.outputs.continue == '1') }} + if: success() # see: https://github.com/actions/toolkit/issues/399 continue-on-error: true with: diff --git a/.github/workflows/ci-pytorch-test-full.yml b/.github/workflows/ci-pytorch-test-full.yml index 7409ce25a5128..386bb012b8cc6 100644 --- a/.github/workflows/ci-pytorch-test-full.yml +++ b/.github/workflows/ci-pytorch-test-full.yml @@ -7,6 +7,12 @@ on: # Trigger the workflow on push or pull request, but only for the master bra pull_request: branches: [master, "release/*"] types: [opened, reopened, ready_for_review, synchronize] + paths: + - "requirements/pytorch/**" + - "src/pytorch_lightning/**" + - "tests/tests_pytorch/**" + - "setup.cfg" # includes pytest config + - ".github/workflows/ci-pytorch-test-full.yml" concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }} @@ -37,67 +43,42 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Get changed files - id: changed-files - uses: tj-actions/changed-files@v24 - - - name: Decide if the test should be skipped - id: skip - shell: bash -l {0} - run: | - FILTER='src/pytorch_lightning|requirements/pytorch|tests/tests_pytorch|examples/pl_*|.github/workflows/ci-pytorch_test-full.yml' - echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr " " "\n" > changed_files.txt - MATCHES=$(cat changed_files.txt | grep -E $FILTER) - echo $MATCHES - if [ -z "$MATCHES" ]; then - echo "Skip" - echo "::set-output name=continue::0" - else - echo "Continue" - echo "::set-output name=continue::1" - fi - - name: Set up Python ${{ matrix.python-version }} - if: ${{ (steps.skip.outputs.continue == '1') }} uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Reset caching - if: ${{ (steps.skip.outputs.continue == '1') }} run: python -c "import time; days = time.time() / 60 / 60 / 24; print(f'TIME_PERIOD=d{int(days / 2) * 2}')" >> $GITHUB_ENV - name: basic setup - if: ${{ (steps.skip.outputs.continue == '1') }} run: | pip --version pip install -q fire # Github Actions: Run step on specific OS: https://stackoverflow.com/a/57948488/4521646 - name: Setup macOS - if: ${{ (runner.os == 'macOS') && (steps.skip.outputs.continue == '1') }} + if: ${{ (runner.os == 'macOS') }} run: | brew install openmpi libuv # Horovod on macOS requires OpenMPI, Gloo not currently supported - name: Setup Windows - if: ${{ (runner.os == 'windows') && (steps.skip.outputs.continue == '1') }} + if: ${{ (runner.os == 'windows') }} run: | python .actions/assistant.py requirements_prune_pkgs horovod - name: Set min. dependencies - if: ${{ (matrix.requires == 'oldest') && (steps.skip.outputs.continue == '1') }} + if: ${{ (matrix.requires == 'oldest') }} run: | python .actions/assistant.py replace_oldest_ver # Note: This uses an internal pip API and may not always work # https://github.com/actions/cache/blob/master/examples.md#multiple-oss-in-a-workflow - name: Get pip cache dir - if: ${{ (steps.skip.outputs.continue == '1') }} id: pip-cache run: echo "::set-output name=dir::$(pip cache dir)" - name: pip cache - if: ${{ (steps.skip.outputs.continue == '1') }} uses: actions/cache@v3 with: path: ${{ steps.pip-cache.outputs.dir }} @@ -106,11 +87,9 @@ jobs: ${{ runner.os }}-pip-td${{ env.TIME_PERIOD }}-py${{ matrix.python-version }}-${{ matrix.release }}-${{ matrix.requires }}- - name: Pull legacy checkpoints - if: ${{ (steps.skip.outputs.continue == '1') }} run: bash .actions/pull_legacy_checkpoints.sh - name: Install dependencies - if: ${{ (steps.skip.outputs.continue == '1') }} env: PACKAGE_NAME: pytorch FREEZE_REQUIREMENTS: 1 @@ -122,12 +101,10 @@ jobs: shell: bash - name: DocTests - if: ${{ (steps.skip.outputs.continue == '1') }} working-directory: ./src run: pytest pytorch_lightning --cov=pytorch_lightning - name: Install extra dependencies - if: ${{ (steps.skip.outputs.continue == '1') }} run: | # adjust versions according installed Torch version python ./requirements/pytorch/adjust-versions.py requirements/pytorch/extra.txt @@ -136,7 +113,7 @@ jobs: shell: bash - name: Reinstall Horovod if necessary - if: ${{ (runner.os != 'windows') && (steps.skip.outputs.continue == '1') }} + if: ${{ (runner.os != 'windows') }} env: HOROVOD_BUILD_ARCH_FLAGS: "-mfma" HOROVOD_WITHOUT_MXNET: 1 @@ -153,43 +130,38 @@ jobs: shell: bash - name: Cache datasets - if: ${{ (steps.skip.outputs.continue == '1') }} uses: actions/cache@v3 with: path: Datasets key: pl-dataset - name: Sanity check - if: ${{ (steps.skip.outputs.continue == '1') }} run: python requirements/pytorch/check-avail-extras.py - name: Testing PyTorch - if: ${{ (steps.skip.outputs.continue == '1') }} working-directory: tests/tests_pytorch # NOTE: do not include coverage report here, see: https://github.com/nedbat/coveragepy/issues/1003 run: coverage run --source pytorch_lightning -m pytest -v --durations=50 --junitxml=results-${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }}.xml - name: Upload pytest results - if: ${{ (failure()) && (steps.skip.outputs.continue == '1') }} + if: failure() uses: actions/upload-artifact@v3 with: name: unittest-results-${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }} path: tests/tests_pytorch/results-${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }}.xml - name: Prepare Examples - if: ${{ (steps.skip.outputs.continue == '1') }} run: | # adjust versions according installed Torch version python ./requirements/pytorch/adjust-versions.py requirements/pytorch/examples.txt pip install -r requirements/pytorch/examples.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --upgrade - name: Run Examples - if: ${{ (steps.skip.outputs.continue == '1') }} working-directory: ./examples run: python -m pytest test_pl_examples.py -v --durations=10 - name: Statistics - if: ${{ (success()) && (steps.skip.outputs.continue == '1') }} + if: success() working-directory: tests/tests_pytorch run: | coverage report @@ -197,7 +169,7 @@ jobs: - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 - if: ${{ (always()) && (steps.skip.outputs.continue == '1') }} + if: always() # see: https://github.com/actions/toolkit/issues/399 continue-on-error: true with: diff --git a/.github/workflows/ci-pytorch-test-slow.yml b/.github/workflows/ci-pytorch-test-slow.yml index 36007d3311451..8e97ea90b2bc4 100644 --- a/.github/workflows/ci-pytorch-test-slow.yml +++ b/.github/workflows/ci-pytorch-test-slow.yml @@ -7,6 +7,12 @@ on: # Trigger the workflow on push or pull request, but only for the master bra pull_request: branches: [master, "release/*"] types: [opened, reopened, ready_for_review, synchronize] + paths: + - "requirements/pytorch/**" + - "src/pytorch_lightning/**" + - "tests/tests_pytorch/**" + - "setup.cfg" # includes pytest config + - ".github/workflows/ci-pytorch-test-slow.yml" concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }} @@ -28,43 +34,19 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Get changed files - id: changed-files - uses: tj-actions/changed-files@v24 - - - name: Decide if the test should be skipped - id: skip - shell: bash -l {0} - run: | - FILTER='src/pytorch_lightning|requirements/pytorch|tests/tests_pytorch|examples/pl_*|.github/workflows/ci-pytorch_test-slow.yml' - echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr " " "\n" > changed_files.txt - MATCHES=$(cat changed_files.txt | grep -E $FILTER) - echo $MATCHES - if [ -z "$MATCHES" ]; then - echo "Skip" - echo "::set-output name=continue::0" - else - echo "Continue" - echo "::set-output name=continue::1" - fi - - uses: actions/setup-python@v4 - if: ${{ (steps.skip.outputs.continue == '1') }} with: python-version: ${{ matrix.python-version }} - name: Reset caching - if: ${{ (steps.skip.outputs.continue == '1') }} run: python -c "import time; days = time.time() / 60 / 60 / 24; print(f'TIME_PERIOD=d{int(days / 2) * 2}')" >> $GITHUB_ENV - name: Get pip cache - if: ${{ (steps.skip.outputs.continue == '1') }} id: pip-cache run: | python -c "from pip._internal.locations import USER_CACHE_DIR; print('::set-output name=dir::' + USER_CACHE_DIR)" - name: Cache pip - if: ${{ (steps.skip.outputs.continue == '1') }} uses: actions/cache@v3 with: path: ${{ steps.pip-cache.outputs.dir }} @@ -73,7 +55,6 @@ jobs: ${{ runner.os }}-pip-td${{ env.TIME_PERIOD }}-py${{ matrix.python-version }}- - name: Install dependencies - if: ${{ (steps.skip.outputs.continue == '1') }} env: PACKAGE_NAME: pytorch FREEZE_REQUIREMENTS: 1 @@ -85,21 +66,20 @@ jobs: shell: bash - name: Testing PyTorch - if: ${{ (steps.skip.outputs.continue == '1') }} working-directory: tests/tests_pytorch run: coverage run --source pytorch_lightning -m pytest -v --junitxml=results-${{ runner.os }}-py${{ matrix.python-version }}.xml env: PL_RUN_SLOW_TESTS: 1 - name: Upload pytest test results - if: ${{ (failure()) && (steps.skip.outputs.continue == '1') }} + if: failure() uses: actions/upload-artifact@v3 with: name: unittest-results-${{ runner.os }}-py${{ matrix.python-version }} path: tests/tests_pytorch/results-${{ runner.os }}-py${{ matrix.python-version }}.xml - name: Statistics - if: ${{ (success()) && (steps.skip.outputs.continue == '1') }} + if: success() working-directory: tests/tests_pytorch run: | coverage report @@ -107,7 +87,7 @@ jobs: - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 - if: ${{ (success()) && (steps.skip.outputs.continue == '1') }} + if: success() # see: https://github.com/actions/toolkit/issues/399 continue-on-error: true with: