From 96ca72014cabd0f99448a96ba66d4bf6b98355ba Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Mon, 6 Feb 2023 14:07:08 +0100 Subject: [PATCH 1/5] add generic macOS workflows --- .github/unittest.sh | 84 ++++++++++++++++++++++++++++++++ .github/workflows/test-macos.yml | 34 +++++++++++++ 2 files changed, 118 insertions(+) create mode 100644 .github/unittest.sh create mode 100644 .github/workflows/test-macos.yml diff --git a/.github/unittest.sh b/.github/unittest.sh new file mode 100644 index 00000000000..98889286685 --- /dev/null +++ b/.github/unittest.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash + +set -euo pipefail + +echo '::group::Prepare conda' +CONDA_PATH=$(which conda) +eval "$(${CONDA_PATH} shell.bash hook)" +# The `setuptools` package installed through `conda` includes a patch that errors if something is installed +# through `setuptools` while the `CONDA_BUILD` environment variable is set. +# https://github.com/AnacondaRecipes/setuptools-feedstock/blob/f5d8d256810ce28fc0cf34170bc34e06d3754041/recipe/patches/0002-disable-downloads-inside-conda-build.patch +# (Although we are not using the `-c conda-forge` channel, the patch is equivalent but not public for +# `setuptools` from the `-c defaults` channel) +# Since we aren't using `conda build` here, we unset it to avoid installation problems later +# TODO: investigate where `CONDA_BUILD` is set and maybe fix unset it there +unset CONDA_BUILD +echo '::endgroup::' + +echo '::group::Set PyTorch conda channel' +# TODO: Can we maybe have this as environment variable in the job template? For example, `IS_RELEASE`. +if [[ (${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then + POSTFIX=test +else + POSTFIX=nightly +fi +PYTORCH_CHANNEL=pytorch-"${POSTFIX}" +echo "${PYTORCH_CHANNEL}" +echo '::endgroup::' + +echo '::group::Set PyTorch GPU mutex' +case $GPU_ARCH_TYPE in + cpu) + PYTORCH_MUTEX=cpuonly + ;; + cuda) + PYTORCH_MUTEX="pytorch-cuda=${GPU_ARCH_VERSION}" + ;; + *) + echo "Unknown GPU_ARCH_TYPE=${GPU_ARCH_TYPE}" + exit 1 + ;; +esac +echo "${PYTORCH_MUTEX}" +echo '::endgroup::' + +echo '::group::Create build environment' +conda create \ + --name ci \ + --quiet --yes \ + python="${PYTHON_VERSION}" pip \ + setuptools ninja \ + libpng jpeg \ + Pillow numpy requests +conda activate ci +pip install 'av<10' +echo '::endgroup::' + +echo '::group::Install PyTorch' +conda install \ + --quiet --yes \ + -c "${PYTORCH_CHANNEL}" \ + -c nvidia \ + pytorch \ + "${PYTORCH_MUTEX}" + +if [[ $GPU_ARCH_TYPE = 'cuda' ]]; then + python3 -c "import torch; exit(not torch.cuda.is_available())" +fi +echo '::endgroup::' + +echo '::group::Install TorchVision' +python setup.py develop +echo '::endgroup::' + +echo '::group::Collect PyTorch environment information' +python -m torch.utils.collect_env +echo '::endgroup::' + +echo '::group::Install testing utilities' +pip install --progress-bar=off pytest pytest-mock pytest-cov +echo '::endgroup::' + +echo '::group::Run tests' +pytest --durations=25 +echo '::endgroup::' diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml new file mode 100644 index 00000000000..1c14c8c3a03 --- /dev/null +++ b/.github/workflows/test-macos.yml @@ -0,0 +1,34 @@ +name: Unit-tests on macOS + +on: + pull_request: + push: + branches: + - nightly + - main + - release/* + workflow_dispatch: + +jobs: + tests: + strategy: + matrix: + python-version: + - "3.8" + - "3.9" + - "3.10" + runner: + - macos-12 + - macos-m1-12 + gpu-arch-type: ["cpu"] + fail-fast: false + uses: pytorch/test-infra/.github/workflows/macos_job.yml@main + with: + repository: pytorch/vision + timeout: 120 + runner: ${{ matrix.runner }} + script: | + export PYTHON_VERSION=${{ matrix.python-version }} + export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }} + + ./.github/unittest.sh From e5d289ca18eafe311d45a145adce189e546caa5b Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Mon, 6 Feb 2023 14:08:38 +0100 Subject: [PATCH 2/5] remove standlaone M1 tests --- .github/workflows/test-m1.yml | 50 ----------------------------------- 1 file changed, 50 deletions(-) delete mode 100644 .github/workflows/test-m1.yml diff --git a/.github/workflows/test-m1.yml b/.github/workflows/test-m1.yml deleted file mode 100644 index c03fa9f76e4..00000000000 --- a/.github/workflows/test-m1.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: Unit-tests on M1 -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - workflow_dispatch: -env: - CHANNEL: "nightly" -jobs: - tests: - name: "Unit-tests on M1" - runs-on: macos-m1-12 - strategy: - matrix: - py_vers: [ "3.8"] - - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - name: Set Release CHANNEL (for release) - if: ${{ (github.event_name == 'pull_request' && startsWith(github.base_ref, 'release')) || startsWith(github.ref, 'refs/heads/release') }} - run: | - echo "CHANNEL=test" >> "$GITHUB_ENV" - - name: Install TorchVision - shell: arch -arch arm64 bash {0} - env: - ENV_NAME: conda-env-${{ github.run_id }} - PY_VERS: ${{ matrix.py_vers }} - run: | - . ~/miniconda3/etc/profile.d/conda.sh - # Needed for JPEG library detection as setup.py detects conda presence by running `shutil.which('conda')` - export PATH=~/miniconda3/bin:$PATH - set -ex - conda create -yp ${ENV_NAME} python=${PY_VERS} numpy libpng jpeg scipy - conda run -p ${ENV_NAME} python3 -mpip install --pre torch --extra-index-url=https://download.pytorch.org/whl/${CHANNEL} - conda run -p ${ENV_NAME} python3 setup.py develop - conda run -p ${ENV_NAME} python3 -mpip install pytest pytest-mock 'av<10' - - name: Run tests - shell: arch -arch arm64 bash {0} - env: - ENV_NAME: conda-env-${{ github.run_id }} - PY_VERS: ${{ matrix.py_vers }} - run: | - . ~/miniconda3/etc/profile.d/conda.sh - set -ex - conda run -p ${ENV_NAME} --no-capture-output python3 -u -mpytest -v --tb=long --durations 20 - conda env remove -p ${ENV_NAME} From ecd4fa03489f16ff0820f724488f42deb4943f1a Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Mon, 6 Feb 2023 16:03:17 +0100 Subject: [PATCH 3/5] add execute permissions --- .github/unittest.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 .github/unittest.sh diff --git a/.github/unittest.sh b/.github/unittest.sh old mode 100644 new mode 100755 From 8fb60685d8006332b984aa189c7084253b08c2da Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 8 Feb 2023 09:34:48 +0100 Subject: [PATCH 4/5] only test M1 against one Python version --- .github/workflows/test-macos.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index 1c14c8c3a03..01fdbc8ac43 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -17,10 +17,10 @@ jobs: - "3.8" - "3.9" - "3.10" - runner: - - macos-12 - - macos-m1-12 - gpu-arch-type: ["cpu"] + runner: ["macos-12"] + include: + python-version: "3.8" + runner: macos-m1-12 fail-fast: false uses: pytorch/test-infra/.github/workflows/macos_job.yml@main with: @@ -29,6 +29,6 @@ jobs: runner: ${{ matrix.runner }} script: | export PYTHON_VERSION=${{ matrix.python-version }} - export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }} + export GPU_ARCH_TYPE=cpu ./.github/unittest.sh From 4d314e6e6f89e3f5bd40ed94bbc527b1c933b8fa Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 8 Feb 2023 22:43:59 +0100 Subject: [PATCH 5/5] fix syntax --- .github/workflows/test-macos.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index 01fdbc8ac43..762a46bf139 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -19,8 +19,8 @@ jobs: - "3.10" runner: ["macos-12"] include: - python-version: "3.8" - runner: macos-m1-12 + - python-version: "3.8" + runner: macos-m1-12 fail-fast: false uses: pytorch/test-infra/.github/workflows/macos_job.yml@main with: