From 913147f0072b074d3e67d6e59934c399b71957cf Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Mon, 2 May 2022 11:17:55 +0200 Subject: [PATCH 01/19] disable CircleCI while debugging --- .circleci/config.yml | 3303 ------------------------------------------ 1 file changed, 3303 deletions(-) delete mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index ece9d1a7b97..00000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,3303 +0,0 @@ -version: 2.1 - -# How to test the Linux jobs: -# - Install CircleCI local CLI: https://circleci.com/docs/2.0/local-cli/ -# - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_linux_wheel_py3.7 -# - Replace binary_linux_wheel_py3.7 with the name of the job you want to test. -# Job names are 'name:' key. - -executors: - windows-cpu: - machine: - resource_class: windows.xlarge - image: windows-server-2019-vs2019:stable - shell: bash.exe - - windows-gpu: - machine: - resource_class: windows.gpu.nvidia.medium - image: windows-server-2019-nvidia:stable - shell: bash.exe - -commands: - checkout_merge: - description: "checkout merge branch" - steps: - - checkout -# - run: -# name: Checkout merge branch -# command: | -# set -ex -# BRANCH=$(git rev-parse --abbrev-ref HEAD) -# if [[ "$BRANCH" != "main" ]]; then -# git fetch --force origin ${CIRCLE_BRANCH}/merge:merged/${CIRCLE_BRANCH} -# git checkout "merged/$CIRCLE_BRANCH" -# fi - designate_upload_channel: - description: "inserts the correct upload channel into ${BASH_ENV}" - steps: - - run: - name: adding UPLOAD_CHANNEL to BASH_ENV - command: | - our_upload_channel=nightly - # On tags upload to test instead - if [[ -n "${CIRCLE_TAG}" ]]; then - our_upload_channel=test - fi - echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV} - - brew_update: - description: "Update Homebrew and install base formulae" - steps: - - run: - name: Update Homebrew - no_output_timeout: "10m" - command: | - set -ex - - # Update repositories manually. - # Running `brew update` produces a comparison between the - # current checkout and the updated checkout, which takes a - # very long time because the existing checkout is 2y old. - for path in $(find /usr/local/Homebrew -type d -name .git) - do - cd $path/.. - git fetch --depth=1 origin - git reset --hard origin/master - done - - export HOMEBREW_NO_AUTO_UPDATE=1 - - # Install expect and moreutils so that we can call `unbuffer` and `ts`. - # moreutils installs a `parallel` executable by default, which conflicts - # with the executable from the GNU `parallel`, so we must unlink GNU - # `parallel` first, and relink it afterwards. - brew install coreutils - brew unlink parallel - brew install moreutils - brew link parallel --overwrite - brew install expect - - brew_install: - description: "Install Homebrew formulae" - parameters: - formulae: - type: string - default: "" - steps: - - run: - name: Install << parameters.formulae >> - no_output_timeout: "10m" - command: | - set -ex - export HOMEBREW_NO_AUTO_UPDATE=1 - brew install << parameters.formulae >> - - run_brew_for_ios_build: - steps: - - brew_update - - brew_install: - formulae: libtool - - apt_install: - parameters: - args: - type: string - descr: - type: string - default: "" - update: - type: boolean - default: true - steps: - - run: - name: > - <<^ parameters.descr >> apt install << parameters.args >> <> - <<# parameters.descr >> << parameters.descr >> <> - command: | - <<# parameters.update >> sudo apt update -qy <> - sudo apt install << parameters.args >> - - pip_install: - parameters: - args: - type: string - descr: - type: string - default: "" - user: - type: boolean - default: true - steps: - - run: - name: > - <<^ parameters.descr >> pip install << parameters.args >> <> - <<# parameters.descr >> << parameters.descr >> <> - command: > - pip install - <<# parameters.user >> --user <> - --progress-bar=off - << parameters.args >> - - install_torchvision: - parameters: - editable: - type: boolean - default: true - steps: - - pip_install: - args: --pre torch --extra-index-url https://download.pytorch.org/whl/nightly/cpu - descr: Install PyTorch from nightly releases - - pip_install: - args: --no-build-isolation <<# parameters.editable >> --editable <> . - descr: Install torchvision <<# parameters.editable >> in editable mode <> - - install_prototype_dependencies: - steps: - - pip_install: - args: iopath - descr: Install third-party dependencies - - pip_install: - args: --pre torchdata --extra-index-url https://download.pytorch.org/whl/nightly/cpu - descr: Install torchdata from nightly releases - - # Most of the test suite is handled by the `unittest` jobs, with completely different workflow and setup. - # This command can be used if only a selection of tests need to be run, for ad-hoc files. - run_tests_selective: - parameters: - file_or_dir: - type: string - steps: - - run: - name: Install test utilities - command: pip install --progress-bar=off pytest pytest-mock - - run: - name: Run tests - command: pytest --junitxml=test-results/junit.xml -v --durations 20 <> - - store_test_results: - path: test-results - - download_model_weights: - parameters: - extract_roots: - type: string - default: "torchvision/models" - background: - type: boolean - default: true - steps: - - apt_install: - args: parallel wget - descr: Install download utilitites - - run: - name: Download model weights - background: << parameters.background >> - command: | - mkdir -p ~/.cache/torch/hub/checkpoints - python scripts/collect_model_urls.py << parameters.extract_roots >> \ - | parallel -j0 'wget --no-verbose -O ~/.cache/torch/hub/checkpoints/`basename {}` {}\?source=ci' - -binary_common: &binary_common - parameters: - # Edit these defaults to do a release - build_version: - description: "version number of release binary; by default, build a nightly" - type: string - default: "" - pytorch_version: - description: "PyTorch version to build against; by default, use a nightly" - type: string - default: "" - # Don't edit these - python_version: - description: "Python version to build against (e.g., 3.7)" - type: string - cu_version: - description: "CUDA version to build against, in CU format (e.g., cpu or cu100)" - type: string - default: "cpu" - unicode_abi: - description: "Python 2.7 wheel only: whether or not we are cp27mu (default: no)" - type: string - default: "" - wheel_docker_image: - description: "Wheel only: what docker image to use" - type: string - default: "pytorch/manylinux-cuda102" - conda_docker_image: - description: "Conda only: what docker image to use" - type: string - default: "pytorch/conda-builder:cpu" - environment: - PYTHON_VERSION: << parameters.python_version >> - PYTORCH_VERSION: << parameters.pytorch_version >> - UNICODE_ABI: << parameters.unicode_abi >> - CU_VERSION: << parameters.cu_version >> - -torchvision_ios_params: &torchvision_ios_params - parameters: - build_environment: - type: string - default: "" - ios_arch: - type: string - default: "" - ios_platform: - type: string - default: "" - environment: - BUILD_ENVIRONMENT: << parameters.build_environment >> - IOS_ARCH: << parameters.ios_arch >> - IOS_PLATFORM: << parameters.ios_platform >> - -torchvision_android_params: &torchvision_android_params - parameters: - build_environment: - type: string - default: "" - environment: - BUILD_ENVIRONMENT: << parameters.build_environment >> - -smoke_test_common: &smoke_test_common - <<: *binary_common - docker: - - image: torchvision/smoke_test:latest - -jobs: - circleci_consistency: - docker: - - image: cimg/python:3.7 - steps: - - checkout - - pip_install: - args: jinja2 pyyaml - - run: - name: Check CircleCI config consistency - command: | - python .circleci/regenerate.py - git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1) - - lint_python_and_config: - docker: - - image: cimg/python:3.7 - steps: - - checkout - - pip_install: - args: pre-commit - descr: Install lint utilities - - run: - name: Install pre-commit hooks - command: pre-commit install-hooks - - run: - name: Lint Python code and config files - command: pre-commit run --all-files - - run: - name: Required lint modifications - when: on_fail - command: git --no-pager diff - - lint_c: - docker: - - image: cimg/python:3.7 - steps: - - apt_install: - args: libtinfo5 - descr: Install additional system libraries - - checkout - - run: - name: Install lint utilities - command: | - curl https://oss-clang-format.s3.us-east-2.amazonaws.com/linux64/clang-format-linux64 -o clang-format - chmod +x clang-format - sudo mv clang-format /opt/clang-format - - run: - name: Lint C code - command: ./.circleci/unittest/linux/scripts/run-clang-format.py -r torchvision/csrc --clang-format-executable /opt/clang-format - - run: - name: Required lint modifications - when: on_fail - command: git --no-pager diff - - type_check_python: - docker: - - image: cimg/python:3.7 - steps: - - apt_install: - args: libturbojpeg-dev - descr: Install additional system libraries - - checkout - - install_torchvision: - editable: true - - install_prototype_dependencies - - pip_install: - args: mypy - descr: Install Python type check utilities - - run: - name: Check Python types statically - command: mypy --install-types --non-interactive --config-file mypy.ini - - unittest_torchhub: - docker: - - image: cimg/python:3.7 - steps: - - checkout - - install_torchvision - - run_tests_selective: - file_or_dir: test/test_hub.py - - unittest_onnx: - docker: - - image: cimg/python:3.7 - steps: - - checkout - - install_torchvision - - pip_install: - args: onnx onnxruntime - descr: Install ONNX - - run_tests_selective: - file_or_dir: test/test_onnx.py - - unittest_extended: - docker: - - image: cimg/python:3.7 - resource_class: xlarge - steps: - - checkout - - download_model_weights - - install_torchvision - - run: - name: Enable extended tests - command: echo 'export PYTORCH_TEST_WITH_EXTENDED=1' >> $BASH_ENV - - run_tests_selective: - file_or_dir: test/test_extended_*.py - - binary_linux_wheel: - <<: *binary_common - docker: - - image: << parameters.wheel_docker_image >> - resource_class: 2xlarge+ - steps: - - checkout_merge - - designate_upload_channel - - run: packaging/build_wheel.sh - - store_artifacts: - path: dist - - persist_to_workspace: - root: dist - paths: - - "*" - - binary_linux_conda: - <<: *binary_common - docker: - - image: "<< parameters.conda_docker_image >>" - resource_class: 2xlarge+ - steps: - - checkout_merge - - designate_upload_channel - - run: packaging/build_conda.sh - - store_artifacts: - path: /opt/conda/conda-bld/linux-64 - - persist_to_workspace: - root: /opt/conda/conda-bld/linux-64 - paths: - - "*" - - store_test_results: - path: build_results/ - - binary_win_conda: - <<: *binary_common - executor: windows-cpu - steps: - - checkout_merge - - designate_upload_channel - - run: - name: Build conda packages - no_output_timeout: 20m - command: | - set -ex - source packaging/windows/internal/vc_install_helper.sh - packaging/windows/internal/cuda_install.bat - eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" - conda activate base - conda install -yq conda-build "conda-package-handling!=1.5.0" - # cudatoolkit >= 11 isn't available for windows in the nvidia channel - if [[ "${CU_VERSION}" =~ cu11.* ]]; then - export CONDA_CHANNEL_FLAGS="-c conda-forge" - fi - packaging/build_conda.sh - rm /C/tools/miniconda3/conda-bld/win-64/vs${VC_YEAR}*.tar.bz2 - - store_artifacts: - path: C:/tools/miniconda3/conda-bld/win-64 - - persist_to_workspace: - root: C:/tools/miniconda3/conda-bld/win-64 - paths: - - "*" - - store_test_results: - path: build_results/ - - binary_win_wheel: - <<: *binary_common - executor: windows-cpu - steps: - - checkout_merge - - designate_upload_channel - - run: - name: Build wheel packages - command: | - set -ex - source packaging/windows/internal/vc_install_helper.sh - packaging/windows/internal/cuda_install.bat - packaging/build_wheel.sh - - store_artifacts: - path: dist - - persist_to_workspace: - root: dist - paths: - - "*" - - store_test_results: - path: build_results/ - - binary_macos_wheel: - <<: *binary_common - macos: - xcode: "12.0" - steps: - - checkout_merge - - designate_upload_channel - - run: - # Cannot easily deduplicate this as source'ing activate - # will set environment variables which we need to propagate - # to build_wheel.sh - command: | - curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh - sh conda.sh -b - source $HOME/miniconda3/bin/activate - packaging/build_wheel.sh - - store_artifacts: - path: dist - - persist_to_workspace: - root: dist - paths: - - "*" - - binary_ios_build: - <<: *torchvision_ios_params - macos: - xcode: "12.0" - steps: - - attach_workspace: - at: ~/workspace - - checkout - - run_brew_for_ios_build - - run: - name: Build - no_output_timeout: "1h" - command: | - script="/Users/distiller/project/.circleci/unittest/ios/scripts/binary_ios_build.sh" - cat "$script" - source "$script" - - persist_to_workspace: - root: /Users/distiller/workspace/ - paths: ios - - binary_ios_upload: - <<: *torchvision_ios_params - macos: - xcode: "12.0" - steps: - - attach_workspace: - at: ~/workspace - - checkout - - run_brew_for_ios_build - - run: - name: Upload - no_output_timeout: "1h" - command: | - script="/Users/distiller/project/.circleci/unittest/ios/scripts/binary_ios_upload.sh" - cat "$script" - source "$script" - - binary_android_build: - <<: *torchvision_android_params - docker: - - image: cimg/android:2021.08-ndk - resource_class: xlarge - steps: - - attach_workspace: - at: ~/workspace - - checkout - - run: - name: Build - no_output_timeout: "1h" - command: | - script="/home/circleci/project/.circleci/unittest/android/scripts/binary_android_build.sh" - cat "$script" - source "$script" - - store_artifacts: - path: ~/workspace/artifacts - - binary_android_upload: - <<: *torchvision_android_params - docker: - - image: cimg/android:2021.08-ndk - resource_class: xlarge - steps: - - attach_workspace: - at: ~/workspace - - checkout - - run: - name: Upload - no_output_timeout: "1h" - command: | - script="/home/circleci/project/.circleci/unittest/android/scripts/binary_android_upload.sh" - cat "$script" - source "$script" - - binary_macos_conda: - <<: *binary_common - macos: - xcode: "12.0" - steps: - - checkout_merge - - designate_upload_channel - - run: - command: | - curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh - sh conda.sh -b - source $HOME/miniconda3/bin/activate - conda install -yq conda-build - packaging/build_conda.sh - - store_artifacts: - path: /Users/distiller/miniconda3/conda-bld/osx-64 - - persist_to_workspace: - root: /Users/distiller/miniconda3/conda-bld/osx-64 - paths: - - "*" - - store_test_results: - path: build_results/ - - # Requires org-member context - binary_conda_upload: - docker: - - image: continuumio/miniconda - steps: - - attach_workspace: - at: ~/workspace - - designate_upload_channel - - run: - command: | - # Prevent credential from leaking - conda install -yq anaconda-client - set -x - anaconda -t "${CONDA_PYTORCHBOT_TOKEN}" upload ~/workspace/*.tar.bz2 -u "pytorch-${UPLOAD_CHANNEL}" --label main --no-progress --force - - # Requires org-member context - binary_wheel_upload: - parameters: - subfolder: - description: "What whl subfolder to upload to, e.g., blank or cu100/ (trailing slash is important)" - type: string - docker: - - image: cimg/python:3.7 - steps: - - attach_workspace: - at: ~/workspace - - designate_upload_channel - - checkout - - pip_install: - args: awscli - - run: - command: | - export PATH="$HOME/.local/bin:$PATH" - # Prevent credential from leaking - set +x - export AWS_ACCESS_KEY_ID="${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}" - export AWS_SECRET_ACCESS_KEY="${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}" - set -x - for pkg in ~/workspace/*.whl; do - aws s3 cp "$pkg" "s3://pytorch/whl/${UPLOAD_CHANNEL}/<< parameters.subfolder >>" --acl public-read - done - - smoke_test_linux_conda: - <<: *smoke_test_common - steps: - - attach_workspace: - at: ~/workspace - - designate_upload_channel - - run: - name: install binaries - command: | - set -x - source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION} - conda install -v -y -c pytorch-nightly pytorch - conda install -v -y $(ls ~/workspace/torchvision*.tar.bz2) - - run: - name: smoke test - command: | - source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION} - python -c "import torchvision" - - smoke_test_linux_pip: - <<: *smoke_test_common - steps: - - attach_workspace: - at: ~/workspace - - designate_upload_channel - - run: - name: install binaries - command: | - set -x - source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION} - - pip_install: - args: $(ls ~/workspace/torchvision*.whl) --pre -f https://download.pytorch.org/whl/nightly/torch_nightly.html - - run: - name: smoke test - command: | - source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION} - python -c "import torchvision" - - smoke_test_docker_image_build: - machine: - image: ubuntu-2004:202104-01 - resource_class: large - environment: - image_name: torchvision/smoke_test - steps: - - checkout - - designate_upload_channel - - run: - name: Build and push Docker image - no_output_timeout: "1h" - command: | - set +x - echo "${DOCKER_HUB_TOKEN}" | docker login --username "${DOCKER_HUB_USERNAME}" --password-stdin - set -x - cd .circleci/smoke_test/docker && docker build . -t ${image_name}:${CIRCLE_WORKFLOW_ID} - docker tag ${image_name}:${CIRCLE_WORKFLOW_ID} ${image_name}:latest - docker push ${image_name}:${CIRCLE_WORKFLOW_ID} - docker push ${image_name}:latest - - smoke_test_win_conda: - <<: *binary_common - executor: - name: windows-cpu - steps: - - attach_workspace: - at: ~/workspace - - designate_upload_channel - - run: - name: install binaries - command: | - set -x - eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" - conda env remove -n python${PYTHON_VERSION} || true - conda create -yn python${PYTHON_VERSION} python=${PYTHON_VERSION} - conda activate python${PYTHON_VERSION} - conda install "Pillow>=5.3.0,!=8.3.*" - conda install -v -y -c pytorch-nightly pytorch - conda install -v -y $(ls ~/workspace/torchvision*.tar.bz2) - - run: - name: smoke test - command: | - eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" - conda activate python${PYTHON_VERSION} - python -c "import torchvision" - - smoke_test_win_pip: - <<: *binary_common - executor: - name: windows-cpu - steps: - - attach_workspace: - at: ~/workspace - - designate_upload_channel - - run: - name: install binaries - command: | - set -x - eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" - conda create -yn python${PYTHON_VERSION} python=${PYTHON_VERSION} - conda activate python${PYTHON_VERSION} - - pip_install: - args: $(ls ~/workspace/torchvision*.whl) --pre -f https://download.pytorch.org/whl/nightly/torch_nightly.html - - run: - name: smoke test - command: | - eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" - conda activate python${PYTHON_VERSION} - python -c "import torchvision" - - unittest_linux_cpu: - <<: *binary_common - docker: - - image: "pytorch/manylinux-cuda102" - resource_class: 2xlarge+ - steps: - - checkout - - designate_upload_channel - - run: - name: Generate cache key - # This will refresh cache on Sundays, nightly build should generate new cache. - command: echo "$(date +"%Y-%U")" > .circleci-weekly - - restore_cache: - - keys: - - env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} - - - run: - name: Setup - command: .circleci/unittest/linux/scripts/setup_env.sh - - save_cache: - - key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} - - paths: - - conda - - env - - run: - name: Install torchvision - command: .circleci/unittest/linux/scripts/install.sh - - run: - name: Run tests - command: .circleci/unittest/linux/scripts/run_test.sh - - run: - name: Post process - command: .circleci/unittest/linux/scripts/post_process.sh - - store_test_results: - path: test-results - - unittest_linux_gpu: - <<: *binary_common - machine: - image: ubuntu-2004-cuda-11.4:202110-01 - resource_class: gpu.nvidia.medium - environment: - image_name: "pytorch/manylinux-cuda102" - PYTHON_VERSION: << parameters.python_version >> - steps: - - checkout - - designate_upload_channel - - run: - name: Generate cache key - # This will refresh cache on Sundays, nightly build should generate new cache. - command: echo "$(date +"%Y-%U")" > .circleci-weekly - - restore_cache: - - keys: - - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} - - - run: - name: Setup - command: docker run -e PYTHON_VERSION -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/setup_env.sh - - save_cache: - - key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} - - paths: - - conda - - env - - run: - # Here we create an envlist file that contains some env variables that we want the docker container to be aware of. - # Normally, the CIRCLECI variable is set and available on all CI workflows: https://circleci.com/docs/2.0/env-vars/#built-in-environment-variables. - # They're avaiable in all the other workflows (OSX and Windows). - # But here, we're running the unittest_linux_gpu workflows in a docker container, where those variables aren't accessible. - # So instead we dump the variables we need in env.list and we pass that file when invoking "docker run". - name: export CIRCLECI env var - command: echo "CIRCLECI=true" >> ./env.list - - run: - name: Install torchvision - command: docker run -t --gpus all -v $PWD:$PWD -w $PWD -e UPLOAD_CHANNEL -e CU_VERSION "${image_name}" .circleci/unittest/linux/scripts/install.sh - - run: - name: Run tests - command: docker run --env-file ./env.list -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/run_test.sh - - run: - name: Post Process - command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/post_process.sh - - store_test_results: - path: test-results - - unittest_windows_cpu: - <<: *binary_common - executor: - name: windows-cpu - steps: - - checkout - - designate_upload_channel - - run: - name: Generate cache key - # This will refresh cache on Sundays, nightly build should generate new cache. - command: echo "$(date +"%Y-%U")" > .circleci-weekly - - restore_cache: - - keys: - - env-v2-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} - - - run: - name: Setup - command: .circleci/unittest/windows/scripts/setup_env.sh - - save_cache: - - key: env-v2-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} - - paths: - - conda - - env - - run: - name: Install torchvision - command: .circleci/unittest/windows/scripts/install.sh - - run: - name: Run tests - command: .circleci/unittest/windows/scripts/run_test.sh - - run: - name: Post process - command: .circleci/unittest/windows/scripts/post_process.sh - - store_test_results: - path: test-results - - unittest_windows_gpu: - <<: *binary_common - executor: - name: windows-gpu - environment: - CUDA_VERSION: "11.3" - PYTHON_VERSION: << parameters.python_version >> - steps: - - checkout - - designate_upload_channel - - run: - name: Generate cache key - # This will refresh cache on Sundays, nightly build should generate new cache. - command: echo "$(date +"%Y-%U")" > .circleci-weekly - - restore_cache: - - keys: - - env-v1-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} - - - run: - name: Setup - command: .circleci/unittest/windows/scripts/setup_env.sh - - save_cache: - - key: env-v1-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} - - paths: - - conda - - env - - run: - name: Install CUDA - command: packaging/windows/internal/cuda_install.bat - - run: - name: Update CUDA driver - command: packaging/windows/internal/driver_update.bat - - run: - name: Install torchvision - command: .circleci/unittest/windows/scripts/install.sh - - run: - name: Run tests - command: .circleci/unittest/windows/scripts/run_test.sh - - run: - name: Post process - command: .circleci/unittest/windows/scripts/post_process.sh - - store_test_results: - path: test-results - - unittest_macos_cpu: - <<: *binary_common - macos: - xcode: "12.0" - resource_class: large - steps: - - checkout - - designate_upload_channel - - run: - name: Install wget - command: HOMEBREW_NO_AUTO_UPDATE=1 brew install wget - # Disable brew auto update which is very slow - - run: - name: Generate cache key - # This will refresh cache on Sundays, nightly build should generate new cache. - command: echo "$(date +"%Y-%U")" > .circleci-weekly - - restore_cache: - - keys: - - env-v3-macos-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} - - - run: - name: Setup - command: .circleci/unittest/linux/scripts/setup_env.sh - - save_cache: - - key: env-v3-macos-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} - - paths: - - conda - - env - - run: - name: Install torchvision - command: .circleci/unittest/linux/scripts/install.sh - - run: - name: Run tests - command: .circleci/unittest/linux/scripts/run_test.sh - - run: - name: Post process - command: .circleci/unittest/linux/scripts/post_process.sh - - store_test_results: - path: test-results - - cmake_linux_cpu: - <<: *binary_common - docker: - - image: "pytorch/manylinux-cuda102" - resource_class: 2xlarge+ - steps: - - checkout_merge - - designate_upload_channel - - run: - name: Setup conda - command: .circleci/unittest/linux/scripts/setup_env.sh - - run: packaging/build_cmake.sh - - cmake_linux_gpu: - <<: *binary_common - machine: - image: ubuntu-2004-cuda-11.4:202110-01 - resource_class: gpu.nvidia.small - environment: - PYTHON_VERSION: << parameters.python_version >> - PYTORCH_VERSION: << parameters.pytorch_version >> - UNICODE_ABI: << parameters.unicode_abi >> - CU_VERSION: << parameters.cu_version >> - steps: - - checkout_merge - - designate_upload_channel - - run: - name: Setup conda - command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> .circleci/unittest/linux/scripts/setup_env.sh - - run: - name: Build torchvision C++ distribution and test - command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -e UPLOAD_CHANNEL -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> packaging/build_cmake.sh - - cmake_macos_cpu: - <<: *binary_common - macos: - xcode: "12.0" - steps: - - checkout_merge - - designate_upload_channel - - run: - command: | - curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh - sh conda.sh -b - source $HOME/miniconda3/bin/activate - conda install -yq conda-build cmake - packaging/build_cmake.sh - - cmake_windows_cpu: - <<: *binary_common - executor: - name: windows-cpu - steps: - - checkout_merge - - designate_upload_channel - - run: - command: | - set -ex - source packaging/windows/internal/vc_install_helper.sh - packaging/build_cmake.sh - - cmake_windows_gpu: - <<: *binary_common - executor: - name: windows-gpu - steps: - - checkout_merge - - designate_upload_channel - - run: - command: | - set -ex - source packaging/windows/internal/vc_install_helper.sh - packaging/windows/internal/cuda_install.bat - packaging/build_cmake.sh - - build_docs: - <<: *binary_common - docker: - - image: cimg/python:3.7 - resource_class: 2xlarge+ - steps: - - attach_workspace: - at: ~/workspace - - checkout - - download_model_weights - - run: - name: Setup - command: .circleci/unittest/linux/scripts/setup_env.sh - - designate_upload_channel - - run: - name: Install torchvision - command: .circleci/unittest/linux/scripts/install.sh - - run: - name: Build docs - command: | - set -ex - # turn v1.12.0rc3 into 1.12.0 - tag=$(echo $CIRCLE_TAG | sed -e 's/v*\([0-9.]*\).*/\1/') - VERSION=${tag:-main} - eval "$(./conda/bin/conda shell.bash hook)" - conda activate ./env - pushd docs - pip install --progress-bar=off -r requirements.txt - make html - popd - - persist_to_workspace: - root: ./ - paths: - - "*" - - store_artifacts: - path: ./docs/build/html - destination: docs - - upload_docs: - <<: *binary_common - docker: - - image: "pytorch/manylinux-cuda100" - resource_class: 2xlarge+ - steps: - - attach_workspace: - at: ~/workspace - - run: - name: Generate netrc - command: | - # set credentials for https pushing - # requires the org-member context - cat > ~/.netrc \< Date: Mon, 2 May 2022 11:21:15 +0200 Subject: [PATCH 02/19] cache third-party dependencies in prototype tests --- .github/workflows/prototype-tests.yml | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/.github/workflows/prototype-tests.yml b/.github/workflows/prototype-tests.yml index 518fa1cedc2..d2d7fee5f49 100644 --- a/.github/workflows/prototype-tests.yml +++ b/.github/workflows/prototype-tests.yml @@ -21,6 +21,20 @@ jobs: with: python-version: 3.7 + - name: Get pip cache path + id: get-cache-path + shell: bash + run: | + pip install appdirs + CACHE_PATH=`python -c "import appdirs; print(appdirs.user_cache_dir('pip', appauthor=False))"` + echo "::set-output name=path::$CACHE_PATH" + + - name: Restore pip cache + uses: actions/cache@v2 + with: + path: ${{ steps.get-cache-path.outputs.path }} + key: ${{ runner.os }}-pip + - name: Upgrade system packages run: python -m pip install --upgrade pip setuptools wheel @@ -34,10 +48,10 @@ jobs: run: pip install --progress-bar=off --no-build-isolation --editable . - name: Install other prototype dependencies - run: pip install --progress-bar=off scipy pycocotools h5py iopath + run: pip install --upgrade --progress-bar=off scipy pycocotools h5py iopath - name: Install test requirements - run: pip install --progress-bar=off pytest pytest-mock + run: pip install --upgrade --progress-bar=off pytest pytest-mock - name: Run prototype tests shell: bash From 65ce8b62b37e9faddd5531de8689660c0b16c3e8 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Mon, 2 May 2022 11:25:13 +0200 Subject: [PATCH 03/19] cleanup --- .github/workflows/prototype-tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/prototype-tests.yml b/.github/workflows/prototype-tests.yml index d2d7fee5f49..159c81cf467 100644 --- a/.github/workflows/prototype-tests.yml +++ b/.github/workflows/prototype-tests.yml @@ -27,6 +27,7 @@ jobs: run: | pip install appdirs CACHE_PATH=`python -c "import appdirs; print(appdirs.user_cache_dir('pip', appauthor=False))"` + echo "pip cache stored in ${CACHE_PATH}" echo "::set-output name=path::$CACHE_PATH" - name: Restore pip cache @@ -42,7 +43,7 @@ jobs: uses: actions/checkout@v3 - name: Install PyTorch nightly builds - run: pip install --progress-bar=off --pre torch torchdata --extra-index-url https://download.pytorch.org/whl/nightly/cpu/ + run: pip install --upgrade --progress-bar=off --pre torch torchdata --extra-index-url https://download.pytorch.org/whl/nightly/cpu/ - name: Install torchvision run: pip install --progress-bar=off --no-build-isolation --editable . From 2605324bb7ac1753c563c010e572a409df86a846 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Tue, 3 May 2022 10:55:35 +0200 Subject: [PATCH 04/19] try caching on date --- .github/workflows/prototype-tests.yml | 77 ++++++++++++++++----------- 1 file changed, 45 insertions(+), 32 deletions(-) diff --git a/.github/workflows/prototype-tests.yml b/.github/workflows/prototype-tests.yml index 159c81cf467..3a98c8dea9b 100644 --- a/.github/workflows/prototype-tests.yml +++ b/.github/workflows/prototype-tests.yml @@ -21,39 +21,52 @@ jobs: with: python-version: 3.7 - - name: Get pip cache path - id: get-cache-path - shell: bash + - name: Debug run: | - pip install appdirs - CACHE_PATH=`python -c "import appdirs; print(appdirs.user_cache_dir('pip', appauthor=False))"` - echo "pip cache stored in ${CACHE_PATH}" - echo "::set-output name=path::$CACHE_PATH" - - - name: Restore pip cache - uses: actions/cache@v2 - with: - path: ${{ steps.get-cache-path.outputs.path }} - key: ${{ runner.os }}-pip - - - name: Upgrade system packages - run: python -m pip install --upgrade pip setuptools wheel - - - name: Checkout repository - uses: actions/checkout@v3 + python -c "import sys; print(sys.path)" + echo "----------------------------------------------------------" + python -c "import pip; print(pip.__path__)" - - name: Install PyTorch nightly builds - run: pip install --upgrade --progress-bar=off --pre torch torchdata --extra-index-url https://download.pytorch.org/whl/nightly/cpu/ - - - name: Install torchvision - run: pip install --progress-bar=off --no-build-isolation --editable . - - - name: Install other prototype dependencies - run: pip install --upgrade --progress-bar=off scipy pycocotools h5py iopath + - name: Create cache key + shell: bash + env: + HOUR: 10 + run: | + if (( `date -u '+%-H'` > $HOUR )); then DATE='today'; else DATE='yesterday'; fi + KEY=`date -u --date=$DATE '+%Y%m%d'` + echo $KEY + echo "::set-output name=key::$KEY" - - name: Install test requirements - run: pip install --upgrade --progress-bar=off pytest pytest-mock +# - name: Get pip cache path +# id: get-cache-path +# shell: bash +# run: | +# pip install appdirs +# CACHE_PATH=`python -c "import appdirs; print(appdirs.user_cache_dir('pip', appauthor=False))"` +# echo "pip cache stored in ${CACHE_PATH}" +# echo "::set-output name=path::$CACHE_PATH" - - name: Run prototype tests - shell: bash - run: pytest --durations=20 test/test_prototype_*.py +# - name: Restore pip cache +# uses: actions/cache@v2 +# with: +# path: ${{ steps.get-cache-path.outputs.path }} +# key: ${{ runner.os }}-pip +# +# - name: Checkout repository +# uses: actions/checkout@v3 +# +# - name: Install PyTorch nightly builds +# run: pip install --progress-bar=off --pre torch torchdata --extra-index-url https://download.pytorch.org/whl/nightly/cpu/ +# +# - name: Install torchvision +# run: pip install --progress-bar=off --no-build-isolation --editable . +# +# - name: Install other prototype dependencies +# run: pip install --progress-bar=off scipy pycocotools h5py iopath +# +# - name: Install test requirements +# run: pip install --progress-bar=off pytest pytest-mock +# +# - name: Run prototype tests +# shell: bash +# run: pytest --durations=20 test/test_prototype_*.py From 5a30081a49796ef0c5b8fa52db7a8491ebe24772 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Tue, 3 May 2022 11:07:04 +0200 Subject: [PATCH 05/19] try use together with cache --- .github/workflows/prototype-tests.yml | 31 ++++++++++++++------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/.github/workflows/prototype-tests.yml b/.github/workflows/prototype-tests.yml index 3a98c8dea9b..bf57687c9a5 100644 --- a/.github/workflows/prototype-tests.yml +++ b/.github/workflows/prototype-tests.yml @@ -21,21 +21,22 @@ jobs: with: python-version: 3.7 - - name: Debug - run: | - python -c "import sys; print(sys.path)" - echo "----------------------------------------------------------" - python -c "import pip; print(pip.__path__)" - - - name: Create cache key + - name: Generate cache params + id: cache-params shell: bash env: + # TODO: explain HOUR: 10 run: | + CACHE_PATH=`python -c 'import sys; print("\n".join(path for path in sys.path if path))'` + echo $CACHE_PATH + echo "::set-output name=path::$CACHE_PATH" + if (( `date -u '+%-H'` > $HOUR )); then DATE='today'; else DATE='yesterday'; fi - KEY=`date -u --date=$DATE '+%Y%m%d'` - echo $KEY - echo "::set-output name=key::$KEY" + echo $DATE + CACHE_DATE=`date -u --date=$DATE '+%Y%m%d'` + echo $CACHE_DATE + echo "::set-output name=date::$CACHE_DATE" # - name: Get pip cache path # id: get-cache-path @@ -46,11 +47,11 @@ jobs: # echo "pip cache stored in ${CACHE_PATH}" # echo "::set-output name=path::$CACHE_PATH" -# - name: Restore pip cache -# uses: actions/cache@v2 -# with: -# path: ${{ steps.get-cache-path.outputs.path }} -# key: ${{ runner.os }}-pip + - name: Restore pip cache + uses: actions/cache@v2 + with: + path: ${{ steps.cache-params.outputs.path }} + key: ${{ runner.os }}-${{ steps.cache-params.outputs.date }} # # - name: Checkout repository # uses: actions/checkout@v3 From 4e67eac7fe570db0ff8ea193e8843c533091b2b9 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Tue, 3 May 2022 11:07:27 +0200 Subject: [PATCH 06/19] use v3 cache action --- .github/workflows/prototype-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/prototype-tests.yml b/.github/workflows/prototype-tests.yml index bf57687c9a5..4ec7c746444 100644 --- a/.github/workflows/prototype-tests.yml +++ b/.github/workflows/prototype-tests.yml @@ -48,7 +48,7 @@ jobs: # echo "::set-output name=path::$CACHE_PATH" - name: Restore pip cache - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ${{ steps.cache-params.outputs.path }} key: ${{ runner.os }}-${{ steps.cache-params.outputs.date }} From e8dc9cae574da99b813acb46c615bd97ea82ba5c Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Tue, 3 May 2022 11:12:27 +0200 Subject: [PATCH 07/19] debug --- .github/workflows/prototype-tests.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/prototype-tests.yml b/.github/workflows/prototype-tests.yml index 4ec7c746444..6e96f1cbbb4 100644 --- a/.github/workflows/prototype-tests.yml +++ b/.github/workflows/prototype-tests.yml @@ -28,13 +28,13 @@ jobs: # TODO: explain HOUR: 10 run: | - CACHE_PATH=`python -c 'import sys; print("\n".join(path for path in sys.path if path))'` + CACHE_PATH=`python -c 'import sys; print("\n".join(path.strip() for path in sys.path if path))'` echo $CACHE_PATH echo "::set-output name=path::$CACHE_PATH" if (( `date -u '+%-H'` > $HOUR )); then DATE='today'; else DATE='yesterday'; fi echo $DATE - CACHE_DATE=`date -u --date=$DATE '+%Y%m%d'` + CACHE_DATE=`date -u -d$DATE '+%Y%m%d'` echo $CACHE_DATE echo "::set-output name=date::$CACHE_DATE" @@ -50,7 +50,8 @@ jobs: - name: Restore pip cache uses: actions/cache@v3 with: - path: ${{ steps.cache-params.outputs.path }} + path: | + ${{ steps.cache-params.outputs.path }} key: ${{ runner.os }}-${{ steps.cache-params.outputs.date }} # # - name: Checkout repository From 113d1f0bcb7134efb34c4a2489286a9ddb5a0e37 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Tue, 3 May 2022 11:25:32 +0200 Subject: [PATCH 08/19] use only one path --- .github/workflows/prototype-tests.yml | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/.github/workflows/prototype-tests.yml b/.github/workflows/prototype-tests.yml index 6e96f1cbbb4..21becd0f302 100644 --- a/.github/workflows/prototype-tests.yml +++ b/.github/workflows/prototype-tests.yml @@ -28,31 +28,20 @@ jobs: # TODO: explain HOUR: 10 run: | - CACHE_PATH=`python -c 'import sys; print("\n".join(path.strip() for path in sys.path if path))'` + CACHE_PATH=`python -c 'import sys; print(sys.path[-1])'` echo $CACHE_PATH echo "::set-output name=path::$CACHE_PATH" if (( `date -u '+%-H'` > $HOUR )); then DATE='today'; else DATE='yesterday'; fi - echo $DATE - CACHE_DATE=`date -u -d$DATE '+%Y%m%d'` + CACHE_DATE=`date -u -d $DATE '+%Y%m%d'` echo $CACHE_DATE echo "::set-output name=date::$CACHE_DATE" -# - name: Get pip cache path -# id: get-cache-path -# shell: bash -# run: | -# pip install appdirs -# CACHE_PATH=`python -c "import appdirs; print(appdirs.user_cache_dir('pip', appauthor=False))"` -# echo "pip cache stored in ${CACHE_PATH}" -# echo "::set-output name=path::$CACHE_PATH" - - name: Restore pip cache uses: actions/cache@v3 with: - path: | - ${{ steps.cache-params.outputs.path }} - key: ${{ runner.os }}-${{ steps.cache-params.outputs.date }} + path: ${{ steps.cache-params.outputs.path }} + key: ${{ runner.os }}-0-${{ steps.cache-params.outputs.date }} # # - name: Checkout repository # uses: actions/checkout@v3 From 20f97ed75948c2b09ce24b02b61c357f4f842c5e Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Tue, 3 May 2022 11:27:43 +0200 Subject: [PATCH 09/19] debug date on macOS --- .github/workflows/prototype-tests.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/prototype-tests.yml b/.github/workflows/prototype-tests.yml index 21becd0f302..e322c95fc83 100644 --- a/.github/workflows/prototype-tests.yml +++ b/.github/workflows/prototype-tests.yml @@ -21,6 +21,10 @@ jobs: with: python-version: 3.7 + - name: Debug + run: | + date --version + - name: Generate cache params id: cache-params shell: bash From c1bb5a2d9dd9cd5369888c9965e203e7d8430735 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Tue, 3 May 2022 11:37:49 +0200 Subject: [PATCH 10/19] use gdate on macos --- .github/workflows/prototype-tests.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/prototype-tests.yml b/.github/workflows/prototype-tests.yml index e322c95fc83..8cc1d7d57d8 100644 --- a/.github/workflows/prototype-tests.yml +++ b/.github/workflows/prototype-tests.yml @@ -23,7 +23,7 @@ jobs: - name: Debug run: | - date --version + gdate --version - name: Generate cache params id: cache-params @@ -31,12 +31,14 @@ jobs: env: # TODO: explain HOUR: 10 + OS: ${{ matrix.os }} run: | CACHE_PATH=`python -c 'import sys; print(sys.path[-1])'` echo $CACHE_PATH echo "::set-output name=path::$CACHE_PATH" - if (( `date -u '+%-H'` > $HOUR )); then DATE='today'; else DATE='yesterday'; fi + if [[ $OS == Macos ]]; then DATE_BIN=gdate; else DATE_BIN=date; fi + if (( `$DATE_BIN -u '+%-H'` > $HOUR )); then DATE='today'; else DATE='yesterday'; fi CACHE_DATE=`date -u -d $DATE '+%Y%m%d'` echo $CACHE_DATE echo "::set-output name=date::$CACHE_DATE" From c142879b16b520114c98d6d860dc9c51c8990540 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Tue, 3 May 2022 11:52:01 +0200 Subject: [PATCH 11/19] try use python shell --- .github/workflows/prototype-tests.yml | 31 +++++++++++---------------- 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/.github/workflows/prototype-tests.yml b/.github/workflows/prototype-tests.yml index 8cc1d7d57d8..8b7fff6a37b 100644 --- a/.github/workflows/prototype-tests.yml +++ b/.github/workflows/prototype-tests.yml @@ -21,33 +21,28 @@ jobs: with: python-version: 3.7 - - name: Debug - run: | - gdate --version - - name: Generate cache params id: cache-params - shell: bash - env: - # TODO: explain - HOUR: 10 - OS: ${{ matrix.os }} + shell: python run: | - CACHE_PATH=`python -c 'import sys; print(sys.path[-1])'` - echo $CACHE_PATH - echo "::set-output name=path::$CACHE_PATH" + import sys + from datetime import timedelta, datetime + + cache_path = sys.path[-1] + print(cache_path) + print(f"::set-output name=date::{cache_path}") - if [[ $OS == Macos ]]; then DATE_BIN=gdate; else DATE_BIN=date; fi - if (( `$DATE_BIN -u '+%-H'` > $HOUR )); then DATE='today'; else DATE='yesterday'; fi - CACHE_DATE=`date -u -d $DATE '+%Y%m%d'` - echo $CACHE_DATE - echo "::set-output name=date::$CACHE_DATE" + today = datetime.utcnow() + yesterday = today - timedelta(1) + cache_date = f"{today if today.hour > 10 else yesterday:%Y%m%d}" + print(cache_date) + print(f"::set-output name=date::{cache_date}") - name: Restore pip cache uses: actions/cache@v3 with: path: ${{ steps.cache-params.outputs.path }} - key: ${{ runner.os }}-0-${{ steps.cache-params.outputs.date }} + key: ${{ runner.os }}-1-${{ steps.cache-params.outputs.date }} # # - name: Checkout repository # uses: actions/checkout@v3 From cb3889264a2135f07b6f0d61d1290527ba9fc3b4 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Tue, 3 May 2022 11:53:00 +0200 Subject: [PATCH 12/19] fix --- .github/workflows/prototype-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/prototype-tests.yml b/.github/workflows/prototype-tests.yml index 8b7fff6a37b..c7ff9f3c0ac 100644 --- a/.github/workflows/prototype-tests.yml +++ b/.github/workflows/prototype-tests.yml @@ -30,7 +30,7 @@ jobs: cache_path = sys.path[-1] print(cache_path) - print(f"::set-output name=date::{cache_path}") + print(f"::set-output name=path::{cache_path}") today = datetime.utcnow() yesterday = today - timedelta(1) From 29afeb7a119ec87f175ca75c1f4fa64b2c9e01c8 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Tue, 3 May 2022 11:56:28 +0200 Subject: [PATCH 13/19] try full caching --- .github/workflows/prototype-tests.yml | 36 +++++++++++++-------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/workflows/prototype-tests.yml b/.github/workflows/prototype-tests.yml index c7ff9f3c0ac..eb0477d1633 100644 --- a/.github/workflows/prototype-tests.yml +++ b/.github/workflows/prototype-tests.yml @@ -38,27 +38,27 @@ jobs: print(cache_date) print(f"::set-output name=date::{cache_date}") - - name: Restore pip cache + - name: Restore cache uses: actions/cache@v3 with: path: ${{ steps.cache-params.outputs.path }} - key: ${{ runner.os }}-1-${{ steps.cache-params.outputs.date }} -# -# - name: Checkout repository -# uses: actions/checkout@v3 -# -# - name: Install PyTorch nightly builds -# run: pip install --progress-bar=off --pre torch torchdata --extra-index-url https://download.pytorch.org/whl/nightly/cpu/ -# -# - name: Install torchvision -# run: pip install --progress-bar=off --no-build-isolation --editable . -# -# - name: Install other prototype dependencies -# run: pip install --progress-bar=off scipy pycocotools h5py iopath -# -# - name: Install test requirements -# run: pip install --progress-bar=off pytest pytest-mock -# + key: ${{ runner.os }}-v2-${{ steps.cache-params.outputs.date }} + + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Install PyTorch nightly builds + run: pip install --progress-bar=off --pre torch torchdata --extra-index-url https://download.pytorch.org/whl/nightly/cpu/ + + - name: Install torchvision + run: pip install --progress-bar=off --no-build-isolation --editable . + + - name: Install optional dependencies + run: pip install --progress-bar=off scipy pycocotools h5py iopath + + - name: Install test requirements + run: pip install --progress-bar=off pytest pytest-mock + # - name: Run prototype tests # shell: bash # run: pytest --durations=20 test/test_prototype_*.py From 6db3322f8ebf96e64f43f1799e0567bf1bae8414 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Tue, 3 May 2022 12:10:15 +0200 Subject: [PATCH 14/19] fix cache path generation --- .github/workflows/prototype-tests.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/prototype-tests.yml b/.github/workflows/prototype-tests.yml index eb0477d1633..218ffd11522 100644 --- a/.github/workflows/prototype-tests.yml +++ b/.github/workflows/prototype-tests.yml @@ -25,15 +25,16 @@ jobs: id: cache-params shell: python run: | + import datetime + import pathlib import sys - from datetime import timedelta, datetime - cache_path = sys.path[-1] + cache_path = pathlib.Path(sys.executable).parents[1] print(cache_path) print(f"::set-output name=path::{cache_path}") - today = datetime.utcnow() - yesterday = today - timedelta(1) + today = datetime.datetime.utcnow() + yesterday = today - datetime.timedelta(1) cache_date = f"{today if today.hour > 10 else yesterday:%Y%m%d}" print(cache_date) print(f"::set-output name=date::{cache_date}") @@ -42,7 +43,7 @@ jobs: uses: actions/cache@v3 with: path: ${{ steps.cache-params.outputs.path }} - key: ${{ runner.os }}-v2-${{ steps.cache-params.outputs.date }} + key: ${{ runner.os }}-v3-${{ steps.cache-params.outputs.date }} - name: Checkout repository uses: actions/checkout@v3 From 44371dad6b81b7f1f216bd93a8f0c9a540ccbc4f Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Tue, 3 May 2022 12:11:38 +0200 Subject: [PATCH 15/19] debug without cache --- .github/workflows/prototype-tests.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/prototype-tests.yml b/.github/workflows/prototype-tests.yml index 218ffd11522..90374adaf79 100644 --- a/.github/workflows/prototype-tests.yml +++ b/.github/workflows/prototype-tests.yml @@ -39,11 +39,11 @@ jobs: print(cache_date) print(f"::set-output name=date::{cache_date}") - - name: Restore cache - uses: actions/cache@v3 - with: - path: ${{ steps.cache-params.outputs.path }} - key: ${{ runner.os }}-v3-${{ steps.cache-params.outputs.date }} +# - name: Restore cache +# uses: actions/cache@v3 +# with: +# path: ${{ steps.cache-params.outputs.path }} +# key: ${{ runner.os }}-v3-${{ steps.cache-params.outputs.date }} - name: Checkout repository uses: actions/checkout@v3 From 3e38e57103189a1aee1e725a9c2d6922c476bd92 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Tue, 3 May 2022 12:20:55 +0200 Subject: [PATCH 16/19] try upgrade build packages --- .github/workflows/prototype-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/prototype-tests.yml b/.github/workflows/prototype-tests.yml index 90374adaf79..b3b1e9d69f9 100644 --- a/.github/workflows/prototype-tests.yml +++ b/.github/workflows/prototype-tests.yml @@ -45,6 +45,9 @@ jobs: # path: ${{ steps.cache-params.outputs.path }} # key: ${{ runner.os }}-v3-${{ steps.cache-params.outputs.date }} + - name: Upgrade build packages + run: python -m pip install --upgrade pip setuptools wheel + - name: Checkout repository uses: actions/checkout@v3 From 62650451542ce35d4a5976aeca29ec0169bcae1d Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Tue, 3 May 2022 12:24:07 +0200 Subject: [PATCH 17/19] re-enable cache --- .github/workflows/prototype-tests.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/prototype-tests.yml b/.github/workflows/prototype-tests.yml index b3b1e9d69f9..d9d4f526742 100644 --- a/.github/workflows/prototype-tests.yml +++ b/.github/workflows/prototype-tests.yml @@ -39,11 +39,11 @@ jobs: print(cache_date) print(f"::set-output name=date::{cache_date}") -# - name: Restore cache -# uses: actions/cache@v3 -# with: -# path: ${{ steps.cache-params.outputs.path }} -# key: ${{ runner.os }}-v3-${{ steps.cache-params.outputs.date }} + - name: Restore cache + uses: actions/cache@v3 + with: + path: ${{ steps.cache-params.outputs.path }} + key: ${{ runner.os }}-v3-${{ steps.cache-params.outputs.date }} - name: Upgrade build packages run: python -m pip install --upgrade pip setuptools wheel From 2e27808fb13310e48905a6382e3e363b6d3ab69c Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Tue, 3 May 2022 13:49:31 +0200 Subject: [PATCH 18/19] w/o cache --- .github/workflows/prototype-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/prototype-tests.yml b/.github/workflows/prototype-tests.yml index d9d4f526742..ddb9361ddf0 100644 --- a/.github/workflows/prototype-tests.yml +++ b/.github/workflows/prototype-tests.yml @@ -43,7 +43,7 @@ jobs: uses: actions/cache@v3 with: path: ${{ steps.cache-params.outputs.path }} - key: ${{ runner.os }}-v3-${{ steps.cache-params.outputs.date }} + key: ${{ runner.os }}-${{ steps.cache-params.outputs.date }} - name: Upgrade build packages run: python -m pip install --upgrade pip setuptools wheel From f516b2109f630b449ddb3b708e8892fa7a8d9866 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Tue, 3 May 2022 13:55:28 +0200 Subject: [PATCH 19/19] w/ cache