From 3812d7373ec6dcb091b5df93045929dbf830157c Mon Sep 17 00:00:00 2001 From: Omkar Salpekar Date: Mon, 15 May 2023 17:12:11 -0700 Subject: [PATCH 1/2] [Nova] Deprecate Remaining CircleCI unittest jobs --- .circleci/config.yml | 137 ---------------------------------------- .circleci/config.yml.in | 129 ------------------------------------- .circleci/regenerate.py | 21 ------ 3 files changed, 287 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1bb9d36b54..7c85d7b53f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -402,132 +402,6 @@ jobs: conda activate python${PYTHON_VERSION} python -c "import torchtext" - unittest_linux: - <<: *binary_common - docker: - - image: "pytorch/manylinux-cuda102" - resource_class: 2xlarge+ - steps: - - checkout - - designate_upload_channel - - load_conda_channel_flags - - run: - name: Generate cache key - # This will refresh cache on Sundays, nightly build should generate new cache. - command: echo "$(date +"%Y-%U")" > .circleci-weekly - - run: - name: Setup - command: .circleci/unittest/linux/scripts/setup_env.sh - - run: - name: Install torchtext - command: .circleci/unittest/linux/scripts/install.sh - - restore_cache: - keys: - - - data-linux-v1-{{ checksum ".circleci-weekly" }} - - - run: - name: Run tests - # Downloading embedding vector takes long time. - no_output_timeout: 30m - command: .circleci/unittest/linux/scripts/run_test.sh - - save_cache: - - key: data-linux-v1-{{ checksum ".circleci-weekly" }} - - paths: - - .vector_cache - - run: - name: Post process - command: .circleci/unittest/linux/scripts/post_process.sh - - store_test_results: - path: test-results - - unittest_macos: - <<: *binary_common - macos: - xcode: "14.0" - resource_class: large - steps: - - checkout - - designate_upload_channel - - load_conda_channel_flags - - run: - name: Generate cache key - # This will refresh cache on Sundays, nightly build should generate new cache. - command: echo "$(date +"%Y-%U")" > .circleci-weekly - - run: - name: Setup - command: .circleci/unittest/linux/scripts/setup_env.sh - - run: - name: Install torchtext - command: .circleci/unittest/linux/scripts/install.sh - - restore_cache: - keys: - - - data-macos-v1-{{ checksum ".circleci-weekly" }} - - - - run: - name: Run tests - # Downloading embedding vector takes long time. - no_output_timeout: 30m - command: .circleci/unittest/linux/scripts/run_test.sh - - save_cache: - keys: - - key: data-macos-v1-{{ checksum ".circleci-weekly" }} - - - paths: - - .vector_cache - - run: - name: Post process - command: .circleci/unittest/linux/scripts/post_process.sh - - store_test_results: - path: test-results - - unittest_windows: - <<: *binary_common - executor: - name: windows-cpu - steps: - - checkout - - designate_upload_channel - - load_conda_channel_flags - - run: - name: Generate cache key - # This will refresh cache on Sundays, nightly build should generate new cache. - command: echo "$(date +"%Y-%U")" > .circleci-weekly - - run: - name: Setup - command: .circleci/unittest/windows/scripts/setup_env.sh - - run: - name: Install torchtext - command: .circleci/unittest/windows/scripts/install.sh - - restore_cache: - keys: - - - data-windows-v1-{{ checksum ".circleci-weekly" }} - - - - run: - name: Run tests - # Downloading embedding vector takes long time. - no_output_timeout: 30m - command: .circleci/unittest/windows/scripts/run_test.sh - - save_cache: - - key: data-windows-v1-{{ checksum ".circleci-weekly" }} - - paths: - - .vector_cache - - run: - name: Post process - command: .circleci/unittest/windows/scripts/post_process.sh - - store_test_results: - path: test-results - stylecheck: <<: *binary_common docker: @@ -664,17 +538,6 @@ workflows: python_version: '3.8' requires: - build_docs - unittest: - jobs: - - unittest_windows: - name: unittest_windows_py3.8 - python_version: '3.8' - - unittest_windows: - name: unittest_windows_py3.9 - python_version: '3.9' - - unittest_windows: - name: unittest_windows_py3.10 - python_version: '3.10' nightly: jobs: - circleci_consistency: diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 68dd97dcfe..40a571445d 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -402,132 +402,6 @@ jobs: conda activate python${PYTHON_VERSION} python -c "import torchtext" - unittest_linux: - <<: *binary_common - docker: - - image: "pytorch/manylinux-cuda102" - resource_class: 2xlarge+ - steps: - - checkout - - designate_upload_channel - - load_conda_channel_flags - - run: - name: Generate cache key - # This will refresh cache on Sundays, nightly build should generate new cache. - command: echo "$(date +"%Y-%U")" > .circleci-weekly - - run: - name: Setup - command: .circleci/unittest/linux/scripts/setup_env.sh - - run: - name: Install torchtext - command: .circleci/unittest/linux/scripts/install.sh - - restore_cache: - keys: - {% raw %} - - data-linux-v1-{{ checksum ".circleci-weekly" }} - {% endraw %} - - run: - name: Run tests - # Downloading embedding vector takes long time. - no_output_timeout: 30m - command: .circleci/unittest/linux/scripts/run_test.sh - - save_cache: - {% raw %} - key: data-linux-v1-{{ checksum ".circleci-weekly" }} - {% endraw %} - paths: - - .vector_cache - - run: - name: Post process - command: .circleci/unittest/linux/scripts/post_process.sh - - store_test_results: - path: test-results - - unittest_macos: - <<: *binary_common - macos: - xcode: "14.0" - resource_class: large - steps: - - checkout - - designate_upload_channel - - load_conda_channel_flags - - run: - name: Generate cache key - # This will refresh cache on Sundays, nightly build should generate new cache. - command: echo "$(date +"%Y-%U")" > .circleci-weekly - - run: - name: Setup - command: .circleci/unittest/linux/scripts/setup_env.sh - - run: - name: Install torchtext - command: .circleci/unittest/linux/scripts/install.sh - - restore_cache: - keys: - {% raw %} - - data-macos-v1-{{ checksum ".circleci-weekly" }} - {% endraw %} - - - run: - name: Run tests - # Downloading embedding vector takes long time. - no_output_timeout: 30m - command: .circleci/unittest/linux/scripts/run_test.sh - - save_cache: - keys: - {% raw %} - key: data-macos-v1-{{ checksum ".circleci-weekly" }} - {% endraw %} - - paths: - - .vector_cache - - run: - name: Post process - command: .circleci/unittest/linux/scripts/post_process.sh - - store_test_results: - path: test-results - - unittest_windows: - <<: *binary_common - executor: - name: windows-cpu - steps: - - checkout - - designate_upload_channel - - load_conda_channel_flags - - run: - name: Generate cache key - # This will refresh cache on Sundays, nightly build should generate new cache. - command: echo "$(date +"%Y-%U")" > .circleci-weekly - - run: - name: Setup - command: .circleci/unittest/windows/scripts/setup_env.sh - - run: - name: Install torchtext - command: .circleci/unittest/windows/scripts/install.sh - - restore_cache: - keys: - {% raw %} - - data-windows-v1-{{ checksum ".circleci-weekly" }} - {% endraw %} - - - run: - name: Run tests - # Downloading embedding vector takes long time. - no_output_timeout: 30m - command: .circleci/unittest/windows/scripts/run_test.sh - - save_cache: - {% raw %} - key: data-windows-v1-{{ checksum ".circleci-weekly" }} - {% endraw %} - paths: - - .vector_cache - - run: - name: Post process - command: .circleci/unittest/windows/scripts/post_process.sh - - store_test_results: - path: test-results - stylecheck: <<: *binary_common docker: @@ -612,9 +486,6 @@ workflows: build: jobs: {{ build_workflows() }} - unittest: - jobs: - {{ unittest_workflows() }} nightly: jobs: - circleci_consistency: diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py index 4c8b3437b4..9094a7cf80 100755 --- a/.circleci/regenerate.py +++ b/.circleci/regenerate.py @@ -158,26 +158,6 @@ def indent(indentation, data_list): return ("\n" + " " * indentation).join(yaml.dump(data_list).splitlines()) -def unittest_workflows(indentation=6): - w = [] - for os_type in ["windows"]: - for python_version in PYTHON_VERSIONS: - # Turn off unit tests for 3.11, unit test are not setup properly in circleci - if python_version == "3.11": - continue - - w.append( - { - f"unittest_{os_type}": { - "name": f"unittest_{os_type}_py{python_version}", - "python_version": python_version, - } - } - ) - - return indent(indentation, w) - - if __name__ == "__main__": d = os.path.dirname(__file__) env = jinja2.Environment( @@ -190,7 +170,6 @@ def unittest_workflows(indentation=6): f.write( env.get_template("config.yml.in").render( build_workflows=build_workflows, - unittest_workflows=unittest_workflows, ) ) f.write("\n") From 0a8ca272d11005b788ae20f2728f3f75e585ce13 Mon Sep 17 00:00:00 2001 From: Omkar Salpekar Date: Mon, 15 May 2023 17:34:41 -0700 Subject: [PATCH 2/2] Remove more unused job deps --- .circleci/config.yml | 103 ---------------------------------------- .circleci/config.yml.in | 103 ---------------------------------------- 2 files changed, 206 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7c85d7b53f..7411b883ee 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -149,23 +149,6 @@ jobs: paths: - "*" - binary_linux_conda: - <<: *binary_common - docker: - - image: "pytorch/conda-cuda" - resource_class: 2xlarge+ - steps: - - checkout - - designate_upload_channel - - load_conda_channel_flags - - run: packaging/build_conda.sh - - store_artifacts: - path: /opt/conda/conda-bld/linux-64 - - persist_to_workspace: - root: /opt/conda - paths: - - "conda-bld/*" - binary_windows_wheel: <<: *binary_common executor: @@ -209,54 +192,6 @@ jobs: paths: - "conda-bld/*" - binary_macos_wheel: - <<: *binary_common - macos: - xcode: "14.0" - steps: - - checkout - - designate_upload_channel - - run: - # Installing cmake with `brew install cmake` takes 30 mins, so we use binary distribution. - command: | - curl -o cmake.tar.gz -L https://github.com/Kitware/CMake/releases/download/v3.17.2/cmake-3.17.2-Darwin-x86_64.tar.gz - tar -xzf cmake.tar.gz - cp cmake-3.17.2-Darwin-x86_64/CMake.app/Contents/bin/* /usr/local/bin/ - cp -r cmake-3.17.2-Darwin-x86_64/CMake.app/Contents/share/* /usr/local/share/ - export PATH="${PATH}:/usr/local/bin" - curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh - sh conda.sh -b - source $HOME/miniconda3/bin/activate - packaging/build_wheel.sh - - store_artifacts: - path: dist - - persist_to_workspace: - root: dist - paths: - - "*" - - binary_macos_conda: - <<: *binary_common - macos: - xcode: "14.0" - steps: - - checkout - - designate_upload_channel - - load_conda_channel_flags - - run: - command: | - curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh - sh conda.sh -b - source $HOME/miniconda3/bin/activate - conda install -yq conda-build - packaging/build_conda.sh - - store_artifacts: - path: /Users/distiller/miniconda3/conda-bld/osx-64 - - persist_to_workspace: - root: /Users/distiller/miniconda3 - paths: - - "conda-bld/*" - # Requires org-member context binary_conda_upload: docker: @@ -294,44 +229,6 @@ jobs: aws s3 cp "$pkg" "s3://pytorch/whl/${UPLOAD_CHANNEL}/" --acl public-read done - smoke_test_linux_conda: - <<: *smoke_test_common - steps: - - attach_workspace: - at: ~/workspace - - designate_upload_channel - - load_conda_channel_flags - - run: - name: install binaries - command: | - set -x - source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION} - conda install -v -y ${CONDA_CHANNEL_FLAGS} -c pytorch-${UPLOAD_CHANNEL} pytorch - conda install -v -y ${CONDA_CHANNEL_FLAGS} -c file://$HOME/workspace/conda-bld torchtext - - run: - name: smoke test - command: | - source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION} - python -c "import torchtext" - - smoke_test_linux_pip: - <<: *smoke_test_common - steps: - - attach_workspace: - at: ~/workspace - - designate_upload_channel - - run: - name: install binaries - command: | - set -x - source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION} - pip install $(ls ~/workspace/torchtext*.whl) --pre -f "https://download.pytorch.org/whl/${UPLOAD_CHANNEL}/cpu/torch_${UPLOAD_CHANNEL}.html" - - run: - name: smoke test - command: | - source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION} - python -c "import torchtext" - smoke_test_docker_image_build: machine: image: ubuntu-1604:201903-01 diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 40a571445d..781abdc81e 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -149,23 +149,6 @@ jobs: paths: - "*" - binary_linux_conda: - <<: *binary_common - docker: - - image: "pytorch/conda-cuda" - resource_class: 2xlarge+ - steps: - - checkout - - designate_upload_channel - - load_conda_channel_flags - - run: packaging/build_conda.sh - - store_artifacts: - path: /opt/conda/conda-bld/linux-64 - - persist_to_workspace: - root: /opt/conda - paths: - - "conda-bld/*" - binary_windows_wheel: <<: *binary_common executor: @@ -209,54 +192,6 @@ jobs: paths: - "conda-bld/*" - binary_macos_wheel: - <<: *binary_common - macos: - xcode: "14.0" - steps: - - checkout - - designate_upload_channel - - run: - # Installing cmake with `brew install cmake` takes 30 mins, so we use binary distribution. - command: | - curl -o cmake.tar.gz -L https://github.com/Kitware/CMake/releases/download/v3.17.2/cmake-3.17.2-Darwin-x86_64.tar.gz - tar -xzf cmake.tar.gz - cp cmake-3.17.2-Darwin-x86_64/CMake.app/Contents/bin/* /usr/local/bin/ - cp -r cmake-3.17.2-Darwin-x86_64/CMake.app/Contents/share/* /usr/local/share/ - export PATH="${PATH}:/usr/local/bin" - curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh - sh conda.sh -b - source $HOME/miniconda3/bin/activate - packaging/build_wheel.sh - - store_artifacts: - path: dist - - persist_to_workspace: - root: dist - paths: - - "*" - - binary_macos_conda: - <<: *binary_common - macos: - xcode: "14.0" - steps: - - checkout - - designate_upload_channel - - load_conda_channel_flags - - run: - command: | - curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh - sh conda.sh -b - source $HOME/miniconda3/bin/activate - conda install -yq conda-build - packaging/build_conda.sh - - store_artifacts: - path: /Users/distiller/miniconda3/conda-bld/osx-64 - - persist_to_workspace: - root: /Users/distiller/miniconda3 - paths: - - "conda-bld/*" - # Requires org-member context binary_conda_upload: docker: @@ -294,44 +229,6 @@ jobs: aws s3 cp "$pkg" "s3://pytorch/whl/${UPLOAD_CHANNEL}/" --acl public-read done - smoke_test_linux_conda: - <<: *smoke_test_common - steps: - - attach_workspace: - at: ~/workspace - - designate_upload_channel - - load_conda_channel_flags - - run: - name: install binaries - command: | - set -x - source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION} - conda install -v -y ${CONDA_CHANNEL_FLAGS} -c pytorch-${UPLOAD_CHANNEL} pytorch - conda install -v -y ${CONDA_CHANNEL_FLAGS} -c file://$HOME/workspace/conda-bld torchtext - - run: - name: smoke test - command: | - source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION} - python -c "import torchtext" - - smoke_test_linux_pip: - <<: *smoke_test_common - steps: - - attach_workspace: - at: ~/workspace - - designate_upload_channel - - run: - name: install binaries - command: | - set -x - source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION} - pip install $(ls ~/workspace/torchtext*.whl) --pre -f "https://download.pytorch.org/whl/${UPLOAD_CHANNEL}/cpu/torch_${UPLOAD_CHANNEL}.html" - - run: - name: smoke test - command: | - source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION} - python -c "import torchtext" - smoke_test_docker_image_build: machine: image: ubuntu-1604:201903-01