From 2e01d88e30174807341305dd8fbc45b54f68f779 Mon Sep 17 00:00:00 2001 From: Omkar Salpekar Date: Mon, 14 Nov 2022 13:34:45 -0800 Subject: [PATCH 1/3] [Nova] Disable Uploads for Linux Wheels from CircleCI --- .circleci/config.yml | 98 +----------------------- .circleci/config.yml.in | 16 ---- .circleci/regenerate.py | 5 +- .github/workflows/build-wheels-linux.yml | 2 +- 4 files changed, 6 insertions(+), 115 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6a3339a34d..c63f3a6ca6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -128,22 +128,6 @@ jobs: name: Code format not compliant with the rules! Run '$ python run-clang-format.py' to fix this. command: exit 1 - binary_linux_wheel: - <<: *binary_common - docker: - - image: "pytorch/manylinux-cuda102" - resource_class: 2xlarge+ - steps: - - checkout - - designate_upload_channel - - run: packaging/build_wheel.sh - - store_artifacts: - path: dist - - persist_to_workspace: - root: dist - paths: - - "*" - binary_linux_conda: <<: *binary_common docker: @@ -692,7 +676,7 @@ workflows: name: build_docs python_version: '3.8' requires: - - binary_linux_wheel_py3.8 + - binary_windows_wheel_py3.8 - upload_docs: context: org-member filters: @@ -756,26 +740,6 @@ workflows: only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ name: nightly_binary_linux_wheel_py3.7 python_version: '3.7' - - binary_wheel_upload: - context: org-member - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_linux_wheel_py3.7_upload - requires: - - nightly_binary_linux_wheel_py3.7 - - smoke_test_linux_pip: - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_linux_wheel_py3.7_smoke_test_pip - python_version: '3.7' - requires: - - nightly_binary_linux_wheel_py3.7_upload - binary_linux_wheel: filters: branches: @@ -784,26 +748,6 @@ workflows: only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ name: nightly_binary_linux_wheel_py3.8 python_version: '3.8' - - binary_wheel_upload: - context: org-member - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_linux_wheel_py3.8_upload - requires: - - nightly_binary_linux_wheel_py3.8 - - smoke_test_linux_pip: - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_linux_wheel_py3.8_smoke_test_pip - python_version: '3.8' - requires: - - nightly_binary_linux_wheel_py3.8_upload - binary_linux_wheel: filters: branches: @@ -812,26 +756,6 @@ workflows: only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ name: nightly_binary_linux_wheel_py3.9 python_version: '3.9' - - binary_wheel_upload: - context: org-member - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_linux_wheel_py3.9_upload - requires: - - nightly_binary_linux_wheel_py3.9 - - smoke_test_linux_pip: - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_linux_wheel_py3.9_smoke_test_pip - python_version: '3.9' - requires: - - nightly_binary_linux_wheel_py3.9_upload - binary_linux_wheel: filters: branches: @@ -840,26 +764,6 @@ workflows: only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ name: nightly_binary_linux_wheel_py3.10 python_version: '3.10' - - binary_wheel_upload: - context: org-member - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_linux_wheel_py3.10_upload - requires: - - nightly_binary_linux_wheel_py3.10 - - smoke_test_linux_pip: - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_linux_wheel_py3.10_smoke_test_pip - python_version: '3.10' - requires: - - nightly_binary_linux_wheel_py3.10_upload - binary_macos_wheel: filters: branches: diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 77dbf581ac..14bda19e38 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -128,22 +128,6 @@ jobs: name: Code format not compliant with the rules! Run '$ python run-clang-format.py' to fix this. command: exit 1 - binary_linux_wheel: - <<: *binary_common - docker: - - image: "pytorch/manylinux-cuda102" - resource_class: 2xlarge+ - steps: - - checkout - - designate_upload_channel - - run: packaging/build_wheel.sh - - store_artifacts: - path: dist - - persist_to_workspace: - root: dist - paths: - - "*" - binary_linux_conda: <<: *binary_common docker: diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py index 53e7e318f0..c95a42a370 100755 --- a/.circleci/regenerate.py +++ b/.circleci/regenerate.py @@ -50,6 +50,9 @@ def build_workflow_pair(btype, os_type, python_version, filter_branch, prefix="" base_workflow_name = f"{prefix}binary_{os_type}_{btype}_py{python_version}" w.append(generate_base_workflow(base_workflow_name, python_version, filter_branch, os_type, btype)) + if filter_branch == "nightly" and os_type == "linux" and btype == "wheel": + upload = False + if upload: w.append(generate_upload_workflow(base_workflow_name, filter_branch, btype)) if filter_branch == "nightly" and os_type in ["linux", "windows"]: @@ -63,7 +66,7 @@ def build_doc_job(filter_branch): "name": "build_docs", "python_version": "3.8", "requires": [ - "binary_linux_wheel_py3.8", + "binary_windows_wheel_py3.8", ], } diff --git a/.github/workflows/build-wheels-linux.yml b/.github/workflows/build-wheels-linux.yml index 6777589c79..5d5f73462a 100644 --- a/.github/workflows/build-wheels-linux.yml +++ b/.github/workflows/build-wheels-linux.yml @@ -40,7 +40,7 @@ jobs: smoke-test-script: ${{ matrix.smoke-test-script }} # Using "development" as trigger event so these binaries are not uploaded # to official channels yet - trigger-event: development + trigger-event: ${{ github.event_name }} secrets: AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }} From 2721106eb80a49f1c08446ea96675409b6ad77ab Mon Sep 17 00:00:00 2001 From: Omkar Salpekar Date: Mon, 14 Nov 2022 13:40:57 -0800 Subject: [PATCH 2/3] change regenerate.py to account for the build --- .circleci/config.yml | 49 ----------------------------------------- .circleci/regenerate.py | 5 ++--- 2 files changed, 2 insertions(+), 52 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c63f3a6ca6..c9ac30dd94 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -590,23 +590,6 @@ workflows: - lint_c build: jobs: - - binary_linux_wheel: - name: binary_linux_wheel_py3.7 - python_version: '3.7' - - binary_linux_wheel: - filters: - branches: - only: /.*/ - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: binary_linux_wheel_py3.8 - python_version: '3.8' - - binary_linux_wheel: - name: binary_linux_wheel_py3.9 - python_version: '3.9' - - binary_linux_wheel: - name: binary_linux_wheel_py3.10 - python_version: '3.10' - binary_macos_wheel: name: binary_macos_wheel_py3.7 python_version: '3.7' @@ -732,38 +715,6 @@ workflows: filters: branches: only: nightly - - binary_linux_wheel: - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_linux_wheel_py3.7 - python_version: '3.7' - - binary_linux_wheel: - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_linux_wheel_py3.8 - python_version: '3.8' - - binary_linux_wheel: - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_linux_wheel_py3.9 - python_version: '3.9' - - binary_linux_wheel: - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_linux_wheel_py3.10 - python_version: '3.10' - binary_macos_wheel: filters: branches: diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py index c95a42a370..f8a599f6f8 100755 --- a/.circleci/regenerate.py +++ b/.circleci/regenerate.py @@ -35,6 +35,8 @@ def build_workflows(prefix="", upload=False, filter_branch=None, indentation=6): if not fb and (os_type == "linux" and btype == "wheel" and python_version == "3.8"): # the fields must match the build_docs "requires" dependency fb = "/.*/" + if os_type == "linux" and btype == "wheel": + continue w += build_workflow_pair(btype, os_type, python_version, fb, prefix, upload) if not filter_branch: @@ -50,9 +52,6 @@ def build_workflow_pair(btype, os_type, python_version, filter_branch, prefix="" base_workflow_name = f"{prefix}binary_{os_type}_{btype}_py{python_version}" w.append(generate_base_workflow(base_workflow_name, python_version, filter_branch, os_type, btype)) - if filter_branch == "nightly" and os_type == "linux" and btype == "wheel": - upload = False - if upload: w.append(generate_upload_workflow(base_workflow_name, filter_branch, btype)) if filter_branch == "nightly" and os_type in ["linux", "windows"]: From 1fe6a85e9340bf1e8cb8d52e9b02fe87e36824f4 Mon Sep 17 00:00:00 2001 From: Omkar Salpekar Date: Mon, 14 Nov 2022 14:09:04 -0800 Subject: [PATCH 3/3] add back py3.8 linux wheel to allow for docs_build --- .circleci/config.yml | 34 +++++++++++++++++++++++++++++++++- .circleci/config.yml.in | 16 ++++++++++++++++ .circleci/regenerate.py | 10 ++++++++-- 3 files changed, 57 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c9ac30dd94..010fb9d09e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -128,6 +128,22 @@ jobs: name: Code format not compliant with the rules! Run '$ python run-clang-format.py' to fix this. command: exit 1 + binary_linux_wheel: + <<: *binary_common + docker: + - image: "pytorch/manylinux-cuda102" + resource_class: 2xlarge+ + steps: + - checkout + - designate_upload_channel + - run: packaging/build_wheel.sh + - store_artifacts: + path: dist + - persist_to_workspace: + root: dist + paths: + - "*" + binary_linux_conda: <<: *binary_common docker: @@ -590,6 +606,14 @@ workflows: - lint_c build: jobs: + - binary_linux_wheel: + filters: + branches: + only: /.*/ + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: binary_linux_wheel_py3.8 + python_version: '3.8' - binary_macos_wheel: name: binary_macos_wheel_py3.7 python_version: '3.7' @@ -659,7 +683,7 @@ workflows: name: build_docs python_version: '3.8' requires: - - binary_windows_wheel_py3.8 + - binary_linux_wheel_py3.8 - upload_docs: context: org-member filters: @@ -715,6 +739,14 @@ workflows: filters: branches: only: nightly + - binary_linux_wheel: + filters: + branches: + only: nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_linux_wheel_py3.8 + python_version: '3.8' - binary_macos_wheel: filters: branches: diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 14bda19e38..77dbf581ac 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -128,6 +128,22 @@ jobs: name: Code format not compliant with the rules! Run '$ python run-clang-format.py' to fix this. command: exit 1 + binary_linux_wheel: + <<: *binary_common + docker: + - image: "pytorch/manylinux-cuda102" + resource_class: 2xlarge+ + steps: + - checkout + - designate_upload_channel + - run: packaging/build_wheel.sh + - store_artifacts: + path: dist + - persist_to_workspace: + root: dist + paths: + - "*" + binary_linux_conda: <<: *binary_common docker: diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py index f8a599f6f8..87acf92830 100755 --- a/.circleci/regenerate.py +++ b/.circleci/regenerate.py @@ -35,7 +35,10 @@ def build_workflows(prefix="", upload=False, filter_branch=None, indentation=6): if not fb and (os_type == "linux" and btype == "wheel" and python_version == "3.8"): # the fields must match the build_docs "requires" dependency fb = "/.*/" - if os_type == "linux" and btype == "wheel": + # We'll stop building Linux Wheels from CircleCI. Keeping + # around the Python3.8 build just for docs builds until those + # are also migrated. + if os_type == "linux" and btype == "wheel" and python_version != "3.8": continue w += build_workflow_pair(btype, os_type, python_version, fb, prefix, upload) @@ -52,6 +55,9 @@ def build_workflow_pair(btype, os_type, python_version, filter_branch, prefix="" base_workflow_name = f"{prefix}binary_{os_type}_{btype}_py{python_version}" w.append(generate_base_workflow(base_workflow_name, python_version, filter_branch, os_type, btype)) + if os_type == "linux" and btype == "wheel" and python_version == "3.8": + upload = False + if upload: w.append(generate_upload_workflow(base_workflow_name, filter_branch, btype)) if filter_branch == "nightly" and os_type in ["linux", "windows"]: @@ -65,7 +71,7 @@ def build_doc_job(filter_branch): "name": "build_docs", "python_version": "3.8", "requires": [ - "binary_windows_wheel_py3.8", + "binary_linux_wheel_py3.8", ], }