From e07978a4b1397527ac9dc3eb3b0ab15a0d5fd1ab Mon Sep 17 00:00:00 2001 From: Omkar Salpekar Date: Fri, 3 Feb 2023 11:06:56 -0800 Subject: [PATCH 1/4] Try python 3.11 wheels builds --- .../actions/setup-binary-builds/action.yml | 31 +++++++++++++------ .../test_build_conda_linux_with_cuda.yml | 2 +- .../test_build_conda_linux_without_cuda.yml | 2 +- .github/workflows/test_build_conda_m1.yml | 2 +- .github/workflows/test_build_conda_macos.yml | 2 +- .../test_build_conda_windows_without_cuda.yml | 2 +- .../test_build_wheels_linux_with_cuda.yml | 20 ++++++------ .github/workflows/test_build_wheels_m1.yml | 2 +- .github/workflows/test_build_wheels_macos.yml | 2 +- .../test_build_wheels_windows_with_cuda.yml | 2 +- ...test_build_wheels_windows_without_cuda.yml | 2 +- tools/scripts/generate_binary_build_matrix.py | 2 +- 12 files changed, 42 insertions(+), 29 deletions(-) diff --git a/.github/actions/setup-binary-builds/action.yml b/.github/actions/setup-binary-builds/action.yml index 7e8eb5f99e..c991d6c606 100644 --- a/.github/actions/setup-binary-builds/action.yml +++ b/.github/actions/setup-binary-builds/action.yml @@ -100,14 +100,27 @@ runs: set -euxo pipefail CONDA_ENV="${RUNNER_TEMP}/conda_environment_${GITHUB_RUN_ID}" - conda create \ - --yes \ - --prefix "${CONDA_ENV}" \ - "python=${PYTHON_VERSION}" \ - cmake=3.22 \ - conda-build=3.21 \ - ninja=1.10 \ - pkg-config=0.29 \ - wheel=0.37 + if [[ "${PYTHON_VERSION}" = "3.11" ]]; then + conda create \ + --yes \ + --prefix "${CONDA_ENV}" \ + -c conda-forge \ + "python=${PYTHON_VERSION}" \ + cmake=3.22 \ + conda-build=3.21 \ + ninja=1.10 \ + pkg-config=0.29 \ + wheel=0.37 + else + conda create \ + --yes \ + --prefix "${CONDA_ENV}" \ + "python=${PYTHON_VERSION}" \ + cmake=3.22 \ + conda-build=3.21 \ + ninja=1.10 \ + pkg-config=0.29 \ + wheel=0.37 + fi echo "CONDA_ENV=${CONDA_ENV}" >> "${GITHUB_ENV}" echo "CONDA_RUN=conda run -p ${CONDA_ENV}" >> "${GITHUB_ENV}" diff --git a/.github/workflows/test_build_conda_linux_with_cuda.yml b/.github/workflows/test_build_conda_linux_with_cuda.yml index dfdaa4bfe5..c898fc8028 100644 --- a/.github/workflows/test_build_conda_linux_with_cuda.yml +++ b/.github/workflows/test_build_conda_linux_with_cuda.yml @@ -6,7 +6,7 @@ on: - .github/actions/setup-binary-builds/action.yml - .github/workflows/test_build_conda_linux.yml - .github/workflows/build_conda_linux.yml - - .github/workflows/generate_binary_build_matrix.yml + # - .github/workflows/generate_binary_build_matrix.yml - tools/scripts/generate_binary_build_matrix.py workflow_dispatch: diff --git a/.github/workflows/test_build_conda_linux_without_cuda.yml b/.github/workflows/test_build_conda_linux_without_cuda.yml index 9331bd8f06..d33bd6e633 100644 --- a/.github/workflows/test_build_conda_linux_without_cuda.yml +++ b/.github/workflows/test_build_conda_linux_without_cuda.yml @@ -7,7 +7,7 @@ on: - .github/workflows/test_build_conda_linux.yml - .github/workflows/build_conda_linux.yml - .github/workflows/generate_binary_build_matrix.yml - - tools/scripts/generate_binary_build_matrix.py + # - tools/scripts/generate_binary_build_matrix.py workflow_dispatch: jobs: diff --git a/.github/workflows/test_build_conda_m1.yml b/.github/workflows/test_build_conda_m1.yml index 4f3c1d207c..2fe2430cca 100644 --- a/.github/workflows/test_build_conda_m1.yml +++ b/.github/workflows/test_build_conda_m1.yml @@ -7,7 +7,7 @@ on: - .github/workflows/test_build_conda_m1.yml - .github/workflows/build_conda_macos.yml - .github/workflows/generate_binary_build_matrix.yml - - tools/scripts/generate_binary_build_matrix.py + # - tools/scripts/generate_binary_build_matrix.py workflow_dispatch: jobs: diff --git a/.github/workflows/test_build_conda_macos.yml b/.github/workflows/test_build_conda_macos.yml index 4b0708f7c5..c3dfa99a71 100644 --- a/.github/workflows/test_build_conda_macos.yml +++ b/.github/workflows/test_build_conda_macos.yml @@ -7,7 +7,7 @@ on: - .github/workflows/test_build_conda_macos.yml - .github/workflows/build_conda_macos.yml - .github/workflows/generate_binary_build_matrix.yml - - tools/scripts/generate_binary_build_matrix.py + # - tools/scripts/generate_binary_build_matrix.py jobs: generate-matrix: diff --git a/.github/workflows/test_build_conda_windows_without_cuda.yml b/.github/workflows/test_build_conda_windows_without_cuda.yml index 2bffc090c8..434fb3e007 100644 --- a/.github/workflows/test_build_conda_windows_without_cuda.yml +++ b/.github/workflows/test_build_conda_windows_without_cuda.yml @@ -7,7 +7,7 @@ on: - .github/workflows/test_build_conda_windows_without_cuda.yml - .github/workflows/build_conda_windows.yml - .github/workflows/generate_binary_build_matrix.yml - - tools/scripts/generate_binary_build_matrix.py + # - tools/scripts/generate_binary_build_matrix.py workflow_dispatch: jobs: diff --git a/.github/workflows/test_build_wheels_linux_with_cuda.yml b/.github/workflows/test_build_wheels_linux_with_cuda.yml index d3ed608f42..3da65fa8d4 100644 --- a/.github/workflows/test_build_wheels_linux_with_cuda.yml +++ b/.github/workflows/test_build_wheels_linux_with_cuda.yml @@ -25,21 +25,21 @@ jobs: fail-fast: false matrix: include: - - repository: pytorch/audio - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: test/smoke_test/smoke_test.py - package-name: torchaudio + # - repository: pytorch/audio + # pre-script: packaging/pre_build_script.sh + # post-script: packaging/post_build_script.sh + # smoke-test-script: test/smoke_test/smoke_test.py + # package-name: torchaudio - repository: pytorch/vision pre-script: packaging/pre_build_script.sh post-script: packaging/post_build_script.sh smoke-test-script: test/smoke_test.py package-name: torchvision - - repository: pytorch/torchrec - pre-script: "" - post-script: "" - smoke-test-script: "" - package-name: torchrec + # - repository: pytorch/torchrec + # pre-script: "" + # post-script: "" + # smoke-test-script: "" + # package-name: torchrec uses: ./.github/workflows/build_wheels_linux.yml name: ${{ matrix.repository }} with: diff --git a/.github/workflows/test_build_wheels_m1.yml b/.github/workflows/test_build_wheels_m1.yml index 6460a81da8..39c4e1aae3 100644 --- a/.github/workflows/test_build_wheels_m1.yml +++ b/.github/workflows/test_build_wheels_m1.yml @@ -7,7 +7,7 @@ on: - .github/workflows/test_build_wheels_m1.yml - .github/workflows/build_wheels_macos.yml - .github/workflows/generate_binary_build_matrix.yml - - tools/scripts/generate_binary_build_matrix.py + # - tools/scripts/generate_binary_build_matrix.py workflow_dispatch: jobs: diff --git a/.github/workflows/test_build_wheels_macos.yml b/.github/workflows/test_build_wheels_macos.yml index bc1b890183..66e481c639 100644 --- a/.github/workflows/test_build_wheels_macos.yml +++ b/.github/workflows/test_build_wheels_macos.yml @@ -7,7 +7,7 @@ on: - .github/workflows/test_build_wheels_macos.yml - .github/workflows/build_wheels_macos.yml - .github/workflows/generate_binary_build_matrix.yml - - tools/scripts/generate_binary_build_matrix.py + # - tools/scripts/generate_binary_build_matrix.py workflow_dispatch: jobs: diff --git a/.github/workflows/test_build_wheels_windows_with_cuda.yml b/.github/workflows/test_build_wheels_windows_with_cuda.yml index 6066bc3bf1..20fcbbd85f 100644 --- a/.github/workflows/test_build_wheels_windows_with_cuda.yml +++ b/.github/workflows/test_build_wheels_windows_with_cuda.yml @@ -7,7 +7,7 @@ on: - .github/workflows/test_build_wheels_windows_with_cuda.yml - .github/workflows/build_wheels_windows.yml - .github/workflows/generate_binary_build_matrix.yml - - tools/scripts/generate_binary_build_matrix.py + # - tools/scripts/generate_binary_build_matrix.py workflow_dispatch: jobs: diff --git a/.github/workflows/test_build_wheels_windows_without_cuda.yml b/.github/workflows/test_build_wheels_windows_without_cuda.yml index c5fd28cc60..fd930a83a7 100644 --- a/.github/workflows/test_build_wheels_windows_without_cuda.yml +++ b/.github/workflows/test_build_wheels_windows_without_cuda.yml @@ -7,7 +7,7 @@ on: - .github/workflows/test_build_wheels_windows_without_cuda.yml - .github/workflows/build_wheels_windows.yml - .github/workflows/generate_binary_build_matrix.yml - - tools/scripts/generate_binary_build_matrix.py + # - tools/scripts/generate_binary_build_matrix.py workflow_dispatch: jobs: diff --git a/tools/scripts/generate_binary_build_matrix.py b/tools/scripts/generate_binary_build_matrix.py index 65b06355e6..5ad5c5d4cc 100644 --- a/tools/scripts/generate_binary_build_matrix.py +++ b/tools/scripts/generate_binary_build_matrix.py @@ -21,7 +21,7 @@ mod = sys.modules[__name__] PYTHON_ARCHES_DICT = { - "nightly": ["3.8", "3.9", "3.10"], + "nightly": ["3.10", "3.11"], "test": ["3.7", "3.8", "3.9", "3.10"], "release": ["3.7", "3.8", "3.9", "3.10"], } From b188446087f606cd555bb7570eab0f153bc0bd5b Mon Sep 17 00:00:00 2001 From: Omkar Salpekar Date: Mon, 6 Feb 2023 13:01:22 -0800 Subject: [PATCH 2/4] fix conflicts in conda env setup --- .github/actions/setup-binary-builds/action.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/actions/setup-binary-builds/action.yml b/.github/actions/setup-binary-builds/action.yml index c991d6c606..7e6963b6f2 100644 --- a/.github/actions/setup-binary-builds/action.yml +++ b/.github/actions/setup-binary-builds/action.yml @@ -105,12 +105,8 @@ runs: --yes \ --prefix "${CONDA_ENV}" \ -c conda-forge \ - "python=${PYTHON_VERSION}" \ - cmake=3.22 \ - conda-build=3.21 \ - ninja=1.10 \ - pkg-config=0.29 \ - wheel=0.37 + "python=${PYTHON_VERSION}" + conda install wheel ninja pkg-config else conda create \ --yes \ From 56d54b6846585f32356ea28a2514434266e63c44 Mon Sep 17 00:00:00 2001 From: Omkar Salpekar Date: Tue, 7 Feb 2023 10:56:33 -0800 Subject: [PATCH 3/4] Versioned libs for 3.11 and remove conda-forge --- .github/actions/setup-binary-builds/action.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/actions/setup-binary-builds/action.yml b/.github/actions/setup-binary-builds/action.yml index 7e6963b6f2..888861cbed 100644 --- a/.github/actions/setup-binary-builds/action.yml +++ b/.github/actions/setup-binary-builds/action.yml @@ -101,12 +101,15 @@ runs: CONDA_ENV="${RUNNER_TEMP}/conda_environment_${GITHUB_RUN_ID}" if [[ "${PYTHON_VERSION}" = "3.11" ]]; then + # conda-build not available for Python 3.11 conda create \ --yes \ --prefix "${CONDA_ENV}" \ - -c conda-forge \ - "python=${PYTHON_VERSION}" - conda install wheel ninja pkg-config + "python=${PYTHON_VERSION}" \ + cmake=3.22 \ + ninja=1.10 \ + pkg-config=0.29 \ + wheel=0.37 else conda create \ --yes \ From 80cb0142217aba96652b27ae52e18557c9a198b2 Mon Sep 17 00:00:00 2001 From: Omkar Salpekar Date: Tue, 7 Feb 2023 13:49:07 -0800 Subject: [PATCH 4/4] clean up unneeded code --- .github/workflows/test_build_conda_linux_with_cuda.yml | 2 +- .../workflows/test_build_conda_linux_without_cuda.yml | 2 +- .github/workflows/test_build_conda_m1.yml | 2 +- .github/workflows/test_build_conda_macos.yml | 2 +- .../test_build_conda_windows_without_cuda.yml | 2 +- .../workflows/test_build_wheels_linux_with_cuda.yml | 10 +++++----- .github/workflows/test_build_wheels_m1.yml | 2 +- .github/workflows/test_build_wheels_macos.yml | 2 +- .../workflows/test_build_wheels_windows_with_cuda.yml | 2 +- .../test_build_wheels_windows_without_cuda.yml | 2 +- tools/scripts/generate_binary_build_matrix.py | 2 +- 11 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/test_build_conda_linux_with_cuda.yml b/.github/workflows/test_build_conda_linux_with_cuda.yml index c898fc8028..dfdaa4bfe5 100644 --- a/.github/workflows/test_build_conda_linux_with_cuda.yml +++ b/.github/workflows/test_build_conda_linux_with_cuda.yml @@ -6,7 +6,7 @@ on: - .github/actions/setup-binary-builds/action.yml - .github/workflows/test_build_conda_linux.yml - .github/workflows/build_conda_linux.yml - # - .github/workflows/generate_binary_build_matrix.yml + - .github/workflows/generate_binary_build_matrix.yml - tools/scripts/generate_binary_build_matrix.py workflow_dispatch: diff --git a/.github/workflows/test_build_conda_linux_without_cuda.yml b/.github/workflows/test_build_conda_linux_without_cuda.yml index d33bd6e633..9331bd8f06 100644 --- a/.github/workflows/test_build_conda_linux_without_cuda.yml +++ b/.github/workflows/test_build_conda_linux_without_cuda.yml @@ -7,7 +7,7 @@ on: - .github/workflows/test_build_conda_linux.yml - .github/workflows/build_conda_linux.yml - .github/workflows/generate_binary_build_matrix.yml - # - tools/scripts/generate_binary_build_matrix.py + - tools/scripts/generate_binary_build_matrix.py workflow_dispatch: jobs: diff --git a/.github/workflows/test_build_conda_m1.yml b/.github/workflows/test_build_conda_m1.yml index 2fe2430cca..4f3c1d207c 100644 --- a/.github/workflows/test_build_conda_m1.yml +++ b/.github/workflows/test_build_conda_m1.yml @@ -7,7 +7,7 @@ on: - .github/workflows/test_build_conda_m1.yml - .github/workflows/build_conda_macos.yml - .github/workflows/generate_binary_build_matrix.yml - # - tools/scripts/generate_binary_build_matrix.py + - tools/scripts/generate_binary_build_matrix.py workflow_dispatch: jobs: diff --git a/.github/workflows/test_build_conda_macos.yml b/.github/workflows/test_build_conda_macos.yml index c3dfa99a71..4b0708f7c5 100644 --- a/.github/workflows/test_build_conda_macos.yml +++ b/.github/workflows/test_build_conda_macos.yml @@ -7,7 +7,7 @@ on: - .github/workflows/test_build_conda_macos.yml - .github/workflows/build_conda_macos.yml - .github/workflows/generate_binary_build_matrix.yml - # - tools/scripts/generate_binary_build_matrix.py + - tools/scripts/generate_binary_build_matrix.py jobs: generate-matrix: diff --git a/.github/workflows/test_build_conda_windows_without_cuda.yml b/.github/workflows/test_build_conda_windows_without_cuda.yml index 434fb3e007..2bffc090c8 100644 --- a/.github/workflows/test_build_conda_windows_without_cuda.yml +++ b/.github/workflows/test_build_conda_windows_without_cuda.yml @@ -7,7 +7,7 @@ on: - .github/workflows/test_build_conda_windows_without_cuda.yml - .github/workflows/build_conda_windows.yml - .github/workflows/generate_binary_build_matrix.yml - # - tools/scripts/generate_binary_build_matrix.py + - tools/scripts/generate_binary_build_matrix.py workflow_dispatch: jobs: diff --git a/.github/workflows/test_build_wheels_linux_with_cuda.yml b/.github/workflows/test_build_wheels_linux_with_cuda.yml index 3da65fa8d4..924b6b73b2 100644 --- a/.github/workflows/test_build_wheels_linux_with_cuda.yml +++ b/.github/workflows/test_build_wheels_linux_with_cuda.yml @@ -25,11 +25,11 @@ jobs: fail-fast: false matrix: include: - # - repository: pytorch/audio - # pre-script: packaging/pre_build_script.sh - # post-script: packaging/post_build_script.sh - # smoke-test-script: test/smoke_test/smoke_test.py - # package-name: torchaudio + - repository: pytorch/audio + pre-script: packaging/pre_build_script.sh + post-script: packaging/post_build_script.sh + smoke-test-script: test/smoke_test/smoke_test.py + package-name: torchaudio - repository: pytorch/vision pre-script: packaging/pre_build_script.sh post-script: packaging/post_build_script.sh diff --git a/.github/workflows/test_build_wheels_m1.yml b/.github/workflows/test_build_wheels_m1.yml index 39c4e1aae3..6460a81da8 100644 --- a/.github/workflows/test_build_wheels_m1.yml +++ b/.github/workflows/test_build_wheels_m1.yml @@ -7,7 +7,7 @@ on: - .github/workflows/test_build_wheels_m1.yml - .github/workflows/build_wheels_macos.yml - .github/workflows/generate_binary_build_matrix.yml - # - tools/scripts/generate_binary_build_matrix.py + - tools/scripts/generate_binary_build_matrix.py workflow_dispatch: jobs: diff --git a/.github/workflows/test_build_wheels_macos.yml b/.github/workflows/test_build_wheels_macos.yml index 66e481c639..bc1b890183 100644 --- a/.github/workflows/test_build_wheels_macos.yml +++ b/.github/workflows/test_build_wheels_macos.yml @@ -7,7 +7,7 @@ on: - .github/workflows/test_build_wheels_macos.yml - .github/workflows/build_wheels_macos.yml - .github/workflows/generate_binary_build_matrix.yml - # - tools/scripts/generate_binary_build_matrix.py + - tools/scripts/generate_binary_build_matrix.py workflow_dispatch: jobs: diff --git a/.github/workflows/test_build_wheels_windows_with_cuda.yml b/.github/workflows/test_build_wheels_windows_with_cuda.yml index 20fcbbd85f..6066bc3bf1 100644 --- a/.github/workflows/test_build_wheels_windows_with_cuda.yml +++ b/.github/workflows/test_build_wheels_windows_with_cuda.yml @@ -7,7 +7,7 @@ on: - .github/workflows/test_build_wheels_windows_with_cuda.yml - .github/workflows/build_wheels_windows.yml - .github/workflows/generate_binary_build_matrix.yml - # - tools/scripts/generate_binary_build_matrix.py + - tools/scripts/generate_binary_build_matrix.py workflow_dispatch: jobs: diff --git a/.github/workflows/test_build_wheels_windows_without_cuda.yml b/.github/workflows/test_build_wheels_windows_without_cuda.yml index fd930a83a7..c5fd28cc60 100644 --- a/.github/workflows/test_build_wheels_windows_without_cuda.yml +++ b/.github/workflows/test_build_wheels_windows_without_cuda.yml @@ -7,7 +7,7 @@ on: - .github/workflows/test_build_wheels_windows_without_cuda.yml - .github/workflows/build_wheels_windows.yml - .github/workflows/generate_binary_build_matrix.yml - # - tools/scripts/generate_binary_build_matrix.py + - tools/scripts/generate_binary_build_matrix.py workflow_dispatch: jobs: diff --git a/tools/scripts/generate_binary_build_matrix.py b/tools/scripts/generate_binary_build_matrix.py index 5ad5c5d4cc..62daf854b0 100644 --- a/tools/scripts/generate_binary_build_matrix.py +++ b/tools/scripts/generate_binary_build_matrix.py @@ -21,7 +21,7 @@ mod = sys.modules[__name__] PYTHON_ARCHES_DICT = { - "nightly": ["3.10", "3.11"], + "nightly": ["3.8", "3.9", "3.10", "3.11"], "test": ["3.7", "3.8", "3.9", "3.10"], "release": ["3.7", "3.8", "3.9", "3.10"], }