From 03810ff2bd40e2243778d97333f4272b97b302fe Mon Sep 17 00:00:00 2001 From: Omkar Salpekar Date: Fri, 3 Feb 2023 11:06:56 -0800 Subject: [PATCH 1/7] Try python 3.11 wheels builds --- .../actions/setup-binary-builds/action.yml | 31 +++++++++++++------ .../test_build_conda_linux_with_cuda.yml | 2 +- .../test_build_conda_linux_without_cuda.yml | 2 +- .github/workflows/test_build_conda_m1.yml | 2 +- .github/workflows/test_build_conda_macos.yml | 2 +- .../test_build_conda_windows_without_cuda.yml | 2 +- .../test_build_wheels_linux_with_cuda.yml | 20 ++++++------ .github/workflows/test_build_wheels_m1.yml | 2 +- .github/workflows/test_build_wheels_macos.yml | 2 +- .../test_build_wheels_windows_with_cuda.yml | 2 +- ...test_build_wheels_windows_without_cuda.yml | 2 +- tools/scripts/generate_binary_build_matrix.py | 2 +- 12 files changed, 42 insertions(+), 29 deletions(-) diff --git a/.github/actions/setup-binary-builds/action.yml b/.github/actions/setup-binary-builds/action.yml index 7e8eb5f99e..c991d6c606 100644 --- a/.github/actions/setup-binary-builds/action.yml +++ b/.github/actions/setup-binary-builds/action.yml @@ -100,14 +100,27 @@ runs: set -euxo pipefail CONDA_ENV="${RUNNER_TEMP}/conda_environment_${GITHUB_RUN_ID}" - conda create \ - --yes \ - --prefix "${CONDA_ENV}" \ - "python=${PYTHON_VERSION}" \ - cmake=3.22 \ - conda-build=3.21 \ - ninja=1.10 \ - pkg-config=0.29 \ - wheel=0.37 + if [[ "${PYTHON_VERSION}" = "3.11" ]]; then + conda create \ + --yes \ + --prefix "${CONDA_ENV}" \ + -c conda-forge \ + "python=${PYTHON_VERSION}" \ + cmake=3.22 \ + conda-build=3.21 \ + ninja=1.10 \ + pkg-config=0.29 \ + wheel=0.37 + else + conda create \ + --yes \ + --prefix "${CONDA_ENV}" \ + "python=${PYTHON_VERSION}" \ + cmake=3.22 \ + conda-build=3.21 \ + ninja=1.10 \ + pkg-config=0.29 \ + wheel=0.37 + fi echo "CONDA_ENV=${CONDA_ENV}" >> "${GITHUB_ENV}" echo "CONDA_RUN=conda run -p ${CONDA_ENV}" >> "${GITHUB_ENV}" diff --git a/.github/workflows/test_build_conda_linux_with_cuda.yml b/.github/workflows/test_build_conda_linux_with_cuda.yml index dfdaa4bfe5..c898fc8028 100644 --- a/.github/workflows/test_build_conda_linux_with_cuda.yml +++ b/.github/workflows/test_build_conda_linux_with_cuda.yml @@ -6,7 +6,7 @@ on: - .github/actions/setup-binary-builds/action.yml - .github/workflows/test_build_conda_linux.yml - .github/workflows/build_conda_linux.yml - - .github/workflows/generate_binary_build_matrix.yml + # - .github/workflows/generate_binary_build_matrix.yml - tools/scripts/generate_binary_build_matrix.py workflow_dispatch: diff --git a/.github/workflows/test_build_conda_linux_without_cuda.yml b/.github/workflows/test_build_conda_linux_without_cuda.yml index 9331bd8f06..d33bd6e633 100644 --- a/.github/workflows/test_build_conda_linux_without_cuda.yml +++ b/.github/workflows/test_build_conda_linux_without_cuda.yml @@ -7,7 +7,7 @@ on: - .github/workflows/test_build_conda_linux.yml - .github/workflows/build_conda_linux.yml - .github/workflows/generate_binary_build_matrix.yml - - tools/scripts/generate_binary_build_matrix.py + # - tools/scripts/generate_binary_build_matrix.py workflow_dispatch: jobs: diff --git a/.github/workflows/test_build_conda_m1.yml b/.github/workflows/test_build_conda_m1.yml index 4f3c1d207c..2fe2430cca 100644 --- a/.github/workflows/test_build_conda_m1.yml +++ b/.github/workflows/test_build_conda_m1.yml @@ -7,7 +7,7 @@ on: - .github/workflows/test_build_conda_m1.yml - .github/workflows/build_conda_macos.yml - .github/workflows/generate_binary_build_matrix.yml - - tools/scripts/generate_binary_build_matrix.py + # - tools/scripts/generate_binary_build_matrix.py workflow_dispatch: jobs: diff --git a/.github/workflows/test_build_conda_macos.yml b/.github/workflows/test_build_conda_macos.yml index 4b0708f7c5..c3dfa99a71 100644 --- a/.github/workflows/test_build_conda_macos.yml +++ b/.github/workflows/test_build_conda_macos.yml @@ -7,7 +7,7 @@ on: - .github/workflows/test_build_conda_macos.yml - .github/workflows/build_conda_macos.yml - .github/workflows/generate_binary_build_matrix.yml - - tools/scripts/generate_binary_build_matrix.py + # - tools/scripts/generate_binary_build_matrix.py jobs: generate-matrix: diff --git a/.github/workflows/test_build_conda_windows_without_cuda.yml b/.github/workflows/test_build_conda_windows_without_cuda.yml index 2bffc090c8..434fb3e007 100644 --- a/.github/workflows/test_build_conda_windows_without_cuda.yml +++ b/.github/workflows/test_build_conda_windows_without_cuda.yml @@ -7,7 +7,7 @@ on: - .github/workflows/test_build_conda_windows_without_cuda.yml - .github/workflows/build_conda_windows.yml - .github/workflows/generate_binary_build_matrix.yml - - tools/scripts/generate_binary_build_matrix.py + # - tools/scripts/generate_binary_build_matrix.py workflow_dispatch: jobs: diff --git a/.github/workflows/test_build_wheels_linux_with_cuda.yml b/.github/workflows/test_build_wheels_linux_with_cuda.yml index d3ed608f42..3da65fa8d4 100644 --- a/.github/workflows/test_build_wheels_linux_with_cuda.yml +++ b/.github/workflows/test_build_wheels_linux_with_cuda.yml @@ -25,21 +25,21 @@ jobs: fail-fast: false matrix: include: - - repository: pytorch/audio - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: test/smoke_test/smoke_test.py - package-name: torchaudio + # - repository: pytorch/audio + # pre-script: packaging/pre_build_script.sh + # post-script: packaging/post_build_script.sh + # smoke-test-script: test/smoke_test/smoke_test.py + # package-name: torchaudio - repository: pytorch/vision pre-script: packaging/pre_build_script.sh post-script: packaging/post_build_script.sh smoke-test-script: test/smoke_test.py package-name: torchvision - - repository: pytorch/torchrec - pre-script: "" - post-script: "" - smoke-test-script: "" - package-name: torchrec + # - repository: pytorch/torchrec + # pre-script: "" + # post-script: "" + # smoke-test-script: "" + # package-name: torchrec uses: ./.github/workflows/build_wheels_linux.yml name: ${{ matrix.repository }} with: diff --git a/.github/workflows/test_build_wheels_m1.yml b/.github/workflows/test_build_wheels_m1.yml index 6460a81da8..39c4e1aae3 100644 --- a/.github/workflows/test_build_wheels_m1.yml +++ b/.github/workflows/test_build_wheels_m1.yml @@ -7,7 +7,7 @@ on: - .github/workflows/test_build_wheels_m1.yml - .github/workflows/build_wheels_macos.yml - .github/workflows/generate_binary_build_matrix.yml - - tools/scripts/generate_binary_build_matrix.py + # - tools/scripts/generate_binary_build_matrix.py workflow_dispatch: jobs: diff --git a/.github/workflows/test_build_wheels_macos.yml b/.github/workflows/test_build_wheels_macos.yml index bc1b890183..66e481c639 100644 --- a/.github/workflows/test_build_wheels_macos.yml +++ b/.github/workflows/test_build_wheels_macos.yml @@ -7,7 +7,7 @@ on: - .github/workflows/test_build_wheels_macos.yml - .github/workflows/build_wheels_macos.yml - .github/workflows/generate_binary_build_matrix.yml - - tools/scripts/generate_binary_build_matrix.py + # - tools/scripts/generate_binary_build_matrix.py workflow_dispatch: jobs: diff --git a/.github/workflows/test_build_wheels_windows_with_cuda.yml b/.github/workflows/test_build_wheels_windows_with_cuda.yml index 6066bc3bf1..20fcbbd85f 100644 --- a/.github/workflows/test_build_wheels_windows_with_cuda.yml +++ b/.github/workflows/test_build_wheels_windows_with_cuda.yml @@ -7,7 +7,7 @@ on: - .github/workflows/test_build_wheels_windows_with_cuda.yml - .github/workflows/build_wheels_windows.yml - .github/workflows/generate_binary_build_matrix.yml - - tools/scripts/generate_binary_build_matrix.py + # - tools/scripts/generate_binary_build_matrix.py workflow_dispatch: jobs: diff --git a/.github/workflows/test_build_wheels_windows_without_cuda.yml b/.github/workflows/test_build_wheels_windows_without_cuda.yml index c5fd28cc60..fd930a83a7 100644 --- a/.github/workflows/test_build_wheels_windows_without_cuda.yml +++ b/.github/workflows/test_build_wheels_windows_without_cuda.yml @@ -7,7 +7,7 @@ on: - .github/workflows/test_build_wheels_windows_without_cuda.yml - .github/workflows/build_wheels_windows.yml - .github/workflows/generate_binary_build_matrix.yml - - tools/scripts/generate_binary_build_matrix.py + # - tools/scripts/generate_binary_build_matrix.py workflow_dispatch: jobs: diff --git a/tools/scripts/generate_binary_build_matrix.py b/tools/scripts/generate_binary_build_matrix.py index 65b06355e6..5ad5c5d4cc 100644 --- a/tools/scripts/generate_binary_build_matrix.py +++ b/tools/scripts/generate_binary_build_matrix.py @@ -21,7 +21,7 @@ mod = sys.modules[__name__] PYTHON_ARCHES_DICT = { - "nightly": ["3.8", "3.9", "3.10"], + "nightly": ["3.10", "3.11"], "test": ["3.7", "3.8", "3.9", "3.10"], "release": ["3.7", "3.8", "3.9", "3.10"], } From 3d2f371504de3c9d86429d2c416c75a15fb8a5c0 Mon Sep 17 00:00:00 2001 From: Omkar Salpekar Date: Mon, 6 Feb 2023 13:01:22 -0800 Subject: [PATCH 2/7] fix conflicts in conda env setup --- .github/actions/setup-binary-builds/action.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/actions/setup-binary-builds/action.yml b/.github/actions/setup-binary-builds/action.yml index c991d6c606..7e6963b6f2 100644 --- a/.github/actions/setup-binary-builds/action.yml +++ b/.github/actions/setup-binary-builds/action.yml @@ -105,12 +105,8 @@ runs: --yes \ --prefix "${CONDA_ENV}" \ -c conda-forge \ - "python=${PYTHON_VERSION}" \ - cmake=3.22 \ - conda-build=3.21 \ - ninja=1.10 \ - pkg-config=0.29 \ - wheel=0.37 + "python=${PYTHON_VERSION}" + conda install wheel ninja pkg-config else conda create \ --yes \ From 089c942d4ef6fa8225ac9942a096738ba3176303 Mon Sep 17 00:00:00 2001 From: Wei Wang Date: Tue, 7 Feb 2023 07:03:29 +0000 Subject: [PATCH 3/7] test torchaudio with py311 --- .../test_build_wheels_linux_with_cuda.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test_build_wheels_linux_with_cuda.yml b/.github/workflows/test_build_wheels_linux_with_cuda.yml index 3da65fa8d4..7ac66fc57d 100644 --- a/.github/workflows/test_build_wheels_linux_with_cuda.yml +++ b/.github/workflows/test_build_wheels_linux_with_cuda.yml @@ -25,16 +25,16 @@ jobs: fail-fast: false matrix: include: - # - repository: pytorch/audio - # pre-script: packaging/pre_build_script.sh - # post-script: packaging/post_build_script.sh - # smoke-test-script: test/smoke_test/smoke_test.py - # package-name: torchaudio - - repository: pytorch/vision + - repository: pytorch/audio pre-script: packaging/pre_build_script.sh post-script: packaging/post_build_script.sh - smoke-test-script: test/smoke_test.py - package-name: torchvision + smoke-test-script: test/smoke_test/smoke_test.py + package-name: torchaudio + #- repository: pytorch/vision + #- pre-script: packaging/pre_build_script.sh + #- post-script: packaging/post_build_script.sh + #- smoke-test-script: test/smoke_test.py + #- package-name: torchvision # - repository: pytorch/torchrec # pre-script: "" # post-script: "" From f0e01a9b7b1051db25fa53eaa53da8f366086402 Mon Sep 17 00:00:00 2001 From: Wei Wang Date: Tue, 7 Feb 2023 07:27:44 +0000 Subject: [PATCH 4/7] M1 needs conda-build --- .github/actions/setup-binary-builds/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/setup-binary-builds/action.yml b/.github/actions/setup-binary-builds/action.yml index 7e6963b6f2..192b1be5e4 100644 --- a/.github/actions/setup-binary-builds/action.yml +++ b/.github/actions/setup-binary-builds/action.yml @@ -106,7 +106,7 @@ runs: --prefix "${CONDA_ENV}" \ -c conda-forge \ "python=${PYTHON_VERSION}" - conda install wheel ninja pkg-config + conda install wheel ninja pkg-config conda-build else conda create \ --yes \ From 86c7bcfd0e4940bf0ec52e22668889dd1a085065 Mon Sep 17 00:00:00 2001 From: Wei Wang Date: Tue, 7 Feb 2023 12:20:35 -0800 Subject: [PATCH 5/7] MacOS needs cmake --- .github/actions/setup-binary-builds/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/setup-binary-builds/action.yml b/.github/actions/setup-binary-builds/action.yml index 192b1be5e4..24e5d56a14 100644 --- a/.github/actions/setup-binary-builds/action.yml +++ b/.github/actions/setup-binary-builds/action.yml @@ -106,7 +106,7 @@ runs: --prefix "${CONDA_ENV}" \ -c conda-forge \ "python=${PYTHON_VERSION}" - conda install wheel ninja pkg-config conda-build + conda install wheel cmake ninja pkg-config conda-build else conda create \ --yes \ From 095d714e04d4a7771daeeb53a3c3da5e4cca10eb Mon Sep 17 00:00:00 2001 From: Wei Wang Date: Tue, 7 Feb 2023 12:44:44 -0800 Subject: [PATCH 6/7] Apply Omkar's change https://github.com/pytorch/test-infra/pull/2105/commits/f7901baff834e2bda6983b2b36d20305ba75da3a --- .github/actions/setup-binary-builds/action.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/actions/setup-binary-builds/action.yml b/.github/actions/setup-binary-builds/action.yml index 24e5d56a14..bb63b6516d 100644 --- a/.github/actions/setup-binary-builds/action.yml +++ b/.github/actions/setup-binary-builds/action.yml @@ -104,9 +104,11 @@ runs: conda create \ --yes \ --prefix "${CONDA_ENV}" \ - -c conda-forge \ "python=${PYTHON_VERSION}" - conda install wheel cmake ninja pkg-config conda-build + cmake=3.22 \ + ninja=1.10 \ + pkg-config=0.29 \ + wheel=0.37 else conda create \ --yes \ From e6c46a2220216e9e741a8a3a27f0c5412201f7b1 Mon Sep 17 00:00:00 2001 From: Wei Wang Date: Tue, 7 Feb 2023 13:42:24 -0800 Subject: [PATCH 7/7] miss "\" oops --- .github/actions/setup-binary-builds/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/setup-binary-builds/action.yml b/.github/actions/setup-binary-builds/action.yml index bb63b6516d..92d780f2b5 100644 --- a/.github/actions/setup-binary-builds/action.yml +++ b/.github/actions/setup-binary-builds/action.yml @@ -104,7 +104,7 @@ runs: conda create \ --yes \ --prefix "${CONDA_ENV}" \ - "python=${PYTHON_VERSION}" + "python=${PYTHON_VERSION}" \ cmake=3.22 \ ninja=1.10 \ pkg-config=0.29 \