From b478170845e6e1a178ff636e60736add47d39534 Mon Sep 17 00:00:00 2001 From: atalman Date: Thu, 2 Jun 2022 06:38:35 -0700 Subject: [PATCH 1/5] Retrieve version from version.txt and small m1 improvements --- .github/workflows/build-m1-binaries.yml | 7 ++++--- packaging/build_conda.sh | 2 +- packaging/build_wheel.sh | 2 +- packaging/pkg_helpers.bash | 13 +++++++++++-- version.txt | 1 + 5 files changed, 18 insertions(+), 7 deletions(-) create mode 100644 version.txt diff --git a/.github/workflows/build-m1-binaries.yml b/.github/workflows/build-m1-binaries.yml index 422c41fb8f..88eb167024 100644 --- a/.github/workflows/build-m1-binaries.yml +++ b/.github/workflows/build-m1-binaries.yml @@ -26,12 +26,13 @@ jobs: echo $PATH . ~/miniconda3/etc/profile.d/conda.sh set -ex - export BUILD_VERSION=0.14.0.dev$(date "+%Y%m%d") + . packaging/pkg_helpers.bash setup_build_version WHL_NAME=torchaudio-${BUILD_VERSION}-cp${PY_VERS/.}-cp${PY_VERS/.}-macosx_11_0_arm64.whl conda create -yp ${ENV_NAME} python=${PY_VERS} numpy cmake ninja libpng openjpeg wheel pkg-config - conda run -p ${ENV_NAME} python3 -mpip install torch --extra-index-url=https://download.pytorch.org/whl/nightly + conda run -p ${ENV_NAME} python3 -mpip install torch --pre --extra-index-url=https://download.pytorch.org/whl/nightly conda run -p ${ENV_NAME} python3 -mpip install delocate conda run -p ${ENV_NAME} python3 setup.py bdist_wheel + export PYTORCH_VERSION="$(conda run -p ${ENV_NAME} python3 -mpip show torch | grep ^Version: | sed 's/Version: *//')" conda run -p ${ENV_NAME} delocate-wheel -v --ignore-missing-dependencies dist/${WHL_NAME} conda env remove -p ${ENV_NAME} - name: Test wheel @@ -43,7 +44,7 @@ jobs: . ~/miniconda3/etc/profile.d/conda.sh set -ex conda create -yp ${ENV_NAME} python=${PY_VERS} numpy - conda run -p ${ENV_NAME} python3 -mpip install torch --extra-index-url=https://download.pytorch.org/whl/nightly + conda run -p ${ENV_NAME} python3 -mpip install torch --pre --extra-index-url=https://download.pytorch.org/whl/nightly conda run -p ${ENV_NAME} python3 -mpip install dist/*.whl # Test torch is importable, by changing cwd and running import commands conda run --cwd /tmp -p ${ENV_NAME} python3 -c "import torchaudio;print('torchaudio version is ', torchaudio.__version__)" diff --git a/packaging/build_conda.sh b/packaging/build_conda.sh index 3cfb98b91f..e326db4455 100755 --- a/packaging/build_conda.sh +++ b/packaging/build_conda.sh @@ -7,7 +7,7 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" . "$script_dir/pkg_helpers.bash" export BUILD_TYPE="conda" -setup_env 0.12.0 +setup_env export SOURCE_ROOT_DIR="$PWD" setup_conda_pytorch_constraint setup_conda_cudatoolkit_constraint diff --git a/packaging/build_wheel.sh b/packaging/build_wheel.sh index 5fa30692bd..a19fd9cdcd 100755 --- a/packaging/build_wheel.sh +++ b/packaging/build_wheel.sh @@ -7,7 +7,7 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" . "$script_dir/pkg_helpers.bash" export BUILD_TYPE="wheel" -setup_env 0.12.0 +setup_env setup_wheel_python pip_install numpy future cmake ninja setup_pip_pytorch_version diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash index ef757da498..dea6e7b7b9 100644 --- a/packaging/pkg_helpers.bash +++ b/packaging/pkg_helpers.bash @@ -97,10 +97,19 @@ setup_cuda() { # BUILD_VERSION (e.g., 0.2.0.dev20190807+cpu) # # Fill BUILD_VERSION if it doesn't exist already with a nightly string +# Or retrieve it from the version.txt # Usage: setup_build_version 0.2.0 setup_build_version() { if [[ -z "$BUILD_VERSION" ]]; then - export BUILD_VERSION="$1.dev$(date "+%Y%m%d")$VERSION_SUFFIX" + if [[ -z "$1" ]]; then + SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + # version.txt for some reason has `a` character after major.minor.rev + # command below yields 0.10.0 from version.txt containing 0.10.0a0 + _VERSION_BASE=$( cut -f 1 -d a "$SCRIPT_DIR/../version.txt" ) + else + _VERSION_BASE="$1" + fi + export BUILD_VERSION="$_VERSION_BASE.dev$(date "+%Y%m%d")$VERSION_SUFFIX" else export BUILD_VERSION="$BUILD_VERSION$VERSION_SUFFIX" fi @@ -119,7 +128,7 @@ setup_macos() { setup_env() { git submodule update --init --recursive setup_cuda - setup_build_version "$1" + setup_build_version setup_macos } diff --git a/version.txt b/version.txt new file mode 100644 index 0000000000..3db2940af4 --- /dev/null +++ b/version.txt @@ -0,0 +1 @@ +0.12.0a0 From dbc86b91f46829d4ede44104483269aed2050479 Mon Sep 17 00:00:00 2001 From: atalman Date: Thu, 2 Jun 2022 08:03:36 -0700 Subject: [PATCH 2/5] Don't upload artifacts if its pull request build --- .github/workflows/build-m1-binaries.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build-m1-binaries.yml b/.github/workflows/build-m1-binaries.yml index 88eb167024..9f0e56ca7b 100644 --- a/.github/workflows/build-m1-binaries.yml +++ b/.github/workflows/build-m1-binaries.yml @@ -51,11 +51,13 @@ jobs: conda run --cwd /tmp -p ${ENV_NAME} python3 -c "import torch;import torchaudio;torchaudio.set_audio_backend('sox_io')" conda env remove -p ${ENV_NAME} - name: Upload wheel to GitHub + if: ${{ github.event_name == 'push' && steps.extract_branch.outputs.branch == 'nightly' }} uses: actions/upload-artifact@v3 with: name: torchaudio-py${{ matrix.py_vers }}-macos11-m1 path: dist/ - name: Upload wheel to S3 + if: ${{ github.event_name == 'push' && steps.extract_branch.outputs.branch == 'nightly' }} shell: arch -arch arm64 bash {0} env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} From 84c3aba37e58689979412145baac4958849b4132 Mon Sep 17 00:00:00 2001 From: atalman Date: Thu, 2 Jun 2022 08:47:19 -0700 Subject: [PATCH 3/5] Fix small typo --- .github/workflows/build-m1-binaries.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-m1-binaries.yml b/.github/workflows/build-m1-binaries.yml index 9f0e56ca7b..9436cf5072 100644 --- a/.github/workflows/build-m1-binaries.yml +++ b/.github/workflows/build-m1-binaries.yml @@ -26,14 +26,15 @@ jobs: echo $PATH . ~/miniconda3/etc/profile.d/conda.sh set -ex - . packaging/pkg_helpers.bash setup_build_version + . packaging/pkg_helpers.bash + setup_build_version WHL_NAME=torchaudio-${BUILD_VERSION}-cp${PY_VERS/.}-cp${PY_VERS/.}-macosx_11_0_arm64.whl conda create -yp ${ENV_NAME} python=${PY_VERS} numpy cmake ninja libpng openjpeg wheel pkg-config conda run -p ${ENV_NAME} python3 -mpip install torch --pre --extra-index-url=https://download.pytorch.org/whl/nightly conda run -p ${ENV_NAME} python3 -mpip install delocate conda run -p ${ENV_NAME} python3 setup.py bdist_wheel export PYTORCH_VERSION="$(conda run -p ${ENV_NAME} python3 -mpip show torch | grep ^Version: | sed 's/Version: *//')" - conda run -p ${ENV_NAME} delocate-wheel -v --ignore-missing-dependencies dist/${WHL_NAME} + conda run -p ${ENV_NAME} DYLD_FALLBACK_LIBRARY_PATH="${ENV_NAME}/lib" delocate-wheel -v --ignore-missing-dependencies dist/${WHL_NAME} conda env remove -p ${ENV_NAME} - name: Test wheel shell: arch -arch arm64 bash {0} From 2d4871edd5a51458841dd5d955bc7b1e0fd58e76 Mon Sep 17 00:00:00 2001 From: atalman Date: Thu, 2 Jun 2022 09:23:21 -0700 Subject: [PATCH 4/5] Simplify version getting logic --- packaging/pkg_helpers.bash | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash index dea6e7b7b9..b1eac3dfd5 100644 --- a/packaging/pkg_helpers.bash +++ b/packaging/pkg_helpers.bash @@ -98,17 +98,13 @@ setup_cuda() { # # Fill BUILD_VERSION if it doesn't exist already with a nightly string # Or retrieve it from the version.txt -# Usage: setup_build_version 0.2.0 +# Usage: setup_build_version setup_build_version() { if [[ -z "$BUILD_VERSION" ]]; then - if [[ -z "$1" ]]; then - SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" - # version.txt for some reason has `a` character after major.minor.rev - # command below yields 0.10.0 from version.txt containing 0.10.0a0 - _VERSION_BASE=$( cut -f 1 -d a "$SCRIPT_DIR/../version.txt" ) - else - _VERSION_BASE="$1" - fi + SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + # version.txt for some reason has `a` character after major.minor.rev + # command below yields 0.10.0 from version.txt containing 0.10.0a0 + _VERSION_BASE=$( cut -f 1 -d a "$SCRIPT_DIR/../version.txt" ) export BUILD_VERSION="$_VERSION_BASE.dev$(date "+%Y%m%d")$VERSION_SUFFIX" else export BUILD_VERSION="$BUILD_VERSION$VERSION_SUFFIX" From 5e2ab89604f850e72f89d73262c5f15035610b39 Mon Sep 17 00:00:00 2001 From: atalman Date: Thu, 2 Jun 2022 09:30:32 -0700 Subject: [PATCH 5/5] Temporary remove m1 changes next pr to follow --- .github/workflows/build-m1-binaries.yml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-m1-binaries.yml b/.github/workflows/build-m1-binaries.yml index 9436cf5072..422c41fb8f 100644 --- a/.github/workflows/build-m1-binaries.yml +++ b/.github/workflows/build-m1-binaries.yml @@ -26,15 +26,13 @@ jobs: echo $PATH . ~/miniconda3/etc/profile.d/conda.sh set -ex - . packaging/pkg_helpers.bash - setup_build_version + export BUILD_VERSION=0.14.0.dev$(date "+%Y%m%d") WHL_NAME=torchaudio-${BUILD_VERSION}-cp${PY_VERS/.}-cp${PY_VERS/.}-macosx_11_0_arm64.whl conda create -yp ${ENV_NAME} python=${PY_VERS} numpy cmake ninja libpng openjpeg wheel pkg-config - conda run -p ${ENV_NAME} python3 -mpip install torch --pre --extra-index-url=https://download.pytorch.org/whl/nightly + conda run -p ${ENV_NAME} python3 -mpip install torch --extra-index-url=https://download.pytorch.org/whl/nightly conda run -p ${ENV_NAME} python3 -mpip install delocate conda run -p ${ENV_NAME} python3 setup.py bdist_wheel - export PYTORCH_VERSION="$(conda run -p ${ENV_NAME} python3 -mpip show torch | grep ^Version: | sed 's/Version: *//')" - conda run -p ${ENV_NAME} DYLD_FALLBACK_LIBRARY_PATH="${ENV_NAME}/lib" delocate-wheel -v --ignore-missing-dependencies dist/${WHL_NAME} + conda run -p ${ENV_NAME} delocate-wheel -v --ignore-missing-dependencies dist/${WHL_NAME} conda env remove -p ${ENV_NAME} - name: Test wheel shell: arch -arch arm64 bash {0} @@ -45,20 +43,18 @@ jobs: . ~/miniconda3/etc/profile.d/conda.sh set -ex conda create -yp ${ENV_NAME} python=${PY_VERS} numpy - conda run -p ${ENV_NAME} python3 -mpip install torch --pre --extra-index-url=https://download.pytorch.org/whl/nightly + conda run -p ${ENV_NAME} python3 -mpip install torch --extra-index-url=https://download.pytorch.org/whl/nightly conda run -p ${ENV_NAME} python3 -mpip install dist/*.whl # Test torch is importable, by changing cwd and running import commands conda run --cwd /tmp -p ${ENV_NAME} python3 -c "import torchaudio;print('torchaudio version is ', torchaudio.__version__)" conda run --cwd /tmp -p ${ENV_NAME} python3 -c "import torch;import torchaudio;torchaudio.set_audio_backend('sox_io')" conda env remove -p ${ENV_NAME} - name: Upload wheel to GitHub - if: ${{ github.event_name == 'push' && steps.extract_branch.outputs.branch == 'nightly' }} uses: actions/upload-artifact@v3 with: name: torchaudio-py${{ matrix.py_vers }}-macos11-m1 path: dist/ - name: Upload wheel to S3 - if: ${{ github.event_name == 'push' && steps.extract_branch.outputs.branch == 'nightly' }} shell: arch -arch arm64 bash {0} env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }}