Skip to content

Commit e808225

Browse files
mthrokmthrok
authored
Add test to validate torchscript backward compatibility (#838)
This CC adds CI test to check the backward compatibility of Torchscript functions/object dumps. The job first dumps Torchscript objects into file from torchaudio 0.6.0 release environment (x Python 3.6, 3.7, 3,8), then load & run the function in master build (in Python 3.6, 3.7, 3.8). If there is a BC-breaking change in master build, (registration schema change), then the test should fail. At this moment, `info` function does not work due to suspected bug in torch side, so the test is disabled for `info` function. See pytorch/pytorch#42258 for the detail. Once pytorch/pytorch#42258 is resolved we can enable it.
1 parent 4a8610f commit e808225

File tree

14 files changed

+558
-129
lines changed

14 files changed

+558
-129
lines changed

‎.circleci/config.yml‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -605,6 +605,42 @@ jobs:
605605
name: Run style check
606606
command: .circleci/unittest/linux/scripts/run_style_checks.sh
607607

608+
torchscript_bc_test:
609+
docker:
610+
- image: "pytorch/torchaudio_unittest_base:manylinux"
611+
resource_class: medium
612+
steps:
613+
- checkout
614+
- generate_cache_key
615+
- restore_cache:
616+
617+
keys:
618+
- torchscript-bc-test-release-envs-v1-{{ arch }}-{{ checksum ".cachekey" }}
619+
620+
- run:
621+
name: Generate Objects
622+
command: |
623+
.circleci/torchscript_bc_test/setup_release_envs.sh
624+
.circleci/torchscript_bc_test/generate_objects.sh
625+
- save_cache:
626+
627+
key: torchscript-bc-test-release-envs-v1-{{ arch }}-{{ checksum ".cachekey" }}
628+
629+
paths:
630+
- conda
631+
- envs
632+
- store_artifacts:
633+
path: test/torchscript_bc_test/assets
634+
- persist_to_workspace:
635+
root: .
636+
paths:
637+
- test/torchscript_bc_test/assets
638+
- run:
639+
name: Run BC check test
640+
command: |
641+
.circleci/torchscript_bc_test/setup_master_envs.sh
642+
.circleci/torchscript_bc_test/validate_objects.sh
643+
608644
workflows:
609645
build:
610646
jobs:
@@ -691,6 +727,9 @@ workflows:
691727
python_version: '3.8'
692728
unittest:
693729
jobs:
730+
- torchscript_bc_test:
731+
requires:
732+
- download_third_parties_nix
694733
- download_third_parties_nix:
695734
name: download_third_parties_nix
696735
- unittest_linux_cpu:

‎.circleci/config.yml.in‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -605,13 +605,52 @@ jobs:
605605
name: Run style check
606606
command: .circleci/unittest/linux/scripts/run_style_checks.sh
607607

608+
torchscript_bc_test:
609+
docker:
610+
- image: "pytorch/torchaudio_unittest_base:manylinux"
611+
resource_class: medium
612+
steps:
613+
- checkout
614+
- generate_cache_key
615+
- restore_cache:
616+
{% raw %}
617+
keys:
618+
- torchscript-bc-test-release-envs-v1-{{ arch }}-{{ checksum ".cachekey" }}
619+
{% endraw %}
620+
- run:
621+
name: Generate Objects
622+
command: |
623+
.circleci/torchscript_bc_test/setup_release_envs.sh
624+
.circleci/torchscript_bc_test/generate_objects.sh
625+
- save_cache:
626+
{% raw %}
627+
key: torchscript-bc-test-release-envs-v1-{{ arch }}-{{ checksum ".cachekey" }}
628+
{% endraw %}
629+
paths:
630+
- conda
631+
- envs
632+
- store_artifacts:
633+
path: test/torchscript_bc_test/assets
634+
- persist_to_workspace:
635+
root: .
636+
paths:
637+
- test/torchscript_bc_test/assets
638+
- run:
639+
name: Run BC check test
640+
command: |
641+
.circleci/torchscript_bc_test/setup_master_envs.sh
642+
.circleci/torchscript_bc_test/validate_objects.sh
643+
608644
workflows:
609645
build:
610646
jobs:
611647
- circleci_consistency
612648
{{ build_workflows() }}
613649
unittest:
614650
jobs:
651+
- torchscript_bc_test:
652+
requires:
653+
- download_third_parties_nix
615654
{{ unittest_workflows() }}
616655
nightly:
617656
jobs:

‎.circleci/torchscript_bc_test/common.sh‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
#!/usr/bin/env bash
2+
3+
declare -a TORCHAUDIO_VERSIONS=("0.6.0")
4+
declare -a PYTHON_VERSIONS=("3.6" "3.7" "3.8")
5+
6+
export TORCHAUDIO_VERSIONS
7+
export PYTHON_VERSIONS
8+
9+
export KALDI_ROOT="${KALDI_ROOT:-$HOME}" # Just to disable warning emitted from kaldi_io
10+
11+
_this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
12+
_root_dir="$(git rev-parse --show-toplevel)"
13+
_conda_dir="${_root_dir}/conda"
14+
case "$(uname -s)" in
15+
Darwin*) _os="MacOSX";;
16+
*) _os="Linux"
17+
esac
18+
19+
install_conda() {
20+
if [ ! -d "${_conda_dir}" ]; then
21+
printf "* Installing conda\n"
22+
wget -nv -O miniconda.sh "http://repo.continuum.io/miniconda/Miniconda3-latest-${_os}-x86_64.sh"
23+
bash ./miniconda.sh -b -f -p "${_conda_dir}"
24+
rm miniconda.sh
25+
fi
26+
}
27+
28+
init_conda() {
29+
eval "$("${_conda_dir}/bin/conda" shell.bash hook)"
30+
}
31+
32+
get_name() {
33+
echo "${1}-py${2}"
34+
}
35+
36+
get_env_dir() {
37+
echo "${_root_dir}/envs/$(get_name "$1" "$2")"
38+
}
39+
40+
create_env() {
41+
env_dir="$(get_env_dir "$1" "$2")"
42+
if [ ! -d "${env_dir}" ]; then
43+
printf "* Creating environment torchaudio: %s, Python: %s\n" "$1" "$2"
44+
conda create -q --prefix "${env_dir}" -y python="$2"
45+
fi
46+
}
47+
48+
activate_env() {
49+
printf "* Activating environment torchaudio: %s, Python: %s\n" "$1" "$2"
50+
conda activate "$(get_env_dir "$1" "$2")"
51+
}
52+
53+
install_release() {
54+
printf "* Installing torchaudio: %s\n" "$1"
55+
conda install -y -q torchaudio="$1" packaging -c pytorch
56+
# packaging is required in test to validate the torchaudio version for dump
57+
}
58+
59+
install_build_dependencies() {
60+
printf "* Installing torchaudio dependencies except PyTorch - (Python: %s)\n" "$1"
61+
conda env update -q --file "${_this_dir}/environment.yml" --prune
62+
if [ "${_os}" == Linux ]; then
63+
pip install clang-format
64+
fi
65+
}
66+
67+
build_master() {
68+
printf "* Installing PyTorch (py%s)\n" "$1"
69+
conda install -y -q pytorch "cpuonly" -c pytorch-nightly
70+
printf "* Installing torchaudio\n"
71+
cd "${_root_dir}" || exit 1
72+
BUILD_SOX=1 python setup.py clean install
73+
}

‎.circleci/torchscript_bc_test/environment.yml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
channels:
2+
- conda-forge
3+
- defaults
4+
dependencies:
5+
- flake8
6+
- numpy
7+
- pytest
8+
- pytest-cov
9+
- codecov
10+
- librosa
11+
- llvmlite==0.31 # See https://github.com/pytorch/audio/pull/766
12+
- pip
13+
- pip:
14+
- kaldi-io
15+
- scipy
16+
- parameterized
17+
- numba==0.48 # See https://github.com/librosa/librosa/issues/1160

‎.circleci/torchscript_bc_test/generate_objects.sh‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
#!/usr/bin/env bash
2+
3+
set -e
4+
5+
this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
6+
pushd "${this_dir}"
7+
. "common.sh"
8+
popd
9+
10+
init_conda
11+
12+
# Move to test directory so that the checked out torchaudio source
13+
# will not shadow the conda-installed version of torchaudio
14+
cd test
15+
16+
for torchaudio in "${TORCHAUDIO_VERSIONS[@]}" ; do
17+
for python in "${PYTHON_VERSIONS[@]}" ; do
18+
activate_env "${torchaudio}" "${python}"
19+
python -m torch.utils.collect_env
20+
printf "***********************************************************\n"
21+
printf "* Generating\n"
22+
printf " Objects: Python: %s, torchaudio: %s\n" "${python}" "${torchaudio}"
23+
printf "***********************************************************\n"
24+
./torchscript_bc_test/main.py --mode generate --version "${torchaudio}"
25+
done
26+
done

‎.circleci/torchscript_bc_test/setup_master_envs.sh‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
#!/usr/bin/env bash
2+
3+
set -e
4+
5+
cd "$( dirname "${BASH_SOURCE[0]}" )"
6+
. "common.sh"
7+
8+
install_conda
9+
init_conda
10+
11+
# Install torchaudio environments
12+
for python in "${PYTHON_VERSIONS[@]}" ; do
13+
create_env master "${python}"
14+
activate_env master "${python}"
15+
install_build_dependencies "${python}"
16+
build_master "${python}"
17+
done

‎.circleci/torchscript_bc_test/setup_release_envs.sh‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
#!/usr/bin/env bash
2+
3+
set -e
4+
5+
cd "$( dirname "${BASH_SOURCE[0]}" )"
6+
. "common.sh"
7+
8+
install_conda
9+
init_conda
10+
11+
# Install torchaudio environments
12+
for torchaudio in "${TORCHAUDIO_VERSIONS[@]}" ; do
13+
for python in "${PYTHON_VERSIONS[@]}" ; do
14+
create_env "${torchaudio}" "${python}"
15+
activate_env "${torchaudio}" "${python}"
16+
install_release "${torchaudio}"
17+
done
18+
done

‎.circleci/torchscript_bc_test/validate_objects.sh‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
#!/usr/bin/env bash
2+
3+
set -e
4+
5+
this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
6+
pushd "${this_dir}"
7+
. "common.sh"
8+
popd
9+
10+
init_conda
11+
12+
# Move to test directory so that the checked out torchaudio source
13+
# will not shadow the conda-installed version of torchaudio
14+
cd test
15+
16+
# Validate torchscript objects for each
17+
for runtime_python in "${PYTHON_VERSIONS[@]}" ; do
18+
activate_env master "${runtime_python}"
19+
python -m torch.utils.collect_env
20+
for object_torchaudio in "${TORCHAUDIO_VERSIONS[@]}" ; do
21+
for object_python in "${PYTHON_VERSIONS[@]}" ; do
22+
printf "***********************************************************\n"
23+
printf "* Validating\n"
24+
printf " Runtime: Python: %s, torchaudio: master (%s)\n" "${runtime_python}" "$(python -c 'import torchaudio;print(torchaudio.__version__)')"
25+
printf " Objects: Python: %s, torchaudio: %s\n" "${object_python}" "${object_torchaudio}"
26+
printf "***********************************************************\n"
27+
./torchscript_bc_test/main.py --mode validate --version "${object_torchaudio}"
28+
done
29+
done
30+
done

0 commit comments

Comments
 (0)