Skip to content

Commit 6c78dfe

Browse files
chensuyuechensuyue
authored
Update framework versions for CI (#1462)
Signed-off-by: chensuyue <[email protected]> Signed-off-by: Xin He <[email protected]>
1 parent 9d41a7d commit 6c78dfe

29 files changed

+105
-1134
lines changed

‎.azure-pipelines/model-test.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -155,9 +155,9 @@ stages:
155155
patterns: "**/*_tuning_info.log"
156156
path: $(OUT_SCRIPT_PATH)
157157
- task: UsePythonVersion@0
158-
displayName: "Use Python 3.8"
158+
displayName: "Use Python 3.10"
159159
inputs:
160-
versionSpec: "3.8"
160+
versionSpec: "3.10"
161161
- script: |
162162
cd ${OUT_SCRIPT_PATH}
163163
mkdir generated

‎.azure-pipelines/scripts/fwk_version.sh‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
#!/bin/bash
2+
3+
echo "export FWs version..."
4+
test_mode=$1
5+
6+
if [ "$test_mode" == "coverage" ] || [ "$test_mode" == "latest" ]; then
7+
export tensorflow_version='2.14.0'
8+
export pytorch_version='2.1.0+cpu'
9+
export torchvision_version='0.16.0+cpu'
10+
export ipex_version='2.1.0+cpu'
11+
export onnx_version='1.14.1'
12+
export onnxruntime_version='1.16.3'
13+
export mxnet_version='1.9.1'
14+
else
15+
export tensorflow_version='2.13.0'
16+
export pytorch_version='2.0.1+cpu'
17+
export torchvision_version='0.15.2+cpu'
18+
export ipex_version='2.0.1+cpu'
19+
export onnx_version='1.14.1'
20+
export onnxruntime_version='1.15.1'
21+
export mxnet_version='1.9.1'
22+
fi
23+
24+
25+
26+
27+

‎.azure-pipelines/scripts/models/run_mxnet_models_trigger.sh‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,10 @@ do
2121
esac
2222
done
2323

24+
echo "specify FWs version..."
25+
source /neural-compressor/.azure-pipelines/scripts/fwk_version.sh 'latest'
2426
FRAMEWORK="mxnet"
25-
FRAMEWORK_VERSION="1.9.1"
27+
FRAMEWORK_VERSION=${mxnet_version}
2628

2729
inc_new_api=false
2830
# ======== set up config for mxnet models ========

‎.azure-pipelines/scripts/models/run_onnxrt_models_trigger.sh‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,10 @@ do
2121
esac
2222
done
2323

24+
echo "specify FWs version..."
25+
source /neural-compressor/.azure-pipelines/scripts/fwk_version.sh 'latest'
2426
FRAMEWORK="onnxrt"
25-
FRAMEWORK_VERSION="1.15.1"
27+
FRAMEWORK_VERSION=${onnxruntime_version}
2628

2729
inc_new_api=false
2830
# ======== set up config for onnxrt models ========

‎.azure-pipelines/scripts/models/run_pytorch_models_trigger.sh‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,11 @@ do
2121
esac
2222
done
2323

24+
echo "specify FWs version..."
25+
source /neural-compressor/.azure-pipelines/scripts/fwk_version.sh 'latest'
2426
FRAMEWORK="pytorch"
25-
FRAMEWORK_VERSION="2.0.1+cpu"
26-
TORCH_VISION_VERSION="0.15.2+cpu"
27+
FRAMEWORK_VERSION=${pytorch_version}
28+
TORCH_VISION_VERSION=${torchvision_version}
2729

2830
inc_new_api=false
2931
# ======== set up config for pytorch models ========

‎.azure-pipelines/scripts/models/run_tensorflow_models_trigger.sh‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,10 @@ do
2121
esac
2222
done
2323

24+
echo "specify FWs version..."
25+
source /neural-compressor/.azure-pipelines/scripts/fwk_version.sh 'latest'
2426
FRAMEWORK="tensorflow"
25-
FRAMEWORK_VERSION="2.13.0"
27+
FRAMEWORK_VERSION=${tensorflow_version}
2628

2729
inc_new_api=false
2830
# ======== set up config for tensorflow models ========

‎.azure-pipelines/scripts/ut/env_setup.sh‎

Lines changed: 17 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -48,19 +48,15 @@ if [[ "${torchvision_version}" != "" ]]; then
4848
pip install torchvision==${torchvision_version} -f https://download.pytorch.org/whl/torch_stable.html
4949
fi
5050

51-
if [[ "${ipex_version}" == "1.13.0+cpu" ]]; then
52-
ipex_whl="https://github.com/intel/intel-extension-for-pytorch/releases/download/v1.13.0%2Bcpu/intel_extension_for_pytorch-1.13.0-cp310-cp310-manylinux2014_x86_64.whl"
53-
pip install $ipex_whl
54-
elif [[ "${ipex_version}" == "2.0.0+cpu" ]]; then
51+
if [[ "${ipex_version}" == "2.0.0+cpu" ]]; then
5552
ipex_whl="https://intel-extension-for-pytorch.s3.amazonaws.com/ipex_stable/cpu/intel_extension_for_pytorch-2.0.0%2Bcpu-cp310-cp310-linux_x86_64.whl"
5653
pip install $ipex_whl
5754
elif [[ "${ipex_version}" == "2.0.1+cpu" ]]; then
5855
ipex_whl="https://intel-extension-for-pytorch.s3.amazonaws.com/ipex_stable/cpu/intel_extension_for_pytorch-2.0.100%2Bcpu-cp310-cp310-linux_x86_64.whl"
5956
pip install $ipex_whl
60-
elif [[ "${ipex_version}" == "2.1.0" ]]; then
61-
pip install /tf_dataset/pt_binary/ww32/torch-*.whl
62-
pip install /tf_dataset/pt_binary/ww32/torchvision-*.whl
63-
pip install /tf_dataset/pt_binary/ww32/intel_extension_for_pytorch-*.whl
57+
elif [[ "${ipex_version}" == "2.1.0+cpu" ]]; then
58+
ipex_whl="https://intel-extension-for-pytorch.s3.amazonaws.com/ipex_stable/cpu/intel_extension_for_pytorch-2.1.0%2Bcpu-cp310-cp310-linux_x86_64.whl"
59+
pip install $ipex_whl
6460
fi
6561

6662
if [[ "${onnx_version}" != "" ]]; then
@@ -88,7 +84,6 @@ fi
8884
# install special test env requirements
8985
# common deps
9086
pip install cmake
91-
pip install horovod
9287
pip install transformers
9388

9489
if [[ $(echo "${test_case}" | grep -c "others") != 0 ]];then
@@ -97,6 +92,10 @@ elif [[ $(echo "${test_case}" | grep -c "nas") != 0 ]]; then
9792
pip install dynast==1.6.0rc1
9893
elif [[ $(echo "${test_case}" | grep -c "tf pruning") != 0 ]]; then
9994
pip install tensorflow-addons
95+
# Workaround
96+
# horovod can't be install in the env with TF and PT together
97+
# so test distribute cases in the env with single fw installed
98+
pip install horovod
10099
fi
101100
# test deps
102101
pip install coverage
@@ -106,3 +105,12 @@ pip list
106105
echo "[DEBUG] list pipdeptree..."
107106
pip install pipdeptree
108107
pipdeptree
108+
109+
# import torch before import tensorflow
110+
if [[ $(echo "${test_case}" | grep -c "run basic api") != 0 ]] || [[ $(echo "${test_case}" | grep -c "run basic others") != 0 ]] || [[ $(echo "${test_case}" | grep -c "run basic adaptor") != 0 ]]; then
111+
cd /neural-compressor/test || exit 1
112+
find . -name "test*.py" | xargs sed -i 's/import tensorflow as tf/import torch; import tensorflow as tf/g'
113+
find . -name "test*.py" | xargs sed -i 's/import tensorflow.compat.v1 as tf/import torch; import tensorflow.compat.v1 as tf/g'
114+
find . -name "test*.py" | xargs sed -i 's/from tensorflow import keras/import torch; from tensorflow import keras/g'
115+
fi
116+

‎.azure-pipelines/scripts/ut/run_basic_adaptor.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ test_case="run basic adaptor"
44
echo "${test_case}"
55

66
echo "specify fwk version..."
7-
source /neural-compressor/.azure-pipelines/scripts/ut/ut_fwk_version.sh $1
7+
source /neural-compressor/.azure-pipelines/scripts/fwk_version.sh $1
88

99
echo "set up UT env..."
1010
bash /neural-compressor/.azure-pipelines/scripts/ut/env_setup.sh "${test_case}"

‎.azure-pipelines/scripts/ut/run_basic_adaptor_tfnewapi.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#!/bin/bash
22
python -c "import neural_compressor as nc;print(nc.version.__version__)"
3-
test_case="run basic adaptor tfnewapi"
3+
test_case="run basic tfnewapi"
44
echo "${test_case}"
55

66
echo "specify fwk version..."

‎.azure-pipelines/scripts/ut/run_basic_api.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
#!/bin/bash
22
python -c "import neural_compressor as nc;print(nc.version.__version__)"
3-
test_case="run basic quantization/benchmark/export/mixed_precision/distillation/scheduler/nas"
3+
test_case="run basic api quantization/benchmark/export/mixed_precision/distillation/scheduler/nas"
44
echo "${test_case}"
55

66
echo "specify fwk version..."
7-
source /neural-compressor/.azure-pipelines/scripts/ut/ut_fwk_version.sh $1
7+
source /neural-compressor/.azure-pipelines/scripts/fwk_version.sh $1
88

99
echo "set up UT env..."
1010
bash /neural-compressor/.azure-pipelines/scripts/ut/env_setup.sh "${test_case}"

0 commit comments

Comments
 (0)