From b62ccf77d53adaf4398487ec09ada4dc820a1dc1 Mon Sep 17 00:00:00 2001 From: peter Date: Fri, 15 May 2020 14:17:47 +0800 Subject: [PATCH 1/3] Add windows binary jobs --- .circleci/config.yml | 141 +++++++++++++++++++++++++++++++++++++ .circleci/config.yml.in | 39 ++++++++++ .circleci/regenerate.py | 2 +- packaging/build_wheel.sh | 4 +- packaging/pkg_helpers.bash | 2 +- 5 files changed, 185 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a6241009a2..9abac70f52 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -132,6 +132,45 @@ jobs: paths: - "*" + binary_windows_wheel: + <<: *binary_common + executor: + name: windows-cpu + steps: + - checkout + - run: + name: build + command: | + eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" + conda activate base + bash packaging/build_wheel.sh + - store_artifacts: + path: dist + - persist_to_workspace: + root: dist + paths: + - "*" + + binary_windows_conda: + <<: *binary_common + executor: + name: windows-cpu + steps: + - checkout + - run: + name: build + command: | + eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" + conda activate base + conda install -yq conda-build "conda-package-handling!=1.5.0" + bash packaging/build_conda.sh + - store_artifacts: + path: /C/tools/miniconda3/conda-bld/win-64 + - persist_to_workspace: + root: /C/tools/miniconda3/conda-bld/win-64 + paths: + - "*" + # Requires org-member context binary_conda_upload: docker: @@ -403,6 +442,15 @@ workflows: - binary_macos_wheel: name: binary_macos_wheel_py3.8 python_version: '3.8' + - binary_windows_wheel: + name: binary_windows_wheel_py3.6 + python_version: '3.6' + - binary_windows_wheel: + name: binary_windows_wheel_py3.7 + python_version: '3.7' + - binary_windows_wheel: + name: binary_windows_wheel_py3.8 + python_version: '3.8' - binary_linux_conda: name: binary_linux_conda_py3.6 python_version: '3.6' @@ -421,6 +469,15 @@ workflows: - binary_macos_conda: name: binary_macos_conda_py3.8 python_version: '3.8' + - binary_windows_conda: + name: binary_windows_conda_py3.6 + python_version: '3.6' + - binary_windows_conda: + name: binary_windows_conda_py3.7 + python_version: '3.7' + - binary_windows_conda: + name: binary_windows_conda_py3.8 + python_version: '3.8' unittest: jobs: - unittest_linux_cpu: @@ -591,6 +648,48 @@ workflows: name: nightly_binary_macos_wheel_py3.8_upload requires: - nightly_binary_macos_wheel_py3.8 + - binary_windows_wheel: + filters: + branches: + only: nightly + name: nightly_binary_windows_wheel_py3.6 + python_version: '3.6' + - binary_wheel_upload: + context: org-member + filters: + branches: + only: nightly + name: nightly_binary_windows_wheel_py3.6_upload + requires: + - nightly_binary_windows_wheel_py3.6 + - binary_windows_wheel: + filters: + branches: + only: nightly + name: nightly_binary_windows_wheel_py3.7 + python_version: '3.7' + - binary_wheel_upload: + context: org-member + filters: + branches: + only: nightly + name: nightly_binary_windows_wheel_py3.7_upload + requires: + - nightly_binary_windows_wheel_py3.7 + - binary_windows_wheel: + filters: + branches: + only: nightly + name: nightly_binary_windows_wheel_py3.8 + python_version: '3.8' + - binary_wheel_upload: + context: org-member + filters: + branches: + only: nightly + name: nightly_binary_windows_wheel_py3.8_upload + requires: + - nightly_binary_windows_wheel_py3.8 - binary_linux_conda: filters: branches: @@ -699,6 +798,48 @@ workflows: name: nightly_binary_macos_conda_py3.8_upload requires: - nightly_binary_macos_conda_py3.8 + - binary_windows_conda: + filters: + branches: + only: nightly + name: nightly_binary_windows_conda_py3.6 + python_version: '3.6' + - binary_conda_upload: + context: org-member + filters: + branches: + only: nightly + name: nightly_binary_windows_conda_py3.6_upload + requires: + - nightly_binary_windows_conda_py3.6 + - binary_windows_conda: + filters: + branches: + only: nightly + name: nightly_binary_windows_conda_py3.7 + python_version: '3.7' + - binary_conda_upload: + context: org-member + filters: + branches: + only: nightly + name: nightly_binary_windows_conda_py3.7_upload + requires: + - nightly_binary_windows_conda_py3.7 + - binary_windows_conda: + filters: + branches: + only: nightly + name: nightly_binary_windows_conda_py3.8 + python_version: '3.8' + - binary_conda_upload: + context: org-member + filters: + branches: + only: nightly + name: nightly_binary_windows_conda_py3.8_upload + requires: + - nightly_binary_windows_conda_py3.8 docker_build: triggers: - schedule: diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index deef1929e5..b96a0e91b1 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -132,6 +132,45 @@ jobs: paths: - "*" + binary_windows_wheel: + <<: *binary_common + executor: + name: windows-cpu + steps: + - checkout + - run: + name: build + command: | + eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" + conda activate base + bash packaging/build_wheel.sh + - store_artifacts: + path: dist + - persist_to_workspace: + root: dist + paths: + - "*" + + binary_windows_conda: + <<: *binary_common + executor: + name: windows-cpu + steps: + - checkout + - run: + name: build + command: | + eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" + conda activate base + conda install -yq conda-build "conda-package-handling!=1.5.0" + bash packaging/build_conda.sh + - store_artifacts: + path: /C/tools/miniconda3/conda-bld/win-64 + - persist_to_workspace: + root: /C/tools/miniconda3/conda-bld/win-64 + paths: + - "*" + # Requires org-member context binary_conda_upload: docker: diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py index 74539db720..c61090a45c 100755 --- a/.circleci/regenerate.py +++ b/.circleci/regenerate.py @@ -25,7 +25,7 @@ def build_workflows(prefix='', upload=False, filter_branch=None, indentation=6): w = [] for btype in ["wheel", "conda"]: - for os_type in ["linux", "macos"]: + for os_type in ["linux", "macos", "windows"]: for python_version in PYTHON_VERSIONS: w += build_workflow_pair(btype, os_type, python_version, filter_branch, prefix, upload) diff --git a/packaging/build_wheel.sh b/packaging/build_wheel.sh index fc0742ece1..eb4e66e648 100755 --- a/packaging/build_wheel.sh +++ b/packaging/build_wheel.sh @@ -8,7 +8,9 @@ export BUILD_TYPE="wheel" export NO_CUDA_PACKAGE=1 setup_env 0.6.0 setup_wheel_python -"$script_dir/build_from_source.sh" "$(pwd)" # Build static dependencies +if [[ "$OSTYPE" != "msys" ]]; then + "$script_dir/build_from_source.sh" "$(pwd)" # Build static dependencies +fi pip_install numpy future setup_pip_pytorch_version python setup.py clean diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash index bc8ec997df..4030128a98 100644 --- a/packaging/pkg_helpers.bash +++ b/packaging/pkg_helpers.bash @@ -122,7 +122,7 @@ retry () { # # Precondition: If Linux, you are in a soumith/manylinux-cuda* Docker image setup_wheel_python() { - if [[ "$(uname)" == Darwin ]]; then + if [[ "$(uname)" == Darwin || "$OSTYPE" == "msys" ]]; then eval "$(conda shell.bash hook)" conda env remove -n "env$PYTHON_VERSION" || true conda create -yn "env$PYTHON_VERSION" python="$PYTHON_VERSION" From d61a37eb97d4e42a33e9666d92fcf3740b98d16c Mon Sep 17 00:00:00 2001 From: peter Date: Fri, 15 May 2020 14:25:49 +0800 Subject: [PATCH 2/3] Fix paths --- .circleci/config.yml | 4 ++-- .circleci/config.yml.in | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9abac70f52..3f6375397a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -165,9 +165,9 @@ jobs: conda install -yq conda-build "conda-package-handling!=1.5.0" bash packaging/build_conda.sh - store_artifacts: - path: /C/tools/miniconda3/conda-bld/win-64 + path: C:/tools/miniconda3/conda-bld/win-64 - persist_to_workspace: - root: /C/tools/miniconda3/conda-bld/win-64 + root: C:/tools/miniconda3/conda-bld/win-64 paths: - "*" diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index b96a0e91b1..284ca38ae9 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -165,9 +165,9 @@ jobs: conda install -yq conda-build "conda-package-handling!=1.5.0" bash packaging/build_conda.sh - store_artifacts: - path: /C/tools/miniconda3/conda-bld/win-64 + path: C:/tools/miniconda3/conda-bld/win-64 - persist_to_workspace: - root: /C/tools/miniconda3/conda-bld/win-64 + root: C:/tools/miniconda3/conda-bld/win-64 paths: - "*" From 94994ecd24a813250dad825918d773a6dd8b9d03 Mon Sep 17 00:00:00 2001 From: peter Date: Sat, 16 May 2020 08:53:00 +0800 Subject: [PATCH 3/3] Make wheel package platform-specific and python-version-specific --- packaging/build_wheel.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packaging/build_wheel.sh b/packaging/build_wheel.sh index eb4e66e648..576cf9f286 100755 --- a/packaging/build_wheel.sh +++ b/packaging/build_wheel.sh @@ -14,4 +14,9 @@ fi pip_install numpy future setup_pip_pytorch_version python setup.py clean -IS_WHEEL=1 python setup.py bdist_wheel +if [[ "$OSTYPE" == "msys" ]]; then + python_tag="$(echo "cp$PYTHON_VERSION" | tr -d '.')" + IS_WHEEL=1 python setup.py bdist_wheel --plat-name win_amd64 --python-tag $python_tag +else + IS_WHEEL=1 python setup.py bdist_wheel +fi