Skip to content

Commit 3b5934d

Browse files
Jeff YangBordas-rog
committed
docker: run ci only docker related files are changed (#5203)
* only run ci on docker related files * docker related files changed! * install pytorch along with cudatoolkit * build docker only on SUN * conda exit status has been fixed * reverts back to old conda version * add more docker related files * conda env update --name * create env and install pytorch again * create env and install pytorch again * ${PYTORCH_CHANNEL} * dont update pytorch with conda env update * Apply suggestions from code review Co-authored-by: Jirka Borovec <[email protected]> * Update dockers/base-conda/Dockerfile * Apply suggestions from code review * remove checks in cron job * Apply suggestions from code review * readd # * readd # Co-authored-by: Jirka Borovec <[email protected]> Co-authored-by: Roger Shieh <[email protected]> (cherry picked from commit cc62435)
1 parent 6eb4508 commit 3b5934d

File tree

3 files changed

+51
-51
lines changed

3 files changed

+51
-51
lines changed

‎.github/workflows/ci_dockers.yml‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,21 @@ name: CI build Docker
22
# https://www.docker.com/blog/first-docker-github-action-is-here
33
# https://github.com/docker/build-push-action
44
# see: https://help.github.com/en/actions/reference/events-that-trigger-workflows
5-
on: # Trigger the workflow on push or pull request, but only for the master branch
5+
on: # Trigger the workflow on push or pull request, but only for the master branch
66
push:
7-
branches: [master, "release/*"] # include release branches like release/1.0.x
7+
branches: [master, "release/*"] # include release branches like release/1.0.x
88
pull_request:
99
branches: [master, "release/*"]
10+
paths:
11+
- "dockers/**"
12+
- "!dockers/README.md"
13+
- "requirements/*.txt"
14+
- "environment.yml"
15+
- "requirements.txt"
16+
- ".github/workflows/ci_dockers.yml"
17+
- ".github/workflows/nightly.yml"
18+
- ".github/workflows/release-docker.yml"
19+
- "setup.py"
1020

1121
jobs:
1222
build-PL:
@@ -55,7 +65,6 @@ jobs:
5565
build-args: |
5666
PYTHON_VERSION=${{ matrix.python_version }}
5767
XLA_VERSION=${{ matrix.xla_version }}
58-
cache-from: pytorchlightning/pytorch_lightning:base-xla-py${{ matrix.python_version }}-torch${{ matrix.xla_version }}
5968
file: dockers/base-xla/Dockerfile
6069
push: false
6170
timeout-minutes: 50
@@ -96,7 +105,6 @@ jobs:
96105
PYTHON_VERSION=${{ matrix.python_version }}
97106
PYTORCH_VERSION=${{ matrix.pytorch_version }}
98107
CUDA_VERSION=${{ steps.extend.outputs.CUDA }}
99-
cache-from: pytorchlightning/pytorch_lightning:base-cuda-py${{ matrix.python_version }}-torch${{ matrix.pytorch_version }}
100108
file: dockers/base-cuda/Dockerfile
101109
push: false
102110
timeout-minutes: 50
@@ -139,7 +147,6 @@ jobs:
139147
PYTORCH_VERSION=${{ matrix.pytorch_version }}
140148
PYTORCH_CHANNEL=${{ steps.extend.outputs.CHANNEL }}
141149
CUDA_VERSION=${{ steps.extend.outputs.CUDA }}
142-
cache-from: pytorchlightning/pytorch_lightning:base-conda-py${{ matrix.python_version }}-torch${{ matrix.pytorch_version }}
143150
file: dockers/base-conda/Dockerfile
144151
push: false
145152
timeout-minutes: 50

‎.github/workflows/nightly.yml‎

Lines changed: 34 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -1,57 +1,56 @@
11
name: Nightly events
22

33
# https://jasonet.co/posts/scheduled-actions/
4+
# https://github.202132.xyzmunity/t/distinct-job-for-each-schedule/17811/2
45
on:
56
schedule:
6-
# At the end of every day
7-
- cron: "0 0 * * *"
7+
- cron: "0 0 * * *" # At the end of every day
88

99
# based on https://github.com/pypa/gh-action-pypi-publish
1010
jobs:
11-
1211
pypi-release:
1312
runs-on: ubuntu-20.04
1413

1514
steps:
16-
# does nightly releases from feature branch
17-
- uses: actions/checkout@v2
18-
with:
19-
ref: release/1.2-dev
20-
- uses: actions/setup-python@v2
21-
with:
22-
python-version: 3.7
23-
24-
- name: Install dependencies
25-
run: >-
26-
python -m pip install --user --upgrade setuptools wheel
27-
28-
- name: Build packages
29-
run: |
30-
python .github/prepare-nightly_version.py
31-
python setup.py sdist bdist_wheel
32-
ls -lh dist/
33-
34-
- name: Delay releasing
35-
uses: juliangruber/sleep-action@v1
36-
with:
37-
time: 5m
38-
39-
# We do this, since failures on test.pypi aren't that bad
40-
- name: Publish to Test PyPI
41-
uses: pypa/[email protected]
42-
with:
43-
user: __token__
44-
password: ${{ secrets.test_pypi_password }}
45-
repository_url: https://test.pypi.org/legacy/
46-
verbose: true
15+
# does nightly releases from feature branch
16+
- uses: actions/checkout@v2
17+
with:
18+
ref: release/1.2-dev
19+
- uses: actions/setup-python@v2
20+
with:
21+
python-version: 3.7
22+
23+
- name: Install dependencies
24+
run: >-
25+
python -m pip install --user --upgrade setuptools wheel
26+
27+
- name: Build packages
28+
run: |
29+
python .github/prepare-nightly_version.py
30+
python setup.py sdist bdist_wheel
31+
ls -lh dist/
32+
33+
- name: Delay releasing
34+
uses: juliangruber/sleep-action@v1
35+
with:
36+
time: 5m
37+
38+
# We do this, since failures on test.pypi aren't that bad
39+
- name: Publish to Test PyPI
40+
uses: pypa/[email protected]
41+
with:
42+
user: __token__
43+
password: ${{ secrets.test_pypi_password }}
44+
repository_url: https://test.pypi.org/legacy/
45+
verbose: true
4746

4847
docker-XLA:
4948
runs-on: ubuntu-20.04
5049
strategy:
5150
fail-fast: false
5251
matrix:
5352
python_version: [3.6, 3.7]
54-
xla_version: [1.6, 1.7] # todo: , "nightly"
53+
xla_version: [1.6, 1.7] # todo: , "nightly"
5554
steps:
5655
- name: Checkout
5756
uses: actions/checkout@v2
@@ -72,8 +71,6 @@ jobs:
7271
build-args: |
7372
PYTHON_VERSION=${{ matrix.python_version }}
7473
XLA_VERSION=${{ matrix.xla_version }}
75-
cache-from: pytorchlightning/pytorch_lightning:base-xla-py${{ matrix.python_version }}-torch${{ matrix.xla_version }}
76-
cache-to: type=inline
7774
file: dockers/base-xla/Dockerfile
7875
push: true
7976
tags: pytorchlightning/pytorch_lightning:base-xla-py${{ matrix.python_version }}-torch${{ matrix.xla_version }}
@@ -118,8 +115,6 @@ jobs:
118115
PYTHON_VERSION=${{ matrix.python_version }}
119116
PYTORCH_VERSION=${{ matrix.pytorch_version }}
120117
CUDA_VERSION=${{ steps.extend.outputs.CUDA }}
121-
cache-from: pytorchlightning/pytorch_lightning:base-cuda-py${{ matrix.python_version }}-torch${{ matrix.pytorch_version }}
122-
cache-to: type=inline
123118
file: dockers/base-cuda/Dockerfile
124119
push: true
125120
tags: pytorchlightning/pytorch_lightning:base-cuda-py${{ matrix.python_version }}-torch${{ matrix.pytorch_version }}
@@ -134,8 +129,6 @@ jobs:
134129
PYTORCH_VERSION=${{ matrix.pytorch_version }}
135130
PYTORCH_CHANNEL=${{ steps.extend.outputs.CHANNEL }}
136131
CUDA_VERSION=${{ steps.extend.outputs.CUDA }}
137-
cache-from: pytorchlightning/pytorch_lightning:base-conda-py${{ matrix.python_version }}-torch${{ matrix.pytorch_version }}
138-
cache-to: type=inline
139132
file: dockers/base-conda/Dockerfile
140133
push: true
141134
tags: pytorchlightning/pytorch_lightning:base-conda-py${{ matrix.python_version }}-torch${{ matrix.pytorch_version }}

‎dockers/base-conda/Dockerfile‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -74,16 +74,16 @@ ENV CONDA_ENV=lightning
7474
COPY environment.yml environment.yml
7575

7676
# conda init
77-
RUN conda create -y --name $CONDA_ENV cudatoolkit=${CUDA_VERSION} && \
77+
RUN conda create -y --name $CONDA_ENV python=${PYTHON_VERSION} pytorch=${PYTORCH_VERSION} cudatoolkit=${CUDA_VERSION} -c ${PYTORCH_CHANNEL} && \
7878
conda init bash && \
7979
# NOTE: this requires that the channel is presented in the yaml before packages
80-
# replace channel to nigtly if needed, fix PT version and remove Horovod as it will be installe later
80+
# replace channel to nigtly if needed, fix PT version and remove Horovod as it will be installed later
8181
python -c "fname = 'environment.yml' ; req = open(fname).read().replace('pytorch', '${PYTORCH_CHANNEL}', 1) ; open(fname, 'w').write(req)" && \
82-
python -c "import re ; fname = 'environment.yml' ; req = re.sub(r'python[>=]+[\d\.]+', 'python=${PYTHON_VERSION}', open(fname).read()) ; open(fname, 'w').write(req)" && \
83-
python -c "import re ; fname = 'environment.yml' ; req = re.sub(r'torch[>=]+[\d\.]+', 'torch=${PYTORCH_VERSION}', open(fname).read()) ; open(fname, 'w').write(req)" && \
82+
python -c "import re ; fname = 'environment.yml' ; req = re.sub(r'- python[>=]+[\d\.]+', '# - python=${PYTHON_VERSION}', open(fname).read()) ; open(fname, 'w').write(req)" && \
83+
python -c "import re ; fname = 'environment.yml' ; req = re.sub(r'- pytorch[>=]+[\d\.]+', '# - pytorch=${PYTORCH_VERSION}', open(fname).read()) ; open(fname, 'w').write(req)" && \
8484
python -c "fname = 'environment.yml' ; req = open(fname).readlines() ; open(fname, 'w').writelines([ln for ln in req if 'horovod' not in ln])" && \
8585
cat environment.yml && \
86-
conda env update --file environment.yml && \
86+
conda env update --name $CONDA_ENV --file environment.yml && \
8787
conda clean -ya && \
8888
rm environment.yml
8989

0 commit comments

Comments
 (0)