Skip to content

Commit 31f1607

Browse files
rohitgr7rohitgr7
authored
Merge branch 'master' into fix/restore_step
2 parents 6ab0608 + 2845e75 commit 31f1607

File tree

266 files changed

+8628
-2741
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

266 files changed

+8628
-2741
lines changed

‎.actions/assistant.py‎

Lines changed: 7 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
from urllib.request import Request, urlopen
1616

1717
import fire
18+
import pkg_resources
1819

1920
REQUIREMENT_FILES = {
2021
"pytorch": (
@@ -78,17 +79,12 @@ def requirements_prune_pkgs(packages: Sequence[str], req_files: Sequence[str] =
7879
@staticmethod
7980
def _prune_packages(req_file: str, packages: Sequence[str]) -> None:
8081
"""Remove some packages from given requirement files."""
81-
with open(req_file) as fp:
82-
lines = fp.readlines()
83-
84-
if isinstance(packages, str):
85-
packages = [packages]
86-
for pkg in packages:
87-
lines = [ln for ln in lines if not ln.startswith(pkg)]
88-
pprint(lines)
89-
90-
with open(req_file, "w") as fp:
91-
fp.writelines(lines)
82+
path = Path(req_file)
83+
assert path.exists()
84+
text = path.read_text()
85+
final = [str(req) for req in pkg_resources.parse_requirements(text) if req.name not in packages]
86+
pprint(final)
87+
path.write_text("\n".join(final))
9288

9389
@staticmethod
9490
def _replace_min(fname: str) -> None:

‎.azure/gpu-benchmark.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,5 +46,6 @@ jobs:
4646
- bash: python -m pytest benchmarks -v --durations=0
4747
env:
4848
PL_RUNNING_BENCHMARKS: 1
49+
PL_RUN_CUDA_TESTS: "1"
4950
workingDirectory: tests/tests_pytorch
5051
displayName: 'Testing: PyTorch benchmarks'

‎.azure/gpu-tests.yml‎

Lines changed: 40 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,18 +12,23 @@ trigger:
1212
- "master"
1313
- "release/*"
1414
- "refs/tags/*"
15+
1516
pr:
1617
- "master"
1718
- "release/*"
1819

20+
variables:
21+
- name: continue
22+
value: '1'
23+
1924
jobs:
2025
- job: testing
2126
strategy:
2227
matrix:
2328
'PyTorch - stable':
2429
image: "pytorchlightning/pytorch_lightning:base-cuda-py3.9-torch1.11"
2530
# how long to run the job before automatically cancelling
26-
timeoutInMinutes: "100"
31+
timeoutInMinutes: "80"
2732
# how much time to give 'run always even if cancelled tasks' before stopping them
2833
cancelTimeoutInMinutes: "2"
2934
pool: azure-jirka-spot
@@ -37,6 +42,21 @@ jobs:
3742

3843
steps:
3944

45+
- bash: |
46+
CHANGED_FILES=$(git diff --name-status origin/master -- . | awk '{print $2}')
47+
FILTER='src/pytorch_lightning|requirements/pytorch|tests/tests_pytorch|examples/pl_*'
48+
echo $CHANGED_FILES > changed_files.txt
49+
MATCHES=$(cat changed_files.txt | grep -E $FILTER)
50+
echo $MATCHES
51+
if [ -z "$MATCHES" ]; then
52+
echo "Skip"
53+
echo "##vso[task.setvariable variable=continue]0"
54+
else
55+
echo "Continue"
56+
echo "##vso[task.setvariable variable=continue]1"
57+
fi
58+
displayName: Skipper
59+
4060
- bash: |
4161
lspci | egrep 'VGA|3D'
4262
whereis nvidia
@@ -46,6 +66,7 @@ jobs:
4666
pip --version
4767
pip list
4868
displayName: 'Image info & NVIDIA'
69+
condition: eq(variables['continue'], '1')
4970
5071
- bash: |
5172
python -c "fname = 'requirements/pytorch/strategies.txt' ; lines = [line for line in open(fname).readlines() if 'horovod' not in line] ; open(fname, 'w').writelines(lines)"
@@ -58,6 +79,7 @@ jobs:
5879
PACKAGE_NAME: pytorch
5980
FREEZE_REQUIREMENTS: 1
6081
displayName: 'Install dependencies'
82+
condition: eq(variables['continue'], '1')
6183
6284
- bash: |
6385
set -e
@@ -66,23 +88,33 @@ jobs:
6688
python requirements/pytorch/check-avail-strategies.py
6789
python requirements/pytorch/check-avail-extras.py
6890
displayName: 'Env details'
91+
condition: eq(variables['continue'], '1')
6992
7093
- bash: bash .actions/pull_legacy_checkpoints.sh
7194
displayName: 'Get legacy checkpoints'
95+
condition: eq(variables['continue'], '1')
7296

7397
- bash: python -m coverage run --source pytorch_lightning -m pytest
7498
workingDirectory: src/pytorch_lightning
7599
displayName: 'Testing: PyTorch doctests'
100+
condition: eq(variables['continue'], '1')
76101

77102
- bash: python -m coverage run --source pytorch_lightning -m pytest --ignore benchmarks -v --junitxml=$(Build.StagingDirectory)/test-results.xml --durations=50
78-
displayName: 'Testing: PyTorch standard'
103+
env:
104+
PL_RUN_CUDA_TESTS: "1"
79105
workingDirectory: tests/tests_pytorch
106+
displayName: 'Testing: PyTorch standard'
107+
timeoutInMinutes: "35"
108+
condition: eq(variables['continue'], '1')
80109

81110
- bash: bash run_standalone_tests.sh
82111
workingDirectory: tests/tests_pytorch
83112
env:
84113
PL_USE_MOCKED_MNIST: "1"
114+
PL_RUN_CUDA_TESTS: "1"
85115
displayName: 'Testing: PyTorch standalone tests'
116+
timeoutInMinutes: "35"
117+
condition: eq(variables['continue'], '1')
86118

87119
- bash: |
88120
python -m coverage report
@@ -92,13 +124,14 @@ jobs:
92124
ls -l
93125
workingDirectory: tests/tests_pytorch
94126
displayName: 'Statistics'
127+
condition: eq(variables['continue'], '1')
95128
96129
- task: PublishTestResults@2
97130
displayName: 'Publish test results'
98131
inputs:
99132
testResultsFiles: '$(Build.StagingDirectory)/test-results.xml'
100133
testRunTitle: '$(Agent.OS) - $(Build.DefinitionName) - Python $(python.version)'
101-
condition: succeededOrFailed()
134+
condition: and(succeededOrFailed(), eq(variables['continue'], '1'))
102135

103136
- script: |
104137
set -e
@@ -110,7 +143,11 @@ jobs:
110143
env:
111144
PL_USE_MOCKED_MNIST: "1"
112145
displayName: 'Testing: PyTorch examples'
146+
condition: eq(variables['continue'], '1')
113147
114148
- bash: python -m pytest benchmarks -v --maxfail=2 --durations=0
115149
workingDirectory: tests/tests_pytorch
150+
env:
151+
PL_RUN_CUDA_TESTS: "1"
116152
displayName: 'Testing: PyTorch benchmarks'
153+
condition: eq(variables['continue'], '1')

‎.azure/hpu-tests.yml‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,15 @@ trigger:
99
- "master"
1010
- "release/*"
1111
- "refs/tags/*"
12+
1213
pr:
1314
- "master"
1415
- "release/*"
1516

17+
variables:
18+
- name: continue
19+
value: '1'
20+
1621
jobs:
1722
- job: testing
1823
# how long to run the job before automatically cancelling
@@ -24,27 +29,46 @@ jobs:
2429
clean: all
2530

2631
steps:
32+
- bash: |
33+
CHANGED_FILES=$(git diff --name-status origin/master -- . | awk '{print $2}')
34+
FILTER='src/pytorch_lightning|requirements/pytorch|tests/tests_pytorch|examples/pl_*'
35+
echo $CHANGED_FILES > changed_files.txt
36+
MATCHES=$(cat changed_files.txt | grep -E $FILTER)
37+
echo $MATCHES
38+
if [ -z "$MATCHES" ]; then
39+
echo "Skip"
40+
echo "##vso[task.setvariable variable=continue]0"
41+
else
42+
echo "Continue"
43+
echo "##vso[task.setvariable variable=continue]1"
44+
fi
45+
displayName: Skipper
46+
2747
- bash: |
2848
apt-get install -y hwinfo
2949
hwinfo --short
3050
displayName: 'Instance HW info'
51+
condition: eq(variables['continue'], '1')
3152
3253
- bash: |
3354
pip install -e .[extra] -r requirements/pytorch/test.txt
3455
env:
3556
PACKAGE_NAME: pytorch
3657
FREEZE_REQUIREMENTS: 1
3758
displayName: 'Install dependencies'
59+
condition: eq(variables['continue'], '1')
3860
3961
- bash: |
4062
python -m pytest -sv accelerators/test_hpu.py --forked --junitxml=hpu1_test-results.xml
4163
workingDirectory: tests/tests_pytorch
4264
displayName: 'Single card HPU test'
65+
condition: eq(variables['continue'], '1')
4366
4467
- bash: |
4568
python -m pytest -sv accelerators/test_hpu.py --forked --hpus 8 --junitxml=hpu8_test-results.xml
4669
workingDirectory: tests/tests_pytorch
4770
displayName: 'Multi card(8) HPU test'
71+
condition: eq(variables['continue'], '1')
4872
4973
- bash: |
5074
python -m pytest -sv plugins/precision/hpu/test_hpu.py --hmp-bf16 \
@@ -53,16 +77,18 @@ jobs:
5377
--junitxml=hpu1_precision_test-results.xml
5478
workingDirectory: tests/tests_pytorch
5579
displayName: 'HPU precision test'
80+
condition: eq(variables['continue'], '1')
5681
5782
- bash: |
5883
export PYTHONPATH="${PYTHONPATH}:$(pwd)"
5984
python "pl_hpu/mnist_sample.py"
6085
workingDirectory: examples
6186
displayName: 'Testing: HPU examples'
87+
condition: eq(variables['continue'], '1')
6288
6389
- task: PublishTestResults@2
6490
inputs:
6591
testResultsFiles: 'hpu*_test-results.xml'
6692
testRunTitle: '$(Agent.OS) - $(Build.DefinitionName) - Python $(python.version)'
67-
condition: succeededOrFailed()
93+
condition: and(succeededOrFailed(), eq(variables['continue'], '1'))
6894
displayName: 'Publish test results'

‎.azure/ipu-tests.yml‎

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,13 +7,16 @@ trigger:
77
- master
88
- release/*
99
- refs/tags/*
10+
1011
pr:
1112
- master
1213
- release/*
1314

1415
variables:
15-
- name: poplar_sdk
16-
value: "poplar_sdk-ubuntu_20_04-2.3.1+793-89796d462d"
16+
- name: poplar_sdk
17+
value: "poplar_sdk-ubuntu_20_04-2.3.1+793-89796d462d"
18+
- name: continue
19+
value: '1'
1720

1821
jobs:
1922
- job: testing
@@ -24,14 +27,31 @@ jobs:
2427
clean: all
2528

2629
steps:
30+
- bash: |
31+
CHANGED_FILES=$(git diff --name-status origin/master -- . | awk '{print $2}')
32+
FILTER='src/pytorch_lightning|requirements/pytorch|tests/tests_pytorch|examples/pl_*'
33+
echo $CHANGED_FILES > changed_files.txt
34+
MATCHES=$(cat changed_files.txt | grep -E $FILTER)
35+
echo $MATCHES
36+
if [ -z "$MATCHES" ]; then
37+
echo "Skip"
38+
echo "##vso[task.setvariable variable=continue]0"
39+
else
40+
echo "Continue"
41+
echo "##vso[task.setvariable variable=continue]1"
42+
fi
43+
displayName: Skipper
44+
2745
- script: tar -xvzf /opt/poplar/${{ variables.poplar_sdk }}.tar.gz
2846
displayName: "Extract Poplar SDK"
47+
condition: eq(variables['continue'], '1')
2948

3049
- script: |
3150
set -eux
3251
pip debug --verbose
3352
pip install ${{ variables.poplar_sdk }}/poptorch-*ubuntu*.whl
3453
displayName: "Install poptorch"
54+
condition: eq(variables['continue'], '1')
3555
3656
- script: |
3757
set -eux
@@ -48,6 +68,7 @@ jobs:
4868
i=$((i + 1))
4969
done
5070
displayName: "Reset IPU devices"
71+
condition: eq(variables['continue'], '1')
5172
5273
- bash: |
5374
export GIT_TERMINAL_PROMPT=1
@@ -59,18 +80,21 @@ jobs:
5980
PACKAGE_NAME: pytorch
6081
FREEZE_REQUIREMENTS: 1
6182
displayName: 'Install dependencies'
83+
condition: eq(variables['continue'], '1')
6284
6385
- bash: |
6486
python requirements/collect_env_details.py
6587
python -c "import torch"
6688
displayName: 'Env details'
89+
condition: eq(variables['continue'], '1')
6790
6891
- script: |
6992
set -eux
7093
source ${{ variables.poplar_sdk }}/poplar-ubuntu*/enable.sh
7194
source ${{ variables.poplar_sdk }}/popart-ubuntu*/enable.sh
7295
python -c "import poptorch; print(poptorch.__version__)"
7396
displayName: "Check poptorch installation"
97+
condition: eq(variables['continue'], '1')
7498
7599
- bash: |
76100
source ${{ variables.poplar_sdk }}/poplar-ubuntu*/enable.sh
@@ -81,3 +105,4 @@ jobs:
81105
POPTORCH_WAIT_FOR_IPU: 1
82106
PL_RUN_IPU_TESTS: 1
83107
displayName: 'Testing: PyTorch standard'
108+
condition: eq(variables['continue'], '1')

‎.circleci/config.yml‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ trigger:
1414
- "master"
1515
- "release/*"
1616
- "refs/tags/*"
17+
1718
pr:
1819
- "master"
1920
- "release/*"
@@ -157,9 +158,9 @@ jobs:
157158

158159
workflows:
159160
version: 2
160-
build-docs:
161-
jobs:
162-
- build-Docs
161+
#build-docs: # FixMe
162+
# jobs:
163+
# - build-Docs
163164
test-on-tpus:
164165
jobs:
165166
- TPU-tests

‎.github/gatekeeper-config_app.yml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
approvals:
2+
# check will fail if there is no approval
3+
minimum: 1
4+
groups:
5+
- name: 'Lightning Apps'
6+
from:
7+
- alecmerdler
8+
- awaelchli
9+
- hhsecond
10+
- lantiga
11+
- manskx
12+
- nohalon
13+
- tchaton

‎.github/approve_config.yml‎ renamed to ‎.github/gatekeeper-config_pytorch.yml‎

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ approvals:
33
minimum: 1
44
groups:
55
- name: 'PyTorch Lightning'
6-
minimum: 1
76
from:
87
- awaelchli
98
- Borda
@@ -15,13 +14,3 @@ approvals:
1514
- rohitgr7
1615
- tchaton
1716
- williamFalcon
18-
- name: 'Lightning Apps'
19-
minimum: 1
20-
from:
21-
- alecmerdler
22-
- awaelchli
23-
- hhsecond
24-
- lantiga
25-
- manskx
26-
- nohalon
27-
- tchaton

0 commit comments

Comments
 (0)