Decouple pulling legacy checkpoints from existing GHA workflows and docker files (#13185)

* Add pull-legacy-checkpoints action
* Replace pulls with the new action and script
* Simplify

Author: akihironitta

.actions/pull_legacy_checkpoints.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+# Run this script from the project root.
+URL="https://pl-public-data.s3.amazonaws.com/legacy/checkpoints.zip"
+mkdir -p legacy
+# wget is simpler but does not work on Windows
+python -c "from urllib.request import urlretrieve; urlretrieve('$URL', 'legacy/checkpoints.zip')"
+ls -l legacy/
+unzip -o legacy/checkpoints.zip -d legacy/
+ls -l legacy/checkpoints/

.azure-pipelines/gpu-tests.yml

@@ -69,10 +69,7 @@ jobs:
         python requirements/check-avail-extras.py
       displayName: 'Env details'
 
-    - bash: |
-        wget https://pl-public-data.s3.amazonaws.com/legacy/checkpoints.zip -P legacy/
-        unzip -o legacy/checkpoints.zip -d legacy/
-        ls -l legacy/checkpoints/
+    - bash: bash .actions/pull_legacy_checkpoints.sh
       displayName: 'Get legacy checkpoints'
 
     - bash: |

.github/workflows/ci_test-conda.yml

@@ -55,13 +55,8 @@ jobs:
         # sanity check
         python requirements/check-avail-extras.py
 
-    - name: Pull checkpoints from S3
-      working-directory: ./legacy
-      run: |
-        # enter legacy and update checkpoints from S3
-        curl https://pl-public-data.s3.amazonaws.com/legacy/checkpoints.zip --output checkpoints.zip
-        unzip -o checkpoints.zip
-        ls -l checkpoints/
+    - name: Pull legacy checkpoints
+      run: bash .actions/pull_legacy_checkpoints.sh
 
     - name: Tests
       run: |

.github/workflows/ci_test-full.yml

@@ -76,14 +76,8 @@ jobs:
         restore-keys: |
           ${{ runner.os }}-pip-td${{ env.TIME_PERIOD }}-py${{ matrix.python-version }}-${{ matrix.release }}-${{ matrix.requires }}-
 
-    - name: Pull checkpoints from S3
-      working-directory: ./legacy
-      run: |
-        # wget is simpler but does not work on Windows
-        python -c "from urllib.request import urlretrieve ; urlretrieve('https://pl-public-data.s3.amazonaws.com/legacy/checkpoints.zip', 'checkpoints.zip')"
-        ls -l .
-        unzip -o checkpoints.zip
-        ls -l checkpoints/
+    - name: Pull legacy checkpoints
+      run: bash .actions/pull_legacy_checkpoints.sh
 
     - name: Install dependencies
       run: |

dockers/tpu-tests/Dockerfile

@@ -22,10 +22,7 @@ LABEL maintainer="PyTorchLightning <https://github.com/PyTorchLightning>"
 COPY ./ ./pytorch-lightning/
 
 # Pull the legacy checkpoints
-RUN cd pytorch-lightning && \
-    wget https://pl-public-data.s3.amazonaws.com/legacy/checkpoints.zip -P legacy/ && \
-    unzip -o legacy/checkpoints.zip -d legacy/ && \
-    ls -l legacy/checkpoints/
+RUN cd pytorch-lightning && bash .actions/pull_legacy_checkpoints.sh
 
 RUN \
     pip install -q fire && \

legacy/README.md

@@ -7,8 +7,7 @@ At this moment we focus on ability running old checkpoints, so the flow here is
 If you want to pull all saved version-checkpoints for local testing/development, call
 
 ```bash
-wget https://pl-public-data.s3.amazonaws.com/legacy/checkpoints.zip
-unzip -o checkpoints.zip
+bash .actions/pull_legacy_checkpoints.sh
 ```
 
 To back populate collection with past version you can use following bash:

tests/README.md

@@ -22,8 +22,7 @@ pre-commit install
 Additionally, for testing backward compatibility with older versions of PyTorch Lightning, you also need to download all saved version-checkpoints from the public AWS storage. Run the following script to get all saved version-checkpoints:
 
 ```bash
-wget https://pl-public-data.s3.amazonaws.com/legacy/checkpoints.zip -P legacy/
-unzip -o legacy/checkpoints.zip -d legacy/
+bash .actions/pull_legacy_checkpoints.sh
 ```
 
 Note: These checkpoints are generated to set baselines for maintaining backward compatibility with legacy versions of PyTorch Lightning. Details of checkpoints for back-compatibility can be found [here](https://github.com/PyTorchLightning/pytorch-lightning/blob/master/legacy/README.md).