From 7ef90df239d16f8aafdd55567274768f4d5cf9dd Mon Sep 17 00:00:00 2001 From: Ee Durbin Date: Mon, 3 Apr 2023 07:16:38 -0400 Subject: [PATCH 01/14] try running CI with this pip, it *should* use built wheels from cache --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 65567d83794e..b82c2818d98d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -76,5 +76,7 @@ jobs: pip install -r requirements.txt --no-deps pip install -r requirements/dev.txt pip check + - name: Install a specific pip + run: pip install 'pip @ git+https://github.com/sbidoul/pip@49d3354e1c807bafdd715225d2aaa5cfc3532434' - name: Run ${{ matrix.name }} run: ${{ matrix.command }} From 634e1b073a345e99740fa05abef9e1721021e4c5 Mon Sep 17 00:00:00 2001 From: Ee Durbin Date: Mon, 3 Apr 2023 07:20:51 -0400 Subject: [PATCH 02/14] d'oh, we need to install pip *before* deps, and drop the cached env --- .github/workflows/ci.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b82c2818d98d..02d5099607bd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,15 +68,13 @@ jobs: path: | .cache .mypy_cache - ${{ env.pythonLocation }} - key: ${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('requirements.txt', 'requirements/*.txt') }} + key: ${{ runner.os }}-${{ hashFiles('requirements.txt', 'requirements/*.txt') }} - name: Install Python dependencies run: | - pip install -U pip setuptools wheel + pip install -U setuptools wheel + pip install 'pip @ git+https://github.com/sbidoul/pip@49d3354e1c807bafdd715225d2aaa5cfc3532434' pip install -r requirements.txt --no-deps pip install -r requirements/dev.txt pip check - - name: Install a specific pip - run: pip install 'pip @ git+https://github.com/sbidoul/pip@49d3354e1c807bafdd715225d2aaa5cfc3532434' - name: Run ${{ matrix.name }} run: ${{ matrix.command }} From 090c06692a9a66ab7d8fecfd1c61d9bc529c8577 Mon Sep 17 00:00:00 2001 From: Ee Durbin Date: Mon, 3 Apr 2023 07:27:46 -0400 Subject: [PATCH 03/14] in theory... --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 02d5099607bd..6303aa6480d4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,7 +54,7 @@ jobs: - name: Install platform dependencies run: | sudo apt -y update - sudo apt -y install libcurl4-openssl-dev libssl-dev pkg-config + sudo apt -y install libcurl4-openssl-dev libssl-dev pkg-config libxml2-dev libxslt-dev - uses: actions/setup-python@v4 with: python-version-file: '.python-version' @@ -68,7 +68,7 @@ jobs: path: | .cache .mypy_cache - key: ${{ runner.os }}-${{ hashFiles('requirements.txt', 'requirements/*.txt') }} + key: ${{ runner.os }} - name: Install Python dependencies run: | pip install -U setuptools wheel From eed2e66c9fb3477090d9cc65c9f59309a41ac9ae Mon Sep 17 00:00:00 2001 From: Ee Durbin Date: Mon, 3 Apr 2023 07:33:19 -0400 Subject: [PATCH 04/14] unsure if this interferes, lets find out --- requirements/main.txt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/requirements/main.txt b/requirements/main.txt index b7a2fa725640..494cad2b3dc9 100644 --- a/requirements/main.txt +++ b/requirements/main.txt @@ -1701,10 +1701,6 @@ zxcvbn==4.4.28 \ # via -r requirements/main.in # The following packages are considered to be unsafe in a requirements file: -pip==23.0.1 \ - --hash=sha256:236bcb61156d76c4b8a05821b988c7b8c35bf0da28a4b614e8d6ab5212c25c6f \ - --hash=sha256:cd015ea1bfb0fcef59d8a286c1f8bebcb983f6317719d415dc5351efb7cd7024 - # via pip-api setuptools==67.6.1 \ --hash=sha256:257de92a9d50a60b8e22abfcbb771571fde0dbf3ec234463212027a4eeecbe9a \ --hash=sha256:e728ca814a823bf7bf60162daf9db95b93d532948c4c0bea762ce62f60189078 From 573b33dcef332b7cd8d0e114c6ab7e909b09dda8 Mon Sep 17 00:00:00 2001 From: Ee Durbin Date: Mon, 3 Apr 2023 08:29:08 -0400 Subject: [PATCH 05/14] do deps in one swoop, if this works out i think it's an overall win for resource usage --- .github/workflows/ci.yml | 42 +++++++++++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6303aa6480d4..1a235586e89f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,7 +13,32 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} cancel-in-progress: true jobs: + deps: + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v3 + - name: Install platform dependencies + run: | + sudo apt -y update + sudo apt -y install libcurl4-openssl-dev libssl-dev pkg-config libxml2-dev libxslt-dev + - uses: actions/setup-python@v4 + with: + python-version-file: '.python-version' + - name: pip cache + uses: actions/cache@v3 + with: + path: .cache + key: pip-${{ runner.os }} + - name: Install Python dependencies + run: | + pip install -U setuptools wheel + pip install 'pip @ git+https://github.com/sbidoul/pip@49d3354e1c807bafdd715225d2aaa5cfc3532434' + pip install -r requirements.txt --no-deps + pip install -r requirements/dev.txt + pip check test: + needs: deps strategy: matrix: include: @@ -58,17 +83,16 @@ jobs: - uses: actions/setup-python@v4 with: python-version-file: '.python-version' - cache: 'pip' - cache-dependency-path: | - requirements.txt - requirements/*.txt - - name: Cache common Python cache paths + - name: mypy cache + uses: actions/cache@v3 + with: + path: .mypy_cache + key: mypy-${{ runner.os }}-${{ hashFiles('requirements.txt', 'requirements/*.txt') }} + - name: pip cache uses: actions/cache@v3 with: - path: | - .cache - .mypy_cache - key: ${{ runner.os }} + path: .cache + key: pip-${{ runner.os }} - name: Install Python dependencies run: | pip install -U setuptools wheel From e10b5901953159bbfb25ac28fd0f0d1dfd179c66 Mon Sep 17 00:00:00 2001 From: Ee Durbin Date: Mon, 3 Apr 2023 08:34:53 -0400 Subject: [PATCH 06/14] be explicit (ish) --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1a235586e89f..9954d9e12606 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,7 +28,7 @@ jobs: - name: pip cache uses: actions/cache@v3 with: - path: .cache + path: ~/.cache/pip key: pip-${{ runner.os }} - name: Install Python dependencies run: | @@ -91,7 +91,7 @@ jobs: - name: pip cache uses: actions/cache@v3 with: - path: .cache + path: ~/.cache/pip key: pip-${{ runner.os }} - name: Install Python dependencies run: | From 63504e098e18dd2306df50c45f3827b7105822b8 Mon Sep 17 00:00:00 2001 From: Ee Durbin Date: Mon, 3 Apr 2023 09:51:30 -0400 Subject: [PATCH 07/14] a lil debuggin --- .github/workflows/ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9954d9e12606..d85c60972147 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,6 +34,7 @@ jobs: run: | pip install -U setuptools wheel pip install 'pip @ git+https://github.com/sbidoul/pip@49d3354e1c807bafdd715225d2aaa5cfc3532434' + pip --version pip install -r requirements.txt --no-deps pip install -r requirements/dev.txt pip check @@ -97,8 +98,9 @@ jobs: run: | pip install -U setuptools wheel pip install 'pip @ git+https://github.com/sbidoul/pip@49d3354e1c807bafdd715225d2aaa5cfc3532434' - pip install -r requirements.txt --no-deps - pip install -r requirements/dev.txt + pip --version + pip install --no-build-isolation -r requirements.txt --no-deps + pip install --no-build-isolation -r requirements/dev.txt pip check - name: Run ${{ matrix.name }} run: ${{ matrix.command }} From ae587913e702e04cf2e6fa16276fcba3cd6cc930 Mon Sep 17 00:00:00 2001 From: Ee Durbin Date: Mon, 3 Apr 2023 09:53:16 -0400 Subject: [PATCH 08/14] debug it all --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d85c60972147..de0793d2c51e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,6 +35,8 @@ jobs: pip install -U setuptools wheel pip install 'pip @ git+https://github.com/sbidoul/pip@49d3354e1c807bafdd715225d2aaa5cfc3532434' pip --version + pip cache dir + pip cache info pip install -r requirements.txt --no-deps pip install -r requirements/dev.txt pip check @@ -99,6 +101,8 @@ jobs: pip install -U setuptools wheel pip install 'pip @ git+https://github.com/sbidoul/pip@49d3354e1c807bafdd715225d2aaa5cfc3532434' pip --version + pip cache dir + pip cache info pip install --no-build-isolation -r requirements.txt --no-deps pip install --no-build-isolation -r requirements/dev.txt pip check From 10c50f943bb0deb3ab94a101bf2d8491cf27ca94 Mon Sep 17 00:00:00 2001 From: Ee Durbin Date: Mon, 3 Apr 2023 09:55:04 -0400 Subject: [PATCH 09/14] at dstuffts req --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index de0793d2c51e..48e06ebc203b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,7 +33,7 @@ jobs: - name: Install Python dependencies run: | pip install -U setuptools wheel - pip install 'pip @ git+https://github.com/sbidoul/pip@49d3354e1c807bafdd715225d2aaa5cfc3532434' + pip install --no-cache-dir 'pip @ git+https://github.com/sbidoul/pip@49d3354e1c807bafdd715225d2aaa5cfc3532434' pip --version pip cache dir pip cache info @@ -99,7 +99,7 @@ jobs: - name: Install Python dependencies run: | pip install -U setuptools wheel - pip install 'pip @ git+https://github.com/sbidoul/pip@49d3354e1c807bafdd715225d2aaa5cfc3532434' + pip install --no-cache-dir 'pip @ git+https://github.com/sbidoul/pip@49d3354e1c807bafdd715225d2aaa5cfc3532434' pip --version pip cache dir pip cache info From 64c03196b017de349382b8f06ddd586b92e28e70 Mon Sep 17 00:00:00 2001 From: Ee Durbin Date: Mon, 3 Apr 2023 13:44:44 -0400 Subject: [PATCH 10/14] Update .github/workflows/ci.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Stéphane Bidoul --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 48e06ebc203b..1f52f53d94af 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,7 +33,7 @@ jobs: - name: Install Python dependencies run: | pip install -U setuptools wheel - pip install --no-cache-dir 'pip @ git+https://github.com/sbidoul/pip@49d3354e1c807bafdd715225d2aaa5cfc3532434' + pip install 'pip @ git+https://github.com/pypa/pip@refs/pull/11897/head' pip --version pip cache dir pip cache info From c0f10861fd045bb73b06fd8708c57e92a39ff1a9 Mon Sep 17 00:00:00 2001 From: Ee Durbin Date: Mon, 3 Apr 2023 13:52:12 -0400 Subject: [PATCH 11/14] allup test --- .github/workflows/ci.yml | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1f52f53d94af..68a25bff6e91 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,9 +34,6 @@ jobs: run: | pip install -U setuptools wheel pip install 'pip @ git+https://github.com/pypa/pip@refs/pull/11897/head' - pip --version - pip cache dir - pip cache info pip install -r requirements.txt --no-deps pip install -r requirements/dev.txt pip check @@ -99,12 +96,9 @@ jobs: - name: Install Python dependencies run: | pip install -U setuptools wheel - pip install --no-cache-dir 'pip @ git+https://github.com/sbidoul/pip@49d3354e1c807bafdd715225d2aaa5cfc3532434' - pip --version - pip cache dir - pip cache info - pip install --no-build-isolation -r requirements.txt --no-deps - pip install --no-build-isolation -r requirements/dev.txt + pip install 'pip @ git+https://github.com/pypa/pip@refs/pull/11897/head' + pip install -r requirements.txt --no-deps + pip install -r requirements/dev.txt pip check - name: Run ${{ matrix.name }} run: ${{ matrix.command }} From 05e3689f930b1efa20b7b425a670b0c27c5d126c Mon Sep 17 00:00:00 2001 From: Ee Durbin Date: Mon, 3 Apr 2023 16:26:57 -0400 Subject: [PATCH 12/14] just wanna go fast --- .github/workflows/ci.yml | 17 +++++++++++------ requirements/main.txt | 4 ++++ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 68a25bff6e91..8ae1e7f7de08 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,6 +37,11 @@ jobs: pip install -r requirements.txt --no-deps pip install -r requirements/dev.txt pip check + - name: Cache built Python environment + uses: actions/cache/save@v3 + with: + path: ${{ env.pythonLocation }} + key: ${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('requirements.txt', 'requirements/*.txt') }} test: needs: deps strategy: @@ -79,20 +84,20 @@ jobs: - name: Install platform dependencies run: | sudo apt -y update - sudo apt -y install libcurl4-openssl-dev libssl-dev pkg-config libxml2-dev libxslt-dev + sudo apt -y install libcurl4-openssl libssl libxml2 libxslt - uses: actions/setup-python@v4 with: python-version-file: '.python-version' + - name: Restore built Python environment from deps + uses: actions/cache/restore@v3 + with: + path: ${{ env.pythonLocation }} + key: ${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('requirements.txt', 'requirements/*.txt') }} - name: mypy cache uses: actions/cache@v3 with: path: .mypy_cache key: mypy-${{ runner.os }}-${{ hashFiles('requirements.txt', 'requirements/*.txt') }} - - name: pip cache - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: pip-${{ runner.os }} - name: Install Python dependencies run: | pip install -U setuptools wheel diff --git a/requirements/main.txt b/requirements/main.txt index 494cad2b3dc9..b7a2fa725640 100644 --- a/requirements/main.txt +++ b/requirements/main.txt @@ -1701,6 +1701,10 @@ zxcvbn==4.4.28 \ # via -r requirements/main.in # The following packages are considered to be unsafe in a requirements file: +pip==23.0.1 \ + --hash=sha256:236bcb61156d76c4b8a05821b988c7b8c35bf0da28a4b614e8d6ab5212c25c6f \ + --hash=sha256:cd015ea1bfb0fcef59d8a286c1f8bebcb983f6317719d415dc5351efb7cd7024 + # via pip-api setuptools==67.6.1 \ --hash=sha256:257de92a9d50a60b8e22abfcbb771571fde0dbf3ec234463212027a4eeecbe9a \ --hash=sha256:e728ca814a823bf7bf60162daf9db95b93d532948c4c0bea762ce62f60189078 From ecc4e9913162cbc1e7c7254d0c25ff8e57be6322 Mon Sep 17 00:00:00 2001 From: Ee Durbin Date: Mon, 3 Apr 2023 16:30:15 -0400 Subject: [PATCH 13/14] looks worth keeping as a whole e2e cache --- .github/workflows/ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8ae1e7f7de08..cd6bb9239985 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,6 +30,11 @@ jobs: with: path: ~/.cache/pip key: pip-${{ runner.os }} + - name: Cache built Python environment + uses: actions/cache@v3 + with: + path: ${{ env.pythonLocation }} + key: ${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('requirements.txt', 'requirements/*.txt') }} - name: Install Python dependencies run: | pip install -U setuptools wheel @@ -37,11 +42,6 @@ jobs: pip install -r requirements.txt --no-deps pip install -r requirements/dev.txt pip check - - name: Cache built Python environment - uses: actions/cache/save@v3 - with: - path: ${{ env.pythonLocation }} - key: ${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('requirements.txt', 'requirements/*.txt') }} test: needs: deps strategy: From c031db02dbd520703af7346d33479b01ac4dbd01 Mon Sep 17 00:00:00 2001 From: Ee Durbin Date: Mon, 3 Apr 2023 16:32:36 -0400 Subject: [PATCH 14/14] pairity to Dockerfile --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cd6bb9239985..763e84beff54 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -84,7 +84,7 @@ jobs: - name: Install platform dependencies run: | sudo apt -y update - sudo apt -y install libcurl4-openssl libssl libxml2 libxslt + sudo apt -y install libxml2 libxslt1.1 libcurl4 - uses: actions/setup-python@v4 with: python-version-file: '.python-version'