Skip to content

Commit 5f80942

Browse files
jonashaagjonashaag
committed
Add ccache to GHA builds
- Refactor duplicated code from GHA workflows to the 'build-pandas' and 'setup' actions. - Use Mamba over Conda everywhere. - Add ccache using 'hendrikmuhs/ccache-action'. - General cleanup of GHA workflows.
1 parent 9cc98a0 commit 5f80942

File tree

11 files changed

+189
-157
lines changed

11 files changed

+189
-157
lines changed

‎.github/actions/build-pandas/action.yml‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
name: Build pandas
2+
description: Rebuilds the C extensions and installs pandas
3+
inputs:
4+
use-login-shell:
5+
description: "Use 'bash -l' as shell (required for Conda envs)"
6+
default: true
7+
runs:
8+
using: composite
9+
steps:
10+
# Create a shell wrapper to be able to call "bash" or "bash -l" depending
11+
# on the "use-login-shell" arguments.
12+
# We need this because GHA does not allow ${{ inputs. }} in "shell: " arguments.
13+
- name: Set shell
14+
shell: bash
15+
run: |
16+
if [ ${{ inputs.use-login-shell }} = true ]; then
17+
args="-l"
18+
fi
19+
echo "exec bash $args \"\$@\"" > /tmp/_build_pandas_shell
20+
cat /tmp/_build_pandas_shell
21+
22+
- name: Environment Detail
23+
shell: bash /tmp/_build_pandas_shell {0}
24+
run: |
25+
if which conda; then
26+
conda info
27+
conda list
28+
fi
29+
if which pip; then
30+
pip list
31+
fi
32+
python --version
33+
34+
- name: Get Python version
35+
id: get-python-version
36+
shell: bash /tmp/_build_pandas_shell {0}
37+
run: python3 -c "import platform as p; print(f'::set-output name=version::{p.python_version()}-{p.python_branch()}')"
38+
39+
- name: Set up ccache
40+
uses: ./.github/actions/setup-ccache
41+
with:
42+
extra-cache-key: ${{ steps.get-python-version.outputs.version }}
43+
if: ${{ runner.os != 'Windows' }}
44+
45+
- name: Build Pandas
46+
shell: bash /tmp/_build_pandas_shell {0}
47+
run: |
48+
if [ -z "$CC_FOR_BUILD" ]; then
49+
export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
50+
else
51+
# "build_ext" uses '$CC.split()[0]' as its linker, which isn't what
52+
# we want if '$CC.split()[0]' is "ccache".
53+
echo -e '#!/bin/bash -eu\n if which "$1"; then exec ccache "$@"; else exec ccache "$CC" "$@"; fi' >> /tmp/ccache-wrapper
54+
chmod +x /tmp/ccache-wrapper
55+
export CC="/tmp/ccache-wrapper $CC"
56+
fi
57+
time python setup.py build_ext -vv -j 2
58+
time python -m pip install -vv -e . --no-build-isolation --no-use-pep517 --no-index
59+
60+
- name: Build Version
61+
shell: bash /tmp/_build_pandas_shell {0}
62+
run: pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd

‎.github/actions/build_pandas/action.yml‎

Lines changed: 0 additions & 17 deletions
This file was deleted.

‎.github/actions/setup-ccache/action.yml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
name: Setup ccache
2+
inputs:
3+
extra-cache-key:
4+
required: false
5+
default: ''
6+
runs:
7+
using: composite
8+
steps:
9+
- name: Get Date
10+
id: get-date
11+
run: echo "::set-output name=today::$(/bin/date -u '+%Y%m%d')"
12+
shell: bash
13+
14+
- name: Setup ccache
15+
uses: hendrikmuhs/ccache-action@2181be813387616fc2b8dca72d3ff8b912b25f73
16+
with:
17+
key: ${{ runner.os }}--${{ runner.arch }}--${{ github.workflow }}--${{ steps.get-date.outputs.today }}--${{ inputs.extra-cache-key }}

‎.github/actions/setup/action.yml‎

Lines changed: 47 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,53 @@
11
name: Set up pandas
22
description: Runs all the setup steps required to have a built pandas ready to use
3+
inputs:
4+
environment-file:
5+
default: environment.yml
6+
is-pypy:
7+
default: false
8+
activate-environment:
9+
default: pandas-dev
10+
python-version:
11+
required: false
312
runs:
413
using: composite
514
steps:
6-
- name: Setting conda path
7-
run: echo "${HOME}/miniconda3/bin" >> $GITHUB_PATH
8-
shell: bash -l {0}
15+
- name: Get Date
16+
id: get-date
17+
run: echo "::set-output name=today::$(/bin/date -u '+%Y%m%d')"
18+
shell: bash
919

10-
- name: Setup environment and build pandas
11-
run: ci/setup_env.sh
12-
shell: bash -l {0}
20+
- name: Cache Conda packages
21+
uses: actions/cache@v2
22+
with:
23+
path: ~/conda_pkgs_dir
24+
key: conda-${{ runner.os }}-${{ runner.arch }}-${{ inputs.environment-file }}-${{ steps.get-date.outputs.today }}
25+
26+
- uses: conda-incubator/setup-miniconda@v2
27+
with:
28+
mamba-version: "0.20"
29+
use-mamba: true
30+
channels: conda-forge
31+
activate-environment: ${{ inputs.activate-environment }}
32+
channel-priority: strict
33+
environment-file: ${{ inputs.environment-file }}
34+
python-version: ${{ inputs.python-version }}
35+
use-only-tar-bz2: true
36+
if: ${{ inputs.is-pypy == 'false' }} # No pypy3.8 support
37+
38+
- name: Pin setuptools
39+
run: mamba install -n ${{ inputs.activate-environment }} 'setuptools<60.0.0'
40+
shell: bash
41+
if: ${{ inputs.is-pypy == 'false' }} # No pypy3.8 support
42+
43+
- name: Setup PyPy
44+
uses: actions/setup-python@v2
45+
with:
46+
python-version: "pypy-3.8"
47+
if: ${{ inputs.is-pypy == 'true' }}
48+
49+
- name: Setup PyPy dependencies
50+
# TODO: re-enable cov, its slowing the tests down though
51+
run: pip install Cython numpy python-dateutil pytz pytest>=6.0 pytest-xdist>=1.31.0 hypothesis>=5.5.3
52+
shell: bash
53+
if: ${{ inputs.is-pypy == 'true' }}

‎.github/workflows/asv-bot.yml‎

Lines changed: 5 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,7 @@ on:
66
- created
77

88
env:
9-
ENV_FILE: environment.yml
10-
COMMENT: ${{github.event.comment.body}}
9+
COMMENT: ${{ github.event.comment.body }}
1110

1211
jobs:
1312
autotune:
@@ -33,20 +32,10 @@ jobs:
3332
with:
3433
fetch-depth: 0
3534

36-
- name: Cache conda
37-
uses: actions/cache@v2
38-
with:
39-
path: ~/conda_pkgs_dir
40-
key: ${{ runner.os }}-conda-${{ hashFiles('${{ env.ENV_FILE }}') }}
41-
42-
# Although asv sets up its own env, deps are still needed
43-
# during discovery process
44-
- uses: conda-incubator/setup-miniconda@v2
45-
with:
46-
activate-environment: pandas-dev
47-
channel-priority: strict
48-
environment-file: ${{ env.ENV_FILE }}
49-
use-only-tar-bz2: true
35+
# Although asv sets up its own env, deps are still needed
36+
# during discovery process
37+
- name: Set up Conda
38+
uses: ./.github/actions/setup
5039

5140
- name: Run benchmarks
5241
id: bench

‎.github/workflows/code-checks.yml‎

Lines changed: 7 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@ on:
1111
- 1.4.x
1212

1313
env:
14-
ENV_FILE: environment.yml
1514
PANDAS_CI: 1
1615

1716
jobs:
@@ -52,20 +51,11 @@ jobs:
5251
with:
5352
fetch-depth: 0
5453

55-
- name: Cache conda
56-
uses: actions/cache@v2
57-
with:
58-
path: ~/conda_pkgs_dir
59-
key: ${{ runner.os }}-conda-${{ hashFiles('${{ env.ENV_FILE }}') }}
54+
- name: Set up Conda
55+
uses: ./.github/actions/setup
6056

61-
- uses: conda-incubator/setup-miniconda@v2
62-
with:
63-
mamba-version: "*"
64-
channels: conda-forge
65-
activate-environment: pandas-dev
66-
channel-priority: strict
67-
environment-file: ${{ env.ENV_FILE }}
68-
use-only-tar-bz2: true
57+
- name: Build Pandas
58+
uses: ./.github/actions/build-pandas
6959

7060
- name: Install node.js (for pyright)
7161
uses: actions/setup-node@v2
@@ -76,29 +66,20 @@ jobs:
7666
# note: keep version in sync with .pre-commit-config.yaml
7767
run: npm install -g [email protected]
7868

79-
- name: Build Pandas
80-
id: build
81-
uses: ./.github/actions/build_pandas
82-
8369
- name: Run checks on imported code
8470
run: ci/code_checks.sh code
85-
if: ${{ steps.build.outcome == 'success' }}
8671

8772
- name: Run doctests
8873
run: ci/code_checks.sh doctests
89-
if: ${{ steps.build.outcome == 'success' }}
9074

9175
- name: Run docstring validation
9276
run: ci/code_checks.sh docstrings
93-
if: ${{ steps.build.outcome == 'success' }}
9477

9578
- name: Run typing validation
9679
run: ci/code_checks.sh typing
97-
if: ${{ steps.build.outcome == 'success' }}
9880

9981
- name: Run docstring validation script tests
10082
run: pytest scripts
101-
if: ${{ steps.build.outcome == 'success' }}
10283

10384
asv-benchmarks:
10485
name: ASV Benchmarks
@@ -118,24 +99,11 @@ jobs:
11899
with:
119100
fetch-depth: 0
120101

121-
- name: Cache conda
122-
uses: actions/cache@v2
123-
with:
124-
path: ~/conda_pkgs_dir
125-
key: ${{ runner.os }}-conda-${{ hashFiles('${{ env.ENV_FILE }}') }}
126-
127-
- uses: conda-incubator/setup-miniconda@v2
128-
with:
129-
mamba-version: "*"
130-
channels: conda-forge
131-
activate-environment: pandas-dev
132-
channel-priority: strict
133-
environment-file: ${{ env.ENV_FILE }}
134-
use-only-tar-bz2: true
102+
- name: Set up Conda
103+
uses: ./.github/actions/setup
135104

136105
- name: Build Pandas
137-
id: build
138-
uses: ./.github/actions/build_pandas
106+
uses: ./.github/actions/build-pandas
139107

140108
- name: Run ASV benchmarks
141109
run: |
@@ -148,7 +116,6 @@ jobs:
148116
if grep "failed" benchmarks.log > /dev/null ; then
149117
exit 1
150118
fi
151-
if: ${{ steps.build.outcome == 'success' }}
152119
153120
- name: Publish benchmarks artifact
154121
uses: actions/upload-artifact@v2

‎.github/workflows/datamanger.yml‎ renamed to ‎.github/workflows/datamanager.yml‎

Lines changed: 14 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -13,14 +13,21 @@ on:
1313
- "doc/**"
1414

1515
env:
16-
ENV_FILE: environment.yml
1716
PANDAS_CI: 1
17+
PYTEST_WORKERS: "auto"
18+
PYTEST_TARGET: pandas
19+
PATTERN: "not network and not clipboard and not single_cpu"
20+
PANDAS_DATA_MANAGER: array
1821

1922
jobs:
2023
data_manager:
2124
name: Test experimental data manager
2225
runs-on: ubuntu-latest
2326
timeout-minutes: 120
27+
defaults:
28+
run:
29+
shell: bash -l {0}
30+
2431
services:
2532
moto:
2633
image: motoserver/moto
@@ -29,6 +36,7 @@ jobs:
2936
AWS_SECRET_ACCESS_KEY: foobar_secret
3037
ports:
3138
- 5000:5000
39+
3240
concurrency:
3341
# https://github.202132.xyzmunity/t/concurrecy-not-work-for-push/183068/7
3442
group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-data_manager
@@ -40,15 +48,11 @@ jobs:
4048
with:
4149
fetch-depth: 0
4250

43-
- name: Set up pandas
51+
- name: Set up Conda
4452
uses: ./.github/actions/setup
4553

54+
- name: Build pandas
55+
uses: ./.github/actions/build-pandas
56+
4657
- name: Run tests
47-
env:
48-
PANDAS_DATA_MANAGER: array
49-
PATTERN: "not network and not clipboard and not single_cpu"
50-
PYTEST_WORKERS: "auto"
51-
PYTEST_TARGET: pandas
52-
run: |
53-
source activate pandas-dev
54-
ci/run_tests.sh
58+
run: ci/run_tests.sh

‎.github/workflows/docbuild-and-upload.yml‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -11,13 +11,15 @@ on:
1111
- 1.4.x
1212

1313
env:
14-
ENV_FILE: environment.yml
1514
PANDAS_CI: 1
1615

1716
jobs:
1817
web_and_docs:
1918
name: Doc Build and Upload
2019
runs-on: ubuntu-latest
20+
defaults:
21+
run:
22+
shell: bash -l {0}
2123

2224
concurrency:
2325
# https://github.202132.xyzmunity/t/concurrecy-not-work-for-push/183068/7
@@ -30,17 +32,17 @@ jobs:
3032
with:
3133
fetch-depth: 0
3234

33-
- name: Set up pandas
35+
- name: Set up Conda
3436
uses: ./.github/actions/setup
3537

38+
- name: Build pandas
39+
uses: ./.github/actions/build-pandas
40+
3641
- name: Build website
37-
run: |
38-
source activate pandas-dev
39-
python web/pandas_web.py web/pandas --target-path=web/build
42+
run: python web/pandas_web.py web/pandas --target-path=web/build
43+
4044
- name: Build documentation
41-
run: |
42-
source activate pandas-dev
43-
doc/make.py --warnings-are-errors
45+
run: doc/make.py --warnings-are-errors
4446

4547
- name: Install ssh key
4648
run: |

0 commit comments

Comments
 (0)