Skip to content

Commit 92840f8

Browse files
jonashaagjonashaag
committed
Add ccache to GHA builds
- Refactor duplicated code from GHA workflows to the 'build-pandas' and 'setup' actions. - Use Mamba over Conda everywhere. - Add sccache using 'hendrikmuhs/ccache-action'. - General cleanup of GHA workflows.
1 parent af8ad6d commit 92840f8

File tree

12 files changed

+235
-160
lines changed

12 files changed

+235
-160
lines changed

‎.github/actions/build-pandas/action.yml‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
name: Build pandas
2+
description: Rebuilds the C extensions and installs pandas
3+
inputs:
4+
use-login-shell:
5+
description: "Use 'bash -l' as shell (required for Conda envs)"
6+
default: true
7+
runs:
8+
using: composite
9+
steps:
10+
# Create a shell wrapper to be able to call "bash" or "bash -l" depending
11+
# on the "use-login-shell" arguments.
12+
# We need this because GHA does not allow ${{ inputs. }} in "shell: " arguments.
13+
- name: Set shell
14+
shell: bash
15+
run: |
16+
if [ ${{ inputs.use-login-shell }} = true ]; then
17+
args="-l"
18+
fi
19+
echo "exec bash $args \"\$@\"" > /tmp/_build_pandas_shell
20+
cat /tmp/_build_pandas_shell
21+
22+
- name: Environment Detail
23+
shell: bash /tmp/_build_pandas_shell {0}
24+
run: |
25+
if which conda; then
26+
conda info
27+
conda list
28+
fi
29+
if which pip; then
30+
pip list
31+
fi
32+
python --version
33+
34+
- name: Get Python version
35+
id: get-python-version
36+
shell: bash /tmp/_build_pandas_shell {0}
37+
run: python3 -c "import platform as p; print(f'::set-output name=version::{p.python_version()}-{p.python_branch()}')"
38+
39+
- name: Set up sccache
40+
uses: ./.github/actions/setup-sccache
41+
with:
42+
extra-cache-key: ${{ steps.get-python-version.outputs.version }}
43+
44+
- name: Build Pandas
45+
shell: bash /tmp/_build_pandas_shell {0}
46+
run: |
47+
time DISTUTILS_C_COMPILER_LAUNCHER=sccache python setup.py build_ext -vv -j 2
48+
python -m pip install -vv -e . --no-build-isolation --no-use-pep517 --no-index
49+
50+
- name: Build Version
51+
shell: bash /tmp/_build_pandas_shell {0}
52+
run: pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd

‎.github/actions/build_pandas/action.yml‎

Lines changed: 0 additions & 17 deletions
This file was deleted.

‎.github/actions/setup-sccache/action.yml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
name: Setup sccache
2+
inputs:
3+
extra-cache-key:
4+
required: false
5+
default: ''
6+
runs:
7+
using: composite
8+
steps:
9+
- name: Get Date
10+
id: get-date
11+
run: echo "::set-output name=today::$(/bin/date -u '+%Y%m%d')"
12+
shell: bash
13+
14+
- name: Setup sccache
15+
uses: jonashaag/ccache-action@sccache-2
16+
with:
17+
sccache: true
18+
key: ${{ runner.os }}--${{ runner.arch }}--${{ github.workflow }}--${{ steps.get-date.outputs.today }}--${{ inputs.extra-cache-key }}

‎.github/actions/setup/action.yml‎

Lines changed: 58 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,64 @@
11
name: Set up pandas
22
description: Runs all the setup steps required to have a built pandas ready to use
3+
inputs:
4+
environment-file:
5+
default: environment.yml
6+
pyarrow-version:
7+
required: false
8+
is-pypy:
9+
default: false
10+
activate-environment:
11+
default: pandas-dev
12+
python-version:
13+
required: false
314
runs:
415
using: composite
516
steps:
6-
- name: Setting conda path
7-
run: echo "${HOME}/miniconda3/bin" >> $GITHUB_PATH
8-
shell: bash -l {0}
17+
- name: Get Date
18+
id: get-date
19+
run: echo "::set-output name=today::$(/bin/date -u '+%Y%m%d')"
20+
shell: bash
921

10-
- name: Setup environment and build pandas
11-
run: ci/setup_env.sh
12-
shell: bash -l {0}
22+
- name: Cache Conda packages
23+
uses: actions/cache@v2
24+
with:
25+
path: ~/conda_pkgs_dir
26+
key: conda-${{ runner.os }}-${{ runner.arch }}-${{ inputs.environment-file }}-${{ steps.get-date.outputs.today }}
27+
28+
- name: Set Arrow version in ${{ inputs.environment-file }} to ${{ inputs.pyarrow-version }}
29+
run: |
30+
grep -q '\- pyarrow' ${{ inputs.environment-file }}
31+
sed -i "s/- pyarrow/- pyarrow=${{ inputs.pyarrow-version }}/" ${{ inputs.environment-file }}
32+
cat ${{ inputs.environment-file }}
33+
shell: bash
34+
if: ${{ inputs.pyarrow-version }}
35+
36+
- name: Setup Mambaforge and install ${{ inputs.environment-file }} (Python ${{ inputs.python-version }})
37+
uses: conda-incubator/setup-miniconda@v2
38+
with:
39+
mamba-version: "0.21.2"
40+
use-mamba: true
41+
channels: conda-forge
42+
activate-environment: ${{ inputs.activate-environment }}
43+
channel-priority: strict
44+
environment-file: ${{ inputs.environment-file }}
45+
python-version: ${{ inputs.python-version }}
46+
use-only-tar-bz2: true
47+
if: ${{ inputs.is-pypy == 'false' }} # No pypy3.8 support
48+
49+
- name: Pin setuptools (GH#44980)
50+
run: mamba install -n ${{ inputs.activate-environment }} 'setuptools<60.0.0'
51+
shell: bash
52+
if: ${{ inputs.is-pypy == 'false' }} # No pypy3.8 support
53+
54+
- name: Setup PyPy
55+
uses: actions/setup-python@v2
56+
with:
57+
python-version: "pypy-3.8"
58+
if: ${{ inputs.is-pypy == 'true' }}
59+
60+
- name: Setup PyPy dependencies
61+
# TODO: re-enable cov, its slowing the tests down though
62+
run: pip install Cython numpy python-dateutil pytz pytest>=6.0 pytest-xdist>=1.31.0 hypothesis>=5.5.3
63+
shell: bash
64+
if: ${{ inputs.is-pypy == 'true' }}

‎.github/workflows/asv-bot.yml‎

Lines changed: 5 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,7 @@ on:
66
- created
77

88
env:
9-
ENV_FILE: environment.yml
10-
COMMENT: ${{github.event.comment.body}}
9+
COMMENT: ${{ github.event.comment.body }}
1110

1211
jobs:
1312
autotune:
@@ -33,20 +32,10 @@ jobs:
3332
with:
3433
fetch-depth: 0
3534

36-
- name: Cache conda
37-
uses: actions/cache@v2
38-
with:
39-
path: ~/conda_pkgs_dir
40-
key: ${{ runner.os }}-conda-${{ hashFiles('${{ env.ENV_FILE }}') }}
41-
42-
# Although asv sets up its own env, deps are still needed
43-
# during discovery process
44-
- uses: conda-incubator/setup-miniconda@v2
45-
with:
46-
activate-environment: pandas-dev
47-
channel-priority: strict
48-
environment-file: ${{ env.ENV_FILE }}
49-
use-only-tar-bz2: true
35+
# Although asv sets up its own env, deps are still needed
36+
# during discovery process
37+
- name: Set up Conda
38+
uses: ./.github/actions/setup
5039

5140
- name: Run benchmarks
5241
id: bench

‎.github/workflows/code-checks.yml‎

Lines changed: 9 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@ on:
1111
- 1.4.x
1212

1313
env:
14-
ENV_FILE: environment.yml
1514
PANDAS_CI: 1
1615

1716
jobs:
@@ -52,20 +51,13 @@ jobs:
5251
with:
5352
fetch-depth: 0
5453

55-
- name: Cache conda
56-
uses: actions/cache@v2
57-
with:
58-
path: ~/conda_pkgs_dir
59-
key: ${{ runner.os }}-conda-${{ hashFiles('${{ env.ENV_FILE }}') }}
54+
- name: Set up Conda
55+
uses: ./.github/actions/setup
6056

61-
- uses: conda-incubator/setup-miniconda@v2
62-
with:
63-
mamba-version: "*"
64-
channels: conda-forge
65-
activate-environment: pandas-dev
66-
channel-priority: strict
67-
environment-file: ${{ env.ENV_FILE }}
68-
use-only-tar-bz2: true
57+
- name: Build Pandas
58+
uses: ./.github/actions/build-pandas
59+
id: build
60+
continue-on-error: true
6961

7062
- name: Install node.js (for pyright)
7163
uses: actions/setup-node@v2
@@ -76,10 +68,6 @@ jobs:
7668
# note: keep version in sync with .pre-commit-config.yaml
7769
run: npm install -g [email protected]
7870

79-
- name: Build Pandas
80-
id: build
81-
uses: ./.github/actions/build_pandas
82-
8371
- name: Run checks on imported code
8472
run: ci/code_checks.sh code
8573
if: ${{ steps.build.outcome == 'success' }}
@@ -94,11 +82,9 @@ jobs:
9482

9583
- name: Run typing validation
9684
run: ci/code_checks.sh typing
97-
if: ${{ steps.build.outcome == 'success' }}
9885

9986
- name: Run docstring validation script tests
10087
run: pytest scripts
101-
if: ${{ steps.build.outcome == 'success' }}
10288

10389
asv-benchmarks:
10490
name: ASV Benchmarks
@@ -118,24 +104,11 @@ jobs:
118104
with:
119105
fetch-depth: 0
120106

121-
- name: Cache conda
122-
uses: actions/cache@v2
123-
with:
124-
path: ~/conda_pkgs_dir
125-
key: ${{ runner.os }}-conda-${{ hashFiles('${{ env.ENV_FILE }}') }}
126-
127-
- uses: conda-incubator/setup-miniconda@v2
128-
with:
129-
mamba-version: "*"
130-
channels: conda-forge
131-
activate-environment: pandas-dev
132-
channel-priority: strict
133-
environment-file: ${{ env.ENV_FILE }}
134-
use-only-tar-bz2: true
107+
- name: Set up Conda
108+
uses: ./.github/actions/setup
135109

136110
- name: Build Pandas
137-
id: build
138-
uses: ./.github/actions/build_pandas
111+
uses: ./.github/actions/build-pandas
139112

140113
- name: Run ASV benchmarks
141114
run: |
@@ -148,7 +121,6 @@ jobs:
148121
if grep "failed" benchmarks.log > /dev/null ; then
149122
exit 1
150123
fi
151-
if: ${{ steps.build.outcome == 'success' }}
152124
153125
- name: Publish benchmarks artifact
154126
uses: actions/upload-artifact@v2

‎.github/workflows/datamanger.yml‎ renamed to ‎.github/workflows/datamanager.yml‎

Lines changed: 14 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -13,14 +13,21 @@ on:
1313
- "doc/**"
1414

1515
env:
16-
ENV_FILE: environment.yml
1716
PANDAS_CI: 1
17+
PYTEST_WORKERS: "auto"
18+
PYTEST_TARGET: pandas
19+
PATTERN: "not network and not clipboard and not single_cpu"
20+
PANDAS_DATA_MANAGER: array
1821

1922
jobs:
2023
data_manager:
2124
name: Test experimental data manager
2225
runs-on: ubuntu-latest
2326
timeout-minutes: 120
27+
defaults:
28+
run:
29+
shell: bash -l {0}
30+
2431
services:
2532
moto:
2633
image: motoserver/moto
@@ -29,6 +36,7 @@ jobs:
2936
AWS_SECRET_ACCESS_KEY: foobar_secret
3037
ports:
3138
- 5000:5000
39+
3240
concurrency:
3341
# https://github.202132.xyzmunity/t/concurrecy-not-work-for-push/183068/7
3442
group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-data_manager
@@ -40,15 +48,11 @@ jobs:
4048
with:
4149
fetch-depth: 0
4250

43-
- name: Set up pandas
51+
- name: Set up Conda
4452
uses: ./.github/actions/setup
4553

54+
- name: Build pandas
55+
uses: ./.github/actions/build-pandas
56+
4657
- name: Run tests
47-
env:
48-
PANDAS_DATA_MANAGER: array
49-
PATTERN: "not network and not clipboard and not single_cpu"
50-
PYTEST_WORKERS: "auto"
51-
PYTEST_TARGET: pandas
52-
run: |
53-
source activate pandas-dev
54-
ci/run_tests.sh
58+
run: ci/run_tests.sh

‎.github/workflows/docbuild-and-upload.yml‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -11,13 +11,15 @@ on:
1111
- 1.4.x
1212

1313
env:
14-
ENV_FILE: environment.yml
1514
PANDAS_CI: 1
1615

1716
jobs:
1817
web_and_docs:
1918
name: Doc Build and Upload
2019
runs-on: ubuntu-latest
20+
defaults:
21+
run:
22+
shell: bash -l {0}
2123

2224
concurrency:
2325
# https://github.202132.xyzmunity/t/concurrecy-not-work-for-push/183068/7
@@ -30,17 +32,17 @@ jobs:
3032
with:
3133
fetch-depth: 0
3234

33-
- name: Set up pandas
35+
- name: Set up Conda
3436
uses: ./.github/actions/setup
3537

38+
- name: Build pandas
39+
uses: ./.github/actions/build-pandas
40+
3641
- name: Build website
37-
run: |
38-
source activate pandas-dev
39-
python web/pandas_web.py web/pandas --target-path=web/build
42+
run: python web/pandas_web.py web/pandas --target-path=web/build
43+
4044
- name: Build documentation
41-
run: |
42-
source activate pandas-dev
43-
doc/make.py --warnings-are-errors
45+
run: doc/make.py --warnings-are-errors
4446

4547
- name: Install ssh key
4648
run: |

0 commit comments

Comments
 (0)