Skip to content

Commit 83bd3fa

Browse files
jonashaagjonashaag
committed
Refactor CI
- Drop Azure (move to GHA). - Use mamba-org/provision-with-micromamba to set up Conda envs. - Cache Conda envs. - Use hendrikmuhs/ccache-action to cache C compiliation. - Add Windows 32 bit build. - Drop Ubuntu 32 bit build. - Factor out common GHA code into reusable actions.
1 parent afec0e9 commit 83bd3fa

30 files changed

+520
-435
lines changed

‎ci/setup_env.sh‎ renamed to ‎.circleci/setup_env.sh‎

File renamed without changes.

‎.github/actions/build-pandas/action.yml‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
name: Build pandas
2+
description: Rebuilds the C extensions and installs pandas
3+
runs:
4+
using: composite
5+
steps:
6+
- name: Environment Detail
7+
run: |
8+
if which micromamba; then
9+
micromamba info
10+
micromamba list
11+
micromamba info | grep -Ei 'environment.+:' | grep -qEiv 'environment.+:.+none'
12+
fi
13+
if which pip; then
14+
pip list
15+
fi
16+
python --version
17+
shell: bash -el {0}
18+
19+
- name: Get Python version
20+
id: get-python-version
21+
run: python3 -c "import platform as p; print(f'::set-output name=version::{p.python_implementation()}-{p.python_version()}')"
22+
shell: bash -el {0}
23+
24+
- name: Set up sccache
25+
uses: ./.github/actions/setup-sccache
26+
with:
27+
extra-cache-key: ${{ steps.get-python-version.outputs.version }}
28+
29+
- name: Build Pandas
30+
run: |
31+
which python
32+
which pip
33+
time DISTUTILS_C_COMPILER_LAUNCHER=sccache python setup.py build_ext -v -j 2
34+
pip install -v -e . --no-build-isolation --no-use-pep517 --no-index
35+
shell: bash -el {0}
36+
37+
- name: Build Version
38+
run: pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd
39+
shell: bash -el {0}

‎.github/actions/build_pandas/action.yml‎

Lines changed: 0 additions & 17 deletions
This file was deleted.

‎.github/actions/run-tests/action.yml‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
name: Run tests and report results
2+
inputs:
3+
check-pyarrow-version:
4+
required: false
5+
runs:
6+
using: composite
7+
steps:
8+
- name: Check PyArrow version
9+
run: |
10+
# Double check that we have the expected PyArrow
11+
pushd /tmp
12+
python -c "import pandas; pandas.show_versions()" | egrep -i "pyarrow.+: ${{ matrix.check-pyarrow-version }}"
13+
popd
14+
shell: bash -el {0}
15+
if: ${{ inputs.check-pyarrow-version }}
16+
17+
- name: Test
18+
run: ci/run_tests.sh
19+
shell: bash -el {0}
20+
21+
- name: Publish test results
22+
uses: actions/upload-artifact@v2
23+
with:
24+
name: Test results
25+
path: test-data.xml
26+
if: failure()
27+
28+
- name: Upload coverage to Codecov
29+
uses: codecov/codecov-action@v2
30+
with:
31+
flags: unittests
32+
name: codecov-pandas
33+
fail_ci_if_error: false
34+
if: failure()

‎.github/actions/setup-python/action.yml‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
name: Setup Python and install requirements
2+
inputs:
3+
python-version:
4+
required: true
5+
architecture:
6+
default: x64
7+
runs:
8+
using: composite
9+
steps:
10+
# TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941
11+
- name: Create temporary requirements.txt
12+
run: |
13+
# Drop cache at least once per month
14+
month_today="$(date '+%Y-%m')"
15+
cat > requirements.txt <<EOF
16+
# $month_today
17+
18+
# Python deps
19+
pip
20+
setuptools<60.0.0
21+
wheel
22+
23+
# Pandas deps
24+
# GH 39416
25+
numpy
26+
cython
27+
python-dateutil
28+
pytz
29+
30+
# Test deps
31+
git+https://github.com/nedbat/coveragepy.git
32+
hypothesis
33+
pytest>=6.2.5
34+
pytest-xdist
35+
pytest-cov
36+
pytest-asyncio>=0.17
37+
EOF
38+
shell: bash -el {0}
39+
40+
- name: Set up Python
41+
uses: actions/setup-python@v3
42+
with:
43+
python-version: ${{ inputs.python-version }}
44+
architecture: ${{ inputs.architecture }}
45+
cache: pip
46+
47+
- name: Fix $PATH on macOS
48+
run: |
49+
# On macOS, the Python version we installed above is too late in $PATH
50+
# to be effective if using "bash -l" (which we need for code that works
51+
# with Conda envs).
52+
cat >> ~/.bash_profile <<EOF
53+
export PATH="\$(echo "\$PATH" | grep -Eio '[^:]+hostedtoolcache/python[^:]+bin'):\$PATH" \
54+
EOF
55+
shell: bash -el {0}
56+
if: ${{ runner.os == 'macOS' }}
57+
58+
- name: Install dependencies
59+
run: |
60+
cat requirements.txt
61+
# TODO https://github.com/numpy/numpy/issues/21196
62+
bits32=$(python -c 'import sys; print(int(sys.maxsize > 2**32))')
63+
NPY_DISABLE_SVML=$bits32 pip install -r requirements.txt
64+
pip list
65+
shell: bash -el {0}

‎.github/actions/setup-sccache/action.yml‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
name: Setup sccache
2+
inputs:
3+
extra-cache-key:
4+
required: false
5+
default: ''
6+
runs:
7+
using: composite
8+
steps:
9+
- name: Get Date
10+
id: get-date
11+
run: echo "::set-output name=today::$(/bin/date -u '+%Y%m%d')"
12+
shell: bash
13+
14+
- name: Fix Windows temporary directory
15+
# On Windows, for some reason the default temporary directory provided to sccache
16+
# may become read-only at some point. Work around by having a private tempdir.
17+
id: mktemp
18+
run: echo "::set-output name=tmpdir::$(mktemp -d)"
19+
shell: bash
20+
if: ${{ runner.os == 'Windows' }}
21+
22+
- name: Setup sccache
23+
uses: hendrikmuhs/[email protected]
24+
with:
25+
variant: sccache
26+
key: ${{ runner.os }}--${{ runner.arch }}--${{ github.workflow }}--${{ steps.get-date.outputs.today }}--${{ inputs.extra-cache-key }}
27+
env:
28+
TMP: "${{ steps.mktemp.outputs.tmpdir }}"

‎.github/actions/setup/action.yml‎

Lines changed: 44 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,50 @@
11
name: Set up pandas
22
description: Runs all the setup steps required to have a built pandas ready to use
3+
inputs:
4+
environment-file:
5+
default: environment.yml
6+
pyarrow-version:
7+
required: false
8+
is-pypy:
9+
default: false
10+
environment-name:
11+
default: pandas-dev
12+
python-version:
13+
required: false
314
runs:
415
using: composite
516
steps:
6-
- name: Setting conda path
7-
run: echo "${HOME}/miniconda3/bin" >> $GITHUB_PATH
8-
shell: bash -l {0}
17+
- name: Set Arrow version in ${{ inputs.environment-file }} to ${{ inputs.pyarrow-version }}
18+
run: |
19+
grep -q ' - pyarrow' ${{ inputs.environment-file }}
20+
sed -i"" -e "s/ - pyarrow/ - pyarrow=${{ inputs.pyarrow-version }}/" ${{ inputs.environment-file }}
21+
cat ${{ inputs.environment-file }}
22+
shell: bash
23+
if: ${{ inputs.pyarrow-version }}
924

10-
- name: Setup environment and build pandas
11-
run: ci/setup_env.sh
12-
shell: bash -l {0}
25+
- name: Pin setuptools (GH#44980)
26+
run: |
27+
echo ' - setuptools <60' >> ${{ inputs.environment-file }}
28+
shell: bash
29+
30+
- name: Install ${{ inputs.environment-file }} (Python ${{ inputs.python-version }})
31+
uses: mamba-org/provision-with-micromamba@main
32+
with:
33+
environment-name: ${{ inputs.environment-name }}
34+
environment-file: ${{ inputs.environment-file }}
35+
extra-specs: |
36+
${{ inputs.python-version && format('python={0}', inputs.python-version) }}
37+
cache-env: true
38+
if: ${{ inputs.is-pypy == 'false' }} # No pypy3.8 support
39+
40+
- name: Setup PyPy
41+
uses: actions/setup-python@v3
42+
with:
43+
python-version: "pypy-3.8"
44+
if: ${{ inputs.is-pypy == 'true' }}
45+
46+
- name: Setup PyPy dependencies
47+
# TODO: re-enable cov, its slowing the tests down though
48+
run: pip install Cython numpy python-dateutil pytz pytest>=6.0 pytest-xdist>=1.31.0 hypothesis>=5.5.3 pytest-asyncio>=0.17
49+
shell: bash
50+
if: ${{ inputs.is-pypy == 'true' }}

‎.github/workflows/asv-bot.yml‎

Lines changed: 7 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,7 @@ on:
66
- created
77

88
env:
9-
ENV_FILE: environment.yml
10-
COMMENT: ${{github.event.comment.body}}
9+
COMMENT: ${{ github.event.comment.body }}
1110

1211
jobs:
1312
autotune:
@@ -17,7 +16,7 @@ jobs:
1716
runs-on: ubuntu-latest
1817
defaults:
1918
run:
20-
shell: bash -l {0}
19+
shell: bash -el {0}
2120

2221
concurrency:
2322
# Set concurrency to prevent abuse(full runs are ~5.5 hours !!!)
@@ -29,24 +28,14 @@ jobs:
2928

3029
steps:
3130
- name: Checkout
32-
uses: actions/checkout@v2
31+
uses: actions/checkout@v3
3332
with:
3433
fetch-depth: 0
3534

36-
- name: Cache conda
37-
uses: actions/cache@v2
38-
with:
39-
path: ~/conda_pkgs_dir
40-
key: ${{ runner.os }}-conda-${{ hashFiles('${{ env.ENV_FILE }}') }}
41-
42-
# Although asv sets up its own env, deps are still needed
43-
# during discovery process
44-
- uses: conda-incubator/setup-miniconda@v2
45-
with:
46-
activate-environment: pandas-dev
47-
channel-priority: strict
48-
environment-file: ${{ env.ENV_FILE }}
49-
use-only-tar-bz2: true
35+
# Although asv sets up its own env, deps are still needed
36+
# during discovery process
37+
- name: Set up Conda
38+
uses: ./.github/actions/setup
5039

5140
- name: Run benchmarks
5241
id: bench

‎.github/workflows/autoupdate-pre-commit-config.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ jobs:
1212
runs-on: ubuntu-latest
1313
steps:
1414
- name: Set up Python
15-
uses: actions/setup-python@v2
15+
uses: actions/setup-python@v3
1616
- name: Cache multiple paths
1717
uses: actions/cache@v2
1818
with:

0 commit comments

Comments
 (0)