diff --git a/.cirrus.yml b/.cirrus.yml new file mode 100644 index 0000000000..d4aedd3955 --- /dev/null +++ b/.cirrus.yml @@ -0,0 +1,227 @@ +# Reference: +# - https://cirrus-ci.org/guide/writing-tasks/ +# - https://cirrus-ci.org/guide/tips-and-tricks/#sharing-configuration-between-tasks +# - https://cirrus-ci.org/guide/linux/ +# - https://cirrus-ci.org/guide/macOS/ +# - https://cirrus-ci.org/guide/windows/ +# - https://hub.docker.com/_/gcc/ +# - https://hub.docker.com/_/python/ + +# +# Global defaults. +# +container: + image: python:3.8 + cpu: 2 + memory: 4G + + +env: + # Maximum cache period (in weeks) before forcing a new cache upload. + CACHE_PERIOD: "2" + # Increment the build number to force new cartopy cache upload. + CARTOPY_CACHE_BUILD: "0" + # Increment the build number to force new conda cache upload. + CONDA_CACHE_BUILD: "0" + # Increment the build number to force new nox cache upload. + NOX_CACHE_BUILD: "0" + # Increment the build number to force new pip cache upload. + PIP_CACHE_BUILD: "0" + # Pip package to be upgraded/installed. + PIP_CACHE_PACKAGES: "pip setuptools wheel nox" + # Git commit hash for iris test data. + IRIS_TEST_DATA_REF: "fffb9b14b9cb472c5eb2ebb7fd19acb7f6414a30" + # Base directory for the iris-test-data. + IRIS_TEST_DATA_DIR: ${HOME}/iris-test-data + + +# +# Linting +# +lint_task: + auto_cancellation: true + name: "${CIRRUS_OS}: flake8 and black" + pip_cache: + folder: ~/.cache/pip + fingerprint_script: + - echo "${CIRRUS_TASK_NAME}" + - echo "$(date +%Y).$(($(date +%U) / ${CACHE_PERIOD})):${PIP_CACHE_BUILD} ${PIP_CACHE_PACKAGES}" + lint_script: + - pip list + - python -m pip install --retries 3 --upgrade ${PIP_CACHE_PACKAGES} + - pip list + - nox --session flake8 + - nox --session black + + +# +# YAML alias for common linux test infra-structure. +# +linux_task_template: &LINUX_TASK_TEMPLATE + auto_cancellation: true + env: + IRIS_REPO_DIR: ${CIRRUS_WORKING_DIR} + PATH: ${HOME}/miniconda/bin:${PATH} + SITE_CFG: ${CIRRUS_WORKING_DIR}/lib/iris/etc/site.cfg + conda_cache: + folder: ${HOME}/miniconda + fingerprint_script: + - wget --quiet https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh + - echo "${CIRRUS_OS} $(sha256sum miniconda.sh)" + - echo "$(date +%Y).$(($(date +%U) / ${CACHE_PERIOD})):${CONDA_CACHE_BUILD}" + populate_script: + - bash miniconda.sh -b -p ${HOME}/miniconda + - conda config --set always_yes yes --set changeps1 no + - conda config --set show_channel_urls True + - conda config --add channels conda-forge + - conda update --quiet --name base conda + - conda install --quiet --name base nox pip + cartopy_cache: + folder: ${HOME}/.local/share/cartopy + fingerprint_script: + - echo "${CIRRUS_OS}" + - echo "$(date +%Y).$(($(date +%U) / ${CACHE_PERIOD})):${CARTOPY_CACHE_BUILD}" + nox_cache: + folder: ${CIRRUS_WORKING_DIR}/.nox + fingerprint_script: + - echo "${CIRRUS_TASK_NAME}" + - echo "$(date +%Y).$(($(date +%U) / ${CACHE_PERIOD})):${NOX_CACHE_BUILD}" + - sha256sum ${CIRRUS_WORKING_DIR}/requirements/ci/py$(echo ${PY_VER} | tr -d ".").yml + + +# +# Testing Minimal (Linux) +# +linux_minimal_task: + matrix: + env: + PY_VER: 3.6 + env: + PY_VER: 3.7 + name: "${CIRRUS_OS}: py${PY_VER} tests (minimal)" + container: + image: gcc:latest + cpu: 2 + memory: 4G + << : *LINUX_TASK_TEMPLATE + tests_script: + - echo "[Resources]" > ${SITE_CFG} + - echo "doc_dir = ${CIRRUS_WORKING_DIR}/docs/iris" >> ${SITE_CFG} + - nox --session tests + + +# +# Testing Full (Linux) +# +linux_task: + matrix: + env: + PY_VER: 3.6 + env: + PY_VER: 3.7 + name: "${CIRRUS_OS}: py${PY_VER} tests (full)" + container: + image: gcc:latest + cpu: 6 + memory: 8G + data_cache: + folder: ${IRIS_TEST_DATA_DIR} + fingerprint_script: + - echo "${IRIS_TEST_DATA_REF}" + populate_script: + - wget --quiet https://github.com/SciTools/iris-test-data/archive/${IRIS_TEST_DATA_REF}.zip -O iris-test-data.zip + - unzip -q iris-test-data.zip + - mv iris-test-data-$(echo "${IRIS_TEST_DATA_REF}" | sed "s/^v//") ${IRIS_TEST_DATA_DIR} + << : *LINUX_TASK_TEMPLATE + tests_script: + - echo "[Resources]" > ${SITE_CFG} + - echo "test_data_dir = ${IRIS_TEST_DATA_DIR}/test_data" >> ${SITE_CFG} + - echo "doc_dir = ${CIRRUS_WORKING_DIR}/docs/iris" >> ${SITE_CFG} + - nox --session tests + + +# +# Testing Documentation Gallery (Linux) +# +gallery_task: + matrix: + env: + PY_VER: 3.6 + env: + PY_VER: 3.7 + name: "${CIRRUS_OS}: py${PY_VER} doc tests (gallery)" + container: + image: gcc:latest + cpu: 2 + memory: 4G + data_cache: + folder: ${IRIS_TEST_DATA_DIR} + fingerprint_script: + - echo "${IRIS_TEST_DATA_REF}" + populate_script: + - wget --quiet https://github.com/SciTools/iris-test-data/archive/${IRIS_TEST_DATA_REF}.zip -O iris-test-data.zip + - unzip -q iris-test-data.zip + - mv iris-test-data-$(echo "${IRIS_TEST_DATA_REF}" | sed "s/^v//") ${IRIS_TEST_DATA_DIR} + << : *LINUX_TASK_TEMPLATE + tests_script: + - echo "[Resources]" > ${SITE_CFG} + - echo "test_data_dir = ${IRIS_TEST_DATA_DIR}/test_data" >> ${SITE_CFG} + - echo "doc_dir = ${CIRRUS_WORKING_DIR}/docs/iris" >> ${SITE_CFG} + - nox --session gallery + + +# +# Testing Documentation (Linux) +# +doctest_task: + matrix: + env: + PY_VER: 3.7 + name: "${CIRRUS_OS}: py${PY_VER} doc tests" + container: + image: gcc:latest + cpu: 2 + memory: 4G + env: + MPL_RC_DIR: ${HOME}/.config/matplotlib + MPL_RC_FILE: ${HOME}/.config/matplotlib/matplotlibrc + data_cache: + folder: ${IRIS_TEST_DATA_DIR} + fingerprint_script: + - echo "${IRIS_TEST_DATA_REF}" + populate_script: + - wget --quiet https://github.com/SciTools/iris-test-data/archive/${IRIS_TEST_DATA_REF}.zip -O iris-test-data.zip + - unzip -q iris-test-data.zip + - mv iris-test-data-$(echo "${IRIS_TEST_DATA_REF}" | sed "s/^v//") ${IRIS_TEST_DATA_DIR} + << : *LINUX_TASK_TEMPLATE + tests_script: + - echo "[Resources]" > ${SITE_CFG} + - echo "test_data_dir = ${IRIS_TEST_DATA_DIR}/test_data" >> ${SITE_CFG} + - echo "doc_dir = ${CIRRUS_WORKING_DIR}/docs/iris" >> ${SITE_CFG} + - mkdir -p ${MPL_RC_DIR} + - echo "backend : agg" > ${MPL_RC_FILE} + - echo "image.cmap : viridis" >> ${MPL_RC_FILE} + - nox --session doctest + + +# +# Testing Documentation Link Check (Linux) +# +link_task: + matrix: + env: + PY_VER: 3.7 + name: "${CIRRUS_OS}: py${PY_VER} doc link check" + container: + image: gcc:latest + cpu: 2 + memory: 4G + env: + MPL_RC_DIR: ${HOME}/.config/matplotlib + MPL_RC_FILE: ${HOME}/.config/matplotlib/matplotlibrc + << : *LINUX_TASK_TEMPLATE + tests_script: + - mkdir -p ${MPL_RC_DIR} + - echo "backend : agg" > ${MPL_RC_FILE} + - echo "image.cmap : viridis" >> ${MPL_RC_FILE} + - nox --session linkcheck diff --git a/.gitignore b/.gitignore index d589c306fe..618913e7ec 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ var sdist develop-eggs .installed.cfg +.nox # Installer logs pip-log.txt diff --git a/.stickler.yml b/.stickler.yml deleted file mode 100644 index 6edee0f6a5..0000000000 --- a/.stickler.yml +++ /dev/null @@ -1,4 +0,0 @@ -linters: - flake8: - python: 3 - config: ./.flake8 \ No newline at end of file diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index ab1accab4a..0000000000 --- a/.travis.yml +++ /dev/null @@ -1,169 +0,0 @@ -# Please update the test data git references below if appropriate. -# -# Note: Contrary to the travis documentation, -# http://about.travis-ci.org/docs/user/languages/python/#Travis-CI-Uses-Isolated-virtualenvs -# we will use conda to give us a much faster setup time. - -language: minimal -dist: xenial - -env: - global: - # The decryption key for the encrypted .github/deploy_key.scitools-docs.enc. - - secure: "N9/qBUT5CqfC7KQBDy5mIWZcGNuUJk3e/qmKJpotWYV+zwOI4GghJsRce6nFnlRiwl65l5oBEcvf3+sBvUfbZqh7U0MdHpw2tHhr2FSCmMB3bkvARZblh9M37f4da9G9VmRkqnyBM5G5TImXtoq4dusvNWKvLW0qETciaipq7ws=" - matrix: - - PYTHON_VERSION='36' TEST_TARGET='default' TEST_MINIMAL=true - - PYTHON_VERSION='36' TEST_TARGET='default' TEST_BLACK=true - - PYTHON_VERSION='36' TEST_TARGET='gallery' - - PYTHON_VERSION='37' TEST_TARGET='default' TEST_MINIMAL=true - - PYTHON_VERSION='37' TEST_TARGET='default' TEST_BLACK=true - - PYTHON_VERSION='37' TEST_TARGET='gallery' - - PYTHON_VERSION='37' TEST_TARGET='doctest' PUSH_BUILT_DOCS=true - - PYTHON_VERSION='37' TEST_TARGET='linkcheck' - # TODO: Dependencies for sphinxcontrib-spelling to be in place before this - # spelling code block is enabled - #- PYTHON_VERSION='37' TEST_TARGET='spelling' - -git: - # We need a deep clone so that we can compute the age of the files using their git history. - depth: 10000 - -install: - - > - export IRIS_TEST_DATA_REF="fffb9b14b9cb472c5eb2ebb7fd19acb7f6414a30"; - export IRIS_TEST_DATA_SUFFIX=$(echo "${IRIS_TEST_DATA_REF}" | sed "s/^v//"); - - # Install miniconda - # ----------------- - - > - echo 'Installing miniconda'; - export CONDA_BASE="https://repo.continuum.io/miniconda/Miniconda"; - wget --quiet ${CONDA_BASE}3-latest-Linux-x86_64.sh -O miniconda.sh; - bash miniconda.sh -b -p ${HOME}/miniconda; - export PATH="${HOME}/miniconda/bin:${PATH}"; - - # Create the testing environment - # ------------------------------ - # Explicitly add defaults channel, see https://github.com/conda/conda/issues/2675 - - > - echo 'Configure conda and create an environment'; - conda config --set always_yes yes --set changeps1 no; - conda config --set show_channel_urls True; - conda config --add channels conda-forge; - conda update --quiet conda; - export ENV_NAME='iris-dev'; - ENV_FILE="requirements/ci/py${PYTHON_VERSION}.yml"; - cat ${ENV_FILE}; - conda env create --quiet --file=${ENV_FILE}; - source activate ${ENV_NAME}; - export PREFIX="${CONDA_PREFIX}"; - - # Output debug info - - > - conda list -n ${ENV_NAME}; - conda list -n ${ENV_NAME} --explicit; - conda info -a; - -# Pre-load Natural Earth data to avoid multiple, overlapping downloads. -# i.e. There should be no DownloadWarning reports in the log. - - python -c 'import cartopy; cartopy.io.shapereader.natural_earth()' - -# iris test data - - > - if [[ "${TEST_MINIMAL}" != true ]]; then - wget --quiet -O iris-test-data.zip https://github.com/SciTools/iris-test-data/archive/${IRIS_TEST_DATA_REF}.zip; - unzip -q iris-test-data.zip; - mv "iris-test-data-${IRIS_TEST_DATA_SUFFIX}" iris-test-data; - fi - -# set config paths - - > - SITE_CFG="lib/iris/etc/site.cfg"; - echo "[Resources]" > ${SITE_CFG}; - echo "test_data_dir = $(pwd)/iris-test-data/test_data" >> ${SITE_CFG}; - echo "doc_dir = $(pwd)/docs/iris" >> ${SITE_CFG}; - echo "[System]" >> ${SITE_CFG}; - echo "udunits2_path = ${PREFIX}/lib/libudunits2.so" >> ${SITE_CFG}; - - - python setup.py --quiet install - -script: - # Capture install-dir: As a test command must be last for get Travis to check - # the RC, so it's best to start each operation with an absolute cd. - - export INSTALL_DIR=$(pwd) - - - > - if [[ "${TEST_BLACK}" == 'true' ]]; then - echo $(black --version); - rm ${INSTALL_DIR}/.gitignore; - black --check ${INSTALL_DIR}; - fi - - - > - if [[ "${TEST_TARGET}" == 'default' ]]; then - export IRIS_REPO_DIR=${INSTALL_DIR}; - python -m iris.tests.runner --default-tests --system-tests; - fi - - - > - if [[ "${TEST_TARGET}" == 'gallery' ]]; then - python -m iris.tests.runner --gallery-tests; - fi - - # Build the docs. - - > - if [[ "${TEST_TARGET}" == 'doctest' ]]; then - MPL_RC_DIR="${HOME}/.config/matplotlib"; - mkdir -p ${MPL_RC_DIR}; - echo 'backend : agg' > ${MPL_RC_DIR}/matplotlibrc; - echo 'image.cmap : viridis' >> ${MPL_RC_DIR}/matplotlibrc; - cd ${INSTALL_DIR}/docs/iris; - make clean html && make doctest; - fi - - # check the links in the docs - - > - if [[ "${TEST_TARGET}" == 'linkcheck' ]]; then - MPL_RC_DIR="${HOME}/.config/matplotlib"; - mkdir -p ${MPL_RC_DIR}; - echo 'backend : agg' > ${MPL_RC_DIR}/matplotlibrc; - echo 'image.cmap : viridis' >> ${MPL_RC_DIR}/matplotlibrc; - cd ${INSTALL_DIR}/docs/iris; - make clean && make linkcheck; - fi - - # TODO: Dependencies for sphinxcontrib-spelling to be in place before this - # spelling code block is enabled - - # check the spelling in the docs - # - > - # if [[ "${TEST_TARGET}" == 'spelling' ]]; then - # MPL_RC_DIR="${HOME}/.config/matplotlib"; - # mkdir -p ${MPL_RC_DIR}; - # echo 'backend : agg' > ${MPL_RC_DIR}/matplotlibrc; - # echo 'image.cmap : viridis' >> ${MPL_RC_DIR}/matplotlibrc; - # cd ${INSTALL_DIR}/docs/iris; - # make clean && make spelling; - # fi - - # Split the organisation out of the slug. See https://stackoverflow.com/a/5257398/741316 for description. - # NOTE: a *separate* "export" command appears to be necessary here : A command of the - # form "export ORG=.." failed to define ORG for the following command (?!) - - > - ORG=$(echo ${TRAVIS_REPO_SLUG} | cut -d/ -f1); - export ORG - - - echo "Travis job context ORG=${ORG}; TRAVIS_EVENT_TYPE=${TRAVIS_EVENT_TYPE}; PUSH_BUILT_DOCS=${PUSH_BUILT_DOCS}" - - # When we merge a change to SciTools/iris, we can push docs to github pages. - # At present, only the Python 3.7 "doctest" job does this. - # Results appear at https://scitools-docs.github.io/iris/<>/index.html - - if [[ "${ORG}" == 'SciTools' && "${TRAVIS_EVENT_TYPE}" == 'push' && "${PUSH_BUILT_DOCS}" == 'true' ]]; then - cd ${INSTALL_DIR}; - conda install --quiet -n ${ENV_NAME} pip; - pip install doctr; - doctr deploy --deploy-repo SciTools-docs/iris --built-docs docs/iris/src/_build/html - --key-path .github/deploy_key.scitools-docs.enc - --no-require-master - ${TRAVIS_BRANCH:-${TRAVIS_TAG}}; - fi diff --git a/README.md b/README.md index aeadb52d93..6339491955 100644 --- a/README.md +++ b/README.md @@ -10,13 +10,12 @@

- - -Travis-CI + +Cirrus-CI - Documentation Status +Documentation Status conda-forge downloads @@ -26,9 +25,6 @@ Latest version - -Stable docs Commits since last release diff --git a/docs/iris/src/common_links.inc b/docs/iris/src/common_links.inc index 94c2f3c92b..0bc8ca60e6 100644 --- a/docs/iris/src/common_links.inc +++ b/docs/iris/src/common_links.inc @@ -25,3 +25,4 @@ .. _sphinx: https://www.sphinx-doc.org/en/master/ .. _napolean: https://sphinxcontrib-napoleon.readthedocs.io/en/latest/sphinxcontrib.napoleon.html .. _legacy documentation: https://scitools.org.uk/iris/docs/v2.4.0/ +.. _cirrus-ci: https://cirrus-ci.com/github/SciTools/iris diff --git a/docs/iris/src/conf.py b/docs/iris/src/conf.py index 9e6276f544..73b60fb982 100644 --- a/docs/iris/src/conf.py +++ b/docs/iris/src/conf.py @@ -261,13 +261,13 @@ def autolog(message): # url link checker. Some links work but report as broken, lets ignore them. # See https://www.sphinx-doc.org/en/1.2/config.html#options-for-the-linkcheck-builder linkcheck_ignore = [ - "https://github.com/SciTools/iris/commit/69597eb3d8501ff16ee3d56aef1f7b8f1c2bb316#diff-1680206bdc5cfaa83e14428f5ba0f848", - "http://www.wmo.int/pages/prog/www/DPFS/documents/485_Vol_I_en_colour.pdf", + "http://cfconventions.org", "http://code.google.com/p/msysgit/downloads/list", + "https://github.com", + "http://www.personal.psu.edu/cab38/ColorBrewer/ColorBrewer_updates.html", "http://schacon.github.com/git", - "https://github.com/SciTools/iris/pull", - "https://github.com/SciTools/iris/issue", - "http://cfconventions.org", + "http://scitools.github.com/cartopy", + "http://www.wmo.int/pages/prog/www/DPFS/documents/485_Vol_I_en_colour.pdf", ] # list of sources to exclude from the build. diff --git a/docs/iris/src/whatsnew/3.0.rst b/docs/iris/src/whatsnew/3.0.rst index 79de8b3edf..59f7ec8735 100644 --- a/docs/iris/src/whatsnew/3.0.rst +++ b/docs/iris/src/whatsnew/3.0.rst @@ -424,6 +424,13 @@ This document explains the changes made to Iris for this release * `@znicholls`_ made :func:`~iris.tests.idiff.step_over_diffs` robust to hyphens (``-``) in the input path (i.e. the ``result_dir`` argument) (:pull:`3902`). +* `@bjlittle`_ migrated the CIaaS from `travis-ci`_ to `cirrus-ci`_. (:pull:`3928`) + +* `@bjlittle`_ introduced `nox`_ as a common and easy entry-point for test automation. + It can be used both from `cirrus-ci`_ in the cloud, and locally by the developer to + run the Iris tests, the doc-tests, the gallery doc-tests, and lint Iris + with `flake8`_ and `black`_. (:pull:`3928`) + .. _Read the Docs: https://scitools-iris.readthedocs.io/en/latest/ .. _Matplotlib: https://matplotlib.org/ .. _CF units rules: https://cfconventions.org/Data/cf-conventions/cf-conventions-1.8/cf-conventions.html#units @@ -469,3 +476,5 @@ This document explains the changes made to Iris for this release .. _GitHub: https://github.com/SciTools/iris/issues/new/choose .. _readthedocs: https://readthedocs.org/ .. _CF Conventions and Metadata: https://cfconventions.org/ +.. _flake8: https://flake8.pycqa.org/en/stable/ +.. _nox: https://nox.thea.codes/en/stable/ diff --git a/lib/iris/tests/results/analysis/sqrt.cml b/lib/iris/tests/results/analysis/sqrt.cml index 0dd0fe20b3..c6b9b88e9a 100644 --- a/lib/iris/tests/results/analysis/sqrt.cml +++ b/lib/iris/tests/results/analysis/sqrt.cml @@ -1,6 +1,6 @@ - + @@ -39,6 +39,6 @@ - + diff --git a/lib/iris/tests/test_basic_maths.py b/lib/iris/tests/test_basic_maths.py index a559ee0e8a..4b3cde95e4 100644 --- a/lib/iris/tests/test_basic_maths.py +++ b/lib/iris/tests/test_basic_maths.py @@ -537,11 +537,12 @@ def test_multiplication_not_in_place(self): class TestExponentiate(tests.IrisTest): def setUp(self): self.cube = iris.tests.stock.global_pp() - self.cube.data = self.cube.data - 260 + # Increase dtype from float32 to float64 in order + # to avoid dtype quantization errors during maths. + self.cube.data = self.cube.data.astype(np.float64) - 260.0 def test_exponentiate(self): a = self.cube - a.data = a.data.astype(np.float64) e = pow(a, 4) self.assertCMLApproxData(e, ("analysis", "exponentiate.cml")) @@ -553,8 +554,8 @@ def test_square_root(self): e = a ** 0.5 - self.assertCML(e, ("analysis", "sqrt.cml")) self.assertArrayEqual(e.data, a.data ** 0.5) + self.assertCML(e, ("analysis", "sqrt.cml")) self.assertRaises(ValueError, iris.analysis.maths.exponentiate, a, 0.3) def test_type_error(self): diff --git a/lib/iris/tests/test_coding_standards.py b/lib/iris/tests/test_coding_standards.py index 00ce7b7d44..79dff535eb 100644 --- a/lib/iris/tests/test_coding_standards.py +++ b/lib/iris/tests/test_coding_standards.py @@ -102,6 +102,7 @@ def last_change_by_fname(): def test_license_headers(self): exclude_patterns = ( "setup.py", + "noxfile.py", "build/*", "dist/*", "docs/iris/gallery_code/*/*.py", diff --git a/lib/iris/tests/test_netcdf.py b/lib/iris/tests/test_netcdf.py index 75266ff3fe..2d1b4a53d5 100644 --- a/lib/iris/tests/test_netcdf.py +++ b/lib/iris/tests/test_netcdf.py @@ -543,17 +543,20 @@ def test_noexist_directory(self): pass def test_bad_permissions(self): - # Non-exhaustive check that wrong permissions results in a suitable - # exception being raised. - dir_name = tempfile.mkdtemp() - fnme = os.path.join(dir_name, "tmp.nc") - try: - os.chmod(dir_name, stat.S_IREAD) - with self.assertRaises(IOError): - iris.fileformats.netcdf.Saver(fnme, "NETCDF4") - self.assertFalse(os.path.exists(fnme)) - finally: - os.rmdir(dir_name) + # Skip this test for the root user. This is applicable to + # running within a Docker container and/or CIaaS hosted testing. + if os.getuid(): + # Non-exhaustive check that wrong permissions results in a suitable + # exception being raised. + dir_name = tempfile.mkdtemp() + fname = os.path.join(dir_name, "tmp.nc") + try: + os.chmod(dir_name, stat.S_IREAD) + with self.assertRaises(PermissionError): + iris.fileformats.netcdf.Saver(fname, "NETCDF4") + self.assertFalse(os.path.exists(fname)) + finally: + shutil.rmtree(dir_name) @tests.skip_data diff --git a/noxfile.py b/noxfile.py new file mode 100644 index 0000000000..cd97e8ef8b --- /dev/null +++ b/noxfile.py @@ -0,0 +1,312 @@ +""" +Perform test automation with nox. + +For further details, see https://nox.thea.codes/en/stable/# + +""" + +import hashlib +import os +from pathlib import Path + +import nox + + +#: Default to reusing any pre-existing nox environments. +nox.options.reuse_existing_virtualenvs = True + +#: Name of the package to test. +PACKAGE = str("lib" / Path("iris")) + +#: Cirrus-CI environment variable hook. +PY_VER = os.environ.get("PY_VER", "3.7") + +#: Default cartopy cache directory. +CARTOPY_CACHE_DIR = os.environ.get("HOME") / Path(".local/share/cartopy") + + +def venv_cached(session): + """ + Determine whether the nox session environment has been cached. + + Parameters + ---------- + session: object + A `nox.sessions.Session` object. + + Returns + ------- + bool + Whether the session has been cached. + + """ + result = False + yml = Path(f"requirements/ci/py{PY_VER.replace('.', '')}.yml") + tmp_dir = Path(session.create_tmp()) + cache = tmp_dir / yml.name + if cache.is_file(): + with open(yml, "rb") as fi: + expected = hashlib.sha256(fi.read()).hexdigest() + with open(cache, "r") as fi: + actual = fi.read() + result = actual == expected + return result + + +def cache_venv(session): + """ + Cache the nox session environment. + + This consists of saving a hexdigest (sha256) of the associated + conda requirements YAML file. + + Parameters + ---------- + session: object + A `nox.sessions.Session` object. + + """ + yml = Path(f"requirements/ci/py{PY_VER.replace('.', '')}.yml") + with open(yml, "rb") as fi: + hexdigest = hashlib.sha256(fi.read()).hexdigest() + tmp_dir = Path(session.create_tmp()) + cache = tmp_dir / yml.name + with open(cache, "w") as fo: + fo.write(hexdigest) + + +def cache_cartopy(session): + """ + Determine whether to cache the cartopy natural earth shapefiles. + + Parameters + ---------- + session: object + A `nox.sessions.Session` object. + + """ + if not CARTOPY_CACHE_DIR.is_dir(): + session.run( + "python", + "-c", + "import cartopy; cartopy.io.shapereader.natural_earth()", + ) + + +@nox.session +def flake8(session): + """ + Perform flake8 linting of iris. + + Parameters + ---------- + session: object + A `nox.sessions.Session` object. + + """ + # Pip install the session requirements. + session.install("flake8") + # Execute the flake8 linter on the package. + session.run("flake8", PACKAGE) + # Execute the flake8 linter on this file. + session.run("flake8", __file__) + + +@nox.session +def black(session): + """ + Perform black format checking of iris. + + Parameters + ---------- + session: object + A `nox.sessions.Session` object. + + """ + # Pip install the session requirements. + session.install("black==20.8b1") + # Execute the black format checker on the package. + session.run("black", "--check", PACKAGE) + # Execute the black format checker on this file. + session.run("black", "--check", __file__) + + +@nox.session(python=[PY_VER], venv_backend="conda") +def tests(session): + """ + Perform iris system, integration and unit tests. + + Parameters + ---------- + session: object + A `nox.sessions.Session` object. + + Notes + ----- + See + - https://github.com/theacodes/nox/issues/346 + - https://github.com/theacodes/nox/issues/260 + + """ + if not venv_cached(session): + # Determine the conda requirements yaml file. + fname = f"requirements/ci/py{PY_VER.replace('.', '')}.yml" + # Back-door approach to force nox to use "conda env update". + command = ( + "conda", + "env", + "update", + f"--prefix={session.virtualenv.location}", + f"--file={fname}", + "--prune", + ) + session._run(*command, silent=True, external="error") + cache_venv(session) + + cache_cartopy(session) + session.run("python", "setup.py", "develop") + session.run( + "python", + "-m", + "iris.tests.runner", + "--default-tests", + "--system-tests", + ) + + +@nox.session(python=[PY_VER], venv_backend="conda") +def gallery(session): + """ + Perform iris gallery doc-tests. + + Parameters + ---------- + session: object + A `nox.sessions.Session` object. + + Notes + ----- + See + - https://github.com/theacodes/nox/issues/346 + - https://github.com/theacodes/nox/issues/260 + + """ + if not venv_cached(session): + # Determine the conda requirements yaml file. + fname = f"requirements/ci/py{PY_VER.replace('.', '')}.yml" + # Back-door approach to force nox to use "conda env update". + command = ( + "conda", + "env", + "update", + f"--prefix={session.virtualenv.location}", + f"--file={fname}", + "--prune", + ) + session._run(*command, silent=True, external="error") + cache_venv(session) + + cache_cartopy(session) + session.run("python", "setup.py", "develop") + session.run( + "python", + "-m", + "iris.tests.runner", + "--gallery-tests", + ) + + +@nox.session(python=[PY_VER], venv_backend="conda") +def doctest(session): + """ + Perform iris doc-tests. + + Parameters + ---------- + session: object + A `nox.sessions.Session` object. + + Notes + ----- + See + - https://github.com/theacodes/nox/issues/346 + - https://github.com/theacodes/nox/issues/260 + + """ + if not venv_cached(session): + # Determine the conda requirements yaml file. + fname = f"requirements/ci/py{PY_VER.replace('.', '')}.yml" + # Back-door approach to force nox to use "conda env update". + command = ( + "conda", + "env", + "update", + f"--prefix={session.virtualenv.location}", + f"--file={fname}", + "--prune", + ) + session._run(*command, silent=True, external="error") + cache_venv(session) + + cache_cartopy(session) + session.run("python", "setup.py", "develop") + session.cd("docs/iris") + session.run( + "make", + "clean", + "html", + external=True, + ) + session.run( + "make", + "doctest", + external=True, + ) + + +@nox.session(python=[PY_VER], venv_backend="conda") +def linkcheck(session): + """ + Perform iris doc link check. + + Parameters + ---------- + session: object + A `nox.sessions.Session` object. + + Notes + ----- + See + - https://github.com/theacodes/nox/issues/346 + - https://github.com/theacodes/nox/issues/260 + + """ + if not venv_cached(session): + # Determine the conda requirements yaml file. + fname = f"requirements/ci/py{PY_VER.replace('.', '')}.yml" + # Back-door approach to force nox to use "conda env update". + command = ( + "conda", + "env", + "update", + f"--prefix={session.virtualenv.location}", + f"--file={fname}", + "--prune", + ) + session._run(*command, silent=True, external="error") + cache_venv(session) + + cache_cartopy(session) + session.run("python", "setup.py", "develop") + session.cd("docs/iris") + session.run( + "make", + "clean", + "html", + external=True, + ) + session.run( + "make", + "linkcheck", + external=True, + ) diff --git a/requirements/ci/py36.yml b/requirements/ci/py36.yml index 3b3328cf1f..2b40fbad4e 100644 --- a/requirements/ci/py36.yml +++ b/requirements/ci/py36.yml @@ -13,7 +13,7 @@ dependencies: # Core dependencies. - cartopy>=0.18 - cf-units>=2 - - cftime<1.3.0 + - cftime<1.3.0 - dask>=2 - matplotlib - netcdf4 @@ -35,6 +35,7 @@ dependencies: - asv - black=20.8b1 - filelock + - flake8 - imagehash>=4.0 - nose - pillow<7 diff --git a/requirements/ci/py37.yml b/requirements/ci/py37.yml index 8817f575b7..0f01f0ef75 100644 --- a/requirements/ci/py37.yml +++ b/requirements/ci/py37.yml @@ -35,6 +35,7 @@ dependencies: - asv - black=20.8b1 - filelock + - flake8 - imagehash>=4.0 - nose - pillow<7 diff --git a/requirements/core.txt b/requirements/core.txt index 0b59c573ec..9e0c4fb1bb 100644 --- a/requirements/core.txt +++ b/requirements/core.txt @@ -2,7 +2,7 @@ cartopy>=0.18 cf-units>=2 -cftime +cftime<1.3.0 dask[array]>=2 matplotlib netcdf4 diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index a87902cbfd..0000000000 --- a/setup.cfg +++ /dev/null @@ -1,44 +0,0 @@ -[flake8] -ignore = E402,\ # Due to conditional imports - E226 # Due to whitespace around operators (e.g. 2*x + 3) -exclude = */iris/std_names.py,\ - */iris/fileformats/cf.py,\ - */iris/fileformats/dot.py,\ - */iris/fileformats/pp_load_rules.py,\ - */iris/fileformats/rules.py,\ - */iris/fileformats/um_cf_map.py,\ - */iris/fileformats/_pyke_rules/compiled_krb/*,\ - */iris/io/__init__.py,\ - */iris/io/format_picker.py,\ - */iris/tests/__init__.py,\ - */iris/tests/pp.py,\ - */iris/tests/system_test.py,\ - */iris/tests/test_analysis.py,\ - */iris/tests/test_analysis_calculus.py,\ - */iris/tests/test_basic_maths.py,\ - */iris/tests/test_cartography.py,\ - */iris/tests/test_cdm.py,\ - */iris/tests/test_cell.py,\ - */iris/tests/test_cf.py,\ - */iris/tests/test_constraints.py,\ - */iris/tests/test_coord_api.py,\ - */iris/tests/test_coord_categorisation.py,\ - */iris/tests/test_coordsystem.py,\ - */iris/tests/test_cube_to_pp.py,\ - */iris/tests/test_file_load.py,\ - */iris/tests/test_file_save.py,\ - */iris/tests/test_hybrid.py,\ - */iris/tests/test_intersect.py,\ - */iris/tests/test_io_init.py,\ - */iris/tests/test_iterate.py,\ - */iris/tests/test_load.py,\ - */iris/tests/test_merge.py,\ - */iris/tests/test_pp_cf.py,\ - */iris/tests/test_pp_module.py,\ - */iris/tests/test_pp_stash.py,\ - */iris/tests/test_pp_to_cube.py,\ - */iris/tests/test_quickplot.py,\ - */iris/tests/test_std_names.py,\ - */iris/tests/test_uri_callback.py,\ - */iris/tests/test_util.py -