diff --git a/.readthedocs.yml b/.readthedocs.yml index 55c24eaddc..bfc8cfa72b 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -4,7 +4,7 @@ build: image: latest conda: - environment: ci/requirements/readthedocs.yml + environment: requirements/ci/readthedocs.yml sphinx: configuration: docs/iris/src/conf.py diff --git a/.travis.yml b/.travis.yml index 5015ac153e..ab1accab4a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,6 @@ # http://about.travis-ci.org/docs/user/languages/python/#Travis-CI-Uses-Isolated-virtualenvs # we will use conda to give us a much faster setup time. - language: minimal dist: xenial @@ -13,17 +12,17 @@ env: # The decryption key for the encrypted .github/deploy_key.scitools-docs.enc. - secure: "N9/qBUT5CqfC7KQBDy5mIWZcGNuUJk3e/qmKJpotWYV+zwOI4GghJsRce6nFnlRiwl65l5oBEcvf3+sBvUfbZqh7U0MdHpw2tHhr2FSCmMB3bkvARZblh9M37f4da9G9VmRkqnyBM5G5TImXtoq4dusvNWKvLW0qETciaipq7ws=" matrix: - - PYTHON_VERSION=3.6 TEST_TARGET=default TEST_MINIMAL=true - - PYTHON_VERSION=3.6 TEST_TARGET=default TEST_BLACK=true - - PYTHON_VERSION=3.6 TEST_TARGET=gallery - - PYTHON_VERSION=3.7 TEST_TARGET=default TEST_MINIMAL=true - - PYTHON_VERSION=3.7 TEST_TARGET=default TEST_BLACK=true - - PYTHON_VERSION=3.7 TEST_TARGET=gallery - - PYTHON_VERSION=3.7 TEST_TARGET=doctest PUSH_BUILT_DOCS=true - - PYTHON_VERSION=3.7 TEST_TARGET=linkcheck + - PYTHON_VERSION='36' TEST_TARGET='default' TEST_MINIMAL=true + - PYTHON_VERSION='36' TEST_TARGET='default' TEST_BLACK=true + - PYTHON_VERSION='36' TEST_TARGET='gallery' + - PYTHON_VERSION='37' TEST_TARGET='default' TEST_MINIMAL=true + - PYTHON_VERSION='37' TEST_TARGET='default' TEST_BLACK=true + - PYTHON_VERSION='37' TEST_TARGET='gallery' + - PYTHON_VERSION='37' TEST_TARGET='doctest' PUSH_BUILT_DOCS=true + - PYTHON_VERSION='37' TEST_TARGET='linkcheck' # TODO: Dependencies for sphinxcontrib-spelling to be in place before this # spelling code block is enabled - #- PYTHON_VERSION=3.7 TEST_TARGET=spelling + #- PYTHON_VERSION='37' TEST_TARGET='spelling' git: # We need a deep clone so that we can compute the age of the files using their git history. @@ -43,8 +42,8 @@ install: bash miniconda.sh -b -p ${HOME}/miniconda; export PATH="${HOME}/miniconda/bin:${PATH}"; - # Create the basic testing environment - # ------------------------------------ + # Create the testing environment + # ------------------------------ # Explicitly add defaults channel, see https://github.com/conda/conda/issues/2675 - > echo 'Configure conda and create an environment'; @@ -52,27 +51,12 @@ install: conda config --set show_channel_urls True; conda config --add channels conda-forge; conda update --quiet conda; - export ENV_NAME='test-environment'; - conda create --quiet -n ${ENV_NAME} python=${PYTHON_VERSION} pip; + export ENV_NAME='iris-dev'; + ENV_FILE="requirements/ci/py${PYTHON_VERSION}.yml"; + cat ${ENV_FILE}; + conda env create --quiet --file=${ENV_FILE}; source activate ${ENV_NAME}; - - # Customise the testing environment - # --------------------------------- - - > - echo 'Install Iris dependencies'; - CONDA_REQS_GROUPS="test"; - if [[ "${TEST_MINIMAL}" != true ]]; then - CONDA_REQS_GROUPS="${CONDA_REQS_GROUPS} all"; - fi; - if [[ "${TEST_TARGET}" == 'doctest' || "${TEST_TARGET}" == 'linkcheck' ]]; then - CONDA_REQS_GROUPS="${CONDA_REQS_GROUPS} docs"; - fi; - CONDA_REQS_FILE="conda-requirements.txt"; - python requirements/gen_conda_requirements.py --groups ${CONDA_REQS_GROUPS} > ${CONDA_REQS_FILE}; - cat ${CONDA_REQS_FILE}; - conda install --quiet -n ${ENV_NAME} --file ${CONDA_REQS_FILE}; - - - export PREFIX="${HOME}/miniconda/envs/${ENV_NAME}" + export PREFIX="${CONDA_PREFIX}"; # Output debug info - > @@ -176,6 +160,7 @@ script: # Results appear at https://scitools-docs.github.io/iris/<>/index.html - if [[ "${ORG}" == 'SciTools' && "${TRAVIS_EVENT_TYPE}" == 'push' && "${PUSH_BUILT_DOCS}" == 'true' ]]; then cd ${INSTALL_DIR}; + conda install --quiet -n ${ENV_NAME} pip; pip install doctr; doctr deploy --deploy-repo SciTools-docs/iris --built-docs docs/iris/src/_build/html --key-path .github/deploy_key.scitools-docs.enc diff --git a/ci/requirements/readthedocs.yml b/ci/requirements/readthedocs.yml deleted file mode 100644 index 121f8a6722..0000000000 --- a/ci/requirements/readthedocs.yml +++ /dev/null @@ -1,60 +0,0 @@ -name: iris-docs - -channels: - - conda-forge - -dependencies: -# Dependencies necessary to run setup.py of iris -# ---------------------------------------------- - - setuptools - - pyke - -# Absolute minimal dependencies for iris -# -------------------------------------- - -# Without these, iris won't even import. - - - cartopy>=0.18 - - cf-units>=2 - - cftime - - dask>=2 - - matplotlib - - netcdf4 - - numpy>=1.14 - - scipy - - python-xxhash - -# Dependencies needed to run the iris tests -#------------------------------------------ - - - black=19.10b0 - - filelock - - pillow<7 - - imagehash>=4.0 - - nose - - pre-commit - - requests - - asv - -# Dependencies for a feature complete installation -# ------------------------------------------------ - -# esmpy regridding not available through pip. - - esmpy>=7.0 -#gdal : under review -- not tested at present - - mo_pack - - nc-time-axis - - pandas - - python-stratify - - pyugrid - - - graphviz - -# Iris sample data is not available through pip. It can be installed from -# https://github.com/SciTools/iris-sample-data/archive/master.zip - - iris-sample-data - - sphinx - - sphinx_rtd_theme - - sphinx-copybutton - - sphinx-gallery - diff --git a/pyproject.toml b/pyproject.toml index bafcf2c451..353f3df53f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,3 +28,9 @@ exclude = ''' | um_cf_map.py ) ''' + +[build-system] +# Defined by PEP 518 +requires = ["scitools-pyke", "setuptools>=40.8.0", "wheel"] +# Defined by PEP 517 +build-backend = "setuptools.build_meta" diff --git a/requirements/all.txt b/requirements/all.txt index 46558e2cbd..6d7bb942cf 100644 --- a/requirements/all.txt +++ b/requirements/all.txt @@ -1,17 +1,7 @@ -# Dependencies for a feature complete installation -# ------------------------------------------------ +# Optional dependencies. -# esmpy regridding not available through pip. -#conda: esmpy>=7.0 -#gdal : under review -- not tested at present mo_pack nc-time-axis pandas -stratify #conda: python-stratify pyugrid - -#conda: graphviz - -# Iris sample data is not available through pip. It can be installed from -# https://github.com/SciTools/iris-sample-data/archive/master.zip -#conda: iris-sample-data +stratify diff --git a/requirements/ci/iris.yml b/requirements/ci/iris.yml new file mode 120000 index 0000000000..e9adb956db --- /dev/null +++ b/requirements/ci/iris.yml @@ -0,0 +1 @@ +py37.yml \ No newline at end of file diff --git a/requirements/ci/py36.yml b/requirements/ci/py36.yml new file mode 100644 index 0000000000..bca0d5af2c --- /dev/null +++ b/requirements/ci/py36.yml @@ -0,0 +1,48 @@ +name: iris-dev + +channels: + - conda-forge + +dependencies: + - python=3.6 + +# Setup dependencies. + - setuptools>=40.8.0 + - pyke + +# Core dependencies. + - cartopy>=0.18 + - cf-units>=2 + - cftime + - dask>=2 + - matplotlib + - netcdf4 + - numpy>=1.14 + - python-xxhash + - scipy + +# Optional dependencies. + - esmpy>=7.0 + - graphviz + - iris-sample-data + - mo_pack + - nc-time-axis + - pandas + - python-stratify + - pyugrid + +# Test dependencies. + - asv + - black=19.10b0 + - filelock + - imagehash>=4.0 + - nose + - pillow<7 + - pre-commit + - requests + +# Documentation dependencies. + - sphinx + - sphinx-copybutton + - sphinx-gallery + - sphinx_rtd_theme diff --git a/requirements/ci/py37.yml b/requirements/ci/py37.yml new file mode 100644 index 0000000000..61fe91b868 --- /dev/null +++ b/requirements/ci/py37.yml @@ -0,0 +1,48 @@ +name: iris-dev + +channels: + - conda-forge + +dependencies: + - python=3.7 + +# Setup dependencies. + - setuptools>=40.8.0 + - pyke + +# Core dependencies. + - cartopy>=0.18 + - cf-units>=2 + - cftime + - dask>=2 + - matplotlib + - netcdf4 + - numpy>=1.14 + - python-xxhash + - scipy + +# Optional dependencies. + - esmpy>=7.0 + - graphviz + - iris-sample-data + - mo_pack + - nc-time-axis + - pandas + - python-stratify + - pyugrid + +# Test dependencies. + - asv + - black=19.10b0 + - filelock + - imagehash>=4.0 + - nose + - pillow<7 + - pre-commit + - requests + +# Documentation dependencies. + - sphinx + - sphinx-copybutton + - sphinx-gallery + - sphinx_rtd_theme diff --git a/requirements/ci/readthedocs.yml b/requirements/ci/readthedocs.yml new file mode 120000 index 0000000000..119da0c856 --- /dev/null +++ b/requirements/ci/readthedocs.yml @@ -0,0 +1 @@ +iris.yml \ No newline at end of file diff --git a/requirements/core.txt b/requirements/core.txt index 4129187597..0b59c573ec 100644 --- a/requirements/core.txt +++ b/requirements/core.txt @@ -1,14 +1,11 @@ -# Absolute minimal dependencies for iris -# -------------------------------------- - -# Without these, iris won't even import. +# Core dependencies. cartopy>=0.18 cf-units>=2 cftime -dask[array]>=2 #conda: dask>=2 +dask[array]>=2 matplotlib netcdf4 numpy>=1.14 scipy -xxhash #conda: python-xxhash +xxhash diff --git a/requirements/docs.txt b/requirements/docs.txt index 2d2c03f688..78585630e5 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -1,4 +1,6 @@ +# Documentation dependencies. + sphinx -sphinx_rtd_theme sphinx-copybutton sphinx-gallery +sphinx_rtd_theme diff --git a/requirements/extensions.txt b/requirements/extensions.txt deleted file mode 100644 index 220fb562ec..0000000000 --- a/requirements/extensions.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Iris extensions (i.e. key tools that depend on Iris) -# ---------------------------------------------------- - -# Note: pip can handle the circularity of these extensions, but conda will -# struggle. To install these extensions, ensure iris[core] has been installed -# first. - -iris-grib #conda: diff --git a/requirements/gen_conda_requirements.py b/requirements/gen_conda_requirements.py deleted file mode 100644 index 5f46d911ac..0000000000 --- a/requirements/gen_conda_requirements.py +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright Iris contributors -# -# This file is part of Iris and is released under the LGPL license. -# See COPYING and COPYING.LESSER in the root of the repository for full -# licensing details. - -import argparse -import os.path - - -REQS_DIR = os.path.dirname(__file__) -CONDA_PATTERN = "#conda:" - - -def read_conda_reqs(fname): - lines = [] - with open(fname, "r") as fh: - for line in fh: - line = line.strip() - if CONDA_PATTERN in line: - line_start = line.index(CONDA_PATTERN) + len(CONDA_PATTERN) - line = line[line_start:].strip() - lines.append(line) - return lines - - -def compute_requirements(requirement_names=("core",)): - conda_reqs_lines = [] - - for req_name in requirement_names: - fname = os.path.join(REQS_DIR, "{}.txt".format(req_name)) - if not os.path.exists(fname): - raise RuntimeError( - "Unable to find the requirements file for {} " - "in {}".format(req_name, fname) - ) - conda_reqs_lines.extend(read_conda_reqs(fname)) - conda_reqs_lines.append("") - - return conda_reqs_lines - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--verbosity", help="increase output verbosity") - parser.add_argument( - "--groups", - nargs="*", - default=[], - help=( - "Gather requirements for these given named groups " - "(as found in the requirements/ folder)" - ), - ) - - args = parser.parse_args() - - requirement_names = args.groups - requirement_names.insert(0, "core") - requirement_names.insert(0, "setup") - - print("\n".join(compute_requirements(requirement_names))) - - -if __name__ == "__main__": - main() diff --git a/requirements/setup.txt b/requirements/setup.txt index abdf7b72f4..2e14da4905 100644 --- a/requirements/setup.txt +++ b/requirements/setup.txt @@ -1,5 +1,6 @@ # Dependencies necessary to run setup.py of iris # ---------------------------------------------- -setuptools -# pyke (not pip installable) #conda: pyke +scitools-pyke +setuptools>=40.8.0 +wheel diff --git a/requirements/test.txt b/requirements/test.txt index 0a0ebb1b8f..1f3275a781 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -1,11 +1,10 @@ -# Dependencies needed to run the iris tests -#------------------------------------------ +# Test dependencies. -black==19.10b0 #conda: black=19.10b0 +asv +black==19.10b0 filelock -pillow<7 imagehash>=4.0 nose +pillow<7 pre-commit requests -asv diff --git a/setup.py b/setup.py index b078e3de1f..b1c8939fdd 100644 --- a/setup.py +++ b/setup.py @@ -10,6 +10,10 @@ from setuptools.command.build_py import build_py +PACKAGE_DIR = os.path.abspath(os.path.dirname(__file__)) +PYPI_NAME = "scitools-iris" + + # Returns the package and all its sub-packages def find_package_tree(root_path, root_package): root_path = root_path.replace("/", os.path.sep) @@ -69,24 +73,25 @@ def temporary_path(directory): with temporary_path("lib/iris/tests/runner"): from _runner import TestRunner # noqa: -SETUP_DIR = os.path.dirname(__file__) - -def pip_requirements(name): - fname = os.path.join(SETUP_DIR, "requirements", "{}.txt".format(name)) - if not os.path.exists(fname): - raise RuntimeError( - "Unable to find the {} requirements file at {}" - "".format(name, fname) +def pip_requirements(*args): + requirements = [] + for name in args: + fname = os.path.join( + PACKAGE_DIR, "requirements", "{}.txt".format(name) ) - reqs = [] - with open(fname, "r") as fh: - for line in fh: - line = line.strip() - if not line or line.startswith("#"): - continue - reqs.append(line) - return reqs + if not os.path.exists(fname): + emsg = ( + f"Unable to find the {name!r} requirements file at {fname!r}." + ) + raise RuntimeError(emsg) + with open(fname, "r") as fh: + for line in fh: + line = line.strip() + if not line or line.startswith("#"): + continue + requirements.append(line) + return requirements class SetupTestRunner(TestRunner, Command): @@ -214,16 +219,23 @@ def run(self): def extract_version(): version = None - fnme = os.path.join(SETUP_DIR, "lib", "iris", "__init__.py") - with open(fnme) as fd: - for line in fd: + fname = os.path.join(PACKAGE_DIR, "lib", "iris", "__init__.py") + with open(fname) as fi: + for line in fi: if line.startswith("__version__"): _, version = line.split("=") - version = version.strip()[1:-1] # Remove quotation characters + version = version.strip()[1:-1] # Remove quotations break return version +def long_description(): + fname = os.path.join(PACKAGE_DIR, "README.md") + with open(fname, "rb") as fi: + result = fi.read().decode("utf-8") + return result + + custom_commands = { "test": SetupTestRunner, "develop": custom_cmd(develop_cmd, [build_std_names, compile_pyke_rules]), @@ -244,20 +256,15 @@ def extract_version(): } -pypi_name = "scitools-iris" - -with open(os.path.join(SETUP_DIR, "README.md"), "r") as fh: - description = "".join(fh.readlines()) - setup( - name=pypi_name, + name=PYPI_NAME, version=extract_version(), url="http://scitools.org.uk/iris/", author="UK Met Office", author_email="scitools-iris-dev@googlegroups.com", description="A powerful, format-agnostic, community-driven Python " "library for analysing and visualising Earth science data", - long_description=description, + long_description=long_description(), long_description_content_type="text/markdown", packages=find_package_tree("lib/iris", "iris"), package_dir={"": "lib"}, @@ -265,11 +272,11 @@ def extract_version(): cmdclass=custom_commands, zip_safe=False, setup_requires=pip_requirements("setup"), - install_requires=pip_requirements("setup") + pip_requirements("core"), - tests_require=["{}[test]".format(pypi_name)], + install_requires=pip_requirements("setup", "core"), + tests_require=[f"{PYPI_NAME}[test]"], extras_require={ - "test": pip_requirements("test"), "all": pip_requirements("all"), - "extensions": pip_requirements("extensions"), + "docs": pip_requirements("docs"), + "test": pip_requirements("test"), }, )