From 54be2fa953e418da65f8f0b45b1be4febd0d0d68 Mon Sep 17 00:00:00 2001 From: lbdreyer Date: Thu, 10 Feb 2022 16:31:26 +0000 Subject: [PATCH 1/4] Fix opendap bug, add docs and extra testing --- lib/iris/__init__.py | 6 ++++++ lib/iris/fileformats/netcdf.py | 4 ++-- lib/iris/io/__init__.py | 8 ++++---- lib/iris/tests/test_load.py | 35 +++++++++++++++++++++++++++------- 4 files changed, 40 insertions(+), 13 deletions(-) diff --git a/lib/iris/__init__.py b/lib/iris/__init__.py index aca4e77e88..d7c33c72fb 100644 --- a/lib/iris/__init__.py +++ b/lib/iris/__init__.py @@ -44,6 +44,8 @@ standard library function :func:`os.path.expanduser` and module :mod:`fnmatch` for more details. + If supplying a URL, only OPeNDAP Data Sources are supported. + * constraints: Either a single constraint, or an iterable of constraints. Each constraint can be either a string, an instance of @@ -287,6 +289,7 @@ def load(uris, constraints=None, callback=None): * uris: One or more filenames/URIs, as a string or :class:`pathlib.PurePath`. + If supplying a URL, only OPeNDAP Data Sources are supported. Kwargs: @@ -315,6 +318,7 @@ def load_cube(uris, constraint=None, callback=None): * uris: One or more filenames/URIs, as a string or :class:`pathlib.PurePath`. + If supplying a URL, only OPeNDAP Data Sources are supported. Kwargs: @@ -354,6 +358,7 @@ def load_cubes(uris, constraints=None, callback=None): * uris: One or more filenames/URIs, as a string or :class:`pathlib.PurePath`. + If supplying a URL, only OPeNDAP Data Sources are supported. Kwargs: @@ -399,6 +404,7 @@ def load_raw(uris, constraints=None, callback=None): * uris: One or more filenames/URIs, as a string or :class:`pathlib.PurePath`. + If supplying a URL, only OPeNDAP Data Sources are supported. Kwargs: diff --git a/lib/iris/fileformats/netcdf.py b/lib/iris/fileformats/netcdf.py index 100ab29daa..dd819fb63f 100644 --- a/lib/iris/fileformats/netcdf.py +++ b/lib/iris/fileformats/netcdf.py @@ -825,12 +825,12 @@ def inner(cf_datavar): def load_cubes(filenames, callback=None, constraints=None): """ - Loads cubes from a list of NetCDF filenames/URLs. + Loads cubes from a list of NetCDF filenames/OPeNDAP URLs. Args: * filenames (string/list): - One or more NetCDF filenames/DAP URLs to load from. + One or more NetCDF filenames/OPeNDAP URLs to load from. Kwargs: diff --git a/lib/iris/io/__init__.py b/lib/iris/io/__init__.py index 034fa4baab..8d5a2e05d2 100644 --- a/lib/iris/io/__init__.py +++ b/lib/iris/io/__init__.py @@ -216,7 +216,7 @@ def load_files(filenames, callback, constraints=None): def load_http(urls, callback): """ - Takes a list of urls and a callback function, and returns a generator + Takes a list of OPeNDAP URLs and a callback function, and returns a generator of Cubes from the given URLs. .. note:: @@ -226,11 +226,11 @@ def load_http(urls, callback): """ # Create default dict mapping iris format handler to its associated filenames + from iris.fileformats import FORMAT_AGENT + handler_map = collections.defaultdict(list) for url in urls: - handling_format_spec = iris.fileformats.FORMAT_AGENT.get_spec( - url, None - ) + handling_format_spec = FORMAT_AGENT.get_spec(url, None) handler_map[handling_format_spec].append(url) # Call each iris format handler with the appropriate filenames diff --git a/lib/iris/tests/test_load.py b/lib/iris/tests/test_load.py index 86ff2f1ece..d21b40ee26 100644 --- a/lib/iris/tests/test_load.py +++ b/lib/iris/tests/test_load.py @@ -12,6 +12,9 @@ import iris.tests as tests # isort:skip import pathlib +from unittest import mock + +import netCDF4 import iris import iris.io @@ -148,19 +151,20 @@ def test_path_object(self): self.assertEqual(len(cubes), 1) -class TestOpenDAP(tests.IrisTest): - def test_load(self): - # Check that calling iris.load_* with a http URI triggers a call to - # ``iris.io.load_http`` +class TestOPeNDAP(tests.IrisTest): + def setUp(self): + self.url = "http://geoport.whoi.edu:80/thredds/dodsC/bathy/gom15" - url = "http://geoport.whoi.edu:80/thredds/dodsC/bathy/gom15" + def test_load_http_called(self): + # Check that calling iris.load_* with an http URI triggers a call to + # ``iris.io.load_http`` class LoadHTTPCalled(Exception): pass def new_load_http(passed_urls, *args, **kwargs): self.assertEqual(len(passed_urls), 1) - self.assertEqual(url, passed_urls[0]) + self.assertEqual(self.url, passed_urls[0]) raise LoadHTTPCalled() try: @@ -174,11 +178,28 @@ def new_load_http(passed_urls, *args, **kwargs): iris.load_cubes, ]: with self.assertRaises(LoadHTTPCalled): - fn(url) + fn(self.url) finally: iris.io.load_http = orig + def test_netCDF_Dataset_call(self): + # Check that load_http calls netCDF4.Dataset and supplies the expected URL. + + # To avoid making a request to an OPeNDAP server in a test, instead + # mock the call to netCDF.Dataset so that it returns a dataset for a + # local file. + filename = tests.get_data_path( + ("NetCDF", "global", "xyt", "SMALL_total_column_co2.nc") + ) + fake_dataset = netCDF4.Dataset(filename) + + with mock.patch( + "netCDF4.Dataset", return_value=fake_dataset + ) as dataset_loader: + next(iris.io.load_http([self.url], callback=None)) + dataset_loader.assert_called_with(self.url, mode="r") + if __name__ == "__main__": tests.main() From 5f0289e309f7f3c5a2bc5bb40a09236f08e1a1c2 Mon Sep 17 00:00:00 2001 From: lbdreyer Date: Fri, 11 Feb 2022 10:12:31 +0000 Subject: [PATCH 2/4] Add whats new entry --- docs/src/whatsnew/3.2.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/src/whatsnew/3.2.rst b/docs/src/whatsnew/3.2.rst index c78e1283d6..37444cae25 100644 --- a/docs/src/whatsnew/3.2.rst +++ b/docs/src/whatsnew/3.2.rst @@ -177,6 +177,9 @@ This document explains the changes made to Iris for this release to indicate cloud fraction greater than 7.9 oktas, rather than 7.5 (:issue:`3305`, :pull:`4535`) +#. `@lbdreyer`_ fixed a bug in `iris.io.load_http` which was missing an import + (:pull:`4580`) + 💣 Incompatible Changes ======================= From ee01c07e2363dd454d6a2aec143644a2ab4b9dcc Mon Sep 17 00:00:00 2001 From: lbdreyer Date: Fri, 11 Feb 2022 10:37:35 +0000 Subject: [PATCH 3/4] Update docs/src/whatsnew/3.2.rst Co-authored-by: Bill Little --- docs/src/whatsnew/3.2.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/whatsnew/3.2.rst b/docs/src/whatsnew/3.2.rst index 37444cae25..9aa6a78846 100644 --- a/docs/src/whatsnew/3.2.rst +++ b/docs/src/whatsnew/3.2.rst @@ -177,7 +177,7 @@ This document explains the changes made to Iris for this release to indicate cloud fraction greater than 7.9 oktas, rather than 7.5 (:issue:`3305`, :pull:`4535`) -#. `@lbdreyer`_ fixed a bug in `iris.io.load_http` which was missing an import +#. `@lbdreyer`_ fixed a bug in :class:`iris.io.load_http` which was missing an import (:pull:`4580`) From a5506dc50c6739955f2cdcc3a4f6a7bd68f03c8f Mon Sep 17 00:00:00 2001 From: lbdreyer Date: Fri, 11 Feb 2022 10:53:29 +0000 Subject: [PATCH 4/4] Add warning box --- lib/iris/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/iris/__init__.py b/lib/iris/__init__.py index d7c33c72fb..95722c69cf 100644 --- a/lib/iris/__init__.py +++ b/lib/iris/__init__.py @@ -44,7 +44,9 @@ standard library function :func:`os.path.expanduser` and module :mod:`fnmatch` for more details. - If supplying a URL, only OPeNDAP Data Sources are supported. + .. warning:: + + If supplying a URL, only OPeNDAP Data Sources are supported. * constraints: Either a single constraint, or an iterable of constraints.