From c4cbc8dcecb2397d0c2005040d8a75e9812444e9 Mon Sep 17 00:00:00 2001 From: Barnaby Sherratt Date: Tue, 9 Nov 2021 13:47:54 +0000 Subject: [PATCH 1/6] Convert size 1 array dtype without becoming scalar --- lib/iris/fileformats/name_loaders.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/iris/fileformats/name_loaders.py b/lib/iris/fileformats/name_loaders.py index c3db0e98de..b044c9154f 100644 --- a/lib/iris/fileformats/name_loaders.py +++ b/lib/iris/fileformats/name_loaders.py @@ -478,7 +478,7 @@ def _generate_cubes( coord_units = _parse_units("FL") if coord.name == "time": coord_units = time_unit - pts = np.float_(time_unit.date2num(coord.values)) + pts = time_unit.date2num(coord.values).astype(float) if coord.dimension is not None: if coord.name == "longitude": @@ -505,7 +505,7 @@ def _generate_cubes( ): dt = coord.values - field_headings["Av or Int period"] bnds = time_unit.date2num(np.vstack((dt, coord.values)).T) - icoord.bounds = np.float_(bnds) + icoord.bounds = bnds.astype(float) else: icoord.guess_bounds() cube.add_dim_coord(icoord, coord.dimension) @@ -522,7 +522,7 @@ def _generate_cubes( ): dt = coord.values - field_headings["Av or Int period"] bnds = time_unit.date2num(np.vstack((dt, coord.values)).T) - icoord.bounds = np.float_(bnds[i, :]) + icoord.bounds = bnds[i, :].astype(float) cube.add_aux_coord(icoord) # Headings/column headings which are encoded elsewhere. @@ -1250,7 +1250,7 @@ def load_NAMEIII_trajectory(filename): long_name = units = None if isinstance(values[0], datetime.datetime): - values = np.float_(time_unit.date2num(values)) + values = time_unit.date2num(values).astype(float) units = time_unit if name == "Time": name = "time" From 740a86bb14f9ff4c47b3d652ed650522edc8bb0b Mon Sep 17 00:00:00 2001 From: Barnaby Sherratt Date: Tue, 9 Nov 2021 18:00:53 +0000 Subject: [PATCH 2/6] Ensure z coords are recognised by guess_coord_axis --- lib/iris/fileformats/name_loaders.py | 1 + .../name_loaders/test__cf_height_from_name.py | 17 +++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/lib/iris/fileformats/name_loaders.py b/lib/iris/fileformats/name_loaders.py index b044c9154f..0250e85e66 100644 --- a/lib/iris/fileformats/name_loaders.py +++ b/lib/iris/fileformats/name_loaders.py @@ -387,6 +387,7 @@ def _cf_height_from_name(z_coord, lower_bound=None, upper_bound=None): standard_name=standard_name, long_name=long_name, bounds=bounds, + attributes={"positive": "up"}, ) return coord diff --git a/lib/iris/tests/unit/fileformats/name_loaders/test__cf_height_from_name.py b/lib/iris/tests/unit/fileformats/name_loaders/test__cf_height_from_name.py index a246edd7fd..7ce66c3fef 100644 --- a/lib/iris/tests/unit/fileformats/name_loaders/test__cf_height_from_name.py +++ b/lib/iris/tests/unit/fileformats/name_loaders/test__cf_height_from_name.py @@ -29,6 +29,7 @@ def _default_coord(self, data): bounds=None, standard_name=None, long_name="z", + attributes={"positive": "up"}, ) @@ -43,6 +44,7 @@ def test_bounded_height_above_ground(self): bounds=np.array([0.0, 100.0]), standard_name="height", long_name="height above ground level", + attributes={"positive": "up"}, ) self.assertEqual(com, res) @@ -55,6 +57,7 @@ def test_bounded_flight_level(self): bounds=np.array([0.0, 100.0]), standard_name=None, long_name="flight_level", + attributes={"positive": "up"}, ) self.assertEqual(com, res) @@ -67,6 +70,7 @@ def test_bounded_height_above_sea_level(self): bounds=np.array([0.0, 100.0]), standard_name="altitude", long_name="altitude above sea level", + attributes={"positive": "up"}, ) self.assertEqual(com, res) @@ -104,6 +108,7 @@ def test_float_bounded_height_above_ground(self): bounds=np.array([0.0, 100.0]), standard_name="height", long_name="height above ground level", + attributes={"positive": "up"}, ) self.assertEqual(com, res) @@ -117,6 +122,7 @@ def test_float_bounded_height_flight_level(self): bounds=np.array([0.0, 100.0]), standard_name=None, long_name="flight_level", + attributes={"positive": "up"}, ) self.assertEqual(com, res) @@ -130,6 +136,7 @@ def test_float_bounded_height_above_sea_level(self): bounds=np.array([0.0, 100.0]), standard_name="altitude", long_name="altitude above sea level", + attributes={"positive": "up"}, ) self.assertEqual(com, res) @@ -151,6 +158,7 @@ def test_pressure(self): bounds=np.array([0.0, 100.0]), standard_name="air_pressure", long_name=None, + attributes={"positive": "up"}, ) self.assertEqual(com, res) @@ -166,6 +174,7 @@ def test_height_above_ground(self): bounds=None, standard_name="height", long_name="height above ground level", + attributes={"positive": "up"}, ) self.assertEqual(com, res) @@ -178,6 +187,7 @@ def test_height_flight_level(self): bounds=None, standard_name=None, long_name="flight_level", + attributes={"positive": "up"}, ) self.assertEqual(com, res) @@ -190,6 +200,7 @@ def test_height_above_sea_level(self): bounds=None, standard_name="altitude", long_name="altitude above sea level", + attributes={"positive": "up"}, ) self.assertEqual(com, res) @@ -227,6 +238,7 @@ def test_integer_height_above_ground(self): bounds=None, standard_name="height", long_name="height above ground level", + attributes={"positive": "up"}, ) self.assertEqual(com, res) @@ -240,6 +252,7 @@ def test_integer_height_flight_level(self): bounds=None, standard_name=None, long_name="flight_level", + attributes={"positive": "up"}, ) self.assertEqual(com, res) @@ -253,6 +266,7 @@ def test_integer_height_above_sea_level(self): bounds=None, standard_name="altitude", long_name="altitude above sea level", + attributes={"positive": "up"}, ) self.assertEqual(com, res) @@ -266,6 +280,7 @@ def test_enotation_height_above_ground(self): bounds=None, standard_name="height", long_name="height above ground level", + attributes={"positive": "up"}, ) self.assertEqual(com, res) @@ -279,6 +294,7 @@ def test_enotation_height_above_sea_level(self): bounds=None, standard_name="altitude", long_name="altitude above sea level", + attributes={"positive": "up"}, ) self.assertEqual(com, res) @@ -292,6 +308,7 @@ def test_pressure(self): bounds=None, standard_name="air_pressure", long_name=None, + attributes={"positive": "up"}, ) self.assertEqual(com, res) From d59a1d714343bb51acf6174c317cca2171ffc374 Mon Sep 17 00:00:00 2001 From: Barnaby Sherratt Date: Wed, 10 Nov 2021 11:53:53 +0000 Subject: [PATCH 3/6] Test time coord creation --- .../name_loaders/test__generate_cubes.py | 69 ++++++++++++++++++- 1 file changed, 66 insertions(+), 3 deletions(-) diff --git a/lib/iris/tests/unit/fileformats/name_loaders/test__generate_cubes.py b/lib/iris/tests/unit/fileformats/name_loaders/test__generate_cubes.py index c0cfe4b041..d50a7fdad1 100644 --- a/lib/iris/tests/unit/fileformats/name_loaders/test__generate_cubes.py +++ b/lib/iris/tests/unit/fileformats/name_loaders/test__generate_cubes.py @@ -12,8 +12,11 @@ # importing anything else. import iris.tests as tests # isort:skip +from datetime import datetime, timedelta from unittest import mock +import numpy as np + from iris.fileformats.name_loaders import NAMECoord, _generate_cubes @@ -66,7 +69,7 @@ def _simulate_with_coords(self, names, values, dimensions): def test_non_circular(self): results = self._simulate_with_coords( - names=["longitude"], values=[[1, 7, 23]], dimensions=[(0,)] + names=["longitude"], values=[[1, 7, 23]], dimensions=[0] ) self.assertEqual(len(results), 1) add_coord_calls = results[0].add_dim_coord.call_args_list @@ -78,7 +81,7 @@ def test_circular(self): results = self._simulate_with_coords( names=["longitude"], values=[[5.0, 95.0, 185.0, 275.0]], - dimensions=[(0,)], + dimensions=[0], ) self.assertEqual(len(results), 1) add_coord_calls = results[0].add_dim_coord.call_args_list @@ -90,7 +93,7 @@ def test_lat_lon_byname(self): results = self._simulate_with_coords( names=["longitude", "latitude"], values=[[5.0, 95.0, 185.0, 275.0], [5.0, 95.0, 185.0, 275.0]], - dimensions=[(0,), (1,)], + dimensions=[0, 1], ) self.assertEqual(len(results), 1) add_coord_calls = results[0].add_dim_coord.call_args_list @@ -101,5 +104,65 @@ def test_lat_lon_byname(self): self.assertEqual(lat_coord.circular, False) +class TestTimeCoord(tests.IrisTest): + def _simulate_with_coords(self, names, values, dimensions): + header = mock.MagicMock() + column_headings = { + "Species": [1, 2, 3], + "Quantity": [4, 5, 6], + "Units": ["m", "m", "m"], + "Av or Int period": [timedelta(hours=24)], + } + coords = [ + NAMECoord(name, dim, np.array(vals)) + for name, vals, dim in zip(names, values, dimensions) + ] + data_arrays = [mock.Mock()] + + self.patch("iris.fileformats.name_loaders._cf_height_from_name") + self.patch("iris.cube.Cube") + cubes = list( + _generate_cubes(header, column_headings, coords, data_arrays) + ) + return cubes + + def test_time_dim(self): + results = self._simulate_with_coords( + names=["longitude", "latitude", "time"], + values=[ + [10, 20], + [30, 40], + [datetime(2015, 6, 7), datetime(2015, 6, 8)], + ], + dimensions=[0, 1, 2], + ) + self.assertEqual(len(results), 1) + result = results[0] + dim_coord_calls = result.add_dim_coord.call_args_list + self.assertEqual(len(dim_coord_calls), 3) # lon, lat, time + t_coord = dim_coord_calls[2][0][0] + self.assertEqual(t_coord.standard_name, "time") + self.assertArrayEqual(t_coord.points, [398232, 398256]) + self.assertArrayEqual(t_coord.bounds[0], [398208, 398232]) + self.assertArrayEqual(t_coord.bounds[-1], [398232, 398256]) + + def test_time_scalar(self): + results = self._simulate_with_coords( + names=["longitude", "latitude", "time"], + values=[[10, 20], [30, 40], [datetime(2015, 6, 7)]], + dimensions=[0, 1, None], + ) + self.assertEqual(len(results), 1) + result = results[0] + dim_coord_calls = result.add_dim_coord.call_args_list + self.assertEqual(len(dim_coord_calls), 2) + aux_coord_calls = result.add_aux_coord.call_args_list + self.assertEqual(len(aux_coord_calls), 1) + t_coord = aux_coord_calls[0][0][0] + self.assertEqual(t_coord.standard_name, "time") + self.assertArrayEqual(t_coord.points, [398232]) + self.assertArrayEqual(t_coord.bounds, [[398208, 398232]]) + + if __name__ == "__main__": tests.main() From 308cd20eb26c4a1cb76deced274ad9c31f1a733d Mon Sep 17 00:00:00 2001 From: Barnaby Sherratt Date: Wed, 10 Nov 2021 16:49:55 +0000 Subject: [PATCH 4/6] Allow integers in lat-lon headings --- lib/iris/fileformats/name_loaders.py | 46 +++++++++------- ...test__build_lat_lon_for_NAME_timeseries.py | 52 +++++++++++++++++++ 2 files changed, 80 insertions(+), 18 deletions(-) create mode 100644 lib/iris/tests/unit/fileformats/name_loaders/test__build_lat_lon_for_NAME_timeseries.py diff --git a/lib/iris/fileformats/name_loaders.py b/lib/iris/fileformats/name_loaders.py index 0250e85e66..34e88aff80 100644 --- a/lib/iris/fileformats/name_loaders.py +++ b/lib/iris/fileformats/name_loaders.py @@ -160,27 +160,37 @@ def _build_lat_lon_for_NAME_timeseries(column_headings): the provided column_headings dictionary. """ - pattern = re.compile(r"\-?[0-9]*\.[0-9]*") - new_Xlocation_column_header = [] - for t in column_headings["X"]: - if "Lat-Long" in t: - matches = pattern.search(t) - new_Xlocation_column_header.append(float(matches.group(0))) - else: - new_Xlocation_column_header.append(t) - column_headings["X"] = new_Xlocation_column_header + # Pattern to match a number + pattern = re.compile( + r""" + [-+]? # Optional sign + (?: + \d+\.\d* # Float: integral part required + | + \d*\.\d+ # Float: fractional part required + | + \d+ # Integer + ) + (?![0-9.]) # Not followed by a numeric character + """, + re.VERBOSE, + ) + + # Extract numbers from the X and Y column headings, which are currently + # strings of the form "X = -1.9 Lat-Long" + for key in ("X", "Y"): + new_headings = [] + for heading in column_headings[key]: + match = pattern.search(heading) + if match and "Lat-Long" in heading: + new_headings.append(float(match.group(0))) + else: + new_headings.append(heading) + column_headings[key] = new_headings + lon = NAMECoord( name="longitude", dimension=None, values=column_headings["X"] ) - - new_Ylocation_column_header = [] - for t in column_headings["Y"]: - if "Lat-Long" in t: - matches = pattern.search(t) - new_Ylocation_column_header.append(float(matches.group(0))) - else: - new_Ylocation_column_header.append(t) - column_headings["Y"] = new_Ylocation_column_header lat = NAMECoord( name="latitude", dimension=None, values=column_headings["Y"] ) diff --git a/lib/iris/tests/unit/fileformats/name_loaders/test__build_lat_lon_for_NAME_timeseries.py b/lib/iris/tests/unit/fileformats/name_loaders/test__build_lat_lon_for_NAME_timeseries.py new file mode 100644 index 0000000000..5954823c54 --- /dev/null +++ b/lib/iris/tests/unit/fileformats/name_loaders/test__build_lat_lon_for_NAME_timeseries.py @@ -0,0 +1,52 @@ +# Copyright Iris contributors +# +# This file is part of Iris and is released under the LGPL license. +# See COPYING and COPYING.LESSER in the root of the repository for full +# licensing details. +""" +Unit tests for :func:`iris.analysis.name_loaders._build_lat_lon_for_NAME_timeseries`. + +""" + +# Import iris.tests first so that some things can be initialised before +# importing anything else. +import iris.tests as tests # isort:skip + +from iris.fileformats.name_loaders import ( + NAMECoord, + _build_lat_lon_for_NAME_timeseries, +) + + +class TestCellMethods(tests.IrisTest): + def test_float(self): + column_headings = { + "X": ["X = -.100 Lat-Long", "X = -1.600 Lat-Long"], + "Y": ["Y = 52.450 Lat-Long", "Y = 51. Lat-Long"], + } + lat, lon = _build_lat_lon_for_NAME_timeseries(column_headings) + self.assertIsInstance(lat, NAMECoord) + self.assertIsInstance(lon, NAMECoord) + self.assertEqual(lat.name, "latitude") + self.assertEqual(lon.name, "longitude") + self.assertIsNone(lat.dimension) + self.assertIsNone(lon.dimension) + self.assertArrayEqual(lat.values, [52.45, 51.0]) + self.assertArrayEqual(lon.values, [-0.1, -1.6]) + + def test_int(self): + column_headings = { + "X": ["X = -1 Lat-Long", "X = -2 Lat-Long"], + "Y": ["Y = 52 Lat-Long", "Y = 51 Lat-Long"], + } + lat, lon = _build_lat_lon_for_NAME_timeseries(column_headings) + self.assertIsInstance(lat, NAMECoord) + self.assertIsInstance(lon, NAMECoord) + self.assertEqual(lat.name, "latitude") + self.assertEqual(lon.name, "longitude") + self.assertIsNone(lat.dimension) + self.assertIsNone(lon.dimension) + self.assertArrayEqual(lat.values, [52.0, 51.0]) + self.assertArrayEqual(lon.values, [-1.0, -2.0]) + self.assertIsInstance(lat.values[0], float) + self.assertIsInstance(lon.values[0], float) From edfe6521a75b1b0394702b821fb80632deb4c746 Mon Sep 17 00:00:00 2001 From: Barnaby Sherratt Date: Wed, 10 Nov 2021 17:45:55 +0000 Subject: [PATCH 5/6] Update test results --- lib/iris/tests/results/name/NAMEIII_field.cml | 30 +++++++++++++++---- .../tests/results/name/NAMEIII_timeseries.cml | 30 +++++++++++++++---- lib/iris/tests/results/name/NAMEII_field.cml | 30 +++++++++++++++---- .../tests/results/name/NAMEII_timeseries.cml | 12 ++++++-- 4 files changed, 85 insertions(+), 17 deletions(-) diff --git a/lib/iris/tests/results/name/NAMEIII_field.cml b/lib/iris/tests/results/name/NAMEIII_field.cml index 674190f642..97b3189bba 100644 --- a/lib/iris/tests/results/name/NAMEIII_field.cml +++ b/lib/iris/tests/results/name/NAMEIII_field.cml @@ -51,7 +51,11 @@ - + + + + + @@ -112,7 +116,11 @@ - + + + + + @@ -172,7 +180,11 @@ - + + + + + @@ -232,7 +244,11 @@ - + + + + + @@ -292,7 +308,11 @@ - + + + + + diff --git a/lib/iris/tests/results/name/NAMEIII_timeseries.cml b/lib/iris/tests/results/name/NAMEIII_timeseries.cml index 976a6b2804..c4e70590a2 100644 --- a/lib/iris/tests/results/name/NAMEIII_timeseries.cml +++ b/lib/iris/tests/results/name/NAMEIII_timeseries.cml @@ -61,7 +61,11 @@ 358354.0, 358355.0, 358356.0, 358357.0]" shape="(72,)" standard_name="time" units="Unit('hours since 1970-01-01 00:00:00', calendar='gregorian')" value_type="float64"/> - + + + + + @@ -128,7 +132,11 @@ 358354.0, 358355.0, 358356.0, 358357.0]" shape="(72,)" standard_name="time" units="Unit('hours since 1970-01-01 00:00:00', calendar='gregorian')" value_type="float64"/> - + + + + + @@ -194,7 +202,11 @@ 358354.0, 358355.0, 358356.0, 358357.0]" shape="(72,)" standard_name="time" units="Unit('hours since 1970-01-01 00:00:00', calendar='gregorian')" value_type="float64"/> - + + + + + @@ -260,7 +272,11 @@ 358354.0, 358355.0, 358356.0, 358357.0]" shape="(72,)" standard_name="time" units="Unit('hours since 1970-01-01 00:00:00', calendar='gregorian')" value_type="float64"/> - + + + + + @@ -326,7 +342,11 @@ 358354.0, 358355.0, 358356.0, 358357.0]" shape="(72,)" standard_name="time" units="Unit('hours since 1970-01-01 00:00:00', calendar='gregorian')" value_type="float64"/> - + + + + + diff --git a/lib/iris/tests/results/name/NAMEII_field.cml b/lib/iris/tests/results/name/NAMEII_field.cml index db9ac94198..664669ef62 100644 --- a/lib/iris/tests/results/name/NAMEII_field.cml +++ b/lib/iris/tests/results/name/NAMEII_field.cml @@ -19,7 +19,11 @@ - + + + + + + + + + + - + + + + + @@ -218,7 +230,11 @@ - + + + + + @@ -275,7 +291,11 @@ - + + + + + diff --git a/lib/iris/tests/results/name/NAMEII_timeseries.cml b/lib/iris/tests/results/name/NAMEII_timeseries.cml index 61d0b2fed0..52aaa8b809 100644 --- a/lib/iris/tests/results/name/NAMEII_timeseries.cml +++ b/lib/iris/tests/results/name/NAMEII_timeseries.cml @@ -39,7 +39,11 @@ 370475.0, 370476.0]" shape="(132,)" standard_name="time" units="Unit('hours since 1970-01-01 00:00:00', calendar='gregorian')" value_type="float64"/> - + + + + + @@ -84,7 +88,11 @@ 370475.0, 370476.0]" shape="(132,)" standard_name="time" units="Unit('hours since 1970-01-01 00:00:00', calendar='gregorian')" value_type="float64"/> - + + + + + From ec404c6caf0085231303e113f1ced4e54c396985 Mon Sep 17 00:00:00 2001 From: Barnaby Sherratt Date: Fri, 12 Nov 2021 09:52:22 +0000 Subject: [PATCH 6/6] What's new --- docs/src/whatsnew/latest.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/src/whatsnew/latest.rst b/docs/src/whatsnew/latest.rst index 2f20a58daf..ded63078ce 100644 --- a/docs/src/whatsnew/latest.rst +++ b/docs/src/whatsnew/latest.rst @@ -88,7 +88,6 @@ This document explains the changes made to Iris for this release 🐛 Bugs Fixed ============= - #. `@rcomer`_ fixed :meth:`~iris.cube.Cube.intersection` for special cases where one cell's bounds align with the requested maximum and negative minimum, fixing :issue:`4221`. (:pull:`4278`) @@ -110,6 +109,10 @@ This document explains the changes made to Iris for this release to indicate that the value of a scalar coordinate may be mismatched, rather than the metadata (:issue:`4096`, :pull:`4387`) +#. `@bsherratt`_ fixed a regression to the NAME file loader introduced in 3.0.4, + as well as some long-standing bugs with vertical coordinates and number + formats. (:pull:`4411`) + 💣 Incompatible Changes =======================