From b4b5b6998daf06935c3169066379fc3ef2d4ad62 Mon Sep 17 00:00:00 2001 From: Peter Killick Date: Wed, 31 May 2017 16:39:16 +0100 Subject: [PATCH 01/10] Maintain input lazy bounds in aux factory derivation --- lib/iris/aux_factory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/iris/aux_factory.py b/lib/iris/aux_factory.py index 729817adf9..fd3b25c00c 100644 --- a/lib/iris/aux_factory.py +++ b/lib/iris/aux_factory.py @@ -187,7 +187,7 @@ def _nd_bounds(self, coord, dims, ndim): # Transpose to be consistent with the Cube. sorted_pairs = sorted(enumerate(dims), key=lambda pair: pair[1]) transpose_order = [pair[0] for pair in sorted_pairs] + [len(dims)] - bounds = coord.bounds + bounds = coord.core_bounds() if dims: bounds = bounds.transpose(transpose_order) From 6344a4f20d3e41f14b07c45c99c71ce6a0ea7edc Mon Sep 17 00:00:00 2001 From: Corinne Bosley Date: Fri, 2 Jun 2017 16:02:10 +0100 Subject: [PATCH 02/10] test for bugfix - first try --- lib/iris/tests/test_coord_api.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/iris/tests/test_coord_api.py b/lib/iris/tests/test_coord_api.py index d6dbe6de13..f8f02ca92c 100644 --- a/lib/iris/tests/test_coord_api.py +++ b/lib/iris/tests/test_coord_api.py @@ -255,6 +255,13 @@ def test_bounded(self): ", standard_name='air_temperature', units=Unit('kelvin'))" ) self.assertEqual(result, str(a)) + + def test_lazy_bounds(self): + # Make sure that bounds arrays loaded as lazy do not get realised + # by the loading process. + cube = iris.load_cube(iris.sample_data_path('hybrid_height.nc')) + cube + self.assertTrue(cube.coord('sigma').has_lazy_bounds()) def test_string_coord_equality(self): b = iris.coords.AuxCoord(['Jan', 'Feb', 'March'], units='no_unit') From 28d6fabebbc18042f17ac4c6cff678b96418c105 Mon Sep 17 00:00:00 2001 From: Corinne Bosley Date: Mon, 5 Jun 2017 10:20:56 +0100 Subject: [PATCH 03/10] test added for derived coord lazy bounds --- lib/iris/tests/test_coord_api.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/iris/tests/test_coord_api.py b/lib/iris/tests/test_coord_api.py index f8f02ca92c..53cff4ff59 100644 --- a/lib/iris/tests/test_coord_api.py +++ b/lib/iris/tests/test_coord_api.py @@ -258,9 +258,11 @@ def test_bounded(self): def test_lazy_bounds(self): # Make sure that bounds arrays loaded as lazy do not get realised - # by the loading process. - cube = iris.load_cube(iris.sample_data_path('hybrid_height.nc')) - cube + # when the cube is printed. + cube = iris.load_cube(tests.get_data_path(['NetCDF', 'testing', + 'small_theta_colpex.nc'])) + self.assertTrue(cube.coord('sigma').has_lazy_bounds()) + printed_cube = str(cube) self.assertTrue(cube.coord('sigma').has_lazy_bounds()) def test_string_coord_equality(self): From ed1fa28e3e9557f43a595984aac00b17534742a7 Mon Sep 17 00:00:00 2001 From: Corinne Bosley Date: Mon, 5 Jun 2017 12:22:13 +0100 Subject: [PATCH 04/10] test data skipper added --- lib/iris/tests/test_coord_api.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/iris/tests/test_coord_api.py b/lib/iris/tests/test_coord_api.py index 53cff4ff59..8cf78a805d 100644 --- a/lib/iris/tests/test_coord_api.py +++ b/lib/iris/tests/test_coord_api.py @@ -256,6 +256,7 @@ def test_bounded(self): ) self.assertEqual(result, str(a)) + @tests.skip_data def test_lazy_bounds(self): # Make sure that bounds arrays loaded as lazy do not get realised # when the cube is printed. From f3950c04f260a222929b989b43069050d54113ac Mon Sep 17 00:00:00 2001 From: Corinne Bosley Date: Tue, 6 Jun 2017 09:48:04 +0100 Subject: [PATCH 05/10] tests moved and updated --- lib/iris/tests/test_coord_api.py | 10 --------- .../unit/aux_factory/test_AuxCoordFactory.py | 22 +++++++++++++++++++ 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/lib/iris/tests/test_coord_api.py b/lib/iris/tests/test_coord_api.py index 8cf78a805d..d6dbe6de13 100644 --- a/lib/iris/tests/test_coord_api.py +++ b/lib/iris/tests/test_coord_api.py @@ -255,16 +255,6 @@ def test_bounded(self): ", standard_name='air_temperature', units=Unit('kelvin'))" ) self.assertEqual(result, str(a)) - - @tests.skip_data - def test_lazy_bounds(self): - # Make sure that bounds arrays loaded as lazy do not get realised - # when the cube is printed. - cube = iris.load_cube(tests.get_data_path(['NetCDF', 'testing', - 'small_theta_colpex.nc'])) - self.assertTrue(cube.coord('sigma').has_lazy_bounds()) - printed_cube = str(cube) - self.assertTrue(cube.coord('sigma').has_lazy_bounds()) def test_string_coord_equality(self): b = iris.coords.AuxCoord(['Jan', 'Feb', 'March'], units='no_unit') diff --git a/lib/iris/tests/unit/aux_factory/test_AuxCoordFactory.py b/lib/iris/tests/unit/aux_factory/test_AuxCoordFactory.py index df9190de45..7a6ee23489 100644 --- a/lib/iris/tests/unit/aux_factory/test_AuxCoordFactory.py +++ b/lib/iris/tests/unit/aux_factory/test_AuxCoordFactory.py @@ -80,5 +80,27 @@ def test_lazy_complex(self): self.assertArrayEqual(result, expected) +@tests.skip_data +class Test_lazy_aux_coords(tests.IrisTest): + def test_lazy_coord_loading(self): + # Test that points and bounds arrays stay lazy upon cube loading + cube = iris.load_cube(tests.get_data_path(['NetCDF', 'testing', + 'small_theta_colpex.nc'])) + for coord in cube.aux_coords: + self.assertTrue(coord.has_lazy_points()) + if coord.has_bounds(): + self.assertTrue(coord.has_lazy_bounds()) + + def test_lazy_coord_printing(self): + # Test that points and bounds arrays stay lazy upon cube printing + cube = iris.load_cube(tests.get_data_path(['NetCDF', 'testing', + 'small_theta_colpex.nc'])) + printed_cube = str(cube) + for coord in cube.aux_coords: + self.assertTrue(coord.has_lazy_points()) + if coord.has_bounds(): + self.assertTrue(coord.has_lazy_bounds()) + + if __name__ == '__main__': tests.main() From cdac7f3ee7ed0f3ff699e95c77f19d32ec863a2a Mon Sep 17 00:00:00 2001 From: Corinne Bosley Date: Tue, 6 Jun 2017 11:49:13 +0100 Subject: [PATCH 06/10] derived coord check added to test --- .../tests/unit/aux_factory/test_AuxCoordFactory.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/lib/iris/tests/unit/aux_factory/test_AuxCoordFactory.py b/lib/iris/tests/unit/aux_factory/test_AuxCoordFactory.py index 7a6ee23489..e83ffd9872 100644 --- a/lib/iris/tests/unit/aux_factory/test_AuxCoordFactory.py +++ b/lib/iris/tests/unit/aux_factory/test_AuxCoordFactory.py @@ -82,21 +82,22 @@ def test_lazy_complex(self): @tests.skip_data class Test_lazy_aux_coords(tests.IrisTest): + def setUp(self): + self.cube = iris.load_cube(tests.get_data_path(['NetCDF', 'testing', + 'small_theta_colpex.nc'])) + def test_lazy_coord_loading(self): # Test that points and bounds arrays stay lazy upon cube loading - cube = iris.load_cube(tests.get_data_path(['NetCDF', 'testing', - 'small_theta_colpex.nc'])) - for coord in cube.aux_coords: + for coord in self.cube.aux_coords and self.cube.derived_coords: self.assertTrue(coord.has_lazy_points()) if coord.has_bounds(): self.assertTrue(coord.has_lazy_bounds()) def test_lazy_coord_printing(self): # Test that points and bounds arrays stay lazy upon cube printing - cube = iris.load_cube(tests.get_data_path(['NetCDF', 'testing', - 'small_theta_colpex.nc'])) + cube = self.cube printed_cube = str(cube) - for coord in cube.aux_coords: + for coord in cube.aux_coords and cube.derived_coords: self.assertTrue(coord.has_lazy_points()) if coord.has_bounds(): self.assertTrue(coord.has_lazy_bounds()) From c38d15b378503bf1799e0b74ac534d4a212dfff2 Mon Sep 17 00:00:00 2001 From: Corinne Bosley Date: Tue, 6 Jun 2017 11:56:38 +0100 Subject: [PATCH 07/10] pep8 --- lib/iris/tests/unit/aux_factory/test_AuxCoordFactory.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/iris/tests/unit/aux_factory/test_AuxCoordFactory.py b/lib/iris/tests/unit/aux_factory/test_AuxCoordFactory.py index e83ffd9872..dbc263eb7a 100644 --- a/lib/iris/tests/unit/aux_factory/test_AuxCoordFactory.py +++ b/lib/iris/tests/unit/aux_factory/test_AuxCoordFactory.py @@ -83,8 +83,9 @@ def test_lazy_complex(self): @tests.skip_data class Test_lazy_aux_coords(tests.IrisTest): def setUp(self): - self.cube = iris.load_cube(tests.get_data_path(['NetCDF', 'testing', - 'small_theta_colpex.nc'])) + self.cube = iris.load_cube(tests.get_data_path + (['NetCDF', 'testing', + 'small_theta_colpex.nc'])) def test_lazy_coord_loading(self): # Test that points and bounds arrays stay lazy upon cube loading From c0463245c6212c243c036d736bae0090108ba225 Mon Sep 17 00:00:00 2001 From: Corinne Bosley Date: Tue, 6 Jun 2017 13:36:44 +0100 Subject: [PATCH 08/10] added class to test _nd_bounds --- lib/iris/aux_factory.py | 3 +- .../unit/aux_factory/test_AuxCoordFactory.py | 55 ++++++++++++++++++- 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/lib/iris/aux_factory.py b/lib/iris/aux_factory.py index fd3b25c00c..07e5b116be 100644 --- a/lib/iris/aux_factory.py +++ b/lib/iris/aux_factory.py @@ -170,7 +170,8 @@ def _dependency_dims(self, coord_dims_func): dependency_dims[key] = coord_dims_func(coord) return dependency_dims - def _nd_bounds(self, coord, dims, ndim): + @staticmethod + def _nd_bounds(coord, dims, ndim): """ Returns the coord's bounds in Cube-orientation and broadcastable to N dimensions. diff --git a/lib/iris/tests/unit/aux_factory/test_AuxCoordFactory.py b/lib/iris/tests/unit/aux_factory/test_AuxCoordFactory.py index dbc263eb7a..410ddf2aa2 100644 --- a/lib/iris/tests/unit/aux_factory/test_AuxCoordFactory.py +++ b/lib/iris/tests/unit/aux_factory/test_AuxCoordFactory.py @@ -34,7 +34,7 @@ class Test__nd_points(tests.IrisTest): - def test_numpy_scalar_cooord(self): + def test_numpy_scalar_coord(self): points = np.arange(1) coord = iris.coords.AuxCoord(points) result = AuxCoordFactory._nd_points(coord, (), 2) @@ -80,6 +80,59 @@ def test_lazy_complex(self): self.assertArrayEqual(result, expected) + +class Test__nd_bounds(tests.IrisTest): + def test_numpy_scalar_coord(self): + bounds = np.arange(2).reshape(1, 2) + coord = iris.coords.AuxCoord(np.arange(1), bounds=bounds) + result = AuxCoordFactory._nd_bounds(coord, (), 2) + expected = bounds[np.newaxis] + self.assertArrayEqual(result, expected) + + def test_numpy_simple(self): + points = np.arange(12).reshape(4, 3) + bounds = np.arange(24).reshape(4, 3, 2) + coord = iris.coords.AuxCoord(points, bounds=bounds) + result = AuxCoordFactory._nd_bounds(coord, (0, 1), 2) + expected = bounds + self.assertArrayEqual(result, expected) + + def test_numpy_complex(self): + points = np.arange(12).reshape(4, 3) + bounds = np.arange(24).reshape(4, 3, 2) + coord = iris.coords.AuxCoord(points, bounds=bounds) + result = AuxCoordFactory._nd_bounds(coord, (3, 2), 5) + expected = bounds.T[np.newaxis, np.newaxis, ..., np.newaxis] + self.assertArrayEqual(result, expected) + + def test_lazy_simple(self): + raw_points = np.arange(12).reshape(4, 3) + points = as_lazy_data(raw_points, 1) + raw_bounds = np.arange(24).reshape(4, 3, 2) + bounds = as_lazy_data(raw_bounds, 1) + coord = iris.coords.AuxCoord(points, bounds=bounds) + self.assertTrue(is_lazy_data(coord.core_bounds())) + result = AuxCoordFactory._nd_bounds(coord, (0, 1), 2) + # Check we haven't triggered the loading of the coordinate values. + self.assertTrue(is_lazy_data(coord.core_bounds())) + self.assertTrue(is_lazy_data(result)) + expected = raw_bounds + self.assertArrayEqual(result, expected) + + def test_lazy_complex(self): + raw_points = np.arange(12).reshape(4, 3) + points = as_lazy_data(raw_points, 1) + raw_bounds = np.arange(24).reshape(4, 3, 2) + bounds = as_lazy_data(raw_bounds, 1) + coord = iris.coords.AuxCoord(points, bounds=bounds) + self.assertTrue(is_lazy_data(coord.core_bounds())) + result = AuxCoordFactory._nd_bounds(coord, (3, 2), 5) + # Check we haven't triggered the loading of the coordinate values. + self.assertTrue(is_lazy_data(coord.core_bounds())) + self.assertTrue(is_lazy_data(result)) + expected = raw_bounds.T[np.newaxis, np.newaxis, ..., np.newaxis] + self.assertArrayEqual(result, expected) + @tests.skip_data class Test_lazy_aux_coords(tests.IrisTest): def setUp(self): From 949aecdc1790625345b8371d7910d7ac4538ac9e Mon Sep 17 00:00:00 2001 From: Bill Little Date: Mon, 5 Jun 2017 16:07:18 +0100 Subject: [PATCH 09/10] Fix aux-factory lazy bounds. --- lib/iris/aux_factory.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/iris/aux_factory.py b/lib/iris/aux_factory.py index 07e5b116be..efd6607fc6 100644 --- a/lib/iris/aux_factory.py +++ b/lib/iris/aux_factory.py @@ -297,7 +297,6 @@ def _remap_with_bounds(self, dependency_dims, derived_dims): # extra dimension to make the shape compatible, so # we just add an extra 1. shape.append(1) - nd_values = np.array(nd_values) nd_values = nd_values.reshape(shape) else: # If no coord, treat value as zero. From 07ee9a08be4adb28aa2167a7fd0486f533c11f01 Mon Sep 17 00:00:00 2001 From: Bill Little Date: Wed, 7 Jun 2017 07:53:18 +0100 Subject: [PATCH 10/10] Review actions. --- .../unit/aux_factory/test_AuxCoordFactory.py | 84 +++++++++++-------- 1 file changed, 51 insertions(+), 33 deletions(-) diff --git a/lib/iris/tests/unit/aux_factory/test_AuxCoordFactory.py b/lib/iris/tests/unit/aux_factory/test_AuxCoordFactory.py index 410ddf2aa2..4b24f3887b 100644 --- a/lib/iris/tests/unit/aux_factory/test_AuxCoordFactory.py +++ b/lib/iris/tests/unit/aux_factory/test_AuxCoordFactory.py @@ -26,39 +26,48 @@ # importing anything else. import iris.tests as tests -from iris._lazy_data import as_lazy_data, is_lazy_data import numpy as np -import iris.coords +import iris +from iris._lazy_data import as_lazy_data, is_lazy_data from iris.aux_factory import AuxCoordFactory +from iris.coords import AuxCoord class Test__nd_points(tests.IrisTest): + def test_numpy_scalar_coord__zero_ndim(self): + points = np.array(1) + coord = AuxCoord(points) + result = AuxCoordFactory._nd_points(coord, (), 0) + expected = np.array([1]) + self.assertArrayEqual(result, expected) + def test_numpy_scalar_coord(self): - points = np.arange(1) - coord = iris.coords.AuxCoord(points) + value = 1 + points = np.array(value) + coord = AuxCoord(points) result = AuxCoordFactory._nd_points(coord, (), 2) - expected = points[np.newaxis] + expected = np.array(value).reshape(1, 1) self.assertArrayEqual(result, expected) def test_numpy_simple(self): points = np.arange(12).reshape(4, 3) - coord = iris.coords.AuxCoord(points) + coord = AuxCoord(points) result = AuxCoordFactory._nd_points(coord, (0, 1), 2) expected = points self.assertArrayEqual(result, expected) def test_numpy_complex(self): points = np.arange(12).reshape(4, 3) - coord = iris.coords.AuxCoord(points) + coord = AuxCoord(points) result = AuxCoordFactory._nd_points(coord, (3, 2), 5) expected = points.T[np.newaxis, np.newaxis, ..., np.newaxis] self.assertArrayEqual(result, expected) def test_lazy_simple(self): raw_points = np.arange(12).reshape(4, 3) - points = as_lazy_data(raw_points, 1) - coord = iris.coords.AuxCoord(points) + points = as_lazy_data(raw_points, raw_points.shape) + coord = AuxCoord(points) self.assertTrue(is_lazy_data(coord.core_points())) result = AuxCoordFactory._nd_points(coord, (0, 1), 2) # Check we haven't triggered the loading of the coordinate values. @@ -69,8 +78,8 @@ def test_lazy_simple(self): def test_lazy_complex(self): raw_points = np.arange(12).reshape(4, 3) - points = as_lazy_data(raw_points, 1) - coord = iris.coords.AuxCoord(points) + points = as_lazy_data(raw_points, raw_points.shape) + coord = AuxCoord(points) self.assertTrue(is_lazy_data(coord.core_points())) result = AuxCoordFactory._nd_points(coord, (3, 2), 5) # Check we haven't triggered the loading of the coordinate values. @@ -80,11 +89,19 @@ def test_lazy_complex(self): self.assertArrayEqual(result, expected) - class Test__nd_bounds(tests.IrisTest): + def test_numpy_scalar_coord__zero_ndim(self): + points = np.array(0.5) + bounds = np.arange(2) + coord = AuxCoord(points, bounds=bounds) + result = AuxCoordFactory._nd_bounds(coord, (), 0) + expected = bounds + self.assertArrayEqual(result, expected) + def test_numpy_scalar_coord(self): + points = np.array(0.5) bounds = np.arange(2).reshape(1, 2) - coord = iris.coords.AuxCoord(np.arange(1), bounds=bounds) + coord = AuxCoord(points, bounds=bounds) result = AuxCoordFactory._nd_bounds(coord, (), 2) expected = bounds[np.newaxis] self.assertArrayEqual(result, expected) @@ -92,7 +109,7 @@ def test_numpy_scalar_coord(self): def test_numpy_simple(self): points = np.arange(12).reshape(4, 3) bounds = np.arange(24).reshape(4, 3, 2) - coord = iris.coords.AuxCoord(points, bounds=bounds) + coord = AuxCoord(points, bounds=bounds) result = AuxCoordFactory._nd_bounds(coord, (0, 1), 2) expected = bounds self.assertArrayEqual(result, expected) @@ -100,17 +117,17 @@ def test_numpy_simple(self): def test_numpy_complex(self): points = np.arange(12).reshape(4, 3) bounds = np.arange(24).reshape(4, 3, 2) - coord = iris.coords.AuxCoord(points, bounds=bounds) + coord = AuxCoord(points, bounds=bounds) result = AuxCoordFactory._nd_bounds(coord, (3, 2), 5) - expected = bounds.T[np.newaxis, np.newaxis, ..., np.newaxis] + expected = bounds.transpose((1, 0, 2)).reshape(1, 1, 3, 4, 1, 2) self.assertArrayEqual(result, expected) def test_lazy_simple(self): raw_points = np.arange(12).reshape(4, 3) - points = as_lazy_data(raw_points, 1) + points = as_lazy_data(raw_points, raw_points.shape) raw_bounds = np.arange(24).reshape(4, 3, 2) - bounds = as_lazy_data(raw_bounds, 1) - coord = iris.coords.AuxCoord(points, bounds=bounds) + bounds = as_lazy_data(raw_bounds, raw_bounds.shape) + coord = AuxCoord(points, bounds=bounds) self.assertTrue(is_lazy_data(coord.core_bounds())) result = AuxCoordFactory._nd_bounds(coord, (0, 1), 2) # Check we haven't triggered the loading of the coordinate values. @@ -121,40 +138,41 @@ def test_lazy_simple(self): def test_lazy_complex(self): raw_points = np.arange(12).reshape(4, 3) - points = as_lazy_data(raw_points, 1) + points = as_lazy_data(raw_points, raw_points.shape) raw_bounds = np.arange(24).reshape(4, 3, 2) - bounds = as_lazy_data(raw_bounds, 1) - coord = iris.coords.AuxCoord(points, bounds=bounds) + bounds = as_lazy_data(raw_bounds, raw_bounds.shape) + coord = AuxCoord(points, bounds=bounds) self.assertTrue(is_lazy_data(coord.core_bounds())) result = AuxCoordFactory._nd_bounds(coord, (3, 2), 5) # Check we haven't triggered the loading of the coordinate values. self.assertTrue(is_lazy_data(coord.core_bounds())) self.assertTrue(is_lazy_data(result)) - expected = raw_bounds.T[np.newaxis, np.newaxis, ..., np.newaxis] + expected = raw_bounds.transpose((1, 0, 2)).reshape(1, 1, 3, 4, 1, 2) self.assertArrayEqual(result, expected) + @tests.skip_data class Test_lazy_aux_coords(tests.IrisTest): def setUp(self): self.cube = iris.load_cube(tests.get_data_path (['NetCDF', 'testing', 'small_theta_colpex.nc'])) + self.coords = self.cube.aux_coords + self.cube.derived_coords - def test_lazy_coord_loading(self): - # Test that points and bounds arrays stay lazy upon cube loading - for coord in self.cube.aux_coords and self.cube.derived_coords: + def _check_lazy(self): + for coord in self.coords: self.assertTrue(coord.has_lazy_points()) if coord.has_bounds(): self.assertTrue(coord.has_lazy_bounds()) + def test_lazy_coord_loading(self): + # Test that points and bounds arrays stay lazy upon cube loading. + self._check_lazy() + def test_lazy_coord_printing(self): - # Test that points and bounds arrays stay lazy upon cube printing - cube = self.cube - printed_cube = str(cube) - for coord in cube.aux_coords and cube.derived_coords: - self.assertTrue(coord.has_lazy_points()) - if coord.has_bounds(): - self.assertTrue(coord.has_lazy_bounds()) + # Test that points and bounds arrays stay lazy after cube printing. + _ = str(self.cube) + self._check_lazy() if __name__ == '__main__':