From 5f95fd301c5ff7493a50d2a1069cf990aff1a3f8 Mon Sep 17 00:00:00 2001 From: "stephen.worsley" Date: Thu, 28 Nov 2019 14:24:37 +0000 Subject: [PATCH 01/10] add concatenate support for ancils and cell measures --- lib/iris/_concatenate.py | 229 ++++++++++++++++++++++++++++++++++++--- lib/iris/coords.py | 103 +++++++++++++++++- 2 files changed, 313 insertions(+), 19 deletions(-) diff --git a/lib/iris/_concatenate.py b/lib/iris/_concatenate.py index 6dabb1d264..167274a2da 100644 --- a/lib/iris/_concatenate.py +++ b/lib/iris/_concatenate.py @@ -162,6 +162,65 @@ def name(self): return self.defn.name() +class _OtherMetaData(namedtuple("OtherMetaData", ["defn", "dims"],)): + """ + Container for the metadata that defines a cell measure or ancillary + variable. + + Args: + + * defn: + The :class:`iris.coords._DMDefn` or :class:`iris.coords._CellMeasureDefn` + metadata that represents a coordinate. + + * dims: + The dimension(s) associated with the coordinate. + + """ + + def __new__(cls, ancil, dims): + """ + Create a new :class:`_OtherMetaData` instance. + + Args: + + * ancil: + The :class:`iris.coord.CellMeasure` or + :class:`iris.coord.AncillaryVariable`. + + * dims: + The dimension(s) associated with ancil. + + Returns: + The new class instance. + + """ + defn = ancil._as_defn() + metadata = super().__new__(cls, defn, dims) + return metadata + + __slots__ = () + + def __hash__(self): + return super().__hash__() + + def __eq__(self, other): + result = NotImplemented + if isinstance(other, _OtherMetaData): + result = self._asdict() == other._asdict() + return result + + def __ne__(self, other): + result = self.__eq__(other) + if result is not NotImplemented: + result = not result + return result + + def name(self): + """Get the name from the coordinate definition.""" + return self.defn.name() + + class _SkeletonCube(namedtuple("SkeletonCube", ["signature", "data"])): """ Basis of a source-cube, containing the associated coordinate metadata, @@ -306,7 +365,10 @@ def __init__(self, cube): self.dim_metadata = [] self.ndim = cube.ndim self.scalar_coords = [] - self.cell_measures_and_dims = cube._cell_measures_and_dims + self.cell_measures_and_dims = [] + self.cm_metadata = [] + self.ancillary_variables_and_dims = [] + self.av_metadata = [] self.dim_mapping = [] # Determine whether there are any anonymous cube dimensions. @@ -349,6 +411,23 @@ def key_func(coord): else: self.scalar_coords.append(coord) + def meta_key_func(dm): + return (dm._as_defn(), dm.cube_dims(cube)) + + for cm in sorted(cube.cell_measures(), key=meta_key_func): + dims = cube.cell_measure_dims(cm) + metadata = _OtherMetaData(cm, dims) + self.cm_metadata.append(metadata) + cm_and_dims = _CoordAndDims(cm, tuple(dims)) + self.cell_measures_and_dims.append(cm_and_dims) + + for av in sorted(cube.ancillary_variables(), key=meta_key_func): + dims = cube.ancillary_variable_dims(av) + metadata = _OtherMetaData(av, dims) + self.av_metadata.append(metadata) + av_and_dims = _CoordAndDims(av, tuple(dims)) + self.ancillary_variables_and_dims.append(av_and_dims) + def _coordinate_differences(self, other, attr): """ Determine the names of the coordinates that differ between `self` and @@ -442,6 +521,18 @@ def match(self, other, error_on_mismatch): msgs.append( msg_template.format("Auxiliary coordinates", *differences) ) + # Check cell measures. + if self.cm_metadata != other.cm_metadata: + differences = self._coordinate_differences(other, "cm_metadata") + msgs.append( + msg_template.format("Auxiliary coordinates", *differences) + ) + # Check ancillary variables. + if self.av_metadata != other.av_metadata: + differences = self._coordinate_differences(other, "av_metadata") + msgs.append( + msg_template.format("Auxiliary coordinates", *differences) + ) # Check scalar coordinates. if self.scalar_coords != other.scalar_coords: differences = self._coordinate_differences(other, "scalar_coords") @@ -463,17 +554,6 @@ def match(self, other, error_on_mismatch): ) ) - # Check _cell_measures_and_dims - if self.cell_measures_and_dims != other.cell_measures_and_dims: - msgs.append( - msg_template.format( - "CellMeasures", - "", - self.cell_measures_and_dims, - other.cell_measures_and_dims, - ) - ) - match = not bool(msgs) if error_on_mismatch and not match: raise iris.exceptions.ConcatenateError(msgs) @@ -500,6 +580,10 @@ def __init__(self, cube_signature): """ self.aux_coords_and_dims = cube_signature.aux_coords_and_dims + self.cell_measures_and_dims = cube_signature.cell_measures_and_dims + self.ancillary_variables_and_dims = ( + cube_signature.ancillary_variables_and_dims + ) self.dim_coords = cube_signature.dim_coords self.dim_mapping = cube_signature.dim_mapping self.dim_extents = [] @@ -676,20 +760,23 @@ def concatenate(self): # Concatenate the new auxiliary coordinates. aux_coords_and_dims = self._build_aux_coordinates() + # Concatenate the new cell measures + cell_measures_and_dims = self._build_cell_measures() + + # Concatenate the new ancillary variables + ancillary_variables_and_dims = self._build_ancillary_variables() + # Concatenate the new data payload. data = self._build_data() # Build the new cube. kwargs = cube_signature.defn._asdict() - new_cm_and_dims = [ - (deepcopy(cm), dims) - for cm, dims in self._cube._cell_measures_and_dims - ] cube = iris.cube.Cube( data, dim_coords_and_dims=dim_coords_and_dims, aux_coords_and_dims=aux_coords_and_dims, - cell_measures_and_dims=new_cm_and_dims, + cell_measures_and_dims=cell_measures_and_dims, + ancillary_variables_and_dims=ancillary_variables_and_dims, **kwargs, ) else: @@ -769,6 +856,36 @@ def register( if not coord_a == coord_b: match = False + # Check for compatible CellMeasures. + if match: + if check_aux_coords: + for coord_a, coord_b in zip( + self._cube_signature.cell_measures_and_dims, + cube_signature.cell_measures_and_dims, + ): + # CellMeasures that span the candidate axis can difffer + if ( + candidate_axis not in coord_a.dims + or candidate_axis not in coord_b.dims + ): + if not coord_a == coord_b: + match = False + + # Check for compatible AncillaryVariables. + if match: + if check_aux_coords: + for coord_a, coord_b in zip( + self._cube_signature.ancillary_variables_and_dims, + cube_signature.ancillary_variables_and_dims, + ): + # AncillaryVariables that span the candidate axis can difffer + if ( + candidate_axis not in coord_a.dims + or candidate_axis not in coord_b.dims + ): + if not coord_a == coord_b: + match = False + if match: # Register the cube as a source-cube for this proto-cube. self._add_skeleton(coord_signature, cube.lazy_data()) @@ -871,6 +988,84 @@ def _build_aux_coordinates(self): return aux_coords_and_dims + def _build_cell_measures(self): + """ + Generate the cell measures with associated dimension(s) + mapping for the new concatenated cube. + + Returns: + A list of cell measures and dimension(s) tuple pairs. + + """ + # Setup convenience hooks. + skeletons = self._skeletons + cube_signature = self._cube_signature + + cell_measures_and_dims = [] + + # Generate all the cell measures for the new concatenated cube. + for i, (cm, dims) in enumerate(cube_signature.cell_measures_and_dims): + # Check whether the cell measure spans the nominated + # dimension of concatenation. + if self.axis in dims: + # Concatenate the data together. + dim = dims.index(self.axis) + data = [ + skton.signature.cell_measures_and_dims[i].coord.data + for skton in skeletons + ] + data = np.concatenate(tuple(data), axis=dim) + + # Generate the associated metadata. + kwargs = cube_signature.cm_metadata[i].defn._asdict() + + # Build the concatenated coordinate. + cm = iris.coords.CellMeasure(data, **kwargs) + + cell_measures_and_dims.append((cm.copy(), dims)) + + return cell_measures_and_dims + + def _build_ancillary_variables(self): + """ + Generate the ancillary variables with associated dimension(s) + mapping for the new concatenated cube. + + Returns: + A list of ancillary variables and dimension(s) tuple pairs. + + """ + # Setup convenience hooks. + skeletons = self._skeletons + cube_signature = self._cube_signature + + ancillary_variables_and_dims = [] + + # Generate all the ancillary variables for the new concatenated cube. + for i, (av, dims) in enumerate( + cube_signature.ancillary_variables_and_dims + ): + # Check whether the ancillary variable spans the nominated + # dimension of concatenation. + if self.axis in dims: + # Concatenate the data together. + dim = dims.index(self.axis) + data = [ + skton.signature.ancillary_variables_and_dims[i].coord.data + for skton in skeletons + ] + data = np.concatenate(tuple(data), axis=dim) + + # Generate the associated metadata. + kwargs = cube_signature.av_metadata[i].defn._asdict() + + # Build the concatenated coordinate. + av = iris.coords.AncillaryVariable(data, **kwargs) + + ancillary_variables_and_dims.append((av.copy(), dims)) + + return ancillary_variables_and_dims + def _build_data(self): """ Generate the data payload for the new concatenated cube. diff --git a/lib/iris/coords.py b/lib/iris/coords.py index e491a65c91..3b3e87b7f4 100644 --- a/lib/iris/coords.py +++ b/lib/iris/coords.py @@ -368,7 +368,7 @@ def __ne__(self, other): return result def _as_defn(self): - defn = ( + defn = _DMDefn( self.standard_name, self.long_name, self.var_name, @@ -861,7 +861,7 @@ def __repr__(self): return result def _as_defn(self): - defn = ( + defn = CellMeasureDefn( self.standard_name, self.long_name, self.var_name, @@ -936,6 +936,105 @@ def _sort_key(defn): return _sort_key(self) < _sort_key(other) +class CellMeasureDefn( + namedtuple( + "CellMeasureDefn", + [ + "standard_name", + "long_name", + "var_name", + "units", + "attributes", + "measure", + ], + ) +): + """ + Criterion for identifying a specific type of :class:`CellMeasure` + based on its metadata. + + """ + + __slots__ = () + + def name(self, default="unknown"): + """ + Returns a human-readable name. + + First it tries self.standard_name, then it tries the 'long_name' + attribute, then the 'var_name' attribute, before falling back to + the value of `default` (which itself defaults to 'unknown'). + + """ + return self.standard_name or self.long_name or self.var_name or default + + def __lt__(self, other): + if not isinstance(other, CoordDefn): + return NotImplemented + + def _sort_key(defn): + # Emulate Python 2 behaviour with None + return ( + defn.standard_name is not None, + defn.standard_name, + defn.long_name is not None, + defn.long_name, + defn.var_name is not None, + defn.var_name, + defn.units is not None, + defn.units, + defn.measure is not None, + defn.measure, + ) + + return _sort_key(self) < _sort_key(other) + + +class _DMDefn( + namedtuple( + "DMDefn", + ["standard_name", "long_name", "var_name", "units", "attributes",], + ) +): + """ + Criterion for identifying a specific type of :class:`_DimensionalMetadata` + based on its metadata. + + """ + + __slots__ = () + + def name(self, default="unknown"): + """ + Returns a human-readable name. + + First it tries self.standard_name, then it tries the 'long_name' + attribute, then the 'var_name' attribute, before falling back to + the value of `default` (which itself defaults to 'unknown'). + + """ + return self.standard_name or self.long_name or self.var_name or default + + def __lt__(self, other): + if not isinstance(other, CoordDefn): + return NotImplemented + + def _sort_key(defn): + # Emulate Python 2 behaviour with None + return ( + defn.standard_name is not None, + defn.standard_name, + defn.long_name is not None, + defn.long_name, + defn.var_name is not None, + defn.var_name, + defn.units is not None, + defn.units, + ) + + return _sort_key(self) < _sort_key(other) + + class CoordExtent( namedtuple( "_CoordExtent", From d5cdafbaf4bdd94218b083a7104fb5661bd5fce6 Mon Sep 17 00:00:00 2001 From: "stephen.worsley" Date: Fri, 29 Nov 2019 12:14:39 +0000 Subject: [PATCH 02/10] added some tests, made some fixes --- lib/iris/coords.py | 4 +- .../concatenate/test_concatenate.py | 66 +++++++++++++++++ lib/iris/tests/test_concatenate.py | 72 ++++++++++++++++++- 3 files changed, 138 insertions(+), 4 deletions(-) diff --git a/lib/iris/coords.py b/lib/iris/coords.py index 3b3e87b7f4..c28c8b36aa 100644 --- a/lib/iris/coords.py +++ b/lib/iris/coords.py @@ -969,7 +969,7 @@ def name(self, default="unknown"): return self.standard_name or self.long_name or self.var_name or default def __lt__(self, other): - if not isinstance(other, CoordDefn): + if not isinstance(other, CellMeasureDefn): return NotImplemented def _sort_key(defn): @@ -1016,7 +1016,7 @@ def name(self, default="unknown"): return self.standard_name or self.long_name or self.var_name or default def __lt__(self, other): - if not isinstance(other, CoordDefn): + if not isinstance(other, _DMDefn): return NotImplemented def _sort_key(defn): diff --git a/lib/iris/tests/integration/concatenate/test_concatenate.py b/lib/iris/tests/integration/concatenate/test_concatenate.py index 1da146b97f..e2a858d9cf 100644 --- a/lib/iris/tests/integration/concatenate/test_concatenate.py +++ b/lib/iris/tests/integration/concatenate/test_concatenate.py @@ -99,6 +99,72 @@ def test_ignore_diff_aux_coord(self): self.assertEqual(result[0].shape, (4, 2)) +class Test_cubes_with_cell_measure(tests.IrisTest): + def create_cube(self): + data = np.arange(4).reshape(2, 2) + + lat = iris.coords.DimCoord( + [0, 30], standard_name="latitude", units="degrees" + ) + volume = iris.coords.CellMeasure( + [0, 15], measure="volume", long_name="volume" + ) + area = iris.coords.CellMeasure([1.5], standard_name="height", units="m") + t_unit = cf_units.Unit( + "hours since 1970-01-01 00:00:00", calendar="gregorian" + ) + time = iris.coords.DimCoord([0, 6], standard_name="time", units=t_unit) + + cube = iris.cube.Cube(data, standard_name="air_temperature", units="K") + cube.add_dim_coord(time, 0) + cube.add_dim_coord(lat, 1) + cube.add_cell_measure(volume, 1) + cube.add_cell_measure(area) + return cube + + def test_diff_cell_measure(self): + cube_a = self.create_cube() + cube_b = cube_a.copy() + cube_b.coord("time").points = [12, 18] + cube_b.cell_measure("volume").data = [120, 150] + + result = concatenate([cube_a, cube_b]) + self.assertEqual(len(result), 2) + + +class Test_cubes_with_ancillary_variables(tests.IrisTest): + def create_cube(self): + data = np.arange(4).reshape(2, 2) + + lat = iris.coords.DimCoord( + [0, 30], standard_name="latitude", units="degrees" + ) + quality = iris.coords.AncillaryVariable( + [0, 15], long_name="quality" + ) + height = iris.coords.AncillaryVariable([1.5], standard_name="height", units="m") + t_unit = cf_units.Unit( + "hours since 1970-01-01 00:00:00", calendar="gregorian" + ) + time = iris.coords.DimCoord([0, 6], standard_name="time", units=t_unit) + + cube = iris.cube.Cube(data, standard_name="air_temperature", units="K") + cube.add_dim_coord(time, 0) + cube.add_dim_coord(lat, 1) + cube.add_ancillary_variable(quality, 1) + cube.add_ancillary_variable(height) + return cube + + def test_diff_cell_measure(self): + cube_a = self.create_cube() + cube_b = cube_a.copy() + cube_b.coord("time").points = [12, 18] + cube_b.ancillary_variable("quality").data = [120, 150] + + result = concatenate([cube_a, cube_b]) + self.assertEqual(len(result), 2) + + class Test_anonymous_dims(tests.IrisTest): def setUp(self): data = np.arange(12).reshape(2, 3, 2) diff --git a/lib/iris/tests/test_concatenate.py b/lib/iris/tests/test_concatenate.py index aafd4d15be..bd81b54974 100644 --- a/lib/iris/tests/test_concatenate.py +++ b/lib/iris/tests/test_concatenate.py @@ -16,11 +16,13 @@ import numpy.ma as ma import iris.cube -from iris.coords import DimCoord, AuxCoord +from iris.coords import DimCoord, AuxCoord, CellMeasure, AncillaryVariable import iris.tests.stock as stock -def _make_cube(x, y, data, aux=None, offset=0, scalar=None): +def _make_cube( + x, y, data, aux=None, cell_measure=None, ancil=None, offset=0, scalar=None +): """ A convenience test function that creates a custom 2D cube. @@ -86,6 +88,36 @@ def _make_cube(x, y, data, aux=None, offset=0, scalar=None): coord = AuxCoord(payload * 100 + offset, long_name="xy-aux") cube.add_aux_coord(coord, (0, 1)) + if cell_measure is not None: + cell_measure = cell_measure.split(",") + if "y" in cell_measure: + cm = CellMeasure(y_range * 10, long_name="y-aux") + cube.add_cell_measure(cm, (0,)) + if "x" in cell_measure: + cm = CellMeasure(x_range * 10, long_name="x-aux") + cube.add_cell_measure(cm, (1,)) + if "xy" in cell_measure: + payload = np.arange(y_size * x_size, dtype=np.float32).reshape( + y_size, x_size + ) + cm = CellMeasure(payload * 100 + offset, long_name="xy-aux") + cube.add_cell_measure(cm, (0, 1)) + + if ancil is not None: + ancil = ancil.split(",") + if "y" in ancil: + av = AncillaryVariable(y_range * 10, long_name="y-aux") + cube.add_ancillary_variable(av, (0,)) + if "x" in ancil: + av = AncillaryVariable(x_range * 10, long_name="x-aux") + cube.add_ancillary_variable(av, (1,)) + if "xy" in ancil: + payload = np.arange(y_size * x_size, dtype=np.float32).reshape( + y_size, x_size + ) + av = AncillaryVariable(payload * 100 + offset, long_name="xy-aux") + cube.add_ancillary_variable(av, (0, 1)) + if scalar is not None: data = np.array([scalar], dtype=np.float32) coord = AuxCoord(data, long_name="height", units="m") @@ -306,6 +338,22 @@ def test_order_difference(self): result = concatenate(cubes) self.assertEqual(len(result), 2) + def test_cell_measure_missing(self): + cubes = [] + y = (0, 2) + cubes.append(_make_cube((0, 2), y, 1, cell_measure="x")) + cubes.append(_make_cube((2, 4), y, 2)) + result = concatenate(cubes) + self.assertEqual(len(result), 2) + + def test_ancil_missing(self): + cubes = [] + y = (0, 2) + cubes.append(_make_cube((0, 2), y, 1, ancil="x")) + cubes.append(_make_cube((2, 4), y, 2)) + result = concatenate(cubes) + self.assertEqual(len(result), 2) + class Test2D(tests.IrisTest): def test_masked_and_unmasked(self): @@ -658,6 +706,26 @@ def test_concat_2x2d_aux_xy_bounds(self): self.assertEqual(len(result), 1) self.assertEqual(result[0].shape, (2, 4)) + def test_concat_2y2d_cell_measure_x_y_xy(self): + cubes = [] + x = (0, 2) + cubes.append(_make_cube(x, (0, 4), 1, cell_measure="x,y,xy")) + cubes.append(_make_cube(x, (4, 6), 2, cell_measure="x,y,xy")) + result = concatenate(cubes) + # self.assertCML(result, ("concatenate", "concat_2y2d_cell_measure_x_y_xy.cml")) + self.assertEqual(len(result), 1) + self.assertEqual(result[0].shape, (6, 2)) + + def test_concat_2y2d_ancil_x_y_xy(self): + cubes = [] + x = (0, 2) + cubes.append(_make_cube(x, (0, 4), 1, ancil="x,y,xy")) + cubes.append(_make_cube(x, (4, 6), 2, ancil="x,y,xy")) + result = concatenate(cubes) + # self.assertCML(result, ("concatenate", "concat_2y2d_ancil_x_y_xy.cml")) + self.assertEqual(len(result), 1) + self.assertEqual(result[0].shape, (6, 2)) + class TestMulti2D(tests.IrisTest): def test_concat_4x2d_aux_xy(self): From 2fd3436860c16d13f415d25adbc086064f39e66a Mon Sep 17 00:00:00 2001 From: "stephen.worsley" Date: Fri, 29 Nov 2019 12:25:47 +0000 Subject: [PATCH 03/10] blackify --- .../tests/integration/concatenate/test_concatenate.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/iris/tests/integration/concatenate/test_concatenate.py b/lib/iris/tests/integration/concatenate/test_concatenate.py index e2a858d9cf..6319fd9b97 100644 --- a/lib/iris/tests/integration/concatenate/test_concatenate.py +++ b/lib/iris/tests/integration/concatenate/test_concatenate.py @@ -109,7 +109,9 @@ def create_cube(self): volume = iris.coords.CellMeasure( [0, 15], measure="volume", long_name="volume" ) - area = iris.coords.CellMeasure([1.5], standard_name="height", units="m") + area = iris.coords.CellMeasure( + [1.5], standard_name="height", units="m" + ) t_unit = cf_units.Unit( "hours since 1970-01-01 00:00:00", calendar="gregorian" ) @@ -139,10 +141,10 @@ def create_cube(self): lat = iris.coords.DimCoord( [0, 30], standard_name="latitude", units="degrees" ) - quality = iris.coords.AncillaryVariable( - [0, 15], long_name="quality" + quality = iris.coords.AncillaryVariable([0, 15], long_name="quality") + height = iris.coords.AncillaryVariable( + [1.5], standard_name="height", units="m" ) - height = iris.coords.AncillaryVariable([1.5], standard_name="height", units="m") t_unit = cf_units.Unit( "hours since 1970-01-01 00:00:00", calendar="gregorian" ) From a09f73929e439ac456591884c0adaef7c02d061b Mon Sep 17 00:00:00 2001 From: "stephen.worsley" Date: Fri, 29 Nov 2019 12:27:09 +0000 Subject: [PATCH 04/10] minor name change --- lib/iris/tests/integration/concatenate/test_concatenate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/iris/tests/integration/concatenate/test_concatenate.py b/lib/iris/tests/integration/concatenate/test_concatenate.py index 6319fd9b97..512ddab6b1 100644 --- a/lib/iris/tests/integration/concatenate/test_concatenate.py +++ b/lib/iris/tests/integration/concatenate/test_concatenate.py @@ -157,7 +157,7 @@ def create_cube(self): cube.add_ancillary_variable(height) return cube - def test_diff_cell_measure(self): + def test_diff_ancillary_variables(self): cube_a = self.create_cube() cube_b = cube_a.copy() cube_b.coord("time").points = [12, 18] From d5496103831b806bb11d1ac02d46fd9d9c0f62e6 Mon Sep 17 00:00:00 2001 From: "stephen.worsley" Date: Fri, 29 Nov 2019 13:21:28 +0000 Subject: [PATCH 05/10] separate strictness keywords --- lib/iris/_concatenate.py | 27 ++++++++--- lib/iris/cube.py | 45 +++++++++++++++++-- .../concatenate/test_concatenate.py | 20 +++++++++ 3 files changed, 83 insertions(+), 9 deletions(-) diff --git a/lib/iris/_concatenate.py b/lib/iris/_concatenate.py index 167274a2da..0263893b65 100644 --- a/lib/iris/_concatenate.py +++ b/lib/iris/_concatenate.py @@ -276,7 +276,13 @@ class _CoordExtent(namedtuple("CoordExtent", ["points", "bounds"])): __slots__ = () -def concatenate(cubes, error_on_mismatch=False, check_aux_coords=True): +def concatenate( + cubes, + error_on_mismatch=False, + check_aux_coords=True, + check_cell_measures=True, + check_ancils=True, +): """ Concatenate the provided cubes over common existing dimensions. @@ -311,7 +317,12 @@ def concatenate(cubes, error_on_mismatch=False, check_aux_coords=True): # Register cube with an existing proto-cube. for proto_cube in proto_cubes: registered = proto_cube.register( - cube, axis, error_on_mismatch, check_aux_coords + cube, + axis, + error_on_mismatch, + check_aux_coords, + check_cell_measures, + check_ancils, ) if registered: axis = proto_cube.axis @@ -787,7 +798,13 @@ def concatenate(self): return cube def register( - self, cube, axis=None, error_on_mismatch=False, check_aux_coords=False + self, + cube, + axis=None, + error_on_mismatch=False, + check_aux_coords=False, + check_cell_measures=False, + check_ancils=False, ): """ Determine whether the given source-cube is suitable for concatenation @@ -858,7 +875,7 @@ def register( # Check for compatible CellMeasures. if match: - if check_aux_coords: + if check_cell_measures: for coord_a, coord_b in zip( self._cube_signature.cell_measures_and_dims, cube_signature.cell_measures_and_dims, @@ -873,7 +890,7 @@ def register( # Check for compatible AncillaryVariables. if match: - if check_aux_coords: + if check_ancils: for coord_a, coord_b in zip( self._cube_signature.ancillary_variables_and_dims, cube_signature.ancillary_variables_and_dims, diff --git a/lib/iris/cube.py b/lib/iris/cube.py index 3ff77fa1ad..e839991661 100644 --- a/lib/iris/cube.py +++ b/lib/iris/cube.py @@ -489,7 +489,12 @@ def _none_sort(item): return merged_cubes - def concatenate_cube(self, check_aux_coords=True): + def concatenate_cube( + self, + check_aux_coords=True, + check_cell_measures=True, + check_ancils=True, + ): """ Return the concatenated contents of the :class:`CubeList` as a single :class:`Cube`. @@ -505,6 +510,16 @@ def concatenate_cube(self, check_aux_coords=True): is not applied to auxilliary coordinates that span the dimension the concatenation is occuring along. Defaults to True. + * check_cell_measures + Checks the cell measures of the cubes match. This check + is not applied to cell measures that span the dimension + the concatenation is occuring along. Defaults to True. + + * check_ancils + Checks the ancillary variables of the cubes match. This check + is not applied to ancillary variables that span the dimension + the concatenation is occuring along. Defaults to True. + .. note:: Concatenation cannot occur along an anonymous dimension. @@ -517,7 +532,11 @@ def concatenate_cube(self, check_aux_coords=True): unique_names = list(OrderedDict.fromkeys(names)) if len(unique_names) == 1: res = iris._concatenate.concatenate( - self, error_on_mismatch=True, check_aux_coords=check_aux_coords + self, + error_on_mismatch=True, + check_aux_coords=check_aux_coords, + check_cell_measures=check_cell_measures, + check_ancils=check_ancils, ) n_res_cubes = len(res) if n_res_cubes == 1: @@ -537,7 +556,12 @@ def concatenate_cube(self, check_aux_coords=True): ) raise iris.exceptions.ConcatenateError(msgs) - def concatenate(self, check_aux_coords=True): + def concatenate( + self, + check_aux_coords=True, + check_cell_measures=True, + check_ancils=True, + ): """ Concatenate the cubes over their common dimensions. @@ -548,6 +572,16 @@ def concatenate(self, check_aux_coords=True): is not applied to auxilliary coordinates that span the dimension the concatenation is occuring along. Defaults to True. + * check_cell_measures + Checks the cell measures of the cubes match. This check + is not applied to cell measures that span the dimension + the concatenation is occuring along. Defaults to True. + + * check_ancils + Checks the ancillary variables of the cubes match. This check + is not applied to ancillary variables that span the dimension + the concatenation is occuring along. Defaults to True. + Returns: A new :class:`iris.cube.CubeList` of concatenated :class:`iris.cube.Cube` instances. @@ -613,7 +647,10 @@ def concatenate(self, check_aux_coords=True): """ return iris._concatenate.concatenate( - self, check_aux_coords=check_aux_coords + self, + check_aux_coords=check_aux_coords, + check_cell_measures=check_cell_measures, + check_ancils=check_ancils, ) def realise_data(self): diff --git a/lib/iris/tests/integration/concatenate/test_concatenate.py b/lib/iris/tests/integration/concatenate/test_concatenate.py index 512ddab6b1..2ec99b6d4c 100644 --- a/lib/iris/tests/integration/concatenate/test_concatenate.py +++ b/lib/iris/tests/integration/concatenate/test_concatenate.py @@ -133,6 +133,16 @@ def test_diff_cell_measure(self): result = concatenate([cube_a, cube_b]) self.assertEqual(len(result), 2) + def test_ignore_diff_cell_measure(self): + cube_a = self.create_cube() + cube_b = cube_a.copy() + cube_b.coord("time").points = [12, 18] + cube_b.cell_measure("volume").data = [120, 150] + + result = concatenate([cube_a, cube_b], check_cell_measures=False) + self.assertEqual(len(result), 1) + self.assertEqual(result[0].shape, (4, 2)) + class Test_cubes_with_ancillary_variables(tests.IrisTest): def create_cube(self): @@ -166,6 +176,16 @@ def test_diff_ancillary_variables(self): result = concatenate([cube_a, cube_b]) self.assertEqual(len(result), 2) + def test_ignore_diff_ancillary_variables(self): + cube_a = self.create_cube() + cube_b = cube_a.copy() + cube_b.coord("time").points = [12, 18] + cube_b.ancillary_variable("quality").data = [120, 150] + + result = concatenate([cube_a, cube_b], check_ancils=False) + self.assertEqual(len(result), 1) + self.assertEqual(result[0].shape, (4, 2)) + class Test_anonymous_dims(tests.IrisTest): def setUp(self): From fb66db76f29691d8457cde331a2e4f1448414dbc Mon Sep 17 00:00:00 2001 From: "stephen.worsley" Date: Fri, 29 Nov 2019 15:14:45 +0000 Subject: [PATCH 06/10] add tests, add name support for remove_ancillary_variables --- lib/iris/_concatenate.py | 12 +++--- lib/iris/cube.py | 6 ++- .../unit/concatenate/test_concatenate.py | 42 +++++++++++++++++++ 3 files changed, 51 insertions(+), 9 deletions(-) diff --git a/lib/iris/_concatenate.py b/lib/iris/_concatenate.py index 0263893b65..646613d114 100644 --- a/lib/iris/_concatenate.py +++ b/lib/iris/_concatenate.py @@ -535,14 +535,12 @@ def match(self, other, error_on_mismatch): # Check cell measures. if self.cm_metadata != other.cm_metadata: differences = self._coordinate_differences(other, "cm_metadata") - msgs.append( - msg_template.format("Auxiliary coordinates", *differences) - ) + msgs.append(msg_template.format("Cell measures", *differences)) # Check ancillary variables. if self.av_metadata != other.av_metadata: differences = self._coordinate_differences(other, "av_metadata") msgs.append( - msg_template.format("Auxiliary coordinates", *differences) + msg_template.format("Ancillary variables", *differences) ) # Check scalar coordinates. if self.scalar_coords != other.scalar_coords: @@ -865,7 +863,7 @@ def register( self._cube_signature.aux_coords_and_dims, cube_signature.aux_coords_and_dims, ): - # AuxCoords that span the candidate axis can difffer + # AuxCoords that span the candidate axis can differ if ( candidate_axis not in coord_a.dims or candidate_axis not in coord_b.dims @@ -880,7 +878,7 @@ def register( self._cube_signature.cell_measures_and_dims, cube_signature.cell_measures_and_dims, ): - # CellMeasures that span the candidate axis can difffer + # CellMeasures that span the candidate axis can differ if ( candidate_axis not in coord_a.dims or candidate_axis not in coord_b.dims @@ -895,7 +893,7 @@ def register( self._cube_signature.ancillary_variables_and_dims, cube_signature.ancillary_variables_and_dims, ): - # AncillaryVariables that span the candidate axis can difffer + # AncillaryVariables that span the candidate axis can differ if ( candidate_axis not in coord_a.dims or candidate_axis not in coord_b.dims diff --git a/lib/iris/cube.py b/lib/iris/cube.py index e839991661..5dc9cfb2ba 100644 --- a/lib/iris/cube.py +++ b/lib/iris/cube.py @@ -1312,10 +1312,12 @@ def remove_ancillary_variable(self, ancillary_variable): Args: - * ancillary_variable (AncillaryVariable) - The AncillaryVariable to remove from the cube. + * ancillary_variable (string or AncillaryVariable) + The (name of the) AncillaryVariable to remove from the cube. """ + ancillary_variable = self.ancillary_variable(ancillary_variable) + self._ancillary_variables_and_dims = [ (ancillary_variable_, dim) for ancillary_variable_, dim in self._ancillary_variables_and_dims diff --git a/lib/iris/tests/unit/concatenate/test_concatenate.py b/lib/iris/tests/unit/concatenate/test_concatenate.py index 2253238e09..e65134fe90 100644 --- a/lib/iris/tests/unit/concatenate/test_concatenate.py +++ b/lib/iris/tests/unit/concatenate/test_concatenate.py @@ -90,6 +90,16 @@ def setUp(self): iris.coords.AuxCoord([0, 1, 2], long_name="foo", units="1"), data_dims=(1,), ) + cube.add_cell_measure( + iris.coords.CellMeasure([0, 1, 2], long_name="bar", units="1"), + data_dims=(1,), + ) + cube.add_ancillary_variable( + iris.coords.AncillaryVariable( + [0, 1, 2], long_name="baz", units="1" + ), + data_dims=(1,), + ) self.cube = cube def test_definition_difference_message(self): @@ -148,6 +158,38 @@ def test_scalar_coords_metadata_difference_message(self): with self.assertRaisesRegex(ConcatenateError, exc_regexp): _ = concatenate([cube_1, cube_2], True) + def test_cell_measure_difference_message(self): + cube_1 = self.cube + cube_2 = cube_1.copy() + cube_2.remove_cell_measure("bar") + exc_regexp = "Cell measures differ: .* != .*" + with self.assertRaisesRegex(ConcatenateError, exc_regexp): + _ = concatenate([cube_1, cube_2], True) + + def test_cell_measure_metadata_difference_message(self): + cube_1 = self.cube + cube_2 = cube_1.copy() + cube_2.cell_measure("bar").units = "m" + exc_regexp = "Cell measures metadata differ: .* != .*" + with self.assertRaisesRegex(ConcatenateError, exc_regexp): + _ = concatenate([cube_1, cube_2], True) + + def test_ancillary_variable_difference_message(self): + cube_1 = self.cube + cube_2 = cube_1.copy() + cube_2.remove_ancillary_variable("baz") + exc_regexp = "Ancillary variables differ: .* != .*" + with self.assertRaisesRegex(ConcatenateError, exc_regexp): + _ = concatenate([cube_1, cube_2], True) + + def test_ancillary_variable_metadata_difference_message(self): + cube_1 = self.cube + cube_2 = cube_1.copy() + cube_2.ancillary_variable("baz").units = "m" + exc_regexp = "Ancillary variables metadata differ: .* != .*" + with self.assertRaisesRegex(ConcatenateError, exc_regexp): + _ = concatenate([cube_1, cube_2], True) + def test_ndim_difference_message(self): cube_1 = self.cube cube_2 = iris.cube.Cube( From 1b56bf506ab84ecacbc3de13a5fd600fa8301557 Mon Sep 17 00:00:00 2001 From: "stephen.worsley" Date: Fri, 29 Nov 2019 15:44:29 +0000 Subject: [PATCH 07/10] remove whitespace --- lib/iris/cube.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/iris/cube.py b/lib/iris/cube.py index 5dc9cfb2ba..b592930b10 100644 --- a/lib/iris/cube.py +++ b/lib/iris/cube.py @@ -1306,7 +1306,6 @@ def remove_cell_measure(self, cell_measure): ] def remove_ancillary_variable(self, ancillary_variable): - """ Removes an ancillary variable from the cube. From 76a6a937a65c6032a1586691334f86f3d50b6325 Mon Sep 17 00:00:00 2001 From: "stephen.worsley" Date: Mon, 2 Dec 2019 11:28:44 +0000 Subject: [PATCH 08/10] added whatsnew --- .../bugfix_2019-Dec-02_cell_measure_concatenate.txt | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 docs/iris/src/whatsnew/contributions_3.0.0/bugfix_2019-Dec-02_cell_measure_concatenate.txt diff --git a/docs/iris/src/whatsnew/contributions_3.0.0/bugfix_2019-Dec-02_cell_measure_concatenate.txt b/docs/iris/src/whatsnew/contributions_3.0.0/bugfix_2019-Dec-02_cell_measure_concatenate.txt new file mode 100644 index 0000000000..151341d9af --- /dev/null +++ b/docs/iris/src/whatsnew/contributions_3.0.0/bugfix_2019-Dec-02_cell_measure_concatenate.txt @@ -0,0 +1,2 @@ +* Concatenating cubes along an axis shared by cell measures would cause concatenation to inappropriately fail. + These cell measures are now concatenated together in the resulting cube. \ No newline at end of file From 5f816b8b41aa68f2748edbe91e8338ecd502efb1 Mon Sep 17 00:00:00 2001 From: "stephen.worsley" Date: Mon, 16 Dec 2019 10:05:57 +0000 Subject: [PATCH 09/10] fix spelling --- lib/iris/cube.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/iris/cube.py b/lib/iris/cube.py index b592930b10..468be4644a 100644 --- a/lib/iris/cube.py +++ b/lib/iris/cube.py @@ -506,19 +506,19 @@ def concatenate_cube( Kwargs: * check_aux_coords - Checks the auxilliary coordinates of the cubes match. This check - is not applied to auxilliary coordinates that span the dimension - the concatenation is occuring along. Defaults to True. + Checks the auxiliary coordinates of the cubes match. This check + is not applied to auxiliary coordinates that span the dimension + the concatenation is occurring along. Defaults to True. * check_cell_measures Checks the cell measures of the cubes match. This check is not applied to cell measures that span the dimension - the concatenation is occuring along. Defaults to True. + the concatenation is occurring along. Defaults to True. * check_ancils Checks the ancillary variables of the cubes match. This check is not applied to ancillary variables that span the dimension - the concatenation is occuring along. Defaults to True. + the concatenation is occurring along. Defaults to True. .. note:: @@ -568,19 +568,19 @@ def concatenate( Kwargs: * check_aux_coords - Checks the auxilliary coordinates of the cubes match. This check - is not applied to auxilliary coordinates that span the dimension - the concatenation is occuring along. Defaults to True. + Checks the auxiliary coordinates of the cubes match. This check + is not applied to auxiliary coordinates that span the dimension + the concatenation is occurring along. Defaults to True. * check_cell_measures Checks the cell measures of the cubes match. This check is not applied to cell measures that span the dimension - the concatenation is occuring along. Defaults to True. + the concatenation is occurring along. Defaults to True. * check_ancils Checks the ancillary variables of the cubes match. This check is not applied to ancillary variables that span the dimension - the concatenation is occuring along. Defaults to True. + the concatenation is occurring along. Defaults to True. Returns: A new :class:`iris.cube.CubeList` of concatenated From e2c1552059f3606c6f40bcee6306121b60892970 Mon Sep 17 00:00:00 2001 From: "stephen.worsley" Date: Tue, 17 Dec 2019 15:00:54 +0000 Subject: [PATCH 10/10] bypass cml checking for ancils and cell measures --- lib/iris/tests/test_concatenate.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/lib/iris/tests/test_concatenate.py b/lib/iris/tests/test_concatenate.py index bd81b54974..bbe5f5eba2 100644 --- a/lib/iris/tests/test_concatenate.py +++ b/lib/iris/tests/test_concatenate.py @@ -97,9 +97,7 @@ def _make_cube( cm = CellMeasure(x_range * 10, long_name="x-aux") cube.add_cell_measure(cm, (1,)) if "xy" in cell_measure: - payload = np.arange(y_size * x_size, dtype=np.float32).reshape( - y_size, x_size - ) + payload = x_range + y_range[:, np.newaxis] cm = CellMeasure(payload * 100 + offset, long_name="xy-aux") cube.add_cell_measure(cm, (0, 1)) @@ -112,9 +110,7 @@ def _make_cube( av = AncillaryVariable(x_range * 10, long_name="x-aux") cube.add_ancillary_variable(av, (1,)) if "xy" in ancil: - payload = np.arange(y_size * x_size, dtype=np.float32).reshape( - y_size, x_size - ) + payload = x_range + y_range[:, np.newaxis] av = AncillaryVariable(payload * 100 + offset, long_name="xy-aux") cube.add_ancillary_variable(av, (0, 1)) @@ -710,21 +706,23 @@ def test_concat_2y2d_cell_measure_x_y_xy(self): cubes = [] x = (0, 2) cubes.append(_make_cube(x, (0, 4), 1, cell_measure="x,y,xy")) - cubes.append(_make_cube(x, (4, 6), 2, cell_measure="x,y,xy")) + cubes.append(_make_cube(x, (4, 6), 1, cell_measure="x,y,xy")) result = concatenate(cubes) - # self.assertCML(result, ("concatenate", "concat_2y2d_cell_measure_x_y_xy.cml")) + com = _make_cube(x, (0, 6), 1, cell_measure="x,y,xy") self.assertEqual(len(result), 1) self.assertEqual(result[0].shape, (6, 2)) + self.assertEqual(result[0], com) def test_concat_2y2d_ancil_x_y_xy(self): cubes = [] x = (0, 2) cubes.append(_make_cube(x, (0, 4), 1, ancil="x,y,xy")) - cubes.append(_make_cube(x, (4, 6), 2, ancil="x,y,xy")) + cubes.append(_make_cube(x, (4, 6), 1, ancil="x,y,xy")) result = concatenate(cubes) - # self.assertCML(result, ("concatenate", "concat_2y2d_ancil_x_y_xy.cml")) + com = _make_cube(x, (0, 6), 1, ancil="x,y,xy") self.assertEqual(len(result), 1) self.assertEqual(result[0].shape, (6, 2)) + self.assertEqual(result[0], com) class TestMulti2D(tests.IrisTest):