diff --git a/docs/iris/src/whatsnew/contributions_3.0.0/docchange_2020-Apr-03_extract_clarification.txt b/docs/iris/src/whatsnew/contributions_3.0.0/docchange_2020-Apr-03_extract_clarification.txt deleted file mode 100644 index 48e2a99e57..0000000000 --- a/docs/iris/src/whatsnew/contributions_3.0.0/docchange_2020-Apr-03_extract_clarification.txt +++ /dev/null @@ -1 +0,0 @@ -* Updated the documentation for the :meth:`iris.cube.CubeList.extract` method, to specify when a single cube, as opposed to a CubeList, may be returned. diff --git a/docs/iris/src/whatsnew/contributions_3.0.0/incompatiblechange_2020-May-22_cubelist_extract_cubes.txt b/docs/iris/src/whatsnew/contributions_3.0.0/incompatiblechange_2020-May-22_cubelist_extract_cubes.txt new file mode 100644 index 0000000000..ed8e6a8e2c --- /dev/null +++ b/docs/iris/src/whatsnew/contributions_3.0.0/incompatiblechange_2020-May-22_cubelist_extract_cubes.txt @@ -0,0 +1,10 @@ +* The method :meth:`~iris.cube.CubeList.extract_strict`, and the 'strict' + keyword to :meth:`~iris.cube.CubeList.extract` method have been removed, and + are replaced by the new routines :meth:`~iris.cube.CubeList.extract_cube` and + :meth:`~iris.cube.CubeList.extract_cubes`. + The new routines perform the same operation, but in a style more like other + Iris functions such as :meth:`iris.load_cube` and :meth:`iris.load_cubes`. + Unlike 'strict extraction', the type of return value is now completely + consistent : :meth:`~iris.cube.CubeList.extract_cube` always returns a cube, + and :meth:`~iris.cube.CubeList.extract_cubes` always returns a CubeList of a + length equal to the number of constraints. diff --git a/lib/iris/_constraints.py b/lib/iris/_constraints.py index ab213dfa7b..37daeec4aa 100644 --- a/lib/iris/_constraints.py +++ b/lib/iris/_constraints.py @@ -428,7 +428,7 @@ def list_of_constraints(constraints): using :func:`as_constraint`. """ - if not isinstance(constraints, (list, tuple)): + if isinstance(constraints, str) or not isinstance(constraints, Iterable): constraints = [constraints] return [as_constraint(constraint) for constraint in constraints] diff --git a/lib/iris/cube.py b/lib/iris/cube.py index fec68c575c..1b1a4d7b9a 100644 --- a/lib/iris/cube.py +++ b/lib/iris/cube.py @@ -268,7 +268,7 @@ def xml(self, checksum=False, order=True, byteorder=True): # return our newly created XML string return doc.toprettyxml(indent=" ") - def extract(self, constraints, strict=False): + def extract(self, constraints): """ Filter each of the cubes which can be filtered by the given constraints. @@ -278,23 +278,57 @@ def extract(self, constraints, strict=False): **n** when filtered with **m** constraints can generate a maximum of **m * n** cubes. - Keywords: + Args: + + * constraints (:class:`~iris.Constraint` or iterable of constraints): + A single constraint or an iterable. + + """ + return self._extract_and_merge(self, constraints, strict=False) + + def extract_cube(self, constraint): + """ + Extract a single cube from a CubeList, and return it. + Raise an error if the extract produces no cubes, or more than one. + + Args: + + * constraint (:class:`~iris.Constraint`): + The constraint to extract with. - * strict - boolean - If strict is True, then there must be exactly one cube which is - filtered per constraint. Note: if a single constraint is given, a - Cube is returned rather than a CubeList. + .. see also:: + :meth:`~iris.cube.CubeList.extract` """ + # Just validate this, so we can accept strings etc, but not multiples. + constraint = iris._constraints.as_constraint(constraint) return self._extract_and_merge( - self, constraints, strict, merge_unique=None + self, constraint, strict=True, return_single_cube=True ) - @staticmethod - def _extract_and_merge(cubes, constraints, strict, merge_unique=False): - # * merge_unique - if None: no merging, if false: non unique merging, - # else unique merging (see merge) + def extract_cubes(self, constraints): + """ + Extract specific cubes from a CubeList, one for each given constraint. + Each constraint must produce exactly one cube, otherwise an error is + raised. + + Args: + * constraints (iterable of, or single, :class:`~iris.Constraint`): + The constraints to extract with. + + .. see also:: + :meth:`~iris.cube.CubeList.extract` + + """ + return self._extract_and_merge( + self, constraints, strict=True, return_single_cube=False + ) + + @staticmethod + def _extract_and_merge( + cubes, constraints, strict=False, return_single_cube=False + ): constraints = iris._constraints.list_of_constraints(constraints) # group the resultant cubes by constraints in a dictionary @@ -307,10 +341,6 @@ def _extract_and_merge(cubes, constraints, strict, merge_unique=False): if sub_cube is not None: cube_list.append(sub_cube) - if merge_unique is not None: - for constraint, cubelist in constraint_groups.items(): - constraint_groups[constraint] = cubelist.merge(merge_unique) - result = CubeList() for constraint in constraints: constraint_cubes = constraint_groups[constraint] @@ -322,18 +352,18 @@ def _extract_and_merge(cubes, constraints, strict, merge_unique=False): raise iris.exceptions.ConstraintMismatchError(msg) result.extend(constraint_cubes) - if strict and len(constraints) == 1: + if return_single_cube: + if len(result) != 1: + # Practically this should never occur, as we now *only* request + # single cube result for 'extract_cube'. + msg = "Got {!s} cubes for constraints {!r}, expecting 1." + raise iris.exceptions.ConstraintMismatchError( + msg.format(len(result), constraints) + ) result = result[0] return result - def extract_strict(self, constraints): - """ - Calls :meth:`CubeList.extract` with the strict keyword set to True. - - """ - return self.extract(constraints, strict=True) - def extract_overlapping(self, coord_names): """ Returns a :class:`CubeList` of cubes extracted over regions diff --git a/lib/iris/tests/test_constraints.py b/lib/iris/tests/test_constraints.py index 1f0bf064ed..0026fe0ee8 100644 --- a/lib/iris/tests/test_constraints.py +++ b/lib/iris/tests/test_constraints.py @@ -292,7 +292,7 @@ class TestCubeListStrictConstraint(StrictConstraintMixin, tests.IrisTest): suffix = "load_strict" def load_match(self, files, constraints): - cubes = iris.load(files).extract_strict(constraints) + cubes = iris.load(files).extract_cubes(constraints) return cubes @@ -317,25 +317,25 @@ def setUp(self): def test_standard_name(self): constraint = iris.Constraint(self.standard_name) - result = self.cubes.extract(constraint, strict=True) + result = self.cubes.extract_cube(constraint) self.assertIsNotNone(result) self.assertEqual(result.standard_name, self.standard_name) def test_long_name(self): constraint = iris.Constraint(self.long_name) - result = self.cubes.extract(constraint, strict=True) + result = self.cubes.extract_cube(constraint) self.assertIsNotNone(result) self.assertEqual(result.long_name, self.long_name) def test_var_name(self): constraint = iris.Constraint(self.var_name) - result = self.cubes.extract(constraint, strict=True) + result = self.cubes.extract_cube(constraint) self.assertIsNotNone(result) self.assertEqual(result.var_name, self.var_name) def test_stash(self): constraint = iris.Constraint(self.stash) - result = self.cubes.extract(constraint, strict=True) + result = self.cubes.extract_cube(constraint) self.assertIsNotNone(result) self.assertEqual(str(result.attributes["STASH"]), self.stash) @@ -348,7 +348,7 @@ def test_unknown(self): cube.attributes = None # Extract the unknown cube. constraint = iris.Constraint("unknown") - result = self.cubes.extract(constraint, strict=True) + result = self.cubes.extract_cube(constraint) self.assertIsNotNone(result) self.assertEqual(result.name(), "unknown") @@ -380,14 +380,14 @@ def test_standard_name(self): # Match. constraint = NameConstraint(standard_name=self.standard_name) - result = self.cubes.extract(constraint, strict=True) + result = self.cubes.extract_cube(constraint) self.assertIsNotNone(result) self.assertEqual(result.standard_name, self.standard_name) # Match - callable. kwargs = dict(standard_name=lambda item: item.startswith("air_pot")) constraint = NameConstraint(**kwargs) - result = self.cubes.extract(constraint, strict=True) + result = self.cubes.extract_cube(constraint) self.assertIsNotNone(result) self.assertEqual(result.standard_name, self.standard_name) @@ -397,7 +397,7 @@ def test_standard_name__None(self): constraint = NameConstraint( standard_name=None, long_name=self.long_name ) - result = self.cubes.extract(constraint, strict=True) + result = self.cubes.extract_cube(constraint) self.assertIsNotNone(result) self.assertIsNone(result.standard_name) self.assertEqual(result.long_name, self.long_name) @@ -410,7 +410,7 @@ def test_long_name(self): # Match. constraint = NameConstraint(long_name=self.long_name) - result = self.cubes.extract(constraint, strict=True) + result = self.cubes.extract_cube(constraint) self.assertIsNotNone(result) self.assertEqual(result.long_name, self.long_name) @@ -420,7 +420,7 @@ def test_long_name(self): and item.startswith("air pot") ) constraint = NameConstraint(**kwargs) - result = self.cubes.extract(constraint, strict=True) + result = self.cubes.extract_cube(constraint) self.assertIsNotNone(result) self.assertEqual(result.long_name, self.long_name) @@ -430,7 +430,7 @@ def test_long_name__None(self): constraint = NameConstraint( standard_name=self.standard_name, long_name=None ) - result = self.cubes.extract(constraint, strict=True) + result = self.cubes.extract_cube(constraint) self.assertIsNotNone(result) self.assertEqual(result.standard_name, self.standard_name) self.assertIsNone(result.long_name) @@ -443,14 +443,14 @@ def test_var_name(self): # Match. constraint = NameConstraint(var_name=self.var_name) - result = self.cubes.extract(constraint, strict=True) + result = self.cubes.extract_cube(constraint) self.assertIsNotNone(result) self.assertEqual(result.var_name, self.var_name) # Match - callable. kwargs = dict(var_name=lambda item: item.startswith("ap")) constraint = NameConstraint(**kwargs) - result = self.cubes.extract(constraint, strict=True) + result = self.cubes.extract_cube(constraint) self.assertIsNotNone(result) self.assertEqual(result.var_name, self.var_name) @@ -460,7 +460,7 @@ def test_var_name__None(self): constraint = NameConstraint( standard_name=self.standard_name, var_name=None ) - result = self.cubes.extract(constraint, strict=True) + result = self.cubes.extract_cube(constraint) self.assertIsNotNone(result) self.assertEqual(result.standard_name, self.standard_name) self.assertIsNone(result.var_name) @@ -473,14 +473,14 @@ def test_stash(self): # Match. constraint = NameConstraint(STASH=self.stash) - result = self.cubes.extract(constraint, strict=True) + result = self.cubes.extract_cube(constraint) self.assertIsNotNone(result) self.assertEqual(str(result.attributes["STASH"]), self.stash) # Match - callable. kwargs = dict(STASH=lambda stash: stash.item == 4) constraint = NameConstraint(**kwargs) - result = self.cubes.extract(constraint, strict=True) + result = self.cubes.extract_cube(constraint) self.assertIsNotNone(result) def test_stash__None(self): @@ -489,7 +489,7 @@ def test_stash__None(self): constraint = NameConstraint( standard_name=self.standard_name, STASH=None ) - result = self.cubes.extract(constraint, strict=True) + result = self.cubes.extract_cube(constraint) self.assertIsNotNone(result) self.assertEqual(result.standard_name, self.standard_name) self.assertIsNone(result.attributes.get("STASH")) @@ -499,7 +499,7 @@ def test_compound(self): constraint = NameConstraint( standard_name=self.standard_name, long_name=self.long_name ) - result = self.cubes.extract(constraint, strict=True) + result = self.cubes.extract_cube(constraint) self.assertIsNotNone(result) self.assertEqual(result.standard_name, self.standard_name) @@ -518,7 +518,7 @@ def test_compound(self): long_name=self.long_name, var_name=self.var_name, ) - result = self.cubes.extract(constraint, strict=True) + result = self.cubes.extract_cube(constraint) self.assertIsNotNone(result) self.assertEqual(result.standard_name, self.standard_name) self.assertEqual(result.long_name, self.long_name) @@ -541,7 +541,7 @@ def test_compound(self): var_name=self.var_name, STASH=self.stash, ) - result = self.cubes.extract(constraint, strict=True) + result = self.cubes.extract_cube(constraint) self.assertIsNotNone(result) self.assertEqual(result.standard_name, self.standard_name) self.assertEqual(result.long_name, self.long_name) @@ -571,7 +571,7 @@ def test_unknown(self): cube.var_name = None cube.attributes = None constraint = NameConstraint(None, None, None, None) - result = self.cubes.extract(constraint, strict=True) + result = self.cubes.extract_cube(constraint) self.assertIsNotNone(result) self.assertIsNone(result.standard_name) self.assertIsNone(result.long_name) diff --git a/lib/iris/tests/test_merge.py b/lib/iris/tests/test_merge.py index a6e2991fd0..1e6aff8666 100644 --- a/lib/iris/tests/test_merge.py +++ b/lib/iris/tests/test_merge.py @@ -79,9 +79,7 @@ def custom_coord_callback(cube, field, filename): # Load slices, decorating a coord with custom attributes cubes = iris.load_raw(self._data_path, callback=custom_coord_callback) # Merge - merged = iris.cube.CubeList._extract_and_merge( - cubes, constraints=None, strict=False, merge_unique=False - ) + merged = iris.cube.CubeList(cubes).merge() # Check the custom attributes are in the merged cube for cube in merged: assert cube.coord("time").attributes["monty"] == "python" diff --git a/lib/iris/tests/unit/cube/test_CubeList.py b/lib/iris/tests/unit/cube/test_CubeList.py index 6870e2367f..985c5b6576 100644 --- a/lib/iris/tests/unit/cube/test_CubeList.py +++ b/lib/iris/tests/unit/cube/test_CubeList.py @@ -17,6 +17,7 @@ from cf_units import Unit import numpy as np +from iris import Constraint from iris.cube import Cube, CubeList from iris.coords import AuxCoord, DimCoord import iris.coord_systems @@ -294,6 +295,252 @@ def test_scalar_cube_data_constraint(self): self.assertEqual(res, expected) +class ExtractMixin: + # Choose "which" extract method to test. + # Effectively "abstract" -- inheritor must define this property : + # method_name = 'extract_cube' / 'extract_cubes' + + def setUp(self): + self.cube_x = Cube(0, long_name="x") + self.cube_y = Cube(0, long_name="y") + self.cons_x = Constraint("x") + self.cons_y = Constraint("y") + self.cons_any = Constraint(cube_func=lambda cube: True) + self.cons_none = Constraint(cube_func=lambda cube: False) + + def check_extract(self, cubes, constraints, expected): + # Check that extracting a cubelist with the given arguments has the + # expected result. + # 'expected' and the operation results can be: + # * None + # * a single cube + # * a list of cubes --> cubelist (with cubes matching) + # * string --> a ConstraintMatchException matching the string + cubelist = CubeList(cubes) + method = getattr(cubelist, self.method_name) + if isinstance(expected, str): + with self.assertRaisesRegex( + iris.exceptions.ConstraintMismatchError, expected + ): + method(constraints) + else: + result = method(constraints) + if expected is None: + self.assertIsNone(result) + elif isinstance(expected, Cube): + self.assertIsInstance(result, Cube) + self.assertEqual(result, expected) + elif isinstance(expected, list): + self.assertIsInstance(result, CubeList) + self.assertEqual(result, expected) + else: + msg = ( + 'Unhandled usage in "check_extract" call: ' + '"expected" arg has type {}, value {}.' + ) + raise ValueError(msg.format(type(expected), expected)) + + +class Test_extract_cube(ExtractMixin, tests.IrisTest): + method_name = "extract_cube" + + def test_empty(self): + self.check_extract([], self.cons_x, "Got 0 cubes .* expecting 1") + + def test_single_cube_ok(self): + self.check_extract([self.cube_x], self.cons_x, self.cube_x) + + def test_single_cube_fail__too_few(self): + self.check_extract( + [self.cube_x], self.cons_y, "Got 0 cubes .* expecting 1" + ) + + def test_single_cube_fail__too_many(self): + self.check_extract( + [self.cube_x, self.cube_y], + self.cons_any, + "Got 2 cubes .* expecting 1", + ) + + def test_string_as_constraint(self): + # Check that we can use a string, that converts to a constraint + # ( via "as_constraint" ). + self.check_extract([self.cube_x], "x", self.cube_x) + + def test_none_as_constraint(self): + # Check that we can use a None, that converts to a constraint + # ( via "as_constraint" ). + self.check_extract([self.cube_x], None, self.cube_x) + + def test_constraint_in_list__fail(self): + # Check that we *cannot* use [constraint] + msg = "cannot be cast to a constraint" + with self.assertRaisesRegex(TypeError, msg): + self.check_extract([], [self.cons_x], []) + + def test_multi_cube_ok(self): + self.check_extract( + [self.cube_x, self.cube_y], self.cons_x, self.cube_x + ) # NOTE: returns a cube + + def test_multi_cube_fail__too_few(self): + self.check_extract( + [self.cube_x, self.cube_y], + self.cons_none, + "Got 0 cubes .* expecting 1", + ) + + def test_multi_cube_fail__too_many(self): + self.check_extract( + [self.cube_x, self.cube_y], + self.cons_any, + "Got 2 cubes .* expecting 1", + ) + + +class ExtractCubesMixin(ExtractMixin): + method_name = "extract_cubes" + + +class Test_extract_cubes__noconstraint(ExtractCubesMixin, tests.IrisTest): + """Test with an empty list of constraints.""" + + def test_empty(self): + self.check_extract([], [], []) + + def test_single_cube(self): + self.check_extract([self.cube_x], [], []) + + def test_multi_cubes(self): + self.check_extract([self.cube_x, self.cube_y], [], []) + + +class ExtractCubesSingleConstraintMixin(ExtractCubesMixin): + """ + Common code for testing extract_cubes with a single constraint. + Generalised, so that we can do the same tests for a "bare" constraint, + and a list containing a single [constraint]. + + """ + + # Effectively "abstract" -- inheritor must define this property : + # wrap_test_constraint_as_list_of_one = True / False + + def check_extract(self, cubes, constraint, result): + # Overload standard test operation. + if self.wrap_test_constraint_as_list_of_one: + constraint = [constraint] + super().check_extract(cubes, constraint, result) + + def test_empty(self): + self.check_extract([], self.cons_x, "Got 0 cubes .* expecting 1") + + def test_single_cube_ok(self): + self.check_extract( + [self.cube_x], self.cons_x, [self.cube_x] + ) # NOTE: always returns list NOT cube + + def test_single_cube__fail_mismatch(self): + self.check_extract( + [self.cube_x], self.cons_y, "Got 0 cubes .* expecting 1" + ) + + def test_multi_cube_ok(self): + self.check_extract( + [self.cube_x, self.cube_y], self.cons_x, [self.cube_x] + ) # NOTE: always returns list NOT cube + + def test_multi_cube__fail_too_few(self): + self.check_extract( + [self.cube_x, self.cube_y], + self.cons_none, + "Got 0 cubes .* expecting 1", + ) + + def test_multi_cube__fail_too_many(self): + self.check_extract( + [self.cube_x, self.cube_y], + self.cons_any, + "Got 2 cubes .* expecting 1", + ) + + +class Test_extract_cubes__bare_single_constraint( + ExtractCubesSingleConstraintMixin, tests.IrisTest +): + """Testing with a single constraint as the argument.""" + + wrap_test_constraint_as_list_of_one = False + + +class Test_extract_cubes__list_single_constraint( + ExtractCubesSingleConstraintMixin, tests.IrisTest +): + """Testing with a list of one constraint as the argument.""" + + wrap_test_constraint_as_list_of_one = True + + +class Test_extract_cubes__multi_constraints(ExtractCubesMixin, tests.IrisTest): + """ + Testing when the 'constraints' arg is a list of multiple constraints. + """ + + def test_empty(self): + # Always fails. + self.check_extract( + [], [self.cons_x, self.cons_any], "Got 0 cubes .* expecting 1" + ) + + def test_single_cube_ok(self): + # Possible if the one cube matches all the constraints. + self.check_extract( + [self.cube_x], + [self.cons_x, self.cons_any], + [self.cube_x, self.cube_x], + ) + + def test_single_cube__fail_too_few(self): + self.check_extract( + [self.cube_x], + [self.cons_x, self.cons_y], + "Got 0 cubes .* expecting 1", + ) + + def test_multi_cube_ok(self): + self.check_extract( + [self.cube_x, self.cube_y], + [self.cons_y, self.cons_x], # N.B. reverse order ! + [self.cube_y, self.cube_x], + ) + + def test_multi_cube_castable_constraint_args(self): + # Check with args that *aren't* constraints, but can be converted + # ( via "as_constraint" ). + self.check_extract( + [self.cube_x, self.cube_y], + ["y", "x", self.cons_y], + [self.cube_y, self.cube_x, self.cube_y], + ) + + # NOTE: not bothering to check we can cast a 'None', as it will anyway + # fail with multiple input cubes. + + def test_multi_cube__fail_too_few(self): + self.check_extract( + [self.cube_x, self.cube_y], + [self.cons_x, self.cons_y, self.cons_none], + "Got 0 cubes .* expecting 1", + ) + + def test_multi_cube__fail_too_many(self): + self.check_extract( + [self.cube_x, self.cube_y], + [self.cons_x, self.cons_y, self.cons_any], + "Got 2 cubes .* expecting 1", + ) + + class Test_iteration(tests.IrisTest): def setUp(self): self.scalar_cubes = CubeList()