From acef7fa9600dfb3d10fa9944aed55989c9707bac Mon Sep 17 00:00:00 2001 From: Daniel Kirkham Date: Thu, 19 Oct 2017 17:33:27 +0100 Subject: [PATCH 1/3] Don't remove mask on no-mask masked arrays returned from methods --- lib/iris/_merge.py | 4 ---- lib/iris/analysis/__init__.py | 2 -- lib/iris/cube.py | 5 ----- lib/iris/experimental/regrid_conservative.py | 4 ---- 4 files changed, 15 deletions(-) diff --git a/lib/iris/_merge.py b/lib/iris/_merge.py index ee87cefe82..01bc531e7d 100644 --- a/lib/iris/_merge.py +++ b/lib/iris/_merge.py @@ -1239,10 +1239,6 @@ def merge(self, unique=True): # normal array. dtype = self._cube_signature.data_type merged_data = as_concrete_data(merged_data) - # Unmask the array if it has no masked points. - if (ma.isMaskedArray(merged_data) and - not ma.is_masked(merged_data)): - merged_data = merged_data.data merged_cube = self._get_cube(merged_data) merged_cubes.append(merged_cube) diff --git a/lib/iris/analysis/__init__.py b/lib/iris/analysis/__init__.py index 0b1d94fbc5..1b884f6fdc 100644 --- a/lib/iris/analysis/__init__.py +++ b/lib/iris/analysis/__init__.py @@ -1050,8 +1050,6 @@ def _percentile(data, axis, percent, fast_percentile_method=False, quantiles = np.array(percent) / 100. result = scipy.stats.mstats.mquantiles(data, quantiles, axis=-1, **kwargs) - if not ma.isMaskedArray(data) and not ma.is_masked(result): - result = np.asarray(result) # Ensure to unflatten any leading dimensions. if shape: diff --git a/lib/iris/cube.py b/lib/iris/cube.py index 4597edc3ee..5610c0a639 100644 --- a/lib/iris/cube.py +++ b/lib/iris/cube.py @@ -2094,11 +2094,6 @@ def new_cell_measure_dims(cm_): # We don't want a view of the data, so take a copy of it. data = deepcopy(data) - # We can turn a masked array into a normal array if it's full. - if ma.isMaskedArray(data): - if ma.count_masked(data) == 0: - data = data.filled() - # XXX: Slicing a single item from a masked array that is masked, # results in numpy (v1.11.1) *always* returning a MaskedConstant # with a dtype of float64, regardless of the original masked diff --git a/lib/iris/experimental/regrid_conservative.py b/lib/iris/experimental/regrid_conservative.py index d8dc2219a0..65072a9771 100644 --- a/lib/iris/experimental/regrid_conservative.py +++ b/lib/iris/experimental/regrid_conservative.py @@ -274,10 +274,6 @@ def _valid_units(coord): # Paste regridded slice back into parent array fullcube_data[slice_indices_tuple] = data - # Remove the data mask if completely unused. - if not np.ma.is_masked(fullcube_data): - fullcube_data = np.array(fullcube_data) - # Generate a full 2d sample grid, as required for regridding orography # NOTE: as seen in "regrid_bilinear_rectilinear_src_and_grid" # TODO: can this not also be wound into the _create_cube method ? From f63d9501f0f23ddb348fa39bff35c37753f8b59b Mon Sep 17 00:00:00 2001 From: Daniel Kirkham Date: Thu, 26 Oct 2017 14:20:29 +0100 Subject: [PATCH 2/3] Undo removing check in _percentile() and conservate regrid --- lib/iris/analysis/__init__.py | 2 ++ lib/iris/experimental/regrid_conservative.py | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/lib/iris/analysis/__init__.py b/lib/iris/analysis/__init__.py index 1b884f6fdc..0b1d94fbc5 100644 --- a/lib/iris/analysis/__init__.py +++ b/lib/iris/analysis/__init__.py @@ -1050,6 +1050,8 @@ def _percentile(data, axis, percent, fast_percentile_method=False, quantiles = np.array(percent) / 100. result = scipy.stats.mstats.mquantiles(data, quantiles, axis=-1, **kwargs) + if not ma.isMaskedArray(data) and not ma.is_masked(result): + result = np.asarray(result) # Ensure to unflatten any leading dimensions. if shape: diff --git a/lib/iris/experimental/regrid_conservative.py b/lib/iris/experimental/regrid_conservative.py index 65072a9771..d8dc2219a0 100644 --- a/lib/iris/experimental/regrid_conservative.py +++ b/lib/iris/experimental/regrid_conservative.py @@ -274,6 +274,10 @@ def _valid_units(coord): # Paste regridded slice back into parent array fullcube_data[slice_indices_tuple] = data + # Remove the data mask if completely unused. + if not np.ma.is_masked(fullcube_data): + fullcube_data = np.array(fullcube_data) + # Generate a full 2d sample grid, as required for regridding orography # NOTE: as seen in "regrid_bilinear_rectilinear_src_and_grid" # TODO: can this not also be wound into the _create_cube method ? From f56b303c440a80f64240983a0f9c1dbb5b90e8a6 Mon Sep 17 00:00:00 2001 From: Daniel Kirkham Date: Thu, 26 Oct 2017 14:38:06 +0100 Subject: [PATCH 3/3] Add unit tests --- lib/iris/tests/test_merge.py | 18 ++++++++++++++++++ lib/iris/tests/unit/cube/test_Cube.py | 7 +++++++ 2 files changed, 25 insertions(+) diff --git a/lib/iris/tests/test_merge.py b/lib/iris/tests/test_merge.py index 97967fc008..c7bdfd8042 100644 --- a/lib/iris/tests/test_merge.py +++ b/lib/iris/tests/test_merge.py @@ -248,6 +248,24 @@ def test__masked_ndarray(self): self.assertEqual(result.dtype, self.dtype) self._check_fill_value(result, fill0=fill) + def test_maksed_array_preserved(self): + for (lazy0, lazy1), (fill,) in self.mixed_combos: + cubes = iris.cube.CubeList() + mask = False + cubes.append(self._make_cube(0, mask=mask, lazy=lazy0, + dtype=self.dtype, + fill_value=fill)) + cubes.append(self._make_cube(1, lazy=lazy1, dtype=self.dtype)) + result = cubes.merge_cube() + mask = False + expected_fill_value = self._expected_fill_value(fill) + expected = self._make_data([0, 1], mask=mask, dtype=self.dtype, + fill_value=expected_fill_value) + self.assertEqual(type(result.data), ma.MaskedArray) + self.assertMaskedArrayEqual(result.data, expected) + self.assertEqual(result.dtype, self.dtype) + self._check_fill_value(result, fill0=fill) + def test_fill_value_invariant_to_order__same_non_None(self): fill_value = 1234 cubes = [self._make_cube(i, mask=True, diff --git a/lib/iris/tests/unit/cube/test_Cube.py b/lib/iris/tests/unit/cube/test_Cube.py index 21b1352feb..89aaef355d 100644 --- a/lib/iris/tests/unit/cube/test_Cube.py +++ b/lib/iris/tests/unit/cube/test_Cube.py @@ -59,6 +59,13 @@ def test_masked(self): self.assertEqual(type(cube.data), ma.MaskedArray) self.assertMaskedArrayEqual(cube.data, data) + def test_masked_no_mask(self): + # ma.MaskedArray should be allowed through even if it has no mask + data = ma.masked_array(np.arange(12).reshape(3, 4), False) + cube = Cube(data) + self.assertEqual(type(cube.data), ma.MaskedArray) + self.assertMaskedArrayEqual(cube.data, data) + def test_matrix(self): # Subclasses of np.ndarray should be coerced back to np.ndarray. # (Except for np.ma.MaskedArray.)