[FB] launch_ancils: Update merge-base to commit 42a1964

docs/iris/src/whatsnew/contributions_3.0.0/bugfix_2019-Nov-21_cell_measure_attributes.txt

@@ -0,0 +1,2 @@
+* Fixed a bug where the attributes of cell measures in netcdf-CF files were discarded on
+  loading.  They now appear on the CellMeasure in the loaded cube.

docs/iris/src/whatsnew/contributions_3.0.0/newfeature_2019-Nov-21_netcdf_ancillary_data.txt

@@ -0,0 +1,2 @@
+* CF Ancillary Variables are now loaded from and saved to netcdf-CF files.
+

lib/iris/fileformats/netcdf.py

@@ -460,7 +460,10 @@ def __setstate__(self, state):
 
 def _assert_case_specific_facts(engine, cf, cf_group):
     # Initialise pyke engine "provides" hooks.
-    engine.provides["coordinates"] = []
+    # These are used to patch non-processed element attributes after rules activation.
+    engine.cube_parts["coordinates"] = []
+    engine.cube_parts["cell_measures"] = []
+    engine.cube_parts["ancillary_variables"] = []
 
     # Assert facts for CF coordinates.
     for cf_name in cf_group.coordinates.keys():
@@ -480,6 +483,12 @@ def _assert_case_specific_facts(engine, cf, cf_group):
             _PYKE_FACT_BASE, "cell_measure", (cf_name,)
         )
 
+    # Assert facts for CF ancillary variables.
+    for cf_name in cf_group.ancillary_variables.keys():
+        engine.add_case_specific_fact(
+            _PYKE_FACT_BASE, "ancillary_variable", (cf_name,)
+        )
+
     # Assert facts for CF grid_mappings.
     for cf_name in cf_group.grid_mappings.keys():
         engine.add_case_specific_fact(
@@ -587,7 +596,7 @@ def _load_cube(engine, cf, cf_var, filename):
     # Initialise pyke engine rule processing hooks.
     engine.cf_var = cf_var
     engine.cube = cube
-    engine.provides = {}
+    engine.cube_parts = {}
     engine.requires = {}
     engine.rule_triggered = set()
     engine.filename = filename
@@ -598,31 +607,38 @@ def _load_cube(engine, cf, cf_var, filename):
     # Run pyke inference engine with forward chaining rules.
     engine.activate(_PYKE_RULE_BASE)
 
-    # Populate coordinate attributes with the untouched attributes from the
-    # associated CF-netCDF variable.
-    coordinates = engine.provides.get("coordinates", [])
-
+    # Having run the rules, now populate the attributes of all the cf elements with the
+    # "unused" attributes from the associated CF-netCDF variable.
+    # That is, all those that aren't CF reserved terms.
     def attribute_predicate(item):
         return item[0] not in _CF_ATTRS
 
-    for coord, cf_var_name in coordinates:
-        tmpvar = filter(
-            attribute_predicate, cf.cf_group[cf_var_name].cf_attrs_unused()
-        )
+    def add_unused_attributes(iris_object, cf_var):
+        tmpvar = filter(attribute_predicate, cf_var.cf_attrs_unused())
         for attr_name, attr_value in tmpvar:
-            _set_attributes(coord.attributes, attr_name, attr_value)
+            _set_attributes(iris_object.attributes, attr_name, attr_value)
+
+    def fix_attributes_all_elements(role_name):
+        elements_and_names = engine.cube_parts.get(role_name, [])
+
+        for iris_object, cf_var_name in elements_and_names:
+            add_unused_attributes(iris_object, cf.cf_group[cf_var_name])
+
+    # Populate the attributes of all coordinates, cell-measures and ancillary-vars.
+    fix_attributes_all_elements("coordinates")
+    fix_attributes_all_elements("ancillary_variables")
+    fix_attributes_all_elements("cell_measures")
 
-    tmpvar = filter(attribute_predicate, cf_var.cf_attrs_unused())
-    # Attach untouched attributes of the associated CF-netCDF data variable to
-    # the cube.
-    for attr_name, attr_value in tmpvar:
-        _set_attributes(cube.attributes, attr_name, attr_value)
+    # Also populate attributes of the top-level cube itself.
+    add_unused_attributes(cube, cf_var)
 
+    # Work out reference names for all the coords.
     names = {
         coord.var_name: coord.standard_name or coord.var_name or "unknown"
         for coord in cube.coords()
     }
 
+    # Add all the cube cell methods.
     cube.cell_methods = [
         iris.coords.CellMethod(
             method=method.method,
@@ -662,7 +678,7 @@ def coord_from_term(term):
             # Convert term names to coordinates (via netCDF variable names).
             name = engine.requires["formula_terms"].get(term, None)
             if name is not None:
-                for coord, cf_var_name in engine.provides["coordinates"]:
+                for coord, cf_var_name in engine.cube_parts["coordinates"]:
                     if cf_var_name == name:
                         return coord
                 warnings.warn(

lib/iris/tests/test_netcdf.py

@@ -32,13 +32,21 @@
 from iris.fileformats.netcdf import load_cubes as nc_load_cubes
 import iris.std_names
 import iris.util
+from iris.coords import AncillaryVariable, CellMeasure
 import iris.coord_systems as icoord_systems
 import iris.tests.stock as stock
 from iris._lazy_data import is_lazy_data
 
 
 @tests.skip_data
 class TestNetCDFLoad(tests.IrisTest):
+    def setUp(self):
+        self.tmpdir = None
+
+    def tearDown(self):
+        if self.tmpdir is not None:
+            shutil.rmtree(self.tmpdir)
+
     def test_monotonic(self):
         cubes = iris.load(
             tests.get_data_path(
@@ -243,6 +251,153 @@ def test_cell_methods(self):
 
         self.assertCML(cubes, ("netcdf", "netcdf_cell_methods.cml"))
 
+    def test_ancillary_variables(self):
+        # Note: using a CDL string as a test data reference, rather than a binary file.
+        ref_cdl = """
+            netcdf cm_attr {
+            dimensions:
+                axv = 3 ;
+            variables:
+                int64 qqv(axv) ;
+                    qqv:long_name = "qq" ;
+                    qqv:units = "1" ;
+                    qqv:ancillary_variables = "my_av" ;
+                int64 axv(axv) ;
+                    axv:units = "1" ;
+                    axv:long_name = "x" ;
+                double my_av(axv) ;
+                    my_av:units = "1" ;
+                    my_av:long_name = "refs" ;
+                    my_av:custom = "extra-attribute";
+            data:
+                axv = 1, 2, 3;
+                my_av = 11., 12., 13.;
+            }
+            """
+        self.tmpdir = tempfile.mkdtemp()
+        cdl_path = os.path.join(self.tmpdir, "tst.cdl")
+        nc_path = os.path.join(self.tmpdir, "tst.nc")
+        # Write CDL string into a temporary CDL file.
+        with open(cdl_path, "w") as f_out:
+            f_out.write(ref_cdl)
+        # Use ncgen to convert this into an actual (temporary) netCDF file.
+        command = "ncgen -o {} {}".format(nc_path, cdl_path)
+        check_call(command, shell=True)
+        # Load with iris.fileformats.netcdf.load_cubes, and check expected content.
+        cubes = list(nc_load_cubes(nc_path))
+        self.assertEqual(len(cubes), 1)
+        avs = cubes[0].ancillary_variables()
+        self.assertEqual(len(avs), 1)
+        expected = AncillaryVariable(
+            np.ma.array([11.0, 12.0, 13.0]),
+            long_name="refs",
+            var_name="my_av",
+            units="1",
+            attributes={"custom": "extra-attribute"},
+        )
+        self.assertEqual(avs[0], expected)
+
+    def test_status_flags(self):
+        # Note: using a CDL string as a test data reference, rather than a binary file.
+        ref_cdl = """
+            netcdf cm_attr {
+            dimensions:
+                axv = 3 ;
+            variables:
+                int64 qqv(axv) ;
+                    qqv:long_name = "qq" ;
+                    qqv:units = "1" ;
+                    qqv:ancillary_variables = "my_av" ;
+                int64 axv(axv) ;
+                    axv:units = "1" ;
+                    axv:long_name = "x" ;
+                byte my_av(axv) ;
+                    my_av:long_name = "qq status_flag" ;
+                    my_av:flag_values = 1b, 2b ;
+                    my_av:flag_meanings = "a b" ;
+            data:
+                axv = 11, 21, 31;
+                my_av = 1b, 1b, 2b;
+            }
+            """
+        self.tmpdir = tempfile.mkdtemp()
+        cdl_path = os.path.join(self.tmpdir, "tst.cdl")
+        nc_path = os.path.join(self.tmpdir, "tst.nc")
+        # Write CDL string into a temporary CDL file.
+        with open(cdl_path, "w") as f_out:
+            f_out.write(ref_cdl)
+        # Use ncgen to convert this into an actual (temporary) netCDF file.
+        command = "ncgen -o {} {}".format(nc_path, cdl_path)
+        check_call(command, shell=True)
+        # Load with iris.fileformats.netcdf.load_cubes, and check expected content.
+        cubes = list(nc_load_cubes(nc_path))
+        self.assertEqual(len(cubes), 1)
+        avs = cubes[0].ancillary_variables()
+        self.assertEqual(len(avs), 1)
+        expected = AncillaryVariable(
+            np.ma.array([1, 1, 2], dtype=np.int8),
+            long_name="qq status_flag",
+            var_name="my_av",
+            units="no_unit",
+            attributes={
+                "flag_values": np.array([1, 2], dtype=np.int8),
+                "flag_meanings": "a b",
+            },
+        )
+        self.assertEqual(avs[0], expected)
+
+    def test_cell_measures(self):
+        # Note: using a CDL string as a test data reference, rather than a binary file.
+        ref_cdl = """
+            netcdf cm_attr {
+            dimensions:
+                axv = 3 ;
+                ayv = 2 ;
+            variables:
+                int64 qqv(ayv, axv) ;
+                    qqv:long_name = "qq" ;
+                    qqv:units = "1" ;
+                    qqv:cell_measures = "area: my_areas" ;
+                int64 ayv(ayv) ;
+                    ayv:units = "1" ;
+                    ayv:long_name = "y" ;
+                int64 axv(axv) ;
+                    axv:units = "1" ;
+                    axv:long_name = "x" ;
+                double my_areas(ayv, axv) ;
+                    my_areas:units = "m2" ;
+                    my_areas:long_name = "standardised cell areas" ;
+                    my_areas:custom = "extra-attribute";
+            data:
+                axv = 11, 12, 13;
+                ayv = 21, 22;
+                my_areas = 110., 120., 130., 221., 231., 241.;
+            }
+            """
+        self.tmpdir = tempfile.mkdtemp()
+        cdl_path = os.path.join(self.tmpdir, "tst.cdl")
+        nc_path = os.path.join(self.tmpdir, "tst.nc")
+        # Write CDL string into a temporary CDL file.
+        with open(cdl_path, "w") as f_out:
+            f_out.write(ref_cdl)
+        # Use ncgen to convert this into an actual (temporary) netCDF file.
+        command = "ncgen -o {} {}".format(nc_path, cdl_path)
+        check_call(command, shell=True)
+        # Load with iris.fileformats.netcdf.load_cubes, and check expected content.
+        cubes = list(nc_load_cubes(nc_path))
+        self.assertEqual(len(cubes), 1)
+        cms = cubes[0].cell_measures()
+        self.assertEqual(len(cms), 1)
+        expected = CellMeasure(
+            np.ma.array([[110.0, 120.0, 130.0], [221.0, 231.0, 241.0]]),
+            measure="area",
+            var_name="my_areas",
+            long_name="standardised cell areas",
+            units="m2",
+            attributes={"custom": "extra-attribute"},
+        )
+        self.assertEqual(cms[0], expected)
+
     def test_deferred_loading(self):
         # Test exercising CF-netCDF deferred loading and deferred slicing.
         # shape (31, 161, 320)

lib/iris/tests/unit/fileformats/netcdf/test__load_aux_factory.py

@@ -26,8 +26,10 @@ def setUp(self):
         self.ap = mock.MagicMock(units="units")
         self.ps = mock.MagicMock(units="units")
         coordinates = [(mock.sentinel.b, "b"), (self.ps, "ps")]
-        self.provides = dict(coordinates=coordinates)
-        self.engine = mock.Mock(requires=self.requires, provides=self.provides)
+        self.cube_parts = dict(coordinates=coordinates)
+        self.engine = mock.Mock(
+            requires=self.requires, cube_parts=self.cube_parts
+        )
         self.cube = mock.create_autospec(Cube, spec_set=True, instance=True)
         # Patch out the check_dependencies functionality.
         func = "iris.aux_factory.HybridPressureFactory._check_dependencies"
@@ -36,7 +38,7 @@ def setUp(self):
         self.addCleanup(patcher.stop)
 
     def test_formula_terms_ap(self):
-        self.provides["coordinates"].append((self.ap, "ap"))
+        self.cube_parts["coordinates"].append((self.ap, "ap"))
         self.requires["formula_terms"] = dict(ap="ap", b="b", ps="ps")
         _load_aux_factory(self.engine, self.cube)
         # Check cube.add_aux_coord method.
@@ -59,7 +61,9 @@ def test_formula_terms_a_p0(self):
             long_name="vertical pressure",
             var_name="ap",
         )
-        self.provides["coordinates"].extend([(coord_a, "a"), (coord_p0, "p0")])
+        self.cube_parts["coordinates"].extend(
+            [(coord_a, "a"), (coord_p0, "p0")]
+        )
         self.requires["formula_terms"] = dict(a="a", b="b", ps="ps", p0="p0")
         _load_aux_factory(self.engine, self.cube)
         # Check cube.coord_dims method.
@@ -84,15 +88,17 @@ def test_formula_terms_a_p0(self):
 
     def test_formula_terms_p0_non_scalar(self):
         coord_p0 = DimCoord(np.arange(5))
-        self.provides["coordinates"].append((coord_p0, "p0"))
+        self.cube_parts["coordinates"].append((coord_p0, "p0"))
         self.requires["formula_terms"] = dict(p0="p0")
         with self.assertRaises(ValueError):
             _load_aux_factory(self.engine, self.cube)
 
     def test_formula_terms_p0_bounded(self):
         coord_a = DimCoord(np.arange(5))
         coord_p0 = DimCoord(1, bounds=[0, 2], var_name="p0")
-        self.provides["coordinates"].extend([(coord_a, "a"), (coord_p0, "p0")])
+        self.cube_parts["coordinates"].extend(
+            [(coord_a, "a"), (coord_p0, "p0")]
+        )
         self.requires["formula_terms"] = dict(a="a", b="b", ps="ps", p0="p0")
         with warnings.catch_warnings(record=True) as warn:
             warnings.simplefilter("always")
@@ -133,14 +139,14 @@ def test_formula_terms_no_delta_terms(self):
 
     def test_formula_terms_no_p0_term(self):
         coord_a = DimCoord(np.arange(5), units="Pa")
-        self.provides["coordinates"].append((coord_a, "a"))
+        self.cube_parts["coordinates"].append((coord_a, "a"))
         self.requires["formula_terms"] = dict(a="a", b="b", ps="ps")
         _load_aux_factory(self.engine, self.cube)
         self._check_no_delta()
 
     def test_formula_terms_no_a_term(self):
         coord_p0 = DimCoord(10, units="1")
-        self.provides["coordinates"].append((coord_p0, "p0"))
+        self.cube_parts["coordinates"].append((coord_p0, "p0"))
         self.requires["formula_terms"] = dict(a="p0", b="b", ps="ps")
         _load_aux_factory(self.engine, self.cube)
         self._check_no_delta()

lib/iris/tests/unit/fileformats/netcdf/test__load_cube.py

@@ -25,7 +25,7 @@ def _patcher(engine, cf, cf_group):
         for coord in cf_group:
             engine.cube.add_aux_coord(coord)
             coordinates.append((coord, coord.name()))
-        engine.provides["coordinates"] = coordinates
+        engine.cube_parts["coordinates"] = coordinates
 
     def setUp(self):
         this = "iris.fileformats.netcdf._assert_case_specific_facts"