Initial insertion of Ugrid into CF loading.

Author: pp-mo

lib/iris/fileformats/_ugrid_cf_reader.py

@@ -0,0 +1,75 @@
+# Copyright Iris contributors
+#
+# This file is part of Iris and is released under the LGPL license.
+# See COPYING and COPYING.LESSER in the root of the repository for full
+# licensing details.
+"""
+Adds a UGRID extension layer to netCDF file loading.
+
+"""
+import os
+
+import netCDF4
+
+from gridded.pyugrid.ugrid import UGrid
+from gridded.pyugrid.read_netcdf import (
+    find_mesh_names,
+    load_grid_from_nc_dataset,
+)
+from iris.fileformats.cf import CFReader
+
+
+_UGRID_ELEMENT_TYPE_NAMES = ("node", "edge", "face", "volume")
+
+_UGRID_LINK_PROPERTIES = [
+    "{}_coordinates".format(elem) for elem in _UGRID_ELEMENT_TYPE_NAMES
+]
+_UGRID_LINK_PROPERTIES += [
+    "{}_{}_connectivity".format(e1, e2)
+    for e1 in _UGRID_ELEMENT_TYPE_NAMES
+    for e2 in _UGRID_ELEMENT_TYPE_NAMES
+]
+
+# print('')
+# print('Ugrid link properties ...')
+# print('\n'.join(_UGRID_LINK_PROPERTIES))
+
+
+class UGridCFReader:
+    def __init__(self, filename, *args, **kwargs):
+        self.filename = os.path.expanduser(filename)
+        dataset = netCDF4.Dataset(self.filename, mode="r")
+        self.dataset = dataset
+        meshes = {}
+        for meshname in find_mesh_names(self.dataset):
+            mesh = UGrid()
+            load_grid_from_nc_dataset(dataset, mesh, mesh_name=meshname)
+            meshes[meshname] = mesh
+        self.meshes = meshes
+        # Generate list of excluded variable names.
+        exclude_vars = list(meshes.keys())
+        for mesh in meshes.values():
+            mesh_var = dataset.variables[mesh.mesh_name]
+            for attr in mesh_var.ncattrs():
+                if attr in _UGRID_LINK_PROPERTIES:
+                    exclude_vars.extend(mesh_var.getncattr(attr).split())
+        print("")
+        print("File {}".format(filename, meshes))
+        print("File meshes={}".format(meshes))
+        print("")
+        print("File exclude vars={}".format(exclude_vars))
+        print("")
+        print(
+            "File remaining vars={}".format(
+                [x for x in dataset.variables.keys() if x not in exclude_vars]
+            )
+        )
+        kwargs["exclude_var_names"] = exclude_vars
+        self.cfreader = CFReader(self.dataset, *args, **kwargs)
+
+    def complete_ugrid_cube(self, cube):
+        pass
+
+    def __del__(self):
+        # Explicitly close dataset to prevent file remaining open.
+        self.dataset.close()

lib/iris/fileformats/cf.py

@@ -17,11 +17,9 @@
 from abc import ABCMeta, abstractmethod
 
 from collections.abc import Iterable, MutableMapping
-import os
 import re
 import warnings
 
-import netCDF4
 import numpy as np
 import numpy.ma as ma
 
@@ -1008,8 +1006,12 @@ class CFReader:
 
     """
 
-    def __init__(self, filename, warn=False, monotonic=False):
-        self._filename = os.path.expanduser(filename)
+    def __init__(
+        self, dataset, warn=False, monotonic=False, exclude_var_names=None
+    ):
+        self._dataset = dataset
+        self._filename = dataset.filepath()
+
         # All CF variable types EXCEPT for the "special cases" of
         # CFDataVariable, CFCoordinateVariable and _CFFormulaTermsVariable.
         self._variable_types = (
@@ -1025,8 +1027,6 @@ def __init__(self, filename, warn=False, monotonic=False):
         #: Collection of CF-netCDF variables associated with this netCDF file
         self.cf_group = CFGroup()
 
-        self._dataset = netCDF4.Dataset(self._filename, mode="r")
-
         # Issue load optimisation warning.
         if warn and self._dataset.file_format in [
             "NETCDF3_CLASSIC",
@@ -1260,10 +1260,6 @@ def _reset(self):
         for nc_var_name in self._dataset.variables.keys():
             self.cf_group[nc_var_name].cf_attrs_reset()
 
-    def __del__(self):
-        # Explicitly close dataset to prevent file remaining open.
-        self._dataset.close()
-
 
 def _getncattr(dataset, attr, default=None):
     """

lib/iris/fileformats/netcdf.py

@@ -44,6 +44,7 @@
 import iris.exceptions
 import iris.fileformats.cf
 import iris.fileformats._pyke_rules
+from iris.fileformats._ugrid_cf_reader import UGridCFReader
 import iris.io
 import iris.util
 from iris._lazy_data import as_lazy_data
@@ -778,14 +779,17 @@ def load_cubes(filenames, callback=None):
 
     for filename in filenames:
         # Ingest the netCDF file.
-        cf = iris.fileformats.cf.CFReader(filename)
+        #         ncreader = iris.fileformats.cf.CFReader(filename)
+        ncreader = UGridCFReader(filename)
 
         # Process each CF data variable.
-        data_variables = list(cf.cf_group.data_variables.values()) + list(
-            cf.cf_group.promoted.values()
-        )
+        data_variables = list(
+            ncreader.cfreader.cf_group.data_variables.values()
+        ) + list(ncreader.cfreader.cf_group.promoted.values())
         for cf_var in data_variables:
-            cube = _load_cube(engine, cf, cf_var, filename)
+            cube = _load_cube(engine, ncreader.cfreader, cf_var, filename)
+
+            ncreader.complete_ugrid_cube(cube)
 
             # Process any associated formula terms and attach
             # the corresponding AuxCoordFactory.

lib/iris/tests/integration/ugrid_cf_reader/__init__.py

@@ -0,0 +1,10 @@
+# Copyright Iris contributors
+#
+# This file is part of Iris and is released under the LGPL license.
+# See COPYING and COPYING.LESSER in the root of the repository for full
+# licensing details.
+"""
+Integration tests for the
+:mod:`iris.fileformats._structured_array_identification` module.
+
+"""

lib/iris/tests/integration/ugrid_cf_reader/test_ugrid_load.py

@@ -0,0 +1,29 @@
+# Copyright Iris contributors
+#
+# This file is part of Iris and is released under the LGPL license.
+# See COPYING and COPYING.LESSER in the root of the repository for full
+# licensing details.
+"""
+Integration tests for the
+:mod:`iris.fileformats._ugrid_cf_reader.UGridCFReader` class.
+
+"""
+
+# Import iris.tests first so that some things can be initialised before
+# importing anything else.
+import iris.tests as tests
+
+import iris
+
+
+class TestUgrid(tests.IrisTest):
+    def test_basic_load(self):
+        fp = "/project/avd/ng-vat/data/sam_adams_tiny_test_files/lfric_diag.nc"
+        cubes = iris.load(fp)
+        print(cubes)
+        cube = iris.load_cube(fp, "theta")
+        print(cube)
+
+
+if __name__ == "__main__":
+    tests.main()

lib/iris/tests/unit/fileformats/structured_array_identification/__init__.py

@@ -5,6 +5,6 @@
 # licensing details.
 """
 Unit tests for the
-:mod:`iris.fileformats._structured_array_identification` module.
+:mod:`iris.fileformats._ugrid_cf_reader` module.
 
 """