diff --git a/.travis.yml b/.travis.yml index 792f6a6195..a9f6bf3bfe 100644 --- a/.travis.yml +++ b/.travis.yml @@ -105,7 +105,7 @@ install: # Conda-forge versioning is out of order (0.9.* is later than 2.12.*). - > if [[ "${TEST_MINIMAL}" != true ]]; then - conda install --quiet -n ${ENV_NAME} python-eccodes=0.9.3; + conda install --quiet -n ${ENV_NAME} python-eccodes">=0.9.1, <2"; conda install --quiet -n ${ENV_NAME} --no-deps iris-grib; fi @@ -162,13 +162,18 @@ script: fi # Split the organisation out of the slug. See https://stackoverflow.com/a/5257398/741316 for description. - - export ORG=(${TRAVIS_REPO_SLUG//\// }) + # NOTE: a *separate* "export" command appears to be necessary here : A command of the + # form "export ORG=.." failed to define ORG for the following command (?!) + - > + ORG=$(echo ${TRAVIS_REPO_SLUG} | cut -d/ -f1); + export ORG + + - echo "Travis job context ORG=${ORG}; TRAVIS_EVENT_TYPE=${TRAVIS_EVENT_TYPE}; PUSH_BUILT_DOCS=${PUSH_BUILT_DOCS}" # When we merge a change to SciTools/iris, we can push docs to github pages. # At present, only the Python 3.7 "doctest" job does this. # Results appear at https://scitools-docs.github.io/iris/<>/index.html - - > - if [[ "${ORG}" == 'SciTools' && "${TRAVIS_EVENT_TYPE}" == 'push' && "${PUSH_BUILT_DOCS}" == 'true' ]]; then + - if [[ "${ORG}" == 'SciTools' && "${TRAVIS_EVENT_TYPE}" == 'push' && "${PUSH_BUILT_DOCS}" == 'true' ]]; then cd ${INSTALL_DIR}; pip install doctr; doctr deploy --deploy-repo SciTools-docs/iris --built-docs docs/iris/build/html diff --git a/docs/iris/src/whatsnew/contributions_3.0.0/docchange_2019-Dec-04_black_code_formatting.txt b/docs/iris/src/whatsnew/contributions_3.0.0/docchange_2019-Dec-04_black_code_formatting.txt new file mode 100644 index 0000000000..500a215bb9 --- /dev/null +++ b/docs/iris/src/whatsnew/contributions_3.0.0/docchange_2019-Dec-04_black_code_formatting.txt @@ -0,0 +1,6 @@ +* Added support for the `black `_ code formatter. + This is now automatically checked on GitHub PRs, replacing the older, unittest-based + "iris.tests.test_coding_standards.TestCodeFormat". + Black provides automatic code format correction for most IDEs. + See the new developer guide section on this : + https://scitools-docs.github.io/iris/master/developers_guide/code_format.html. diff --git a/lib/iris/experimental/regrid.py b/lib/iris/experimental/regrid.py index 87d9e7e902..afbf87dee8 100644 --- a/lib/iris/experimental/regrid.py +++ b/lib/iris/experimental/regrid.py @@ -473,6 +473,24 @@ def _regrid_area_weighted_array( grid. """ + # Determine which grid bounds are within src extent. + y_within_bounds = _within_bounds( + src_y_bounds, grid_y_bounds, grid_y_decreasing + ) + x_within_bounds = _within_bounds( + src_x_bounds, grid_x_bounds, grid_x_decreasing + ) + + # Cache which src_bounds are within grid bounds + cached_x_bounds = [] + cached_x_indices = [] + for (x_0, x_1) in grid_x_bounds: + if grid_x_decreasing: + x_0, x_1 = x_1, x_0 + x_bounds, x_indices = _cropped_bounds(src_x_bounds, x_0, x_1) + cached_x_bounds.append(x_bounds) + cached_x_indices.append(x_indices) + # Create empty data array to match the new grid. # Note that dtype is not preserved and that the array is # masked to allow for regions that do not overlap. @@ -497,24 +515,6 @@ def _regrid_area_weighted_array( # Assign to mask to explode it, allowing indexed assignment. new_data.mask = False - # Determine which grid bounds are within src extent. - y_within_bounds = _within_bounds( - src_y_bounds, grid_y_bounds, grid_y_decreasing - ) - x_within_bounds = _within_bounds( - src_x_bounds, grid_x_bounds, grid_x_decreasing - ) - - # Cache which src_bounds are within grid bounds - cached_x_bounds = [] - cached_x_indices = [] - for (x_0, x_1) in grid_x_bounds: - if grid_x_decreasing: - x_0, x_1 = x_1, x_0 - x_bounds, x_indices = _cropped_bounds(src_x_bounds, x_0, x_1) - cached_x_bounds.append(x_bounds) - cached_x_indices.append(x_indices) - # Axes of data over which the weighted mean is calculated. axes = [] if y_dim is not None: @@ -565,15 +565,15 @@ def _regrid_area_weighted_array( raise RuntimeError( "Cannot handle split bounds " "in both x and y." ) + # Calculate weights based on areas of cropped bounds. + weights = area_func(y_bounds, x_bounds) + if x_dim is not None: indices[x_dim] = x_indices if y_dim is not None: indices[y_dim] = y_indices data = src_data[tuple(indices)] - # Calculate weights based on areas of cropped bounds. - weights = area_func(y_bounds, x_bounds) - # Transpose weights to match dim ordering in data. weights_shape_y = weights.shape[0] weights_shape_x = weights.shape[1] diff --git a/lib/iris/fileformats/_pyke_rules/fc_rules_cf.krb b/lib/iris/fileformats/_pyke_rules/fc_rules_cf.krb index 5ecfeb77b1..815d71a5f4 100644 --- a/lib/iris/fileformats/_pyke_rules/fc_rules_cf.krb +++ b/lib/iris/fileformats/_pyke_rules/fc_rules_cf.krb @@ -498,6 +498,22 @@ fc_build_cell_measure python engine.rule_triggered.add(rule.name) +# +# Context: +# This rule will trigger for each ancillary_variable case specific fact. +# +# Purpose: +# Add the ancillary variable to the cube. +# +fc_build_ancil_var + foreach + facts_cf.ancillary_variable($var) + assert + python ancil_var = engine.cf_var.cf_group.ancillary_variables[$var] + python build_ancil_var(engine, ancil_var) + python engine.rule_triggered.add(rule.name) + + # # Context: # This rule will trigger iff a CF latitude coordinate exists and @@ -1941,25 +1957,26 @@ fc_extras # Add it to the cube cube.add_aux_coord(coord, data_dims) - # Update the coordinate to CF-netCDF variable mapping. + # Make a list with names, stored on the engine, so we can find them all later. engine.provides['coordinates'].append((coord, cf_coord_var.cf_name)) ################################################################################ - def build_cell_measures(engine, cf_cm_attr, coord_name=None): + def build_cell_measures(engine, cf_cm_var): """Create a CellMeasure instance and add it to the cube.""" cf_var = engine.cf_var cube = engine.cube attributes = {} # Get units - attr_units = get_attr_units(cf_cm_attr, attributes) + attr_units = get_attr_units(cf_cm_var, attributes) - data = _get_cf_var_data(cf_cm_attr, engine.filename) + # Get (lazy) content array + data = _get_cf_var_data(cf_cm_var, engine.filename) # Determine the name of the dimension/s shared between the CF-netCDF data variable # and the coordinate being built. - common_dims = [dim for dim in cf_cm_attr.dimensions + common_dims = [dim for dim in cf_cm_var.dimensions if dim in cf_var.dimensions] data_dims = None if common_dims: @@ -1967,10 +1984,10 @@ fc_extras data_dims = [cf_var.dimensions.index(dim) for dim in common_dims] # Determine the standard_name, long_name and var_name - standard_name, long_name, var_name = get_names(cf_cm_attr, coord_name, attributes) + standard_name, long_name, var_name = get_names(cf_cm_var, None, attributes) # Obtain the cf_measure. - measure = cf_cm_attr.cf_measure + measure = cf_cm_var.cf_measure # Create the CellMeasure cell_measure = iris.coords.CellMeasure(data, @@ -1984,6 +2001,51 @@ fc_extras # Add it to the cube cube.add_cell_measure(cell_measure, data_dims) + # Make a list with names, stored on the engine, so we can find them all later. + engine.provides['cell_measures'].append((cell_measure, cf_cm_var.cf_name)) + + + + ################################################################################ + def build_ancil_var(engine, cf_av_var): + """Create an AncillaryVariable instance and add it to the cube.""" + cf_var = engine.cf_var + cube = engine.cube + attributes = {} + + # Get units + attr_units = get_attr_units(cf_av_var, attributes) + + # Get (lazy) content array + data = _get_cf_var_data(cf_av_var, engine.filename) + + # Determine the name of the dimension/s shared between the CF-netCDF data variable + # and the AV being built. + common_dims = [dim for dim in cf_av_var.dimensions + if dim in cf_var.dimensions] + data_dims = None + if common_dims: + # Calculate the offset of each common dimension. + data_dims = [cf_var.dimensions.index(dim) for dim in common_dims] + + # Determine the standard_name, long_name and var_name + standard_name, long_name, var_name = get_names(cf_av_var, None, attributes) + + # Create the AncillaryVariable + av = iris.coords.AncillaryVariable( + data, + standard_name=standard_name, + long_name=long_name, + var_name=var_name, + units=attr_units, + attributes=attributes) + + # Add it to the cube + cube.add_ancillary_variable(av, data_dims) + + # Make a list with names, stored on the engine, so we can find them all later. + engine.provides['ancillary_variables'].append((av, cf_av_var.cf_name)) + ################################################################################ diff --git a/lib/iris/fileformats/netcdf.py b/lib/iris/fileformats/netcdf.py index 4d7ddedc61..08b079c3ed 100644 --- a/lib/iris/fileformats/netcdf.py +++ b/lib/iris/fileformats/netcdf.py @@ -459,7 +459,10 @@ def __setstate__(self, state): def _assert_case_specific_facts(engine, cf, cf_group): # Initialise pyke engine "provides" hooks. + # These are used to patch non-processed element attributes after rules activation. engine.provides["coordinates"] = [] + engine.provides["cell_measures"] = [] + engine.provides["ancillary_variables"] = [] # Assert facts for CF coordinates. for cf_name in cf_group.coordinates.keys(): @@ -479,6 +482,12 @@ def _assert_case_specific_facts(engine, cf, cf_group): _PYKE_FACT_BASE, "cell_measure", (cf_name,) ) + # Assert facts for CF ancillary variables. + for cf_name in cf_group.ancillary_variables.keys(): + engine.add_case_specific_fact( + _PYKE_FACT_BASE, "ancillary_variable", (cf_name,) + ) + # Assert facts for CF grid_mappings. for cf_name in cf_group.grid_mappings.keys(): engine.add_case_specific_fact( @@ -597,31 +606,38 @@ def _load_cube(engine, cf, cf_var, filename): # Run pyke inference engine with forward chaining rules. engine.activate(_PYKE_RULE_BASE) - # Populate coordinate attributes with the untouched attributes from the - # associated CF-netCDF variable. - coordinates = engine.provides.get("coordinates", []) - + # Having run the rules, now populate the attributes of all the cf elements with the + # "unused" attributes from the associated CF-netCDF variable. + # That is, all those that aren't CF reserved terms. def attribute_predicate(item): return item[0] not in _CF_ATTRS - for coord, cf_var_name in coordinates: - tmpvar = filter( - attribute_predicate, cf.cf_group[cf_var_name].cf_attrs_unused() - ) + def add_unused_attributes(iris_object, cf_var): + tmpvar = filter(attribute_predicate, cf_var.cf_attrs_unused()) for attr_name, attr_value in tmpvar: - _set_attributes(coord.attributes, attr_name, attr_value) + _set_attributes(iris_object.attributes, attr_name, attr_value) + + def fix_attributes_all_elements(role_name): + elements_and_names = engine.provides.get(role_name, []) + + for iris_object, cf_var_name in elements_and_names: + add_unused_attributes(iris_object, cf.cf_group[cf_var_name]) + + # Populate the attributes of all coordinates, cell-measures and ancillary-vars. + fix_attributes_all_elements("coordinates") + fix_attributes_all_elements("ancillary_variables") + fix_attributes_all_elements("cell_measures") - tmpvar = filter(attribute_predicate, cf_var.cf_attrs_unused()) - # Attach untouched attributes of the associated CF-netCDF data variable to - # the cube. - for attr_name, attr_value in tmpvar: - _set_attributes(cube.attributes, attr_name, attr_value) + # Also populate attributes of the top-level cube itself. + add_unused_attributes(cube, cf_var) + # Work out reference names for all the coords. names = { coord.var_name: coord.standard_name or coord.var_name or "unknown" for coord in cube.coords() } + # Add all the cube cell methods. cube.cell_methods = [ iris.coords.CellMethod( method=method.method, diff --git a/lib/iris/tests/__init__.py b/lib/iris/tests/__init__.py index f260bd0d26..d689738008 100644 --- a/lib/iris/tests/__init__.py +++ b/lib/iris/tests/__init__.py @@ -1297,12 +1297,6 @@ class MyPlotTests(test.GraphicsTest): ) -# TODO: remove these skips when iris-grib is fixed -skip_grib_fail = unittest.skipIf( - True, "Test(s) are failing due to known problems " 'with "iris-grib".' -) - - skip_sample_data = unittest.skipIf( not SAMPLE_DATA_AVAILABLE, ('Test(s) require "iris-sample-data", ' "which is not available."), diff --git a/lib/iris/tests/integration/test_grib2.py b/lib/iris/tests/integration/test_grib2.py index 1c94d424b0..691c4469d3 100644 --- a/lib/iris/tests/integration/test_grib2.py +++ b/lib/iris/tests/integration/test_grib2.py @@ -24,6 +24,7 @@ if tests.GRIB_AVAILABLE: from iris_grib import load_pairs_from_fields from iris_grib.message import GribMessage + from iris_grib.grib_phenom_translation import GRIBCode @tests.skip_data @@ -36,7 +37,6 @@ def test_gdt1(self): cube = load_cube(path) self.assertCMLApproxData(cube) - @tests.skip_grib_fail def test_gdt90_with_bitmap(self): path = tests.get_data_path(("GRIB", "umukv", "ukv_chan9.grib2")) cube = load_cube(path) @@ -156,6 +156,7 @@ def test_save_load(self): cube.add_aux_coord(tcoord) cube.add_aux_coord(fpcoord) cube.attributes["WMO_constituent_type"] = 0 + cube.attributes["GRIB_PARAM"] = GRIBCode("GRIB2:d000c014n000") with self.temp_filename("test_grib_pdt40.grib2") as temp_file_path: save(cube, temp_file_path) @@ -232,9 +233,12 @@ def test_save_load(self): self.assertEqual(test_cube.shape, (744, 744)) self.assertEqual(test_cube.cell_methods, ()) - # Check no cube attributes on the re-loaded cube. + # Check only the GRIB_PARAM attribute exists on the re-loaded cube. # Note: this does *not* match the original, but is as expected. - self.assertEqual(cube_loaded_from_saved.attributes, {}) + self.assertEqual( + cube_loaded_from_saved.attributes, + {"GRIB_PARAM": GRIBCode("GRIB2:d000c003n001")}, + ) # Now remaining to check: coordinates + data... @@ -300,7 +304,6 @@ def test_regular(self): cube = load_cube(path) self.assertCMLApproxData(cube) - @tests.skip_grib_fail def test_reduced(self): path = tests.get_data_path(("GRIB", "reduced", "reduced_gg.grib2")) cube = load_cube(path) diff --git a/lib/iris/tests/integration/test_grib_load.py b/lib/iris/tests/integration/test_grib_load.py index 230c792756..0e7548ee34 100644 --- a/lib/iris/tests/integration/test_grib_load.py +++ b/lib/iris/tests/integration/test_grib_load.py @@ -136,7 +136,6 @@ def test_reduced_ll(self): ) self.assertCML(cube, ("grib_load", "reduced_ll_grib1.cml")) - @tests.skip_grib_fail def test_reduced_gg(self): cube = iris.load_cube( tests.get_data_path(("GRIB", "reduced", "reduced_gg.grib2")) diff --git a/lib/iris/tests/results/grib_load/3_layer.cml b/lib/iris/tests/results/grib_load/3_layer.cml index 24f24ed777..76cc41a04a 100644 --- a/lib/iris/tests/results/grib_load/3_layer.cml +++ b/lib/iris/tests/results/grib_load/3_layer.cml @@ -1,6 +1,9 @@ + + + @@ -31,6 +34,9 @@ + + + @@ -82,6 +88,9 @@ + + + diff --git a/lib/iris/tests/results/grib_load/earth_shape_0.cml b/lib/iris/tests/results/grib_load/earth_shape_0.cml index 1e1e491d58..bb51db3201 100644 --- a/lib/iris/tests/results/grib_load/earth_shape_0.cml +++ b/lib/iris/tests/results/grib_load/earth_shape_0.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/grib_load/earth_shape_1.cml b/lib/iris/tests/results/grib_load/earth_shape_1.cml index dd409ebb20..774e9921b5 100644 --- a/lib/iris/tests/results/grib_load/earth_shape_1.cml +++ b/lib/iris/tests/results/grib_load/earth_shape_1.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/grib_load/earth_shape_2.cml b/lib/iris/tests/results/grib_load/earth_shape_2.cml index 0e3a4a14ea..3ff9ccccb5 100644 --- a/lib/iris/tests/results/grib_load/earth_shape_2.cml +++ b/lib/iris/tests/results/grib_load/earth_shape_2.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/grib_load/earth_shape_3.cml b/lib/iris/tests/results/grib_load/earth_shape_3.cml index 0213c4a4a0..47d11467ee 100644 --- a/lib/iris/tests/results/grib_load/earth_shape_3.cml +++ b/lib/iris/tests/results/grib_load/earth_shape_3.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/grib_load/earth_shape_4.cml b/lib/iris/tests/results/grib_load/earth_shape_4.cml index 2573e867d1..e6aa14e45a 100644 --- a/lib/iris/tests/results/grib_load/earth_shape_4.cml +++ b/lib/iris/tests/results/grib_load/earth_shape_4.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/grib_load/earth_shape_5.cml b/lib/iris/tests/results/grib_load/earth_shape_5.cml index 56462c684b..1257c9c2ad 100644 --- a/lib/iris/tests/results/grib_load/earth_shape_5.cml +++ b/lib/iris/tests/results/grib_load/earth_shape_5.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/grib_load/earth_shape_6.cml b/lib/iris/tests/results/grib_load/earth_shape_6.cml index 1ad54d1f77..eb96657104 100644 --- a/lib/iris/tests/results/grib_load/earth_shape_6.cml +++ b/lib/iris/tests/results/grib_load/earth_shape_6.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/grib_load/earth_shape_7.cml b/lib/iris/tests/results/grib_load/earth_shape_7.cml index cea76b2739..d27ce04a4c 100644 --- a/lib/iris/tests/results/grib_load/earth_shape_7.cml +++ b/lib/iris/tests/results/grib_load/earth_shape_7.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/grib_load/ineg_jneg.cml b/lib/iris/tests/results/grib_load/ineg_jneg.cml index 344fbbacf2..a7d7741092 100644 --- a/lib/iris/tests/results/grib_load/ineg_jneg.cml +++ b/lib/iris/tests/results/grib_load/ineg_jneg.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/grib_load/ineg_jpos.cml b/lib/iris/tests/results/grib_load/ineg_jpos.cml index 14967e6a88..f578fceadb 100644 --- a/lib/iris/tests/results/grib_load/ineg_jpos.cml +++ b/lib/iris/tests/results/grib_load/ineg_jpos.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/grib_load/ipos_jneg.cml b/lib/iris/tests/results/grib_load/ipos_jneg.cml index 1e1e491d58..bb51db3201 100644 --- a/lib/iris/tests/results/grib_load/ipos_jneg.cml +++ b/lib/iris/tests/results/grib_load/ipos_jneg.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/grib_load/ipos_jpos.cml b/lib/iris/tests/results/grib_load/ipos_jpos.cml index 373d8fc475..4dc6d7f980 100644 --- a/lib/iris/tests/results/grib_load/ipos_jpos.cml +++ b/lib/iris/tests/results/grib_load/ipos_jpos.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/grib_load/lambert_grib2.cml b/lib/iris/tests/results/grib_load/lambert_grib2.cml index e8b3f1c4c6..dc938f0aca 100644 --- a/lib/iris/tests/results/grib_load/lambert_grib2.cml +++ b/lib/iris/tests/results/grib_load/lambert_grib2.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/grib_load/missing_values_grib2.cml b/lib/iris/tests/results/grib_load/missing_values_grib2.cml index b090d56a92..c4c0d81915 100644 --- a/lib/iris/tests/results/grib_load/missing_values_grib2.cml +++ b/lib/iris/tests/results/grib_load/missing_values_grib2.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/grib_load/reduced_gg_grib2.cml b/lib/iris/tests/results/grib_load/reduced_gg_grib2.cml index f34938ce3f..fa3ba45e3d 100644 --- a/lib/iris/tests/results/grib_load/reduced_gg_grib2.cml +++ b/lib/iris/tests/results/grib_load/reduced_gg_grib2.cml @@ -2,6 +2,7 @@ + diff --git a/lib/iris/tests/results/grib_load/regular_gg_grib2.cml b/lib/iris/tests/results/grib_load/regular_gg_grib2.cml index 20230aee0f..14213c1602 100644 --- a/lib/iris/tests/results/grib_load/regular_gg_grib2.cml +++ b/lib/iris/tests/results/grib_load/regular_gg_grib2.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/grib_load/time_bound_grib2.cml b/lib/iris/tests/results/grib_load/time_bound_grib2.cml index 1e1e491d58..bb51db3201 100644 --- a/lib/iris/tests/results/grib_load/time_bound_grib2.cml +++ b/lib/iris/tests/results/grib_load/time_bound_grib2.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/integration/grib2/TestDRT3/grid_complex_spatial_differencing.cml b/lib/iris/tests/results/integration/grib2/TestDRT3/grid_complex_spatial_differencing.cml index b15c6a4308..2cfe06f8f6 100644 --- a/lib/iris/tests/results/integration/grib2/TestDRT3/grid_complex_spatial_differencing.cml +++ b/lib/iris/tests/results/integration/grib2/TestDRT3/grid_complex_spatial_differencing.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/integration/grib2/TestGDT30/lambert.cml b/lib/iris/tests/results/integration/grib2/TestGDT30/lambert.cml index a33d0b04ba..215a0de88d 100644 --- a/lib/iris/tests/results/integration/grib2/TestGDT30/lambert.cml +++ b/lib/iris/tests/results/integration/grib2/TestGDT30/lambert.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/integration/grib2/TestGDT40/reduced.cml b/lib/iris/tests/results/integration/grib2/TestGDT40/reduced.cml index f2ca666998..3a963b3203 100644 --- a/lib/iris/tests/results/integration/grib2/TestGDT40/reduced.cml +++ b/lib/iris/tests/results/integration/grib2/TestGDT40/reduced.cml @@ -2,6 +2,7 @@ + diff --git a/lib/iris/tests/results/integration/grib2/TestGDT40/regular.cml b/lib/iris/tests/results/integration/grib2/TestGDT40/regular.cml index fb6445b8b1..e5eea0fc7c 100644 --- a/lib/iris/tests/results/integration/grib2/TestGDT40/regular.cml +++ b/lib/iris/tests/results/integration/grib2/TestGDT40/regular.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/integration/grib2/TestImport/gdt1.cml b/lib/iris/tests/results/integration/grib2/TestImport/gdt1.cml index d3cc6b4732..d304d8a843 100644 --- a/lib/iris/tests/results/integration/grib2/TestImport/gdt1.cml +++ b/lib/iris/tests/results/integration/grib2/TestImport/gdt1.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/integration/grib2/TestImport/gdt90_with_bitmap.cml b/lib/iris/tests/results/integration/grib2/TestImport/gdt90_with_bitmap.cml index 9f950b5e1f..3118f86823 100644 --- a/lib/iris/tests/results/integration/grib2/TestImport/gdt90_with_bitmap.cml +++ b/lib/iris/tests/results/integration/grib2/TestImport/gdt90_with_bitmap.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/integration/name_grib/NAMEII/0_TRACER_AIR_CONCENTRATION.cml b/lib/iris/tests/results/integration/name_grib/NAMEII/0_TRACER_AIR_CONCENTRATION.cml index 4d0fddbba5..b0daf50907 100644 --- a/lib/iris/tests/results/integration/name_grib/NAMEII/0_TRACER_AIR_CONCENTRATION.cml +++ b/lib/iris/tests/results/integration/name_grib/NAMEII/0_TRACER_AIR_CONCENTRATION.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/integration/name_grib/NAMEII/1_TRACER_DOSAGE.cml b/lib/iris/tests/results/integration/name_grib/NAMEII/1_TRACER_DOSAGE.cml index fd61a67eb6..aef4988ce6 100644 --- a/lib/iris/tests/results/integration/name_grib/NAMEII/1_TRACER_DOSAGE.cml +++ b/lib/iris/tests/results/integration/name_grib/NAMEII/1_TRACER_DOSAGE.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/integration/name_grib/NAMEII/2_TRACER_WET_DEPOSITION.cml b/lib/iris/tests/results/integration/name_grib/NAMEII/2_TRACER_WET_DEPOSITION.cml deleted file mode 100644 index 029aa022ea..0000000000 --- a/lib/iris/tests/results/integration/name_grib/NAMEII/2_TRACER_WET_DEPOSITION.cml +++ /dev/null @@ -1,31 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/lib/iris/tests/results/integration/name_grib/NAMEII/3_TRACER_DRY_DEPOSITION.cml b/lib/iris/tests/results/integration/name_grib/NAMEII/3_TRACER_DRY_DEPOSITION.cml index 429d9db2ff..5787c19643 100644 --- a/lib/iris/tests/results/integration/name_grib/NAMEII/3_TRACER_DRY_DEPOSITION.cml +++ b/lib/iris/tests/results/integration/name_grib/NAMEII/3_TRACER_DRY_DEPOSITION.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/integration/name_grib/NAMEII/4_TRACER_TOTAL_DEPOSITION.cml b/lib/iris/tests/results/integration/name_grib/NAMEII/4_TRACER_TOTAL_DEPOSITION.cml index 429d9db2ff..5787c19643 100644 --- a/lib/iris/tests/results/integration/name_grib/NAMEII/4_TRACER_TOTAL_DEPOSITION.cml +++ b/lib/iris/tests/results/integration/name_grib/NAMEII/4_TRACER_TOTAL_DEPOSITION.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/integration/name_grib/NAMEIII/0_TRACER_AIR_CONCENTRATION.cml b/lib/iris/tests/results/integration/name_grib/NAMEIII/0_TRACER_AIR_CONCENTRATION.cml index 8412a4f814..1a31427de0 100644 --- a/lib/iris/tests/results/integration/name_grib/NAMEIII/0_TRACER_AIR_CONCENTRATION.cml +++ b/lib/iris/tests/results/integration/name_grib/NAMEIII/0_TRACER_AIR_CONCENTRATION.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/integration/name_grib/NAMEIII/1_TRACER_AIR_CONCENTRATION.cml b/lib/iris/tests/results/integration/name_grib/NAMEIII/1_TRACER_AIR_CONCENTRATION.cml index 590e8ef463..7007836e62 100644 --- a/lib/iris/tests/results/integration/name_grib/NAMEIII/1_TRACER_AIR_CONCENTRATION.cml +++ b/lib/iris/tests/results/integration/name_grib/NAMEIII/1_TRACER_AIR_CONCENTRATION.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/integration/name_grib/NAMEIII/2_TRACER_DRY_DEPOSITION.cml b/lib/iris/tests/results/integration/name_grib/NAMEIII/2_TRACER_DRY_DEPOSITION.cml index d3edb03a56..850ef89ed2 100644 --- a/lib/iris/tests/results/integration/name_grib/NAMEIII/2_TRACER_DRY_DEPOSITION.cml +++ b/lib/iris/tests/results/integration/name_grib/NAMEIII/2_TRACER_DRY_DEPOSITION.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/integration/name_grib/NAMEIII/3_TRACER_WET_DEPOSITION.cml b/lib/iris/tests/results/integration/name_grib/NAMEIII/3_TRACER_WET_DEPOSITION.cml index 3e8d62ef3d..ade4cea92d 100644 --- a/lib/iris/tests/results/integration/name_grib/NAMEIII/3_TRACER_WET_DEPOSITION.cml +++ b/lib/iris/tests/results/integration/name_grib/NAMEIII/3_TRACER_WET_DEPOSITION.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/integration/name_grib/NAMEIII/4_TRACER_DEPOSITION.cml b/lib/iris/tests/results/integration/name_grib/NAMEIII/4_TRACER_DEPOSITION.cml index 586aaa6b56..088b622c46 100644 --- a/lib/iris/tests/results/integration/name_grib/NAMEIII/4_TRACER_DEPOSITION.cml +++ b/lib/iris/tests/results/integration/name_grib/NAMEIII/4_TRACER_DEPOSITION.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/system/supported_filetype_.grib2.cml b/lib/iris/tests/results/system/supported_filetype_.grib2.cml index c230684fbd..f334b13863 100644 --- a/lib/iris/tests/results/system/supported_filetype_.grib2.cml +++ b/lib/iris/tests/results/system/supported_filetype_.grib2.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/results/uri_callback/grib_global.cml b/lib/iris/tests/results/uri_callback/grib_global.cml index a7a23e7235..aef0310a96 100644 --- a/lib/iris/tests/results/uri_callback/grib_global.cml +++ b/lib/iris/tests/results/uri_callback/grib_global.cml @@ -1,6 +1,9 @@ + + + diff --git a/lib/iris/tests/test_netcdf.py b/lib/iris/tests/test_netcdf.py index a550e1ed4b..91e37dd3a8 100644 --- a/lib/iris/tests/test_netcdf.py +++ b/lib/iris/tests/test_netcdf.py @@ -16,6 +16,7 @@ import os.path import shutil import stat +from subprocess import check_call import tempfile from unittest import mock @@ -27,8 +28,10 @@ import iris.analysis.trajectory import iris.fileformats._pyke_rules.compiled_krb.fc_rules_cf_fc as pyke_rules import iris.fileformats.netcdf +from iris.fileformats.netcdf import load_cubes as nc_load_cubes import iris.std_names import iris.util +from iris.coords import AncillaryVariable, CellMeasure import iris.coord_systems as icoord_systems import iris.tests.stock as stock from iris._lazy_data import is_lazy_data @@ -36,6 +39,13 @@ @tests.skip_data class TestNetCDFLoad(tests.IrisTest): + def setUp(self): + self.tmpdir = None + + def tearDown(self): + if self.tmpdir is not None: + shutil.rmtree(self.tmpdir) + def test_monotonic(self): cubes = iris.load( tests.get_data_path( @@ -240,6 +250,104 @@ def test_cell_methods(self): self.assertCML(cubes, ("netcdf", "netcdf_cell_methods.cml")) + def test_ancillary_variables(self): + # Note: using a CDL string as a test data reference, rather than a binary file. + ref_cdl = """ + netcdf cm_attr { + dimensions: + axv = 3 ; + variables: + int64 qqv(axv) ; + qqv:long_name = "qq" ; + qqv:units = "1" ; + qqv:ancillary_variables = "my_av" ; + int64 axv(axv) ; + axv:units = "1" ; + axv:long_name = "x" ; + double my_av(axv) ; + my_av:units = "1" ; + my_av:long_name = "refs" ; + my_av:custom = "extra-attribute"; + data: + axv = 1, 2, 3; + my_av = 11., 12., 13.; + } + """ + self.tmpdir = tempfile.mkdtemp() + cdl_path = os.path.join(self.tmpdir, "tst.cdl") + nc_path = os.path.join(self.tmpdir, "tst.nc") + # Write CDL string into a temporary CDL file. + with open(cdl_path, "w") as f_out: + f_out.write(ref_cdl) + # Use ncgen to convert this into an actual (temporary) netCDF file. + command = "ncgen -o {} {}".format(nc_path, cdl_path) + check_call(command, shell=True) + # Load with iris.fileformats.netcdf.load_cubes, and check expected content. + cubes = list(nc_load_cubes(nc_path)) + self.assertEqual(len(cubes), 1) + avs = cubes[0].ancillary_variables() + self.assertEqual(len(avs), 1) + expected = AncillaryVariable( + np.ma.array([11.0, 12.0, 13.0]), + long_name="refs", + var_name="my_av", + units="1", + attributes={"custom": "extra-attribute"}, + ) + self.assertEqual(avs[0], expected) + + def test_cell_measures(self): + # Note: using a CDL string as a test data reference, rather than a binary file. + ref_cdl = """ + netcdf cm_attr { + dimensions: + axv = 3 ; + ayv = 2 ; + variables: + int64 qqv(ayv, axv) ; + qqv:long_name = "qq" ; + qqv:units = "1" ; + qqv:cell_measures = "area: my_areas" ; + int64 ayv(ayv) ; + ayv:units = "1" ; + ayv:long_name = "y" ; + int64 axv(axv) ; + axv:units = "1" ; + axv:long_name = "x" ; + double my_areas(ayv, axv) ; + my_areas:units = "m2" ; + my_areas:long_name = "standardised cell areas" ; + my_areas:custom = "extra-attribute"; + data: + axv = 11, 12, 13; + ayv = 21, 22; + my_areas = 110., 120., 130., 221., 231., 241.; + } + """ + self.tmpdir = tempfile.mkdtemp() + cdl_path = os.path.join(self.tmpdir, "tst.cdl") + nc_path = os.path.join(self.tmpdir, "tst.nc") + # Write CDL string into a temporary CDL file. + with open(cdl_path, "w") as f_out: + f_out.write(ref_cdl) + # Use ncgen to convert this into an actual (temporary) netCDF file. + command = "ncgen -o {} {}".format(nc_path, cdl_path) + check_call(command, shell=True) + # Load with iris.fileformats.netcdf.load_cubes, and check expected content. + cubes = list(nc_load_cubes(nc_path)) + self.assertEqual(len(cubes), 1) + cms = cubes[0].cell_measures() + self.assertEqual(len(cms), 1) + expected = CellMeasure( + np.ma.array([[110.0, 120.0, 130.0], [221.0, 231.0, 241.0]]), + measure="area", + var_name="my_areas", + long_name="standardised cell areas", + units="m2", + attributes={"custom": "extra-attribute"}, + ) + self.assertEqual(cms[0], expected) + def test_deferred_loading(self): # Test exercising CF-netCDF deferred loading and deferred slicing. # shape (31, 161, 320)