Skip to content

Translate text PP save rules into Python #2795

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 9 commits into from
Oct 16, 2017
Merged

Translate text PP save rules into Python #2795

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
488827e
Translate text PP save rules to Python
DPeterK Oct 12, 2017
00d50a7
Update rule function names
DPeterK Oct 13, 2017
8817b84
Plumb in changes
DPeterK Oct 13, 2017
d76f996
Changes from running tests
DPeterK Oct 13, 2017
5879f3a
Final test-related changes
DPeterK Oct 13, 2017
4f8941c
Changes to get the tests passing; a review action
DPeterK Oct 16, 2017
4bbaae6
License headers
DPeterK Oct 16, 2017
336cfe4
Reorganisation of pp lbproc_pairs feature. (#12)
pp-mo Oct 16, 2017
fe54f04
Remove unneeded lines
DPeterK Oct 16, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
784 changes: 0 additions & 784 deletions lib/iris/etc/pp_save_rules.txt
View file
Open in desktop

This file was deleted.

48 changes: 48 additions & 0 deletions lib/iris/fileformats/_pp_lbproc_pairs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
# (C) British Crown Copyright 2017, Met Office
#
# This file is part of Iris.
#
# Iris is free software: you can redistribute it and/or modify it under
# the terms of the GNU Lesser General Public License as published by the
# Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Iris is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with Iris. If not, see <http://www.gnu.org/licenses/>.

from __future__ import (absolute_import, division, print_function)
from six.moves import (filter, input, map, range, zip) # noqa
import six

import itertools


# LBPROC codes and their English equivalents
LBPROC_PAIRS = ((1, "Difference from another experiment"),
(2, "Difference from zonal (or other spatial) mean"),
(4, "Difference from time mean"),
(8, "X-derivative (d/dx)"),
(16, "Y-derivative (d/dy)"),
(32, "Time derivative (d/dt)"),
(64, "Zonal mean field"),
(128, "Time mean field"),
(256, "Product of two fields"),
(512, "Square root of a field"),
(1024, "Difference between fields at levels BLEV and BRLEV"),
(2048, "Mean over layer between levels BLEV and BRLEV"),
(4096, "Minimum value of field during time period"),
(8192, "Maximum value of field during time period"),
(16384, "Magnitude of a vector, not specifically wind speed"),
(32768, "Log10 of a field"),
(65536, "Variance of a field"),
(131072, "Mean over an ensemble of parallel runs"))

# lbproc_map is dict mapping lbproc->English and English->lbproc
# essentially a one to one mapping
LBPROC_MAP = {x: y for x, y in
itertools.chain(LBPROC_PAIRS, ((y, x) for x, y in LBPROC_PAIRS))}
58 changes: 8 additions & 50 deletions lib/iris/fileformats/pp.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@
import abc
import collections
from copy import deepcopy
import itertools
import operator
import os
import re
Expand All @@ -41,8 +40,14 @@
from iris._deprecation import warn_deprecated
from iris._lazy_data import as_concrete_data, as_lazy_data, is_lazy_data
import iris.config
import iris.fileformats.rules
import iris.fileformats.pp_rules
from iris.fileformats.pp_save_rules import verify

# NOTE: this is for backwards-compatitibility *ONLY*
# We could simply remove it for v2.0 ?
from iris.fileformats._pp_lbproc_pairs import (LBPROC_PAIRS,
LBPROC_MAP as lbproc_map)
import iris.fileformats.rules
import iris.coord_systems


Expand All @@ -60,10 +65,6 @@
EARTH_RADIUS = 6371229.0


# Cube->PP rules are loaded on first use
_save_rules = None


PP_HEADER_DEPTH = 256
PP_WORD_DEPTH = 4
NUM_LONG_HEADERS = 45
Expand Down Expand Up @@ -221,31 +222,6 @@
'default': np.dtype('>f4'),
}

# LBPROC codes and their English equivalents
Copy link
Member

@pp-mo pp-mo Oct 16, 2017 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think moving these to iris.fileformats.rules is a mistake, as it the information is specific to PP (and FF), whereas iris.fileformats.rules "ought" to be the home for fileformat-independent rules processing.
They aren't specially tied to implementation of text rules anyway.

In any case, "LBPROC_PAIRS" and "lbproc_map" are perfectly reasonable public elements of the API, so you can't just move them anyway.

In an ideal world I think these would be in "iris.fileformats.um_cf_map", and also "lbroc_map" should be capitalised, but it's a bit late for that...

Copy link
Member

@pp-mo pp-mo Oct 16, 2017 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

(from discussion with @dkillick)
The move was introduced because of a circularity problem when iris.fileformats.pp imports iris.fileformats.pp_save_rules, which imports iris.fileformats.pp ...
By putting 'lbproc_map' into rules, then pp_save_rules no longer needed to import pp.

Alternative suggestion: Put LBPROC_PAIRS and lbproc_map inside pp_save_rules itself.
Then to avoid breaking existing API, publish them also in 'pp' by importing them from 'pp_save_rules'.

For better measure, we can fix naming + make them private in pp_save_rules.
Ideally we would "deprecate" this, but I also can't see how?
So, perhaps this really is a suitable case for a major-version breaking change ??

Copy link
Member

@pp-mo pp-mo Oct 16, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

More trouble...
I missed that "lbproc_map" is also used in "pp_rules"
( i.e. "pp_load_rules", perhaps it should now be called that ? )

You can import it there from "pp_save_rules", but that is obviously a rather peculiar dependency ordering.
You cannot import it from 'pp', if 'pp' is to import 'pp_rules' (same circularity problem as before)

In my view, this info really belongs with the other similar stuff in 'um_cf_map'.
To do this properly, we should really put this info into metarelate, and modifytools/gen_translations.py to support it, but I really don't think we have time for that.
In which case, we should just host this translation info in a separate module.
See : DPeterK#12 for worked suggestion ...

Copy link
Member Author

@DPeterK DPeterK Oct 16, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i.e. "pp_load_rules", perhaps it should now be called that ?

See #2790.

we should just host this translation info in a separate module.

Agreed 👍 Thanks for spotting this, @pp-mo

LBPROC_PAIRS = ((1, "Difference from another experiment"),
(2, "Difference from zonal (or other spatial) mean"),
(4, "Difference from time mean"),
(8, "X-derivative (d/dx)"),
(16, "Y-derivative (d/dy)"),
(32, "Time derivative (d/dt)"),
(64, "Zonal mean field"),
(128, "Time mean field"),
(256, "Product of two fields"),
(512, "Square root of a field"),
(1024, "Difference between fields at levels BLEV and BRLEV"),
(2048, "Mean over layer between levels BLEV and BRLEV"),
(4096, "Minimum value of field during time period"),
(8192, "Maximum value of field during time period"),
(16384, "Magnitude of a vector, not specifically wind speed"),
(32768, "Log10 of a field"),
(65536, "Variance of a field"),
(131072, "Mean over an ensemble of parallel runs"))

# lbproc_map is dict mapping lbproc->English and English->lbproc
# essentially a one to one mapping
lbproc_map = {x: y for x, y in
itertools.chain(LBPROC_PAIRS, ((y, x) for x, y in LBPROC_PAIRS))}


class STASH(collections.namedtuple('STASH', 'model section item')):
"""
Expand Down Expand Up @@ -1786,21 +1762,6 @@ def _field_gen(filename, read_data_bytes, little_ended=False):
yield pp_field


def _ensure_save_rules_loaded():
"""Makes sure the standard save rules are loaded."""

# Uses these module-level variables
global _save_rules

if _save_rules is None:
# Load the pp save rules
rules_filename = os.path.join(iris.config.CONFIG_PATH,
'pp_save_rules.txt')
with iris.fileformats.rules._disable_deprecation_warnings():
_save_rules = iris.fileformats.rules.RulesContainer(
rules_filename, iris.fileformats.rules.ProcedureRule)


# Stash codes not to be filtered (reference altitude and pressure fields).
_STASH_ALLOW = [STASH(1, 0, 33), STASH(1, 0, 1)]

Expand Down Expand Up @@ -2102,8 +2063,6 @@ def save_pairs_from_cube(cube, field_coords=None, target=None):
# On the flip side, record which Cube metadata has been "used" and flag up
# unused?

_ensure_save_rules_loaded()

n_dims = len(cube.shape)
if n_dims < 2:
raise ValueError('Unable to save a cube of fewer than 2 dimensions.')
Expand Down Expand Up @@ -2150,8 +2109,7 @@ def save_pairs_from_cube(cube, field_coords=None, target=None):

# Run the PP save rules on the slice2D, to fill the PPField,
# recording the rules that were used
rules_result = _save_rules.verify(slice2D, pp_field)
verify_rules_ran = rules_result.matching_rules
pp_field = verify(slice2D, pp_field)

yield (slice2D, pp_field)

Expand Down
5 changes: 3 additions & 2 deletions lib/iris/fileformats/pp_rules.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,10 @@
from iris.coords import AuxCoord, CellMethod, DimCoord
from iris.fileformats.rules import (ConversionMetadata, Factory, Reference,
ReferenceTarget)
import iris.fileformats.pp
from iris.fileformats._pp_lbproc_pairs import LBPROC_MAP
from iris.fileformats.um_cf_map import (LBFC_TO_CF, STASH_TO_CF,
STASHCODE_IMPLIED_HEIGHTS)
import iris.fileformats.pp


###############################################################################
Expand Down Expand Up @@ -1027,7 +1028,7 @@ def _all_other_rules(f):
if unhandled_lbproc:
attributes["ukmo__process_flags"] = tuple(sorted(
[name
for value, name in six.iteritems(iris.fileformats.pp.lbproc_map)
for value, name in six.iteritems(LBPROC_MAP)
if isinstance(value, int) and f.lbproc & value]))

if (f.lbsrce % 10000) == 1111:
Expand Down
Loading