From 7d6d96e553545e0020a2f4f4f72a499fb3f98c11 Mon Sep 17 00:00:00 2001 From: Corinne Bosley Date: Tue, 10 Oct 2017 15:58:15 +0100 Subject: [PATCH 1/6] broke some rules --- lib/iris/fileformats/rules.py | 99 ----------------------------------- 1 file changed, 99 deletions(-) diff --git a/lib/iris/fileformats/rules.py b/lib/iris/fileformats/rules.py index eec201dfa1..a9854841d0 100644 --- a/lib/iris/fileformats/rules.py +++ b/lib/iris/fileformats/rules.py @@ -290,105 +290,6 @@ def run_actions(self, cube, field): return factories -class FunctionRule(Rule): - """ - A Rule with values returned by its actions. - - .. deprecated:: 1.10 - - """ - def _create_action_method(self, i, action): - # CM loading style action. Returns an object, such as a coord. - # Compile a new method for the operation. - rules_globals = _rules_execution_environment() - compile_locals = {} - exec( - compile( - 'def _f(self, field, f, pp, grib, cm): return %s' % (action, ), - '', - 'exec'), - rules_globals, compile_locals) - # Make it a method of ours. - _f = compile_locals['_f'] - method = six.create_bound_method(_f, self) - setattr(self, '_exec_action_%d' % (i, ), method) - # Add to our list of actions. - self._exec_actions.append(method) - - def _process_action_result(self, obj, cube): - """Process the result of an action.""" - - factory = None - - # NB. The names such as 'CellMethod' are defined by the - # "deferred import" performed by Rule.run_actions() above. - - #cell methods - not yet implemented - if isinstance(obj, CellMethod): - cube.add_cell_method(obj) - - elif isinstance(obj, CMAttribute): - # Temporary code to deal with invalid standard names from the translation table. - # TODO: when name is "standard_name" force the value to be a real standard name - if obj.name == 'standard_name' and obj.value is not None: - cube.rename(obj.value) - elif obj.name == 'units': - # Graceful loading of units. - try: - setattr(cube, obj.name, obj.value) - except ValueError: - msg = 'Ignoring PP invalid units {!r}'.format(obj.value) - warnings.warn(msg) - cube.attributes['invalid_units'] = obj.value - cube.units = cf_units._UNKNOWN_UNIT_STRING - else: - setattr(cube, obj.name, obj.value) - - elif isinstance(obj, Factory): - factory = obj - - # The function returned nothing, like the pp save actions, "lbft = 3" - elif obj is None: - pass - - else: - raise Exception("Object could not be added to cube. Unknown type: " + obj.__class__.__name__) - - return factory - - -class ProcedureRule(Rule): - """ - A Rule with nothing returned by its actions. - - .. deprecated:: 1.10 - - """ - def _create_action_method(self, i, action): - # PP saving style action. No return value, e.g. "pp.lbft = 3". - rules_globals = _rules_execution_environment() - compile_locals = {} - exec(compile('def _f(self, field, f, pp, grib, cm): %s' % (action, ), - '', - 'exec'), - rules_globals, compile_locals) - # Make it a method of ours. - _f = compile_locals['_f'] - method = six.create_bound_method(_f, self) - setattr(self, '_exec_action_%d' % (i, ), method) - # Add to our list of actions. - self._exec_actions.append(method) - - def _process_action_result(self, obj, cube): - # This should always be None, as our rules won't create anything. - pass - - def conditional_warning(self, condition, warning): - pass # without this pass statement it alsp print, " Args:" on a new line. - if condition: - warnings.warn(warning) - - class RulesContainer(object): """ A collection of :class:`Rule` instances, with the ability to read rule From efbc48ac8b589b0b78594b8390b4eb373cbeeee8 Mon Sep 17 00:00:00 2001 From: Corinne Bosley Date: Wed, 11 Oct 2017 11:26:48 +0100 Subject: [PATCH 2/6] removed RulesContainer and corrected some tests --- lib/iris/fileformats/rules.py | 90 ----------------------------------- 1 file changed, 90 deletions(-) diff --git a/lib/iris/fileformats/rules.py b/lib/iris/fileformats/rules.py index a9854841d0..cb5e804991 100644 --- a/lib/iris/fileformats/rules.py +++ b/lib/iris/fileformats/rules.py @@ -290,96 +290,6 @@ def run_actions(self, cube, field): return factories -class RulesContainer(object): - """ - A collection of :class:`Rule` instances, with the ability to read rule - definitions from files and run the rules against given fields. - - .. deprecated:: 1.10 - - """ - def __init__(self, filepath=None, rule_type=FunctionRule): - """Create a new rule set, optionally adding rules from the specified file. - - The rule_type defaults to :class:`FunctionRule`, - e.g for CM loading actions that return objects, such as *AuxCoord(...)* - - rule_type can also be set to :class:`ProcedureRule` - e.g for PP saving actions that do not return anything, such as *pp.lbuser[3] = 16203* - """ - if _enable_rules_deprecations: - warn_deprecated( - "the `iris.fileformats.rules.RulesContainer class is deprecated.") - self._rules = [] - self.rule_type = rule_type - if filepath is not None: - self.import_rules(filepath) - - def import_rules(self, filepath): - """Extend the rule collection with the rules defined in the specified file.""" - # Define state constants - IN_CONDITION = 1 - IN_ACTION = 2 - - rule_file = os.path.expanduser(filepath) - conditions = [] - actions = [] - state = None - - with open(rule_file, 'r') as file: - for line in file: - line = line.rstrip() - if line == "IF": - if conditions and actions: - self._rules.append(self.rule_type(conditions, actions)) - conditions = [] - actions = [] - state = IN_CONDITION - elif line == "THEN": - state = IN_ACTION - elif len(line) == 0: - pass - elif line.strip().startswith('#'): - pass - elif state == IN_CONDITION: - conditions.append(line) - elif state == IN_ACTION: - actions.append(line) - else: - raise Exception('Rule file not read correctly at line: ' + - line) - if conditions and actions: - self._rules.append(self.rule_type(conditions, actions)) - - def verify(self, cube, field): - """ - Add to the given :class:`iris.cube.Cube` by running this set of - rules with the given field. - - Args: - - * cube: - An instance of :class:`iris.cube.Cube`. - * field: - A field object relevant to the rule set. - - Returns: (cube, matching_rules) - - * cube - the resultant cube - * matching_rules - a list of rules which matched - - """ - matching_rules = [] - factories = [] - for rule in self._rules: - if rule.evaluates_true(cube, field): - matching_rules.append(rule) - rule_factories = rule.run_actions(cube, field) - if rule_factories: - factories.extend(rule_factories) - return RuleResult(cube, matching_rules, factories) - - def scalar_coord(cube, coord_name): """Try to find a single-valued coord with the given name.""" found_coord = None From e54acd4078476ac3216b0688e087db01e62891ce Mon Sep 17 00:00:00 2001 From: Corinne Bosley Date: Wed, 11 Oct 2017 13:19:36 +0100 Subject: [PATCH 3/6] fixed more tests.... --- lib/iris/fileformats/pp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/iris/fileformats/pp.py b/lib/iris/fileformats/pp.py index a75bb0bba9..34f697aeec 100644 --- a/lib/iris/fileformats/pp.py +++ b/lib/iris/fileformats/pp.py @@ -2004,7 +2004,7 @@ def save(cube, target, append=False, field_coords=None): def save_pairs_from_cube(cube, field_coords=None, target=None): """ - Use the PP saving rules (and any user rules) to convert a cube or + Use the PP saving rules to convert a cube or iterable of cubes to an iterable of (2D cube, PP field) pairs. Args: From 2165b06ff36fb0e8a2ca3d79e7cf5d2eed8a5097 Mon Sep 17 00:00:00 2001 From: Corinne Bosley Date: Tue, 17 Oct 2017 08:41:47 +0100 Subject: [PATCH 4/6] =?UTF-8?q?Removed=20Rule,=20which=20is=20also=20depre?= =?UTF-8?q?cated=20and=20covered=20by=20this=20PR=C3=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/iris/fileformats/rules.py | 127 ---------------------------------- 1 file changed, 127 deletions(-) diff --git a/lib/iris/fileformats/rules.py b/lib/iris/fileformats/rules.py index cb5e804991..40cbc6c62b 100644 --- a/lib/iris/fileformats/rules.py +++ b/lib/iris/fileformats/rules.py @@ -163,133 +163,6 @@ class Reference(iris.util._OrderedHashable): """ -class Rule(object): - """ - A collection of condition expressions and their associated action expressions. - - Example rule:: - - IF - f.lbuser[6] == 2 - f.lbuser[3] == 101 - THEN - CMAttribute('standard_name', 'sea_water_potential_temperature') - CMAttribute('units', 'Celsius') - - .. deprecated:: 1.10 - - """ - def __init__(self, conditions, actions): - """Create instance methods from our conditions and actions.""" - if _enable_rules_deprecations: - warn_deprecated( - "the `iris.fileformats.rules.Rule class is deprecated.") - if not hasattr(conditions, '__iter__'): - raise TypeError('Variable conditions should be iterable, got: '+ type(conditions)) - if not hasattr(actions, '__iter__'): - raise TypeError('Variable actions should be iterable, got: '+ type(actions)) - - self._conditions = conditions - self._actions = actions - self._exec_actions = [] - - self.id = str(hash((tuple(self._conditions), tuple(self._actions)))) - - for i, condition in enumerate(conditions): - self._conditions[i] = condition - - # Create the conditions method. - self._create_conditions_method() - - # Create the action methods. - for i, action in enumerate(self._actions): - if not action: - action = 'None' - self._create_action_method(i, action) - - def _create_conditions_method(self): - # Bundle all the conditions into one big string. - conditions = '(%s)' % ') and ('.join(self._conditions) - if not conditions: - conditions = 'None' - # Create a method to evaluate the conditions. - # NB. This creates the name '_f' in the 'compile_locals' namespace, - # which is then used below. - code = 'def _f(self, field, f, pp, grib, cm): return %s' % conditions - rules_globals = _rules_execution_environment() - compile_locals = {} - exec(compile(code, '', 'exec'), rules_globals, compile_locals) - # Make it a method of ours. - _f = compile_locals['_f'] - self._exec_conditions = six.create_bound_method(_f, self) - - @abc.abstractmethod - def _create_action_method(self, i, action): - pass - - @abc.abstractmethod - def _process_action_result(self, obj, cube): - pass - - def __repr__(self): - string = "IF\n" - string += '\n'.join(self._conditions) - string += "\nTHEN\n" - string += '\n'.join(self._actions) - return string - - def evaluates_true(self, cube, field): - """Returns True if and only if all the conditions evaluate to True for the given field.""" - field = field - f = field - pp = field - grib = field - cm = cube - - try: - result = self._exec_conditions(field, f, pp, grib, cm) - except Exception as err: - print('Condition failed to run conditions: %s : %s' % (self._conditions, err), file=sys.stderr) - raise err - - return result - - def _matches_field(self, field): - """Simple wrapper onto evaluates_true in the case where cube is None.""" - return self.evaluates_true(None, field) - - def run_actions(self, cube, field): - """ - Adds to the given cube based on the return values of all the actions. - - """ - # Define the variables which the eval command should be able to see - f = field - pp = field - grib = field - cm = cube - - factories = [] - for i, action in enumerate(self._actions): - try: - # Run this action. - obj = self._exec_actions[i](field, f, pp, grib, cm) - # Process the return value (if any), e.g a CM object or None. - action_factory = self._process_action_result(obj, cube) - if action_factory: - factories.append(action_factory) - - except iris.exceptions.CoordinateNotFoundError as err: - print('Failed (msg:%(error)s) to find coordinate, perhaps consider running last: %(command)s' % {'command':action, 'error': err}, file=sys.stderr) - except AttributeError as err: - print('Failed to get value (%(error)s) to execute: %(command)s' % {'command':action, 'error': err}, file=sys.stderr) - except Exception as err: - print('Failed (msg:%(error)s) to run:\n %(command)s\nFrom the rule:\n%(me)r' % {'me':self, 'command':action, 'error': err}, file=sys.stderr) - raise err - - return factories - - def scalar_coord(cube, coord_name): """Try to find a single-valued coord with the given name.""" found_coord = None From ffcc48ad75f19046a41f146effa7427b16e9babc Mon Sep 17 00:00:00 2001 From: Corinne Bosley Date: Tue, 17 Oct 2017 09:04:00 +0100 Subject: [PATCH 5/6] Removed RuleResult which is only used in RulesContainer --- lib/iris/fileformats/rules.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/iris/fileformats/rules.py b/lib/iris/fileformats/rules.py index 40cbc6c62b..0fa7a052b4 100644 --- a/lib/iris/fileformats/rules.py +++ b/lib/iris/fileformats/rules.py @@ -44,7 +44,6 @@ import iris.fileformats.um_cf_map from iris.util import is_regular, regular_step -RuleResult = collections.namedtuple('RuleResult', ['cube', 'matching_rules', 'factories']) Factory = collections.namedtuple('Factory', ['factory_class', 'args']) ReferenceTarget = collections.namedtuple('ReferenceTarget', ('name', 'transform')) From 2002960cafc4db4fe6d1c90181d0ab617295e011 Mon Sep 17 00:00:00 2001 From: Corinne Bosley Date: Tue, 17 Oct 2017 10:08:56 +0100 Subject: [PATCH 6/6] Also removed rules_execution_environment and bits that go with it --- lib/iris/fileformats/rules.py | 37 ----------------------------------- 1 file changed, 37 deletions(-) diff --git a/lib/iris/fileformats/rules.py b/lib/iris/fileformats/rules.py index 0fa7a052b4..695fea6407 100644 --- a/lib/iris/fileformats/rules.py +++ b/lib/iris/fileformats/rules.py @@ -87,43 +87,6 @@ def as_cube(self): return self._final_cube -# Controls the deferred import of all the symbols from iris.coords. -# This "import all" is used as the rules file does not use fully qualified class names. -_rules_globals = None -_import_pending = True -def _rules_execution_environment(): - """ - Return an environment with the globals needed for rules code execution. - - This is needed as the rules file does not use fully qualified class names. - If something is needed for rules execution, it can be added here. - - A master environment is built only when needed (the first call). - This allows the import of various modules to be deferred, so we don't load - all of those when we merely import this module. - - """ - global _import_pending, _rules_globals - if _import_pending: - # Get all module globals, and add other deferred imports. - import iris.aux_factory - import iris.coords - import iris.coord_systems - import iris.fileformats.um_cf_map - # Take a copy of all this module's globals. - _rules_globals = globals().copy() - # Add various other stuff. - # NOTE: these are equivalent to "from xx import *": not tidy ! - _rules_globals.update(iris.aux_factory.__dict__) - _rules_globals.update(iris.coords.__dict__) - _rules_globals.update(iris.coord_systems.__dict__) - _rules_globals.update(iris.fileformats.um_cf_map.__dict__) - _rules_globals.update(cf_units.__dict__) - _import_pending = False - - return _rules_globals.copy() - - # A flag to control all the text-rules and rules-logging deprecation warnings. _enable_rules_deprecations = True