diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 436ac373e3..f8fd885841 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -55,7 +55,8 @@ from easybuild.base import fancylogger from easybuild.framework.easyconfig import EASYCONFIGS_PKG_SUBDIR from easybuild.framework.easyconfig.easyconfig import ITERATE_OPTIONS, EasyConfig, ActiveMNS, get_easyblock_class -from easybuild.framework.easyconfig.easyconfig import get_module_path, letter_dir_for, resolve_template +from easybuild.framework.easyconfig.easyconfig import get_module_path, get_parallel_ec_param_value +from easybuild.framework.easyconfig.easyconfig import letter_dir_for, resolve_template from easybuild.framework.easyconfig.format.format import SANITY_CHECK_PATHS_DIRS, SANITY_CHECK_PATHS_FILES from easybuild.framework.easyconfig.parser import fetch_parameters_from_easyconfig from easybuild.framework.easyconfig.style import MAX_LINE_LENGTH @@ -92,7 +93,7 @@ from easybuild.tools.package.utilities import package from easybuild.tools.py2vs3 import extract_method_name, string_type from easybuild.tools.repository.repository import init_repository -from easybuild.tools.systemtools import check_linked_shared_libs, det_parallelism, get_shared_lib_ext, use_group +from easybuild.tools.systemtools import check_linked_shared_libs, get_shared_lib_ext, use_group from easybuild.tools.utilities import INDENT_4SPACES, get_class_for, nub, quote_str from easybuild.tools.utilities import remove_unwanted_chars, time2str, trace_msg from easybuild.tools.version import this_is_easybuild, VERBOSE_VERSION, VERSION @@ -212,6 +213,8 @@ def __init__(self, ec): self.postmsg = '' # allow a post message to be set, which can be shown as last output self.current_step = None + self.orig_parallel = None + # list of loaded modules self.loaded_modules = [] @@ -1789,20 +1792,13 @@ def det_iter_cnt(self): def set_parallel(self): """Set 'parallel' easyconfig parameter to determine how many cores can/should be used for parallel builds.""" - # set level of parallelism for build - par = build_option('parallel') - cfg_par = self.cfg['parallel'] - if cfg_par is None: - self.log.debug("Desired parallelism specified via 'parallel' build option: %s", par) - elif par is None: - par = cfg_par - self.log.debug("Desired parallelism specified via 'parallel' easyconfig parameter: %s", par) - else: - par = min(int(par), int(cfg_par)) - self.log.debug("Desired parallelism: minimum of 'parallel' build option/easyconfig parameter: %s", par) + # keep track of original value for 'parallel', so it can be restored before determining level + # of parallelism that can be used for extensions + self.orig_parallel = self.cfg['parallel'] + + par = get_parallel_ec_param_value(self.cfg, self.log) - par = det_parallelism(par, maxpar=self.cfg['maxparallel']) - self.log.info("Setting parallelism: %s" % par) + self.log.info("Setting parallelism: %s", par) self.cfg['parallel'] = par def remove_module_file(self): diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 8bc89606ce..4d1c000313 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -73,7 +73,7 @@ from easybuild.tools.module_naming_scheme.utilities import det_hidden_modname, is_valid_module_name from easybuild.tools.modules import modules_tool from easybuild.tools.py2vs3 import OrderedDict, create_base_metaclass, string_type -from easybuild.tools.systemtools import check_os_dependency, pick_dep_version +from easybuild.tools.systemtools import check_os_dependency, det_parallelism, pick_dep_version from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME, is_system_toolchain from easybuild.tools.toolchain.toolchain import TOOLCHAIN_CAPABILITIES, TOOLCHAIN_CAPABILITY_CUDA from easybuild.tools.toolchain.utilities import get_toolchain, search_toolchain @@ -1830,6 +1830,30 @@ def get_cuda_cc_template_value(self, key): raise EasyBuildError(error_msg, key) +def get_parallel_ec_param_value(cfg, log): + """ + Get value for 'parallel' easyconfig parameter for given EasyConfig instance. + Takes into account: + * --parallel EasyBuild configuration option (if defined) + * 'parallel' easyconfig parameter (if defined) + * 'maxparallel' easyconfig parameter (if defined) + * number of available cores (incl. affinity of active EasyBuild session) + """ + # set level of parallelism for build + par = build_option('parallel') + cfg_par = cfg['parallel'] + if cfg_par is None: + log.debug("Desired parallelism specified via 'parallel' build option: %s", par) + elif par is None: + par = cfg_par + log.debug("Desired parallelism specified via 'parallel' easyconfig parameter: %s", par) + else: + par = min(int(par), int(cfg_par)) + log.debug("Desired parallelism: minimum of 'parallel' build option/easyconfig parameter: %s", par) + + return det_parallelism(par, maxpar=cfg['maxparallel']) + + def det_installversion(version, toolchain_name, toolchain_version, prefix, suffix): """Deprecated 'det_installversion' function, to determine exact install version, based on supplied parameters.""" old_fn = 'framework.easyconfig.easyconfig.det_installversion' diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index fcd2cc9596..ae42c1d9a5 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -36,7 +36,7 @@ import copy import os -from easybuild.framework.easyconfig.easyconfig import resolve_template +from easybuild.framework.easyconfig.easyconfig import get_parallel_ec_param_value, resolve_template from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, template_constant_dict from easybuild.tools.build_log import EasyBuildError, raise_nosupport from easybuild.tools.filetools import change_dir @@ -138,6 +138,12 @@ def __init__(self, mself, ext, extra_params=None): self.log.debug("Skipping unknown custom easyconfig parameter '%s' for extension %s/%s: %s", key, name, version, value) + self.cfg['parallel'] = self.master.orig_parallel + par = get_parallel_ec_param_value(self.cfg, self.log) + self.log.info("Setting parallelism: %s", par) + self.log.info("Setting parallelism to %d for extension %s", par, name) + self.cfg['parallel'] = par + self.sanity_check_fail_msgs = [] @property diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 18bfcf07b0..8e8fb9852c 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -1025,10 +1025,16 @@ def test_init_extensions(self): test_ec = os.path.join(self.test_prefix, 'test.eb') test_ec_txt = toy_ec_txt.replace("('barbar', '0.0', {", "('barbar', '0.0', {'easyblock': 'DummyExtension',") + test_ec_txt = test_ec_txt.replace("('barbar', '0.0', {", "('barbar', '0.0', {'maxparallel': 3,") + test_ec_txt = test_ec_txt.replace("('bar', '0.0', {", "('bar', '0.0', {'maxparallel': 1,") + test_ec_txt += "\nparallel = 5" write_file(test_ec, test_ec_txt) ec = process_easyconfig(test_ec)[0] eb = get_easyblock_instance(ec) + # require to trigger setting of 'parallel' value + eb.check_readiness_step() + eb.prepare_for_extensions() eb.init_ext_instances() ext_inst_class_names = [x.__class__.__name__ for x in eb.ext_instances] @@ -1040,6 +1046,21 @@ def test_init_extensions(self): ] self.assertEqual(ext_inst_class_names, expected) + # default parallel is inherited from parent + self.assertEqual(eb.cfg['parallel'], 5) + self.assertEqual(eb.ext_instances[0].cfg['parallel'], 5) + self.assertEqual(eb.ext_instances[3].cfg['parallel'], 5) + + bar_ext = eb.ext_instances[1] + self.assertEqual(bar_ext.name, 'bar') + # parallel should be set to 1 for barbar, due to maxparallel + self.assertEqual(bar_ext.cfg['parallel'], 1) + + barbar_ext = eb.ext_instances[2] + self.assertEqual(barbar_ext.name, 'barbar') + # parallel should be set to 3 for barbar, due to maxparallel + self.assertEqual(barbar_ext.cfg['parallel'], 3) + # check what happen if we specify an easyblock that doesn't derive from Extension, # and hence can't be used to install extensions... test_ec = os.path.join(self.test_prefix, 'test_broken.eb')