diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index d8829d8587..436ac373e3 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1791,18 +1791,19 @@ def set_parallel(self): """Set 'parallel' easyconfig parameter to determine how many cores can/should be used for parallel builds.""" # set level of parallelism for build par = build_option('parallel') - if self.cfg['parallel'] is not None: - if par is None: - par = self.cfg['parallel'] - self.log.debug("Desired parallelism specified via 'parallel' easyconfig parameter: %s", par) - else: - par = min(int(par), int(self.cfg['parallel'])) - self.log.debug("Desired parallelism: minimum of 'parallel' build option/easyconfig parameter: %s", par) - else: + cfg_par = self.cfg['parallel'] + if cfg_par is None: self.log.debug("Desired parallelism specified via 'parallel' build option: %s", par) + elif par is None: + par = cfg_par + self.log.debug("Desired parallelism specified via 'parallel' easyconfig parameter: %s", par) + else: + par = min(int(par), int(cfg_par)) + self.log.debug("Desired parallelism: minimum of 'parallel' build option/easyconfig parameter: %s", par) - self.cfg['parallel'] = det_parallelism(par=par, maxpar=self.cfg['maxparallel']) - self.log.info("Setting parallelism: %s" % self.cfg['parallel']) + par = det_parallelism(par, maxpar=self.cfg['maxparallel']) + self.log.info("Setting parallelism: %s" % par) + self.cfg['parallel'] = par def remove_module_file(self): """Remove module file (if it exists), and check for ghost installation directory (and deal with it).""" diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index 108cdb4571..a43338e147 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -976,31 +976,42 @@ def det_parallelism(par=None, maxpar=None): Determine level of parallelism that should be used. Default: educated guess based on # cores and 'ulimit -u' setting: min(# cores, ((ulimit -u) - 15) // 6) """ - if par is not None: - if not isinstance(par, int): + def get_default_parallelism(): + try: + # Get cache value if any + par = det_parallelism._default_parallelism + except AttributeError: + # No cache -> Calculate value from current system values + par = get_avail_core_count() + # check ulimit -u + out, ec = run_cmd('ulimit -u', force_in_dry_run=True, trace=False, stream_output=False) try: - par = int(par) + if out.startswith("unlimited"): + maxuserproc = 2 ** 32 - 1 + else: + maxuserproc = int(out) except ValueError as err: - raise EasyBuildError("Specified level of parallelism '%s' is not an integer value: %s", par, err) - else: - par = get_avail_core_count() - # check ulimit -u - out, ec = run_cmd('ulimit -u', force_in_dry_run=True, trace=False, stream_output=False) - try: - if out.startswith("unlimited"): - out = 2 ** 32 - 1 - maxuserproc = int(out) + raise EasyBuildError("Failed to determine max user processes (%s, %s): %s", ec, out, err) # assume 6 processes per build thread + 15 overhead - par_guess = int((maxuserproc - 15) // 6) + par_guess = (maxuserproc - 15) // 6 if par_guess < par: par = par_guess - _log.info("Limit parallel builds to %s because max user processes is %s" % (par, out)) + _log.info("Limit parallel builds to %s because max user processes is %s", par, out) + # Cache value + det_parallelism._default_parallelism = par + return par + + if par is None: + par = get_default_parallelism() + else: + try: + par = int(par) except ValueError as err: - raise EasyBuildError("Failed to determine max user processes (%s, %s): %s", ec, out, err) + raise EasyBuildError("Specified level of parallelism '%s' is not an integer value: %s", par, err) if maxpar is not None and maxpar < par: - _log.info("Limiting parallellism from %s to %s" % (par, maxpar)) - par = min(par, maxpar) + _log.info("Limiting parallellism from %s to %s", par, maxpar) + par = maxpar return par diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 96ce325e6e..48aaeb86a4 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -3052,6 +3052,10 @@ def test_template_constant_dict(self): self.assertEqual(res, expected) # mock get_avail_core_count which is used by set_parallel -> det_parallelism + try: + del st.det_parallelism._default_parallelism # Remove cache value + except AttributeError: + pass # Ignore if not present orig_get_avail_core_count = st.get_avail_core_count st.get_avail_core_count = lambda: 42