From dda3a48aec2dd6ce813565d3724c1a9cfdb86e66 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 30 Aug 2021 16:46:04 +0200 Subject: [PATCH 1/3] Speedup EasyBlock.set_parallel - Small refactoring to reduce dictionary access to cfg[parallel] option - Cache default parallel value in det_parallelism to avoid systemcalls --- easybuild/framework/easyblock.py | 16 +++++++------ easybuild/tools/systemtools.py | 41 ++++++++++++++++++++------------ 2 files changed, 35 insertions(+), 22 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index d8829d8587..22a13e630a 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1791,18 +1791,20 @@ def set_parallel(self): """Set 'parallel' easyconfig parameter to determine how many cores can/should be used for parallel builds.""" # set level of parallelism for build par = build_option('parallel') - if self.cfg['parallel'] is not None: + cfg_par = self.cfg['parallel'] + if cfg_par is None: + self.log.debug("Desired parallelism specified via 'parallel' build option: %s", par) + else: if par is None: - par = self.cfg['parallel'] + par = cfg_par self.log.debug("Desired parallelism specified via 'parallel' easyconfig parameter: %s", par) else: - par = min(int(par), int(self.cfg['parallel'])) + par = min(int(par), int(cfg_par)) self.log.debug("Desired parallelism: minimum of 'parallel' build option/easyconfig parameter: %s", par) - else: - self.log.debug("Desired parallelism specified via 'parallel' build option: %s", par) - self.cfg['parallel'] = det_parallelism(par=par, maxpar=self.cfg['maxparallel']) - self.log.info("Setting parallelism: %s" % self.cfg['parallel']) + par = det_parallelism(par, maxpar=self.cfg['maxparallel']) + self.log.info("Setting parallelism: %s" % par) + self.cfg['parallel'] = par def remove_module_file(self): """Remove module file (if it exists), and check for ghost installation directory (and deal with it).""" diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index 108cdb4571..16d471f37c 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -976,31 +976,42 @@ def det_parallelism(par=None, maxpar=None): Determine level of parallelism that should be used. Default: educated guess based on # cores and 'ulimit -u' setting: min(# cores, ((ulimit -u) - 15) // 6) """ - if par is not None: - if not isinstance(par, int): + def get_default_parallelism(): + try: + # Get cache value if any + par = det_parallelism._default_parallelism + except AttributeError: + # No cache -> Calculate value from current system values + par = get_avail_core_count() + # check ulimit -u + out, ec = run_cmd('ulimit -u', force_in_dry_run=True, trace=False, stream_output=False) try: - par = int(par) + if out.startswith("unlimited"): + maxuserproc = 2 ** 32 - 1 + else: + maxuserproc = int(out) except ValueError as err: - raise EasyBuildError("Specified level of parallelism '%s' is not an integer value: %s", par, err) - else: - par = get_avail_core_count() - # check ulimit -u - out, ec = run_cmd('ulimit -u', force_in_dry_run=True, trace=False, stream_output=False) - try: - if out.startswith("unlimited"): - out = 2 ** 32 - 1 - maxuserproc = int(out) + raise EasyBuildError("Failed to determine max user processes (%s, %s): %s", ec, out, err) # assume 6 processes per build thread + 15 overhead - par_guess = int((maxuserproc - 15) // 6) + par_guess = (maxuserproc - 15) // 6 if par_guess < par: par = par_guess _log.info("Limit parallel builds to %s because max user processes is %s" % (par, out)) + # Cache value + det_parallelism._default_parallelism = par + return par + + if par is None: + par = get_default_parallelism() + else: + try: + par = int(par) except ValueError as err: - raise EasyBuildError("Failed to determine max user processes (%s, %s): %s", ec, out, err) + raise EasyBuildError("Specified level of parallelism '%s' is not an integer value: %s", par, err) if maxpar is not None and maxpar < par: _log.info("Limiting parallellism from %s to %s" % (par, maxpar)) - par = min(par, maxpar) + par = maxpar return par From 0f19f1c664160d7411bb49d80687d4c80c5af8c3 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 30 Aug 2021 18:14:59 +0200 Subject: [PATCH 2/3] handle test failure --- test/framework/easyconfig.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 96ce325e6e..48aaeb86a4 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -3052,6 +3052,10 @@ def test_template_constant_dict(self): self.assertEqual(res, expected) # mock get_avail_core_count which is used by set_parallel -> det_parallelism + try: + del st.det_parallelism._default_parallelism # Remove cache value + except AttributeError: + pass # Ignore if not present orig_get_avail_core_count = st.get_avail_core_count st.get_avail_core_count = lambda: 42 From a99001b2faa24846c4727604ca24a5f6a7e69734 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 31 Aug 2021 17:29:14 +0200 Subject: [PATCH 3/3] trivial style fixes --- easybuild/framework/easyblock.py | 11 +++++------ easybuild/tools/systemtools.py | 4 ++-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 22a13e630a..436ac373e3 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1794,13 +1794,12 @@ def set_parallel(self): cfg_par = self.cfg['parallel'] if cfg_par is None: self.log.debug("Desired parallelism specified via 'parallel' build option: %s", par) + elif par is None: + par = cfg_par + self.log.debug("Desired parallelism specified via 'parallel' easyconfig parameter: %s", par) else: - if par is None: - par = cfg_par - self.log.debug("Desired parallelism specified via 'parallel' easyconfig parameter: %s", par) - else: - par = min(int(par), int(cfg_par)) - self.log.debug("Desired parallelism: minimum of 'parallel' build option/easyconfig parameter: %s", par) + par = min(int(par), int(cfg_par)) + self.log.debug("Desired parallelism: minimum of 'parallel' build option/easyconfig parameter: %s", par) par = det_parallelism(par, maxpar=self.cfg['maxparallel']) self.log.info("Setting parallelism: %s" % par) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index 16d471f37c..a43338e147 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -996,7 +996,7 @@ def get_default_parallelism(): par_guess = (maxuserproc - 15) // 6 if par_guess < par: par = par_guess - _log.info("Limit parallel builds to %s because max user processes is %s" % (par, out)) + _log.info("Limit parallel builds to %s because max user processes is %s", par, out) # Cache value det_parallelism._default_parallelism = par return par @@ -1010,7 +1010,7 @@ def get_default_parallelism(): raise EasyBuildError("Specified level of parallelism '%s' is not an integer value: %s", par, err) if maxpar is not None and maxpar < par: - _log.info("Limiting parallellism from %s to %s" % (par, maxpar)) + _log.info("Limiting parallellism from %s to %s", par, maxpar) par = maxpar return par