Merge pull request #3812 from Flamefire/set_parallel-speedup

speed up set_parallel method in EasyBlock class

Author: boegel

easybuild/framework/easyblock.py

@@ -1791,18 +1791,19 @@ def set_parallel(self):
         """Set 'parallel' easyconfig parameter to determine how many cores can/should be used for parallel builds."""
         # set level of parallelism for build
         par = build_option('parallel')
-        if self.cfg['parallel'] is not None:
-            if par is None:
-                par = self.cfg['parallel']
-                self.log.debug("Desired parallelism specified via 'parallel' easyconfig parameter: %s", par)
-            else:
-                par = min(int(par), int(self.cfg['parallel']))
-                self.log.debug("Desired parallelism: minimum of 'parallel' build option/easyconfig parameter: %s", par)
-        else:
+        cfg_par = self.cfg['parallel']
+        if cfg_par is None:
             self.log.debug("Desired parallelism specified via 'parallel' build option: %s", par)
+        elif par is None:
+            par = cfg_par
+            self.log.debug("Desired parallelism specified via 'parallel' easyconfig parameter: %s", par)
+        else:
+            par = min(int(par), int(cfg_par))
+            self.log.debug("Desired parallelism: minimum of 'parallel' build option/easyconfig parameter: %s", par)
 
-        self.cfg['parallel'] = det_parallelism(par=par, maxpar=self.cfg['maxparallel'])
-        self.log.info("Setting parallelism: %s" % self.cfg['parallel'])
+        par = det_parallelism(par, maxpar=self.cfg['maxparallel'])
+        self.log.info("Setting parallelism: %s" % par)
+        self.cfg['parallel'] = par
 
     def remove_module_file(self):
         """Remove module file (if it exists), and check for ghost installation directory (and deal with it)."""

easybuild/tools/systemtools.py

@@ -976,31 +976,42 @@ def det_parallelism(par=None, maxpar=None):
     Determine level of parallelism that should be used.
     Default: educated guess based on # cores and 'ulimit -u' setting: min(# cores, ((ulimit -u) - 15) // 6)
     """
-    if par is not None:
-        if not isinstance(par, int):
+    def get_default_parallelism():
+        try:
+            # Get cache value if any
+            par = det_parallelism._default_parallelism
+        except AttributeError:
+            # No cache -> Calculate value from current system values
+            par = get_avail_core_count()
+            # check ulimit -u
+            out, ec = run_cmd('ulimit -u', force_in_dry_run=True, trace=False, stream_output=False)
             try:
-                par = int(par)
+                if out.startswith("unlimited"):
+                    maxuserproc = 2 ** 32 - 1
+                else:
+                    maxuserproc = int(out)
             except ValueError as err:
-                raise EasyBuildError("Specified level of parallelism '%s' is not an integer value: %s", par, err)
-    else:
-        par = get_avail_core_count()
-        # check ulimit -u
-        out, ec = run_cmd('ulimit -u', force_in_dry_run=True, trace=False, stream_output=False)
-        try:
-            if out.startswith("unlimited"):
-                out = 2 ** 32 - 1
-            maxuserproc = int(out)
+                raise EasyBuildError("Failed to determine max user processes (%s, %s): %s", ec, out, err)
             # assume 6 processes per build thread + 15 overhead
-            par_guess = int((maxuserproc - 15) // 6)
+            par_guess = (maxuserproc - 15) // 6
             if par_guess < par:
                 par = par_guess
-                _log.info("Limit parallel builds to %s because max user processes is %s" % (par, out))
+                _log.info("Limit parallel builds to %s because max user processes is %s", par, out)
+            # Cache value
+            det_parallelism._default_parallelism = par
+        return par
+
+    if par is None:
+        par = get_default_parallelism()
+    else:
+        try:
+            par = int(par)
         except ValueError as err:
-            raise EasyBuildError("Failed to determine max user processes (%s, %s): %s", ec, out, err)
+            raise EasyBuildError("Specified level of parallelism '%s' is not an integer value: %s", par, err)
 
     if maxpar is not None and maxpar < par:
-        _log.info("Limiting parallellism from %s to %s" % (par, maxpar))
-        par = min(par, maxpar)
+        _log.info("Limiting parallellism from %s to %s", par, maxpar)
+        par = maxpar
 
     return par
 

test/framework/easyconfig.py

@@ -3052,6 +3052,10 @@ def test_template_constant_dict(self):
         self.assertEqual(res, expected)
 
         # mock get_avail_core_count which is used by set_parallel -> det_parallelism
+        try:
+            del st.det_parallelism._default_parallelism  # Remove cache value
+        except AttributeError:
+            pass  # Ignore if not present
         orig_get_avail_core_count = st.get_avail_core_count
         st.get_avail_core_count = lambda: 42