add --max-parallel configuration option to control maximum number of cores to use, set to 16 by default

Author: boegel

easybuild/framework/easyblock.py

@@ -2447,7 +2447,22 @@ def set_parallel(self):
             else:
                 par = min(int(par), int(cfg_par))
 
-        par = det_parallelism(par=par, maxpar=self.cfg['max_parallel'])
+        # --max-parallel specifies global maximum for parallelism
+        max_par_cfg = int(build_option('max_parallel'))
+        # note: 'max_parallel' and 'maxparallel; are the same easyconfig parameter,
+        # since 'max_parallel' is an alternative name for 'maxparallel'
+        max_par = self.cfg['max_parallel']
+        # take into account that False is a valid value for max_parallel
+        if max_par is False:
+            max_par = 1
+        # if max_parallel is not specified in easyconfig, we take the global value
+        if max_par is None:
+            max_par = max_par_cfg
+        # take minimum value if both are specified
+        else:
+            max_par = min(int(max_par), max_par_cfg)
+
+        par = det_parallelism(par=par, maxpar=max_par)
         self.log.info(f"Setting parallelism: {par}")
         self.cfg.parallel = par
 

easybuild/tools/config.py

@@ -100,6 +100,7 @@
 DEFAULT_JOB_EB_CMD = 'eb'
 DEFAULT_LOGFILE_FORMAT = ("easybuild", "easybuild-%(name)s-%(version)s-%(date)s.%(time)s.log")
 DEFAULT_MAX_FAIL_RATIO_PERMS = 0.5
+DEFAULT_MAX_PARALLEL = 16
 DEFAULT_MINIMAL_BUILD_ENV = 'CC:gcc,CXX:g++'
 DEFAULT_MNS = 'EasyBuildMNS'
 DEFAULT_MODULE_SYNTAX = 'Lua'
@@ -395,6 +396,9 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
     DEFAULT_MAX_FAIL_RATIO_PERMS: [
         'max_fail_ratio_adjust_permissions',
     ],
+    DEFAULT_MAX_PARALLEL: [
+        'max_parallel',
+    ],
     DEFAULT_MINIMAL_BUILD_ENV: [
         'minimal_build_env',
     ],

easybuild/tools/options.py

@@ -67,15 +67,15 @@
 from easybuild.tools.config import DEFAULT_ENV_FOR_SHEBANG, DEFAULT_ENVVAR_USERS_MODULES
 from easybuild.tools.config import DEFAULT_FORCE_DOWNLOAD, DEFAULT_INDEX_MAX_AGE, DEFAULT_JOB_BACKEND
 from easybuild.tools.config import DEFAULT_JOB_EB_CMD, DEFAULT_LOGFILE_FORMAT, DEFAULT_MAX_FAIL_RATIO_PERMS
-from easybuild.tools.config import DEFAULT_MINIMAL_BUILD_ENV, DEFAULT_MNS, DEFAULT_MODULE_SYNTAX, DEFAULT_MODULES_TOOL
+from easybuild.tools.config import DEFAULT_MAX_PARALLEL, DEFAULT_MINIMAL_BUILD_ENV, DEFAULT_MNS
+from easybuild.tools.config import DEFAULT_MOD_SEARCH_PATH_HEADERS, DEFAULT_MODULE_SYNTAX, DEFAULT_MODULES_TOOL
 from easybuild.tools.config import DEFAULT_MODULECLASSES, DEFAULT_PATH_SUBDIRS, DEFAULT_PKG_RELEASE, DEFAULT_PKG_TOOL
-from easybuild.tools.config import DEFAULT_MOD_SEARCH_PATH_HEADERS, MOD_SEARCH_PATH_HEADERS
 from easybuild.tools.config import DEFAULT_PKG_TYPE, DEFAULT_PNS, DEFAULT_PREFIX, DEFAULT_EXTRA_SOURCE_URLS
 from easybuild.tools.config import DEFAULT_REPOSITORY, DEFAULT_WAIT_ON_LOCK_INTERVAL, DEFAULT_WAIT_ON_LOCK_LIMIT
 from easybuild.tools.config import DEFAULT_PR_TARGET_ACCOUNT, DEFAULT_FILTER_RPATH_SANITY_LIBS
 from easybuild.tools.config import EBROOT_ENV_VAR_ACTIONS, ERROR, FORCE_DOWNLOAD_CHOICES, GENERAL_CLASS, IGNORE
 from easybuild.tools.config import JOB_DEPS_TYPE_ABORT_ON_ERROR, JOB_DEPS_TYPE_ALWAYS_RUN, LOADED_MODULES_ACTIONS
-from easybuild.tools.config import LOCAL_VAR_NAMING_CHECK_WARN, LOCAL_VAR_NAMING_CHECKS
+from easybuild.tools.config import LOCAL_VAR_NAMING_CHECK_WARN, LOCAL_VAR_NAMING_CHECKS, MOD_SEARCH_PATH_HEADERS
 from easybuild.tools.config import OUTPUT_STYLE_AUTO, OUTPUT_STYLES, WARN, build_option
 from easybuild.tools.config import get_pretend_installpath, init, init_build_options, mk_full_default_path
 from easybuild.tools.config import BuildOptions, ConfigurationVariables
@@ -475,6 +475,8 @@ def override_options(self):
                                   None, 'store_true', True),
             'max-fail-ratio-adjust-permissions': ("Maximum ratio for failures to allow when adjusting permissions",
                                                   'float', 'store', DEFAULT_MAX_FAIL_RATIO_PERMS),
+            'max-parallel': ("Specify maximum level of parallelism that should be used during build procedure",
+                             'int', 'store', DEFAULT_MAX_PARALLEL),
             'minimal-build-env': ("Minimal build environment to define when using system toolchain, "
                                   "specified as a comma-separated list that defines a mapping between name of "
                                   "environment variable and its value separated by a colon (':')",
@@ -496,9 +498,10 @@ def override_options(self):
             'output-style': ("Control output style; auto implies using Rich if available to produce rich output, "
                              "with fallback to basic colored output",
                              'choice', 'store', OUTPUT_STYLE_AUTO, OUTPUT_STYLES),
-            'parallel': ("Specify (maximum) level of parallelism used during build procedure "
-                         "(actual value is determined by available cores + 'max_parallel' easyconfig parameter)",
-                         'int', 'store', 16),
+            'parallel': ("Specify level of parallelism that should be used during build procedure, "
+                         "(bypasses auto-detection of number of available cores; "
+                         "actual value is determined by this value + 'max_parallel' easyconfig parameter)",
+                         'int', 'store', None),
             'parallel-extensions-install': ("Install list of extensions in parallel (if supported)",
                                             None, 'store_true', False),
             'pre-create-installdir': ("Create installation directory before submitting build jobs",

test/framework/easyblock.py

@@ -49,7 +49,7 @@
 from easybuild.framework.extensioneasyblock import ExtensionEasyBlock
 from easybuild.tools import LooseVersion, config
 from easybuild.tools.build_log import EasyBuildError
-from easybuild.tools.config import get_module_syntax, update_build_option
+from easybuild.tools.config import get_module_syntax
 from easybuild.tools.environment import modify_env
 from easybuild.tools.filetools import change_dir, copy_dir, copy_file, mkdir, read_file, remove_dir, remove_file
 from easybuild.tools.filetools import symlink, verify_checksum, write_file
@@ -2529,7 +2529,7 @@ def test_parallel(self):
         write_file(toy_ec5, toytxt + "\nmaxparallel = False")
 
         # default: parallelism is derived from # available cores + ulimit
-        # Note that maxparallel has a default of 16, so we need a lower auto_parallel value here
+        # Note that --max-parallel has a default of 16, so we need a lower auto_parallel value here
         auto_parallel = 16 - 4  # Using + 3 below which must still be less
         st.det_parallelism._default_parallelism = auto_parallel
 
@@ -2565,7 +2565,10 @@ def test_parallel(self):
         buildopt_parallel = 11
         # When build option is given the auto-parallelism is ignored. Verify by setting it very low
         st.det_parallelism._default_parallelism = 2
-        init_config(build_options={'parallel': str(buildopt_parallel), 'validate': False})
+        init_config(build_options={
+            'parallel': str(buildopt_parallel),
+            'validate': False,
+        })
 
         test_cases = {
             '': buildopt_parallel,
@@ -2583,6 +2586,61 @@ def test_parallel(self):
             'parallel = 8\nmaxparallel = False': 1,
         }
 
+        for txt, expected in test_cases.items():
+            with self.subTest(ec_params=txt):
+                self.contents = toytxt + '\n' + txt
+                self.writeEC()
+                with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr():
+                    test_eb = EasyBlock(EasyConfig(self.eb_file))
+                test_eb.post_init()
+                self.assertEqual(test_eb.cfg.parallel, expected)
+                with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr():
+                    self.assertEqual(test_eb.cfg['parallel'], expected)
+
+        # re-check when --max-parallel is used instead
+        buildopt_max_parallel = 8
+        st.det_parallelism._default_parallelism = 16
+        init_config(build_options={
+            'max_parallel': buildopt_max_parallel,
+            'validate': False,
+        })
+
+        test_cases = {
+            '': buildopt_max_parallel,
+            'parallel = False': 1,
+            'parallel = 1': 1,
+            'parallel = 6': 6,
+            # --max-parallel value limits max. parallelism, so only 8 cores will be used when 'parallel = 10' is used
+            f'parallel = {buildopt_max_parallel + 2}': buildopt_max_parallel,
+            'maxparallel = False': 1,
+            'maxparallel = 1': 1,
+            'maxparallel = 6': 6,
+            # minimum of 'maxparallel' easyconfig parameter and --max-parallel configuration option is used
+            f'maxparallel = {buildopt_max_parallel + 2}': buildopt_max_parallel,
+            'parallel = 8\nmaxparallel = 6': 6,
+            'parallel = 8\nmaxparallel = 9': 8,
+            'parallel = False\nmaxparallel = 6': 1,
+            'parallel = 8\nmaxparallel = False': 1,
+        }
+
+        for txt, expected in test_cases.items():
+            with self.subTest(ec_params=txt):
+                self.contents = toytxt + '\n' + txt
+                self.writeEC()
+                with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr():
+                    test_eb = EasyBlock(EasyConfig(self.eb_file))
+                test_eb.post_init()
+                self.assertEqual(test_eb.cfg.parallel, expected)
+                with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr():
+                    self.assertEqual(test_eb.cfg['parallel'], expected)
+
+        # re-check when both --max-parallel and --parallel are used (--max-parallel wins)
+        init_config(build_options={
+            'max_parallel': buildopt_max_parallel,
+            'parallel': buildopt_parallel,
+            'validate': False,
+        })
+
         for txt, expected in test_cases.items():
             with self.subTest(ec_params=txt):
                 self.contents = toytxt + '\n' + txt
@@ -2617,13 +2675,13 @@ def test_parallel(self):
         self.writeEC()
         with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr():
             test_eb = EasyBlock(EasyConfig(self.eb_file))
-            parallel = buildopt_parallel - 2
+            parallel = buildopt_max_parallel - 2
             test_eb.cfg['parallel'] = parallel  # Old Easyblocks might change that before the ready step
             test_eb.post_init()
             self.assertEqual(test_eb.cfg.parallel, parallel)
             self.assertEqual(test_eb.cfg['parallel'], parallel)
             # Afterwards it also gets reflected directly ignoring maxparallel
-            parallel = buildopt_parallel * 3
+            parallel = buildopt_max_parallel * 3
             test_eb.cfg['parallel'] = parallel
             self.assertEqual(test_eb.cfg.parallel, parallel)
             self.assertEqual(test_eb.cfg['parallel'], parallel)

test/framework/systemtools.py

@@ -39,7 +39,6 @@
 
 import easybuild.tools.systemtools as st
 from easybuild.tools.build_log import EasyBuildError
-from easybuild.tools.config import update_build_option
 from easybuild.tools.filetools import adjust_permissions, read_file, symlink, which, write_file
 from easybuild.tools.run import RunShellCmdResult, run_shell_cmd
 from easybuild.tools.systemtools import CPU_ARCHITECTURES, AARCH32, AARCH64, POWER, X86_64