diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index a824fb70a1..9bb3e2dd2e 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2436,18 +2436,33 @@ def set_parallel(self): # set level of parallelism for build par = build_option('parallel') if par is not None: - self.log.debug("Desired parallelism specified via 'parallel' build option: %s", par) + self.log.debug(f"Desired parallelism specified via 'parallel' build option: {par}") # Transitional only in case some easyblocks still set/change cfg['parallel'] # Use _parallelLegacy to avoid deprecation warnings - cfg_par = self.cfg['_parallelLegacy'] - if cfg_par is not None: + par_ec = self.cfg['_parallelLegacy'] + if par_ec is not None: if par is None: - par = cfg_par + par = par_ec else: - par = min(int(par), int(cfg_par)) + par = min(int(par), int(par_ec)) + + # --max-parallel specifies global maximum for parallelism + max_par_global = int(build_option('max_parallel')) + # note: 'max_parallel' and 'maxparallel; are the same easyconfig parameter, + # since 'max_parallel' is an alternative name for 'maxparallel' + max_par_ec = self.cfg['max_parallel'] + # take into account that False is a valid value for max_parallel + if max_par_ec is False: + max_par_ec = 1 + # if max_parallel is not specified in easyconfig, we take the global value + if max_par_ec is None: + max_par = max_par_global + # take minimum value if both are specified + else: + max_par = min(int(max_par_ec), max_par_global) - par = det_parallelism(par=par, maxpar=self.cfg['max_parallel']) + par = det_parallelism(par=par, maxpar=max_par) self.log.info(f"Setting parallelism: {par}") self.cfg.parallel = par diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 4a853a75c9..664080d8f2 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -100,6 +100,7 @@ DEFAULT_JOB_EB_CMD = 'eb' DEFAULT_LOGFILE_FORMAT = ("easybuild", "easybuild-%(name)s-%(version)s-%(date)s.%(time)s.log") DEFAULT_MAX_FAIL_RATIO_PERMS = 0.5 +DEFAULT_MAX_PARALLEL = 16 DEFAULT_MINIMAL_BUILD_ENV = 'CC:gcc,CXX:g++' DEFAULT_MNS = 'EasyBuildMNS' DEFAULT_MODULE_SYNTAX = 'Lua' @@ -395,6 +396,9 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): DEFAULT_MAX_FAIL_RATIO_PERMS: [ 'max_fail_ratio_adjust_permissions', ], + DEFAULT_MAX_PARALLEL: [ + 'max_parallel', + ], DEFAULT_MINIMAL_BUILD_ENV: [ 'minimal_build_env', ], diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 05f0d775da..4068aa1ad6 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -67,15 +67,15 @@ from easybuild.tools.config import DEFAULT_ENV_FOR_SHEBANG, DEFAULT_ENVVAR_USERS_MODULES from easybuild.tools.config import DEFAULT_FORCE_DOWNLOAD, DEFAULT_INDEX_MAX_AGE, DEFAULT_JOB_BACKEND from easybuild.tools.config import DEFAULT_JOB_EB_CMD, DEFAULT_LOGFILE_FORMAT, DEFAULT_MAX_FAIL_RATIO_PERMS -from easybuild.tools.config import DEFAULT_MINIMAL_BUILD_ENV, DEFAULT_MNS, DEFAULT_MODULE_SYNTAX, DEFAULT_MODULES_TOOL +from easybuild.tools.config import DEFAULT_MAX_PARALLEL, DEFAULT_MINIMAL_BUILD_ENV, DEFAULT_MNS +from easybuild.tools.config import DEFAULT_MOD_SEARCH_PATH_HEADERS, DEFAULT_MODULE_SYNTAX, DEFAULT_MODULES_TOOL from easybuild.tools.config import DEFAULT_MODULECLASSES, DEFAULT_PATH_SUBDIRS, DEFAULT_PKG_RELEASE, DEFAULT_PKG_TOOL -from easybuild.tools.config import DEFAULT_MOD_SEARCH_PATH_HEADERS, MOD_SEARCH_PATH_HEADERS from easybuild.tools.config import DEFAULT_PKG_TYPE, DEFAULT_PNS, DEFAULT_PREFIX, DEFAULT_EXTRA_SOURCE_URLS from easybuild.tools.config import DEFAULT_REPOSITORY, DEFAULT_WAIT_ON_LOCK_INTERVAL, DEFAULT_WAIT_ON_LOCK_LIMIT from easybuild.tools.config import DEFAULT_PR_TARGET_ACCOUNT, DEFAULT_FILTER_RPATH_SANITY_LIBS from easybuild.tools.config import EBROOT_ENV_VAR_ACTIONS, ERROR, FORCE_DOWNLOAD_CHOICES, GENERAL_CLASS, IGNORE from easybuild.tools.config import JOB_DEPS_TYPE_ABORT_ON_ERROR, JOB_DEPS_TYPE_ALWAYS_RUN, LOADED_MODULES_ACTIONS -from easybuild.tools.config import LOCAL_VAR_NAMING_CHECK_WARN, LOCAL_VAR_NAMING_CHECKS +from easybuild.tools.config import LOCAL_VAR_NAMING_CHECK_WARN, LOCAL_VAR_NAMING_CHECKS, MOD_SEARCH_PATH_HEADERS from easybuild.tools.config import OUTPUT_STYLE_AUTO, OUTPUT_STYLES, WARN, build_option from easybuild.tools.config import get_pretend_installpath, init, init_build_options, mk_full_default_path from easybuild.tools.config import BuildOptions, ConfigurationVariables @@ -475,6 +475,8 @@ def override_options(self): None, 'store_true', True), 'max-fail-ratio-adjust-permissions': ("Maximum ratio for failures to allow when adjusting permissions", 'float', 'store', DEFAULT_MAX_FAIL_RATIO_PERMS), + 'max-parallel': ("Specify maximum level of parallelism that should be used during build procedure", + 'int', 'store', DEFAULT_MAX_PARALLEL), 'minimal-build-env': ("Minimal build environment to define when using system toolchain, " "specified as a comma-separated list that defines a mapping between name of " "environment variable and its value separated by a colon (':')", @@ -496,9 +498,10 @@ def override_options(self): 'output-style': ("Control output style; auto implies using Rich if available to produce rich output, " "with fallback to basic colored output", 'choice', 'store', OUTPUT_STYLE_AUTO, OUTPUT_STYLES), - 'parallel': ("Specify (maximum) level of parallelism used during build procedure " - "(actual value is determined by available cores + 'max_parallel' easyconfig parameter)", - 'int', 'store', 16), + 'parallel': ("Specify level of parallelism that should be used during build procedure, " + "(bypasses auto-detection of number of available cores; " + "actual value is determined by this value + 'max_parallel' easyconfig parameter)", + 'int', 'store', None), 'parallel-extensions-install': ("Install list of extensions in parallel (if supported)", None, 'store_true', False), 'pre-create-installdir': ("Create installation directory before submitting build jobs", diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 62b95c0b3e..de3b3e5198 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -2529,7 +2529,7 @@ def test_parallel(self): write_file(toy_ec5, toytxt + "\nmaxparallel = False") # default: parallelism is derived from # available cores + ulimit - # Note that maxparallel has a default of 16, so we need a lower auto_parallel value here + # Note that --max-parallel has a default of 16, so we need a lower auto_parallel value here auto_parallel = 16 - 4 # Using + 3 below which must still be less st.det_parallelism._default_parallelism = auto_parallel @@ -2565,7 +2565,10 @@ def test_parallel(self): buildopt_parallel = 11 # When build option is given the auto-parallelism is ignored. Verify by setting it very low st.det_parallelism._default_parallelism = 2 - init_config(build_options={'parallel': str(buildopt_parallel), 'validate': False}) + init_config(build_options={ + 'parallel': str(buildopt_parallel), + 'validate': False, + }) test_cases = { '': buildopt_parallel, @@ -2583,6 +2586,61 @@ def test_parallel(self): 'parallel = 8\nmaxparallel = False': 1, } + for txt, expected in test_cases.items(): + with self.subTest(ec_params=txt): + self.contents = toytxt + '\n' + txt + self.writeEC() + with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr(): + test_eb = EasyBlock(EasyConfig(self.eb_file)) + test_eb.post_init() + self.assertEqual(test_eb.cfg.parallel, expected) + with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr(): + self.assertEqual(test_eb.cfg['parallel'], expected) + + # re-check when --max-parallel is used instead + buildopt_max_parallel = 8 + st.det_parallelism._default_parallelism = 16 + init_config(build_options={ + 'max_parallel': buildopt_max_parallel, + 'validate': False, + }) + + test_cases = { + '': buildopt_max_parallel, + 'parallel = False': 1, + 'parallel = 1': 1, + 'parallel = 6': 6, + # --max-parallel value limits max. parallelism, so only 8 cores will be used when 'parallel = 10' is used + f'parallel = {buildopt_max_parallel + 2}': buildopt_max_parallel, + 'maxparallel = False': 1, + 'maxparallel = 1': 1, + 'maxparallel = 6': 6, + # minimum of 'maxparallel' easyconfig parameter and --max-parallel configuration option is used + f'maxparallel = {buildopt_max_parallel + 2}': buildopt_max_parallel, + 'parallel = 8\nmaxparallel = 6': 6, + 'parallel = 8\nmaxparallel = 9': 8, + 'parallel = False\nmaxparallel = 6': 1, + 'parallel = 8\nmaxparallel = False': 1, + } + + for txt, expected in test_cases.items(): + with self.subTest(ec_params=txt): + self.contents = toytxt + '\n' + txt + self.writeEC() + with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr(): + test_eb = EasyBlock(EasyConfig(self.eb_file)) + test_eb.post_init() + self.assertEqual(test_eb.cfg.parallel, expected) + with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr(): + self.assertEqual(test_eb.cfg['parallel'], expected) + + # re-check when both --max-parallel and --parallel are used (--max-parallel wins) + init_config(build_options={ + 'max_parallel': buildopt_max_parallel, + 'parallel': buildopt_parallel, + 'validate': False, + }) + for txt, expected in test_cases.items(): with self.subTest(ec_params=txt): self.contents = toytxt + '\n' + txt @@ -2617,13 +2675,13 @@ def test_parallel(self): self.writeEC() with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr(): test_eb = EasyBlock(EasyConfig(self.eb_file)) - parallel = buildopt_parallel - 2 + parallel = buildopt_max_parallel - 2 test_eb.cfg['parallel'] = parallel # Old Easyblocks might change that before the ready step test_eb.post_init() self.assertEqual(test_eb.cfg.parallel, parallel) self.assertEqual(test_eb.cfg['parallel'], parallel) # Afterwards it also gets reflected directly ignoring maxparallel - parallel = buildopt_parallel * 3 + parallel = buildopt_max_parallel * 3 test_eb.cfg['parallel'] = parallel self.assertEqual(test_eb.cfg.parallel, parallel) self.assertEqual(test_eb.cfg['parallel'], parallel) diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index 30ae336668..7529166d72 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -848,11 +848,13 @@ def test_det_parallelism_mocked(self): # mock number of available cores to 8 st.get_avail_core_count = lambda: 8 self.assertTrue(det_parallelism(), 8) + # make 'ulimit -u' return '40', which should result in default (max) parallelism of 4 ((40-15)/6) + del det_parallelism._default_parallelism st.run_shell_cmd = mocked_run_shell_cmd - self.assertTrue(det_parallelism(), 4) - self.assertTrue(det_parallelism(par=6), 4) - self.assertTrue(det_parallelism(maxpar=2), 2) + self.assertEqual(det_parallelism(), 4) + self.assertEqual(det_parallelism(par=6), 6) + self.assertEqual(det_parallelism(maxpar=2), 2) st.get_avail_core_count = orig_get_avail_core_count