From eaa6205c34c3ab76dd61ce9818d0b084db98a783 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 15 Mar 2025 14:23:47 +0100 Subject: [PATCH 1/3] fix test_det_parallelism_mocked: should use assertEqual instead of assertTrue --- test/framework/systemtools.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index 30ae336668..438a0559b5 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -39,6 +39,7 @@ import easybuild.tools.systemtools as st from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.config import update_build_option from easybuild.tools.filetools import adjust_permissions, read_file, symlink, which, write_file from easybuild.tools.run import RunShellCmdResult, run_shell_cmd from easybuild.tools.systemtools import CPU_ARCHITECTURES, AARCH32, AARCH64, POWER, X86_64 @@ -848,11 +849,13 @@ def test_det_parallelism_mocked(self): # mock number of available cores to 8 st.get_avail_core_count = lambda: 8 self.assertTrue(det_parallelism(), 8) + # make 'ulimit -u' return '40', which should result in default (max) parallelism of 4 ((40-15)/6) + del det_parallelism._default_parallelism st.run_shell_cmd = mocked_run_shell_cmd - self.assertTrue(det_parallelism(), 4) - self.assertTrue(det_parallelism(par=6), 4) - self.assertTrue(det_parallelism(maxpar=2), 2) + self.assertEqual(det_parallelism(), 4) + self.assertEqual(det_parallelism(par=6), 6) + self.assertEqual(det_parallelism(maxpar=2), 2) st.get_avail_core_count = orig_get_avail_core_count From 683fe00951d64850e8fb8ce159170462484225ed Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 15 Mar 2025 15:25:56 +0100 Subject: [PATCH 2/3] add --max-parallel configuration option to control maximum number of cores to use, set to 16 by default --- easybuild/framework/easyblock.py | 17 +++++++- easybuild/tools/config.py | 4 ++ easybuild/tools/options.py | 15 +++++--- test/framework/easyblock.py | 66 ++++++++++++++++++++++++++++++-- test/framework/systemtools.py | 1 - 5 files changed, 91 insertions(+), 12 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index a824fb70a1..1e0a0c3174 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2447,7 +2447,22 @@ def set_parallel(self): else: par = min(int(par), int(cfg_par)) - par = det_parallelism(par=par, maxpar=self.cfg['max_parallel']) + # --max-parallel specifies global maximum for parallelism + max_par_cfg = int(build_option('max_parallel')) + # note: 'max_parallel' and 'maxparallel; are the same easyconfig parameter, + # since 'max_parallel' is an alternative name for 'maxparallel' + max_par = self.cfg['max_parallel'] + # take into account that False is a valid value for max_parallel + if max_par is False: + max_par = 1 + # if max_parallel is not specified in easyconfig, we take the global value + if max_par is None: + max_par = max_par_cfg + # take minimum value if both are specified + else: + max_par = min(int(max_par), max_par_cfg) + + par = det_parallelism(par=par, maxpar=max_par) self.log.info(f"Setting parallelism: {par}") self.cfg.parallel = par diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 4a853a75c9..664080d8f2 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -100,6 +100,7 @@ DEFAULT_JOB_EB_CMD = 'eb' DEFAULT_LOGFILE_FORMAT = ("easybuild", "easybuild-%(name)s-%(version)s-%(date)s.%(time)s.log") DEFAULT_MAX_FAIL_RATIO_PERMS = 0.5 +DEFAULT_MAX_PARALLEL = 16 DEFAULT_MINIMAL_BUILD_ENV = 'CC:gcc,CXX:g++' DEFAULT_MNS = 'EasyBuildMNS' DEFAULT_MODULE_SYNTAX = 'Lua' @@ -395,6 +396,9 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): DEFAULT_MAX_FAIL_RATIO_PERMS: [ 'max_fail_ratio_adjust_permissions', ], + DEFAULT_MAX_PARALLEL: [ + 'max_parallel', + ], DEFAULT_MINIMAL_BUILD_ENV: [ 'minimal_build_env', ], diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 05f0d775da..4068aa1ad6 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -67,15 +67,15 @@ from easybuild.tools.config import DEFAULT_ENV_FOR_SHEBANG, DEFAULT_ENVVAR_USERS_MODULES from easybuild.tools.config import DEFAULT_FORCE_DOWNLOAD, DEFAULT_INDEX_MAX_AGE, DEFAULT_JOB_BACKEND from easybuild.tools.config import DEFAULT_JOB_EB_CMD, DEFAULT_LOGFILE_FORMAT, DEFAULT_MAX_FAIL_RATIO_PERMS -from easybuild.tools.config import DEFAULT_MINIMAL_BUILD_ENV, DEFAULT_MNS, DEFAULT_MODULE_SYNTAX, DEFAULT_MODULES_TOOL +from easybuild.tools.config import DEFAULT_MAX_PARALLEL, DEFAULT_MINIMAL_BUILD_ENV, DEFAULT_MNS +from easybuild.tools.config import DEFAULT_MOD_SEARCH_PATH_HEADERS, DEFAULT_MODULE_SYNTAX, DEFAULT_MODULES_TOOL from easybuild.tools.config import DEFAULT_MODULECLASSES, DEFAULT_PATH_SUBDIRS, DEFAULT_PKG_RELEASE, DEFAULT_PKG_TOOL -from easybuild.tools.config import DEFAULT_MOD_SEARCH_PATH_HEADERS, MOD_SEARCH_PATH_HEADERS from easybuild.tools.config import DEFAULT_PKG_TYPE, DEFAULT_PNS, DEFAULT_PREFIX, DEFAULT_EXTRA_SOURCE_URLS from easybuild.tools.config import DEFAULT_REPOSITORY, DEFAULT_WAIT_ON_LOCK_INTERVAL, DEFAULT_WAIT_ON_LOCK_LIMIT from easybuild.tools.config import DEFAULT_PR_TARGET_ACCOUNT, DEFAULT_FILTER_RPATH_SANITY_LIBS from easybuild.tools.config import EBROOT_ENV_VAR_ACTIONS, ERROR, FORCE_DOWNLOAD_CHOICES, GENERAL_CLASS, IGNORE from easybuild.tools.config import JOB_DEPS_TYPE_ABORT_ON_ERROR, JOB_DEPS_TYPE_ALWAYS_RUN, LOADED_MODULES_ACTIONS -from easybuild.tools.config import LOCAL_VAR_NAMING_CHECK_WARN, LOCAL_VAR_NAMING_CHECKS +from easybuild.tools.config import LOCAL_VAR_NAMING_CHECK_WARN, LOCAL_VAR_NAMING_CHECKS, MOD_SEARCH_PATH_HEADERS from easybuild.tools.config import OUTPUT_STYLE_AUTO, OUTPUT_STYLES, WARN, build_option from easybuild.tools.config import get_pretend_installpath, init, init_build_options, mk_full_default_path from easybuild.tools.config import BuildOptions, ConfigurationVariables @@ -475,6 +475,8 @@ def override_options(self): None, 'store_true', True), 'max-fail-ratio-adjust-permissions': ("Maximum ratio for failures to allow when adjusting permissions", 'float', 'store', DEFAULT_MAX_FAIL_RATIO_PERMS), + 'max-parallel': ("Specify maximum level of parallelism that should be used during build procedure", + 'int', 'store', DEFAULT_MAX_PARALLEL), 'minimal-build-env': ("Minimal build environment to define when using system toolchain, " "specified as a comma-separated list that defines a mapping between name of " "environment variable and its value separated by a colon (':')", @@ -496,9 +498,10 @@ def override_options(self): 'output-style': ("Control output style; auto implies using Rich if available to produce rich output, " "with fallback to basic colored output", 'choice', 'store', OUTPUT_STYLE_AUTO, OUTPUT_STYLES), - 'parallel': ("Specify (maximum) level of parallelism used during build procedure " - "(actual value is determined by available cores + 'max_parallel' easyconfig parameter)", - 'int', 'store', 16), + 'parallel': ("Specify level of parallelism that should be used during build procedure, " + "(bypasses auto-detection of number of available cores; " + "actual value is determined by this value + 'max_parallel' easyconfig parameter)", + 'int', 'store', None), 'parallel-extensions-install': ("Install list of extensions in parallel (if supported)", None, 'store_true', False), 'pre-create-installdir': ("Create installation directory before submitting build jobs", diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 62b95c0b3e..de3b3e5198 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -2529,7 +2529,7 @@ def test_parallel(self): write_file(toy_ec5, toytxt + "\nmaxparallel = False") # default: parallelism is derived from # available cores + ulimit - # Note that maxparallel has a default of 16, so we need a lower auto_parallel value here + # Note that --max-parallel has a default of 16, so we need a lower auto_parallel value here auto_parallel = 16 - 4 # Using + 3 below which must still be less st.det_parallelism._default_parallelism = auto_parallel @@ -2565,7 +2565,10 @@ def test_parallel(self): buildopt_parallel = 11 # When build option is given the auto-parallelism is ignored. Verify by setting it very low st.det_parallelism._default_parallelism = 2 - init_config(build_options={'parallel': str(buildopt_parallel), 'validate': False}) + init_config(build_options={ + 'parallel': str(buildopt_parallel), + 'validate': False, + }) test_cases = { '': buildopt_parallel, @@ -2583,6 +2586,61 @@ def test_parallel(self): 'parallel = 8\nmaxparallel = False': 1, } + for txt, expected in test_cases.items(): + with self.subTest(ec_params=txt): + self.contents = toytxt + '\n' + txt + self.writeEC() + with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr(): + test_eb = EasyBlock(EasyConfig(self.eb_file)) + test_eb.post_init() + self.assertEqual(test_eb.cfg.parallel, expected) + with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr(): + self.assertEqual(test_eb.cfg['parallel'], expected) + + # re-check when --max-parallel is used instead + buildopt_max_parallel = 8 + st.det_parallelism._default_parallelism = 16 + init_config(build_options={ + 'max_parallel': buildopt_max_parallel, + 'validate': False, + }) + + test_cases = { + '': buildopt_max_parallel, + 'parallel = False': 1, + 'parallel = 1': 1, + 'parallel = 6': 6, + # --max-parallel value limits max. parallelism, so only 8 cores will be used when 'parallel = 10' is used + f'parallel = {buildopt_max_parallel + 2}': buildopt_max_parallel, + 'maxparallel = False': 1, + 'maxparallel = 1': 1, + 'maxparallel = 6': 6, + # minimum of 'maxparallel' easyconfig parameter and --max-parallel configuration option is used + f'maxparallel = {buildopt_max_parallel + 2}': buildopt_max_parallel, + 'parallel = 8\nmaxparallel = 6': 6, + 'parallel = 8\nmaxparallel = 9': 8, + 'parallel = False\nmaxparallel = 6': 1, + 'parallel = 8\nmaxparallel = False': 1, + } + + for txt, expected in test_cases.items(): + with self.subTest(ec_params=txt): + self.contents = toytxt + '\n' + txt + self.writeEC() + with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr(): + test_eb = EasyBlock(EasyConfig(self.eb_file)) + test_eb.post_init() + self.assertEqual(test_eb.cfg.parallel, expected) + with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr(): + self.assertEqual(test_eb.cfg['parallel'], expected) + + # re-check when both --max-parallel and --parallel are used (--max-parallel wins) + init_config(build_options={ + 'max_parallel': buildopt_max_parallel, + 'parallel': buildopt_parallel, + 'validate': False, + }) + for txt, expected in test_cases.items(): with self.subTest(ec_params=txt): self.contents = toytxt + '\n' + txt @@ -2617,13 +2675,13 @@ def test_parallel(self): self.writeEC() with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr(): test_eb = EasyBlock(EasyConfig(self.eb_file)) - parallel = buildopt_parallel - 2 + parallel = buildopt_max_parallel - 2 test_eb.cfg['parallel'] = parallel # Old Easyblocks might change that before the ready step test_eb.post_init() self.assertEqual(test_eb.cfg.parallel, parallel) self.assertEqual(test_eb.cfg['parallel'], parallel) # Afterwards it also gets reflected directly ignoring maxparallel - parallel = buildopt_parallel * 3 + parallel = buildopt_max_parallel * 3 test_eb.cfg['parallel'] = parallel self.assertEqual(test_eb.cfg.parallel, parallel) self.assertEqual(test_eb.cfg['parallel'], parallel) diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index 438a0559b5..7529166d72 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -39,7 +39,6 @@ import easybuild.tools.systemtools as st from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.config import update_build_option from easybuild.tools.filetools import adjust_permissions, read_file, symlink, which, write_file from easybuild.tools.run import RunShellCmdResult, run_shell_cmd from easybuild.tools.systemtools import CPU_ARCHITECTURES, AARCH32, AARCH64, POWER, X86_64 From 82670a4f87f650ee36619f7c89f2bdb699962883 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 15 Mar 2025 15:47:09 +0100 Subject: [PATCH 3/3] avoid potential confusion between max_parallel easyconfig parameter and global configuration option in EasyBlock.set_parallel --- easybuild/framework/easyblock.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 1e0a0c3174..9bb3e2dd2e 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2436,31 +2436,31 @@ def set_parallel(self): # set level of parallelism for build par = build_option('parallel') if par is not None: - self.log.debug("Desired parallelism specified via 'parallel' build option: %s", par) + self.log.debug(f"Desired parallelism specified via 'parallel' build option: {par}") # Transitional only in case some easyblocks still set/change cfg['parallel'] # Use _parallelLegacy to avoid deprecation warnings - cfg_par = self.cfg['_parallelLegacy'] - if cfg_par is not None: + par_ec = self.cfg['_parallelLegacy'] + if par_ec is not None: if par is None: - par = cfg_par + par = par_ec else: - par = min(int(par), int(cfg_par)) + par = min(int(par), int(par_ec)) # --max-parallel specifies global maximum for parallelism - max_par_cfg = int(build_option('max_parallel')) + max_par_global = int(build_option('max_parallel')) # note: 'max_parallel' and 'maxparallel; are the same easyconfig parameter, # since 'max_parallel' is an alternative name for 'maxparallel' - max_par = self.cfg['max_parallel'] + max_par_ec = self.cfg['max_parallel'] # take into account that False is a valid value for max_parallel - if max_par is False: - max_par = 1 + if max_par_ec is False: + max_par_ec = 1 # if max_parallel is not specified in easyconfig, we take the global value - if max_par is None: - max_par = max_par_cfg + if max_par_ec is None: + max_par = max_par_global # take minimum value if both are specified else: - max_par = min(int(max_par), max_par_cfg) + max_par = min(int(max_par_ec), max_par_global) par = det_parallelism(par=par, maxpar=max_par) self.log.info(f"Setting parallelism: {par}")