From e346d636ac448b71825c0d5a5350ac11bc7f0595 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 19 Dec 2023 09:36:11 +0100 Subject: [PATCH 1/3] add terse support to `--missing-modules` It is useful to have the missing modules in a format that can be processed in a script, i.e. one EC name per line. So allow `--terse` in combination with `--missing-modules`. --- easybuild/main.py | 6 +++--- easybuild/tools/robot.py | 8 +++++--- test/framework/options.py | 24 ++++++++++++++++++------ 3 files changed, 26 insertions(+), 12 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 73550e3998..826880aad1 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -495,7 +495,7 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session # dry_run: print all easyconfigs and dependencies, and whether they are already built elif dry_run_mode: if options.missing_modules: - txt = missing_deps(easyconfigs, modtool) + txt = missing_deps(easyconfigs, modtool, terse=options.terse) else: txt = dry_run(easyconfigs, modtool, short=not options.dry_run) print_msg(txt, log=_log, silent=testing, prefix=False) @@ -684,7 +684,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, pr options.list_prs, options.merge_pr, options.review_pr, - options.terse, + options.terse and not options.missing_modules, search_query, ] if any(early_stop_options): @@ -721,7 +721,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, pr # stop logging and cleanup tmp log file, unless one build failed (individual logs are located in eb_tmpdir) stop_logging(logfile, logtostdout=options.logtostdout) if do_cleanup: - cleanup(logfile, eb_tmpdir, testing, silent=False) + cleanup(logfile, eb_tmpdir, testing, silent=options.terse) def prepare_main(args=None, logfile=None, testing=None): diff --git a/easybuild/tools/robot.py b/easybuild/tools/robot.py index 68a4c9028b..3a21bc2f7f 100644 --- a/easybuild/tools/robot.py +++ b/easybuild/tools/robot.py @@ -288,16 +288,18 @@ def dry_run(easyconfigs, modtool, short=False): return '\n'.join(lines) -def missing_deps(easyconfigs, modtool): +def missing_deps(easyconfigs, modtool, terse=False): """ Determine subset of easyconfigs for which no module is installed yet. """ ordered_ecs = resolve_dependencies(easyconfigs, modtool, retain_all_deps=True, raise_error_missing_ecs=False) missing = skip_available(ordered_ecs, modtool) - if missing: + if terse: + lines = [os.path.basename(x['ec'].path) for x in missing] + elif missing: lines = ['', "%d out of %d required modules missing:" % (len(missing), len(ordered_ecs)), ''] - for ec in [x['ec'] for x in missing]: + for ec in (x['ec'] for x in missing): if ec.short_mod_name != ec.full_mod_name: modname = '%s | %s' % (ec.mod_subdir, ec.short_mod_name) else: diff --git a/test/framework/options.py b/test/framework/options.py index e6be49ab4f..a246311727 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1472,14 +1472,26 @@ def test_missing(self): ]) for opt in ['-M', '--missing-modules']: - self.mock_stderr(True) - self.mock_stdout(True) - self.eb_main(args + [opt], testing=False, raise_error=True) - stderr, stdout = self.get_stderr(), self.get_stdout() - self.mock_stderr(False) - self.mock_stdout(False) + with self.mocked_stdout_stderr(): + self.eb_main(args + [opt], testing=False, raise_error=True) + stderr, stdout = self.get_stderr(), self.get_stdout() self.assertFalse(stderr) self.assertIn(expected, stdout) + # --terse + with self.mocked_stdout_stderr(): + self.eb_main(args + ['-M', '--terse'], testing=False, raise_error=True) + stderr, stdout = self.get_stderr(), self.get_stdout() + self.assertFalse(stderr) + if mns == 'HierarchicalMNS': + expected = '\n'.join([ + "GCC-4.6.3.eb", + "intel-2018a.eb", + "toy-0.0-deps.eb", + "gzip-1.4-GCC-4.6.3.eb", + ]) + else: + expected = 'gzip-1.4-GCC-4.6.3.eb' + self.assertEqual(stdout, expected + '\n') def test_dry_run_short(self): """Test dry run (short format).""" From b1efd5ced3f46b2dad972ff3e0100d4962f454b5 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 19 Dec 2023 10:06:59 +0100 Subject: [PATCH 2/3] Don't print index status when using `--terse` --- easybuild/tools/config.py | 1 + easybuild/tools/filetools.py | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 6c0a173fe4..f025ce831c 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -306,6 +306,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'skip_test_cases', 'skip_test_step', 'sticky_bit', + 'terse', 'trace', 'unit_testing_mode', 'upload_test_report', diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 6e4bd2d83e..21d9d5e5e0 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -966,11 +966,12 @@ def load_index(path, ignore_dirs=None): # check whether index is still valid if valid_ts: curr_ts = datetime.datetime.now() + terse = build_option('terse') if curr_ts > valid_ts: - print_warning("Index for %s is no longer valid (too old), so ignoring it...", path) + print_warning("Index for %s is no longer valid (too old), so ignoring it...", path, silent=terse) index = None else: - print_msg("found valid index for %s, so using it...", path) + print_msg("found valid index for %s, so using it...", path, silent=terse) return index or None From 640142adcefc164ac91ef51f2072cf59da1ceda8 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 3 Apr 2024 21:37:01 +0200 Subject: [PATCH 3/3] add comment to clarify why there's no early stop when --terse is combined with --missing-modules --- easybuild/main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easybuild/main.py b/easybuild/main.py index 826880aad1..caaa4695e3 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -684,6 +684,8 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, pr options.list_prs, options.merge_pr, options.review_pr, + # --missing-modules is processed by process_eb_args, + # so we can't exit just yet here if it's used in combination with --terse options.terse and not options.missing_modules, search_query, ]