Merge pull request #4868 from lexming/fake-module-per-extension

Update fake module for each extension installed

Author: boegel

easybuild/framework/easyblock.py

@@ -56,6 +56,7 @@
 import time
 import traceback
 from concurrent.futures import ThreadPoolExecutor
+from contextlib import contextmanager
 from datetime import datetime
 from string import ascii_letters
 from textwrap import indent
@@ -1268,9 +1269,6 @@ def make_devel_module(self, create_in_builddir=False):
 
         self.log.info("Making devel module...")
 
-        # load fake module
-        fake_mod_data = self.load_fake_module(purge=True)
-
         header = self.module_generator.MODULE_SHEBANG
         if header:
             header += '\n'
@@ -1314,9 +1312,6 @@ def make_devel_module(self, create_in_builddir=False):
         txt = ''.join([header] + load_lines + env_lines)
         write_file(filename, txt)
 
-        # cleanup: unload fake module, remove fake module dir
-        self.clean_up_fake_module(fake_mod_data)
-
     def make_module_deppaths(self):
         """
         Add specific 'module use' actions to module file, in order to find
@@ -1646,7 +1641,7 @@ def make_module_group_check(self):
 
         return txt
 
-    def make_module_req(self, fake=False):
+    def make_module_req(self):
         """
         Generate the environment-variables required to run the module.
         """
@@ -1687,11 +1682,10 @@ def make_module_req(self, fake=False):
             mod_lines.append(self.module_generator.comment(note))
 
         for env_var, search_paths in env_var_requirements.items():
-            if self.dry_run or fake:
+            if self.dry_run:
                 # Don't expand globs or do any filtering for dry run
                 mod_req_paths = search_paths
-                if self.dry_run:
-                    self.dry_run_msg(f" ${env_var}:{', '.join(mod_req_paths)}")
+                self.dry_run_msg(f" ${env_var}:{', '.join(mod_req_paths)}")
             else:
                 mod_req_paths = [
                     expanded_path for unexpanded_path in search_paths
@@ -1888,7 +1882,6 @@ def load_fake_module(self, purge=False, extra_modules=None, verbose=False):
         # load fake module
         self.modules_tool.prepend_module_path(os.path.join(fake_mod_path, self.mod_subdir), priority=10000)
         self.load_module(purge=purge, extra_modules=extra_modules, verbose=verbose)
-
         return (fake_mod_path, env)
 
     def clean_up_fake_module(self, fake_mod_data):
@@ -1972,10 +1965,11 @@ def skip_extensions(self):
         if not exts_filter or len(exts_filter) == 0:
             raise EasyBuildError("Skipping of extensions, but no exts_filter set in easyconfig")
 
-        if build_option('parallel_extensions_install'):
-            self.skip_extensions_parallel(exts_filter)
-        else:
-            self.skip_extensions_sequential(exts_filter)
+        with self.fake_module_environment():
+            if build_option('parallel_extensions_install'):
+                self.skip_extensions_parallel(exts_filter)
+            else:
+                self.skip_extensions_sequential(exts_filter)
 
     def skip_extensions_sequential(self, exts_filter):
         """
@@ -2103,31 +2097,27 @@ def install_extensions_sequential(self, install=True):
                 msg = "\n* installing extension %s %s using '%s' easyblock\n" % tup
                 self.dry_run_msg(msg)
 
-            self.log.debug("List of loaded modules: %s", self.modules_tool.list())
-
-            # prepare toolchain build environment, but only when not doing a dry run
-            # since in that case the build environment is the same as for the parent
             if self.dry_run:
                 self.dry_run_msg("defining build environment based on toolchain (options) and dependencies...")
-            else:
-                # don't reload modules for toolchain, there is no need since they will be loaded already;
-                # the (fake) module for the parent software gets loaded before installing extensions
-                ext.toolchain.prepare(onlymod=self.cfg['onlytcmod'], silent=True, loadmod=False,
-                                      rpath_filter_dirs=self.rpath_filter_dirs,
-                                      rpath_include_dirs=self.rpath_include_dirs,
-                                      rpath_wrappers_dir=self.rpath_wrappers_dir)
 
             # actual installation of the extension
-            if install:
-                try:
-                    ext.install_extension_substep("pre_install_extension")
-                    with self.module_generator.start_module_creation():
-                        txt = ext.install_extension_substep("install_extension")
-                    if txt:
-                        self.module_extra_extensions += txt
-                    ext.install_extension_substep("post_install_extension")
-                finally:
-                    if not self.dry_run:
+            if install and not self.dry_run:
+                with self.fake_module_environment(with_build_deps=True):
+                    self.log.debug("List of loaded modules: %s", self.modules_tool.list())
+                    # don't reload modules for toolchain, there is no need
+                    # since they will be loaded already by the fake module
+                    ext.toolchain.prepare(onlymod=self.cfg['onlytcmod'], silent=True, loadmod=False,
+                                          rpath_filter_dirs=self.rpath_filter_dirs,
+                                          rpath_include_dirs=self.rpath_include_dirs,
+                                          rpath_wrappers_dir=self.rpath_wrappers_dir)
+                    try:
+                        ext.install_extension_substep("pre_install_extension")
+                        with self.module_generator.start_module_creation():
+                            txt = ext.install_extension_substep("install_extension")
+                        if txt:
+                            self.module_extra_extensions += txt
+                        ext.install_extension_substep("post_install_extension")
+                    finally:
                         ext_duration = datetime.now() - start_time
                         if ext_duration.total_seconds() >= 1:
                             print_msg("\t... (took %s)", time2str(ext_duration), log=self.log, silent=self.silent)
@@ -2274,17 +2264,18 @@ def update_exts_progress_bar_helper(running_exts, progress_size):
                     tup = (ext.name, ext.version or '')
                     print_msg("starting installation of extension %s %s..." % tup, silent=self.silent, log=self.log)
 
-                    # don't reload modules for toolchain, there is no need since they will be loaded already;
-                    # the (fake) module for the parent software gets loaded before installing extensions
-                    ext.toolchain.prepare(onlymod=self.cfg['onlytcmod'], silent=True, loadmod=False,
-                                          rpath_filter_dirs=self.rpath_filter_dirs,
-                                          rpath_include_dirs=self.rpath_include_dirs,
-                                          rpath_wrappers_dir=self.rpath_wrappers_dir)
-                    if install:
-                        ext.install_extension_substep("pre_install_extension")
-                        ext.async_cmd_task = ext.install_extension_substep("install_extension_async", thread_pool)
-                        running_exts.append(ext)
-                        self.log.info(f"Started installation of extension {ext.name} in the background...")
+                    if install and not self.dry_run:
+                        with self.fake_module_environment(with_build_deps=True):
+                            # don't reload modules for toolchain, there is no
+                            # need since they will be loaded by the fake module
+                            ext.toolchain.prepare(onlymod=self.cfg['onlytcmod'], silent=True, loadmod=False,
+                                                  rpath_filter_dirs=self.rpath_filter_dirs,
+                                                  rpath_include_dirs=self.rpath_include_dirs,
+                                                  rpath_wrappers_dir=self.rpath_wrappers_dir)
+                            ext.install_extension_substep("pre_install_extension")
+                            ext.async_cmd_task = ext.install_extension_substep("install_extension_async", thread_pool)
+                            running_exts.append(ext)
+                            self.log.info(f"Started installation of extension {ext.name} in the background...")
                         update_exts_progress_bar_helper(running_exts, 0)
 
             # print progress info after every iteration (unless that info is already shown via progress bar)
@@ -2308,6 +2299,25 @@ def start_dir(self):
         """Start directory in build directory"""
         return self.cfg['start_dir']
 
+    @contextmanager
+    def fake_module_environment(self, extra_modules=None, with_build_deps=False):
+        """
+        Load/Unload fake module
+        """
+        fake_mod_data = None
+
+        if with_build_deps:
+            # load modules for build dependencies as extra modules
+            extra_modules = [dep['short_mod_name'] for dep in self.cfg.dependencies(build_only=True)]
+
+        fake_mod_data = self.load_fake_module(purge=True, extra_modules=extra_modules)
+
+        yield
+
+        # cleanup (unload fake module, remove fake module dir)
+        if fake_mod_data:
+            self.clean_up_fake_module(fake_mod_data)
+
     def guess_start_dir(self):
         """
         Return the directory where to start the whole configure/make/make install cycle from
@@ -3130,14 +3140,9 @@ def extensions_step(self, fetch=False, install=True):
             self.log.debug("No extensions in exts_list")
             return
 
-        # load fake module
-        fake_mod_data = None
-        if install and not self.dry_run:
-
-            # load modules for build dependencies as extra modules
-            build_dep_mods = [dep['short_mod_name'] for dep in self.cfg.dependencies(build_only=True)]
-
-            fake_mod_data = self.load_fake_module(purge=True, extra_modules=build_dep_mods)
+        # we really need a default class
+        if not self.cfg['exts_defaultclass'] and install:
+            raise EasyBuildError("ERROR: No default extension class set for %s", self.name)
 
         start_progress_bar(PROGRESS_BAR_EXTENSIONS, len(self.cfg.get_ref('exts_list')))
 
@@ -3153,22 +3158,13 @@ def extensions_step(self, fetch=False, install=True):
         if install:
             self.log.info("Installing extensions")
 
-        # we really need a default class
-        if not self.cfg['exts_defaultclass'] and fake_mod_data:
-            self.clean_up_fake_module(fake_mod_data)
-            raise EasyBuildError("ERROR: No default extension class set for %s", self.name)
-
         self.init_ext_instances()
 
         if self.skip:
             self.skip_extensions()
 
         self.install_all_extensions(install=install)
 
-        # cleanup (unload fake module, remove fake module dir)
-        if fake_mod_data:
-            self.clean_up_fake_module(fake_mod_data)
-
         stop_progress_bar(PROGRESS_BAR_EXTENSIONS, visible=False)
 
     def package_step(self):
@@ -4078,7 +4074,7 @@ def make_module_step(self, fake=False):
             txt += self.make_module_deppaths()
             txt += self.make_module_dep()
             txt += self.make_module_extend_modpath()
-            txt += self.make_module_req(fake=fake)
+            txt += self.make_module_req()
             txt += self.make_module_extra()
             txt += self.make_module_footer()
 
@@ -4122,13 +4118,14 @@ def make_module_step(self, fake=False):
             self.module_generator.create_symlinks(mod_symlink_paths, fake=fake)
 
             if ActiveMNS().mns.det_make_devel_module() and not fake and build_option('generate_devel_module'):
-                try:
-                    self.make_devel_module()
-                except EasyBuildError as error:
-                    if build_option('module_only') or self.cfg['module_only']:
-                        self.log.info("Using --module-only so can recover from error: %s", error)
-                    else:
-                        raise error
+                with self.fake_module_environment():
+                    try:
+                        self.make_devel_module()
+                    except EasyBuildError as error:
+                        if build_option('module_only') or self.cfg['module_only']:
+                            self.log.info("Using --module-only so can recover from error: %s", error)
+                        else:
+                            raise error
             else:
                 self.log.info("Skipping devel module...")
 

easybuild/framework/extensioneasyblock.py

@@ -176,29 +176,22 @@ def sanity_check_step(self, exts_filter=None, custom_paths=None, custom_commands
             # make sure Extension sanity check step is run once, by using a single empty list of extra modules
             lists_of_extra_modules = [[]]
 
-        for extra_modules in lists_of_extra_modules:
-
-            fake_mod_data = None
-
-            # only load fake module + extra modules for stand-alone installations (not for extensions),
-            # since for extension the necessary modules should already be loaded at this point;
-            # take into account that module may already be loaded earlier in sanity check
-            if not (self.sanity_check_module_loaded or self.is_extension or self.dry_run):
-                # load fake module
-                fake_mod_data = self.load_fake_module(purge=True, extra_modules=extra_modules)
-
-                if extra_modules:
-                    info_msg = "Running extension sanity check with extra modules: %s" % ', '.join(extra_modules)
-                    self.log.info(info_msg)
-                    trace_msg(info_msg)
-
-            # perform extension sanity check
+        # only load fake module + extra modules for stand-alone installations (not for extensions),
+        # since for extension the necessary modules should already be loaded at this point;
+        # take into account that module may already be loaded earlier in sanity check
+        if not (self.sanity_check_module_loaded or self.is_extension or self.dry_run):
+            for extra_modules in lists_of_extra_modules:
+                with self.fake_module_environment(extra_modules=extra_modules):
+                    if extra_modules:
+                        info_msg = f"Running extension sanity check with extra modules: {', '.join(extra_modules)}"
+                        self.log.info(info_msg)
+                        trace_msg(info_msg)
+                    # perform sanity check for stand-alone extension
+                    (sanity_check_ok, fail_msg) = Extension.sanity_check_step(self)
+        else:
+            # perform single sanity check for extension
             (sanity_check_ok, fail_msg) = Extension.sanity_check_step(self)
 
-            if fake_mod_data:
-                # unload fake module and clean up
-                self.clean_up_fake_module(fake_mod_data)
-
         if custom_paths or custom_commands or not self.is_extension:
             super().sanity_check_step(custom_paths=custom_paths,
                                       custom_commands=custom_commands,

test/framework/easyblock.py

@@ -1324,7 +1324,6 @@ def test_extensions_step_deprecations(self):
             'description = "test easyconfig"',
             'toolchain = SYSTEM',
             'exts_defaultclass = "DummyExtension"',
-            'exts_list = ["ext1"]',
             'exts_list = [',
             '    "dummy_ext",',
             '    ("custom_ext", "0.0", {"easyblock": "CustomDummyExtension"}),',
@@ -1416,7 +1415,7 @@ def test_extension_source_tmpl(self):
             "toolchain = SYSTEM",
             "exts_list = [",
             "    ('bar', '0.0', {",
-            "         'source_tmpl': [SOURCE_TAR_GZ],",
+            "        'source_tmpl': [SOURCE_TAR_GZ],",
             "    }),",
             "]",
         ])
@@ -1490,6 +1489,46 @@ def test_skip_extensions_step(self):
         eb.close_log()
         os.remove(eb.logfile)
 
+    def test_extension_fake_modules(self):
+        """
+        Test that extensions relying on installation files from previous extensions work
+        Search paths of fake module should update for each extension and resolve any globs
+        """
+        self.contents = cleandoc("""
+            easyblock = 'ConfigureMake'
+            name = 'toy'
+            version = '0.0'
+            homepage = 'https://example.com'
+            description = 'test'
+            toolchain = SYSTEM
+            exts_list = [
+                ('bar', '0.0', {
+                   'postinstallcmds': [
+                       'mkdir -p %(installdir)s/custom_bin',
+                       'touch %(installdir)s/custom_bin/bar.sh',
+                       'chmod +x %(installdir)s/custom_bin/bar.sh',
+                    ],
+                }),
+                ('barbar', '0.0', {
+                   'postinstallcmds': ['bar.sh'],
+                }),
+            ]
+            exts_defaultclass = "DummyExtension"
+            modextrapaths = {'PATH': 'custom*'}
+        """)
+        self.writeEC()
+        eb = EasyBlock(EasyConfig(self.eb_file))
+        eb.builddir = config.build_path()
+        eb.installdir = config.install_path()
+
+        self.mock_stdout(True)
+        eb.extensions_step(fetch=True)
+        stdout = self.get_stdout()
+        self.mock_stdout(False)
+
+        pattern = r">> running shell command:\n\s+bar.sh(\n\s+\[.*\]){3}\n\s+>> command completed: exit 0"
+        self.assertTrue(re.search(pattern, stdout, re.M))
+
     def test_make_module_step(self):
         """Test the make_module_step"""
 
@@ -2489,6 +2528,7 @@ def test_extensions_sanity_check(self):
             exts_list = toy_ec['exts_list']
             exts_list[-1][2]['exts_filter'] = ("thisshouldfail", '')
             toy_ec['exts_list'] = exts_list
+            toy_ec['exts_defaultclass'] = 'DummyExtension'
 
         eb = EB_toy(toy_ec)
         eb.silent = True
@@ -2503,6 +2543,7 @@ def test_extensions_sanity_check(self):
         # sanity check commands are checked after checking sanity check paths, so this should work
         toy_ec = EasyConfig(toy_ec_fn)
         toy_ec.update('sanity_check_commands', [("%(installdir)s/bin/toy && rm %(installdir)s/bin/toy", '')])
+        toy_ec['exts_defaultclass'] = 'DummyExtension'
         eb = EB_toy(toy_ec)
         eb.silent = True
         with self.mocked_stdout_stderr():

test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-2018a-test.eb

@@ -27,6 +27,7 @@ local_bar_buildopts = " && gcc bar.c -o anotherbar && "
 # used to check whether $TOY_LIBS_PATH is defined even when 'lib' subdirectory doesn't exist yet
 local_bar_buildopts += 'echo "TOY_EXAMPLES=$TOY_EXAMPLES" > %(installdir)s/toy_libs_path.txt'
 
+exts_defaultclass = 'DummyExtension'
 exts_list = [
     'ulimit',  # extension that is part of "standard library"
     ('bar', '0.0', {
@@ -59,7 +60,7 @@ sanity_check_paths = {
     'dirs': [],
 }
 
-modextrapaths = {'TOY_EXAMPLES': 'examples'}
+modextravars = {'TOY_EXAMPLES': 'examples'}
 
 postinstallcmds = ["echo TOY > %(installdir)s/README"]