[GR-40090] Support bundling language homes with native image

PullRequest: graal/12471

Author: jchalou

common.json

@@ -1,7 +1,7 @@
 {
   "README": "This file contains definitions that are useful for the hocon and jsonnet CI files of multiple repositories.",
 
-  "mx_version" : "6.11.3",
+  "mx_version" : "6.11.4",
 
   "jdks": {
     "openjdk11":          {"name": "jpg-jdk",   "version": "11.0.11", "build_id": "9", "open": true, "release": true, "platformspecific": true },

compiler/mx.compiler/suite.py

@@ -1,5 +1,5 @@
 suite = {
-  "mxversion" : "6.11.2",
+  "mxversion" : "6.11.4",
   "name" : "compiler",
   "sourceinprojectwhitelist" : [],
 

docs/reference-manual/embedding/embed-languages.md

@@ -390,15 +390,26 @@ native-image --language:js -cp . HelloPolyglot
 
 Please note that some languages (e.g. Python, Ruby) need their language home directories to work without limitations.
 If the polyglot application runs on a JVM (e.g. [here](#compile-and-run-a-polyglot-application)), the language homes are discovered automatically.
-However, for native images, the language homes have to be specified. We are currently working on a feature that will enable the `native-image` builder to automatically bundle
-the necessary language homes to the same directory as the produced executable/library. Until that feature is available, we recommend specifying
-a GraalVM home at runtime using the option `-Dorg.graalvm.home=$GRAALVM_HOME`, assuming the environment variable `GRAALVM_HOME` is populated with an absolute path to a GraalVM home directory.
-Language homes are automatically discovered in the specified directory. For example:
+However, for native images, paths to language homes have to be stored in the image or specified at runtime.
+
+By default, the `native-image` builder copies the necessary language homes to the `resources` directory located in the same directory as the produced image.
+The paths to the copied homes are written to the image's build artifacts file and also stored in the image itself so that the homes are automatically discovered as long as their relative paths with respect to the image file stay the same.
+That means that the `resources` directory should be always distributed together with the image file.     
 
 ```shell
 native-image --language:python -cp . HelloPolyglot
+./hellopolyglot
+```
+
+In case an installed GraalVM is available, it is possible to use language homes from the GraalVM home directory. A GraalVM home can be specified at runtime using the option `-Dorg.graalvm.home=$GRAALVM_HOME`, assuming the environment variable `GRAALVM_HOME` is populated with an absolute path to the GraalVM home directory.
+Language homes are automatically discovered in the specified directory. For example:
+
+```shell
 ./hellopolyglot -Dorg.graalvm.home=$GRAALVM_HOME
 ```
+
+> Note: The `-Dorg.graalvm.home` option has precedence over any relative language home paths stored in the image.
+
 > Note: The version of GraalVM the home of which is specified at runtime must match the version of GraalVM used to build the native executable/library. 
 
 ### Excluding the JIT compiler 

java-benchmarks/mx.java-benchmarks/suite.py

@@ -1,5 +1,5 @@
 suite = {
-  "mxversion" : "6.11.2",
+  "mxversion" : "6.11.4",
   "name": "java-benchmarks",
 
   "ignore_suite_commit_info": True,

sdk/CHANGELOG.md

@@ -8,6 +8,7 @@ This changelog summarizes major changes between GraalVM SDK versions. The main f
 * (GR-25849) (GR-41634) Added a new way to configure the IO access using the new class `IOAccess`. The IO access configuration determines how a guest language can access the host IO. The `IOAccess` class provides a predefined configuration to [disable](https://www.graalvm.org/sdk/javadoc/org/graalvm/polyglot/io/IOAccess.html#NONE) host IO access, or to [enable](https://www.graalvm.org/sdk/javadoc/org/graalvm/polyglot/io/IOAccess.html#ALL) full host IO access. A custom configuration can be created using an IOAccess [builder](https://www.graalvm.org/sdk/javadoc/org/graalvm/polyglot/io/IOAccess.html#newBuilder--).
 * Deprecated `Context.Builder#allowIO(boolean)` To migrate, use `builder.allowIO(IOAccess.ALL)` to enable unrestricted IO operations on the host system, or `builder.allowIO(IOAccess.NONE)` to disable IO operations.
 * Deprecated `Context.Builder#fileSystem(FileSystem)`. To migrate, use `builder.allowIO(IOAccess.newBuilder().fileSystem(fileSystem).build())`.
+* Added automatic copying of language resources for embedding Truffle languages in native image. Documentation available [here](https://www.graalvm.org/reference-manual/embed-languages/#build-native-executables-from-polyglot-applications).
 
 ## Version 22.3.0
 * (GR-39852) Native Image API: Added FieldValueTransformer API

sdk/mx.sdk/mx_sdk_vm.py

@@ -399,6 +399,11 @@ def __init__(self, suite, name, short_name, license_files, third_party_license_f
 
 
 class GraalVmLanguage(GraalVmTruffleComponent):
+    """
+    :param support_distributions: distributions the contents of which is added to the language's home directory.
+    The contents of support distributions setting the `fileListPurpose` attribute to `native-image-resources` will end up as file list in the `native-image-resources.filelist` file in this language's home directory.
+    As a part of a native image build that includes this language, the files in the merged file list will be copied as resources to a directory named `resources` next to the produced image.
+    """
     pass
 
 

sdk/mx.sdk/mx_sdk_vm_impl.py

@@ -353,6 +353,7 @@ def __init__(self, suite, name, deps, components, is_graalvm, exclLibs, platform
                  base_dir=None,
                  path=None,
                  stage1=False,
+                 include_native_image_resources_filelists=False,
                  **kw_args): # pylint: disable=super-init-not-called
         self.components = components or registered_graalvm_components(stage1)
         self.stage1 = stage1
@@ -555,6 +556,8 @@ def _add_link(_dest, _target, _component=None, _dest_base_name=None):
         graalvm_dists = set()  # the jar distributions mentioned by launchers and libraries
         component_dists = set()  # the jar distributions directly mentioned by components
 
+        _lang_homes_with_ni_resources = []
+
         for _component in sorted(self.components, key=lambda c: c.name):
             mx.logv('Adding {} ({}) to the {} {}'.format(_component.name, _component.__class__.__name__, name, self.__class__.__name__))
             _component_type_base = _get_component_type_base(_component)
@@ -584,6 +587,16 @@ def _add_link(_dest, _target, _component=None, _dest_base_name=None):
                 'exclude': _component.license_files if _no_licenses() else [],
                 'path': None,
             } for d in _component.support_distributions], _component)
+            if include_native_image_resources_filelists and isinstance(_component, mx_sdk.GraalVmLanguage) and _component.support_distributions:
+                # A support distribution of a GraalVmLanguage component might have the `fileListPurpose` attribute with value 'native-image-resources' specifying that all the files from the distribution should
+                # be used as native image resources. Any value of the attribute other than 'native-image-resources' or None for a support distribution of a GraalVmLanguage is invalid. If the attribute is specified,
+                # there is a '<distribution archive file path>.filename' file containing a file list of all the files from the distribution. The support distributions specifying the attribute together specify
+                # a subset of files from this component's home directory. The file lists will be merged by the NativeImageResourcesFileList project into a single file `native-image-resources.filelist` that will be
+                # written into this component's home directory. As a part of a native image build that includes this component, the files in the merged file list will be copied as resources to a directory named
+                # `resources` next to the produced image. This impacts only the native images built by GraalVM that are not a part of the GraalVM itself.
+                if not _component_base in _lang_homes_with_ni_resources:
+                    _add(layout, _component_base, 'dependency:{}/native-image-resources.filelist'.format(NativeImageResourcesFileList.project_name(_component.dir_name)), _component)
+                    _lang_homes_with_ni_resources.append(_component_base)
             _add(layout, '<jdk_base>/include/', [{
                 'source_type': 'extracted-dependency',
                 'dependency': d,
@@ -864,6 +877,7 @@ def __init__(self, base_name, theLicense=None, stage1=False, components=None, **
             base_dir=base_dir,
             path=None,
             stage1=stage1,
+            include_native_image_resources_filelists=not stage1,
             **kw_args)
 
     @staticmethod
@@ -1873,6 +1887,74 @@ def clean(self, forBuild=False):
     def __str__(self):
         return 'Building {}'.format(self.subject.name)
 
+class NativeImageResourcesFileList(_with_metaclass(ABCMeta, GraalVmProject)):
+    def __init__(self, component, components, language_dir, deps, **kw_args):
+        super(NativeImageResourcesFileList, self).__init__(component, NativeImageResourcesFileList.project_name(language_dir), deps, **kw_args)
+        self.language_dir = language_dir
+        self.components = components
+
+    def native_image_resources_filelist_file(self):
+        return join(self.get_output_base(), self.name, "native-image-resources.filelist")
+
+    def getArchivableResults(self, use_relpath=True, single=False):
+        out = self.native_image_resources_filelist_file()
+        yield out, basename(out)
+
+    def get_containing_graalvm(self):
+        return get_final_graalvm_distribution()
+
+    def getBuildTask(self, args):
+        return NativeImageResourcesFileListBuildTask(self, args)
+
+    @staticmethod
+    def project_name(language_dir):
+        return "org.graalvm.langugage." + language_dir + ".ni_resources_filelist"
+
+
+class NativeImageResourcesFileListBuildTask(_with_metaclass(ABCMeta, mx.ProjectBuildTask)):
+    def __init__(self, project, args):
+        super(NativeImageResourcesFileListBuildTask, self).__init__(args, 1, project)
+        self._native_image_resources_filelist_contents = None
+
+    def needsBuild(self, newestInput):
+        reason = _file_needs_build(newestInput, self.subject.native_image_resources_filelist_file(), self.native_image_resources_filelist_contents)
+        if reason:
+            return True, reason
+        return False, None
+
+    def native_image_resources_filelist_contents(self):
+        if self._native_image_resources_filelist_contents is None:
+            contents = []
+            for c in self.subject.components:
+                for dep in c.support_distributions:
+                    d = mx.dependency(dep)
+                    if d.fileListPurpose:
+                        if d.fileListPurpose != 'native-image-resources':
+                            mx.abort("Since distribution {} is a GraalVmLanguage support distribution, the only allowed value of its fileListPurpose attribute is 'native-image-resources', but was {}.".format(d.name, d.fileListPurpose))
+                        (filelist_file,) = (p for p, n in d.getArchivableResults(single=False) if n.endswith(".filelist"))
+                        if not exists(filelist_file):
+                            mx.abort("Distribution {} specifies a fileListPurpose {}, but file {} was not found.".format(d.name, d.fileListPurpose, filelist_file))
+                        with open(filelist_file, "r") as fp:
+                            for line in fp:
+                                contents.append(line.strip())
+
+            self._native_image_resources_filelist_contents = os.linesep.join(contents)
+        return self._native_image_resources_filelist_contents
+
+    def newestOutput(self):
+        paths = [self.subject.native_image_resources_filelist_file()]
+        return mx.TimeStampFile.newest(paths)
+
+    def build(self):
+        with mx.SafeFileCreation(self.subject.native_image_resources_filelist_file()) as sfc, io.open(sfc.tmpFd, mode='w', closefd=False, encoding='utf-8') as f:
+            f.write(self.native_image_resources_filelist_contents())
+
+    def clean(self, forBuild=False):
+        if exists(self.subject.native_image_resources_filelist_file()):
+            os.unlink(self.subject.native_image_resources_filelist_file())
+
+    def __str__(self):
+        return 'Building {}'.format(self.subject.name)
 
 class GraalVmNativeImageBuildTask(_with_metaclass(ABCMeta, mx.ProjectBuildTask)):
     def __init__(self, args, parallelism, project):
@@ -2860,6 +2942,7 @@ def mx_register_dynamic_suite_constituents(register_project, register_distributi
     installables = {}
     jvmci_parent_jars = []
     modified_jmods = {}
+    dir_name_to_ni_resources_components = {}
 
     for component in registered_graalvm_components(stage1=False):
         if component.name in names:
@@ -2895,11 +2978,26 @@ def mx_register_dynamic_suite_constituents(register_project, register_distributi
                     register_project(launcher_project)
                     polyglot_config_project = PolyglotConfig(component, library_config)
                     register_project(polyglot_config_project)
+            if isinstance(component, mx_sdk.GraalVmLanguage) and component.support_distributions:
+                ni_resources_components = dir_name_to_ni_resources_components.get(component.dir_name)
+                if not ni_resources_components:
+                    ni_resources_components = []
+                    dir_name_to_ni_resources_components[component.dir_name] = ni_resources_components
+                ni_resources_components.append(component)
+
         if component.installable and not _disable_installable(component):
             installables.setdefault(component.installable_id, []).append(component)
         if libpolyglot_component is not None and GraalVmLibPolyglotNativeProperties.needs_lib_polyglot_native_properties(component):
             register_distribution(GraalVmLibPolyglotNativeProperties(component))
 
+    for dir_name in dir_name_to_ni_resources_components:
+        ni_resources_components = dir_name_to_ni_resources_components.get(dir_name)
+        deps = []
+        for component in ni_resources_components:
+            deps.extend(component.support_distributions)
+        native_image_resources_filelist_project = NativeImageResourcesFileList(None, ni_resources_components, dir_name, deps)
+        register_project(native_image_resources_filelist_project)
+
     # Create installables
     for components in installables.values():
         main_component = _get_main_component(components)

sdk/mx.sdk/suite.py

@@ -39,7 +39,7 @@
 # SOFTWARE.
 #
 suite = {
-  "mxversion" : "6.11.2",
+  "mxversion" : "6.11.4",
   "name" : "sdk",
   "version" : "23.0.0",
   "release" : False,

sdk/src/org.graalvm.home/src/org/graalvm/home/impl/DefaultHomeFinder.java

@@ -65,9 +65,9 @@ public final class DefaultHomeFinder extends HomeFinder {
 
     private static final boolean STATIC_VERBOSE = Boolean.getBoolean("com.oracle.graalvm.locator.verbose");
 
-    private static final Path FORCE_GRAAL_HOME;
-    private static final Path GRAAL_HOME_RELATIVE_PATH;
-    private static final Map<String, Path> LANGUAGE_RELATIVE_HOMES = new HashMap<>();
+    static final Path FORCE_GRAAL_HOME;
+    static final Path GRAAL_HOME_RELATIVE_PATH;
+    static final Map<String, Path> LANGUAGE_RELATIVE_HOMES = new HashMap<>();
 
     static {
         final String forcedHome = System.getProperty("org.graalvm.launcher.home");
@@ -275,6 +275,21 @@ public Map<String, Path> getToolHomes() {
         return res;
     }
 
+    /*
+     * Sets a relative path to a language home if it wasn't already set by system properties that
+     * are read in the static initializer of this class.
+     *
+     * NOTE: this method is called reflectively by TruffleBaseFeature
+     */
+    @SuppressWarnings("unused")
+    private static boolean setRelativeLanguageHomeIfNotAlreadySet(String languageId, Path homePath) {
+        if (FORCE_GRAAL_HOME == null && GRAAL_HOME_RELATIVE_PATH == null && LANGUAGE_RELATIVE_HOMES.get(languageId) == null) {
+            LANGUAGE_RELATIVE_HOMES.put(languageId, homePath);
+            return true;
+        }
+        return false;
+    }
+
     private static Map<String, Path> collectHomes(Path folder) {
         Map<String, Path> res = new HashMap<>();
         if (Files.exists(folder)) {

substratevm/mx.substratevm/suite.py

@@ -1,6 +1,6 @@
 # pylint: disable=line-too-long
 suite = {
-    "mxversion" : "6.11.2",
+    "mxversion" : "6.11.4",
     "name": "substratevm",
     "version" : "23.0.0",
     "release" : False,