[GR-31840] [GR-31681] Allow native-image shared libs to be built on module-path and fix resource bundle lookup.

PullRequest: graal/9053

Author: olpaw

compiler/mx.compiler/suite.py

@@ -2169,13 +2169,14 @@
         ],
         "exports" : [
           "* to com.oracle.graal.graal_enterprise,org.graalvm.nativeimage.pointsto,org.graalvm.nativeimage.builder,org.graalvm.nativeimage.llvm,com.oracle.svm.svm_enterprise",
-          "org.graalvm.compiler.core.common            to jdk.internal.vm.compiler.management",
+          "org.graalvm.compiler.core.common            to jdk.internal.vm.compiler.management,org.graalvm.nativeimage.agent.tracing",
           "org.graalvm.compiler.debug                  to jdk.internal.vm.compiler.management,org.graalvm.nativeimage.objectfile",
           "org.graalvm.compiler.hotspot                to jdk.internal.vm.compiler.management",
           "org.graalvm.compiler.nodes.graphbuilderconf to org.graalvm.nativeimage.driver",
           "org.graalvm.compiler.options                to jdk.internal.vm.compiler.management,org.graalvm.nativeimage.driver,org.graalvm.nativeimage.librarysupport",
+          "org.graalvm.compiler.phases.common          to org.graalvm.nativeimage.agent.tracing,org.graalvm.nativeimage.configure",
           "org.graalvm.compiler.phases.common.jmx      to jdk.internal.vm.compiler.management",
-          "org.graalvm.compiler.serviceprovider        to jdk.internal.vm.compiler.management,org.graalvm.nativeimage.driver",
+          "org.graalvm.compiler.serviceprovider        to jdk.internal.vm.compiler.management,org.graalvm.nativeimage.driver,org.graalvm.nativeimage.agent.jvmtibase,org.graalvm.nativeimage.agent.diagnostics",
           "org.graalvm.compiler.truffle.jfr            to jdk.internal.vm.compiler.truffle.jfr",
           "org.graalvm.libgraal                        to jdk.internal.vm.compiler.management",
           "org.graalvm.util                            to jdk.internal.vm.compiler.management",

sdk/mx.sdk/mx_sdk_vm.py

@@ -133,6 +133,15 @@ def get_add_exports_list(required_exports, custom_target_module_str=None):
             add_exports.append('--add-exports=' + required_module_name + '/' + required_package_name + "=" + target_modules_str)
         return sorted(add_exports)
 
+    def get_add_exports(self, missing_jars):
+        if self.use_modules is None:
+            return ''
+        distributions = self.jar_distributions
+        distributions_transitive = mx.classpath_entries(distributions)
+        distributions_transitive_clean = [entry for entry in distributions_transitive if str(entry) not in missing_jars]
+        required_exports = mx_javamodules.requiredExports(distributions_transitive_clean, base_jdk())
+        return AbstractNativeImageConfig.get_add_exports_list(required_exports)
+
 
 class LauncherConfig(AbstractNativeImageConfig):
     def __init__(self, destination, jar_distributions, main_class, build_args, is_main_launcher=True,
@@ -148,10 +157,10 @@ def __init__(self, destination, jar_distributions, main_class, build_args, is_ma
         :param str custom_launcher_script: Custom launcher script, to be used when not compiled as a native image
         """
         super(LauncherConfig, self).__init__(destination, jar_distributions, build_args, use_modules, home_finder=home_finder, **kwargs)
+        self.main_module = main_module
+        assert self.use_modules is None or self.main_module
         self.main_class = main_class
         self.is_main_launcher = is_main_launcher
-        assert use_modules is None or main_module
-        self.main_module = main_module
         self.default_symlinks = default_symlinks
         self.is_sdk_launcher = is_sdk_launcher
         self.custom_launcher_script = custom_launcher_script
@@ -166,15 +175,6 @@ def add_relative_home_path(self, language, path):
                 language, self.relative_home_paths[language], path, self.destination))
         self.relative_home_paths[language] = path
 
-    def get_add_exports(self, missing_jars):
-        if self.use_modules is None:
-            return ''
-        distributions = self.jar_distributions
-        distributions_transitive = mx.classpath_entries(distributions)
-        distributions_transitive_clean = [entry for entry in distributions_transitive if str(entry) not in missing_jars]
-        required_exports = mx_javamodules.requiredExports(distributions_transitive_clean, base_jdk())
-        return ' '.join(AbstractNativeImageConfig.get_add_exports_list(required_exports))
-
 
 class LanguageLauncherConfig(LauncherConfig):
     def __init__(self, destination, jar_distributions, main_class, build_args, language,
@@ -191,11 +191,11 @@ def __init__(self, destination, jar_distributions, main_class, build_args, langu
 
 
 class LibraryConfig(AbstractNativeImageConfig):
-    def __init__(self, destination, jar_distributions, build_args, jvm_library=False, **kwargs):
+    def __init__(self, destination, jar_distributions, build_args, jvm_library=False, use_modules=None, **kwargs):
         """
         :param bool jvm_library
         """
-        super(LibraryConfig, self).__init__(destination, jar_distributions, build_args, **kwargs)
+        super(LibraryConfig, self).__init__(destination, jar_distributions, build_args, use_modules, **kwargs)
         self.jvm_library = jvm_library
 
 

sdk/mx.sdk/mx_sdk_vm_impl.py

@@ -704,6 +704,7 @@ def _find_escaping_links(root_dir):
                     _add(layout, _component_base, 'dependency:{}/polyglot.config'.format(launcher_project), _component)
             for _library_config in sorted(_get_library_configs(_component), key=lambda c: c.destination):
                 graalvm_dists.update(_library_config.jar_distributions)
+                self.jimage_ignore_jars.update(_library_config.jar_distributions)
                 if _library_config.jvm_library:
                     assert isinstance(_component, (mx_sdk.GraalVmJdkComponent, mx_sdk.GraalVmJreComponent))
                     _svm_library_home = _jvm_library_dest
@@ -1113,8 +1114,11 @@ def __init__(self, component, image_config, **kw_args):
         :type component: mx_sdk.GraalVmComponent | None
         :type image_config: mx_sdk.AbstractNativeImageConfig
         """
-        deps = []
         self.image_config = image_config
+        # With Java > 8 there are cases where image_config.get_add_exports is getting called in
+        # mx_sdk_vm_impl.NativePropertiesBuildTask.contents. This only works after the jar_distributions
+        # are made into proper modules. Therefore they have to be specified as dependencies here.
+        deps = [] if _src_jdk_version == 8 else list(image_config.jar_distributions)
         super(GraalVmNativeProperties, self).__init__(component, GraalVmNativeProperties.project_name(image_config), deps=deps, **kw_args)
 
     @staticmethod
@@ -1272,6 +1276,11 @@ def contents(self):
                     raise mx.abort("Profiles for an image must have unique filenames.\nThis is not the case for {}: {}.".format(canonical_name, profiles))
                 build_args += ['--pgo=' + ','.join(('${.}/' + n for n in basenames))]
 
+            build_with_module_path = image_config.use_modules == 'image'
+            if build_with_module_path:
+                export_deps_to_exclude = [str(dep) for dep in mx.classpath_entries(['substratevm:LIBRARY_SUPPORT'])] + list(_known_missing_jars)
+                build_args += image_config.get_add_exports(set(export_deps_to_exclude))
+
             requires = [arg[2:] for arg in build_args if arg.startswith('--language:') or arg.startswith('--tool:') or arg.startswith('--macro:')]
             build_args = [arg for arg in build_args if not (arg.startswith('--language:') or arg.startswith('--tool:') or arg.startswith('--macro:'))]
 
@@ -1291,7 +1300,6 @@ def _write_ln(s):
             _write_ln(u'ImagePath=' + java_properties_escape("${.}/" + relpath(dirname(graalvm_image_destination), graalvm_location).replace(os.sep, '/')))
             if requires:
                 _write_ln(u'Requires=' + java_properties_escape(' '.join(requires), ' ', len('Requires')))
-            build_with_module_path = image_config.use_modules == 'image'
             if isinstance(image_config, mx_sdk.LauncherConfig):
                 _write_ln(u'ImageClass=' + java_properties_escape(image_config.main_class))
                 if build_with_module_path:
@@ -1898,7 +1906,7 @@ def _get_launcher_args():
             return ''
 
         def _get_add_exports():
-            res = self.subject.native_image_config.get_add_exports(_known_missing_jars)
+            res = ' '.join(self.subject.native_image_config.get_add_exports(_known_missing_jars))
             if mx.is_windows():
                 res = ' '.join(('"{}"'.format(a) for a in res.split()))
             return res

substratevm/mx.substratevm/mx_substratevm.py

@@ -831,19 +831,21 @@ def _native_image_launcher_extra_jvm_args():
     launcher_configs=[
         mx_sdk_vm.LauncherConfig(
             use_modules='image' if USE_NI_JPMS else 'launcher' if not svm_java8() else None,
+            main_module="org.graalvm.nativeimage.driver",
             destination="bin/<exe:native-image>",
             jar_distributions=["substratevm:SVM_DRIVER"],
-            main_module="org.graalvm.nativeimage.driver",
             main_class=_native_image_launcher_main_class(),
             build_args=[],
             extra_jvm_args=_native_image_launcher_extra_jvm_args(),
         ),
     ],
     library_configs=[
         mx_sdk_vm.LibraryConfig(
+            use_modules='image' if USE_NI_JPMS else 'launcher' if not svm_java8() else None,
             destination="<lib:native-image-agent>",
             jvm_library=True,
             jar_distributions=[
+                'substratevm:SVM_CONFIGURE',
                 'substratevm:JVMTI_AGENT_BASE',
                 'substratevm:SVM_AGENT',
             ],
@@ -853,6 +855,7 @@ def _native_image_launcher_extra_jvm_args():
             ],
         ),
         mx_sdk_vm.LibraryConfig(
+            use_modules='image' if USE_NI_JPMS else 'launcher' if not svm_java8() else None,
             destination="<lib:native-image-diagnostics-agent>",
             jvm_library=True,
             jar_distributions=[
@@ -1006,6 +1009,8 @@ def _native_image_configure_extra_jvm_args():
     support_distributions=[],
     launcher_configs=[
         mx_sdk_vm.LauncherConfig(
+            use_modules='image' if USE_NI_JPMS else 'launcher' if not svm_java8() else None,
+            main_module="org.graalvm.nativeimage.configure",
             destination="bin/<exe:native-image-configure>",
             jar_distributions=["substratevm:SVM_CONFIGURE"],
             main_class="com.oracle.svm.configure.ConfigurationTool",
@@ -1086,17 +1091,28 @@ def hellomodule(args):
     proj_dir = join(suite.dir, 'src', 'native-image-module-tests', 'hello.app')
     mx.run_maven(['-e', 'install'], cwd=proj_dir)
     module_path.append(join(proj_dir, 'target', 'hello-app-1.0-SNAPSHOT.jar'))
-    config = GraalVMConfig.build(native_images=['native-image'])
+    config = GraalVMConfig.build(native_images=['native-image', 'lib:native-image-agent', 'lib:native-image-diagnostics-agent'])
     with native_image_context(hosted_assertions=False, config=config) as native_image:
+        module_path_sep = ';' if mx.is_windows() else ':'
+        moduletest_run_args = [
+            '--add-exports=moduletests.hello.lib/hello.privateLib=moduletests.hello.app',
+            '--add-opens=moduletests.hello.lib/hello.privateLib2=moduletests.hello.app',
+            '-p', module_path_sep.join(module_path), '-m', 'moduletests.hello.app'
+        ]
+        mx.log('Running module-tests on JVM:')
         build_dir = join(svmbuild_dir(), 'hellomodule')
+        mx.run([
+            vm_executable_path('java', config),
+            # also test if native-image-agent works
+            '-agentlib:native-image-agent=config-output-dir=' + join(build_dir, 'config-output-dir-{pid}-{datetime}/'),
+            ] + moduletest_run_args)
+
         # Build module into native image
         mx.log('Building image from java modules: ' + str(module_path))
-        module_path_sep = ';' if mx.is_windows() else ':'
         built_image = native_image([
-            '--verbose', '-ea', '-H:Path=' + build_dir,
-            '--add-exports=moduletests.hello.lib/hello.privateLib=moduletests.hello.app',
-            '--add-exports=moduletests.hello.lib/hello.privateLib2=moduletests.hello.app',
-            '-p', module_path_sep.join(module_path), '-m', 'moduletests.hello.app'])
+            '--verbose', '-H:Path=' + build_dir,
+            '--trace-class-initialization=hello.lib.Greeter', # also test native-image-diagnostics-agent
+            ] + moduletest_run_args)
         mx.log('Running image ' + built_image + ' built from module:')
         mx.run([built_image])
 

substratevm/mx.substratevm/suite.py

@@ -1,3 +1,4 @@
+# pylint: disable=line-too-long
 suite = {
     "mxversion": "5.301.0",
     "name": "substratevm",
@@ -1090,7 +1091,8 @@
                     "com.oracle.svm.hosted.agent                  to java.instrument",
                     "com.oracle.svm.core.graal.thread             to jdk.internal.vm.compiler",
                     "com.oracle.svm.core.classinitialization      to jdk.internal.vm.compiler",
-                    "* to org.graalvm.nativeimage.driver,org.graalvm.nativeimage.librarysupport,org.graalvm.nativeimage.llvm,com.oracle.svm.svm_enterprise",
+                    "com.oracle.svm.truffle.api                   to org.graalvm.truffle",
+                    "* to org.graalvm.nativeimage.driver,org.graalvm.nativeimage.configure,org.graalvm.nativeimage.librarysupport,org.graalvm.nativeimage.llvm,org.graalvm.nativeimage.agent.jvmtibase,org.graalvm.nativeimage.agent.tracing,org.graalvm.nativeimage.agent.diagnostics,com.oracle.svm.svm_enterprise",
                 ],
                 "opens" : [
                     "com.oracle.svm.core.nodes                    to jdk.internal.vm.compiler",
@@ -1109,6 +1111,8 @@
                 ],
                 "uses" : [
                     "org.graalvm.nativeimage.Platform",
+                    "com.oracle.truffle.api.TruffleLanguage.Provider",
+                    "com.oracle.truffle.api.instrumentation.TruffleInstrument.Provider",
                 ],
                 "requiresConcealed": {
                     "jdk.internal.vm.ci": [
@@ -1167,6 +1171,13 @@
                 "LIBRARY_SUPPORT",
                 "SVM_DRIVER",
             ],
+            "moduleInfo" : {
+                "name" : "org.graalvm.nativeimage.agent.jvmtibase",
+                "exports" : [
+                    "com.oracle.svm.jvmtiagentbase",
+                    "com.oracle.svm.jvmtiagentbase.jvmti",
+                ],
+            },
         },
 
         "LIBRARY_SUPPORT": {
@@ -1278,6 +1289,7 @@
               "name" : "org.graalvm.nativeimage.driver",
               "exports" : [
                 "com.oracle.svm.driver",
+                "com.oracle.svm.driver.metainf",
               ],
               "uses" : [
                 "org.graalvm.compiler.options.OptionDescriptors",
@@ -1304,10 +1316,19 @@
                 "JVMTI_AGENT_BASE",
                 "LIBRARY_SUPPORT",
                 "SVM_DRIVER",
+                "SVM_CONFIGURE"
             ],
-            "overlaps" : [
-                "SVM_CONFIGURE",
-            ],
+            "moduleInfo" : {
+                "name" : "org.graalvm.nativeimage.agent.tracing",
+                "exports" : [
+                    "com.oracle.svm.agent",
+                ],
+                "requiresConcealed" : {
+                    "jdk.internal.vm.ci" : [
+                        "jdk.vm.ci.meta",
+                    ],
+                }
+            },
             # vm: included as binary, tool descriptor intentionally not copied
         },
 
@@ -1321,6 +1342,12 @@
                 "JVMTI_AGENT_BASE",
                 "LIBRARY_SUPPORT",
             ],
+            "moduleInfo" : {
+                "name" : "org.graalvm.nativeimage.agent.diagnostics",
+                "exports" : [
+                    "com.oracle.svm.diagnosticsagent",
+                ],
+            },
         },
 
         "SVM_CONFIGURE": {
@@ -1333,6 +1360,13 @@
             "distDependencies": [
                 "LIBRARY_SUPPORT",
             ],
+            "moduleInfo" : {
+                "name" : "org.graalvm.nativeimage.configure",
+                "exports" : [
+                    "* to org.graalvm.nativeimage.agent.tracing",
+                    "com.oracle.svm.configure",
+                ],
+            },
         },
 
 

substratevm/src/com.oracle.objectfile.jdk11/src/com/oracle/objectfile/ModuleAccess.java

@@ -31,11 +31,18 @@ public class ModuleAccess {
     public static void openModuleByClass(Class<?> declaringClass, Class<?> accessingClass) {
         Module declaringModule = declaringClass.getModule();
         String packageName = declaringClass.getPackageName();
-        Module accessingModule = accessingClass == null ? null : accessingClass.getModule();
-        if (accessingModule != null && accessingModule.isNamed()) {
-            if (!declaringModule.isOpen(packageName, accessingModule)) {
-                Modules.addOpens(declaringModule, packageName, accessingModule);
+        Module namedAccessingModule = null;
+        if (accessingClass != null) {
+            Module accessingModule = accessingClass.getModule();
+            if (accessingModule.isNamed()) {
+                namedAccessingModule = accessingModule;
             }
+        }
+        if (namedAccessingModule != null ? declaringModule.isOpen(packageName, namedAccessingModule) : declaringModule.isOpen(packageName)) {
+            return;
+        }
+        if (namedAccessingModule != null) {
+            Modules.addOpens(declaringModule, packageName, namedAccessingModule);
         } else {
             Modules.addOpensToAllUnnamed(declaringModule, packageName);
         }

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/ClassLoaderQuery.java renamed to substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/ClassLoaderSupport.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,11 +24,28 @@
  */
 package com.oracle.svm.core;
 
+import java.util.List;
+import java.util.Locale;
+import java.util.ResourceBundle;
+
 import org.graalvm.nativeimage.Platform;
 import org.graalvm.nativeimage.Platforms;
 
 @Platforms(Platform.HOSTED_ONLY.class)
-public interface ClassLoaderQuery {
+public abstract class ClassLoaderSupport {
+
+    public boolean isNativeImageClassLoader(ClassLoader classLoader) {
+        ClassLoader loader = classLoader;
+        while (loader != null) {
+            if (isNativeImageClassLoaderImpl(loader)) {
+                return true;
+            }
+            loader = loader.getParent();
+        }
+        return false;
+    }
+
+    protected abstract boolean isNativeImageClassLoaderImpl(ClassLoader classLoader);
 
-    boolean isNativeImageClassLoader(ClassLoader c);
+    public abstract List<ResourceBundle> getResourceBundle(String bundleName, Locale locale);
 }

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/RuntimeAssertionsSupport.java

@@ -184,7 +184,7 @@ private boolean desiredAssertionStatusImpl(String name, boolean fallback) {
     }
 
     private boolean desiredAssertionStatusImpl(String name, ClassLoader classLoader) {
-        boolean isNativeImageClassLoader = ImageSingletons.lookup(ClassLoaderQuery.class).isNativeImageClassLoader(classLoader);
+        boolean isNativeImageClassLoader = ImageSingletons.lookup(ClassLoaderSupport.class).isNativeImageClassLoader(classLoader);
         return desiredAssertionStatusImpl(name, isNativeImageClassLoader ? defaultAssertionStatus : systemAssertionStatus);
     }