diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index fd6ceead2c57..b5dce652c33e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -84,7 +84,6 @@ jobs: JDK: "labsjdk-ce-11" GATE: "hellomodule" PRIMARY: "substratevm" - USE_NATIVE_IMAGE_JAVA_PLATFORM_MODULE_SYSTEM: true - env: JDK: "labsjdk-ce-11" GATE: "style,fullbuild" diff --git a/compiler/src/org.graalvm.compiler.options.jdk11/src/org/graalvm/compiler/options/ModuleSupport.java b/compiler/src/org.graalvm.compiler.options.jdk11/src/org/graalvm/compiler/options/ModuleSupport.java index 5e952e899372..04a67c8a9616 100644 --- a/compiler/src/org.graalvm.compiler.options.jdk11/src/org/graalvm/compiler/options/ModuleSupport.java +++ b/compiler/src/org.graalvm.compiler.options.jdk11/src/org/graalvm/compiler/options/ModuleSupport.java @@ -28,7 +28,7 @@ public class ModuleSupport { - public static final boolean USE_NI_JPMS = System.getenv().getOrDefault("USE_NATIVE_IMAGE_JAVA_PLATFORM_MODULE_SYSTEM", "false").toLowerCase().equals("true"); + public static final boolean USE_NI_JPMS = Boolean.parseBoolean(System.getenv().get("USE_NATIVE_IMAGE_JAVA_PLATFORM_MODULE_SYSTEM")); static Iterable getOptionsLoader() { /* diff --git a/sdk/mx.sdk/mx_sdk_vm_impl.py b/sdk/mx.sdk/mx_sdk_vm_impl.py index 79bcabb6ba4d..0acb219f353e 100644 --- a/sdk/mx.sdk/mx_sdk_vm_impl.py +++ b/sdk/mx.sdk/mx_sdk_vm_impl.py @@ -1183,6 +1183,9 @@ def getBuildTask(self, args): class NativePropertiesBuildTask(mx.ProjectBuildTask): + + implicit_excludes = ['substratevm:LIBRARY_SUPPORT'] + def __init__(self, subject, args): """ :type subject: GraalVmNativeProperties @@ -1204,12 +1207,14 @@ def _get_location_classpath(self): graalvm_dist = get_final_graalvm_distribution() image_config = self.subject.image_config graalvm_location = dirname(graalvm_dist.find_single_source_location('dependency:' + self.subject.name)) - self._location_classpath = NativePropertiesBuildTask.get_launcher_classpath(graalvm_dist, graalvm_location, image_config, self.subject.component) + self._location_classpath = NativePropertiesBuildTask.get_launcher_classpath(graalvm_dist, graalvm_location, image_config, self.subject.component, exclude_implicit=True) return self._location_classpath @staticmethod - def get_launcher_classpath(graalvm_dist, start, image_config, component): - location_cp = graalvm_home_relative_classpath(image_config.jar_distributions, start, graal_vm=graalvm_dist) + def get_launcher_classpath(graalvm_dist, start, image_config, component, exclude_implicit=False): + with_substratevm = 'substratevm' in [s.name for s in mx.suites()] + exclude_names = NativePropertiesBuildTask.implicit_excludes if with_substratevm and exclude_implicit else None + location_cp = graalvm_home_relative_classpath(image_config.jar_distributions, start, graal_vm=graalvm_dist, exclude_names=exclude_names) location_classpath = location_cp.split(os.pathsep) if location_cp else [] if image_config.dir_jars: if not component: @@ -1268,7 +1273,7 @@ def contents(self): if isinstance(image_config, mx_sdk.LauncherConfig): if image_config.is_sdk_launcher: - launcher_classpath = NativePropertiesBuildTask.get_launcher_classpath(graalvm_dist, graalvm_home, image_config, self.subject.component) + launcher_classpath = NativePropertiesBuildTask.get_launcher_classpath(graalvm_dist, graalvm_home, image_config, self.subject.component, exclude_implicit=True) build_args += [ '-H:-ParseRuntimeOptions', '-Dorg.graalvm.launcher.classpath=' + os.pathsep.join(launcher_classpath), @@ -1825,7 +1830,7 @@ def polyglot_config_contents(self): assert self.with_polyglot_config() graalvm_dist = self.subject.get_containing_graalvm() graalvm_location = dirname(graalvm_dist.find_single_source_location('dependency:{}/polyglot.config'.format(self.subject.name))) - classpath = NativePropertiesBuildTask.get_launcher_classpath(graalvm_dist, graalvm_location, image_config, self.subject.component) + classpath = NativePropertiesBuildTask.get_launcher_classpath(graalvm_dist, graalvm_location, image_config, self.subject.component, exclude_implicit=True) main_class = image_config.main_class return u"|".join((u":".join(classpath), main_class)) return self._polyglot_config_contents @@ -1978,8 +1983,7 @@ def _get_graalvm_archive_path(jdk_path, graal_vm=None): 'JDK_TOOLS', } - -def graalvm_home_relative_classpath(dependencies, start=None, with_boot_jars=False, graal_vm=None): +def graalvm_home_relative_classpath(dependencies, start=None, with_boot_jars=False, graal_vm=None, exclude_names=None): if graal_vm is None: graal_vm = get_final_graalvm_distribution() start = start or _get_graalvm_archive_path('', graal_vm=graal_vm) @@ -1993,7 +1997,15 @@ def graalvm_home_relative_classpath(dependencies, start=None, with_boot_jars=Fal jimage = mx.project('graalvm-jimage', fatalIfMissing=False) jimage_deps = jimage.deps if jimage else None mx.logv("Composing classpath for " + str(dependencies) + ". Entries:\n" + '\n'.join(('- {}:{}'.format(d.suite, d.name) for d in mx.classpath_entries(dependencies)))) - for _cp_entry in mx.classpath_entries(dependencies): + cp_entries = mx.classpath_entries(dependencies) + + # Compute the set-difference of the transitive dependencies of `dependencies` and the transitive dependencies of `exclude_names` + if exclude_names: + for exclude_entry in mx.classpath_entries(names=exclude_names): + if exclude_entry in cp_entries: + cp_entries.remove(exclude_entry) + + for _cp_entry in cp_entries: if jimage_deps and _jlink_libraries() and _cp_entry in jimage_deps: continue if _cp_entry.isJdkLibrary() or _cp_entry.isJreLibrary(): diff --git a/substratevm/ci_includes/gate.hocon b/substratevm/ci_includes/gate.hocon index e5943932e121..fe41f3535dbf 100644 --- a/substratevm/ci_includes/gate.hocon +++ b/substratevm/ci_includes/gate.hocon @@ -45,9 +45,6 @@ builds += [ } ${labsjdk-ee-11} ${svm-common-linux-gate} ${linux-deploy} { name: "gate-svm-modules-basic" - environment : { - USE_NATIVE_IMAGE_JAVA_PLATFORM_MODULE_SYSTEM : "true" - } run: [ ${svm-cmd-gate} ["build,hellomodule,test"] ] diff --git a/substratevm/mx.substratevm/mx_substratevm.py b/substratevm/mx.substratevm/mx_substratevm.py index 76ced4e52a25..10559f76de6b 100644 --- a/substratevm/mx.substratevm/mx_substratevm.py +++ b/substratevm/mx.substratevm/mx_substratevm.py @@ -60,8 +60,6 @@ else: from io import StringIO -USE_NI_JPMS = os.environ.get('USE_NATIVE_IMAGE_JAVA_PLATFORM_MODULE_SYSTEM', 'false').lower() == 'true' - suite = mx.suite('substratevm') svmSuites = [suite] @@ -71,13 +69,25 @@ def svm_java_compliance(): def svm_java8(): return svm_java_compliance() <= mx.JavaCompliance('1.8') -def graal_compiler_flags(all_unnamed=True): +def graal_compiler_flags(): version_tag = svm_java_compliance().value - compiler_flags = mx.dependency('substratevm:svm-compiler-flags-builder').compute_graal_compiler_flags_map(all_unnamed=all_unnamed) + compiler_flags = mx.dependency('substratevm:svm-compiler-flags-builder').compute_graal_compiler_flags_map() if version_tag not in compiler_flags: missing_flags_message = 'Missing graal-compiler-flags for {0}.\n Did you forget to run "mx build"?' mx.abort(missing_flags_message.format(version_tag)) - return compiler_flags[version_tag] + def adjusted_exports(line): + """ + Turns e.g. + --add-exports=jdk.internal.vm.ci/jdk.vm.ci.code.stack=jdk.internal.vm.compiler,org.graalvm.nativeimage.builder + into: + --add-exports=jdk.internal.vm.ci/jdk.vm.ci.code.stack=ALL-UNNAMED + """ + if line.startswith('--add-exports='): + before, sep, _ = line.rpartition('=') + return before + sep + 'ALL-UNNAMED' + else: + return line + return [adjusted_exports(line) for line in compiler_flags[version_tag]] def svm_unittest_config_participant(config): vmArgs, mainClass, mainClassArgs = config @@ -839,7 +849,7 @@ def _native_image_launcher_extra_jvm_args(): support_distributions=['substratevm:NATIVE_IMAGE_GRAALVM_SUPPORT'], launcher_configs=[ mx_sdk_vm.LauncherConfig( - use_modules='image' if USE_NI_JPMS else 'launcher' if not svm_java8() else None, + use_modules='image' if not svm_java8() else None, main_module="org.graalvm.nativeimage.driver", destination="bin/", jar_distributions=["substratevm:SVM_DRIVER"], @@ -850,7 +860,7 @@ def _native_image_launcher_extra_jvm_args(): ], library_configs=[ mx_sdk_vm.LibraryConfig( - use_modules='image' if USE_NI_JPMS else 'launcher' if not svm_java8() else None, + use_modules='image' if not svm_java8() else None, destination="", jvm_library=True, jar_distributions=[ @@ -864,7 +874,7 @@ def _native_image_launcher_extra_jvm_args(): ], ), mx_sdk_vm.LibraryConfig( - use_modules='image' if USE_NI_JPMS else 'launcher' if not svm_java8() else None, + use_modules='image' if not svm_java8() else None, destination="", jvm_library=True, jar_distributions=[ @@ -1017,7 +1027,7 @@ def _native_image_configure_extra_jvm_args(): support_distributions=[], launcher_configs=[ mx_sdk_vm.LauncherConfig( - use_modules='image' if USE_NI_JPMS else 'launcher' if not svm_java8() else None, + use_modules='image' if not svm_java8() else None, main_module="org.graalvm.nativeimage.configure", destination="bin/", jar_distributions=["substratevm:SVM_CONFIGURE"], @@ -1088,8 +1098,6 @@ def hellomodule(args): """ if svm_java8(): mx.abort('Experimental module support requires Java 11+') - if os.environ.get('USE_NATIVE_IMAGE_JAVA_PLATFORM_MODULE_SYSTEM', 'false') != 'true': - mx.abort('Experimental module support requires USE_NATIVE_IMAGE_JAVA_PLATFORM_MODULE_SYSTEM=true for "mx build" and "mx hellomodule"') # Build a helloworld Java module with maven module_path = [] @@ -1383,8 +1391,19 @@ def clean(self, forBuild=False): def __str__(self): return 'JvmFuncsFallbacksBuildTask {}'.format(self.subject) +def mx_register_dynamic_suite_constituents(register_project, _): + register_project(SubstrateCompilerFlagsBuilder()) + class SubstrateCompilerFlagsBuilder(mx.ArchivableProject): + flags_build_dependencies = [ + 'substratevm:SVM' + ] + + def __init__(self): + mx.ArchivableProject.__init__(self, suite, 'svm-compiler-flags-builder', [], None, None) + self.buildDependencies = list(SubstrateCompilerFlagsBuilder.flags_build_dependencies) + def config_file(self, ver): return 'graal-compiler-flags-' + str(ver) + '.config' @@ -1440,7 +1459,7 @@ def config_file_update(self, file_path, lines, file_paths): # If renaming or moving this method, please update the error message in # com.oracle.svm.driver.NativeImage.BuildConfiguration.getBuilderJavaArgs(). - def compute_graal_compiler_flags_map(self, all_unnamed=not USE_NI_JPMS): + def compute_graal_compiler_flags_map(self): graal_compiler_flags_map = dict() graal_compiler_flags_map[8] = [ '-d64', @@ -1454,18 +1473,12 @@ def compute_graal_compiler_flags_map(self, all_unnamed=not USE_NI_JPMS): ] # Packages to add-export - distributions_transitive = mx.classpath_entries(self.deps) + distributions_transitive = mx.classpath_entries(self.buildDependencies) jdk = mx.get_jdk(tag='default') required_exports = mx_javamodules.requiredExports(distributions_transitive, jdk) - target_module = 'ALL-UNNAMED' if all_unnamed else None - exports_flags = mx_sdk_vm.AbstractNativeImageConfig.get_add_exports_list(required_exports, target_module) + exports_flags = mx_sdk_vm.AbstractNativeImageConfig.get_add_exports_list(required_exports) graal_compiler_flags_map[11].extend(exports_flags) - - # Currently JDK 13, 14, 15 and JDK 11 have the same flags - graal_compiler_flags_map[13] = graal_compiler_flags_map[11] - graal_compiler_flags_map[14] = graal_compiler_flags_map[11] - graal_compiler_flags_map[15] = graal_compiler_flags_map[11] - graal_compiler_flags_map[16] = graal_compiler_flags_map[11] + # Currently JDK 17 and JDK 11 have the same flags graal_compiler_flags_map[17] = graal_compiler_flags_map[11] # DO NOT ADD ANY NEW ADD-OPENS OR ADD-EXPORTS HERE! # diff --git a/substratevm/mx.substratevm/suite.py b/substratevm/mx.substratevm/suite.py index 1e7b60819843..b823d090a035 100644 --- a/substratevm/mx.substratevm/suite.py +++ b/substratevm/mx.substratevm/suite.py @@ -624,13 +624,6 @@ "spotbugs": "false", }, - "svm-compiler-flags-builder": { - "class" : "SubstrateCompilerFlagsBuilder", - "dependencies" : [ - "SVM", - ], - }, - "com.oracle.svm.junit": { "subDir": "src", "sourceDirs": ["src"], @@ -1174,6 +1167,11 @@ "com.oracle.svm.jvmtiagentbase", "com.oracle.svm.jvmtiagentbase.jvmti", ], + "requires" : [ + "static com.oracle.mxtool.junit", + "static junit", + "static hamcrest", + ], }, }, @@ -1220,6 +1218,7 @@ "com.oracle.objectfile", "com.oracle.objectfile.io", "com.oracle.objectfile.debuginfo", + "com.oracle.objectfile.macho", ], "requires" : ["jdk.unsupported"], @@ -1323,6 +1322,11 @@ "exports" : [ "com.oracle.svm.agent", ], + "requires" : [ + "static com.oracle.mxtool.junit", + "static junit", + "static hamcrest", + ], "requiresConcealed" : { "jdk.internal.vm.ci" : [ "jdk.vm.ci.meta", @@ -1347,6 +1351,11 @@ "exports" : [ "com.oracle.svm.diagnosticsagent", ], + "requires" : [ + "static com.oracle.mxtool.junit", + "static junit", + "static hamcrest", + ], }, }, @@ -1367,6 +1376,11 @@ "* to org.graalvm.nativeimage.agent.tracing", "com.oracle.svm.configure", ], + "requires" : [ + "static com.oracle.mxtool.junit", + "static junit", + "static hamcrest", + ], }, }, diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/MacroOptionHandler.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/MacroOptionHandler.java index ed8c60b12df8..f53010eba43c 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/MacroOptionHandler.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/MacroOptionHandler.java @@ -91,11 +91,7 @@ private void applyEnabled(MacroOption.EnabledOption enabledOption, String argume boolean explicitImageClasspath = enabledOption.forEachPropertyValue( config, "ImageClasspath", entry -> nativeImage.addImageClasspath(ClasspathUtils.stringToClasspath(entry)), PATH_SEPARATOR_REGEX); if (!explicitImageModulePath && !explicitImageClasspath) { - if (NativeImage.USE_NI_JPMS) { - NativeImage.getJars(imageJarsDirectory).forEach(nativeImage::addImageModulePath); - } else { - NativeImage.getJars(imageJarsDirectory).forEach(nativeImage::addImageClasspath); - } + NativeImage.getJars(imageJarsDirectory).forEach(nativeImage::addImageClasspath); } String imageName = enabledOption.getProperty(config, "ImageName"); diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index b73b7045d57b..22ac4521e448 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -31,9 +31,6 @@ import java.io.InputStreamReader; import java.lang.management.ManagementFactory; import java.lang.management.OperatingSystemMXBean; -import java.lang.reflect.InvocationHandler; -import java.lang.reflect.Method; -import java.lang.reflect.Proxy; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.InvalidPathException; @@ -53,7 +50,6 @@ import java.util.Map; import java.util.Optional; import java.util.Properties; -import java.util.Scanner; import java.util.function.BiConsumer; import java.util.function.BiFunction; import java.util.function.Consumer; @@ -94,8 +90,7 @@ public class NativeImage { - /* Used to enable native-image JPMS support (WIP) - will become default once completed */ - static final boolean USE_NI_JPMS = System.getenv().getOrDefault("USE_NATIVE_IMAGE_JAVA_PLATFORM_MODULE_SYSTEM", "false").equalsIgnoreCase("true"); + private static final String ENV_VAR_USE_MODULE_SYSTEM = "USE_NATIVE_IMAGE_JAVA_PLATFORM_MODULE_SYSTEM"; private static final String DEFAULT_GENERATOR_CLASS_NAME = NativeImageGeneratorRunner.class.getName(); private static final String DEFAULT_GENERATOR_MODULE_NAME = ModuleSupport.getModuleName(NativeImageGeneratorRunner.class); @@ -263,11 +258,60 @@ private static String oR(OptionKey option) { private final List excludedConfigs = new ArrayList<>(); - public interface BuildConfiguration { + static class BuildConfiguration { + + boolean modulePathBuild; + + protected final Path workDir; + protected final Path rootDir; + protected final List args; + + BuildConfiguration(BuildConfiguration original) { + modulePathBuild = original.modulePathBuild; + workDir = original.workDir; + rootDir = original.rootDir; + args = new ArrayList<>(original.args); + } + + BuildConfiguration(List args) { + this(null, null, args); + } + + @SuppressWarnings("deprecation") + BuildConfiguration(Path rootDir, Path workDir, List args) { + modulePathBuild = Boolean.parseBoolean(System.getenv().get(ENV_VAR_USE_MODULE_SYSTEM)); + this.args = args; + this.workDir = workDir != null ? workDir : Paths.get(".").toAbsolutePath().normalize(); + if (rootDir != null) { + this.rootDir = rootDir; + } else { + if (IS_AOT) { + Path executablePath = Paths.get(ProcessProperties.getExecutableName()); + assert executablePath != null; + Path binDir = executablePath.getParent(); + Path rootDirCandidate = binDir.getParent(); + if (rootDirCandidate.endsWith(platform)) { + rootDirCandidate = rootDirCandidate.getParent(); + } + if (rootDirCandidate.endsWith(Paths.get("lib", "svm"))) { + rootDirCandidate = rootDirCandidate.getParent().getParent(); + } + this.rootDir = rootDirCandidate; + } else { + String rootDirProperty = "native-image.root"; + String rootDirString = System.getProperty(rootDirProperty); + if (rootDirString == null) { + rootDirString = System.getProperty("java.home"); + } + this.rootDir = Paths.get(rootDirString); + } + } + } + /** * @return the name of the image generator main class. */ - default String getGeneratorMainClass() { + public String getGeneratorMainClass() { String generatorClassName = DEFAULT_GENERATOR_CLASS_NAME; if (useJavaModules()) { generatorClassName += DEFAULT_GENERATOR_9PLUS_SUFFIX; @@ -279,26 +323,44 @@ default String getGeneratorMainClass() { * @return relative path usage get resolved against this path (also default path for image * building) */ - Path getWorkingDirectory(); + public Path getWorkingDirectory() { + return workDir; + } /** * @return java.home that is associated with this BuildConfiguration */ - default Path getJavaHome() { - throw VMError.unimplemented(); + public Path getJavaHome() { + return useJavaModules() ? rootDir : rootDir.getParent(); } /** * @return path to Java executable */ - default Path getJavaExecutable() { - throw VMError.unimplemented(); + public Path getJavaExecutable() { + Path binJava = Paths.get("bin", OS.getCurrent() == OS.WINDOWS ? "java.exe" : "java"); + if (Files.isExecutable(rootDir.resolve(binJava))) { + return rootDir.resolve(binJava); + } + + String javaHome = System.getenv("JAVA_HOME"); + if (javaHome == null) { + throw showError("Environment variable JAVA_HOME is not set"); + } + Path javaHomeDir = Paths.get(javaHome); + if (!Files.isDirectory(javaHomeDir)) { + throw showError("Environment variable JAVA_HOME does not refer to a directory"); + } + if (!Files.isExecutable(javaHomeDir.resolve(binJava))) { + throw showError("Environment variable JAVA_HOME does not refer to a directory with a " + binJava + " executable"); + } + return javaHomeDir.resolve(binJava); } /** * @return true if Java modules system should be used */ - default boolean useJavaModules() { + public boolean useJavaModules() { try { Class.forName("java.lang.Module"); } catch (ClassNotFoundException e) { @@ -310,77 +372,82 @@ default boolean useJavaModules() { /** * @return classpath for SubstrateVM image builder components */ - default List getBuilderClasspath() { - throw VMError.unimplemented(); + public List getBuilderClasspath() { + if (modulePathBuild) { + return Collections.emptyList(); + } + List result = new ArrayList<>(); + if (useJavaModules()) { + result.addAll(getJars(rootDir.resolve(Paths.get("lib", "jvmci")), "graal-sdk", "graal", "enterprise-graal")); + } + result.addAll(getJars(rootDir.resolve(Paths.get("lib", "svm", "builder")))); + return result; } /** * @return base clibrary paths needed for general image building */ - default List getBuilderCLibrariesPaths() { - throw VMError.unimplemented(); + public List getBuilderCLibrariesPaths() { + return Collections.singletonList(rootDir.resolve(Paths.get("lib", "svm", "clibraries"))); } /** * @return path to content of the inspect web server (points-to analysis debugging) */ - default Path getBuilderInspectServerPath() { - throw VMError.unimplemented(); + public Path getBuilderInspectServerPath() { + Path inspectPath = rootDir.resolve(Paths.get("lib", "svm", "inspect")); + if (Files.isDirectory(inspectPath)) { + return inspectPath; + } + return null; } /** * @return base image classpath needed for every image (e.g. LIBRARY_SUPPORT) */ - default List getImageProvidedClasspath() { - throw VMError.unimplemented(); + public List getImageProvidedClasspath() { + return getImageProvidedJars(); } /** * @return base image module-path needed for every image (e.g. LIBRARY_SUPPORT) */ - default List getImageProvidedModulePath() { - throw VMError.unimplemented(); + public List getImageProvidedModulePath() { + return getImageProvidedJars(); + } + + private List getImageProvidedJars() { + return getJars(rootDir.resolve(Paths.get("lib", "svm"))); } /** * @return JVMCI API classpath for image builder (jvmci + graal jars) */ - default List getBuilderJVMCIClasspath() { - throw VMError.unimplemented(); + public List getBuilderJVMCIClasspath() { + return getJars(rootDir.resolve(Paths.get("lib", "jvmci"))); } /** * @return entries for jvmci.class.path.append system property (if needed) */ - default List getBuilderJVMCIClasspathAppend() { - throw VMError.unimplemented(); + public List getBuilderJVMCIClasspathAppend() { + return getBuilderJVMCIClasspath().stream() + .filter(f -> f.getFileName().toString().toLowerCase().endsWith("graal.jar")) + .collect(Collectors.toList()); } /** * @return boot-classpath for image builder (graal-sdk.jar) */ - default List getBuilderBootClasspath() { - throw VMError.unimplemented(); + public List getBuilderBootClasspath() { + return getJars(rootDir.resolve(Paths.get("lib", "boot"))); } /** * @return additional arguments for JVM that runs image builder */ - default List getBuilderJavaArgs() { - String javaVersion = String.valueOf(JavaVersionUtil.JAVA_SPEC); - String[] flagsForVersion = graalCompilerFlags.get(javaVersion); - if (flagsForVersion == null) { - String suffix = ""; - if (System.getProperty("java.home").contains("-dev")) { - suffix = " Update SubstrateCompilerFlagsBuilder.compute_graal_compiler_flags_map() in " + - "mx_substratevm.py to add a configuration for a new Java version."; - } - showError(String.format("Image building not supported for Java version %s in %s with VM configuration \"%s\".%s", - System.getProperty("java.version"), - System.getProperty("java.home"), - System.getProperty("java.vm.name"), - suffix)); - } + public List getBuilderJavaArgs() { + ArrayList builderJavaArgs = new ArrayList<>(); if (useJVMCINativeLibrary == null) { useJVMCINativeLibrary = false; @@ -414,198 +481,50 @@ default List getBuilderJavaArgs() { } } - ArrayList builderJavaArgs = new ArrayList<>(); - builderJavaArgs.addAll(Arrays.asList(flagsForVersion)); if (useJVMCINativeLibrary) { builderJavaArgs.add("-XX:+UseJVMCINativeLibrary"); } else { builderJavaArgs.add("-XX:-UseJVMCICompiler"); } - return builderJavaArgs; - } - - /** - * @return entries for the --module-path of the image builder - */ - default List getBuilderModulePath() { - throw VMError.unimplemented(); - } - - /** - * @return entries for the --upgrade-module-path of the image builder - */ - default List getBuilderUpgradeModulePath() { - throw VMError.unimplemented(); - } - /** - * @return classpath for image (the classes the user wants to build an image from) - */ - default List getImageClasspath() { - throw VMError.unimplemented(); - } - - /** - * @return native-image (i.e. image build) arguments - */ - default List getBuildArgs() { - throw VMError.unimplemented(); - } - - /** - * @return true for fallback image building - */ - default boolean buildFallbackImage() { - return false; - } - - default Path getAgentJAR() { - return null; - } - - /** - * ResourcesJar packs resources files needed for some jdk services such as xml - * serialization. - * - * @return the path to the resources.jar file - */ - default Optional getResourcesJar() { - return Optional.empty(); - } - } - - protected static class DefaultBuildConfiguration implements BuildConfiguration { - protected final Path workDir; - protected final Path rootDir; - protected final List args; - - protected DefaultBuildConfiguration(List args) { - this(null, null, args); - } - - @SuppressWarnings("deprecation") - DefaultBuildConfiguration(Path rootDir, Path workDir, List args) { - this.args = args; - this.workDir = workDir != null ? workDir : Paths.get(".").toAbsolutePath().normalize(); - if (rootDir != null) { - this.rootDir = rootDir; - } else { - if (IS_AOT) { - Path executablePath = Paths.get(ProcessProperties.getExecutableName()); - assert executablePath != null; - Path binDir = executablePath.getParent(); - Path rootDirCandidate = binDir.getParent(); - if (rootDirCandidate.endsWith(platform)) { - rootDirCandidate = rootDirCandidate.getParent(); - } - if (rootDirCandidate.endsWith(Paths.get("lib", "svm"))) { - rootDirCandidate = rootDirCandidate.getParent().getParent(); - } - this.rootDir = rootDirCandidate; - } else { - String rootDirProperty = "native-image.root"; - String rootDirString = System.getProperty(rootDirProperty); - if (rootDirString == null) { - rootDirString = System.getProperty("java.home"); - } - this.rootDir = Paths.get(rootDirString); + String javaVersion = String.valueOf(JavaVersionUtil.JAVA_SPEC); + String[] flagsForVersion = graalCompilerFlags.get(javaVersion); + if (flagsForVersion == null) { + String suffix = ""; + if (System.getProperty("java.home").contains("-dev")) { + suffix = " Update SubstrateCompilerFlagsBuilder.compute_graal_compiler_flags_map() in " + + "mx_substratevm.py to add a configuration for a new Java version."; } + showError(String.format("Image building not supported for Java version %s in %s with VM configuration \"%s\".%s", + System.getProperty("java.version"), + System.getProperty("java.home"), + System.getProperty("java.vm.name"), + suffix)); } - } - - @Override - public Path getWorkingDirectory() { - return workDir; - } - - @Override - public Path getJavaHome() { - return useJavaModules() ? rootDir : rootDir.getParent(); - } - - @Override - public Path getJavaExecutable() { - Path binJava = Paths.get("bin", OS.getCurrent() == OS.WINDOWS ? "java.exe" : "java"); - if (Files.isExecutable(rootDir.resolve(binJava))) { - return rootDir.resolve(binJava); - } - - String javaHome = System.getenv("JAVA_HOME"); - if (javaHome == null) { - throw showError("Environment variable JAVA_HOME is not set"); - } - Path javaHomeDir = Paths.get(javaHome); - if (!Files.isDirectory(javaHomeDir)) { - throw showError("Environment variable JAVA_HOME does not refer to a directory"); - } - if (!Files.isExecutable(javaHomeDir.resolve(binJava))) { - throw showError("Environment variable JAVA_HOME does not refer to a directory with a " + binJava + " executable"); - } - return javaHomeDir.resolve(binJava); - } - - @Override - public List getBuilderClasspath() { - if (USE_NI_JPMS) { - return Collections.emptyList(); - } - List result = new ArrayList<>(); - if (useJavaModules()) { - result.addAll(getJars(rootDir.resolve(Paths.get("lib", "jvmci")), "graal-sdk", "graal", "enterprise-graal")); - } - result.addAll(getJars(rootDir.resolve(Paths.get("lib", "svm", "builder")))); - return result; - } - - @Override - public List getBuilderCLibrariesPaths() { - return Collections.singletonList(rootDir.resolve(Paths.get("lib", "svm", "clibraries"))); - } - @Override - public List getImageProvidedClasspath() { - return getImageProvidedJars(); - } - - @Override - public List getImageProvidedModulePath() { - return getImageProvidedJars(); - } - - private List getImageProvidedJars() { - return getJars(rootDir.resolve(Paths.get("lib", "svm"))); - } - - @Override - public Path getBuilderInspectServerPath() { - Path inspectPath = rootDir.resolve(Paths.get("lib", "svm", "inspect")); - if (Files.isDirectory(inspectPath)) { - return inspectPath; + for (String line : flagsForVersion) { + if (!modulePathBuild && line.startsWith("--add-exports=")) { + /*- + * Turns e.g. + * --add-exports=jdk.internal.vm.ci/jdk.vm.ci.code.stack=jdk.internal.vm.compiler,org.graalvm.nativeimage.builder + * into: + * --add-exports=jdk.internal.vm.ci/jdk.vm.ci.code.stack=ALL-UNNAMED + */ + builderJavaArgs.add(line.substring(0, line.lastIndexOf('=') + 1) + "ALL-UNNAMED"); + } else { + builderJavaArgs.add(line); + } } - return null; - } - - @Override - public List getBuilderJVMCIClasspath() { - return getJars(rootDir.resolve(Paths.get("lib", "jvmci"))); - } - @Override - public List getBuilderJVMCIClasspathAppend() { - return getBuilderJVMCIClasspath().stream() - .filter(f -> f.getFileName().toString().toLowerCase().endsWith("graal.jar")) - .collect(Collectors.toList()); - } - - @Override - public List getBuilderBootClasspath() { - return getJars(rootDir.resolve(Paths.get("lib", "boot"))); + return builderJavaArgs; } - @Override + /** + * @return entries for the --module-path of the image builder + */ public List getBuilderModulePath() { List result = new ArrayList<>(); - if (USE_NI_JPMS) { + if (modulePathBuild) { result.addAll(getJars(rootDir.resolve(Paths.get("lib", "svm", "builder")))); } else { result.addAll(getJars(rootDir.resolve(Paths.get("lib", "jvmci")), "graal-sdk", "enterprise-graal")); @@ -614,17 +533,23 @@ public List getBuilderModulePath() { return result; } - @Override + /** + * @return entries for the --upgrade-module-path of the image builder + */ public List getBuilderUpgradeModulePath() { return getJars(rootDir.resolve(Paths.get("lib", "jvmci")), "graal", "graal-management"); } - @Override + /** + * @return classpath for image (the classes the user wants to build an image from) + */ public List getImageClasspath() { return Collections.emptyList(); } - @Override + /** + * @return native-image (i.e. image build) arguments + */ public List getBuildArgs() { if (args.isEmpty()) { return Collections.emptyList(); @@ -636,12 +561,23 @@ public List getBuildArgs() { return buildArgs; } - @Override + /** + * @return true for fallback image building + */ + public boolean buildFallbackImage() { + return false; + } + public Path getAgentJAR() { return rootDir.resolve(Paths.get("lib", "svm", "builder", "svm.jar")); } - @Override + /** + * ResourcesJar packs resources files needed for some jdk services such as xml + * serialization. + * + * @return the path to the resources.jar file + */ public Optional getResourcesJar() { return Optional.of(rootDir.resolve(Paths.get("lib", "resources.jar"))); } @@ -761,35 +697,28 @@ private ArrayList createFallbackBuildArgs() { return buildArgs; } - private static final class FallbackBuildConfiguration implements InvocationHandler { - private final NativeImage original; - private final List buildArgs; + private static final class FallbackBuildConfiguration extends BuildConfiguration { + + private final List fallbackBuildArgs; private FallbackBuildConfiguration(NativeImage original) { - this.original = original; - this.buildArgs = original.createFallbackBuildArgs(); + super(original.config); + fallbackBuildArgs = original.createFallbackBuildArgs(); } - static BuildConfiguration create(NativeImage imageName) { - FallbackBuildConfiguration handler = new FallbackBuildConfiguration(imageName); - BuildConfiguration fallback = (BuildConfiguration) Proxy.newProxyInstance(BuildConfiguration.class.getClassLoader(), - new Class[]{BuildConfiguration.class}, - handler); - return fallback; + @Override + public List getImageClasspath() { + return Collections.emptyList(); } @Override - public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { - switch (method.getName()) { - case "getImageClasspath": - return Collections.emptyList(); - case "getBuildArgs": - return buildArgs; - case "buildFallbackImage": - return true; - default: - return method.invoke(original.config, args); - } + public List getBuildArgs() { + return fallbackBuildArgs; + } + + @Override + public boolean buildFallbackImage() { + return true; } } @@ -859,7 +788,6 @@ protected static void ensureDirectoryExists(Path dir) { } private void prepareImageBuildArgs() { - config.getBuilderJavaArgs().forEach(this::addImageBuilderJavaArgs); addImageBuilderJavaArgs("-Xss10m"); addImageBuilderJavaArgs(oXms + getXmsValue()); String xmxVal = getXmxValue(1); @@ -888,32 +816,6 @@ private void prepareImageBuildArgs() { * non-system class and and triggers a warning. */ addImageBuilderJavaArgs("-Xshare:off"); - config.getBuilderClasspath().forEach(this::addImageBuilderClasspath); - - if (config.getBuilderInspectServerPath() != null) { - addPlainImageBuilderArg(oHInspectServerContentPath + config.getBuilderInspectServerPath()); - } - - if (config.useJavaModules()) { - config.getBuilderModulePath().forEach(this::addImageBuilderModulePath); - String upgradeModulePath = config.getBuilderUpgradeModulePath().stream() - .map(p -> canonicalize(p).toString()) - .collect(Collectors.joining(File.pathSeparator)); - if (!upgradeModulePath.isEmpty()) { - addImageBuilderJavaArgs(Arrays.asList("--upgrade-module-path", upgradeModulePath)); - } - } else { - config.getBuilderJVMCIClasspath().forEach((Consumer) this::addImageBuilderClasspath); - if (!config.getBuilderJVMCIClasspathAppend().isEmpty()) { - String builderJavaArg = config.getBuilderJVMCIClasspathAppend() - .stream().map(path -> canonicalize(path).toString()) - .collect(Collectors.joining(File.pathSeparator, "-Djvmci.class.path.append=", "")); - addImageBuilderJavaArgs(builderJavaArg); - } - - config.getBuilderBootClasspath().forEach((Consumer) this::addImageBuilderBootClasspath); - config.getResourcesJar().ifPresent(this::addImageBuilderClasspath); - } config.getImageClasspath().forEach(this::addCustomImageClasspath); } @@ -1090,6 +992,33 @@ void handleClassPathAttribute(Path jarFilePath, Attributes mainAttributes) { private int completeImageBuild() { List leftoverArgs = processNativeImageArgs(); + config.getBuilderClasspath().forEach(this::addImageBuilderClasspath); + + if (config.getBuilderInspectServerPath() != null) { + addPlainImageBuilderArg(oHInspectServerContentPath + config.getBuilderInspectServerPath()); + } + + if (config.useJavaModules()) { + config.getBuilderModulePath().forEach(this::addImageBuilderModulePath); + String upgradeModulePath = config.getBuilderUpgradeModulePath().stream() + .map(p -> canonicalize(p).toString()) + .collect(Collectors.joining(File.pathSeparator)); + if (!upgradeModulePath.isEmpty()) { + addImageBuilderJavaArgs(Arrays.asList("--upgrade-module-path", upgradeModulePath)); + } + } else { + config.getBuilderJVMCIClasspath().forEach((Consumer) this::addImageBuilderClasspath); + if (!config.getBuilderJVMCIClasspathAppend().isEmpty()) { + String builderJavaArg = config.getBuilderJVMCIClasspathAppend() + .stream().map(path -> canonicalize(path).toString()) + .collect(Collectors.joining(File.pathSeparator, "-Djvmci.class.path.append=", "")); + addImageBuilderJavaArgs(builderJavaArg); + } + + config.getBuilderBootClasspath().forEach((Consumer) this::addImageBuilderBootClasspath); + config.getResourcesJar().ifPresent(this::addImageBuilderClasspath); + } + completeOptionArgs(); addTargetArguments(); @@ -1169,7 +1098,7 @@ private int completeImageBuild() { boolean hasMainClass = mainClass != null && !mainClass.isEmpty(); if (extraImageArgs.isEmpty()) { if (buildExecutable && !hasMainClassModule && !hasMainClass) { - String moduleMsg = USE_NI_JPMS ? " (or /)" : ""; + String moduleMsg = config.modulePathBuild ? " (or /)" : ""; showError("Please specify class" + moduleMsg + " containing the main entry point method. (see --help)"); } } else if (!moduleOptionMode) { @@ -1223,7 +1152,7 @@ private int completeImageBuild() { finalImageClasspath.addAll(imageClasspath); List imageProvidedJars; - if (USE_NI_JPMS) { + if (config.modulePathBuild) { imageProvidedJars = config.getImageProvidedModulePath(); finalImageModulePath.addAll(imageProvidedJars); } else { @@ -1236,7 +1165,9 @@ private int completeImageBuild() { /* Bypass regular build and proceed with fallback image building */ return 2; } - return buildImage(imageBuilderJavaArgs, imageBuilderBootClasspath, imageBuilderClasspath, imageBuilderModulePath, imageBuilderArgs, finalImageClasspath, finalImageModulePath); + + List finalImageBuilderJavaArgs = Stream.concat(config.getBuilderJavaArgs().stream(), imageBuilderJavaArgs.stream()).collect(Collectors.toList()); + return buildImage(finalImageBuilderJavaArgs, imageBuilderBootClasspath, imageBuilderClasspath, imageBuilderModulePath, imageBuilderArgs, finalImageClasspath, finalImageModulePath); } private static String getLocationAgnosticArgPrefix(String argPrefix) { @@ -1412,7 +1343,7 @@ protected int buildImage(List javaArgs, LinkedHashSet bcp, LinkedH arguments.addAll(strings); } - if (USE_NI_JPMS) { + if (config.modulePathBuild) { arguments.addAll(Arrays.asList("--module", DEFAULT_GENERATOR_MODULE_NAME + "/" + DEFAULT_GENERATOR_CLASS_NAME)); } else { arguments.add(config.getGeneratorMainClass()); @@ -1458,6 +1389,9 @@ protected int buildImage(List javaArgs, LinkedHashSet bcp, LinkedH try { ProcessBuilder pb = new ProcessBuilder(); pb.command(command); + if (config.modulePathBuild) { + pb.environment().put(ENV_VAR_USE_MODULE_SYSTEM, Boolean.toString(true)); + } p = pb.inheritIO().start(); exitStatus = p.waitFor(); } catch (IOException | InterruptedException e) { @@ -1473,7 +1407,7 @@ protected int buildImage(List javaArgs, LinkedHashSet bcp, LinkedH private static final Function defaultNativeImageProvider = config -> new NativeImage(config); public static void main(String[] args) { - performBuild(new DefaultBuildConfiguration(Arrays.asList(args)), defaultNativeImageProvider); + performBuild(new BuildConfiguration(Arrays.asList(args)), defaultNativeImageProvider); } public static void build(BuildConfiguration config) { @@ -1481,7 +1415,7 @@ public static void build(BuildConfiguration config) { } public static void agentBuild(Path javaHome, Path workDir, List buildArgs) { - performBuild(new DefaultBuildConfiguration(javaHome, workDir, buildArgs), NativeImage::new); + performBuild(new BuildConfiguration(javaHome, workDir, buildArgs), NativeImage::new); } private static void performBuild(BuildConfiguration config, Function nativeImageProvider) { @@ -1519,7 +1453,7 @@ protected static void build(BuildConfiguration config, Function builderModuleNames = null; - LinkedHashSet getBuilderModuleNames() { - if (builderModuleNames == null) { - ProcessBuilder pb = new ProcessBuilder(); - List command = pb.command(); - command.add(config.getJavaExecutable().toString()); - command.add("--module-path"); - command.add(Stream.concat(config.getBuilderModulePath().stream(), config.getImageProvidedModulePath().stream()) - .map(this::canonicalize).map(Path::toString) - .collect(Collectors.joining(File.pathSeparator))); - command.add("--list-modules"); - pb.redirectErrorStream(); - Process process = null; - int exitValue = -1; - LinkedHashSet modules = new LinkedHashSet<>(); - String message = "Unable to determine image builder modules"; - try { - process = pb.start(); - try (Scanner scanner = new Scanner(process.getInputStream())) { - while (scanner.hasNextLine()) { - String token = scanner.next(); - modules.add(token.split("@", 2)[0]); - scanner.nextLine(); - } - } - exitValue = process.waitFor(); - } catch (IOException | InterruptedException e) { - throw showError(message, e); - } finally { - if (process != null) { - process.destroy(); - } - if (exitValue != 0) { - throw showError(message + " (nonzero exit value)"); - } - } - - builderModuleNames = modules; - } - return builderModuleNames; - } - void addImageModulePath(Path modulePathEntry) { addImageModulePath(modulePathEntry, true); } @@ -1697,36 +1590,12 @@ void addImageModulePath(Path modulePathEntry, boolean strict) { /* Duplicate entries are silently ignored like with the java command */ return; } - List moduleNames = getModuleNames(mpEntry); - if (moduleNames.size() == 1) { - String moduleName = moduleNames.get(0); - if (getBuilderModuleNames().contains(moduleName)) { - /* Modules provided by the image-builder are not allowed on the -imagemp */ - String modulePathEntryStr = "module " + moduleName + " from " + modulePathEntry; - showWarning("Ignoring " + modulePathEntryStr + " (implicitly provided by builder)"); - return; - } - } + config.modulePathBuild = true; imageModulePath.add(mpEntry); processClasspathNativeImageMetaInf(mpEntry); } - @SuppressWarnings("unchecked") - List getModuleNames(Path... modulePathEntries) { - try { - Class moduleAccess = Class.forName("com.oracle.svm.driver.jdk11.ModuleAccess"); - Method getModuleNames = moduleAccess.getMethod("getModuleNames", Path[].class); - return (List) getModuleNames.invoke(null, (Object) modulePathEntries); - } catch (ClassNotFoundException e) { - throw showError("com.oracle.svm.driver.jdk11.ModuleAccess.getModuleNames only available on Java > 8"); - } catch (ReflectiveOperationException e) { - String entries = Arrays.stream(modulePathEntries) - .map(Path::toString).collect(Collectors.joining(", ", "[", "]")); - throw showError("Failed to get module-names for " + entries, e); - } - } - /** * For adding classpath elements that are put *explicitly* on the image-classpath (i.e. when * specified as -cp/-classpath/--class-path entry). This method handles invalid classpath @@ -1795,6 +1664,7 @@ void setJarOptionMode(boolean val) { void setModuleOptionMode(boolean val) { moduleOptionMode = val; + config.modulePathBuild = true; } boolean isVerbose() {