diff --git a/compiler/mx.compiler/mx_compiler.py b/compiler/mx.compiler/mx_compiler.py index b55e715b6102..bfdc3624b1d2 100644 --- a/compiler/mx.compiler/mx_compiler.py +++ b/compiler/mx.compiler/mx_compiler.py @@ -993,6 +993,16 @@ def __opened__(self, arc, srcArc, services): self.arc = arc def __add__(self, arcname, contents): # pylint: disable=unexpected-special-method-signature + + def add_serviceprovider(service, provider, version): + if version is None: + # Non-versioned service + self.services.setdefault(service, []).append(provider) + else: + # Versioned service + services = self.services.setdefault(int(version), {}) + services.setdefault(service, []).append(provider) + m = GraalArchiveParticipant.providersRE.match(arcname) if m: if self.isTest: @@ -1005,23 +1015,25 @@ def __add__(self, arcname, contents): # pylint: disable=unexpected-special-metho for service in _decode(contents).strip().split(os.linesep): assert service version = m.group(1) - if version is None: - # Non-versioned service - self.services.setdefault(service, []).append(provider) - else: - # Versioned service - services = self.services.setdefault(int(version), {}) - services.setdefault(service, []).append(provider) + add_serviceprovider(service, provider, version) return True - elif arcname.endswith('_OptionDescriptors.class') and not arcname.startswith('META-INF/'): + elif arcname.endswith('_OptionDescriptors.class'): if self.isTest: mx.warn('@Option defined in test code will be ignored: ' + arcname) else: # Need to create service files for the providers of the # jdk.internal.vm.ci.options.Options service created by # jdk.internal.vm.ci.options.processor.OptionProcessor. + version_prefix = 'META-INF/versions/' + if arcname.startswith(version_prefix): + # If OptionDescriptor is version-specific, get version + # from arcname and adjust arcname to non-version form + version, _, arcname = arcname[len(version_prefix):].partition('/') + else: + version = None provider = arcname[:-len('.class'):].replace('/', '.') - self.services.setdefault('org.graalvm.compiler.options.OptionDescriptors', []).append(provider) + service = 'org.graalvm.compiler.options.OptionDescriptors' + add_serviceprovider(service, provider, version) return False def __addsrc__(self, arcname, contents): diff --git a/compiler/src/org.graalvm.compiler.truffle.compiler/src/org/graalvm/compiler/truffle/compiler/TruffleCompilerImpl.java b/compiler/src/org.graalvm.compiler.truffle.compiler/src/org/graalvm/compiler/truffle/compiler/TruffleCompilerImpl.java index 1868cd81a9f6..3b539eeea0a9 100644 --- a/compiler/src/org.graalvm.compiler.truffle.compiler/src/org/graalvm/compiler/truffle/compiler/TruffleCompilerImpl.java +++ b/compiler/src/org.graalvm.compiler.truffle.compiler/src/org/graalvm/compiler/truffle/compiler/TruffleCompilerImpl.java @@ -277,7 +277,6 @@ public final void doCompile(TruffleDebugContext truffleDebug, TruffleCompilationTask task, TruffleCompilerListener inListener) { Objects.requireNonNull(compilation, "Compilation must be non null."); - Objects.requireNonNull(task, "Compilation task must be non null."); org.graalvm.options.OptionValues options = getOptionsForCompiler(optionsMap); TruffleCompilationIdentifier compilationId = asTruffleCompilationIdentifier(compilation); CompilableTruffleAST compilable = compilationId.getCompilable(); diff --git a/sdk/mx.sdk/mx_sdk_vm.py b/sdk/mx.sdk/mx_sdk_vm.py index 9f35ff3c2fa9..75b91f518cc9 100644 --- a/sdk/mx.sdk/mx_sdk_vm.py +++ b/sdk/mx.sdk/mx_sdk_vm.py @@ -93,17 +93,19 @@ def __prepare__(mcs, name, this_bases): class AbstractNativeImageConfig(_with_metaclass(ABCMeta, object)): - def __init__(self, destination, jar_distributions, build_args, links=None, is_polyglot=False, dir_jars=False, home_finder=False, build_time=1): # pylint: disable=super-init-not-called + def __init__(self, destination, jar_distributions, build_args, use_modules=None, links=None, is_polyglot=False, dir_jars=False, home_finder=False, build_time=1): # pylint: disable=super-init-not-called """ :type destination: str :type jar_distributions: list[str] :type build_args: list[str] :type links: list[str] + :param str use_modules: Run (with 'laucher') or run and build image with module support (with 'image'). :param bool dir_jars: If true, all jars in the component directory are added to the classpath. """ self.destination = mx_subst.path_substitutions.substitute(destination) self.jar_distributions = jar_distributions self.build_args = build_args + self.use_modules = use_modules self.links = [mx_subst.path_substitutions.substitute(link) for link in links] if links else [] self.is_polyglot = is_polyglot self.dir_jars = dir_jars @@ -133,19 +135,21 @@ def get_add_exports_list(required_exports, custom_target_module_str=None): class LauncherConfig(AbstractNativeImageConfig): def __init__(self, destination, jar_distributions, main_class, build_args, is_main_launcher=True, default_symlinks=True, is_sdk_launcher=False, custom_launcher_script=None, extra_jvm_args=None, - is_module_launcher=False, option_vars=None, home_finder=True, **kwargs): + use_modules=None, main_module=None, option_vars=None, home_finder=True, **kwargs): """ :param str main_class :param bool is_main_launcher :param bool default_symlinks :param bool is_sdk_launcher: Whether it uses org.graalvm.launcher.Launcher - :param bool is_module_launcher: Whether it uses classpath or module-path for the application + :param str use_modules: Run (with 'laucher') or run and build image with module support (with 'image'). + :param str main_module: Specifies the main module. Mandatory if use_modules is not None :param str custom_launcher_script: Custom launcher script, to be used when not compiled as a native image """ - super(LauncherConfig, self).__init__(destination, jar_distributions, build_args, home_finder=home_finder, **kwargs) + super(LauncherConfig, self).__init__(destination, jar_distributions, build_args, use_modules, home_finder=home_finder, **kwargs) self.main_class = main_class self.is_main_launcher = is_main_launcher - self.module_launcher = is_module_launcher + assert use_modules is None or main_module + self.main_module = main_module self.default_symlinks = default_symlinks self.is_sdk_launcher = is_sdk_launcher self.custom_launcher_script = custom_launcher_script @@ -161,7 +165,7 @@ def add_relative_home_path(self, language, path): self.relative_home_paths[language] = path def get_add_exports(self, missing_jars): - if not self.module_launcher: + if self.use_modules is None: return '' distributions = self.jar_distributions distributions_transitive = mx.classpath_entries(distributions) diff --git a/sdk/mx.sdk/mx_sdk_vm_impl.py b/sdk/mx.sdk/mx_sdk_vm_impl.py index 1a30a82e2abb..ec2f0d2b6e96 100644 --- a/sdk/mx.sdk/mx_sdk_vm_impl.py +++ b/sdk/mx.sdk/mx_sdk_vm_impl.py @@ -1075,7 +1075,6 @@ def native_image(self, build_args, output_file, allow_server=False, nonZeroIsFat native_image_project_name = GraalVmLauncher.launcher_project_name(mx_sdk.LauncherConfig(mx.exe_suffix('native-image'), [], "", []), stage1=True) native_image_bin = join(stage1.output, stage1.find_single_source_location('dependency:' + native_image_project_name)) native_image_command = [native_image_bin] + build_args - # currently, when building with the bash version of native-image, --no-server is implied (and can not be passed) output_directory = dirname(output_file) native_image_command += [ '-H:Path=' + output_directory or ".", @@ -1292,10 +1291,14 @@ def _write_ln(s): _write_ln(u'ImagePath=' + java_properties_escape("${.}/" + relpath(dirname(graalvm_image_destination), graalvm_location).replace(os.sep, '/'))) if requires: _write_ln(u'Requires=' + java_properties_escape(' '.join(requires), ' ', len('Requires'))) + build_with_module_path = image_config.use_modules == 'image' if isinstance(image_config, mx_sdk.LauncherConfig): _write_ln(u'ImageClass=' + java_properties_escape(image_config.main_class)) + if build_with_module_path: + _write_ln(u'ImageModule=' + java_properties_escape(image_config.main_module)) if location_classpath: - _write_ln(u'ImageClasspath=' + java_properties_escape(':'.join(("${.}/" + e.replace(os.sep, '/') for e in location_classpath)), ':', len('ImageClasspath'))) + image_path_arg = u'ImageModulePath=' if build_with_module_path else u'ImageClasspath=' + _write_ln(image_path_arg + java_properties_escape(':'.join(("${.}/" + e.replace(os.sep, '/') for e in location_classpath)), ':', len(image_path_arg))) _write_ln(u'Args=' + java_properties_escape(' '.join(build_args), ' ', len('Args'))) return self._contents @@ -1848,7 +1851,10 @@ def _get_main_class(): return self.subject.native_image_config.main_class def _is_module_launcher(): - return str(self.subject.native_image_config.module_launcher) + return str(self.subject.native_image_config.use_modules is not None) + + def _get_main_module(): + return str(self.subject.native_image_config.main_module) def _get_extra_jvm_args(): image_config = self.subject.native_image_config @@ -1885,6 +1891,7 @@ def _get_add_exports(): _template_subst.register_no_arg('classpath', _get_classpath) _template_subst.register_no_arg('jre_bin', _get_jre_bin) _template_subst.register_no_arg('main_class', _get_main_class) + _template_subst.register_no_arg('main_module', _get_main_module) _template_subst.register_no_arg('extra_jvm_args', _get_extra_jvm_args) _template_subst.register_no_arg('macro_name', GraalVmNativeProperties.macro_name(self.subject.native_image_config)) _template_subst.register_no_arg('option_vars', _get_option_vars) diff --git a/sdk/mx.sdk/vm/launcher_template.cmd b/sdk/mx.sdk/vm/launcher_template.cmd index 237295245db2..22ba14b9c559 100644 --- a/sdk/mx.sdk/vm/launcher_template.cmd +++ b/sdk/mx.sdk/vm/launcher_template.cmd @@ -78,6 +78,7 @@ set "module_launcher=" :: The list of --add-exports can easily exceed the 8191 command :: line character limit so pass them in a command line arguments file. if "%module_launcher%"=="True" ( + set "main_class=--module /" set "app_path_arg=--module-path" set exports_file="%location%.!basename!.exports" if not exist "!exports_file!" ( @@ -87,12 +88,13 @@ if "%module_launcher%"=="True" ( ) set "jvm_args=!jvm_args! @!exports_file!" ) else ( + set "main_class=" set "app_path_arg=-cp" ) if "%VERBOSE_GRAALVM_LAUNCHERS%"=="true" echo on -"%location%\java" %jvm_args% %app_path_arg% "%absolute_cp%" %launcher_args% +"%location%\java" %jvm_args% %app_path_arg% "%absolute_cp%" %main_class% %launcher_args% exit /b %errorlevel% :: Function are defined via labels, so have to be defined at the end of the file and skipped diff --git a/sdk/mx.sdk/vm/launcher_template.sh b/sdk/mx.sdk/vm/launcher_template.sh index d4f059d00f07..e3fc71777c8b 100644 --- a/sdk/mx.sdk/vm/launcher_template.sh +++ b/sdk/mx.sdk/vm/launcher_template.sh @@ -102,6 +102,7 @@ cp_or_mp="$(IFS=: ; echo "${absolute_cp[*]}")" module_launcher="" if [[ "${module_launcher}" == "True" ]]; then + main_class='--module /' app_path_arg="--module-path" IFS=" " read -ra add_exports <<< "" for e in "${add_exports[@]}"; do @@ -109,10 +110,11 @@ if [[ "${module_launcher}" == "True" ]]; then done else app_path_arg="-cp" + main_class='' fi if [[ "${VERBOSE_GRAALVM_LAUNCHERS}" == "true" ]]; then set -x fi -exec "${location}//java" "${jvm_args[@]}" ${app_path_arg} "${cp_or_mp}" '' "${launcher_args[@]}" \ No newline at end of file +exec "${location}//java" "${jvm_args[@]}" ${app_path_arg} "${cp_or_mp}" ${main_class} "${launcher_args[@]}" \ No newline at end of file diff --git a/substratevm/mx.substratevm/mx_substratevm.py b/substratevm/mx.substratevm/mx_substratevm.py index 81d6754f3490..9715e3b085c9 100644 --- a/substratevm/mx.substratevm/mx_substratevm.py +++ b/substratevm/mx.substratevm/mx_substratevm.py @@ -29,7 +29,6 @@ from __future__ import print_function import os -import time import re import tempfile from glob import glob @@ -73,18 +72,13 @@ def svm_java_compliance(): def svm_java8(): return svm_java_compliance() <= mx.JavaCompliance('1.8') -def graal_compiler_flags(version_tag=None): - version_tag = version_tag or svm_java_compliance().value - config_path = mx.dependency('substratevm:svm-compiler-flags-builder').result_file_path(version_tag) - if not exists(config_path): - missing_flags_message = ''' -Missing graal-compiler-flags config-file {0}. Possible causes: -* Forgot to run "mx build" before using SubstrateVM. -* Generating config-file for Java {1} missing in SubstrateCompilerFlagsBuilder.compute_graal_compiler_flags_map(). -''' - mx.abort(missing_flags_message.format(config_path, version_tag)) - with open(config_path, 'r') as config_file: - return config_file.read().splitlines() +def graal_compiler_flags(all_unnamed=True): + version_tag = svm_java_compliance().value + compiler_flags = mx.dependency('substratevm:svm-compiler-flags-builder').compute_graal_compiler_flags_map(all_unnamed=all_unnamed) + if version_tag not in compiler_flags: + missing_flags_message = 'Missing graal-compiler-flags for {0}.\n Did you forget to run "mx build"?' + mx.abort(missing_flags_message.format(version_tag)) + return compiler_flags[version_tag] def svm_unittest_config_participant(config): vmArgs, mainClass, mainClassArgs = config @@ -241,14 +235,10 @@ def native_image_context(common_args=None, hosted_assertions=True, native_image_ common_args = [] if common_args is None else common_args base_args = ['--no-fallback', '-H:+EnforceMaxRuntimeCompileMethods'] base_args += ['-H:Path=' + svmbuild_dir()] - has_server = mx.get_os() != 'windows' if mx.get_opts().verbose: base_args += ['--verbose'] if mx.get_opts().very_verbose: - if has_server: - base_args += ['--verbose-server'] - else: - base_args += ['--verbose'] + base_args += ['--verbose'] if hosted_assertions: base_args += native_image_context.hosted_assertions if native_image_cmd: @@ -301,30 +291,15 @@ def remove_quotes(val): return result - server_use = set() def native_image_func(args, **kwargs): all_args = base_args + common_args + args - if '--experimental-build-server' in all_args: - server_use.add(True) path = query_native_image(all_args, r'^-H:Path(@[^=]*)?=') name = query_native_image(all_args, r'^-H:Name(@[^=]*)?=') image = join(path, name) - if not has_server and '--no-server' in all_args: - all_args = [arg for arg in all_args if arg != '--no-server'] - _native_image(all_args, **kwargs) return image - try: - if exists(native_image_cmd) and has_server and server_use: - _native_image(['--server-wipe']) - yield native_image_func - finally: - if exists(native_image_cmd) and has_server and server_use: - def timestr(): - return time.strftime('%d %b %Y %H:%M:%S') + ' - ' - mx.log(timestr() + 'Shutting down image build servers for ' + native_image_cmd) - _native_image(['--server-shutdown']) - mx.log(timestr() + 'Shutting down completed') + + yield native_image_func native_image_context.hosted_assertions = ['-J-ea', '-J-esa'] _native_unittest_features = '--features=com.oracle.svm.test.ImageInfoTest$TestFeature,com.oracle.svm.test.ServiceLoaderTest$TestFeature,com.oracle.svm.test.SecurityServiceTest$TestFeature' @@ -569,7 +544,7 @@ def stderr_collector(x): def build_js(native_image): - return native_image(['--macro:js-launcher', '--no-server']) + return native_image(['--macro:js-launcher']) def test_js(js, benchmarks, bin_args=None): bench_location = join(suite.dir, '..', '..', 'js-benchmarks') @@ -854,9 +829,10 @@ def _native_image_launcher_extra_jvm_args(): support_distributions=['substratevm:NATIVE_IMAGE_GRAALVM_SUPPORT'], launcher_configs=[ mx_sdk_vm.LauncherConfig( - is_module_launcher=not svm_java8(), + use_modules='image' if USE_NI_JPMS else 'launcher' if not svm_java8() else None, destination="bin/", jar_distributions=["substratevm:SVM_DRIVER"], + main_module="org.graalvm.nativeimage.driver", main_class=_native_image_launcher_main_class(), build_args=[], extra_jvm_args=_native_image_launcher_extra_jvm_args(), @@ -1108,12 +1084,17 @@ def hellomodule(args): proj_dir = join(suite.dir, 'src', 'native-image-module-tests', 'hello.app') mx.run_maven(['-e', 'install'], cwd=proj_dir) module_path.append(join(proj_dir, 'target', 'hello-app-1.0-SNAPSHOT.jar')) - with native_image_context(hosted_assertions=False, build_if_missing=False) as native_image: + config = GraalVMConfig.build(native_images=['native-image']) + with native_image_context(hosted_assertions=False, config=config) as native_image: build_dir = join(svmbuild_dir(), 'hellomodule') # Build module into native image mx.log('Building image from java modules: ' + str(module_path)) module_path_sep = ';' if mx.is_windows() else ':' - built_image = native_image(['--verbose', '-ea', '-H:Path=' + build_dir, '-p', module_path_sep.join(module_path), '-m', 'moduletests.hello.app']) + built_image = native_image([ + '--verbose', '-ea', '-H:Path=' + build_dir, + '--add-exports=moduletests.hello.lib/hello.privateLib=moduletests.hello.app', + '--add-exports=moduletests.hello.lib/hello.privateLib2=moduletests.hello.app', + '-p', module_path_sep.join(module_path), '-m', 'moduletests.hello.app']) mx.log('Running image ' + built_image + ' built from module:') mx.run([built_image]) @@ -1424,7 +1405,7 @@ def config_file_update(self, file_path, lines, file_paths): # If renaming or moving this method, please update the error message in # com.oracle.svm.driver.NativeImage.BuildConfiguration.getBuilderJavaArgs(). - def compute_graal_compiler_flags_map(self): + def compute_graal_compiler_flags_map(self, all_unnamed=not USE_NI_JPMS): graal_compiler_flags_map = dict() graal_compiler_flags_map[8] = [ '-d64', @@ -1441,7 +1422,7 @@ def compute_graal_compiler_flags_map(self): distributions_transitive = mx.classpath_entries(self.deps) jdk = mx.get_jdk(tag='default') required_exports = mx_javamodules.requiredExports(distributions_transitive, jdk) - target_module = None if USE_NI_JPMS else 'ALL-UNNAMED' + target_module = 'ALL-UNNAMED' if all_unnamed else None exports_flags = mx_sdk_vm.AbstractNativeImageConfig.get_add_exports_list(required_exports, target_module) graal_compiler_flags_map[11].extend(exports_flags) diff --git a/substratevm/mx.substratevm/rebuild-images.sh b/substratevm/mx.substratevm/rebuild-images.sh index 2736f38fba84..6a7892179722 100755 --- a/substratevm/mx.substratevm/rebuild-images.sh +++ b/substratevm/mx.substratevm/rebuild-images.sh @@ -76,10 +76,6 @@ fi function common() { cmd_line+=("${graalvm_home}/bin/native-image") - if $(${graalvm_home}/bin/native-image --help-extra | grep -q "\-\-no\-server"); then - cmd_line+=("--no-server") - fi - if [[ -f "${graalvm_home}/lib/svm/builder/svm-enterprise.jar" ]]; then cmd_line+=("-g") fi diff --git a/substratevm/mx.substratevm/suite.py b/substratevm/mx.substratevm/suite.py index a6bc1877007e..2f7c09b4e67f 100644 --- a/substratevm/mx.substratevm/suite.py +++ b/substratevm/mx.substratevm/suite.py @@ -399,6 +399,7 @@ "requires" : ["java.instrument"], "requiresConcealed" : { "jdk.internal.vm.ci": ["jdk.vm.ci.meta"], + "java.base" : ["jdk.internal.module"], }, "javaCompliance": "11+", "checkstyle" : "com.oracle.svm.hosted", @@ -449,7 +450,7 @@ }, "": { "": { - "cflags": ["-g", "-fPIC", "-O2", "-D_LITTLE_ENDIAN", "-ffunction-sections", "-fdata-sections", "-fvisibility=hidden", "-D_FORTIFY_SOURCE=0"], + "cflags": ["-g", "-gdwarf-4", "-fPIC", "-O2", "-D_LITTLE_ENDIAN", "-ffunction-sections", "-fdata-sections", "-fvisibility=hidden", "-D_FORTIFY_SOURCE=0"], }, }, }, @@ -485,7 +486,7 @@ }, "linux": { "" : { - "cflags": ["-g", "-fPIC", "-O2", "-ffunction-sections", "-fdata-sections", "-fvisibility=hidden", "-D_FORTIFY_SOURCE=0"], + "cflags": ["-g", "-gdwarf-4", "-fPIC", "-O2", "-ffunction-sections", "-fdata-sections", "-fvisibility=hidden", "-D_FORTIFY_SOURCE=0"], }, }, "": { @@ -559,6 +560,25 @@ "spotbugs": "false", }, + "com.oracle.svm.driver.jdk11": { + "subDir": "src", + "sourceDirs": [ + "src", + ], + "dependencies": [ + "com.oracle.svm.driver", + ], + "checkstyle": "com.oracle.svm.driver", + "workingSets": "SVM", + "annotationProcessors": [ + "compiler:GRAAL_PROCESSOR", + ], + "javaCompliance" : "11+", + "multiReleaseJarVersion": "11", + "overlayTarget" : "com.oracle.svm.driver", + "spotbugs": "false", + }, + "svm-compiler-flags-builder": { "class" : "SubstrateCompilerFlagsBuilder", "dependencies" : [ @@ -1021,7 +1041,6 @@ "name" : "org.graalvm.nativeimage.builder", "exports" : [ "com.oracle.svm.hosted to java.base", - "com.oracle.svm.hosted.server to java.base", "com.oracle.svm.hosted.agent to java.instrument", "com.oracle.svm.core.graal.thread to jdk.internal.vm.compiler", "com.oracle.svm.core.classinitialization to jdk.internal.vm.compiler", @@ -1145,6 +1164,8 @@ "name" : "org.graalvm.nativeimage.objectfile", "exports" : [ "com.oracle.objectfile", + "com.oracle.objectfile.io", + "com.oracle.objectfile.debuginfo", ], "requiresConcealed" : { "java.base" : [ @@ -1216,9 +1237,9 @@ "org.graalvm.compiler.options.OptionDescriptors", ], "requires" : [ + "org.graalvm.nativeimage.builder", "java.management", "jdk.management", - # Already defined static/optional in LIBRARY_SUPPORT. Apparently that is not enough. "static com.oracle.mxtool.junit", "static junit", "static hamcrest", diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java index 03d3283cfa6a..6cca86f12e1c 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java @@ -487,9 +487,6 @@ public static Path getDebugInfoSourceCacheRoot() { @Option(help = "Omit generation of DebugLineInfo originating from inlined methods") // public static final HostedOptionKey OmitInlinedMethodDebugLineInfo = new HostedOptionKey<>(true); - /** Command line option to disable image build server. */ - public static final String NO_SERVER = "--no-server"; - @Fold public static boolean supportCompileInIsolates() { UserError.guarantee(!ConcealedOptions.SupportCompileInIsolates.getValue() || SpawnIsolates.getValue(), diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/TimeZoneSubstitutions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/TimeZoneSubstitutions.java index 47b6d283421b..aac20a99471b 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/TimeZoneSubstitutions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/TimeZoneSubstitutions.java @@ -24,19 +24,16 @@ */ package com.oracle.svm.core.jdk; -import com.oracle.svm.core.LibCHelper; -import com.oracle.svm.core.OS; -import com.oracle.svm.core.annotate.Alias; -import com.oracle.svm.core.annotate.AutomaticFeature; -import com.oracle.svm.core.annotate.RecomputeFieldValue; -import com.oracle.svm.core.annotate.Substitute; -import com.oracle.svm.core.util.VMError; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.ZoneId; +import java.util.TimeZone; + import org.graalvm.collections.EconomicMap; import org.graalvm.compiler.options.Option; import org.graalvm.compiler.options.OptionKey; - -import com.oracle.svm.core.annotate.TargetClass; -import com.oracle.svm.core.option.HostedOptionKey; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.PinnedObject; import org.graalvm.nativeimage.c.type.CCharPointer; @@ -44,11 +41,15 @@ import org.graalvm.nativeimage.hosted.Feature; import org.graalvm.word.WordFactory; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.TimeZone; +import com.oracle.svm.core.LibCHelper; +import com.oracle.svm.core.OS; +import com.oracle.svm.core.annotate.Alias; +import com.oracle.svm.core.annotate.AutomaticFeature; +import com.oracle.svm.core.annotate.RecomputeFieldValue; +import com.oracle.svm.core.annotate.Substitute; +import com.oracle.svm.core.annotate.TargetClass; +import com.oracle.svm.core.option.HostedOptionKey; +import com.oracle.svm.core.util.VMError; /** * The following classes aim to provide full support for time zones for native-image. This @@ -182,6 +183,29 @@ public void afterRegistration(AfterRegistrationAccess access) { "of your JDK usually found: " + tzMappingsPath.toAbsolutePath(), e); } } + + @Override + public void beforeAnalysis(BeforeAnalysisAccess access) { + ensureStableTzdbZoneRulesProviderProvideRules(); + } + + private static void ensureStableTzdbZoneRulesProviderProvideRules() { + /** + * Workaround to ensure stable configuration in TzdbZoneRulesProvider.provideRules + * ConcurrentHashMap prior to analysis. Calling {@link ZoneId#systemDefault()} causes + * {@link java.time.zone.TzdbZoneRulesProvider#provideRules(String, boolean)} to be called + * for the default time-zone. This will modify the map entry for the default time-zone to + * contain a ZoneRules instance instead of the binary data form of that ZoneRules instance + * (see {@code Ser.read(dis)} in {@code provideRules}). Without forcing this entry to exist + * in the map prior to analysis it is very likely that the map will see this update later + * during the image build (e.g. as a side effect of using ZipFileSystem somewhere along + * image building, see {@link jdk.nio.zipfs.ZipUtils#dosToJavaTime(long)}) causing the + * intermittently created ZoneRule instance to be reported via + * {@link com.oracle.svm.hosted.image.NativeImageHeap#reportIllegalType} whenever the + * TzdbZoneRulesProvider ends up being in the image (true for most non-trivial images). + */ + ZoneId.systemDefault(); + } } class TimeZoneSubstitutions { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/APIOption.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/APIOption.java index bfd24c603637..86908d9edf44 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/APIOption.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/APIOption.java @@ -56,6 +56,11 @@ */ String[] name(); + /** + * This option should only be shown with --help-extra. + */ + boolean extra() default false; + /** * Make a boolean option part of a group of boolean options. **/ diff --git a/substratevm/src/com.oracle.svm.driver.jdk11/src/com/oracle/svm/driver/ModuleAccess.java b/substratevm/src/com.oracle.svm.driver.jdk11/src/com/oracle/svm/driver/ModuleAccess.java new file mode 100644 index 000000000000..b55263bfc28c --- /dev/null +++ b/substratevm/src/com.oracle.svm.driver.jdk11/src/com/oracle/svm/driver/ModuleAccess.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2021, 2021, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.driver; + +import java.lang.module.ModuleDescriptor; +import java.lang.module.ModuleFinder; +import java.lang.module.ModuleReference; +import java.nio.file.Path; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +public class ModuleAccess { + @SuppressWarnings("unused") + public static List getModuleNames(Path... modulePathEntries) { + Set moduleReferences = ModuleFinder.of(modulePathEntries).findAll(); + return moduleReferences.stream() + .map(ModuleReference::descriptor) + .map(ModuleDescriptor::name) + .collect(Collectors.toList()); + } +} diff --git a/substratevm/src/com.oracle.svm.driver/resources/HelpExtra.txt b/substratevm/src/com.oracle.svm.driver/resources/HelpExtra.txt index 4a4068054c6b..875667c6ba7f 100644 --- a/substratevm/src/com.oracle.svm.driver/resources/HelpExtra.txt +++ b/substratevm/src/com.oracle.svm.driver/resources/HelpExtra.txt @@ -17,6 +17,3 @@ Non-standard options help: -V= provide values for placeholders in native-image.properties files - --help-experimental-build-server - displays help for experimental image-build server use - diff --git a/substratevm/src/com.oracle.svm.driver/resources/HelpServer.txt b/substratevm/src/com.oracle.svm.driver/resources/HelpServer.txt deleted file mode 100644 index 33c540fc3a01..000000000000 --- a/substratevm/src/com.oracle.svm.driver/resources/HelpServer.txt +++ /dev/null @@ -1,18 +0,0 @@ -Server options help: - - --no-server do not use experimental image-build server (default) - --experimental-build-server - use experimental image-build server - - --server-list lists current experimental image-build servers - --server-list-details same as -server-list with more details - --server-cleanup remove stale experimental image-build servers entries - --server-shutdown shutdown experimental image-build servers under current session ID - --server-shutdown-all shutdown all experimental image-build servers - - --server-session= - use custom session name instead of system provided - session ID of the calling process - - --verbose-server enable verbose output for experimental image-build server handling - \ No newline at end of file diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/APIOptionHandler.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/APIOptionHandler.java index 6f8d4fa3418b..b29b41f800cb 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/APIOptionHandler.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/APIOptionHandler.java @@ -70,12 +70,13 @@ static final class OptionInfo { final boolean hasPathArguments; final boolean defaultFinal; final String deprecationWarning; + final boolean extra; final List> valueTransformers; final APIOptionGroup group; OptionInfo(String[] variants, char valueSeparator, String builderOption, String defaultValue, String helpText, boolean hasPathArguments, boolean defaultFinal, String deprecationWarning, - List> valueTransformers, APIOptionGroup group) { + List> valueTransformers, APIOptionGroup group, boolean extra) { this.variants = variants; this.valueSeparator = valueSeparator; this.builderOption = builderOption; @@ -86,6 +87,7 @@ static final class OptionInfo { this.deprecationWarning = deprecationWarning; this.valueTransformers = valueTransformers; this.group = group; + this.extra = extra; } boolean isDeprecated() { @@ -115,8 +117,9 @@ static SortedMap extractOptions(List apiOptions = new TreeMap<>(); Map> groupDefaults = new HashMap<>(); - hostedOptions.getValues().forEach(o -> extractOption(NativeImage.oH, o, apiOptions, groupDefaults)); - runtimeOptions.getValues().forEach(o -> extractOption(NativeImage.oR, o, apiOptions, groupDefaults)); + Map, APIOptionGroup> groupInstances = new HashMap<>(); + hostedOptions.getValues().forEach(o -> extractOption(NativeImage.oH, o, apiOptions, groupDefaults, groupInstances)); + runtimeOptions.getValues().forEach(o -> extractOption(NativeImage.oR, o, apiOptions, groupDefaults, groupInstances)); groupDefaults.forEach((groupName, defaults) -> { if (defaults.size() > 1) { VMError.shouldNotReachHere(String.format("APIOptionGroup %s must only have a single default (but has: %s)", @@ -126,8 +129,8 @@ static SortedMap extractOptions(List apiOptions, Map> groupDefaults) { + private static void extractOption(String optionPrefix, OptionDescriptor optionDescriptor, SortedMap apiOptions, + Map> groupDefaults, Map, APIOptionGroup> groupInstances) { try { Field optionField = optionDescriptor.getDeclaringClass().getDeclaredField(optionDescriptor.getFieldName()); APIOption[] apiAnnotations = optionField.getAnnotationsByType(APIOption.class); @@ -152,7 +155,7 @@ private static void extractOption(String optionPrefix, OptionDescriptor optionDe if (!apiAnnotation.group().equals(APIOption.NullGroup.class)) { try { Class groupClass = apiAnnotation.group(); - group = ReflectionUtil.newInstance(groupClass); + group = groupInstances.computeIfAbsent(groupClass, ReflectionUtil::newInstance); String groupName = APIOption.Utils.groupName(group); if (group.helpText() == null || group.helpText().isEmpty()) { VMError.shouldNotReachHere(String.format("APIOptionGroup %s(%s) needs to provide help text", groupClass.getName(), group.name())); @@ -233,7 +236,7 @@ private static void extractOption(String optionPrefix, OptionDescriptor optionDe apiOptions.put(apiOptionName, new APIOptionHandler.OptionInfo(apiAnnotation.name(), apiAnnotation.valueSeparator(), builderOption, defaultValue, helpText, apiAnnotation.kind().equals(APIOptionKind.Paths), - booleanOption || apiAnnotation.fixedValue().length > 0, apiAnnotation.deprecated(), valueTransformers, group)); + booleanOption || apiAnnotation.fixedValue().length > 0, apiAnnotation.deprecated(), valueTransformers, group, apiAnnotation.extra())); } } catch (NoSuchFieldException e) { /* Does not qualify as APIOption */ @@ -322,10 +325,10 @@ private String tryCanonicalize(String path) { } } - void printOptions(Consumer println) { + void printOptions(Consumer println, boolean extra) { SortedMap> optionInfo = new TreeMap<>(); apiOptions.forEach((optionName, option) -> { - if (option.isDeprecated()) { + if (option.isDeprecated() || option.extra != extra) { return; } String groupOrOptionName = option.group != null ? APIOption.Utils.groupName(option.group) : optionName; diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/DefaultOptionHandler.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/DefaultOptionHandler.java index db171960ddbc..07037e38c03a 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/DefaultOptionHandler.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/DefaultOptionHandler.java @@ -33,7 +33,6 @@ import org.graalvm.compiler.options.OptionType; -import com.oracle.svm.core.SubstrateOptions; import com.oracle.svm.driver.MacroOption.MacroOptionKind; import com.oracle.svm.driver.NativeImage.ArgumentQueue; @@ -45,7 +44,9 @@ class DefaultOptionHandler extends NativeImage.OptionHandler { static final String helpText = NativeImage.getResource("/Help.txt"); static final String helpExtraText = NativeImage.getResource("/HelpExtra.txt"); - static final String noServerOption = SubstrateOptions.NO_SERVER; + + /* Defunct legacy options that we have to accept to maintain backward compatibility */ + static final String noServerOption = "--no-server"; static final String verboseServerOption = "--verbose-server"; static final String serverOptionPrefix = "--server-"; @@ -72,7 +73,7 @@ public boolean consume(ArgumentQueue args) { singleArgumentCheck(args, headArg); nativeImage.showMessage(helpText); nativeImage.showNewline(); - nativeImage.apiOptionHandler.printOptions(nativeImage::showMessage); + nativeImage.apiOptionHandler.printOptions(nativeImage::showMessage, false); nativeImage.showNewline(); nativeImage.optionRegistry.showOptions(null, true, nativeImage::showMessage); nativeImage.showNewline(); @@ -95,6 +96,8 @@ public boolean consume(ArgumentQueue args) { args.poll(); singleArgumentCheck(args, headArg); nativeImage.showMessage(helpExtraText); + nativeImage.apiOptionHandler.printOptions(nativeImage::showMessage, true); + nativeImage.showNewline(); nativeImage.optionRegistry.showOptions(MacroOptionKind.Macro, true, nativeImage::showMessage); nativeImage.showNewline(); System.exit(0); @@ -264,6 +267,14 @@ public boolean consume(ArgumentQueue args) { if (headArg.startsWith(serverOptionPrefix)) { args.poll(); NativeImage.showWarning("Ignoring server-mode native-image argument " + headArg + "."); + String serverOptionCommand = headArg.substring(serverOptionPrefix.length()); + if (!serverOptionCommand.startsWith("session=")) { + /* + * All but the --server-session=... option used to exit(0). We want to simulate that + * behaviour for proper backward compatibility. + */ + System.exit(0); + } return true; } return false; @@ -279,7 +290,7 @@ private void processClasspathArgs(String cpArgs) { private void processModulePathArgs(String mpArgs) { for (String mpEntry : mpArgs.split(File.pathSeparator, Integer.MAX_VALUE)) { - nativeImage.addImageModulePath(Paths.get(mpEntry)); + nativeImage.addImageModulePath(Paths.get(mpEntry), false); } } diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/MacroOptionHandler.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/MacroOptionHandler.java index c8fe9e42af55..7c7d5dff0085 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/MacroOptionHandler.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/MacroOptionHandler.java @@ -98,7 +98,7 @@ private void applyEnabled(MacroOption.EnabledOption enabledOption, String argume NativeImage.getJars(imageJarsDirectory).forEach(nativeImage::addImageClasspath); } - enabledOption.forEachPropertyValue(config, "ImageModulePath", entry -> nativeImage.addImageModulePath(ClasspathUtils.stringToClasspath(entry)), PATH_SEPARATOR_REGEX); + enabledOption.forEachPropertyValue(config, "ImageModulePath", entry -> nativeImage.addImageModulePath(ClasspathUtils.stringToClasspath(entry), true), PATH_SEPARATOR_REGEX); String imageName = enabledOption.getProperty(config, "ImageName"); if (imageName != null) { @@ -115,6 +115,11 @@ private void applyEnabled(MacroOption.EnabledOption enabledOption, String argume nativeImage.addPlainImageBuilderArg(nativeImage.oHClass + imageClass); } + String imageModule = enabledOption.getProperty(config, "ImageModule"); + if (imageModule != null) { + nativeImage.addPlainImageBuilderArg(nativeImage.oHModule + imageModule); + } + enabledOption.forEachPropertyValue(config, "JavaArgs", nativeImage::addImageBuilderJavaArgs); String origin = enabledOption.getOption().getDescription(true); if (argumentOrigin != null) { diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/server/NativeImageThreadFactory.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/ModuleAccess.java similarity index 67% rename from substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/server/NativeImageThreadFactory.java rename to substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/ModuleAccess.java index 6c7584af0394..2d9439a306b2 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/server/NativeImageThreadFactory.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/ModuleAccess.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2017, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2021, 2021, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -22,17 +22,19 @@ * or visit www.oracle.com if you need additional information or have any * questions. */ -package com.oracle.svm.hosted.server; -import java.util.concurrent.ForkJoinPool; -import java.util.concurrent.ForkJoinWorkerThread; +package com.oracle.svm.driver; -public class NativeImageThreadFactory implements ForkJoinPool.ForkJoinWorkerThreadFactory { - @Override - public ForkJoinWorkerThread newThread(ForkJoinPool pool) { - ForkJoinWorkerThread thread = new ForkJoinWorkerThread(pool) { - }; - thread.setContextClassLoader(NativeImageBuildServer.class.getClassLoader()); - return thread; +import java.nio.file.Path; +import java.util.List; + +import org.graalvm.compiler.serviceprovider.JavaVersionUtil; + +public class ModuleAccess { + @SuppressWarnings("unused") + public static List getModuleNames(Path... modulePathEntries) { + assert JavaVersionUtil.JAVA_SPEC <= 8; + /* For Java 8 this method does not have any effect */ + return null; } } diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index e72553992425..aeeb01eb207b 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -58,6 +58,7 @@ import java.util.Map; import java.util.Optional; import java.util.Properties; +import java.util.Scanner; import java.util.function.BiConsumer; import java.util.function.BiFunction; import java.util.function.Consumer; @@ -393,8 +394,9 @@ default List getBuilderJavaArgs() { command.add(getJavaExecutable().toString()); command.add("-XX:+PrintFlagsFinal"); command.add("-version"); + Process process = null; try { - Process process = pb.start(); + process = pb.start(); try (java.util.Scanner inputScanner = new java.util.Scanner(process.getInputStream())) { while (inputScanner.hasNextLine()) { String line = inputScanner.nextLine(); @@ -408,9 +410,12 @@ default List getBuilderJavaArgs() { } } process.waitFor(); - process.destroy(); } catch (Exception e) { /* Probing fails silently */ + } finally { + if (process != null) { + process.destroy(); + } } } @@ -844,7 +849,7 @@ private void prepareImageBuildArgs() { if (config.useJavaModules()) { String modulePath = config.getBuilderModulePath().stream() - .map(p -> canonicalize(p).toString()) + .map(this::canonicalize).map(Path::toString) .collect(Collectors.joining(File.pathSeparator)); if (!modulePath.isEmpty()) { addImageBuilderJavaArgs(Arrays.asList("--module-path", modulePath)); @@ -1502,18 +1507,24 @@ protected int buildImage(List javaArgs, LinkedHashSet bcp, LinkedH } int exitStatus = 1; + + Process p = null; try { ProcessBuilder pb = new ProcessBuilder(); pb.command(command); - Process p = pb.inheritIO().start(); + p = pb.inheritIO().start(); exitStatus = p.waitFor(); } catch (IOException | InterruptedException e) { throw showError(e.getMessage()); + } finally { + if (p != null) { + p.destroy(); + } } return exitStatus; } - private static final Function defaultNativeImageProvider = config -> IS_AOT ? NativeImageServer.create(config) : new NativeImage(config); + private static final Function defaultNativeImageProvider = config -> new NativeImage(config); public static void main(String[] args) { performBuild(new DefaultBuildConfiguration(Arrays.asList(args)), defaultNativeImageProvider); @@ -1686,15 +1697,83 @@ void addImageClasspath(Path classpath) { addImageClasspathEntry(imageClasspath, classpath, true); } - /** - * For adding classpath elements that are automatically put on the image-classpath. - */ - void addImageModulePath(Path mpEntry) { - if (!imageModulePath.contains(mpEntry)) { - imageModulePath.add(mpEntry); - /* TODO Also collect native-image properties from modules */ - /* processClasspathNativeImageMetaInf(mpEntry); */ + LinkedHashSet builderModuleNames = null; + + LinkedHashSet getBuilderModuleNames() { + if (builderModuleNames == null) { + ProcessBuilder pb = new ProcessBuilder(); + List command = pb.command(); + command.add(config.getJavaExecutable().toString()); + command.add("--module-path"); + command.add(Stream.concat(config.getBuilderModulePath().stream(), config.getImageProvidedModulePath().stream()) + .map(this::canonicalize).map(Path::toString) + .collect(Collectors.joining(File.pathSeparator))); + command.add("--list-modules"); + pb.redirectErrorStream(); + Process process = null; + int exitValue = -1; + LinkedHashSet modules = new LinkedHashSet<>(); + String message = "Unable to determine image builder modules"; + try { + process = pb.start(); + try (Scanner scanner = new Scanner(process.getInputStream())) { + while (scanner.hasNextLine()) { + String token = scanner.next(); + modules.add(token.split("@", 2)[0]); + scanner.nextLine(); + } + } + exitValue = process.waitFor(); + } catch (IOException | InterruptedException e) { + throw showError(message, e); + } finally { + if (process != null) { + process.destroy(); + } + if (exitValue != 0) { + throw showError(message + " (nonzero exit value)"); + } + } + + builderModuleNames = modules; } + return builderModuleNames; + } + + void addImageModulePath(Path modulePathEntry, boolean strict) { + Path mpEntry; + try { + mpEntry = canonicalize(modulePathEntry); + } catch (NativeImageError e) { + if (strict) { + throw e; + } + + if (isVerbose()) { + showWarning("Invalid module-path entry: " + modulePathEntry); + } + /* Allow non-existent module-path entries to comply with `java` command behaviour. */ + imageModulePath.add(canonicalize(modulePathEntry, false)); + return; + } + + if (imageModulePath.contains(mpEntry)) { + /* Duplicate entries are silently ignored like with the java command */ + return; + } + List moduleNames = ModuleAccess.getModuleNames(mpEntry); + if (moduleNames.size() == 1) { + String moduleName = moduleNames.get(0); + if (getBuilderModuleNames().contains(moduleName)) { + /* Modules provided by the image-builder are not allowed on the -imagemp */ + String modulePathEntryStr = "module " + moduleName + " from " + modulePathEntry; + showWarning("Ignoring " + modulePathEntryStr + " (implicitly provided by builder)"); + return; + } + } + + imageModulePath.add(mpEntry); + processClasspathNativeImageMetaInf(mpEntry); } /** diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImageServer.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImageServer.java deleted file mode 100644 index 2737fa431423..000000000000 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImageServer.java +++ /dev/null @@ -1,833 +0,0 @@ -/* - * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Oracle designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ -package com.oracle.svm.driver; - -import java.io.BufferedReader; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.OutputStream; -import java.nio.channels.FileChannel; -import java.nio.channels.FileLock; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.StandardOpenOption; -import java.nio.file.attribute.BasicFileAttributes; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.time.Instant; -import java.time.temporal.ChronoUnit; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Properties; -import java.util.concurrent.TimeUnit; -import java.util.function.Consumer; -import java.util.stream.Collectors; - -import org.graalvm.nativeimage.ProcessProperties; -import org.graalvm.word.WordFactory; - -import com.oracle.svm.core.SubstrateUtil; -import com.oracle.svm.core.option.SubstrateOptionsParser; -import com.oracle.svm.core.util.ClasspathUtils; -import com.oracle.svm.hosted.server.NativeImageBuildClient; -import com.oracle.svm.hosted.server.NativeImageBuildServer; -import com.oracle.svm.hosted.server.SubstrateServerMessage.ServerCommand; - -final class NativeImageServer extends NativeImage { - - private static final String serverDirPrefix = "server-id-"; - private static final String machineDirPrefix = "machine-id-"; - private static final String sessionDirPrefix = "session-id-"; - private static final String defaultLockFileName = ".lock"; - - private static final String pKeyMaxServers = "MaxServers"; - private static final String machineProperties = "machine.properties"; - - private boolean useServer = false; - private boolean verboseServer = false; - private String sessionName = null; - - private volatile Server building = null; - private final List openFileChannels = new ArrayList<>(); - - private final ServerOptionHandler serverOptionHandler; - - private NativeImageServer(BuildConfiguration buildConfiguration) { - super(buildConfiguration); - serverOptionHandler = new ServerOptionHandler(this); - registerOptionHandler(serverOptionHandler); - } - - static NativeImage create(BuildConfiguration config) { - if (NativeImageServerHelper.isInConfiguration()) { - return new NativeImageServer(config); - } - return new NativeImage(config); - } - - @SuppressWarnings("serial") - private static final class ServerInstanceError extends RuntimeException { - } - - private final class Server { - private static final String serverProperties = "server.properties"; - private static final String buildRequestLog = "build-request.log"; - - private static final String pKeyPort = "Port"; - private static final String pKeyPID = "PID"; - private static final String pKeyJavaArgs = "JavaArgs"; - private static final String pKeyBCP = "BootClasspath"; - private static final String pKeyCP = "Classpath"; - - final Path serverDir; - final Instant since; - Instant lastBuildRequest; - final int port; - final int pid; - final LinkedHashSet serverJavaArgs; - final LinkedHashSet serverBootClasspath; - final LinkedHashSet serverClasspath; - - private Server(Path serverDir) throws Exception { - this.serverDir = serverDir; - Path serverPropertiesPath = serverDir.resolve(serverProperties); - Map properties = loadProperties(serverPropertiesPath); - this.pid = Integer.parseInt(properties.get(pKeyPID)); - this.port = Integer.parseInt(properties.get(pKeyPort)); - if (this.port == 0) { - ProcessProperties.destroyForcibly(this.pid); - deleteAllFiles(this.serverDir); - throw new ServerInstanceError(); - } - this.serverJavaArgs = new LinkedHashSet<>(Arrays.asList(properties.get(pKeyJavaArgs).split(" "))); - this.serverBootClasspath = readClasspath(properties.get(pKeyBCP)); - this.serverClasspath = readClasspath(properties.get(pKeyCP)); - BasicFileAttributes sinceAttrs = Files.readAttributes(serverPropertiesPath, BasicFileAttributes.class); - this.since = sinceAttrs.creationTime().toInstant(); - updateLastBuildRequest(); - } - - private void updateLastBuildRequest() throws IOException { - Path buildRequestLogPath = serverDir.resolve(buildRequestLog); - if (Files.isReadable(buildRequestLogPath)) { - BasicFileAttributes buildAttrs = Files.readAttributes(buildRequestLogPath, BasicFileAttributes.class); - lastBuildRequest = buildAttrs.lastModifiedTime().toInstant().plusSeconds(1); - } else { - lastBuildRequest = since; - } - } - - private LinkedHashSet readClasspath(String rawClasspathString) { - LinkedHashSet result = new LinkedHashSet<>(); - for (String pathStr : rawClasspathString.split(" ")) { - result.add(ClasspathUtils.stringToClasspath(pathStr)); - } - return result; - } - - private int sendRequest(Consumer out, Consumer err, ServerCommand serverCommand, String... args) { - List argList = Arrays.asList(args); - showVerboseMessage(verboseServer, "Sending to server ["); - showVerboseMessage(verboseServer, serverCommand.toString()); - if (argList.size() > 0) { - showVerboseMessage(verboseServer, String.join(" \\\n", argList)); - } - showVerboseMessage(verboseServer, "]"); - int exitCode = NativeImageBuildClient.sendRequest(serverCommand, String.join("\n", argList).getBytes(), port, out, err); - showVerboseMessage(verboseServer, "Server returns: " + exitCode); - return exitCode; - } - - int sendBuildRequest(LinkedHashSet imageCP, LinkedHashSet imagemp, ArrayList imageArgs) { - int[] requestStatus = {1}; - withLockDirFileChannel(serverDir, lockFileChannel -> { - boolean abortedOnce = false; - boolean finished = false; - while (!finished) { - try (FileLock lock = lockFileChannel.tryLock()) { - if (lock != null) { - finished = true; - if (abortedOnce) { - showMessage("DONE."); - } - } else { - /* Cancel strategy */ - if (!abortedOnce) { - showMessagePart("A previous build is in progress. Aborting previous build..."); - abortTask(); - abortedOnce = true; - } - try { - Thread.sleep(3_000); - } catch (InterruptedException e) { - throw showError("Woke up from waiting for previous build to abort", e); - } - continue; - } - - /* Now we have the server-lock and can send the build-request */ - List command = new ArrayList<>(); - command.add(NativeImageBuildServer.TASK_PREFIX + "com.oracle.svm.hosted.NativeImageGeneratorRunner"); - - LinkedHashSet imagecp = new LinkedHashSet<>(serverClasspath); - imagecp.addAll(imageCP); - command.addAll(createImageBuilderArgs(imageArgs, imagecp, imagemp)); - - showVerboseMessage(isVerbose(), "SendBuildRequest ["); - showVerboseMessage(isVerbose(), String.join("\n", command)); - showVerboseMessage(isVerbose(), "]"); - try { - /* logfile main purpose is to know when was the last build request */ - String logFileEntry = command.stream().collect(Collectors.joining(" ", Instant.now().toString() + ": ", "\n")); - Files.write(serverDir.resolve(buildRequestLog), logFileEntry.getBytes(), StandardOpenOption.CREATE, StandardOpenOption.APPEND); - updateLastBuildRequest(); - } catch (IOException e) { - throw showError("Could not read/write into build-request log file", e); - } - // Checkstyle: stop - requestStatus[0] = sendRequest( - byteStreamToByteConsumer(System.out), - byteStreamToByteConsumer(System.err), - ServerCommand.BUILD_IMAGE, - command.toArray(new String[command.size()])); - // Checkstyle: resume - } catch (IOException e) { - throw showError("Error while trying to lock ServerDir " + serverDir, e); - } - } - }); - return requestStatus[0]; - } - - boolean isAlive() { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - boolean alive = sendRequest(byteStreamToByteConsumer(baos), byteStreamToByteConsumer(baos), ServerCommand.GET_VERSION) == NativeImageBuildClient.EXIT_SUCCESS; - showVerboseMessage(verboseServer, "Server version response: " + new String(baos.toByteArray())); - return alive; - } - - void abortTask() { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - sendRequest(byteStreamToByteConsumer(baos), byteStreamToByteConsumer(baos), ServerCommand.ABORT_BUILD); - showVerboseMessage(verboseServer, "Server abort response:" + new String(baos.toByteArray())); - } - - synchronized void shutdown() { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - sendRequest(byteStreamToByteConsumer(baos), byteStreamToByteConsumer(baos), ServerCommand.STOP_SERVER); - showVerboseMessage(verboseServer, "Server stop response:" + new String(baos.toByteArray())); - long terminationTimeout = System.currentTimeMillis() + 20_000; - long killTimeout = terminationTimeout + 40_000; - long killedTimeout = killTimeout + 2_000; - showVerboseMessage(verboseServer, "Waiting for " + this + " to shutdown"); - /* Release port only after server stops responding to it */ - boolean sentSIGTERM = false; - boolean sentSIGKILL = false; - do { - try { - Thread.sleep(500); - } catch (InterruptedException e) { - throw showError("Woke up from waiting for " + this + " to shutdown", e); - } - long now = System.currentTimeMillis(); - if (!sentSIGTERM && terminationTimeout < now) { - showWarning(this + " keeps responding to port " + port + " even after sending STOP_SERVER"); - ProcessProperties.destroy(pid); - sentSIGTERM = true; - } else if (!sentSIGKILL && killTimeout < now) { - showWarning(this + " keeps responding to port " + port + " even after destroying"); - ProcessProperties.destroyForcibly(pid); - sentSIGKILL = true; - } else if (killedTimeout < now) { - throw showError(this + " keeps responding to port " + port + " even after destroying forcefully"); - } - } while (isAlive()); - deleteAllFiles(serverDir); - } - - String getServerInfo() { - StringBuilder sb = new StringBuilder(); - sb.append(getClass().getName()); - sb.append("\nServerDir: ").append(serverDir); - sb.append("\nRunning for: ").append(getDurationString(since)); - sb.append("\nLast build: "); - if (since.equals(lastBuildRequest)) { - sb.append("None"); - } else { - sb.append(getDurationString(lastBuildRequest)); - } - sb.append("\nPID: ").append(pid); - sb.append("\nPort: ").append(port); - sb.append("\nJavaArgs: ").append(String.join(" ", serverJavaArgs)); - sb.append("\nBootClasspath: ").append(serverBootClasspath.stream().map(Path::toString).collect(Collectors.joining(File.pathSeparator))); - sb.append("\nClasspath: ").append(serverClasspath.stream().map(Path::toString).collect(Collectors.joining(File.pathSeparator))); - return sb.append('\n').toString(); - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append(getClass().getSimpleName()); - sb.append("("); - sb.append("pid: ").append(pid); - sb.append(", "); - sb.append("port: ").append(port); - sb.append(")"); - if (since.equals(lastBuildRequest)) { - /* Subtle way to show that nothing was built yet */ - sb.append('*'); - } - return sb.toString(); - } - - private String getLivenessInfo(boolean alive) { - if (alive) { - return " running for: " + getDurationString(since); - } else { - return " DEAD"; - } - } - - private String getLastBuildInfo() { - if (lastBuildRequest.equals(since)) { - return " "; - } else { - return " last build: " + getDurationString(lastBuildRequest); - } - } - } - - private static Consumer byteStreamToByteConsumer(OutputStream stream) { - return b -> { - try { - stream.write(b); - } catch (IOException e) { - throw new RuntimeException("Byte stream write failed."); - } - }; - } - - private String getSessionID() { - if (sessionName != null) { - return sessionDirPrefix + sessionName; - } - return sessionDirPrefix + Long.toHexString(Unistd.getsid(Math.toIntExact(ProcessProperties.getProcessID()))); - } - - private static String getMachineID() { - try { - return Files.lines(Paths.get("/etc/machine-id")).collect(Collectors.joining("", machineDirPrefix, "")); - } catch (Exception e) { - /* Fallback - involves DNS (much slower) */ - return machineDirPrefix + "hostid-" + Long.toHexString(Unistd.gethostid()); - } - } - - private Path getMachineDir() { - Path machineDir = getUserConfigDir().resolve(getMachineID()); - ensureDirectoryExists(machineDir); - return machineDir; - } - - private Path getSessionDir() { - Path sessionDir = getMachineDir().resolve((getSessionID())); - ensureDirectoryExists(sessionDir); - return sessionDir; - } - - private static String getDurationString(Instant since) { - long seconds = ChronoUnit.SECONDS.between(since, Instant.now()); - long hours = TimeUnit.SECONDS.toHours(seconds); - seconds -= TimeUnit.HOURS.toSeconds(hours); - long minutes = TimeUnit.SECONDS.toMinutes(seconds); - seconds -= TimeUnit.MINUTES.toSeconds(minutes); - return String.format("%02d:%02d:%02d", hours, minutes, seconds); - } - - @SuppressWarnings("try") - private Server getServerInstance(LinkedHashSet classpath, LinkedHashSet bootClasspath, List javaArgs) { - Server[] result = {null}; - /* Important - Creating new servers is a machine-exclusive operation */ - withFileChannel(getMachineDir().resolve("create-server.lock"), lockFileChannel -> { - try (FileLock lock = lockFileChannel(lockFileChannel)) { - /* Ensure that all dead server entries are gone before we start */ - List aliveServers = cleanupServers(false, true, true); - - /* Determine how many ports are allowed to get used for build-servers */ - String maxServersStr = loadProperties(getMachineDir().resolve(machineProperties)).get(pKeyMaxServers); - if (maxServersStr == null || maxServersStr.isEmpty()) { - maxServersStr = getUserConfigProperties().get(pKeyMaxServers); - } - int maxServers; - if (maxServersStr == null || maxServersStr.isEmpty()) { - maxServers = 2; - } else { - maxServers = Math.max(1, Integer.parseInt(maxServersStr)); - } - - /* Maximize reuse by using same VM mem-args for all server-based image builds */ - String xmxValueStr = getXmxValue(maxServers); - if (!"0".equals(xmxValueStr)) { - replaceArg(javaArgs, oXmx, xmxValueStr); - String xmsValueStr = getXmsValue(); - long xmxValue = SubstrateOptionsParser.parseLong(xmxValueStr); - long xmsValue = SubstrateOptionsParser.parseLong(xmsValueStr); - if (WordFactory.unsigned(xmsValue).aboveThan(WordFactory.unsigned(xmxValue))) { - xmsValueStr = Long.toUnsignedString(xmxValue); - } - replaceArg(javaArgs, oXms, xmsValueStr); - } - Path sessionDir = getSessionDir(); - List> builderPaths = new ArrayList<>(Arrays.asList(classpath, bootClasspath)); - if (config.useJavaModules()) { - builderPaths.addAll(Arrays.asList(config.getBuilderModulePath(), config.getBuilderUpgradeModulePath())); - } - Path javaExePath = canonicalize(config.getJavaExecutable()); - String serverUID = imageServerUID(javaExePath, javaArgs, builderPaths); - Path serverDir = sessionDir.resolve(serverDirPrefix + serverUID); - Optional reusableServer = aliveServers.stream().filter(s -> s.serverDir.equals(serverDir)).findFirst(); - if (reusableServer.isPresent()) { - Server server = reusableServer.get(); - if (!server.isAlive()) { - throw showError("Found defunct image-build server:" + server.getServerInfo()); - } - showVerboseMessage(verboseServer, "Reuse existing image-build server: " + server); - result[0] = server; - } else { - if (aliveServers.size() >= maxServers) { - /* Server limit reached */ - showVerboseMessage(verboseServer, "Image-build server limit reached -> remove least recently used"); - /* Shutdown least recently used within session */ - Server victim = findVictim(aliveServers); - /* If none found also consider servers from other sessions on machine */ - if (victim != null) { - showMessage("Shutdown " + victim); - victim.shutdown(); - } else { - showWarning("Image-build server limit exceeded. Use options --server{-list,-shutdown[-all]} to fix the problem."); - } - } - /* Instantiate new server and write properties file */ - Server server = startServer(javaExePath, serverDir, 0, classpath, bootClasspath, javaArgs); - if (server == null) { - showWarning("Creating image-build server failed. Fallback to one-shot image building ..."); - } - result[0] = server; - } - } catch (IOException e) { - throw showError("ServerInstance-creation locking failed", e); - } - }); - return result[0]; - } - - private static Server findVictim(List aliveServers) { - return aliveServers.stream() - .filter(Server::isAlive) - .min(Comparator.comparing(s -> s.lastBuildRequest)) - .orElse(null); - } - - private List getSessionDirs(boolean machineWide) { - List sessionDirs = Collections.singletonList(getSessionDir()); - if (machineWide) { - try { - sessionDirs = Files.list(getMachineDir()) - .filter(Files::isDirectory) - .filter(sessionDir -> sessionDir.getFileName().toString().startsWith(sessionDirPrefix)) - .collect(Collectors.toList()); - } catch (IOException e) { - throw showError("Accessing MachineDir " + getMachineDir() + " failed", e); - } - } - return sessionDirs; - } - - private List findServers(List sessionDirs) { - ArrayList servers = new ArrayList<>(); - for (Path sessionDir : sessionDirs) { - try { - Files.list(sessionDir).filter(Files::isDirectory).forEach(serverDir -> { - if (serverDir.getFileName().toString().startsWith(serverDirPrefix)) { - try { - servers.add(new Server(serverDir)); - } catch (Exception e) { - showVerboseMessage(verboseServer, "Found corrupt ServerDir " + serverDir); - deleteAllFiles(serverDir); - } - } - }); - } catch (IOException e) { - throw showError("Accessing SessionDir " + sessionDir + " failed", e); - } - } - return servers; - } - - void listServers(boolean machineWide, boolean details) { - List servers = findServers(getSessionDirs(machineWide)); - for (Server server : servers) { - String sessionInfo = machineWide ? "Session " + server.serverDir.getParent().getFileName() + " " : ""; - showMessage(sessionInfo + server + server.getLivenessInfo(server.isAlive()) + server.getLastBuildInfo()); - if (details) { - showMessage("Details:"); - showMessage(server.getServerInfo()); - } - } - } - - void wipeMachineDir() { - deleteAllFiles(getMachineDir()); - } - - @SuppressWarnings("try") - List cleanupServers(boolean serverShutdown, boolean machineWide, boolean quiet) { - List sessionDirs = getSessionDirs(machineWide); - - /* Cleanup nfs-mounted previously deleted sessionDirs */ - for (Path sessionDir : sessionDirs) { - if (isDeletedPath(sessionDir)) { - deleteAllFiles(sessionDir); - } - } - - List aliveServers = new ArrayList<>(); - for (Path sessionDir : sessionDirs) { - withLockDirFileChannel(sessionDir, lockFileChannel -> { - try (FileLock lock = lockFileChannel.lock()) { - List servers = findServers(Collections.singletonList(sessionDir)); - for (Server server : servers) { - boolean alive = server.isAlive(); - if (!alive || serverShutdown) { - if (!quiet) { - showMessage("Cleanup " + server + server.getLivenessInfo(alive)); - } - server.shutdown(); - } else { - aliveServers.add(server); - } - } - } catch (IOException e) { - throw showError("Locking SessionDir " + sessionDir + " failed", e); - } - }); - } - - return aliveServers; - } - - private Server startServer(Path javaExePath, Path serverDir, int serverPort, LinkedHashSet classpath, LinkedHashSet bootClasspath, List javaArgs) { - ProcessBuilder pb = new ProcessBuilder(); - pb.directory(serverDir.toFile()); - pb.redirectErrorStream(true); - List command = pb.command(); - command.add(javaExePath.toString()); - if (!bootClasspath.isEmpty()) { - command.add(bootClasspath.stream().map(Path::toString).collect(Collectors.joining(File.pathSeparator, "-Xbootclasspath/a:", ""))); - } - command.addAll(Arrays.asList("-cp", classpath.stream().map(Path::toString).collect(Collectors.joining(File.pathSeparator)))); - command.addAll(javaArgs); - // Ensure Graal logs to System.err so users can see log output during server-based building. - command.add("-Dgraal.LogFile=%e"); - command.add("com.oracle.svm.hosted.server.NativeImageBuildServer"); - command.add(NativeImageBuildServer.PORT_PREFIX + serverPort); - Path logFilePath = serverDir.resolve("server.log"); - command.add(NativeImageBuildServer.LOG_PREFIX + logFilePath); - showVerboseMessage(isVerbose(), "StartServer ["); - showVerboseMessage(isVerbose(), SubstrateUtil.getShellCommandString(command, true)); - showVerboseMessage(isVerbose(), "]"); - int childPid = NativeImageServerHelper.daemonize(() -> { - try { - ensureDirectoryExists(serverDir); - showVerboseMessage(verboseServer, "Starting new server ..."); - Process process = pb.start(); - long serverPID = ProcessProperties.getProcessID(process); - showVerboseMessage(verboseServer, "New image-build server pid: " + serverPID); - int selectedPort = serverPort; - if (selectedPort == 0) { - try (BufferedReader serverStdout = new BufferedReader(new InputStreamReader(process.getInputStream(), StandardCharsets.UTF_8))) { - String line; - int readLineTries = 60; - ArrayList lines = new ArrayList<>(readLineTries); - while ((line = serverStdout.readLine()) != null && --readLineTries > 0) { - lines.add(line); - if (line.startsWith(NativeImageBuildServer.PORT_LOG_MESSAGE_PREFIX)) { - String portStr = line.substring(NativeImageBuildServer.PORT_LOG_MESSAGE_PREFIX.length()); - try { - selectedPort = Integer.parseInt(portStr); - break; - } catch (NumberFormatException ex) { - /* Fall through */ - } - } - } - if (selectedPort == 0) { - String serverOutputMessage = ""; - if (!lines.isEmpty()) { - serverOutputMessage = "\nServer stdout/stderr:\n" + String.join("\n", lines); - } - throw showError("Could not determine port for sending image-build requests." + serverOutputMessage); - } - showVerboseMessage(verboseServer, "Image-build server selected port " + selectedPort); - } - } - writeServerFile(serverDir, selectedPort, serverPID, classpath, bootClasspath, javaArgs); - } catch (Throwable e) { - deleteAllFiles(serverDir); - throw showError("Starting image-build server instance failed", e); - } - }); - - int exitStatus = ProcessProperties.waitForProcessExit(childPid); - showVerboseMessage(verboseServer, "Exit status forked child process: " + exitStatus); - if (exitStatus == 0) { - Server server; - try { - server = new Server(serverDir); - } catch (Exception e) { - showVerboseMessage(verboseServer, "Image-build server unusable."); - /* Build without server */ - return null; - } - - for (int i = 0; i < 6; i += 1) { - /* Check if it is alive (accepts commands) */ - if (server.isAlive()) { - showVerboseMessage(verboseServer, "Image-build server found."); - return server; - } - try { - Thread.sleep(200); - } catch (InterruptedException e) { - break; - } - } - showVerboseMessage(verboseServer, "Image-build server not responding."); - server.shutdown(); - } - return null; - } - - private static void writeServerFile(Path serverDir, int port, long pid, LinkedHashSet classpath, LinkedHashSet bootClasspath, List javaArgs) throws Exception { - Properties sp = new Properties(); - sp.setProperty(Server.pKeyPort, String.valueOf(port)); - sp.setProperty(Server.pKeyPID, String.valueOf(pid)); - sp.setProperty(Server.pKeyJavaArgs, String.join(" ", javaArgs)); - sp.setProperty(Server.pKeyBCP, bootClasspath.stream().map(ClasspathUtils::classpathToString).collect(Collectors.joining(" "))); - sp.setProperty(Server.pKeyCP, classpath.stream().map(ClasspathUtils::classpathToString).collect(Collectors.joining(" "))); - Path serverPropertiesPath = serverDir.resolve(Server.serverProperties); - try (OutputStream os = Files.newOutputStream(serverPropertiesPath)) { - sp.store(os, ""); - } - } - - private void withLockDirFileChannel(Path lockDir, Consumer consumer) { - withFileChannel(lockDir.resolve(defaultLockFileName), consumer); - } - - private void withFileChannel(Path filePath, Consumer consumer) { - try { - FileChannel fileChannel; - synchronized (openFileChannels) { - fileChannel = FileChannel.open(filePath, StandardOpenOption.CREATE, StandardOpenOption.READ, StandardOpenOption.WRITE); - openFileChannels.add(fileChannel); - } - - consumer.accept(fileChannel); - - synchronized (openFileChannels) { - openFileChannels.remove(fileChannel); - fileChannel.close(); - } - } catch (IOException e) { - throw showError("Using FileChannel for " + filePath + " failed", e); - } - } - - private static FileLock lockFileChannel(FileChannel channel) throws IOException { - Thread lockWatcher = new Thread(() -> { - try { - Thread.sleep(TimeUnit.MINUTES.toMillis(10)); - try { - showWarning("Timeout while waiting for FileChannel.lock"); - /* Trigger AsynchronousCloseException in channel.lock() */ - channel.close(); - } catch (IOException e) { - throw showError("LockWatcher closing FileChannel of LockFile failed", e); - } - } catch (InterruptedException e) { - /* Sleep interrupted -> Lock acquired") */ - } - }); - lockWatcher.start(); - FileLock lock = channel.lock(); - lockWatcher.interrupt(); - return lock; - } - - private final class AbortBuildSignalHandler implements sun.misc.SignalHandler { - private int attemptCount = 0; - - @Override - public void handle(sun.misc.Signal signal) { - Server current = building; - if (attemptCount >= 3 || current == null) { - killServer(); - closeFileChannels(); - System.exit(1); - } else { - attemptCount += 1; - current.abortTask(); - } - } - - private void closeFileChannels() { - showVerboseMessage(isVerbose(), "CleanupHandler Begin"); - synchronized (openFileChannels) { - for (FileChannel fileChannel : openFileChannels) { - try { - showVerboseMessage(isVerbose(), "Closing open FileChannel: " + fileChannel); - fileChannel.close(); - } catch (Exception e) { - throw showError("Closing FileChannel failed", e); - } - } - } - showVerboseMessage(isVerbose(), "CleanupHandler End"); - } - - private void killServer() { - showVerboseMessage(isVerbose(), "Caught interrupt. Kill Server."); - Server current = building; - if (current != null) { - current.shutdown(); - } - } - } - - @Override - protected int buildImage(List javaArgs, LinkedHashSet bcp, LinkedHashSet cp, ArrayList imageArgs, LinkedHashSet imagecp, LinkedHashSet imagemp) { - boolean printFlags = imageArgs.stream().anyMatch(arg -> arg.contains(enablePrintFlags) || arg.contains(enablePrintFlagsWithExtraHelp)); - if (useServer && !printFlags && !useDebugAttach()) { - AbortBuildSignalHandler signalHandler = new AbortBuildSignalHandler(); - sun.misc.Signal.handle(new sun.misc.Signal("TERM"), signalHandler); - sun.misc.Signal.handle(new sun.misc.Signal("INT"), signalHandler); - - Server server = getServerInstance(cp, bcp, javaArgs); - if (server != null) { - showVerboseMessage(verboseServer, "\n" + server.getServerInfo() + "\n"); - /* Send image build job to server */ - showMessage("Build on " + server); - building = server; - int status = server.sendBuildRequest(imagecp, imagemp, imageArgs); - if (!server.isAlive()) { - /* If server does not respond after image-build -> cleanup */ - cleanupServers(false, false, true); - } - return status; - } - } - return super.buildImage(javaArgs, bcp, cp, imageArgs, imagecp, imagemp); - } - - private static String imageServerUID(Path javaExecutable, List vmArgs, List> builderPaths) { - MessageDigest digest; - try { - digest = MessageDigest.getInstance("SHA-512"); - } catch (NoSuchAlgorithmException e) { - throw showError("SHA-512 digest is not available", e); - } - digest.update(javaExecutable.toString().getBytes()); - for (Collection paths : builderPaths) { - for (Path path : paths) { - digest.update(path.toString().getBytes()); - updateHash(digest, path); - } - } - for (String string : vmArgs) { - digest.update(string.getBytes()); - } - - byte[] digestBytes = digest.digest(); - StringBuilder sb = new StringBuilder(digestBytes.length * 2); - for (byte b : digestBytes) { - sb.append(String.format("%02x", b & 0xff)); - } - return sb.toString(); - } - - private static void updateHash(MessageDigest md, Path pathElement) { - try { - if (!(Files.isReadable(pathElement) && pathElement.getFileName().toString().endsWith(".jar"))) { - throw showError("Build server classpath must only contain valid jar-files: " + pathElement); - } - md.update(Files.readAllBytes(pathElement)); - } catch (IOException e) { - throw showError("Problem reading classpath entries: " + e.getMessage()); - } - } - - void setUseServer(boolean val) { - useServer = val; - } - - boolean useServer() { - return useServer; - } - - @Override - protected void setDryRun(boolean val) { - super.setDryRun(val); - if (val) { - useServer = false; - } - } - - void setVerboseServer(boolean val) { - verboseServer = val; - } - - boolean verboseServer() { - return verboseServer; - } - - void setSessionName(String val) { - if (val != null && val.isEmpty()) { - throw showError("Empty string not allowed as session-name"); - } - sessionName = val; - } -} diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImageServerHelper.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImageServerHelper.java deleted file mode 100644 index 588fdd0aaabe..000000000000 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImageServerHelper.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Oracle designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ - -package com.oracle.svm.driver; - -import java.util.Arrays; -import java.util.List; - -import org.graalvm.compiler.api.replacements.Fold; -import org.graalvm.nativeimage.Platform; -import org.graalvm.nativeimage.Platforms; -import org.graalvm.nativeimage.c.CContext; -import org.graalvm.nativeimage.c.function.CFunction; - -public class NativeImageServerHelper { - @Fold - public static boolean isInConfiguration() { - return Platform.includedIn(Platform.LINUX.class) || Platform.includedIn(Platform.DARWIN.class); - } - - /* - * Ensures started server keeps running even after native-image completes. - */ - @Platforms({Platform.LINUX.class, Platform.DARWIN.class}) - static int daemonize(Runnable runnable) { - int pid = Unistd.fork(); - switch (pid) { - case 0: - break; - default: - return pid; - } - - /* The server should not get signals from the native-image during the first run. */ - Unistd.setsid(); - - runnable.run(); - System.exit(0); - return -1; - } -} - -@Platforms({Platform.LINUX.class, Platform.DARWIN.class}) -class UnistdDirectives implements CContext.Directives { - @Override - public boolean isInConfiguration() { - return NativeImageServerHelper.isInConfiguration(); - } - - @Override - public List getHeaderFiles() { - return Arrays.asList(new String[]{""}); - } - - @Override - public List getMacroDefinitions() { - return Arrays.asList("_GNU_SOURCE", "_LARGEFILE64_SOURCE"); - } -} - -@Platforms({Platform.LINUX.class, Platform.DARWIN.class}) -@CContext(UnistdDirectives.class) -class Unistd { - /** - * Create a new session with the calling process as its leader. The process group IDs of the - * session and the calling process are set to the process ID of the calling process, which is - * returned. - */ - @CFunction - public static native int setsid(); - - /** Return the session ID of the given process. */ - @CFunction - public static native int getsid(int pid); - - /** Return identifier for the current host. */ - @CFunction - public static native long gethostid(); - - /** - * Clone the calling process, creating an exact copy. Return -1 for errors, 0 to the new - * process, and the process ID of the new process to the old process. - */ - @CFunction - public static native int fork(); -} diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/ServerOptionHandler.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/ServerOptionHandler.java deleted file mode 100644 index 470bacea63db..000000000000 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/ServerOptionHandler.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Oracle designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ -package com.oracle.svm.driver; - -import java.util.List; - -import com.oracle.svm.driver.NativeImage.ArgumentQueue; - -class ServerOptionHandler extends NativeImage.OptionHandler { - - private static final String helpTextServer = NativeImage.getResource("/HelpServer.txt"); - private static final String enableServerOption = "--experimental-build-server"; - - ServerOptionHandler(NativeImageServer nativeImage) { - super(nativeImage); - } - - @Override - public boolean consume(ArgumentQueue args) { - String headArg = args.peek(); - switch (headArg) { - case "--help-experimental-build-server": - args.poll(); - nativeImage.showMessage(helpTextServer); - nativeImage.showNewline(); - System.exit(0); - return true; - case DefaultOptionHandler.noServerOption: - args.poll(); - nativeImage.setUseServer(false); - return true; - case enableServerOption: - args.poll(); - if (!nativeImage.isDryRun()) { - nativeImage.setUseServer(true); - } - return true; - case DefaultOptionHandler.verboseServerOption: - args.poll(); - nativeImage.setVerboseServer(true); - return true; - } - - if (headArg.startsWith(DefaultOptionHandler.serverOptionPrefix)) { - String optionTail = args.poll().substring(DefaultOptionHandler.serverOptionPrefix.length()); - boolean machineWide = false; - String oList = "list"; - String oCleanup = "cleanup"; - String oShutdown = "shutdown"; - String oWipe = "wipe"; - String oSession = "session="; - boolean serverCleanup = false; - boolean serverShutdown = false; - if (optionTail.startsWith(oList)) { - optionTail = optionTail.substring(oList.length()); - boolean listDetails = false; - machineWide = true; - String oDetails = "-details"; - if (optionTail.startsWith(oDetails)) { - optionTail = optionTail.substring(oDetails.length()); - listDetails = true; - } - if (optionTail.isEmpty()) { - nativeImage.listServers(machineWide, listDetails); - System.exit(0); - } - } else if (optionTail.startsWith(oCleanup)) { - optionTail = optionTail.substring(oCleanup.length()); - serverCleanup = true; - machineWide = true; - } else if (optionTail.startsWith(oShutdown)) { - optionTail = optionTail.substring(oShutdown.length()); - serverShutdown = true; - String oAll = "-all"; - if (optionTail.startsWith(oAll)) { - optionTail = optionTail.substring(oAll.length()); - machineWide = true; - } - } else if (optionTail.equals(oWipe)) { - nativeImage.wipeMachineDir(); - System.exit(0); - } else if (optionTail.startsWith(oSession)) { - nativeImage.setSessionName(optionTail.substring(oSession.length())); - return true; - } - if (optionTail.isEmpty() && (serverCleanup || serverShutdown)) { - nativeImage.cleanupServers(serverShutdown, machineWide, false); - System.exit(0); - } - NativeImage.showError("Invalid server option: " + headArg); - } - return false; - } - - @Override - void addFallbackBuildArgs(List buildArgs) { - if (!nativeImage.useServer()) { - buildArgs.add(DefaultOptionHandler.noServerOption); - } - if (nativeImage.verboseServer()) { - buildArgs.add(DefaultOptionHandler.verboseServerOption); - } - } -} diff --git a/substratevm/src/com.oracle.svm.hosted.jdk11/src/com/oracle/svm/hosted/NativeImageClassLoaderOptions.java b/substratevm/src/com.oracle.svm.hosted.jdk11/src/com/oracle/svm/hosted/NativeImageClassLoaderOptions.java new file mode 100644 index 000000000000..c2b93fade6d3 --- /dev/null +++ b/substratevm/src/com.oracle.svm.hosted.jdk11/src/com/oracle/svm/hosted/NativeImageClassLoaderOptions.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2021, 2021, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.hosted; + +import org.graalvm.compiler.options.Option; + +import com.oracle.svm.core.option.APIOption; +import com.oracle.svm.core.option.HostedOptionKey; +import com.oracle.svm.core.option.LocatableMultiOptionValue; + +public class NativeImageClassLoaderOptions { + static final String AddExportsAndOpensFormat = "/=(,)*"; + + @APIOption(name = "add-exports", extra = true)// + @Option(help = "Value " + AddExportsAndOpensFormat + " updates to export to , regardless of module declaration." + + " can be ALL-UNNAMED to export to all unnamed modules.")// + public static final HostedOptionKey AddExports = new HostedOptionKey<>(new LocatableMultiOptionValue.Strings()); + + @APIOption(name = "add-opens", extra = true)// + @Option(help = "Value " + AddExportsAndOpensFormat + " updates to open to , regardless of module declaration.")// + public static final HostedOptionKey AddOpens = new HostedOptionKey<>(new LocatableMultiOptionValue.Strings()); +} diff --git a/substratevm/src/com.oracle.svm.hosted.jdk11/src/com/oracle/svm/hosted/NativeImageClassLoaderSupport.java b/substratevm/src/com.oracle.svm.hosted.jdk11/src/com/oracle/svm/hosted/NativeImageClassLoaderSupport.java index 6a4dd65c9728..bed2e489d569 100644 --- a/substratevm/src/com.oracle.svm.hosted.jdk11/src/com/oracle/svm/hosted/NativeImageClassLoaderSupport.java +++ b/substratevm/src/com.oracle.svm.hosted.jdk11/src/com/oracle/svm/hosted/NativeImageClassLoaderSupport.java @@ -26,30 +26,39 @@ import java.io.File; import java.lang.module.Configuration; +import java.lang.module.ModuleDescriptor; import java.lang.module.ModuleFinder; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.concurrent.ForkJoinPool; -import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; +import org.graalvm.collections.Pair; +import org.graalvm.compiler.options.OptionValues; + +import com.oracle.svm.core.option.LocatableMultiOptionValue; +import com.oracle.svm.core.option.SubstrateOptionsParser; import com.oracle.svm.core.util.UserError; +import com.oracle.svm.core.util.VMError; import com.oracle.svm.util.ModuleSupport; +import jdk.internal.module.Modules; + public class NativeImageClassLoaderSupport extends AbstractNativeImageClassLoaderSupport { private final List imagemp; private final List buildmp; private final ClassLoader classLoader; - private final Function> moduleFinder; - private final ModuleLayer.Controller moduleController; + private final ModuleLayer moduleLayerForImageBuild; NativeImageClassLoaderSupport(ClassLoader defaultSystemClassLoader, String[] classpath, String[] modulePath) { super(defaultSystemClassLoader, classpath); @@ -57,22 +66,63 @@ public class NativeImageClassLoaderSupport extends AbstractNativeImageClassLoade imagemp = Arrays.stream(modulePath).map(Paths::get).collect(Collectors.toUnmodifiableList()); buildmp = Arrays.stream(System.getProperty("jdk.module.path", "").split(File.pathSeparator)).map(Paths::get).collect(Collectors.toUnmodifiableList()); - moduleController = createModuleController(imagemp.toArray(Path[]::new), classPathClassLoader); - ModuleLayer moduleLayer = moduleController.layer(); - moduleFinder = moduleLayer::findModule; + ModuleLayer moduleLayer = createModuleLayer(imagemp.toArray(Path[]::new), classPathClassLoader); if (moduleLayer.modules().isEmpty()) { + this.moduleLayerForImageBuild = null; classLoader = classPathClassLoader; } else { - classLoader = moduleLayer.modules().iterator().next().getClassLoader(); + adjustBootLayerQualifiedExports(moduleLayer); + this.moduleLayerForImageBuild = moduleLayer; + classLoader = getSingleClassloader(moduleLayer); } } - private static ModuleLayer.Controller createModuleController(Path[] modulePaths, ClassLoader parent) { + private static ModuleLayer createModuleLayer(Path[] modulePaths, ClassLoader parent) { ModuleFinder finder = ModuleFinder.of(modulePaths); List parents = List.of(ModuleLayer.boot().configuration()); Set moduleNames = finder.findAll().stream().map(moduleReference -> moduleReference.descriptor().name()).collect(Collectors.toSet()); Configuration configuration = Configuration.resolve(finder, parents, finder, moduleNames); - return ModuleLayer.defineModulesWithOneLoader(configuration, List.of(ModuleLayer.boot()), parent); + /** + * For the modules we want to build an image for, a ModuleLayer is needed that can be + * accessed with a single classloader so we can use it for {@link ImageClassLoader}. + */ + return ModuleLayer.defineModulesWithOneLoader(configuration, List.of(ModuleLayer.boot()), parent).layer(); + } + + private void adjustBootLayerQualifiedExports(ModuleLayer layer) { + /* + * For all qualified exports packages of modules in the the boot layer we check if layer + * contains modules that satisfy such qualified exports. If we find a match we perform a + * addExports. + */ + for (Module module : ModuleLayer.boot().modules()) { + for (ModuleDescriptor.Exports export : module.getDescriptor().exports()) { + for (String target : export.targets()) { + Optional optExportTargetModule = layer.findModule(target); + if (optExportTargetModule.isEmpty()) { + continue; + } + Module exportTargetModule = optExportTargetModule.get(); + if (module.isExported(export.source(), exportTargetModule)) { + continue; + } + Modules.addExports(module, export.source(), exportTargetModule); + } + } + } + } + + private static ClassLoader getSingleClassloader(ModuleLayer moduleLayer) { + ClassLoader singleClassloader = null; + for (Module module : moduleLayer.modules()) { + ClassLoader moduleClassLoader = module.getClassLoader(); + if (singleClassloader == null) { + singleClassloader = moduleClassLoader; + } else { + VMError.guarantee(singleClassloader == moduleClassLoader); + } + } + return singleClassloader; } @Override @@ -86,8 +136,93 @@ List applicationModulePath() { } @Override - public Optional findModule(String moduleName) { - return Optional.ofNullable(moduleFinder).flatMap(f -> f.apply(moduleName)); + public Optional findModule(String moduleName) { + if (moduleLayerForImageBuild == null) { + return Optional.empty(); + } + return moduleLayerForImageBuild.findModule(moduleName); + } + + @Override + void processAddExportsAndAddOpens(OptionValues parsedHostedOptions) { + LocatableMultiOptionValue.Strings addExports = NativeImageClassLoaderOptions.AddExports.getValue(parsedHostedOptions); + addExports.getValuesWithOrigins().map(this::asAddExportsAndOpensFormatValue).forEach(val -> { + if (val.targetModules.isEmpty()) { + Modules.addExportsToAllUnnamed(val.module, val.packageName); + } else { + for (Module targetModule : val.targetModules) { + Modules.addExports(val.module, val.packageName, targetModule); + } + } + }); + LocatableMultiOptionValue.Strings addOpens = NativeImageClassLoaderOptions.AddOpens.getValue(parsedHostedOptions); + addOpens.getValuesWithOrigins().map(this::asAddExportsAndOpensFormatValue).forEach(val -> { + if (val.targetModules.isEmpty()) { + Modules.addOpensToAllUnnamed(val.module, val.packageName); + } else { + for (Module targetModule : val.targetModules) { + Modules.addOpens(val.module, val.packageName, targetModule); + } + } + }); + } + + private static final class AddExportsAndOpensFormatValue { + private final Module module; + private final String packageName; + private final List targetModules; + + private AddExportsAndOpensFormatValue(Module module, String packageName, List targetModules) { + this.module = module; + this.packageName = packageName; + this.targetModules = targetModules; + } + } + + private AddExportsAndOpensFormatValue asAddExportsAndOpensFormatValue(Pair valueOrigin) { + String optionOrigin = valueOrigin.getRight(); + String optionValue = valueOrigin.getLeft(); + + String syntaxErrorMessage = " Allowed value format: " + NativeImageClassLoaderOptions.AddExportsAndOpensFormat; + + String[] modulePackageAndTargetModules = optionValue.split("=", 2); + if (modulePackageAndTargetModules.length != 2) { + throw userErrorAddExportsAndOpens(optionOrigin, optionValue, syntaxErrorMessage); + } + String modulePackage = modulePackageAndTargetModules[0]; + String targetModuleNames = modulePackageAndTargetModules[1]; + + String[] moduleAndPackage = modulePackage.split("/"); + if (moduleAndPackage.length != 2) { + throw userErrorAddExportsAndOpens(optionOrigin, optionValue, syntaxErrorMessage); + } + String moduleName = moduleAndPackage[0]; + String packageName = moduleAndPackage[1]; + + List targetModuleNamesList = Arrays.asList(targetModuleNames.split(",")); + if (targetModuleNamesList.isEmpty()) { + throw userErrorAddExportsAndOpens(optionOrigin, optionValue, syntaxErrorMessage); + } + + Module module = findModule(moduleName).orElseThrow(() -> { + return userErrorAddExportsAndOpens(optionOrigin, optionValue, " Specified module '" + moduleName + "' is unknown."); + }); + List targetModules; + if (targetModuleNamesList.contains("ALL-UNNAMED")) { + targetModules = Collections.emptyList(); + } else { + targetModules = targetModuleNamesList.stream().map(mn -> { + return findModule(mn).orElseThrow(() -> { + throw userErrorAddExportsAndOpens(optionOrigin, optionValue, " Specified target-module '" + mn + "' is unknown."); + }); + }).collect(Collectors.toList()); + } + return new AddExportsAndOpensFormatValue(module, packageName, targetModules); + } + + private static UserError.UserException userErrorAddExportsAndOpens(String origin, String value, String detailMessage) { + Objects.requireNonNull(detailMessage, "missing detailMessage"); + return UserError.abort("Invalid option %s provided by %s." + detailMessage, SubstrateOptionsParser.commandArgument(NativeImageClassLoaderOptions.AddExports, value), origin); } @Override diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/AbstractNativeImageClassLoaderSupport.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/AbstractNativeImageClassLoaderSupport.java index 03eb016d6457..a891896cbb9d 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/AbstractNativeImageClassLoaderSupport.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/AbstractNativeImageClassLoaderSupport.java @@ -55,6 +55,8 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import org.graalvm.compiler.options.OptionValues; + import com.oracle.svm.core.SubstrateOptions; import com.oracle.svm.core.util.ClasspathUtils; import com.oracle.svm.core.util.InterruptImageBuilding; @@ -109,7 +111,9 @@ ClassLoader getClassLoader() { abstract List applicationModulePath(); - abstract Optional findModule(String moduleName); + abstract Optional findModule(String moduleName); + + abstract void processAddExportsAndAddOpens(OptionValues parsedHostedOptions); abstract void initAllClasses(ForkJoinPool executor, ImageClassLoader imageClassLoader); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ImageBuildTask.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ImageBuildTask.java deleted file mode 100644 index f40dd5a1390b..000000000000 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ImageBuildTask.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Oracle designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ -package com.oracle.svm.hosted; - -import com.oracle.svm.hosted.server.NativeImageBuildServer; - -/** - * Interface for executing SVM image building inside the SVM image build server ( - * {@link NativeImageBuildServer}). - */ -public interface ImageBuildTask { - - /** - * Main function for remote image building which is invoked on every image building request sent - * to the server. - * - * API NOTE: Standard error and standard output for image building will be reported remotely - * only if printed within this method. After {@code build} finishes, the static state of the JDK - * and {@link NativeImageBuildServer} must not have pointers to classes loaded by - * {@code compilationClassLoader}. - * - * @param args arguments passed with the request to the SVM image builder - * @param imageClassLoader the classloader used for this image building task - * @return exit status of compilation - * @see NativeImageBuildServer - */ - int build(String[] args, ImageClassLoader imageClassLoader); - - /** - * Requests interruption of the image build. - */ - void interruptBuild(); - -} diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ImageClassLoader.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ImageClassLoader.java index cabaf5246d60..322eea7660a9 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ImageClassLoader.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ImageClassLoader.java @@ -44,6 +44,7 @@ import java.util.stream.StreamSupport; import org.graalvm.collections.EconomicSet; +import org.graalvm.compiler.options.OptionValues; import org.graalvm.compiler.word.Word; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; @@ -420,6 +421,10 @@ public Class loadClassFromModule(Object module, String className) throws Clas public Optional findModule(String moduleName) { return classLoaderSupport.findModule(moduleName); } + + public void processAddExportsAndAddOpens(OptionValues parsedHostedOptions) { + classLoaderSupport.processAddExportsAndAddOpens(parsedHostedOptions); + } } class ClassLoaderQueryImpl implements ClassLoaderQuery { diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageClassLoaderSupport.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageClassLoaderSupport.java index d88bca85ccb0..a9f209bcbb76 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageClassLoaderSupport.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageClassLoaderSupport.java @@ -30,6 +30,8 @@ import java.util.Optional; import java.util.concurrent.ForkJoinPool; +import org.graalvm.compiler.options.OptionValues; + public class NativeImageClassLoaderSupport extends AbstractNativeImageClassLoaderSupport { NativeImageClassLoaderSupport(ClassLoader defaultSystemClassLoader, String[] classpath, @SuppressWarnings("unused") String[] modulePath) { @@ -51,6 +53,11 @@ public Optional findModule(String moduleName) { return Optional.empty(); } + @Override + void processAddExportsAndAddOpens(OptionValues parsedHostedOptions) { + /* Nothing to do for Java 8 */ + } + @Override Class loadClassFromModule(Object module, String className) throws ClassNotFoundException { if (module != null) { diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java index 3a3e38fe30fb..df431cb0b30b 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java @@ -29,6 +29,7 @@ import static org.graalvm.compiler.replacements.StandardGraphBuilderPlugins.registerInvocationPlugins; import java.io.IOException; +import java.io.PrintWriter; import java.lang.ref.Reference; import java.lang.reflect.Method; import java.lang.reflect.Modifier; @@ -53,6 +54,7 @@ import java.util.concurrent.ForkJoinPool; import java.util.concurrent.ForkJoinWorkerThread; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Consumer; import java.util.stream.Collectors; import java.util.stream.StreamSupport; @@ -299,7 +301,6 @@ public class NativeImageGenerator { protected final ImageClassLoader loader; protected final HostedOptionProvider optionProvider; - private ForkJoinPool buildExecutor; private DeadlockWatchdog watchdog; private AnalysisUniverse aUniverse; private HostedUniverse hUniverse; @@ -310,6 +311,8 @@ public class NativeImageGenerator { private Pair mainEntryPoint; + final Map> buildArtifacts = new EnumMap<>(ArtifactType.class); + public NativeImageGenerator(ImageClassLoader loader, HostedOptionProvider optionProvider, Pair mainEntryPoint) { this.loader = loader; this.mainEntryPoint = mainEntryPoint; @@ -461,6 +464,7 @@ public void run(Map entryPoints, SubstitutionProcessor harnessSubstitutions, ForkJoinPool compilationExecutor, ForkJoinPool analysisExecutor, EconomicSet allOptionNames) { + ForkJoinPool executor = null; try { if (!buildStarted.compareAndSet(false, true)) { throw UserError.abort("An image build has already been performed with this generator."); @@ -480,9 +484,8 @@ public void run(Map entryPoints, setSystemPropertiesForImageLate(k); ImageSingletonsSupportImpl.HostedManagement.installInThread(new ImageSingletonsSupportImpl.HostedManagement()); - this.buildExecutor = createForkJoinPool(compilationExecutor.getParallelism()); + ForkJoinPool buildExecutor = executor = createForkJoinPool(compilationExecutor.getParallelism()); - Map> buildArtifacts = new EnumMap<>(ArtifactType.class); buildExecutor.submit(() -> { ImageSingletons.add(BuildArtifacts.class, (type, artifact) -> buildArtifacts.computeIfAbsent(type, t -> new ArrayList<>()).add(artifact)); ImageSingletons.add(ClassLoaderQuery.class, new ClassLoaderQueryImpl(loader.getClassLoader())); @@ -491,31 +494,20 @@ public void run(Map entryPoints, watchdog = new DeadlockWatchdog(); try (TemporaryBuildDirectoryProviderImpl tempDirectoryProvider = new TemporaryBuildDirectoryProviderImpl()) { ImageSingletons.add(TemporaryBuildDirectoryProvider.class, tempDirectoryProvider); - doRun(entryPoints, javaMainSupport, imageName, k, harnessSubstitutions, compilationExecutor, analysisExecutor); + doRun(entryPoints, javaMainSupport, imageName, k, harnessSubstitutions, compilationExecutor, analysisExecutor, buildExecutor); } finally { watchdog.close(); } }).get(); - - Path buildDir = generatedFiles(HostedOptionValues.singleton()); - ReportUtils.report("build artifacts", buildDir.resolve(imageName + ".build_artifacts.txt"), - writer -> buildArtifacts.forEach((artifactType, paths) -> { - writer.println("[" + artifactType + "]"); - if (artifactType == ArtifactType.JDK_LIB_SHIM) { - writer.println("# Note that shim JDK libraries depend on this"); - writer.println("# particular native image (including its name)"); - writer.println("# and therefore cannot be used with others."); - } - paths.stream().map(buildDir::relativize).forEach(writer::println); - writer.println(); - })); } catch (InterruptedException | CancellationException e) { System.out.println("Interrupted!"); throw new InterruptImageBuilding(e); } catch (ExecutionException e) { rethrow(e.getCause()); } finally { - shutdownBuildExecutor(); + if (executor != null) { + executor.shutdownNow(); + } } } @@ -569,14 +561,14 @@ protected void onTermination(Throwable exception) { private void doRun(Map entryPoints, JavaMainSupport javaMainSupport, String imageName, NativeImageKind k, SubstitutionProcessor harnessSubstitutions, - ForkJoinPool compilationExecutor, ForkJoinPool analysisExecutor) { + ForkJoinPool compilationExecutor, ForkJoinPool analysisExecutor, ForkJoinPool buildExecutor) { List hostedEntryPoints = new ArrayList<>(); OptionValues options = HostedOptionValues.singleton(); SnippetReflectionProvider originalSnippetReflection = GraalAccess.getOriginalSnippetReflection(); try (DebugContext debug = new Builder(options, new GraalDebugHandlersFactory(originalSnippetReflection)).build(); DebugCloseable featureCleanup = () -> featureHandler.forEachFeature(Feature::cleanup)) { - setupNativeImage(imageName, options, entryPoints, javaMainSupport, harnessSubstitutions, analysisExecutor, originalSnippetReflection, debug); + setupNativeImage(imageName, options, entryPoints, javaMainSupport, harnessSubstitutions, analysisExecutor, buildExecutor, originalSnippetReflection, debug); boolean returnAfterAnalysis = runPointsToAnalysis(imageName, options, debug); if (returnAfterAnalysis) { @@ -717,6 +709,21 @@ private void doRun(Map entryPoints, } } + void reportBuildArtifacts(String imageName) { + Path buildDir = generatedFiles(HostedOptionValues.singleton()); + Consumer writerConsumer = writer -> buildArtifacts.forEach((artifactType, paths) -> { + writer.println("[" + artifactType + "]"); + if (artifactType == BuildArtifacts.ArtifactType.JDK_LIB_SHIM) { + writer.println("# Note that shim JDK libraries depend on this"); + writer.println("# particular native image (including its name)"); + writer.println("# and therefore cannot be used with others."); + } + paths.stream().map(Path::toAbsolutePath).map(buildDir::relativize).forEach(writer::println); + writer.println(); + }); + ReportUtils.report("build artifacts", buildDir.resolve(imageName + ".build_artifacts.txt"), writerConsumer); + } + @SuppressWarnings("try") private boolean runPointsToAnalysis(String imageName, OptionValues options, DebugContext debug) { try (Indent ignored = debug.logAndIndent("run analysis")) { @@ -847,7 +854,7 @@ private boolean runPointsToAnalysis(String imageName, OptionValues options, Debu @SuppressWarnings("try") private void setupNativeImage(String imageName, OptionValues options, Map entryPoints, JavaMainSupport javaMainSupport, SubstitutionProcessor harnessSubstitutions, - ForkJoinPool analysisExecutor, SnippetReflectionProvider originalSnippetReflection, DebugContext debug) { + ForkJoinPool analysisExecutor, ForkJoinPool buildExecutor, SnippetReflectionProvider originalSnippetReflection, DebugContext debug) { try (Indent ignored = debug.logAndIndent("setup native-image builder")) { try (StopTimer ignored1 = new Timer(imageName, "setup").start()) { SubstrateTargetDescription target = createTarget(loader.platform); @@ -1130,16 +1137,6 @@ private static void recordRestrictHeapAccessCallees(Collection m ((RestrictHeapAccessCalleesImpl) ImageSingletons.lookup(RestrictHeapAccessCallees.class)).aggregateMethods(methods); } - public void interruptBuild() { - shutdownBuildExecutor(); - } - - private void shutdownBuildExecutor() { - if (buildExecutor != null) { - buildExecutor.shutdownNow(); - } - } - @Platforms(Platform.HOSTED_ONLY.class) static class SubstitutionInvocationPlugins extends InvocationPlugins { diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGeneratorRunner.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGeneratorRunner.java index dd6ddbe7e3a3..6bbc89c115e1 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGeneratorRunner.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGeneratorRunner.java @@ -43,7 +43,6 @@ import org.graalvm.collections.Pair; import org.graalvm.compiler.debug.DebugContext; -import org.graalvm.compiler.debug.DebugContext.Builder; import org.graalvm.compiler.options.OptionValues; import org.graalvm.compiler.printer.GraalDebugHandlersFactory; import org.graalvm.compiler.serviceprovider.JavaVersionUtil; @@ -79,7 +78,7 @@ import jdk.vm.ci.amd64.AMD64; import jdk.vm.ci.code.Architecture; -public class NativeImageGeneratorRunner implements ImageBuildTask { +public class NativeImageGeneratorRunner { private volatile NativeImageGenerator generator; public static final String IMAGE_BUILDER_ARG_FILE_OPTION = "--image-args-file="; @@ -246,6 +245,7 @@ private int buildImage(String[] arguments, ImageClassLoader classLoader) { if (!verifyValidJavaVersionAndPlatform()) { return 1; } + String imageName = null; Timer totalTimer = new Timer("[total]", false); ForkJoinPool analysisExecutor = null; ForkJoinPool compilationExecutor = null; @@ -267,9 +267,9 @@ private int buildImage(String[] arguments, ImageClassLoader classLoader) { * to pass the OptionValues explicitly when accessing options. */ parsedHostedOptions = new OptionValues(optionParser.getHostedValues()); - DebugContext debug = new Builder(parsedHostedOptions, new GraalDebugHandlersFactory(GraalAccess.getOriginalSnippetReflection())).build(); + DebugContext debug = new DebugContext.Builder(parsedHostedOptions, new GraalDebugHandlersFactory(GraalAccess.getOriginalSnippetReflection())).build(); - String imageName = SubstrateOptions.Name.getValue(parsedHostedOptions); + imageName = SubstrateOptions.Name.getValue(parsedHostedOptions); if (imageName.length() == 0) { throw UserError.abort("No output file name specified. Use '%s'.", SubstrateOptionsParser.commandArgument(SubstrateOptions.Name, "")); } @@ -306,6 +306,7 @@ private int buildImage(String[] arguments, ImageClassLoader classLoader) { } if (!className.isEmpty() || !moduleName.isEmpty()) { + classLoader.processAddExportsAndAddOpens(parsedHostedOptions); Method mainEntryPoint; Class mainClass; try { @@ -433,10 +434,13 @@ private int buildImage(String[] arguments, ImageClassLoader classLoader) { NativeImageGeneratorRunner.reportFatalError(e); return 1; } finally { + totalTimer.print(); + if (imageName != null && generator != null) { + generator.reportBuildArtifacts(imageName); + } NativeImageGenerator.clearSystemPropertiesForImage(); ImageSingletonsSupportImpl.HostedManagement.clearInThread(); } - totalTimer.print(); return 0; } @@ -535,19 +539,10 @@ private static void warn(String msg) { System.err.println("Warning: " + msg); } - @Override public int build(String[] args, ImageClassLoader imageClassLoader) { return buildImage(args, imageClassLoader); } - @Override - public void interruptBuild() { - final NativeImageGenerator generatorInstance = generator; - if (generatorInstance != null) { - generatorInstance.interruptBuild(); - } - } - /** * Command line entry point when running on JDK9+. This is required to dynamically export Graal * to SVM and it requires {@code --add-exports=java.base/jdk.internal.module=ALL-UNNAMED} to be diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/SVMHost.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/SVMHost.java index f8e504c2ed60..aa5bda813f24 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/SVMHost.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/SVMHost.java @@ -536,7 +536,6 @@ public void checkType(ResolvedJavaType type, AnalysisUniverse universe) { message += "This can happen when using the image build server. "; message += "To fix the issue you must reset all static state from the bootclasspath and application classpath that points to the application objects. "; message += "If the offending code is in JDK code please file a bug with GraalVM. "; - message += "As an workaround you can disable the image build server by adding " + SubstrateOptions.NO_SERVER + " to the command line. "; throw new UnsupportedFeatureException(message); } } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/phases/AnalysisGraphBuilderPhase.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/phases/AnalysisGraphBuilderPhase.java index 3bcec908f33e..264501904c61 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/phases/AnalysisGraphBuilderPhase.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/phases/AnalysisGraphBuilderPhase.java @@ -32,12 +32,14 @@ import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderConfiguration; import org.graalvm.compiler.nodes.graphbuilderconf.InlineInvokePlugin.InlineInfo; import org.graalvm.compiler.nodes.graphbuilderconf.IntrinsicContext; +import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugin; import org.graalvm.compiler.nodes.spi.CoreProviders; import org.graalvm.compiler.phases.OptimisticOptimizations; import org.graalvm.compiler.word.WordTypes; import com.oracle.graal.pointsto.meta.AnalysisMethod; import com.oracle.svm.core.SubstrateOptions; +import com.oracle.svm.util.ModuleSupport; import jdk.vm.ci.meta.JavaKind; import jdk.vm.ci.meta.ResolvedJavaMethod; @@ -69,6 +71,21 @@ protected boolean tryInvocationPlugin(InvokeKind invokeKind, ValueNode[] args, R return result; } + @Override + protected boolean applyInvocationPlugin(InvokeKind invokeKind, ValueNode[] args, ResolvedJavaMethod targetMethod, JavaKind resultType, InvocationPlugin plugin) { + Class accessingClass = plugin.getClass(); + /* + * The annotation-processor creates InvocationPlugins in classes in modules that e.g. + * use the @Fold annotation. This way InvocationPlugins can be in various classes in + * various modules. For these InvocationPlugins to do their work they need access to + * bits of graal. Thus the modules that contain such plugins need to be allowed such + * access. + */ + ModuleSupport.exportAndOpenPackageToClass("jdk.internal.vm.ci", "jdk.vm.ci.meta", false, accessingClass); + ModuleSupport.exportAndOpenPackageToClass("jdk.internal.vm.compiler", "org.graalvm.compiler.nodes", false, accessingClass); + return super.applyInvocationPlugin(invokeKind, args, targetMethod, resultType, plugin); + } + private final boolean parseOnce = SubstrateOptions.parseOnce(); @Override diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/server/NativeImageBuildClient.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/server/NativeImageBuildClient.java deleted file mode 100644 index 869896785294..000000000000 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/server/NativeImageBuildClient.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Copyright (c) 2017, 2017, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Oracle designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ -package com.oracle.svm.hosted.server; - -import static com.oracle.svm.hosted.server.NativeImageBuildServer.PORT_PREFIX; -import static com.oracle.svm.hosted.server.NativeImageBuildServer.extractArg; - -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.IOException; -import java.net.Socket; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Optional; -import java.util.function.Consumer; - -import com.oracle.svm.hosted.server.SubstrateServerMessage.ServerCommand; - -public class NativeImageBuildClient { - - private static final String COMMAND_PREFIX = "-command="; - private static final int EXIT_FAIL = -1; - public static final int EXIT_SUCCESS = 0; - - private static void usage(Consumer out) { - out.accept("Usage:"); - out.accept(String.format(" java -cp " + NativeImageBuildClient.class.getName() + " %s [%s] []", COMMAND_PREFIX, - PORT_PREFIX)); - } - - public static int run(String[] argsArray, Consumer out, Consumer err) { - Consumer outln = s -> out.accept((s + "\n").getBytes()); - final List args = new ArrayList<>(Arrays.asList(argsArray)); - if (args.size() < 1) { - usage(outln); - return EXIT_FAIL; - } else if (args.size() == 1 && (args.get(0).equals("--help"))) { - usage(outln); - return EXIT_SUCCESS; - } - - final Optional command = extractArg(args, COMMAND_PREFIX).map(arg -> arg.substring(COMMAND_PREFIX.length())); - final Optional port = NativeImageBuildServer.extractPort(args); - - if (port.isPresent() && command.isPresent()) { - ServerCommand serverCommand = ServerCommand.valueOf(command.get()); - return sendRequest(serverCommand, String.join(" ", args).getBytes(), port.get(), out, err); - } else { - usage(outln); - return EXIT_FAIL; - } - } - - public static int sendRequest(ServerCommand command, byte[] payload, int port, Consumer out, Consumer err) { - Consumer outln = s -> out.accept((s + "\n").getBytes()); - Consumer errln = s -> err.accept((s + "\n").getBytes()); - - try ( - Socket svmClient = new Socket((String) null, port); - DataOutputStream os = new DataOutputStream(svmClient.getOutputStream()); - DataInputStream is = new DataInputStream(svmClient.getInputStream())) { - - SubstrateServerMessage.send(new SubstrateServerMessage(command, payload), os); - if (ServerCommand.GET_VERSION.equals(command)) { - SubstrateServerMessage response = SubstrateServerMessage.receive(is); - if (response != null) { - outln.accept(new String(response.payload)); - } - } else { - - SubstrateServerMessage serverCommand; - while ((serverCommand = SubstrateServerMessage.receive(is)) != null) { - Consumer selectedConsumer; - switch (serverCommand.command) { - case WRITE_OUT: - selectedConsumer = out; - break; - case WRITE_ERR: - selectedConsumer = err; - break; - case SEND_STATUS: - /* Exit with exit status sent by server */ - return ByteBuffer.wrap(serverCommand.payload).getInt(); - default: - throw new RuntimeException("Invalid command sent by the image build server: " + serverCommand.command); - } - if (selectedConsumer != null) { - selectedConsumer.accept(serverCommand.payload); - } - } - /* Report failure if communication does not end with ExitStatus */ - return EXIT_FAIL; - } - } catch (IOException e) { - if (!ServerCommand.GET_VERSION.equals(command)) { - errln.accept("Could not connect to image build server running on port " + port); - errln.accept("Underlying exception: " + e); - } - return EXIT_FAIL; - } - return EXIT_SUCCESS; - } - - public static void main(String[] argsArray) { - System.exit(run(argsArray, System.out::print, System.err::print)); - } -} diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/server/NativeImageBuildServer.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/server/NativeImageBuildServer.java deleted file mode 100644 index 28730cb05568..000000000000 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/server/NativeImageBuildServer.java +++ /dev/null @@ -1,564 +0,0 @@ -/* - * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Oracle designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ -package com.oracle.svm.hosted.server; - -import static com.oracle.svm.hosted.NativeImageGeneratorRunner.verifyValidJavaVersionAndPlatform; - -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.PrintStream; -import java.lang.ref.Reference; -import java.lang.ref.ReferenceQueue; -import java.lang.reflect.Field; -import java.lang.reflect.InvocationTargetException; -import java.net.InetAddress; -import java.net.InetSocketAddress; -import java.net.ServerSocket; -import java.net.Socket; -import java.net.SocketException; -import java.net.SocketTimeoutException; -import java.nio.ByteBuffer; -import java.time.Duration; -import java.time.Instant; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.IdentityHashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Optional; -import java.util.Properties; -import java.util.ResourceBundle; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ForkJoinPool; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; -import java.util.function.Supplier; -import java.util.logging.LogManager; -import java.util.stream.Collectors; - -import org.graalvm.collections.EconomicSet; -import org.graalvm.compiler.serviceprovider.JavaVersionUtil; - -import com.oracle.svm.core.SubstrateOptions; -import com.oracle.svm.core.util.UserError; -import com.oracle.svm.core.util.VMError; -import com.oracle.svm.hosted.ImageBuildTask; -import com.oracle.svm.hosted.ImageClassLoader; -import com.oracle.svm.hosted.NativeImageGeneratorRunner; -import com.oracle.svm.hosted.server.SubstrateServerMessage.ServerCommand; -import com.oracle.svm.util.ModuleSupport; -import com.oracle.svm.util.ReflectionUtil; - -/** - * A server for SVM image building that keeps the classpath and JIT compiler code caches warm over - * consecutive runs. Each compilation is defined by an {@link ImageBuildTask}. - */ -public final class NativeImageBuildServer { - - public static final String PORT_LOG_MESSAGE_PREFIX = "Started image build server on port: "; - public static final String TASK_PREFIX = "-task="; - public static final String PORT_PREFIX = "-port="; - public static final String LOG_PREFIX = "-logFile="; - private static final int TIMEOUT_MINUTES = 240; - private static final String GRAALVM_VERSION_PROPERTY = "org.graalvm.version"; - private static final int SERVER_THREAD_POOL_SIZE = 4; - private static final int FAILED_EXIT_STATUS = -1; - - private static Set tasks = Collections.synchronizedSet(new HashSet<>()); - - private boolean terminated; - private final int port; - private PrintStream logOutput; - - /* - * This is done as System.err and System.logOutput are replaced by reference during analysis. - */ - private final StreamingServerMessageOutputStream outJSONStream = new StreamingServerMessageOutputStream(ServerCommand.WRITE_OUT, null); - private final StreamingServerMessageOutputStream errorJSONStream = new StreamingServerMessageOutputStream(ServerCommand.WRITE_ERR, null); - private final PrintStream serverStdout = new PrintStream(outJSONStream, true); - private final PrintStream serverStderr = new PrintStream(errorJSONStream, true); - - private final AtomicLong activeBuildTasks = new AtomicLong(); - private Instant lastKeepAliveAction = Instant.now(); - private ThreadPoolExecutor threadPoolExecutor; - - private NativeImageBuildServer(int port, PrintStream logOutput) { - this.port = port; - this.logOutput = logOutput; - threadPoolExecutor = new ThreadPoolExecutor(SERVER_THREAD_POOL_SIZE, SERVER_THREAD_POOL_SIZE, Long.MAX_VALUE, TimeUnit.DAYS, new LinkedBlockingQueue<>()); - - /* - * Set the right classloader in the process reaper - */ - String executorClassHolder = JavaVersionUtil.JAVA_SPEC <= 8 ? "java.lang.UNIXProcess" : "java.lang.ProcessHandleImpl"; - - withGlobalStaticField(executorClassHolder, "processReaperExecutor", f -> { - ThreadPoolExecutor executor = (ThreadPoolExecutor) f.get(null); - final ThreadFactory factory = executor.getThreadFactory(); - executor.setThreadFactory(r -> { - Thread t = factory.newThread(r); - t.setContextClassLoader(NativeImageBuildServer.class.getClassLoader()); - return t; - }); - }); - - System.setProperty("java.util.concurrent.ForkJoinPool.common.threadFactory", NativeImageThreadFactory.class.getName()); - /* initialize the default fork join pool with the application loader */ - if (ForkJoinPool.commonPool().getFactory().getClass() != NativeImageThreadFactory.class) { - throw VMError.shouldNotReachHere("Wrong thread pool factory: " + ForkJoinPool.commonPool().getFactory().getClass()); - } - } - - private void log(String commandLine, Object... args) { - logOutput.printf(commandLine, args); - logOutput.flush(); - } - - private static void printUsageAndExit() { - System.out.println("Usage:"); - System.out.println(String.format(" java -cp " + NativeImageBuildServer.class.getName() + " %s %s", PORT_PREFIX, LOG_PREFIX)); - System.exit(FAILED_EXIT_STATUS); - } - - public static void main(String[] argsArray) { - ModuleSupport.exportAndOpenAllPackagesToUnnamed("org.graalvm.truffle", false); - ModuleSupport.exportAndOpenAllPackagesToUnnamed("jdk.internal.vm.compiler", false); - ModuleSupport.exportAndOpenAllPackagesToUnnamed("com.oracle.graal.graal_enterprise", true); - ModuleSupport.exportAndOpenPackageToUnnamed("java.base", "sun.text.spi", false); - if (JavaVersionUtil.JAVA_SPEC >= 14) { - ModuleSupport.exportAndOpenPackageToUnnamed("java.base", "jdk.internal.loader", false); - } - if (JavaVersionUtil.JAVA_SPEC >= 16) { - ModuleSupport.exportAndOpenPackageToUnnamed("java.base", "sun.reflect.annotation", false); - ModuleSupport.exportAndOpenPackageToUnnamed("java.base", "sun.security.jca", false); - ModuleSupport.exportAndOpenPackageToUnnamed("jdk.jdeps", "com.sun.tools.classfile", false); - } - - if (!verifyValidJavaVersionAndPlatform()) { - System.exit(FAILED_EXIT_STATUS); - } - List args = new ArrayList<>(Arrays.asList(argsArray)); - if (args.size() < 1) { - printUsageAndExit(); - } - Optional port = extractPort(args); - if (!port.isPresent()) { - printUsageAndExit(); - } else { - Optional logFile = extractLogFile(args); - PrintStream output = System.out; - try { - if (logFile.isPresent()) { - File file = new File(logFile.get()); - if (!file.createNewFile()) { - System.err.println("The log file already exists, or could not be created."); - System.exit(FAILED_EXIT_STATUS); - } - output = new PrintStream(new FileOutputStream(file)); - } - new NativeImageBuildServer(port.get(), output).serve(); - } catch (IOException e) { - System.err.println("Starting server failed with an exception: " + e); - System.exit(FAILED_EXIT_STATUS); - } finally { - if (logFile.isPresent()) { - output.flush(); - output.close(); - } - } - } - } - - private static Optional extractLogFile(List args) { - Optional portArg = extractArg(args, LOG_PREFIX); - return portArg.map(arg -> arg.substring(LOG_PREFIX.length())); - } - - static Optional extractPort(List args) { - Optional portArg = extractArg(args, PORT_PREFIX); - try { - return portArg.map(arg -> Integer.parseInt(arg.substring(PORT_PREFIX.length()))); - } catch (Throwable ignored) { - System.err.println("error: invalid port number format"); - } - return Optional.empty(); - } - - static Optional extractArg(List args, String argPrefix) { - Optional portArg = args.stream().filter(x -> x.startsWith(argPrefix)).reduce((first, second) -> second); - args.removeIf(a -> a.startsWith(argPrefix)); - return portArg; - } - - @SuppressWarnings("InfiniteLoopStatement") - private void serve() { - threadPoolExecutor.purge(); - if (port == 0) { - log("Server selects ephemeral port\n"); - } else { - log("Try binding server to port " + port + "...\n"); - } - try (ServerSocket serverSocket = new ServerSocket()) { - serverSocket.setReuseAddress(true); - serverSocket.setSoTimeout((int) TimeUnit.MINUTES.toMillis(TIMEOUT_MINUTES)); - serverSocket.bind(new InetSocketAddress(InetAddress.getLoopbackAddress(), port)); - - /* NOTE: the following command line gets parsed externally */ - String portLogMessage = PORT_LOG_MESSAGE_PREFIX + serverSocket.getLocalPort(); - System.out.println(portLogMessage); - System.out.flush(); - log(portLogMessage); - - while (true) { - Socket socket = serverSocket.accept(); - - log("Accepted request from " + socket.getInetAddress().getHostName() + ". Queuing to position: " + threadPoolExecutor.getQueue().size() + "\n"); - threadPoolExecutor.execute(() -> { - if (!processRequest(socket)) { - closeServerSocket(serverSocket); - } - }); - } - } catch (SocketTimeoutException ste) { - log("Compilation server timed out. Shutting down...\n"); - } catch (SocketException se) { - log("Terminated: " + se.getMessage() + "\n"); - if (!terminated) { - log("Server error: " + se.getMessage() + "\n"); - } - } catch (IOException e) { - log("IOException in the socket operation.", e); - } finally { - log("Shutting down server...\n"); - try { - Thread.sleep(10000); - } catch (InterruptedException e) { - e.printStackTrace(); - } - threadPoolExecutor.shutdownNow(); - } - } - - private void closeServerSocket(ServerSocket serverSocket) { - try { - log("Terminating..."); - terminated = true; - serverSocket.close(); - } catch (IOException e) { - throw VMError.shouldNotReachHere(e); - } - } - - private boolean processRequest(Socket socket) { - try { - DataOutputStream output = new DataOutputStream(socket.getOutputStream()); - DataInputStream input = new DataInputStream(socket.getInputStream()); - try { - return processCommand(socket, SubstrateServerMessage.receive(input)); - } catch (Throwable t) { - log("Execution failed: " + t + "\n"); - t.printStackTrace(logOutput); - sendExitStatus(output, 1); - } - } catch (IOException ioe) { - log("Failed fetching the output stream."); - } finally { - closeConnection(socket); - log("Connection with the client closed.\n"); - // Remove the application class loader and save a GC on the next compilation - System.gc(); - System.runFinalization(); - System.gc(); - log("Available Memory: " + Runtime.getRuntime().freeMemory() + "\n"); - } - return true; - } - - private static void closeConnection(Socket socket) { - try { - socket.close(); - } catch (IOException e) { - throw VMError.shouldNotReachHere(e); - } - } - - private boolean processCommand(Socket socket, SubstrateServerMessage serverCommand) throws IOException { - DataOutputStream output = new DataOutputStream(socket.getOutputStream()); - switch (serverCommand.command) { - case STOP_SERVER: - log("Received 'stop' request. Shutting down server.\n"); - sendExitStatus(output, 0); - return false; - case GET_VERSION: - log("Received 'version' request. Responding with " + System.getProperty(GRAALVM_VERSION_PROPERTY) + ".\n"); - SubstrateServerMessage.send(new SubstrateServerMessage(serverCommand.command, System.getProperty(GRAALVM_VERSION_PROPERTY).getBytes()), output); - return Instant.now().isBefore(lastKeepAliveAction.plus(Duration.ofMinutes(TIMEOUT_MINUTES))); - case BUILD_IMAGE: - try { - long activeTasks = activeBuildTasks.incrementAndGet(); - if (activeTasks > 1) { - String message = "Can not build image: tasks are already running in the server.\n"; - log(message); - sendError(output, message); - sendExitStatus(output, -1); - } else { - log("Starting compilation for request:\n%s\n", serverCommand.payloadString()); - final ArrayList arguments = new ArrayList<>(Arrays.asList(serverCommand.payloadString().split("\n"))); - - errorJSONStream.writingInterrupted(false); - errorJSONStream.setOriginal(socket.getOutputStream()); - outJSONStream.writingInterrupted(false); - outJSONStream.setOriginal(socket.getOutputStream()); - - int exitStatus = withJVMContext( - serverStdout, - serverStderr, - () -> executeCompilation(arguments)); - sendExitStatus(output, exitStatus); - log("Image building completed.\n"); - - lastKeepAliveAction = Instant.now(); - } - } finally { - activeBuildTasks.decrementAndGet(); - } - return true; - case ABORT_BUILD: - log("Received 'abort' request. Interrupting all image build tasks.\n"); - /* - * Busy wait for all writing to complete, otherwise JSON messages are malformed. - */ - errorJSONStream.writingInterrupted(true); - outJSONStream.writingInterrupted(true); - - // Checkstyle: stop - // noinspection StatementWithEmptyBody - while (errorJSONStream.isWriting() || outJSONStream.isWriting()) { - } - // Checkstyle: start - - outJSONStream.flush(); - errorJSONStream.flush(); - for (ImageBuildTask task : tasks) { - threadPoolExecutor.submit(task::interruptBuild); - } - sendExitStatus(output, 0); - return true; - default: - log("Invalid command: " + serverCommand.command); - sendExitStatus(output, 1); - return true; - } - } - - private static void sendExitStatus(DataOutputStream output, int exitStatus) { - try { - SubstrateServerMessage.send(new SubstrateServerMessage(ServerCommand.SEND_STATUS, ByteBuffer.allocate(4).putInt(exitStatus).array()), output); - } catch (IOException e) { - throw VMError.shouldNotReachHere(e); - } - } - - private static void sendError(DataOutputStream output, String message) { - try { - SubstrateServerMessage.send(new SubstrateServerMessage(ServerCommand.WRITE_ERR, message.getBytes()), output); - } catch (IOException e) { - throw VMError.shouldNotReachHere(e); - } - } - - private static Integer executeCompilation(ArrayList arguments) { - String[] classpath = NativeImageGeneratorRunner.extractImagePathEntries(arguments, SubstrateOptions.IMAGE_CLASSPATH_PREFIX); - ClassLoader applicationClassLoader = Thread.currentThread().getContextClassLoader(); - try { - ImageClassLoader imageClassLoader = NativeImageGeneratorRunner.installNativeImageClassLoader(classpath, new String[0]); - ImageBuildTask task = loadCompilationTask(arguments, imageClassLoader.getClassLoader()); - try { - tasks.add(task); - return task.build(arguments.toArray(new String[0]), imageClassLoader); - } finally { - tasks.remove(task); - } - } finally { - NativeImageGeneratorRunner.uninstallNativeImageClassLoader(); - Thread.currentThread().setContextClassLoader(applicationClassLoader); - } - } - - private static int withJVMContext(PrintStream out, PrintStream err, Supplier body) { - Properties previousProperties = (Properties) System.getProperties().clone(); - PrintStream previousOut = System.out; - PrintStream previousErr = System.err; - - System.setOut(out); - System.setErr(err); - ResourceBundle.clearCache(); - try { - return body.get(); - } catch (Throwable t) { - t.printStackTrace(); - throw t; - } finally { - System.setProperties(previousProperties); - System.setOut(previousOut); - System.setErr(previousErr); - resetGlobalStateInLoggers(); - resetGlobalStateMXBeanLookup(); - resetResourceBundle(); - resetJarFileFactoryCaches(); - resetGlobalStateInGraal(); - withGlobalStaticField("java.lang.ApplicationShutdownHooks", "hooks", f -> { - @SuppressWarnings("unchecked") - IdentityHashMap hooks = (IdentityHashMap) f.get(null); - hooks.forEach((x, y) -> { - x.setContextClassLoader(NativeImageBuildServer.class.getClassLoader()); - y.setContextClassLoader(NativeImageBuildServer.class.getClassLoader()); - }); - }); - } - } - - private static void resetGlobalStateMXBeanLookup() { - withGlobalStaticField("com.sun.jmx.mbeanserver.MXBeanLookup", "currentLookup", f -> { - ThreadLocal currentLookup = (ThreadLocal) f.get(null); - currentLookup.remove(); - }); - withGlobalStaticField("com.sun.jmx.mbeanserver.MXBeanLookup", "mbscToLookup", f -> { - try { - Object mbscToLookup = f.get(null); - Map map = ReflectionUtil.readField(Class.forName("com.sun.jmx.mbeanserver.WeakIdentityHashMap"), "map", mbscToLookup); - map.clear(); - ReferenceQueue refQueue = ReflectionUtil.readField(Class.forName("com.sun.jmx.mbeanserver.WeakIdentityHashMap"), "refQueue", mbscToLookup); - Reference ref; - do { - ref = refQueue.poll(); - } while (ref != null); - } catch (ClassNotFoundException e) { - throw VMError.shouldNotReachHere(e); - } - }); - } - - private static void resetGlobalStateInLoggers() { - LogManager.getLogManager().reset(); - withGlobalStaticField("java.util.logging.Level$KnownLevel", "nameToLevels", NativeImageBuildServer::removeImageLoggers); - withGlobalStaticField("java.util.logging.Level$KnownLevel", "intToLevels", NativeImageBuildServer::removeImageLoggers); - } - - private static void removeImageLoggers(Field f) throws IllegalAccessException { - HashMap newHashMap = new HashMap<>(); - HashMap currentNameToLevels = (HashMap) f.get(null); - currentNameToLevels.entrySet().stream() - .filter(NativeImageBuildServer::isSystemLoaderLogLevelEntry) - .forEach(e -> newHashMap.put(e.getKey(), e.getValue())); - f.set(null, newHashMap); - } - - private static boolean isSystemLoaderLogLevelEntry(Entry e) { - if (JavaVersionUtil.JAVA_SPEC <= 8) { - return ((List) e.getValue()).stream() - .map(x -> getFieldValueOfObject("java.util.logging.Level$KnownLevel", "levelObject", x)) - .allMatch(NativeImageBuildServer::isSystemClassLoader); - } else { - return ((List) e.getValue()).stream() - .map(x -> getFieldValueOfObject("java.util.logging.Level$KnownLevel", "mirroredLevel", x)) - .allMatch(NativeImageBuildServer::isSystemClassLoader); - } - } - - private static Object getFieldValueOfObject(String className, String fieldName, Object o) { - try { - return ReflectionUtil.readField(Class.forName(className), fieldName, o); - } catch (ClassNotFoundException ex) { - throw VMError.shouldNotReachHere(ex); - } - } - - private static boolean isSystemClassLoader(Object obj) { - return obj.getClass().getClassLoader() == null || - obj.getClass().getClassLoader() == ClassLoader.getSystemClassLoader() || - obj.getClass().getClassLoader() == ClassLoader.getSystemClassLoader().getParent(); - } - - interface FieldAction { - void perform(Field f) throws IllegalAccessException; - } - - private static void withGlobalStaticField(String className, String fieldName, FieldAction action) { - try { - Field field = ReflectionUtil.lookupField(Class.forName(className), fieldName); - action.perform(field); - } catch (ClassNotFoundException | IllegalAccessException ex) { - throw VMError.shouldNotReachHere(ex); - } - } - - private static void resetGlobalStateInGraal() { - withGlobalStaticField("org.graalvm.compiler.nodes.NamedLocationIdentity$DB", "map", f -> ((EconomicSet) f.get(null)).clear()); - withGlobalStaticField("org.graalvm.compiler.debug.DebugContext$Immutable", "CACHE", f -> { - Object[] cache = (Object[]) f.get(null); - for (int i = 0; i < cache.length; i++) { - cache[i] = null; - } - }); - } - - private static void resetResourceBundle() { - withGlobalStaticField("java.util.ResourceBundle", "cacheList", list -> ((ConcurrentHashMap) list.get(null)).clear()); - } - - private static void resetJarFileFactoryCaches() { - withGlobalStaticField("sun.net.www.protocol.jar.JarFileFactory", "fileCache", list -> ((HashMap) list.get(null)).clear()); - withGlobalStaticField("sun.net.www.protocol.jar.JarFileFactory", "urlCache", list -> ((HashMap) list.get(null)).clear()); - } - - private static ImageBuildTask loadCompilationTask(ArrayList arguments, ClassLoader classLoader) { - Optional taskParameter = arguments.stream().filter(arg -> arg.startsWith(TASK_PREFIX)).findFirst(); - if (!taskParameter.isPresent()) { - throw UserError.abort("Image building task not specified. Provide the fully qualified task name after the \"%s\" argument.", TASK_PREFIX); - } - arguments.removeAll(arguments.stream().filter(arg -> arg.startsWith(TASK_PREFIX)).collect(Collectors.toList())); - final String task = taskParameter.get().substring(TASK_PREFIX.length()); - try { - Class imageTaskClass = Class.forName(task, true, classLoader); - return (ImageBuildTask) imageTaskClass.getDeclaredConstructor().newInstance(); - } catch (ClassNotFoundException e) { - throw UserError.abort("Image building task %1$s can not be found. Make sure that %1$s is present on the classpath.", task); - } catch (IllegalArgumentException | InstantiationException | InvocationTargetException | NoSuchMethodException | IllegalAccessException e) { - throw UserError.abort("Image building task %s must have a public constructor without parameters.", task); - } - } -} diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/server/StreamingServerMessageOutputStream.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/server/StreamingServerMessageOutputStream.java deleted file mode 100644 index 194e720e7030..000000000000 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/server/StreamingServerMessageOutputStream.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Copyright (c) 2017, 2017, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Oracle designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ -package com.oracle.svm.hosted.server; - -import java.io.DataOutputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.nio.channels.ClosedByInterruptException; - -import com.oracle.svm.hosted.server.SubstrateServerMessage.ServerCommand; - -/** - * Converts the data stream to streaming {@link SubstrateServerMessage messages} containing the - * content as well as the command that accompanies the content. Note: Should be used with - * BufferedOutputStream for better performance and with standard OutputStreams for responsiveness. - */ -public class StreamingServerMessageOutputStream extends OutputStream { - - private final ServerCommand command; - private DataOutputStream original; - private volatile boolean interrupted; - private volatile boolean writing; - - public void setOriginal(OutputStream original) { - this.original = new DataOutputStream(original); - } - - StreamingServerMessageOutputStream(ServerCommand command, OutputStream original) { - this.command = command; - this.original = new DataOutputStream(original); - } - - @Override - public void write(int b) throws IOException { - write(new byte[]{(byte) b}, 0, 1); - } - - @Override - public void write(byte[] b, int off, int len) throws IOException { - if (interrupted) { - throw new ClosedByInterruptException(); - } - writing = true; - try { - SubstrateServerMessage message = new SubstrateServerMessage(command, b, off, len); - SubstrateServerMessage.send(message, original); - } finally { - writing = false; - } - } - - @Override - public void flush() throws IOException { - original.flush(); - } - - @Override - public void close() throws IOException { - original.close(); - } - - void writingInterrupted(boolean value) { - this.interrupted = value; - } - - public boolean isWriting() { - return writing; - } -} diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/server/SubstrateServerMessage.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/server/SubstrateServerMessage.java deleted file mode 100644 index bedea1672bda..000000000000 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/server/SubstrateServerMessage.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright (c) 2017, 2017, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Oracle designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ -package com.oracle.svm.hosted.server; - -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.EOFException; -import java.io.IOException; - -public class SubstrateServerMessage { - final ServerCommand command; - final byte[] payload; - final int offset; - final int length; - - SubstrateServerMessage(ServerCommand command, byte[] payload) { - this(command, payload, 0, payload.length); - } - - SubstrateServerMessage(ServerCommand command, byte[] payload, int offset, int length) { - this.command = command; - this.payload = payload; - this.offset = offset; - this.length = length; - } - - static void send(SubstrateServerMessage message, DataOutputStream os) throws IOException { - os.writeInt(message.command.ordinal()); - os.writeInt(message.length); - os.write(message.payload, message.offset, message.length); - os.flush(); - } - - static SubstrateServerMessage receive(DataInputStream is) throws IOException { - try { - ServerCommand command = ServerCommand.values()[is.readInt()]; - int length = is.readInt(); - byte[] payload = new byte[length]; - is.readFully(payload); - return new SubstrateServerMessage(command, payload); - } catch (EOFException ex) { - return null; - } - } - - public String payloadString() { - return new String(payload); - } - - public enum ServerCommand { - GET_VERSION, - STOP_SERVER, - BUILD_IMAGE, - ABORT_BUILD, - SEND_STATUS, - WRITE_ERR, - WRITE_OUT - } -} diff --git a/substratevm/src/com.oracle.svm.util.jdk11/src/com/oracle/svm/util/ModuleSupport.java b/substratevm/src/com.oracle.svm.util.jdk11/src/com/oracle/svm/util/ModuleSupport.java index a2c25e1e44d9..546fb833ebcc 100644 --- a/substratevm/src/com.oracle.svm.util.jdk11/src/com/oracle/svm/util/ModuleSupport.java +++ b/substratevm/src/com.oracle.svm.util.jdk11/src/com/oracle/svm/util/ModuleSupport.java @@ -128,15 +128,15 @@ public static void openModuleByClass(Class declaringClass, Class accessing } /** - * Exports and opens a single package {@code packageName} in the module named {@code name} to - * all unnamed modules. + * Exports and opens a single package {@code packageName} in the module named {@code moduleName} + * to all unnamed modules. */ @SuppressWarnings("unused") - public static void exportAndOpenPackageToClass(String name, String packageName, boolean optional, Class accessingClass) { - Optional value = ModuleLayer.boot().findModule(name); + public static void exportAndOpenPackageToClass(String moduleName, String packageName, boolean optional, Class accessingClass) { + Optional value = ModuleLayer.boot().findModule(moduleName); if (value.isEmpty()) { if (!optional) { - throw new NoSuchElementException(name); + throw new NoSuchElementException(moduleName); } return; } diff --git a/substratevm/src/com.oracle.svm.util/src/com/oracle/svm/util/ModuleSupport.java b/substratevm/src/com.oracle.svm.util/src/com/oracle/svm/util/ModuleSupport.java index 1dd691e5f6ce..d8b957358cfc 100644 --- a/substratevm/src/com.oracle.svm.util/src/com/oracle/svm/util/ModuleSupport.java +++ b/substratevm/src/com.oracle.svm.util/src/com/oracle/svm/util/ModuleSupport.java @@ -83,11 +83,11 @@ public static void openModuleByClass(Class declaringClass, Class accessing } /** - * Exports and opens a single package {@code packageName} in the module named {@code name} to - * all unnamed modules. + * Exports and opens a single package {@code packageName} in the module named {@code moduleName} + * to all unnamed modules. */ @SuppressWarnings("unused") - public static void exportAndOpenPackageToClass(String name, String packageName, boolean optional, Class accessingClass) { + public static void exportAndOpenPackageToClass(String moduleName, String packageName, boolean optional, Class accessingClass) { /* Nothing to do in JDK 8 version. JDK 11 version provides a proper implementation. */ assert JavaVersionUtil.JAVA_SPEC <= 8; } diff --git a/substratevm/src/native-image-module-tests/hello.app/pom.xml b/substratevm/src/native-image-module-tests/hello.app/pom.xml index 2b4b13917e75..a80c3542f343 100644 --- a/substratevm/src/native-image-module-tests/hello.app/pom.xml +++ b/substratevm/src/native-image-module-tests/hello.app/pom.xml @@ -50,6 +50,11 @@ questions. 11 11 + + --add-exports=moduletests.hello.lib/hello.privateLib=moduletests.hello.app + --add-exports=moduletests.hello.lib/hello.privateLib2=moduletests.hello.app + --add-opens=moduletests.hello.lib/hello.privateLib2=moduletests.hello.app + diff --git a/substratevm/src/native-image-module-tests/hello.app/src/main/java/hello/Main.java b/substratevm/src/native-image-module-tests/hello.app/src/main/java/hello/Main.java index 2a718c1c714f..be8b80253545 100644 --- a/substratevm/src/native-image-module-tests/hello.app/src/main/java/hello/Main.java +++ b/substratevm/src/native-image-module-tests/hello.app/src/main/java/hello/Main.java @@ -26,8 +26,11 @@ import hello.lib.Greeter; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; + public class Main { - public static void main(String[] args) { + public static void main(String[] args) throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { Module helloAppModule = Main.class.getModule(); assert helloAppModule.getName().equals("moduletests.hello.app"); assert helloAppModule.isExported("hello"); @@ -41,5 +44,13 @@ public static void main(String[] args) { System.out.println("Basic Module test involving " + helloAppModule + " and " + helloLibModule); Greeter.greet(); + + System.out.println("Now accessing package that is not exported in " + helloLibModule); + hello.privateLib.Greeter.greet(); + + System.out.println("Now accessing private method from not exported package in " + helloLibModule); + Method greetMethod = hello.privateLib2.PrivateGreeter.class.getDeclaredMethod("greet"); + greetMethod.setAccessible(true); + greetMethod.invoke(null); } } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/server/package-info.java b/substratevm/src/native-image-module-tests/hello.lib/src/main/java/hello/privateLib/Greeter.java similarity index 79% rename from substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/server/package-info.java rename to substratevm/src/native-image-module-tests/hello.lib/src/main/java/hello/privateLib/Greeter.java index 73faedaab92c..1cf115449226 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/server/package-info.java +++ b/substratevm/src/native-image-module-tests/hello.lib/src/main/java/hello/privateLib/Greeter.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2021, 2021, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -22,10 +22,10 @@ * or visit www.oracle.com if you need additional information or have any * questions. */ +package hello.privateLib; -/* - * This package is intentionally not annotated as HOSTED_ONLY. It contains - * the server message classes that are used by the driver, i.e., by code - * distributed as a native image. - */ -package com.oracle.svm.hosted.server; +public class Greeter { + public static void greet() { + System.out.println("Seeing this requires --add-exports"); + } +} diff --git a/substratevm/src/native-image-module-tests/hello.lib/src/main/java/hello/privateLib2/PrivateGreeter.java b/substratevm/src/native-image-module-tests/hello.lib/src/main/java/hello/privateLib2/PrivateGreeter.java new file mode 100644 index 000000000000..8bba22bdf4f5 --- /dev/null +++ b/substratevm/src/native-image-module-tests/hello.lib/src/main/java/hello/privateLib2/PrivateGreeter.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2021, 2021, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package hello.privateLib2; + +public class PrivateGreeter { + private static void greet() { + System.out.println("Seeing this requires --add-opens"); + } +} diff --git a/vm/mx.vm/mx_vm_benchmark.py b/vm/mx.vm/mx_vm_benchmark.py index d9fdef163d8a..7a75d46bcc09 100644 --- a/vm/mx.vm/mx_vm_benchmark.py +++ b/vm/mx.vm/mx_vm_benchmark.py @@ -146,7 +146,7 @@ def __init__(self, vm, bm_suite, args): self.log_dir = self.output_dir self.analysis_report_path = os.path.join(self.output_dir, self.executable_name + '-analysis.json') self.base_image_build_args = [os.path.join(mx_sdk_vm_impl.graalvm_home(fatalIfMissing=True), 'bin', 'native-image')] - self.base_image_build_args += ['--no-fallback', '--no-server', '-g', '--allow-incomplete-classpath'] + self.base_image_build_args += ['--no-fallback', '-g', '--allow-incomplete-classpath'] self.base_image_build_args += ['-H:+VerifyGraalGraphs', '-H:+VerifyPhases'] if vm.is_gate else [] self.base_image_build_args += ['-J-ea', '-J-esa'] if vm.is_gate and not bm_suite.skip_build_assertions(self.benchmark_name) else [] self.base_image_build_args += self.system_properties @@ -633,8 +633,7 @@ def create_log_files(self, config, executable_name, stage): class NativeImageBuildVm(GraalVm): def run(self, cwd, args): - default_args = ['--no-server'] if mx_sdk_vm_impl.has_svm_launcher('svm') else [] - return self.run_launcher('native-image', default_args + args, cwd) + return self.run_launcher('native-image', args, cwd) class GuVm(GraalVm): diff --git a/vm/mx.vm/mx_vm_gate.py b/vm/mx.vm/mx_vm_gate.py index fd89d84f0b6d..004219b3cdd0 100644 --- a/vm/mx.vm/mx_vm_gate.py +++ b/vm/mx.vm/mx_vm_gate.py @@ -261,7 +261,6 @@ def _svm_truffle_tck(native_image, svm_suite, language_suite, language_id): '-H:+EnforceMaxRuntimeCompileMethods', '-cp', cp, - '--no-server', '-H:-FoldSecurityManagerGetter', '-H:TruffleTCKPermissionsReportFile={}'.format(report_file), '-H:Path={}'.format(svmbuild),