Skip to content

[GR-57117] Make --layer-create=package=... take package name instead of cp entry. #10291

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Dec 12, 2024
Merged

[GR-57117] Make --layer-create=package=... take package name instead of cp entry. #10291

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 10 additions & 3 deletions substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,11 +130,14 @@ protected void onValueUpdate(EconomicMap<OptionKey<?>, Object> values, Boolean o
}
});

// @APIOption(name = "layer-create")//
public static final String LAYER_OPTION_PREFIX = "-H:Layer"; // "--layer"
public static final String LAYER_CREATE_OPTION = LAYER_OPTION_PREFIX + "Create"; // "-create"
// @APIOption(name = LAYER_CREATE_OPTION) // use when non-experimental
@Option(help = "Experimental: Build a Native Image layer.")//
public static final HostedOptionKey<AccumulatingLocatableMultiOptionValue.Strings> LayerCreate = new HostedOptionKey<>(AccumulatingLocatableMultiOptionValue.Strings.build());

// @APIOption(name = "layer-use")//
// public static final String LAYER_USE_OPTION = LAYER_OPTION_PREFIX + "-use";
// @APIOption(name = LAYER_USE_OPTION) // use when non-experimental
@Option(help = "Experimental: Build an image based on a Native Image layer.")//
@BundleMember(role = Role.Input) //
public static final HostedOptionKey<AccumulatingLocatableMultiOptionValue.Paths> LayerUse = new HostedOptionKey<>(AccumulatingLocatableMultiOptionValue.Paths.build());
Expand Down Expand Up @@ -1286,11 +1289,15 @@ public enum ReportingMode {
@Option(help = "Include all classes, methods, fields, and resources from given paths", type = OptionType.Debug) //
public static final HostedOptionKey<AccumulatingLocatableMultiOptionValue.Strings> IncludeAllFromPath = new HostedOptionKey<>(AccumulatingLocatableMultiOptionValue.Strings.build());

@Option(help = "Include all classes, methods and fields from the given packages", type = OptionType.Debug) //
public static final HostedOptionKey<AccumulatingLocatableMultiOptionValue.Strings> IncludeAllFromPackage = new HostedOptionKey<>(
AccumulatingLocatableMultiOptionValue.Strings.buildWithCommaDelimiter());

@Option(help = "Include all classes, methods, fields, and resources from the class path", type = OptionType.Debug) //
public static final HostedOptionKey<Boolean> IncludeAllFromClassPath = new HostedOptionKey<>(false);

public static boolean includeAll() {
return IncludeAllFromModule.hasBeenSet() || IncludeAllFromPath.hasBeenSet() || IncludeAllFromClassPath.hasBeenSet();
return IncludeAllFromModule.hasBeenSet() || IncludeAllFromPath.hasBeenSet() || IncludeAllFromPackage.hasBeenSet() || IncludeAllFromClassPath.hasBeenSet();
}

@Option(help = "Force include include all public types and methods that can be reached using normal Java access rules.")//
Expand Down
23 changes: 23 additions & 0 deletions substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/CmdLineOptionHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,10 +31,14 @@
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;

import com.oracle.svm.core.SubstrateOptions;
import com.oracle.svm.core.VM;
import com.oracle.svm.core.option.OptionOrigin;
import com.oracle.svm.core.util.ExitStatus;
import com.oracle.svm.driver.NativeImage.ArgumentQueue;
import com.oracle.svm.hosted.imagelayer.LayerArchiveSupport;
import com.oracle.svm.hosted.imagelayer.LayerOptionsSupport.ExtendedOption;
import com.oracle.svm.hosted.imagelayer.LayerOptionsSupport.LayerOption;
import com.oracle.svm.util.LogUtils;

import jdk.graal.compiler.options.OptionType;
Expand Down Expand Up @@ -138,6 +142,25 @@ private boolean consume(ArgumentQueue args, String headArg) {
return true;
}

if (headArg.startsWith(SubstrateOptions.LAYER_CREATE_OPTION)) {
String layerCreateValue = headArg.substring(SubstrateOptions.LAYER_CREATE_OPTION.length());
if (!layerCreateValue.isEmpty()) {
LayerOption layerOption = LayerOption.parse(layerCreateValue);
for (ExtendedOption option : layerOption.extendedOptions()) {
switch (option.key()) {
case LayerArchiveSupport.PACKAGE_OPTION -> {
String packageName = option.value();
String moduleName = nativeImage.systemPackagesToModules.get(packageName);
if (moduleName != null) {
nativeImage.addAddedModules(moduleName);
}
}
}
}
}
return false;
}

if (headArg.startsWith(DEBUG_ATTACH_OPTION)) {
if (useDebugAttach) {
throw NativeImage.showError("The " + DEBUG_ATTACH_OPTION + " option can only be used once.");
Expand Down
15 changes: 14 additions & 1 deletion substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,19 @@ private static String flagsFileName(String versionTag) {

static final Map<String, String[]> graalCompilerFlags = getCompilerFlags();

private static Map<String, String> getSystemPackages() {
Map<String, String> res = new HashMap<>();
for (ModuleReference moduleRef : ModuleFinder.ofSystem().findAll()) {
ModuleDescriptor moduleDescriptor = moduleRef.descriptor();
for (String packageName : moduleDescriptor.packages()) {
res.put(packageName, moduleDescriptor.name());
}
}
return Map.copyOf(res);
}

final Map<String, String> systemPackagesToModules = getSystemPackages();

static String getResource(String resourceName) {
try (InputStream input = NativeImage.class.getResourceAsStream(resourceName)) {
BufferedReader reader = new BufferedReader(new InputStreamReader(input, StandardCharsets.UTF_8));
Expand Down Expand Up @@ -963,7 +976,7 @@ private void prepareImageBuildArgs() {

/*
* The presence of CDS and custom system class loaders disables the use of archived
* non-system class and and triggers a warning.
* non-system class and triggers a warning.
*/
addImageBuilderJavaArgs("-Xshare:off");

Expand Down
16 changes: 4 additions & 12 deletions substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ModuleLayerFeature.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -217,7 +217,7 @@ public void afterAnalysis(AfterAnalysisAccess access) {
* Parse explicitly added modules via --add-modules. This is done early as this information
* is required when filtering the analysis reachable module set.
*/
Set<String> extraModules = ModuleLayerFeatureUtils.parseModuleSetModifierProperty(ModuleSupport.PROPERTY_IMAGE_EXPLICITLY_ADDED_MODULES);
Set<String> extraModules = ModuleSupport.parseModuleSetModifierProperty(ModuleSupport.PROPERTY_IMAGE_EXPLICITLY_ADDED_MODULES);
extraModules.addAll(Resources.getIncludedResourcesModules());
extraModules.stream().filter(Predicate.not(ModuleSupport.nonExplicitModules::contains)).forEach(moduleName -> {
Optional<?> module = accessImpl.imageClassLoader.findModule(moduleName);
Expand Down Expand Up @@ -282,7 +282,7 @@ private Set<String> calculateRootModules(Collection<String> addModules) {
ModuleFinder upgradeModulePath = NativeImageClassLoaderSupport.finderFor("jdk.module.upgrade.path");
ModuleFinder appModulePath = moduleLayerFeatureUtils.getAppModuleFinder();
String mainModule = ModuleLayerFeatureUtils.getMainModuleName();
Set<String> limitModules = ModuleLayerFeatureUtils.parseModuleSetModifierProperty(ModuleSupport.PROPERTY_IMAGE_EXPLICITLY_LIMITED_MODULES);
Set<String> limitModules = ModuleSupport.parseModuleSetModifierProperty(ModuleSupport.PROPERTY_IMAGE_EXPLICITLY_LIMITED_MODULES);

Object systemModules = null;
ModuleFinder systemModuleFinder;
Expand Down Expand Up @@ -658,7 +658,8 @@ private static final class ModuleLayerFeatureUtils {
ModuleLayerFeatureUtils(ImageClassLoader cl) {
runtimeModules = new HashMap<>();
imageClassLoader = cl;
nativeAccessEnabled = NativeImageClassLoaderOptions.EnableNativeAccess.getValue().values().stream().flatMap(m -> Arrays.stream(SubstrateUtil.split(m, ",")))
nativeAccessEnabled = NativeImageClassLoaderOptions.EnableNativeAccess.getValue().values().stream()
.flatMap(m -> Arrays.stream(SubstrateUtil.split(m, ",")))
.collect(Collectors.toSet());

Method classGetDeclaredMethods0Method = ReflectionUtil.lookupMethod(Class.class, "getDeclaredFields0", boolean.class);
Expand Down Expand Up @@ -770,15 +771,6 @@ static String formatModule(Module module) {
}
}

static Set<String> parseModuleSetModifierProperty(String prop) {
Set<String> specifiedModules = new HashSet<>();
String args = System.getProperty(prop, "");
if (!args.isEmpty()) {
specifiedModules.addAll(Arrays.asList(SubstrateUtil.split(args, ",")));
}
return specifiedModules;
}

static int distanceFromBootModuleLayer(ModuleLayer layer) {
if (layer == ModuleLayer.boot()) {
return 0;
Expand Down
83 changes: 53 additions & 30 deletions ...vm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageClassLoaderSupport.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@

import static com.oracle.svm.core.SubstrateOptions.IncludeAllFromClassPath;
import static com.oracle.svm.core.SubstrateOptions.IncludeAllFromModule;
import static com.oracle.svm.core.SubstrateOptions.IncludeAllFromPackage;
import static com.oracle.svm.core.SubstrateOptions.IncludeAllFromPath;
import static com.oracle.svm.core.util.VMError.guarantee;
import static com.oracle.svm.core.util.VMError.shouldNotReachHere;
Expand Down Expand Up @@ -74,14 +75,14 @@
import java.util.stream.Collectors;
import java.util.stream.Stream;

import com.oracle.svm.core.SubstrateUtil;
import org.graalvm.collections.EconomicMap;
import org.graalvm.collections.EconomicSet;
import org.graalvm.collections.MapCursor;
import org.graalvm.nativeimage.impl.AnnotationExtractor;

import com.oracle.svm.core.NativeImageClassLoaderOptions;
import com.oracle.svm.core.SubstrateOptions;
import com.oracle.svm.core.SubstrateUtil;
import com.oracle.svm.core.option.AccumulatingLocatableMultiOptionValue;
import com.oracle.svm.core.option.HostedOptionKey;
import com.oracle.svm.core.option.LocatableMultiOptionValue.ValueWithOrigin;
Expand Down Expand Up @@ -129,13 +130,15 @@ public final class NativeImageClassLoaderSupport {
public final AnnotationExtractor annotationExtractor;

private Set<String> javaModuleNamesToInclude;
private Set<String> javaPackagesToInclude;
private Set<Path> javaPathsToInclude;
private boolean includeAllFromClassPath;

private Optional<LibGraalClassLoaderBase> libGraalLoader;
private List<ClassLoader> classLoaders;

private final Set<Class<?>> classesToIncludeUnconditionally = Collections.newSetFromMap(new ConcurrentHashMap<>());
private final Set<Class<?>> classesToIncludeUnconditionally = ConcurrentHashMap.newKeySet();
private final Set<String> includedJavaPackages = ConcurrentHashMap.newKeySet();

private final Method implAddReadsAllUnnamed = ReflectionUtil.lookupMethod(Module.class, "implAddReadsAllUnnamed");
private final Method implAddEnableNativeAccess = ReflectionUtil.lookupMethod(Module.class, "implAddEnableNativeAccess");
Expand Down Expand Up @@ -244,7 +247,7 @@ private static Path stringToPath(String path) {
}

public void loadAllClasses(ForkJoinPool executor, ImageClassLoader imageClassLoader) {
guarantee(javaModuleNamesToInclude == null, "This method should be executed only once.");
guarantee(javaModuleNamesToInclude == null && javaPackagesToInclude == null, "This method should be executed only once.");
javaModuleNamesToInclude = Collections.unmodifiableSet(new HashSet<>(IncludeAllFromModule.getValue(parsedHostedOptions).values()));
/* Verify all modules are present */
final Set<String> allModules = Stream.concat(modulepathModuleFinder.findAll().stream(), upgradeAndSystemModuleFinder.findAll().stream())
Expand All @@ -254,6 +257,8 @@ public void loadAllClasses(ForkJoinPool executor, ImageClassLoader imageClassLoa
.filter(m -> !allModules.contains(m))
.findAny().ifPresent(m -> missingFromSetOfEntriesError(m, allModules, "module-path", IncludeAllFromModule));

javaPackagesToInclude = Set.copyOf(IncludeAllFromPackage.getValue(parsedHostedOptions).values());

javaPathsToInclude = IncludeAllFromPath.getValue(parsedHostedOptions).values().stream()
.map(NativeImageClassLoaderSupport::stringToPath)
.map(Path::toAbsolutePath)
Expand Down Expand Up @@ -722,32 +727,44 @@ private void run() {
System.out.println("Total processed entries: " + entriesProcessed.longValue() + ", current entry: " + currentlyProcessedEntry);
}, 5, 1, TimeUnit.MINUTES);

List<String> requiresInit = new ArrayList<>(Arrays.asList(
"jdk.internal.vm.ci", "jdk.graal.compiler", "com.oracle.graal.graal_enterprise",
var requiresInit = new HashSet<>(List.of("jdk.internal.vm.ci", "jdk.graal.compiler", "com.oracle.graal.graal_enterprise",
"org.graalvm.nativeimage", "org.graalvm.truffle", "org.graalvm.truffle.runtime",
"org.graalvm.truffle.compiler", "com.oracle.truffle.enterprise", "org.graalvm.jniutils",
"org.graalvm.nativebridge"));

Set<String> additionalSystemModules = upgradeAndSystemModuleFinder.findAll().stream().map(v -> v.descriptor().name()).collect(Collectors.toSet());
Set<String> additionalSystemModules = upgradeAndSystemModuleFinder.findAll().stream()
.map(v -> v.descriptor().name())
.collect(Collectors.toSet());
additionalSystemModules.retainAll(getJavaModuleNamesToInclude());
requiresInit.addAll(additionalSystemModules);

Set<String> explicitlyAddedModules = ModuleSupport.parseModuleSetModifierProperty(ModuleSupport.PROPERTY_IMAGE_EXPLICITLY_ADDED_MODULES);

for (ModuleReference moduleReference : upgradeAndSystemModuleFinder.findAll()) {
if (requiresInit.contains(moduleReference.descriptor().name())) {
initModule(moduleReference);
String moduleName = moduleReference.descriptor().name();
boolean moduleRequiresInit = requiresInit.contains(moduleName);
if (moduleRequiresInit || explicitlyAddedModules.contains(moduleName)) {
initModule(moduleReference, moduleRequiresInit);
}
}
for (ModuleReference moduleReference : modulepathModuleFinder.findAll()) {
initModule(moduleReference);
initModule(moduleReference, true);
}

classpath().parallelStream().forEach(this::loadClassesFromPath);
} finally {
scheduledExecutor.shutdown();
}

/* Verify all package inclusion requests were successful */
for (String packageName : javaPackagesToInclude) {
if (!includedJavaPackages.contains(packageName)) {
missingFromSetOfEntriesError(packageName, includedJavaPackages, "package", IncludeAllFromPackage);
}
}
}

private void initModule(ModuleReference moduleReference) {
private void initModule(ModuleReference moduleReference, boolean moduleRequiresInit) {
String moduleReferenceLocation = moduleReference.location().map(URI::toString).orElse("UnknownModuleReferenceLocation");
currentlyProcessedEntry = moduleReferenceLocation;
Optional<Module> optionalModule = findModule(moduleReference.descriptor().name());
Expand All @@ -766,7 +783,7 @@ private void initModule(ModuleReference moduleReference) {
String className = extractClassName(moduleResource, fileSystemSeparatorChar);
if (className != null) {
currentlyProcessedEntry = moduleReferenceLocation + fileSystemSeparatorChar + moduleResource;
executor.execute(() -> handleClassFileName(container, module, className, includeUnconditionally));
executor.execute(() -> handleClassFileName(container, module, className, includeUnconditionally, moduleRequiresInit));
}
entriesProcessed.increment();
});
Expand Down Expand Up @@ -832,7 +849,7 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
String className = extractClassName(fileName, fileSystemSeparatorChar);
if (className != null) {
currentlyProcessedEntry = file.toUri().toString();
executor.execute(() -> handleClassFileName(container, null, className, includeUnconditionally));
executor.execute(() -> handleClassFileName(container, null, className, includeUnconditionally, true));
}
entriesProcessed.increment();
return FileVisitResult.CONTINUE;
Expand Down Expand Up @@ -916,24 +933,26 @@ private String extractClassName(String fileName, char fileSystemSeparatorChar) {
return strippedClassFileName.equals("module-info") ? null : strippedClassFileName.replace(fileSystemSeparatorChar, '.');
}

private void handleClassFileName(URI container, Module module, String className, boolean includeUnconditionally) {
synchronized (classes) {
EconomicSet<String> classNames = classes.get(container);
if (classNames == null) {
classNames = EconomicSet.create();
classes.put(container, classNames);
private void handleClassFileName(URI container, Module module, String className, boolean includeUnconditionally, boolean classRequiresInit) {
if (classRequiresInit) {
synchronized (classes) {
EconomicSet<String> classNames = classes.get(container);
if (classNames == null) {
classNames = EconomicSet.create();
classes.put(container, classNames);
}
classNames.add(className);
}
classNames.add(className);
}
int packageSep = className.lastIndexOf('.');
String packageName = packageSep > 0 ? className.substring(0, packageSep) : "";
synchronized (packages) {
EconomicSet<String> packageNames = packages.get(container);
if (packageNames == null) {
packageNames = EconomicSet.create();
packages.put(container, packageNames);
int packageSep = className.lastIndexOf('.');
String packageName = packageSep > 0 ? className.substring(0, packageSep) : "";
synchronized (packages) {
EconomicSet<String> packageNames = packages.get(container);
if (packageNames == null) {
packageNames = EconomicSet.create();
packages.put(container, packageNames);
}
packageNames.add(packageName);
}
packageNames.add(packageName);
}

Class<?> clazz = null;
Expand All @@ -945,10 +964,14 @@ private void handleClassFileName(URI container, Module module, String className,
ImageClassLoader.handleClassLoadingError(t);
}
if (clazz != null) {
if (includeUnconditionally) {
String packageName = clazz.getPackageName();
includedJavaPackages.add(packageName);
if (includeUnconditionally || javaPackagesToInclude.contains(packageName)) {
classesToIncludeUnconditionally.add(clazz);
}
imageClassLoader.handleClass(clazz);
if (classRequiresInit) {
imageClassLoader.handleClass(clazz);
}
}
imageClassLoader.watchdog.recordActivity();
}
Expand Down
Loading