[GR-41209] Module-aware Resource storage

ivan-ristovic · ivan-ristovic · commit ee790c2d3fc4 · 2023-06-01T10:40:12.000Z
PullRequest: graal/14430
diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java
@@ -35,7 +35,12 @@
 import java.util.Date;
 import java.util.Enumeration;
 import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
 
+import com.oracle.svm.core.BuildPhaseProvider;
 import org.graalvm.collections.EconomicMap;
 import org.graalvm.collections.Pair;
 import org.graalvm.nativeimage.ImageSingletons;
@@ -67,11 +72,13 @@ public static Resources singleton() {
     }
 
     /**
-     * The hosted map used to collect registered resources. Using a {@link Pair} of (moduleName,
+     * The hosted map used to collect registered resources. Using a {@link Pair} of (module,
      * resourceName) provides implementations for {@code hashCode()} and {@code equals()} needed for
-     * the map keys.
+     * the map keys. Hosted module instances differ to runtime instances, so the map that ends up in
+     * the image heap is computed after the runtime module instances have been computed {see
+     * com.oracle.svm.hosted.ModuleLayerFeature}.
      */
-    private final EconomicMap<Pair<String, String>, ResourceStorageEntry> resources = ImageHeapMap.create();
+    private final EconomicMap<Pair<Module, String>, ResourceStorageEntry> resources = ImageHeapMap.create();
 
     /**
      * Embedding a resource into an image is counted as a modification. Since all resources are
@@ -83,7 +90,7 @@ public static Resources singleton() {
     Resources() {
     }
 
-    public EconomicMap<Pair<String, String>, ResourceStorageEntry> getResourceStorage() {
+    public EconomicMap<Pair<Module, String>, ResourceStorageEntry> getResourceStorage() {
         return resources;
     }
 
@@ -103,6 +110,19 @@ public static String moduleName(Module module) {
         return module == null ? null : module.getName();
     }
 
+    private static Pair<Module, String> createStorageKey(Module module, String resourceName) {
+        Module m = module != null && module.isNamed() ? module : null;
+        return Pair.create(m, resourceName);
+    }
+
+    public static Set<String> getIncludedResourcesModules() {
+        return StreamSupport.stream(singleton().resources.getKeys().spliterator(), false)
+                        .map(Pair::getLeft)
+                        .filter(Objects::nonNull)
+                        .map(Module::getName)
+                        .collect(Collectors.toSet());
+    }
+
     public static byte[] inputStreamToByteArray(InputStream is) {
         try {
             return is.readAllBytes();
@@ -111,11 +131,16 @@ public static byte[] inputStreamToByteArray(InputStream is) {
         }
     }
 
+    @Platforms(Platform.HOSTED_ONLY.class)
     private static void addEntry(Module module, String resourceName, boolean isDirectory, byte[] data, boolean fromJar) {
-        String moduleName = moduleName(module);
+        VMError.guarantee(!BuildPhaseProvider.isAnalysisFinished(), "Trying to add a resource entry after analysis.");
+        Module m = module != null && module.isNamed() ? module : null;
+        if (m != null) {
+            m = RuntimeModuleSupport.instance().getRuntimeModuleForHostedModule(m);
+        }
         var resources = singleton().resources;
         synchronized (resources) {
-            Pair<String, String> key = Pair.create(moduleName, resourceName);
+            Pair<Module, String> key = createStorageKey(m, resourceName);
             ResourceStorageEntry entry = resources.get(key);
             if (entry == null) {
                 if (singleton().lastModifiedTime == INVALID_TIMESTAMP) {
@@ -205,8 +230,7 @@ public static ResourceStorageEntry get(String name) {
 
     public static ResourceStorageEntry get(Module module, String resourceName) {
         String canonicalResourceName = toCanonicalForm(resourceName);
-        String moduleName = moduleName(module);
-        ResourceStorageEntry entry = singleton().resources.get(Pair.create(moduleName, canonicalResourceName));
+        ResourceStorageEntry entry = singleton().resources.get(createStorageKey(module, canonicalResourceName));
         if (entry == null) {
             return null;
         }
@@ -267,7 +291,7 @@ public static InputStream createInputStream(Module module, String resourceName)
              * If module is not specified or is an unnamed module and entry was not found as
              * classpath-resource we have to search for the resource in all modules in the image.
              */
-            for (Module m : BootModuleLayerSupport.instance().getBootLayer().modules()) {
+            for (Module m : RuntimeModuleSupport.instance().getBootLayer().modules()) {
                 entry = Resources.get(m, resourceName);
                 if (entry != null) {
                     break;
@@ -296,7 +320,7 @@ public static Enumeration<URL> createURLs(Module module, String resourceName) {
         boolean shouldAppendTrailingSlash = hasTrailingSlash(resourceName);
         /* If module was unspecified or unnamed, we have to consider all modules in the image */
         if (moduleName(module) == null) {
-            for (Module m : BootModuleLayerSupport.instance().getBootLayer().modules()) {
+            for (Module m : RuntimeModuleSupport.instance().getBootLayer().modules()) {
                 ResourceStorageEntry entry = Resources.get(m, resourceName);
                 addURLEntries(resourcesURLs, entry, m, shouldAppendTrailingSlash ? canonicalResourceName + '/' : canonicalResourceName);
             }
diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/RuntimeModuleSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/RuntimeModuleSupport.java
@@ -30,20 +30,35 @@
 
 import com.oracle.svm.core.heap.UnknownObjectField;
 
-public final class BootModuleLayerSupport {
+import java.util.function.Function;
 
-    public static BootModuleLayerSupport instance() {
-        return ImageSingletons.lookup(BootModuleLayerSupport.class);
+public final class RuntimeModuleSupport {
+
+    public static RuntimeModuleSupport instance() {
+        return ImageSingletons.lookup(RuntimeModuleSupport.class);
     }
 
     @UnknownObjectField private ModuleLayer bootLayer;
 
-    @Platforms(Platform.HOSTED_ONLY.class)
+    @Platforms(Platform.HOSTED_ONLY.class) //
+    private Function<Module, Module> hostedToRuntimeModuleMapper;
+
+    @Platforms(Platform.HOSTED_ONLY.class) //
     public void setBootLayer(ModuleLayer bootLayer) {
         this.bootLayer = bootLayer;
     }
 
     public ModuleLayer getBootLayer() {
         return bootLayer;
     }
+
+    @Platforms(Platform.HOSTED_ONLY.class) //
+    public void setHostedToRuntimeModuleMapper(Function<Module, Module> hostedToRuntimeModuleMapper) {
+        this.hostedToRuntimeModuleMapper = hostedToRuntimeModuleMapper;
+    }
+
+    public Module getRuntimeModuleForHostedModule(Module hostedModule) {
+        return hostedToRuntimeModuleMapper.apply(hostedModule);
+    }
+
 }
diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Target_java_lang_ModuleLayer.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Target_java_lang_ModuleLayer.java
@@ -34,6 +34,6 @@ final class Target_java_lang_ModuleLayer {
     @SuppressWarnings("unused")
     @Substitute
     public static ModuleLayer boot() {
-        return BootModuleLayerSupport.instance().getBootLayer();
+        return RuntimeModuleSupport.instance().getBootLayer();
     }
 }
diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/resources/NativeImageResourceFileSystem.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/resources/NativeImageResourceFileSystem.java
@@ -651,7 +651,7 @@ private void update(Entry e) {
     }
 
     private void readAllEntries() {
-        MapCursor<Pair<String, String>, ResourceStorageEntry> entries = Resources.singleton().getResourceStorage().getEntries();
+        MapCursor<Pair<Module, String>, ResourceStorageEntry> entries = Resources.singleton().getResourceStorage().getEntries();
         while (entries.advance()) {
             byte[] name = getBytes(entries.getKey().getRight());
             IndexNode newIndexNode = new IndexNode(name, entries.getValue().isDirectory());
diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ModuleLayerFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ModuleLayerFeature.java
@@ -54,6 +54,7 @@
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
+import com.oracle.svm.core.jdk.Resources;
 import jdk.internal.module.DefaultRoots;
 import jdk.internal.module.ModuleBootstrap;
 import jdk.internal.module.SystemModuleFinders;
@@ -63,11 +64,13 @@
 import com.oracle.svm.core.SubstrateOptions;
 import com.oracle.svm.core.SubstrateUtil;
 import com.oracle.svm.core.feature.InternalFeature;
-import com.oracle.svm.core.jdk.BootModuleLayerSupport;
+import com.oracle.svm.core.jdk.RuntimeModuleSupport;
 import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature;
 import com.oracle.svm.core.util.VMError;
 import com.oracle.svm.util.ModuleSupport;
 import com.oracle.svm.util.ReflectionUtil;
+import org.graalvm.nativeimage.Platform;
+import org.graalvm.nativeimage.Platforms;
 
 /**
  * This feature:
@@ -126,7 +129,8 @@ public void duringSetup(DuringSetupAccess access) {
                         .collect(Collectors.toSet());
         Function<String, ClassLoader> clf = moduleLayerFeatureUtils::getClassLoaderForBootLayerModule;
         ModuleLayer runtimeBootLayer = synthesizeRuntimeModuleLayer(new ArrayList<>(List.of(ModuleLayer.empty())), accessImpl.imageClassLoader, baseModules, Set.of(), clf, null);
-        BootModuleLayerSupport.instance().setBootLayer(runtimeBootLayer);
+        RuntimeModuleSupport.instance().setBootLayer(runtimeBootLayer);
+        RuntimeModuleSupport.instance().setHostedToRuntimeModuleMapper(moduleLayerFeatureUtils::getOrCreateRuntimeModuleForHostedModule);
 
         /*
          * Register an object replacer that will ensure all references to hosted module instances
@@ -136,21 +140,20 @@ public void duringSetup(DuringSetupAccess access) {
     }
 
     private Object replaceHostedModules(Object source) {
-        if (source instanceof Module) {
-            Module module = (Module) source;
-            return moduleLayerFeatureUtils.getOrCreateRuntimeModuleForHostedModule(module, module.getDescriptor());
+        if (source instanceof Module module) {
+            return moduleLayerFeatureUtils.getOrCreateRuntimeModuleForHostedModule(module);
         }
         return source;
     }
 
     @Override
     public void afterRegistration(AfterRegistrationAccess access) {
-        ImageSingletons.add(BootModuleLayerSupport.class, new BootModuleLayerSupport());
+        ImageSingletons.add(RuntimeModuleSupport.class, new RuntimeModuleSupport());
 
         List<Module> bootLayerAutomaticModules = ModuleLayer.boot().modules()
                         .stream()
                         .filter(m -> m.isNamed() && m.getDescriptor().isAutomatic())
-                        .collect(Collectors.toList());
+                        .toList();
         if (!bootLayerAutomaticModules.isEmpty()) {
             System.out.println("Warning: Detected automatic module(s) on the module-path of the image builder:" + System.lineSeparator() +
                             bootLayerAutomaticModules.stream().map(ModuleLayerFeatureUtils::formatModule).collect(Collectors.joining(System.lineSeparator())) +
@@ -176,12 +179,7 @@ public void afterAnalysis(AfterAnalysisAccess access) {
          * is required when filtering the analysis reachable module set.
          */
         Set<String> extraModules = ModuleLayerFeatureUtils.parseModuleSetModifierProperty(ModuleSupport.PROPERTY_IMAGE_EXPLICITLY_ADDED_MODULES);
-        Set<String> includedResourceModules = ImageSingletons.lookup(ResourcesFeature.class).includedResourcesModules
-                        .stream()
-                        .map(Module::getName)
-                        .filter(Objects::nonNull)
-                        .collect(Collectors.toSet());
-        extraModules.addAll(includedResourceModules);
+        extraModules.addAll(Resources.getIncludedResourcesModules());
         extraModules.stream().filter(Predicate.not(ModuleSupport.nonExplicitModules::contains)).forEach(moduleName -> {
             Optional<?> module = accessImpl.imageClassLoader.findModule(moduleName);
             if (module.isEmpty()) {
@@ -219,7 +217,7 @@ public void afterAnalysis(AfterAnalysisAccess access) {
         Set<String> rootModules = calculateRootModules(extraModules);
         List<ModuleLayer> runtimeModuleLayers = synthesizeRuntimeModuleLayers(accessImpl, reachableModuleLayers, runtimeImageNamedModules, analysisReachableSyntheticModules, rootModules);
         ModuleLayer runtimeBootLayer = runtimeModuleLayers.get(0);
-        BootModuleLayerSupport.instance().setBootLayer(runtimeBootLayer);
+        RuntimeModuleSupport.instance().setBootLayer(runtimeBootLayer);
 
         /*
          * Ensure that runtime modules have the same relations (i.e., reads, opens and exports) as
@@ -416,7 +414,7 @@ private ModuleLayer synthesizeRuntimeModuleLayer(List<ModuleLayer> parentLayers,
             runtimeModuleLayer = moduleLayerFeatureUtils.createNewModuleLayerInstance(runtimeModuleLayerConfiguration);
             Map<String, Module> nameToModule = moduleLayerFeatureUtils.synthesizeNameToModule(runtimeModuleLayer, clf);
             for (Module syntheticModule : syntheticModules) {
-                Module runtimeSyntheticModule = moduleLayerFeatureUtils.getOrCreateRuntimeModuleForHostedModule(syntheticModule, syntheticModule.getDescriptor());
+                Module runtimeSyntheticModule = moduleLayerFeatureUtils.getOrCreateRuntimeModuleForHostedModule(syntheticModule);
                 nameToModule.putIfAbsent(runtimeSyntheticModule.getName(), runtimeSyntheticModule);
                 moduleLayerFeatureUtils.patchModuleLayerField(runtimeSyntheticModule, runtimeModuleLayer);
             }
@@ -540,6 +538,7 @@ private void patchRuntimeModuleLayer(ModuleLayer runtimeModuleLayer, Map<String,
         runtimeModuleLayer.modules();
     }
 
+    @Platforms(Platform.HOSTED_ONLY.class)
     private static final class ModuleLayerFeatureUtils {
         private final Map<ClassLoader, Map<String, Module>> runtimeModules;
         private final ImageClassLoader imageClassLoader;
@@ -731,9 +730,9 @@ public Module getRuntimeModuleForHostedModule(ClassLoader loader, String hostedM
             }
         }
 
-        public Module getOrCreateRuntimeModuleForHostedModule(Module hostedModule, ModuleDescriptor runtimeModuleDescriptor) {
+        public Module getOrCreateRuntimeModuleForHostedModule(Module hostedModule) {
             if (hostedModule.isNamed()) {
-                return getOrCreateRuntimeModuleForHostedModule(hostedModule.getClassLoader(), hostedModule.getName(), runtimeModuleDescriptor);
+                return getOrCreateRuntimeModuleForHostedModule(hostedModule.getClassLoader(), hostedModule.getName(), hostedModule.getDescriptor());
             } else {
                 return hostedModule == everyoneModule ? everyoneModule : allUnnamedModule;
             }
diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java
@@ -37,7 +37,6 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
 import java.util.Set;
@@ -47,7 +46,6 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.LongAdder;
 import java.util.regex.Pattern;
-import java.util.stream.Collectors;
 
 import org.graalvm.compiler.api.replacements.SnippetReflectionProvider;
 import org.graalvm.compiler.debug.DebugContext;
@@ -137,8 +135,6 @@ public static class Options {
     private int loadedConfigurations;
     private ImageClassLoader imageClassLoader;
 
-    public final Set<Module> includedResourcesModules = new HashSet<>();
-
     private class ResourcesRegistryImpl extends ConditionalConfigurationRegistry implements ResourcesRegistry {
         private final ConfigurationTypeResolver configurationTypeResolver;
 
@@ -228,7 +224,6 @@ private static final class ResourceCollectorImpl implements ResourceCollector {
         private final DebugContext debugContext;
         private final ResourcePattern[] includePatterns;
         private final ResourcePattern[] excludePatterns;
-        private final Set<Module> includedResourcesModules;
 
         private static final int WATCHDOG_RESET_AFTER_EVERY_N_RESOURCES = 1000;
         private static final int WATCHDOG_INITIAL_WARNING_AFTER_N_SECONDS = 60;
@@ -239,12 +234,10 @@ private static final class ResourceCollectorImpl implements ResourceCollector {
         private volatile String currentlyProcessedEntry;
         ScheduledExecutorService scheduledExecutor;
 
-        private ResourceCollectorImpl(DebugContext debugContext, ResourcePattern[] includePatterns, ResourcePattern[] excludePatterns, Set<Module> includedResourcesModules,
-                        Runnable heartbeatCallback) {
+        private ResourceCollectorImpl(DebugContext debugContext, ResourcePattern[] includePatterns, ResourcePattern[] excludePatterns, Runnable heartbeatCallback) {
             this.debugContext = debugContext;
             this.includePatterns = includePatterns;
             this.excludePatterns = excludePatterns;
-            this.includedResourcesModules = includedResourcesModules;
 
             this.heartbeatCallback = heartbeatCallback;
             this.reachedResourceEntries = new LongAdder();
@@ -307,21 +300,13 @@ public boolean isIncluded(Module module, String resourceName, URI resource) {
 
         @Override
         public void addResource(Module module, String resourceName, InputStream resourceStream, boolean fromJar) {
-            collectModule(module);
             registerResource(debugContext, module, resourceName, resourceStream, fromJar);
         }
 
         @Override
         public void addDirectoryResource(Module module, String dir, String content, boolean fromJar) {
-            collectModule(module);
             registerDirectoryResource(debugContext, module, dir, content, fromJar);
         }
-
-        private void collectModule(Module module) {
-            if (module != null && module.isNamed()) {
-                includedResourcesModules.add(module);
-            }
-        }
     }
 
     @Override
@@ -337,7 +322,7 @@ public void duringAnalysis(DuringAnalysisAccess access) {
         ResourcePattern[] includePatterns = compilePatterns(resourcePatternWorkSet);
         ResourcePattern[] excludePatterns = compilePatterns(excludedResourcePatterns);
         DebugContext debugContext = duringAnalysisAccess.getDebugContext();
-        ResourceCollectorImpl collector = new ResourceCollectorImpl(debugContext, includePatterns, excludePatterns, includedResourcesModules, duringAnalysisAccess.bb.getHeartbeatCallback());
+        ResourceCollectorImpl collector = new ResourceCollectorImpl(debugContext, includePatterns, excludePatterns, duringAnalysisAccess.bb.getHeartbeatCallback());
         try {
             collector.prepareProgressReporter();
             ImageSingletons.lookup(ClassLoaderSupport.class).collectResources(collector);
@@ -351,7 +336,7 @@ private ResourcePattern[] compilePatterns(Set<String> patterns) {
         return patterns.stream()
                         .filter(s -> s.length() > 0)
                         .map(this::makeResourcePattern)
-                        .collect(Collectors.toList())
+                        .toList()
                         .toArray(new ResourcePattern[]{});
     }
 

Original file line number	Diff line number	Diff line change
`@@ -34,6 +34,6 @@ final class Target_java_lang_ModuleLayer {`
`34`	`34`	`@SuppressWarnings("unused")`
`35`	`35`	`@Substitute`
`36`	`36`	`public static ModuleLayer boot() {`
`37`		`- return BootModuleLayerSupport.instance().getBootLayer();`
	`37`	`+ return RuntimeModuleSupport.instance().getBootLayer();`
`38`	`38`	`}`
`39`	`39`	`}`
Original file line number	Diff line number	Diff line change
`@@ -651,7 +651,7 @@ private void update(Entry e) {`
`651`	`651`	`}`
`652`	`652`
`653`	`653`	`private void readAllEntries() {`
`654`		`- MapCursor<Pair<String, String>, ResourceStorageEntry> entries = Resources.singleton().getResourceStorage().getEntries();`
	`654`	`+ MapCursor<Pair<Module, String>, ResourceStorageEntry> entries = Resources.singleton().getResourceStorage().getEntries();`
`655`	`655`	`while (entries.advance()) {`
`656`	`656`	`byte[] name = getBytes(entries.getKey().getRight());`
`657`	`657`	`IndexNode newIndexNode = new IndexNode(name, entries.getValue().isDirectory());`