[GR-47922] Enforce -H:±UnlockExperimentalVMOptions in CI.

.github/workflows/main.yml

@@ -42,6 +42,8 @@ env:
   MX_GIT_CACHE: refcache
   MX_PATH: ${{ github.workspace }}/mx
   MX_PYTHON: python3.8
+  # Enforce experimental option checking in CI (GR-47922)
+  NATIVE_IMAGE_EXPERIMENTAL_OPTIONS_ARE_FATAL: "true"
 
 permissions:
   contents: read # to fetch code (actions/checkout)

ci/common.jsonnet

@@ -277,6 +277,10 @@ local common_json = import "../common.json";
       # Keep in sync with com.oracle.svm.hosted.NativeImageOptions#DEFAULT_ERROR_FILE_NAME
       " (?P<filename>.+/svm_err_b_\\d+T\\d+\\.\\d+_pid\\d+\\.md)",
     ],
+    environment+: {
+      # Enforce experimental option checking in CI (GR-47922)
+      NATIVE_IMAGE_EXPERIMENTAL_OPTIONS_ARE_FATAL: "true",
+    },
   },
 
   // OS specific file handling

substratevm/mx.substratevm/mx_substratevm.py

@@ -268,7 +268,7 @@ def native_image_context(common_args=None, hosted_assertions=True, native_image_
             raise mx.abort('The built GraalVM for config ' + str(config) + ' does not contain a native-image command')
 
     def _native_image(args, **kwargs):
-        mx.run([native_image_cmd] + _maybe_convert_to_args_file(args), **kwargs)
+        return mx.run([native_image_cmd] + _maybe_convert_to_args_file(args), **kwargs)
 
     def is_launcher(launcher_path):
         with open(launcher_path, 'rb') as fp:
@@ -282,32 +282,45 @@ def is_launcher(launcher_path):
         verbose_image_build_option = ['--verbose'] if mx.get_opts().verbose else []
         _native_image(verbose_image_build_option + ['--macro:native-image-launcher'])
 
-    def query_native_image(all_args, option):
-
+    def query_native_image(all_args):
         stdoutdata = []
         def stdout_collector(x):
             stdoutdata.append(x.rstrip())
-        _native_image(['--dry-run', '--verbose'] + all_args, out=stdout_collector)
+        stderrdata = []
+        def stderr_collector(x):
+            stderrdata.append(x.rstrip())
+        exit_code = _native_image(['--dry-run', '--verbose'] + all_args, nonZeroIsFatal=False, out=stdout_collector, err=stderr_collector)
+        if exit_code != 0:
+            for line in stdoutdata:
+                print(line)
+            for line in stderrdata:
+                print(line)
+            mx.abort('Failed to query native-image.')
 
         def remove_quotes(val):
             if len(val) >= 2 and val.startswith("'") and val.endswith("'"):
                 return val[1:-1].replace("\\'", "'")
             else:
                 return val
 
-        result = None
+        path_regex = re.compile(r'^-H:Path(@[^=]*)?=')
+        name_regex = re.compile(r'^-H:Name(@[^=]*)?=')
+        path = name = None
         for line in stdoutdata:
             arg = remove_quotes(line.rstrip('\\').strip())
-            m = re.match(option, arg)
-            if m:
-                result = arg[m.end():]
+            path_matcher = path_regex.match(arg)
+            if path_matcher:
+                path = arg[path_matcher.end():]
+            name_matcher = name_regex.match(arg)
+            if name_matcher:
+                name = arg[name_matcher.end():]
 
-        return result
+        assert path is not None and name is not None
+        return path, name
 
     def native_image_func(args, **kwargs):
         all_args = base_args + common_args + args
-        path = query_native_image(all_args, r'^-H:Path(@[^=]*)?=')
-        name = query_native_image(all_args, r'^-H:Name(@[^=]*)?=')
+        path, name = query_native_image(all_args)
         image = join(path, name)
         _native_image(all_args, **kwargs)
         return image

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java

@@ -1022,4 +1022,12 @@ public enum ReportingMode {
 
     @Option(help = "Include all classes, methods, fields, and resources from given paths", type = OptionType.Debug) //
     public static final HostedOptionKey<LocatableMultiOptionValue.Strings> IncludeAllFromPath = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.build());
+
+    public static class TruffleStableOptions {
+
+        @Option(help = "Automatically copy the necessary language resources to the resources/languages directory next to the produced image." +
+                        "Language resources for each language are specified in the native-image-resources.filelist file located in the language home directory." +
+                        "If there is no native-image-resources.filelist file in the language home directory or the file is empty, then no resources are copied.", type = User, stability = OptionStability.STABLE)//
+        public static final HostedOptionKey<Boolean> CopyLanguageResources = new HostedOptionKey<>(true);
+    }
 }

substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/APIOptionHandler.java

@@ -97,7 +97,7 @@ static final class PathsOptionInfo {
     private final Map<String, PathsOptionInfo> pathOptions;
     private final Set<String> stableOptionNames;
 
-    private boolean experimentalOptionsAreUnlocked = false;
+    private int numberOfActiveUnlockExperimentalVMOptions = 0;
     private Set<String> illegalExperimentalOptions = new HashSet<>(0);
 
     APIOptionHandler(NativeImage nativeImage) {
@@ -307,11 +307,21 @@ boolean consume(ArgumentQueue args) {
             return true;
         }
         if (ENTER_UNLOCK_SCOPE.equals(headArg)) {
-            experimentalOptionsAreUnlocked = true;
+            if (args.numberOfFirstObservedActiveUnlockExperimentalVMOptions < 0) {
+                /*
+                 * Remember numberOfExperimentalOptionsUnlocks per ArgumentQueue for verification
+                 * purposes only. Each queue cannot lock more than it unlocks.
+                 */
+                args.numberOfFirstObservedActiveUnlockExperimentalVMOptions = numberOfActiveUnlockExperimentalVMOptions;
+            }
+            numberOfActiveUnlockExperimentalVMOptions++;
         } else if (LEAVE_UNLOCK_SCOPE.equals(headArg)) {
-            VMError.guarantee(experimentalOptionsAreUnlocked, "ensureConsistentUnlockScopes() missed an open unlock scope");
-            experimentalOptionsAreUnlocked = false;
-        } else if (!experimentalOptionsAreUnlocked && !OptionOrigin.isAPI(args.argumentOrigin) && headArg.startsWith(NativeImage.oH) && stableOptionNames.stream().noneMatch(p -> headArg.matches(p))) {
+            if (numberOfActiveUnlockExperimentalVMOptions <= 0 || numberOfActiveUnlockExperimentalVMOptions <= args.numberOfFirstObservedActiveUnlockExperimentalVMOptions) {
+                throw VMError.shouldNotReachHere("Unlocking of experimental options in inconsistent state: trying to lock more scopes than exist or allowed.");
+            }
+            numberOfActiveUnlockExperimentalVMOptions--;
+        } else if (numberOfActiveUnlockExperimentalVMOptions == 0 && !OptionOrigin.isAPI(args.argumentOrigin) && headArg.startsWith(NativeImage.oH) &&
+                        stableOptionNames.stream().noneMatch(p -> headArg.matches(p))) {
             illegalExperimentalOptions.add(headArg);
         }
         for (Entry<String, GroupInfo> entry : groupInfos.entrySet()) {

substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java

@@ -161,6 +161,7 @@ static class ArgumentQueue {
 
         private final ArrayDeque<String> queue;
         public final String argumentOrigin;
+        public int numberOfFirstObservedActiveUnlockExperimentalVMOptions = -1;
 
         ArgumentQueue(String argumentOrigin) {
             queue = new ArrayDeque<>();
@@ -714,11 +715,13 @@ public boolean isExcluded(Path resourcePath, Path entry) {
 
     private ArrayList<String> createFallbackBuildArgs() {
         ArrayList<String> buildArgs = new ArrayList<>();
+        buildArgs.add(oHEnabled(SubstrateOptions.UnlockExperimentalVMOptions));
         Collection<String> fallbackSystemProperties = customJavaArgs.stream()
                         .filter(s -> s.startsWith("-D"))
                         .collect(Collectors.toCollection(LinkedHashSet::new));
+        String fallbackExecutorSystemPropertyOption = oH(FallbackExecutor.Options.FallbackExecutorSystemProperty);
         for (String property : fallbackSystemProperties) {
-            buildArgs.add(oH(FallbackExecutor.Options.FallbackExecutorSystemProperty) + property);
+            buildArgs.add(injectHostedOptionOrigin(fallbackExecutorSystemPropertyOption + property, OptionOrigin.originDriver));
         }
 
         List<String> runtimeJavaArgs = imageBuilderArgs.stream()
@@ -735,7 +738,6 @@ private ArrayList<String> createFallbackBuildArgs() {
             buildArgs.add(fallbackExecutorJavaArg);
         }
 
-        buildArgs.add(oHEnabled(SubstrateOptions.UnlockExperimentalVMOptions));
         buildArgs.add(oHEnabled(SubstrateOptions.BuildOutputSilent));
         buildArgs.add(oHEnabled(SubstrateOptions.ParseRuntimeOptions));
         Path imagePathPath;

substratevm/src/com.oracle.svm.truffle/src/com/oracle/svm/truffle/TruffleBaseFeature.java

@@ -26,7 +26,6 @@
 
 import static com.oracle.svm.core.util.VMError.shouldNotReachHere;
 import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
-import static jdk.graal.compiler.options.OptionType.User;
 
 import java.io.IOException;
 import java.lang.annotation.Annotation;
@@ -60,23 +59,6 @@
 import java.util.stream.Stream;
 
 import org.graalvm.collections.Pair;
-import jdk.graal.compiler.api.replacements.SnippetReflectionProvider;
-import jdk.graal.compiler.nodes.ConstantNode;
-import jdk.graal.compiler.nodes.ValueNode;
-import jdk.graal.compiler.nodes.graphbuilderconf.GraphBuilderConfiguration.Plugins;
-import jdk.graal.compiler.nodes.graphbuilderconf.GraphBuilderContext;
-import jdk.graal.compiler.nodes.graphbuilderconf.InvocationPlugin;
-import jdk.graal.compiler.nodes.graphbuilderconf.InvocationPlugin.RequiredInlineOnlyInvocationPlugin;
-import jdk.graal.compiler.nodes.graphbuilderconf.InvocationPlugin.RequiredInvocationPlugin;
-import jdk.graal.compiler.nodes.graphbuilderconf.InvocationPlugins;
-import jdk.graal.compiler.nodes.graphbuilderconf.InvocationPlugins.Registration;
-import jdk.graal.compiler.options.Option;
-import jdk.graal.compiler.options.OptionStability;
-import jdk.graal.compiler.phases.tiers.Suites;
-import jdk.graal.compiler.phases.util.Providers;
-import jdk.graal.compiler.truffle.host.InjectImmutableFrameFieldsPhase;
-import jdk.graal.compiler.truffle.host.TruffleHostEnvironment;
-import jdk.graal.compiler.truffle.substitutions.TruffleInvocationPlugins;
 import org.graalvm.home.HomeFinder;
 import org.graalvm.home.impl.DefaultHomeFinder;
 import org.graalvm.nativeimage.AnnotationAccess;
@@ -95,6 +77,7 @@
 import com.oracle.svm.core.NeverInline;
 import com.oracle.svm.core.ParsingReason;
 import com.oracle.svm.core.RuntimeAssertionsSupport;
+import com.oracle.svm.core.SubstrateOptions;
 import com.oracle.svm.core.SubstrateUtil;
 import com.oracle.svm.core.annotate.Alias;
 import com.oracle.svm.core.annotate.AnnotateOriginal;
@@ -150,6 +133,22 @@
 import com.oracle.truffle.api.staticobject.StaticProperty;
 import com.oracle.truffle.api.staticobject.StaticShape;
 
+import jdk.graal.compiler.api.replacements.SnippetReflectionProvider;
+import jdk.graal.compiler.nodes.ConstantNode;
+import jdk.graal.compiler.nodes.ValueNode;
+import jdk.graal.compiler.nodes.graphbuilderconf.GraphBuilderConfiguration.Plugins;
+import jdk.graal.compiler.nodes.graphbuilderconf.GraphBuilderContext;
+import jdk.graal.compiler.nodes.graphbuilderconf.InvocationPlugin;
+import jdk.graal.compiler.nodes.graphbuilderconf.InvocationPlugin.RequiredInlineOnlyInvocationPlugin;
+import jdk.graal.compiler.nodes.graphbuilderconf.InvocationPlugin.RequiredInvocationPlugin;
+import jdk.graal.compiler.nodes.graphbuilderconf.InvocationPlugins;
+import jdk.graal.compiler.nodes.graphbuilderconf.InvocationPlugins.Registration;
+import jdk.graal.compiler.options.Option;
+import jdk.graal.compiler.phases.tiers.Suites;
+import jdk.graal.compiler.phases.util.Providers;
+import jdk.graal.compiler.truffle.host.InjectImmutableFrameFieldsPhase;
+import jdk.graal.compiler.truffle.host.TruffleHostEnvironment;
+import jdk.graal.compiler.truffle.substitutions.TruffleInvocationPlugins;
 import jdk.internal.misc.Unsafe;
 import jdk.vm.ci.meta.JavaKind;
 import jdk.vm.ci.meta.ResolvedJavaMethod;
@@ -183,11 +182,6 @@ public static Class<?> lookupClass(String className) {
 
     public static class Options {
 
-        @Option(help = "Automatically copy the necessary language resources to the resources/languages directory next to the produced image." +
-                        "Language resources for each language are specified in the native-image-resources.filelist file located in the language home directory." +
-                        "If there is no native-image-resources.filelist file in the language home directory or the file is empty, then no resources are copied.", type = User, stability = OptionStability.STABLE)//
-        public static final HostedOptionKey<Boolean> CopyLanguageResources = new HostedOptionKey<>(true);
-
         @Option(help = "Check that context pre-initialization does not introduce absolute TruffleFiles into the image heap.")//
         public static final HostedOptionKey<Boolean> TruffleCheckPreinitializedFiles = new HostedOptionKey<>(true);
     }
@@ -402,7 +396,7 @@ public void duringSetup(DuringSetupAccess access) {
         StaticObjectSupport.duringSetup(access);
 
         HomeFinder hf = HomeFinder.getInstance();
-        if (Options.CopyLanguageResources.getValue()) {
+        if (SubstrateOptions.TruffleStableOptions.CopyLanguageResources.getValue()) {
             if (!(hf instanceof DefaultHomeFinder)) {
                 VMError.shouldNotReachHere(String.format("HomeFinder %s cannot be used if CopyLanguageResources option of TruffleBaseFeature is enabled", hf.getClass().getName()));
             }
@@ -1037,7 +1031,7 @@ static void onBuildInvocation(Class<?> storageSuperClass, Class<?> factoryInterf
 
     @Override
     public void afterImageWrite(AfterImageWriteAccess access) {
-        if (Options.CopyLanguageResources.getValue()) {
+        if (SubstrateOptions.TruffleStableOptions.CopyLanguageResources.getValue()) {
             Path buildDir = access.getImagePath();
             if (buildDir != null) {
                 Path parent = buildDir.getParent();
@@ -1390,7 +1384,7 @@ public ValueAvailability valueAvailability() {
 
         @Override
         public Object transform(Object receiver, Object originalValue) {
-            return TruffleBaseFeature.Options.CopyLanguageResources.getValue();
+            return SubstrateOptions.TruffleStableOptions.CopyLanguageResources.getValue();
         }
     }
 }

vm/mx.vm/mx_vm_benchmark.py

@@ -172,58 +172,59 @@ def __init__(self, vm, bm_suite, args):
             self.latest_profile_path = self.profile_path_no_extension + '-latest' + self.profile_file_extension
             self.config_dir = os.path.join(self.output_dir, 'config')
             self.log_dir = self.output_dir
-            self.base_image_build_args = [os.path.join(vm.home(), 'bin', 'native-image')]
-            self.base_image_build_args += ['--no-fallback', '-g']
-            self.base_image_build_args += svm_experimental_options(['-H:+VerifyGraalGraphs', '-H:+VerifyPhases', '--diagnostics-mode']) if vm.is_gate else []
-            self.base_image_build_args += ['-H:+ReportExceptionStackTraces']
-            self.base_image_build_args += bm_suite.build_assertions(self.benchmark_name, vm.is_gate)
+            base_image_build_args = ['--no-fallback', '-g']
+            base_image_build_args += ['-H:+VerifyGraalGraphs', '-H:+VerifyPhases', '--diagnostics-mode'] if vm.is_gate else []
+            base_image_build_args += ['-H:+ReportExceptionStackTraces']
+            base_image_build_args += bm_suite.build_assertions(self.benchmark_name, vm.is_gate)
 
-            self.base_image_build_args += self.system_properties
+            base_image_build_args += self.system_properties
             self.bundle_path = self.get_bundle_path_if_present()
             self.bundle_create_path = self.get_bundle_create_path_if_present()
             if not self.bundle_path:
-                self.base_image_build_args += self.classpath_arguments
-                self.base_image_build_args += self.modulepath_arguments
-                self.base_image_build_args += self.executable
-                self.base_image_build_args += svm_experimental_options(['-H:Path=' + self.output_dir])
-            self.base_image_build_args += svm_experimental_options([
+                base_image_build_args += self.classpath_arguments
+                base_image_build_args += self.modulepath_arguments
+                base_image_build_args += self.executable
+                base_image_build_args += ['-H:Path=' + self.output_dir]
+            base_image_build_args += [
                 '-H:ConfigurationFileDirectories=' + self.config_dir,
                 '-H:+PrintAnalysisStatistics',
                 '-H:+PrintCallEdges',
                 '-H:+CollectImageBuildStatistics',
-            ])
+            ]
             self.image_build_reports_directory = os.path.join(self.output_dir, 'reports')
             if self.bundle_create_path is not None:
                 self.image_build_reports_directory = os.path.join(self.output_dir, self.bundle_create_path)
             self.image_build_stats_file = os.path.join(self.image_build_reports_directory, 'image_build_statistics.json')
 
             if vm.is_quickbuild:
-                self.base_image_build_args += ['-Ob']
+                base_image_build_args += ['-Ob']
             if vm.use_string_inlining:
-                self.base_image_build_args += svm_experimental_options(['-H:+UseStringInlining'])
+                base_image_build_args += ['-H:+UseStringInlining']
             if vm.is_llvm:
-                self.base_image_build_args += ['--features=org.graalvm.home.HomeFinderFeature'] + svm_experimental_options(['-H:CompilerBackend=llvm', '-H:DeadlockWatchdogInterval=0'])
+                base_image_build_args += ['--features=org.graalvm.home.HomeFinderFeature'] + ['-H:CompilerBackend=llvm', '-H:DeadlockWatchdogInterval=0']
             if vm.gc:
-                self.base_image_build_args += ['--gc=' + vm.gc] + svm_experimental_options(['-H:+SpawnIsolates'])
+                base_image_build_args += ['--gc=' + vm.gc] + ['-H:+SpawnIsolates']
             if vm.native_architecture:
-                self.base_image_build_args += ['-march=native']
+                base_image_build_args += ['-march=native']
             if vm.analysis_context_sensitivity:
-                self.base_image_build_args += svm_experimental_options(['-H:AnalysisContextSensitivity=' + vm.analysis_context_sensitivity, '-H:-RemoveSaturatedTypeFlows', '-H:+AliasArrayTypeFlows'])
+                base_image_build_args += ['-H:AnalysisContextSensitivity=' + vm.analysis_context_sensitivity, '-H:-RemoveSaturatedTypeFlows', '-H:+AliasArrayTypeFlows']
             if vm.no_inlining_before_analysis:
-                self.base_image_build_args += svm_experimental_options(['-H:-InlineBeforeAnalysis'])
+                base_image_build_args += ['-H:-InlineBeforeAnalysis']
             if vm.optimization_level:
-                self.base_image_build_args += ['-' + vm.optimization_level]
+                base_image_build_args += ['-' + vm.optimization_level]
             if vm.async_sampler:
-                self.base_image_build_args += ['-R:+FlightRecorder',
+                base_image_build_args += ['-R:+FlightRecorder',
                                                '-R:StartFlightRecording=filename=default.jfr',
                                                '--enable-monitoring=jfr']
                 for stage in ('instrument-image', 'instrument-run'):
                     if stage in self.stages:
                         self.stages.remove(stage)
             if self.image_vm_args is not None:
-                self.base_image_build_args += self.image_vm_args
+                base_image_build_args += self.image_vm_args
             self.is_runnable = self.check_runnable()
-            self.base_image_build_args += self.extra_image_build_arguments
+            base_image_build_args += self.extra_image_build_arguments
+            # benchmarks are allowed to use experimental options
+            self.base_image_build_args = [os.path.join(vm.home(), 'bin', 'native-image')] + svm_experimental_options(base_image_build_args)
 
         def check_runnable(self):
             # TODO remove once there is load available for the specified benchmarks