[GR-41069] Revert "[GR-28461] Partially Collect Profiles in NI CE"

PullRequest: graal/12679

Author: cugurm

graal-common.json

@@ -1,7 +1,7 @@
 {
   "README": "This file contains definitions that are useful for the hocon and jsonnet CI files of the graal and graal-enterprise repositories.",
   "ci": {
-    "overlay": "7fe3eba6396a9b4babac6171e7d2fb95da4dc65c"
+    "overlay": "b84cafb1bb9555b3b13443815280c85df77cfc75"
   },
 
   "mx_version" : "6.9.1"

substratevm/src/com.oracle.svm.configure/src/com/oracle/svm/configure/ConfigurationTool.java

@@ -484,7 +484,7 @@ private static void generateFilterRules(Iterator<String> argsIter) throws IOExce
                             exportedInclusion = ConfigurationFilter.Inclusion.Include;
                             unexportedInclusion = ConfigurationFilter.Inclusion.Exclude;
                         }
-                        filter.setHierarchyFilterNode(ModuleFilterTools.generateFromModules(moduleNames, rootInclusion, exportedInclusion, unexportedInclusion, reduce));
+                        rootNode = ModuleFilterTools.generateFromModules(moduleNames, rootInclusion, exportedInclusion, unexportedInclusion, reduce);
                     } else {
                         throw new UsageException(current + " is currently not supported in the native-image build of this tool.");
                     }
@@ -501,20 +501,20 @@ private static void generateFilterRules(Iterator<String> argsIter) throws IOExce
 
                 case "--include-classes":
                     filterModified = true;
-                    addSingleRule(filter.getHierarchyFilterNode(), current, value, ConfigurationFilter.Inclusion.Include);
+                    addSingleRule(rootNode, current, value, ConfigurationFilter.Inclusion.Include);
                     break;
 
                 case "--exclude-classes":
                     filterModified = true;
-                    addSingleRule(filter.getHierarchyFilterNode(), current, value, ConfigurationFilter.Inclusion.Exclude);
+                    addSingleRule(rootNode, current, value, ConfigurationFilter.Inclusion.Exclude);
                     break;
 
                 default:
                     throw new UsageException("Unknown argument: " + current);
             }
         }
 
-        filter.getHierarchyFilterNode().removeRedundantNodes();
+        rootNode.removeRedundantNodes();
         if (outputPath != null) {
             try (FileOutputStream os = new FileOutputStream(outputPath.toFile())) {
                 printFilterToStream(filter, os);

substratevm/src/com.oracle.svm.configure/src/com/oracle/svm/configure/filters/ComplexFilter.java

@@ -30,7 +30,7 @@
 import java.util.Map;
 
 public class ComplexFilter implements ConfigurationFilter {
-    private HierarchyFilterNode hierarchyFilterNode;
+    private final HierarchyFilterNode hierarchyFilterNode;
     private final RegexFilter regexFilter = new RegexFilter();
 
     public ComplexFilter(HierarchyFilterNode hierarchyFilterNode) {
@@ -42,7 +42,6 @@ public void printJson(JsonWriter writer) throws IOException {
         writer.append('{');
         writer.indent().newline();
         hierarchyFilterNode.printJson(writer);
-        writer.append(",\n").newline();
         regexFilter.printJson(writer);
         writer.unindent().newline();
         writer.append('}').newline();
@@ -62,8 +61,4 @@ public boolean includes(String qualifiedName) {
     public HierarchyFilterNode getHierarchyFilterNode() {
         return hierarchyFilterNode;
     }
-
-    public void setHierarchyFilterNode(HierarchyFilterNode hierarchyFilterNode) {
-        this.hierarchyFilterNode = hierarchyFilterNode;
-    }
 }

substratevm/src/com.oracle.svm.configure/src/com/oracle/svm/configure/filters/RegexFilter.java

@@ -58,6 +58,8 @@ public void printJson(JsonWriter writer) throws IOException {
 
         writer.unindent().newline();
         writer.append("]");
+        writer.unindent().newline();
+        writer.append("}");
     }
 
     @SuppressWarnings({"rawtypes", "unchecked"})

substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/CompileQueue.java

@@ -227,6 +227,7 @@ public interface CompileFunction {
 
     private SnippetReflectionProvider snippetReflection;
     private final FeatureHandler featureHandler;
+    protected final OptionValues compileOptions;
     protected final GlobalMetrics metricValues = new GlobalMetrics();
 
     private volatile boolean inliningProgress;
@@ -402,6 +403,7 @@ public CompileQueue(DebugContext debug, FeatureHandler featureHandler, HostedUni
         this.executor = new CompletionExecutor(universe.getBigBang(), executorService, universe.getBigBang().getHeartbeatCallback());
         this.featureHandler = featureHandler;
         this.snippetReflection = snippetReflection;
+        this.compileOptions = getCustomizedOptions(debug);
 
         callForReplacements(debug, runtimeConfig);
     }
@@ -465,6 +467,7 @@ public void finish(DebugContext debug) {
         if (ImageSingletons.contains(HostedHeapDumpFeature.class)) {
             ImageSingletons.lookup(HostedHeapDumpFeature.class).compileQueueAfterCompilation();
         }
+        metricValues.print(compileOptions);
     }
 
     private boolean suitesNotCreated() {
@@ -871,7 +874,6 @@ private StructuredGraph transplantGraph(DebugContext debug, HostedMethod hMethod
          * The static analysis always needs NodeSourcePosition. But for AOT compilation, we only
          * need to preserve them when explicitly enabled, to reduce memory pressure.
          */
-        OptionValues compileOptions = getCustomizedOptions(hMethod, debug);
         boolean trackNodeSourcePosition = GraalOptions.TrackNodeSourcePosition.getValue(compileOptions);
         StructuredGraph graph = aGraph.copy(universe.lookup(aGraph.method()), compileOptions, debug, trackNodeSourcePosition);
 
@@ -1119,7 +1121,7 @@ private void defaultParseFunction(DebugContext debug, HostedMethod method, Compi
                     Bytecode code = new ResolvedJavaMethodBytecode(method);
                     // DebugContext debug = new DebugContext(options,
                     // providers.getSnippetReflection());
-                    graph = new SubstrateIntrinsicGraphBuilder(getCustomizedOptions(method, debug), debug, providers,
+                    graph = new SubstrateIntrinsicGraphBuilder(compileOptions, debug, providers,
                                     code).buildGraph(plugin);
                 }
             }
@@ -1129,7 +1131,7 @@ private void defaultParseFunction(DebugContext debug, HostedMethod method, Compi
             }
             if (graph == null) {
                 needParsing = true;
-                graph = new StructuredGraph.Builder(getCustomizedOptions(method, debug), debug)
+                graph = new StructuredGraph.Builder(compileOptions, debug)
                                 .method(method)
                                 .recordInlinedMethods(false)
                                 .build();
@@ -1176,7 +1178,7 @@ private void defaultParseFunction(DebugContext debug, HostedMethod method, Compi
                 beforeEncode(method, graph);
                 assert GraphOrder.assertSchedulableGraph(graph);
                 method.compilationInfo.encodeGraph(graph);
-                method.compilationInfo.setCompileOptions(getCustomizedOptions(method, debug));
+                method.compilationInfo.setCompileOptions(compileOptions);
                 checkTrivial(method, graph);
 
             } catch (Throwable ex) {
@@ -1218,7 +1220,7 @@ private void ensureParsed(HostedMethod method, CompileReason reason, CallTargetN
     protected void beforeEncode(HostedMethod method, StructuredGraph graph) {
     }
 
-    protected OptionValues getCustomizedOptions(@SuppressWarnings("unused") HostedMethod method, DebugContext debug) {
+    protected OptionValues getCustomizedOptions(DebugContext debug) {
         return debug.getOptions();
     }
 

vm/mx.vm/mx_vm_benchmark.py

@@ -27,9 +27,6 @@
 # ----------------------------------------------------------------------------------------------------
 import os
 import re
-import tempfile
-import json
-from genericpath import exists
 from os.path import basename, dirname, getsize, join
 from traceback import print_tb
 import inspect
@@ -149,10 +146,6 @@ def __init__(self, vm, bm_suite, args):
                            'extra-agent-profile-run-arg', 'benchmark-output-dir', 'stages', 'skip-agent-assertions']
             self.profile_file_extension = '.iprof'
             self.stages = bm_suite.stages(args)
-            if vm.jdk_profiles_collect:  # forbid image build/run in the profile collection execution mode
-                for stage in ('image', 'run'):
-                    if stage in self.stages:
-                        self.stages.remove(stage)
             self.last_stage = self.stages[-1]
             self.skip_agent_assertions = bm_suite.skip_agent_assertions(self.benchmark_name, args)
             self.root_dir = self.benchmark_output_dir if self.benchmark_output_dir else mx.suite('vm').get_output_root(platformDependent=False, jdkDependent=False)
@@ -218,8 +211,6 @@ def __init__(self, name, config_name, extra_java_args=None, extra_launcher_args=
         self.config = None
         self.stages = None
         self.ml = None
-        self.jdk_profiles_collect = False
-        self.cached_jdk_pgo = False
         self.analysis_context_sensitivity = None
         self.no_inlining_before_analysis = False
         self._configure_from_name(config_name)
@@ -240,7 +231,7 @@ def _configure_from_name(self, config_name):
 
         # This defines the allowed config names for NativeImageVM. The ones registered will be available via --jvm-config
         rule = r'^(?P<native_architecture>native-architecture-)?(?P<string_inlining>string-inlining-)?(?P<gate>gate-)?(?P<upx>upx-)?(?P<quickbuild>quickbuild-)?(?P<gc>g1gc-)?(?P<llvm>llvm-)?(?P<pgo>pgo-|pgo-ctx-insens-)?(?P<inliner>aot-inline-|iterative-|inline-explored-)?' \
-               r'(?P<analysis_context_sensitivity>insens-|allocsens-|1obj-|2obj1h-|3obj2h-|4obj3h-)?(?P<no_inlining_before_analysis>no-inline-)?(?P<ml>ml-profile-inference-)?(?P<jdk_profiles>jdk-profiles-collect-|cached-jdk-pgo-)?(?P<edition>ce-|ee-)?$'
+               r'(?P<analysis_context_sensitivity>insens-|allocsens-|1obj-|2obj1h-|3obj2h-|4obj3h-)?(?P<no_inlining_before_analysis>no-inline-)?(?P<ml>ml-profile-inference-)?(?P<edition>ce-|ee-)?$'
 
         mx.logv("== Registering configuration: {}".format(config_name))
         match_name = "{}-".format(config_name)  # adding trailing dash to simplify the regex
@@ -311,42 +302,6 @@ def _configure_from_name(self, config_name):
         if matching.group("ml") is not None:
             self.ml = matching.group("ml")[:-1]
 
-        if matching.group("jdk_profiles") is not None:
-            config = matching.group("jdk_profiles")[:-1]
-            if config == 'jdk-profiles-collect':
-                self.jdk_profiles_collect = True
-                self.pgo_instrumented_iterations = 1
-
-                def generate_profiling_package_prefixes():
-                    # run the native-image-configure tool to gather the jdk package prefixes
-                    native_image_configure_command = mx.cmd_suffix(join(mx_sdk_vm.graalvm_home(), 'bin', 'native-image-configure'))
-                    if not exists(native_image_configure_command):
-                        mx.abort('Failed to find the native-image-configure command at {}.'.format(native_image_configure_command))
-                    tmp = tempfile.NamedTemporaryFile()
-                    ret = mx.run([native_image_configure_command, 'generate-filters',
-                                  '--include-packages-from-modules=java.base',
-                                  '--exclude-classes=org.graalvm.**', '--exclude-classes=com.oracle.**',  # remove internal packages
-                                  '--output-file={}'.format(tmp.name)], nonZeroIsFatal=True)
-                    if ret != 0:
-                        mx.abort('Native image configure command failed.')
-
-                    # format the profiling package prefixes
-                    with open(tmp.name, 'r') as f:
-                        prefixes = json.loads(f.read())
-                        if 'rules' not in prefixes:
-                            mx.abort('Native image configure command failed. Can not generate rules.')
-                        rules = prefixes['rules']
-                        rules = map(lambda r: r['includeClasses'][:-2], filter(lambda r: 'includeClasses' in r, rules))
-                        return ','.join(rules)
-                self.generate_profiling_package_prefixes = generate_profiling_package_prefixes
-            elif config == 'cached-jdk-pgo':
-                self.cached_jdk_pgo = True
-            else:
-                mx.abort('Unknown jdk profiles configuration: {}'.format(config))
-            # choose appropriate profiles
-            jdk_profiles = 'JDK{}_PROFILES'.format(str(mx.get_jdk().version).split('.')[0])
-            self.cached_profiles_base_dir = mx.library(jdk_profiles).get_path(True)
-
         if matching.group("edition") is not None:
             edition = matching.group("edition")[:-1]
             mx.logv("GraalVM edition is set to: {}".format(edition))
@@ -818,8 +773,6 @@ def run_stage_instrument_image(self, config, stages, out, i, instrumentation_ima
         pgo_args += ['-H:' + ('+' if self.pgo_context_sensitive else '-') + 'PGOContextSensitivityEnabled']
         pgo_args += ['-H:+AOTInliner'] if self.pgo_aot_inline else ['-H:-AOTInliner']
         instrument_args = ['--pgo-instrument'] + ([] if i == 0 else pgo_args)
-        if self.jdk_profiles_collect:
-            instrument_args += ['-H:+ProfilingEnabled', '-H:+AOTPriorityInline', '-H:ProfilingPackagePrefixes={}'.format(self.generate_profiling_package_prefixes())]
 
         with stages.set_command(config.base_image_build_args + executable_name_args + instrument_args) as s:
             s.execute_command()
@@ -844,12 +797,7 @@ def run_stage_image(self, config, stages):
         pgo_args += ['-H:+AOTInliner'] if self.pgo_aot_inline else ['-H:-AOTInliner']
         instrumented_iterations = self.pgo_instrumented_iterations if config.pgo_iteration_num is None else int(config.pgo_iteration_num)
         ml_args = ['-H:+ProfileInference'] if self.ml == 'ml-profile-inference' else []
-        if self.cached_jdk_pgo:
-            cached_profiles = ','.join(list(map(lambda f: os.path.join(self.cached_profiles_base_dir, f), os.listdir(self.cached_profiles_base_dir))))
-            jdk_profiles_args = ['-H:CachedPGOEnabled={}'.format(cached_profiles)]
-        else:
-            jdk_profiles_args = []
-        final_image_command = config.base_image_build_args + executable_name_args + (pgo_args if instrumented_iterations > 0 else []) + ml_args + jdk_profiles_args
+        final_image_command = config.base_image_build_args + executable_name_args + (pgo_args if instrumented_iterations > 0 else []) + ml_args
         with stages.set_command(final_image_command) as s:
             s.execute_command()
             if self.use_upx: