From f01a12fe37c2d9cffa6d47957d6ac2dd962e852e Mon Sep 17 00:00:00 2001 From: Severin Gehwolf Date: Fri, 24 Jun 2022 13:30:26 +0200 Subject: [PATCH 1/8] Initial implementation for json file build output This uses a simple backing Map (HashMap), for collecting the data. It's entirely driven by ProgressReporter and if not requested by the user, by specifying: `-H:BuildOutputJSONFile=` on the command line, entirely disabled. It also adds a JSON schema to the native image manual as an asset. --- .../native-image/BuildOutput.md | 2 + .../assets/build-output-schema-v1.json | 515 ++++++++++++++++++ .../com/oracle/svm/core/SubstrateOptions.java | 5 + .../oracle/svm/hosted/ProgressReporter.java | 92 +++- .../ProgressReporterJsonStatsHelper.java | 293 ++++++++++ 5 files changed, 898 insertions(+), 9 deletions(-) create mode 100644 docs/reference-manual/native-image/assets/build-output-schema-v1.json create mode 100644 substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporterJsonStatsHelper.java diff --git a/docs/reference-manual/native-image/BuildOutput.md b/docs/reference-manual/native-image/BuildOutput.md index 9888c405fdaf..2a6456290337 100644 --- a/docs/reference-manual/native-image/BuildOutput.md +++ b/docs/reference-manual/native-image/BuildOutput.md @@ -190,6 +190,8 @@ Run `native-image --expert-options-all | grep "BuildOutput"` to see all build ou -H:±BuildOutputBreakdowns Show code and heap breakdowns as part of the build output. Default: + (enabled). -H:±BuildOutputColorful Colorize build output. Default: + (enabled). -H:±BuildOutputGCWarnings Print GC warnings as part of build output. Default: + (enabled). +-H:BuildOutputJSONFile="" Print build output statistics as JSON to the specified file. The output is according to the JSON schema located at: + docs/reference-manual/native-image/assets/build-output-schema-v1.json. -H:±BuildOutputLinks Show links in build output. Default: + (enabled). -H:±BuildOutputPrefix Prefix build output with ':'. Default: - (disabled). -H:±BuildOutputProgress Report progress in build output. Default: + (enabled). diff --git a/docs/reference-manual/native-image/assets/build-output-schema-v1.json b/docs/reference-manual/native-image/assets/build-output-schema-v1.json new file mode 100644 index 000000000000..53ffc28d657e --- /dev/null +++ b/docs/reference-manual/native-image/assets/build-output-schema-v1.json @@ -0,0 +1,515 @@ +{ + "$schema": "https://json-schema.org/draft/2019-09/schema", + "$id": "https://github.com/oracle/graal/blob/master/docs/reference-manual/native-image/assets/build-output-schema-v1.json", + "type": "object", + "default": {}, + "title": "Schema for the JSON build output of GraalVM Native Image", + "required": [ + "general_info", + "analysis_results", + "image_details", + "resource_usage" + ], + "properties": { + "general_info": { + "type": "object", + "default": {}, + "title": "General information about the image build process", + "required": [ + "name", + "graalvm_version", + "java_version", + "c_compiler", + "garbage_collector" + ], + "properties": { + "name": { + "type": "string", + "default": "", + "title": "The native image name" + }, + "graalvm_version": { + "type": "string", + "default": "", + "title": "The GraalVM Version" + }, + "java_version": { + "type": ["string", "null"], + "default": null, + "title": "The precise Java version used for running the image builder (or null if not available)" + }, + "c_compiler": { + "type": ["string", "null"], + "default": null, + "title": "The C compiler used for native compilation and linkage (or null if not available)" + }, + "garbage_collector": { + "type": "string", + "default": "", + "title": "The garbage collection algorithm in use for the generated native image" + } + }, + "examples": [ + { + "name": "helloworld", + "graalvm_version": "GraalVM 22.2.0-dev Java 17 CE", + "java_version": "17.0.4+5-jvmci-22.2-b03", + "c_compiler": "gcc (linux, x86_64, 9.3.0)", + "garbage_collector": "Serial GC" + } + ] + }, + "analysis_results": { + "type": "object", + "default": {}, + "title": "Schema for the image generator statistics of the analysis phase", + "required": [ + "classes", + "fields", + "methods" + ], + "properties": { + "classes": { + "type": "object", + "default": {}, + "title": "Number of classes detected during analysis phase", + "required": [ + "total", + "reachable", + "reflection", + "jni" + ], + "properties": { + "total": { + "type": "integer", + "default": 0, + "title": "The total number of classes" + }, + "reachable": { + "type": "integer", + "default": 0, + "title": "The reachable reachable classes" + }, + "reflection": { + "type": "integer", + "default": 0, + "title": "The classes registered for reflective access" + }, + "jni": { + "type": "integer", + "default": -1, + "title": "The classes registered for JNI access (or -1 if unset)" + } + } + }, + "fields": { + "type": "object", + "default": {}, + "title": "Number of fields detected during analysis phase", + "required": [ + "total", + "reachable", + "reflection", + "jni" + ], + "properties": { + "total": { + "type": "integer", + "default": 0, + "title": "The total number of fields" + }, + "reachable": { + "type": "integer", + "default": 0, + "title": "The reachable number of fields" + }, + "reflection": { + "type": "integer", + "default": 0, + "title": "The number of fields registered for reflective access" + }, + "jni": { + "type": "integer", + "default": -1, + "title": "The number of fields registered for JNI access (or -1 if unset)" + } + } + }, + "methods": { + "type": "object", + "default": {}, + "title": "Number of methods detected during analysis phase", + "required": [ + "total", + "reachable", + "reflection", + "jni" + ], + "properties": { + "total": { + "type": "integer", + "default": 0, + "title": "The total number of methods" + }, + "reachable": { + "type": "integer", + "default": 0, + "title": "The reachable number of methods" + }, + "reflection": { + "type": "integer", + "default": 0, + "title": "The number of methods registered for reflective access" + }, + "jni": { + "type": "integer", + "default": -1, + "title": "The number of methods registered for JNI access (or -1 if unset)" + } + } + } + }, + "examples": [ + { + "classes": { + "total": 3850, + "reachable": 2839, + "reflection": 28, + "jni": 58 + }, + "fields": { + "total": 6665, + "reachable": 3400, + "reflection": 0, + "jni": 58 + }, + "methods": { + "total": 29038, + "reachable": 12916, + "reflection": 332, + "jni": 52 + } + } + ] + }, + "image_details": { + "type": "object", + "default": {}, + "title": "Schema for statistics of the image creation phase of the image generator", + "required": [ + "total_bytes", + "code_area", + "image_heap" + ], + "properties": { + "total_bytes": { + "type": "integer", + "default": 0, + "title": "The total image size (in bytes)" + }, + "code_area": { + "type": "object", + "default": {}, + "title": "The code area statistics", + "required": [ + "bytes", + "compilation_units" + ], + "properties": { + "bytes": { + "type": "integer", + "default": 0, + "title": "Bytes used for the code area" + }, + "compilation_units": { + "type": "integer", + "default": 0, + "title": "The number of compilation units in the image" + } + } + }, + "image_heap": { + "type": "object", + "default": {}, + "title": "The image heap statistics", + "required": [ + "bytes", + "resources" + ], + "properties": { + "bytes": { + "type": "integer", + "default": 0, + "title": "Number of bytes used for the image heap" + }, + "resources": { + "type": "object", + "default": {}, + "title": "Resource statistics of the image", + "required": [ + "count", + "bytes" + ], + "properties": { + "count": { + "type": "integer", + "default": 0, + "title": "Number of resources included in the image" + }, + "bytes": { + "type": "integer", + "default": 0, + "title": "The number of bytes used for resource data" + } + } + } + } + }, + "debug_info": { + "type": "object", + "default": {}, + "title": "The debug info statistics", + "required": [ + "bytes" + ], + "properties": { + "bytes": { + "type": "integer", + "default": 0, + "title": "The number of bytes used for debug info" + } + } + } + }, + "examples": [ + { + "total_bytes": 13057934, + "code_area": { + "bytes": 4610048, + "compilation_units": 67007 + }, + "image_heap": { + "bytes": 7307264, + "resources": { + "count": 134, + "bytes": 10200 + } + }, + "debug_info": { + "bytes": 1140622 + } + } + ] + }, + "resource_usage": { + "type": "object", + "default": {}, + "title": "Schema for image generator ressource usage statistics", + "required": [ + "cpu", + "garbage_collection", + "memory" + ], + "properties": { + "cpu": { + "type": "object", + "default": {}, + "title": "The cpu resource schema", + "required": [ + "load", + "total_cores" + ], + "properties": { + "load": { + "type": "number", + "default": -1, + "title": "The cpu load of the build system after image building (or -1 if unavailable)" + }, + "total_cores": { + "type": "integer", + "default": 0, + "title": "The total number of cores on the build system" + } + } + }, + "garbage_collection": { + "type": "object", + "default": {}, + "title": "The garbage collection resource usage schema", + "required": [ + "count", + "total_secs" + ], + "properties": { + "count": { + "type": "integer", + "default": 0, + "title": "The number of GC cycles performed during image generation" + }, + "total_secs": { + "type": "number", + "default": 0.0, + "title": "The total time spent in GC (in seconds)" + } + } + }, + "memory": { + "type": "object", + "default": {}, + "title": "The memory statistics schema", + "required": [ + "system_total", + "peak_rss_bytes" + ], + "properties": { + "system_total": { + "type": "integer", + "default": 0, + "title": "The total system memory" + }, + "peak_rss_bytes": { + "type": "integer", + "default": -1, + "title": "Peak RSS value of the image builder process in bytes (or -1 if unavailable)" + } + } + } + }, + "examples": [ + { + "cpu": { + "load": 8.38, + "total_cores": 10 + }, + "garbage_collection": { + "count": 17, + "total_secs": 0.9245 + }, + "memory": { + "system_total": 33254146048, + "peak_rss_bytes": 3506065408 + } + } + ] + } + }, + "examples": [ + { + "general_info": { + "name": "helloworld", + "graalvm_version": "GraalVM 22.2.0-dev Java 17 CE", + "java_version": "17.0.4+5-jvmci-22.2-b03", + "c_compiler": "gcc (linux, x86_64, 9.3.0)", + "garbage_collector": "Serial GC" + }, + "analysis_results": { + "classes": { + "total": 3850, + "reachable": 2839, + "reflection": 28, + "jni": 58 + }, + "fields": { + "total": 6665, + "reachable": 3400, + "reflection": 0, + "jni": 58 + }, + "methods": { + "total": 29038, + "reachable": 12916, + "reflection": 332, + "jni": 52 + } + }, + "image_details": { + "total_bytes": 13057934, + "code_area": { + "bytes": 4610048, + "compilation_units": 67007 + }, + "image_heap": { + "bytes": 7307264, + "resources": { + "count": 134, + "bytes": 10200 + } + }, + "debug_info": { + "bytes": 1140622 + } + }, + "resource_usage": { + "cpu": { + "load": 8.38, + "total_cores": 10 + }, + "garbage_collection": { + "count": 17, + "total_secs": 0.9245 + }, + "memory": { + "system_total": 33254146048, + "peak_rss_bytes": 3506065408 + } + } + }, + { + "general_info": { + "name": "helloworld", + "graalvm_version": "GraalVM 22.2.0-dev Java 17 CE", + "java_version": null, + "c_compiler": null, + "garbage_collector": "Serial GC" + }, + "analysis_results": { + "classes": { + "total": 3850, + "reachable": 2839, + "reflection": 28, + "jni": -1 + }, + "fields": { + "total": 6665, + "reachable": 3400, + "reflection": 0, + "jni": -1 + }, + "methods": { + "total": 29038, + "reachable": 12916, + "reflection": 332, + "jni": -1 + } + }, + "image_details": { + "total_bytes": 13057934, + "code_area": { + "bytes": 4610048, + "compilation_units": 67007 + }, + "image_heap": { + "bytes": 7307264, + "resources": { + "count": 134, + "bytes": 10200 + } + } + }, + "resource_usage": { + "cpu": { + "load": -1, + "total_cores": 10 + }, + "garbage_collection": { + "count": 17, + "total_secs": 0.9245 + }, + "memory": { + "system_total": 33254146048, + "peak_rss_bytes": -1 + } + } + } + ] +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java index 3e587205b031..3f7a82b5bcc9 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java @@ -412,6 +412,11 @@ public Boolean getValue(OptionValues values) { @Option(help = "Print GC warnings as part of build output", type = OptionType.User)// public static final HostedOptionKey BuildOutputGCWarnings = new HostedOptionKey<>(true); + @Option(help = "Print build output statistics as JSON to the specified file. " + + "The output is according to the JSON schema located at: " + + "docs/reference-manual/native-image/assets/build-output-schema-v1.json", type = OptionType.User)// + public static final HostedOptionKey BuildOutputJSONFile = new HostedOptionKey<>(""); + /* * Object and array allocation options. */ diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java index f0100ecfb051..c61eba930261 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java @@ -49,7 +49,8 @@ import java.util.function.Consumer; import java.util.stream.Collectors; -import com.oracle.graal.pointsto.api.PointstoOptions; +import org.graalvm.collections.EconomicMap; +import org.graalvm.collections.Pair; import org.graalvm.compiler.debug.DebugOptions; import org.graalvm.compiler.options.OptionValues; import org.graalvm.compiler.serviceprovider.GraalServices; @@ -58,6 +59,7 @@ import org.graalvm.nativeimage.impl.ImageSingletonsSupport; import com.oracle.graal.pointsto.BigBang; +import com.oracle.graal.pointsto.api.PointstoOptions; import com.oracle.graal.pointsto.meta.AnalysisField; import com.oracle.graal.pointsto.meta.AnalysisMethod; import com.oracle.graal.pointsto.meta.AnalysisUniverse; @@ -77,6 +79,11 @@ import com.oracle.svm.core.option.HostedOptionValues; import com.oracle.svm.core.reflect.ReflectionMetadataDecoder; import com.oracle.svm.core.util.VMError; +import com.oracle.svm.hosted.ProgressReporterJsonStatsHelper.AnalysisResults; +import com.oracle.svm.hosted.ProgressReporterJsonStatsHelper.GeneralInfo; +import com.oracle.svm.hosted.ProgressReporterJsonStatsHelper.ImageDetailKey; +import com.oracle.svm.hosted.ProgressReporterJsonStatsHelper.JsonMetric; +import com.oracle.svm.hosted.ProgressReporterJsonStatsHelper.ResourceUsageKey; import com.oracle.svm.hosted.c.codegen.CCompilerInvoker; import com.oracle.svm.hosted.code.CompileQueue.CompileTask; import com.oracle.svm.hosted.image.AbstractImage.NativeImageKind; @@ -86,6 +93,7 @@ import com.oracle.svm.util.ReflectionUtil; public class ProgressReporter { + private static final long UNAVAILABLE_METRIC = -1; private static final int CHARACTERS_PER_LINE; private static final String HEADLINE_SEPARATOR; private static final String LINE_SEPARATOR; @@ -101,6 +109,7 @@ public class ProgressReporter { private final NativeImageSystemIOWrappers builderIO; + private final ProgressReporterJsonStatsHelper jsonHelper; private final DirectPrinter linePrinter = new DirectPrinter(); private final StagePrinter stagePrinter; private final ColorStrategy colorStrategy; @@ -166,6 +175,11 @@ public static ProgressReporter singleton() { public ProgressReporter(OptionValues options) { builderIO = NativeImageSystemIOWrappers.singleton(); + if (SubstrateOptions.BuildOutputJSONFile.hasBeenSet(options)) { + jsonHelper = new ProgressReporterJsonStatsHelper(SubstrateOptions.BuildOutputJSONFile.getValue(options)); + } else { + jsonHelper = null; + } usePrefix = SubstrateOptions.BuildOutputPrefix.getValue(options); boolean enableColors = !IS_DUMB_TERM && !IS_CI && OS.getCurrent() != OS.WINDOWS && System.getenv("NO_COLOR") == null /* https://no-color.org/ */; @@ -234,6 +248,7 @@ public void printStart(String imageName, NativeImageKind imageKind) { stagePrinter.progressBarStart += outputPrefix.length(); } l().printHeadlineSeparator(); + recordJsonMetrics(GeneralInfo.IMAGE_NAME, imageName); String imageKindName = imageKind.name().toLowerCase().replace('_', ' '); l().blueBold().link("GraalVM Native Image", "https://www.graalvm.org/native-image/").reset() .a(": Generating '").bold().a(imageName).reset().a("' (").doclink(imageKindName, "#glossary-imagekind").a(")...").println(); @@ -249,15 +264,23 @@ public void printUnsuccessfulInitializeEnd() { public void printInitializeEnd() { stagePrinter.end(getTimer(TimerCollection.Registry.CLASSLIST).getTotalTime() + getTimer(TimerCollection.Registry.SETUP).getTotalTime()); - l().a(" ").doclink("Version info", "#glossary-version-info").a(": '").a(ImageSingletons.lookup(VM.class).version).a("'").println(); + String version = ImageSingletons.lookup(VM.class).version; + recordJsonMetrics(GeneralInfo.GRAALVM_VERSION, version); + l().a(" ").doclink("Version info", "#glossary-version-info").a(": '").a(version).a("'").println(); String javaVersion = System.getProperty("java.runtime.version"); + recordJsonMetrics(GeneralInfo.JAVA_VERSION, javaVersion); if (javaVersion != null) { l().a(" ").doclink("Java version info", "#glossary-java-version-info").a(": '").a(javaVersion).a("'").println(); } + String cCompilerShort = null; if (ImageSingletons.contains(CCompilerInvoker.class)) { - l().a(" ").doclink("C compiler", "#glossary-ccompiler").a(": ").a(ImageSingletons.lookup(CCompilerInvoker.class).compilerInfo.getShortDescription()).println(); + cCompilerShort = ImageSingletons.lookup(CCompilerInvoker.class).compilerInfo.getShortDescription(); + l().a(" ").doclink("C compiler", "#glossary-ccompiler").a(": ").a(cCompilerShort).println(); } - l().a(" ").doclink("Garbage collector", "#glossary-gc").a(": ").a(Heap.getHeap().getGC().getName()).println(); + recordJsonMetrics(GeneralInfo.CC, cCompilerShort); + String gcAlgo = Heap.getHeap().getGC().getName(); + recordJsonMetrics(GeneralInfo.GC, gcAlgo); + l().a(" ").doclink("Garbage collector", "#glossary-gc").a(": ").a(gcAlgo).println(); } public void printFeatures(List features) { @@ -305,16 +328,22 @@ private void printAnalysisStatistics(AnalysisUniverse universe, Collection t.isReachable()).count(); long totalClasses = universe.getTypes().size(); + recordJsonMetrics(AnalysisResults.CLASS_TOTAL, totalClasses); + recordJsonMetrics(AnalysisResults.CLASS_REACHABLE, reachableClasses); l().a(actualVsTotalFormat, reachableClasses, reachableClasses / (double) totalClasses * 100, totalClasses) .a(" classes ").doclink("reachable", "#glossary-reachability").println(); Collection fields = universe.getFields(); long reachableFields = fields.stream().filter(f -> f.isAccessed()).count(); int totalFields = fields.size(); + recordJsonMetrics(AnalysisResults.FIELD_TOTAL, totalFields); + recordJsonMetrics(AnalysisResults.FIELD_REACHABLE, reachableFields); l().a(actualVsTotalFormat, reachableFields, reachableFields / (double) totalFields * 100, totalFields) .a(" fields ").doclink("reachable", "#glossary-reachability").println(); Collection methods = universe.getMethods(); long reachableMethods = methods.stream().filter(m -> m.isReachable()).count(); int totalMethods = methods.size(); + recordJsonMetrics(AnalysisResults.METHOD_TOTAL, totalMethods); + recordJsonMetrics(AnalysisResults.METHOD_REACHABLE, reachableMethods); l().a(actualVsTotalFormat, reachableMethods, reachableMethods / (double) totalMethods * 100, totalMethods) .a(" methods ").doclink("reachable", "#glossary-reachability").println(); if (numRuntimeCompiledMethods >= 0) { @@ -323,8 +352,17 @@ private void printAnalysisStatistics(AnalysisUniverse universe, Collection 0 ? numJNIMethods : UNAVAILABLE_METRIC)); + recordJsonMetrics(AnalysisResults.CLASS_JNI, (numJNIClasses >= 0 ? numJNIClasses : UNAVAILABLE_METRIC)); + recordJsonMetrics(AnalysisResults.FIELD_JNI, (numJNIClasses > 0 ? numJNIFields : UNAVAILABLE_METRIC)); if (numJNIClasses > 0) { l().a(classesFieldsMethodFormat, numJNIClasses, numJNIFields, numJNIMethods) .doclink("registered for JNI access", "#glossary-jni-access-registrations").println(); @@ -406,10 +444,14 @@ public void printCreationEnd(int imageSize, int numHeapObjects, long imageHeapSi String format = "%9s (%5.2f%%) for "; l().a(format, Utils.bytesToHuman(codeAreaSize), codeAreaSize / (double) imageSize * 100) .doclink("code area", "#glossary-code-area").a(":%,10d compilation units", numCompilations).println(); - int numResources = Resources.singleton().resources().size(); + EconomicMap, ResourceStorageEntry> resources = Resources.singleton().resources(); + int numResources = resources.size(); + recordJsonMetrics(ImageDetailKey.IMAGE_HEAP_RESOURCE_COUNT, numResources); + recordJsonResourceMetrics(resources); l().a(format, Utils.bytesToHuman(imageHeapSize), imageHeapSize / (double) imageSize * 100) .doclink("image heap", "#glossary-image-heap").a(":%,9d objects and %,d resources", numHeapObjects, numResources).println(); if (debugInfoSize > 0) { + recordJsonMetrics(ImageDetailKey.DEBUG_INFO_SIZE, debugInfoSize); // Optional metric DirectPrinter l = l().a(format, Utils.bytesToHuman(debugInfoSize), debugInfoSize / (double) imageSize * 100) .doclink("debug info", "#glossary-debug-info"); if (debugInfoTimer != null) { @@ -418,6 +460,10 @@ public void printCreationEnd(int imageSize, int numHeapObjects, long imageHeapSi l.println(); } long otherBytes = imageSize - codeAreaSize - imageHeapSize - debugInfoSize; + recordJsonMetrics(ImageDetailKey.IMAGE_HEAP_SIZE, imageHeapSize); + recordJsonMetrics(ImageDetailKey.TOTAL_SIZE, imageSize); + recordJsonMetrics(ImageDetailKey.CODE_AREA_SIZE, codeAreaSize); + recordJsonMetrics(ImageDetailKey.NUM_COMP_UNITS, numCompilations); l().a(format, Utils.bytesToHuman(otherBytes), otherBytes / (double) imageSize * 100) .doclink("other data", "#glossary-other-data").println(); l().a("%9s in total", Utils.bytesToHuman(imageSize)).println(); @@ -556,7 +602,7 @@ public void printEpilog(String imageName, NativeImageGenerator generator, boolea Map> artifacts = generator.getBuildArtifacts(); if (!artifacts.isEmpty()) { l().printLineSeparator(); - printArtifacts(imageName, generator, parsedHostedOptions, artifacts); + printArtifacts(imageName, generator, parsedHostedOptions, artifacts, wasSuccessfulBuild); } l().printHeadlineSeparator(); @@ -573,7 +619,7 @@ public void printEpilog(String imageName, NativeImageGenerator generator, boolea executor.shutdown(); } - private void printArtifacts(String imageName, NativeImageGenerator generator, OptionValues parsedHostedOptions, Map> artifacts) { + private void printArtifacts(String imageName, NativeImageGenerator generator, OptionValues parsedHostedOptions, Map> artifacts, boolean wasSuccessfulBuild) { l().yellowBold().a("Produced artifacts:").reset().println(); // Use TreeMap to sort paths alphabetically. Map> pathToTypes = new TreeMap<>(); @@ -582,6 +628,10 @@ private void printArtifacts(String imageName, NativeImageGenerator generator, Op pathToTypes.computeIfAbsent(path, p -> new ArrayList<>()).add(artifactType.name().toLowerCase()); } }); + if (jsonHelper != null && wasSuccessfulBuild) { + Path jsonMetric = jsonHelper.printToFile(); + pathToTypes.computeIfAbsent(jsonMetric, p -> new ArrayList<>()).add("raw"); + } if (generator.getBigbang() != null && ImageBuildStatistics.Options.CollectImageBuildStatistics.getValue(parsedHostedOptions)) { Path buildStatisticsPath = reportImageBuildStatistics(imageName, generator.getBigbang()); pathToTypes.computeIfAbsent(buildStatisticsPath, p -> new ArrayList<>()).add("raw"); @@ -626,17 +676,23 @@ private void printResourceStatistics() { GCStats gcStats = GCStats.getCurrent(); double gcSeconds = Utils.millisToSeconds(gcStats.totalTimeMillis); CenteredTextPrinter p = new CenteredTextPrinter(); + recordJsonMetrics(ResourceUsageKey.GC_COUNT, gcStats.totalCount); + recordJsonMetrics(ResourceUsageKey.GC_SECS, gcSeconds); p.a("%.1fs (%.1f%% of total time) in %d ", gcSeconds, gcSeconds / totalProcessTimeSeconds * 100, gcStats.totalCount) .doclink("GCs", "#glossary-garbage-collections"); long peakRSS = ProgressReporterCHelper.getPeakRSS(); if (peakRSS >= 0) { p.a(" | ").doclink("Peak RSS", "#glossary-peak-rss").a(": ").a("%.2fGB", Utils.bytesToGiB(peakRSS)); } + recordJsonMetrics(ResourceUsageKey.PEAK_RSS, (peakRSS >= 0 ? peakRSS : UNAVAILABLE_METRIC)); OperatingSystemMXBean osMXBean = ManagementFactory.getOperatingSystemMXBean(); long processCPUTime = ((com.sun.management.OperatingSystemMXBean) osMXBean).getProcessCpuTime(); + double cpuLoad = UNAVAILABLE_METRIC; if (processCPUTime > 0) { - p.a(" | ").doclink("CPU load", "#glossary-cpu-load").a(": ").a("%.2f", Utils.nanosToSeconds(processCPUTime) / totalProcessTimeSeconds); + cpuLoad = Utils.nanosToSeconds(processCPUTime) / totalProcessTimeSeconds; + p.a(" | ").doclink("CPU load", "#glossary-cpu-load").a(": ").a("%.2f", cpuLoad); } + recordJsonMetrics(ResourceUsageKey.CPU_LOAD, cpuLoad); p.flushln(); } @@ -658,6 +714,24 @@ private void checkForExcessiveGarbageCollection() { lastGCStats = currentGCStats; } + private void recordJsonMetrics(JsonMetric metric, Object value) { + if (jsonHelper != null) { + metric.record(jsonHelper, value); + } + } + + private void recordJsonResourceMetrics(EconomicMap, ResourceStorageEntry> resources) { + if (jsonHelper != null) { + long totalResourceBytes = 0; + for (ResourceStorageEntry entry : resources.getValues()) { + for (byte[] data : entry.getData()) { + totalResourceBytes += data.length; + } + } + ImageDetailKey.RESOURCE_SIZE_BYTES.record(jsonHelper, totalResourceBytes); + } + } + /* * HELPERS */ diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporterJsonStatsHelper.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporterJsonStatsHelper.java new file mode 100644 index 000000000000..d95684a7d59c --- /dev/null +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporterJsonStatsHelper.java @@ -0,0 +1,293 @@ +/* + * Copyright (c) 2021, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.hosted; + +import java.io.File; +import java.io.PrintWriter; +import java.lang.management.ManagementFactory; +import java.lang.management.OperatingSystemMXBean; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.function.Consumer; + +import com.oracle.graal.pointsto.reports.ReportUtils; +import com.oracle.svm.core.util.VMError; + +class ProgressReporterJsonStatsHelper { + private static final String ANALYSIS_RESULTS_KEY = "analysis_results"; + private static final String GENERAL_INFO_KEY = "general_info"; + private static final String IMAGE_DETAILS_KEY = "image_details"; + private static final String RESSOURCE_USAGE_KEY = "resource_usage"; + + private final Map statsHolder = new HashMap<>(); + private final String jsonOutputFile; + + ProgressReporterJsonStatsHelper(String outFile) { + this.jsonOutputFile = outFile; + } + + private void recordSystemFixedValues() { + putResourceUsage(ResourceUsageKey.CPU_CORES_TOTAL, Runtime.getRuntime().availableProcessors()); + putResourceUsage(ResourceUsageKey.MEMORY_TOTAL, getTotalSystemMemory()); + } + + @SuppressWarnings("deprecation") + private static long getTotalSystemMemory() { + OperatingSystemMXBean osMXBean = ManagementFactory.getOperatingSystemMXBean(); + return ((com.sun.management.OperatingSystemMXBean) osMXBean).getTotalPhysicalMemorySize(); + } + + @SuppressWarnings("unchecked") + public void putAnalysisResults(AnalysisResults key, long value) { + Map analysisMap = (Map) statsHolder.computeIfAbsent(ANALYSIS_RESULTS_KEY, k -> new HashMap<>()); + Map bucketMap = (Map) analysisMap.computeIfAbsent(key.bucket(), bk -> new HashMap<>()); + bucketMap.put(key.jsonKey(), value); + } + + @SuppressWarnings("unchecked") + public void putGeneralInfo(GeneralInfo info, String value) { + Map generalInfoMap = (Map) statsHolder.computeIfAbsent(GENERAL_INFO_KEY, gi -> new HashMap<>()); + generalInfoMap.put(info.jsonKey(), value); + } + + @SuppressWarnings("unchecked") + public void putImageDetails(ImageDetailKey key, Object value) { + Map imageDetailsMap = (Map) statsHolder.computeIfAbsent(IMAGE_DETAILS_KEY, id -> new HashMap<>()); + if (!key.hasBucket() && !key.hasSubBucket()) { + imageDetailsMap.put(key.jsonKey(), value); + } else if (!key.hasSubBucket()) { + assert key.hasBucket(); + Map bucketMap = (Map) imageDetailsMap.computeIfAbsent(key.bucket(), sb -> new HashMap<>()); + bucketMap.put(key.jsonKey(), value); + } else { + assert key.hasSubBucket(); + Map bucketMap = (Map) imageDetailsMap.computeIfAbsent(key.bucket(), sb -> new HashMap<>()); + Map subbucketMap = (Map) bucketMap.computeIfAbsent(key.subBucket(), sb -> new HashMap<>()); + subbucketMap.put(key.jsonKey(), value); + } + } + + @SuppressWarnings("unchecked") + public void putResourceUsage(ResourceUsageKey key, Object value) { + Map resUsageMap = (Map) statsHolder.computeIfAbsent(RESSOURCE_USAGE_KEY, ru -> new HashMap<>()); + Map subMap = (Map) resUsageMap.computeIfAbsent(key.bucket, k -> new HashMap<>()); + subMap.put(key.key, value); + } + + public Path printToFile() { + recordSystemFixedValues(); + final File file = new File(jsonOutputFile); + String description = "image statistics in json"; + return ReportUtils.report(description, file.getAbsoluteFile().toPath(), getReporter(), false); + } + + private Consumer getReporter() { + return out -> { + out.println(toJson()); + }; + } + + private String toJson() { + return mapToJson(statsHolder); + } + + private String mapToJson(Map map) { + // base case + if (map.isEmpty()) { + return "{}"; + } + StringBuilder builder = new StringBuilder(); + builder.append("{"); + Iterator keySetIter = map.keySet().iterator(); + while (keySetIter.hasNext()) { + String key = keySetIter.next(); + Object value = map.get(key); + builder.append("\"" + key + "\":"); + if (value == null) { + builder.append("null"); // null string + } else if (value instanceof Map) { + // Always a map + @SuppressWarnings("unchecked") + Map subMap = (Map) value; + builder.append(mapToJson(subMap)); + } else if (value instanceof String) { + builder.append("\"" + value + "\""); + } else { + assert value instanceof Number; + // Numeric value + builder.append(value); + } + if (keySetIter.hasNext()) { + builder.append(","); + } + } + builder.append("}"); + return builder.toString(); + } + + interface JsonMetric { + void record(ProgressReporterJsonStatsHelper helper, Object value); + } + + enum ImageDetailKey implements JsonMetric { + TOTAL_SIZE(null, null, "total_bytes"), + CODE_AREA_SIZE("code_area", null, "bytes"), + NUM_COMP_UNITS("code_area", null, "compilation_units"), + IMAGE_HEAP_SIZE("image_heap", null, "bytes"), + DEBUG_INFO_SIZE("debug_info", null, "bytes"), + IMAGE_HEAP_RESOURCE_COUNT("image_heap", "resources", "count"), + RESOURCE_SIZE_BYTES("image_heap", "resources", "bytes"); + + private String bucket; + private String key; + private String subBucket; + + ImageDetailKey(String bucket, String subBucket, String key) { + this.bucket = bucket; + this.key = key; + this.subBucket = subBucket; + } + + /** + * + * @return true iff the json is represented via a sub object. + */ + public boolean hasBucket() { + return bucket != null; + } + + public boolean hasSubBucket() { + return subBucket != null; + } + + public String bucket() { + return bucket; + } + + public String subBucket() { + return subBucket; + } + + public String jsonKey() { + return key; + } + + @Override + public void record(ProgressReporterJsonStatsHelper helper, Object value) { + helper.putImageDetails(this, value); + } + } + + enum ResourceUsageKey implements JsonMetric { + CPU_LOAD("cpu", "load"), + CPU_CORES_TOTAL("cpu", "total_cores"), + GC_COUNT("garbage_collection", "count"), + GC_SECS("garbage_collection", "total_secs"), + PEAK_RSS("memory", "peak_rss_bytes"), + MEMORY_TOTAL("memory", "system_total"); + + private String bucket; + private String key; + + ResourceUsageKey(String bucket, String key) { + this.key = key; + this.bucket = bucket; + } + + @Override + public void record(ProgressReporterJsonStatsHelper helper, Object value) { + helper.putResourceUsage(this, value); + } + + } + + enum AnalysisResults implements JsonMetric { + CLASS_TOTAL("classes", "total"), + CLASS_REACHABLE("classes", "reachable"), + CLASS_JNI("classes", "jni"), + CLASS_REFLECT("classes", "reflection"), + METHOD_TOTAL("methods", "total"), + METHOD_REACHABLE("methods", "reachable"), + METHOD_JNI("methods", "jni"), + METHOD_REFLECT("methods", "reflection"), + FIELD_TOTAL("fields", "total"), + FIELD_REACHABLE("fields", "reachable"), + FIELD_JNI("fields", "jni"), + FIELD_REFLECT("fields", "reflection"); + + private String key; + private String bucket; + + AnalysisResults(String bucket, String key) { + this.key = key; + this.bucket = bucket; + } + + public String jsonKey() { + return key; + } + + public String bucket() { + return bucket; + } + + @Override + public void record(ProgressReporterJsonStatsHelper helper, Object value) { + if (value instanceof Integer) { + helper.putAnalysisResults(this, (Integer) value); + } else if (value instanceof Long) { + helper.putAnalysisResults(this, (Long) value); + } else { + VMError.shouldNotReachHere("Imcompatible type of 'value': " + value.getClass()); + } + } + } + + enum GeneralInfo implements JsonMetric { + IMAGE_NAME("name"), + JAVA_VERSION("java_version"), + GRAALVM_VERSION("graalvm_version"), + GC("garbage_collector"), + CC("c_compiler"); + + private String key; + + GeneralInfo(String key) { + this.key = key; + } + + public String jsonKey() { + return key; + } + + @Override + public void record(ProgressReporterJsonStatsHelper helper, Object value) { + helper.putGeneralInfo(this, (String) value); + } + } +} From 58f38e123dd615c1a8a4f546fa35a5f3616b0ad1 Mon Sep 17 00:00:00 2001 From: Fabio Niephaus Date: Tue, 28 Jun 2022 18:15:11 +0200 Subject: [PATCH 2/8] Clean up JSON build output implementation. --- .../oracle/svm/hosted/ProgressReporter.java | 29 ++++++++++--------- ...r.java => ProgressReporterJsonHelper.java} | 17 ++++++----- 2 files changed, 24 insertions(+), 22 deletions(-) rename substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/{ProgressReporterJsonStatsHelper.java => ProgressReporterJsonHelper.java} (94%) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java index c61eba930261..d4c0d3561c82 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java @@ -24,6 +24,8 @@ */ package com.oracle.svm.hosted; +import static com.oracle.svm.hosted.ProgressReporterJsonHelper.UNAVAILABLE_METRIC; + import java.io.File; import java.io.PrintWriter; import java.lang.management.GarbageCollectorMXBean; @@ -79,11 +81,11 @@ import com.oracle.svm.core.option.HostedOptionValues; import com.oracle.svm.core.reflect.ReflectionMetadataDecoder; import com.oracle.svm.core.util.VMError; -import com.oracle.svm.hosted.ProgressReporterJsonStatsHelper.AnalysisResults; -import com.oracle.svm.hosted.ProgressReporterJsonStatsHelper.GeneralInfo; -import com.oracle.svm.hosted.ProgressReporterJsonStatsHelper.ImageDetailKey; -import com.oracle.svm.hosted.ProgressReporterJsonStatsHelper.JsonMetric; -import com.oracle.svm.hosted.ProgressReporterJsonStatsHelper.ResourceUsageKey; +import com.oracle.svm.hosted.ProgressReporterJsonHelper.AnalysisResults; +import com.oracle.svm.hosted.ProgressReporterJsonHelper.GeneralInfo; +import com.oracle.svm.hosted.ProgressReporterJsonHelper.ImageDetailKey; +import com.oracle.svm.hosted.ProgressReporterJsonHelper.JsonMetric; +import com.oracle.svm.hosted.ProgressReporterJsonHelper.ResourceUsageKey; import com.oracle.svm.hosted.c.codegen.CCompilerInvoker; import com.oracle.svm.hosted.code.CompileQueue.CompileTask; import com.oracle.svm.hosted.image.AbstractImage.NativeImageKind; @@ -93,7 +95,6 @@ import com.oracle.svm.util.ReflectionUtil; public class ProgressReporter { - private static final long UNAVAILABLE_METRIC = -1; private static final int CHARACTERS_PER_LINE; private static final String HEADLINE_SEPARATOR; private static final String LINE_SEPARATOR; @@ -109,7 +110,7 @@ public class ProgressReporter { private final NativeImageSystemIOWrappers builderIO; - private final ProgressReporterJsonStatsHelper jsonHelper; + private final ProgressReporterJsonHelper jsonHelper; private final DirectPrinter linePrinter = new DirectPrinter(); private final StagePrinter stagePrinter; private final ColorStrategy colorStrategy; @@ -176,7 +177,7 @@ public ProgressReporter(OptionValues options) { builderIO = NativeImageSystemIOWrappers.singleton(); if (SubstrateOptions.BuildOutputJSONFile.hasBeenSet(options)) { - jsonHelper = new ProgressReporterJsonStatsHelper(SubstrateOptions.BuildOutputJSONFile.getValue(options)); + jsonHelper = new ProgressReporterJsonHelper(SubstrateOptions.BuildOutputJSONFile.getValue(options)); } else { jsonHelper = null; } @@ -278,9 +279,9 @@ public void printInitializeEnd() { l().a(" ").doclink("C compiler", "#glossary-ccompiler").a(": ").a(cCompilerShort).println(); } recordJsonMetrics(GeneralInfo.CC, cCompilerShort); - String gcAlgo = Heap.getHeap().getGC().getName(); - recordJsonMetrics(GeneralInfo.GC, gcAlgo); - l().a(" ").doclink("Garbage collector", "#glossary-gc").a(": ").a(gcAlgo).println(); + String gcName = Heap.getHeap().getGC().getName(); + recordJsonMetrics(GeneralInfo.GC, gcName); + l().a(" ").doclink("Garbage collector", "#glossary-gc").a(": ").a(gcName).println(); } public void printFeatures(List features) { @@ -360,10 +361,10 @@ private void printAnalysisStatistics(AnalysisUniverse universe, Collection 0 ? numJNIMethods : UNAVAILABLE_METRIC)); + recordJsonMetrics(AnalysisResults.METHOD_JNI, (numJNIMethods >= 0 ? numJNIMethods : UNAVAILABLE_METRIC)); recordJsonMetrics(AnalysisResults.CLASS_JNI, (numJNIClasses >= 0 ? numJNIClasses : UNAVAILABLE_METRIC)); - recordJsonMetrics(AnalysisResults.FIELD_JNI, (numJNIClasses > 0 ? numJNIFields : UNAVAILABLE_METRIC)); - if (numJNIClasses > 0) { + recordJsonMetrics(AnalysisResults.FIELD_JNI, (numJNIFields >= 0 ? numJNIFields : UNAVAILABLE_METRIC)); + if (numJNIClasses >= 0) { l().a(classesFieldsMethodFormat, numJNIClasses, numJNIFields, numJNIMethods) .doclink("registered for JNI access", "#glossary-jni-access-registrations").println(); } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporterJsonStatsHelper.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporterJsonHelper.java similarity index 94% rename from substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporterJsonStatsHelper.java rename to substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporterJsonHelper.java index d95684a7d59c..26a0a7ca49c4 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporterJsonStatsHelper.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporterJsonHelper.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -38,7 +38,8 @@ import com.oracle.graal.pointsto.reports.ReportUtils; import com.oracle.svm.core.util.VMError; -class ProgressReporterJsonStatsHelper { +class ProgressReporterJsonHelper { + protected static final long UNAVAILABLE_METRIC = -1; private static final String ANALYSIS_RESULTS_KEY = "analysis_results"; private static final String GENERAL_INFO_KEY = "general_info"; private static final String IMAGE_DETAILS_KEY = "image_details"; @@ -47,7 +48,7 @@ class ProgressReporterJsonStatsHelper { private final Map statsHolder = new HashMap<>(); private final String jsonOutputFile; - ProgressReporterJsonStatsHelper(String outFile) { + ProgressReporterJsonHelper(String outFile) { this.jsonOutputFile = outFile; } @@ -151,7 +152,7 @@ private String mapToJson(Map map) { } interface JsonMetric { - void record(ProgressReporterJsonStatsHelper helper, Object value); + void record(ProgressReporterJsonHelper helper, Object value); } enum ImageDetailKey implements JsonMetric { @@ -198,7 +199,7 @@ public String jsonKey() { } @Override - public void record(ProgressReporterJsonStatsHelper helper, Object value) { + public void record(ProgressReporterJsonHelper helper, Object value) { helper.putImageDetails(this, value); } } @@ -220,7 +221,7 @@ enum ResourceUsageKey implements JsonMetric { } @Override - public void record(ProgressReporterJsonStatsHelper helper, Object value) { + public void record(ProgressReporterJsonHelper helper, Object value) { helper.putResourceUsage(this, value); } @@ -257,7 +258,7 @@ public String bucket() { } @Override - public void record(ProgressReporterJsonStatsHelper helper, Object value) { + public void record(ProgressReporterJsonHelper helper, Object value) { if (value instanceof Integer) { helper.putAnalysisResults(this, (Integer) value); } else if (value instanceof Long) { @@ -286,7 +287,7 @@ public String jsonKey() { } @Override - public void record(ProgressReporterJsonStatsHelper helper, Object value) { + public void record(ProgressReporterJsonHelper helper, Object value) { helper.putGeneralInfo(this, (String) value); } } From c15736bcf91235d7f378f5b53190222f6f572c8e Mon Sep 17 00:00:00 2001 From: Fabio Niephaus Date: Wed, 29 Jun 2022 10:17:50 +0200 Subject: [PATCH 3/8] Drop `recordJsonResourceMetrics`. and rename `recordJsonMetrics` to `recordJsonMetric`. --- .../oracle/svm/hosted/ProgressReporter.java | 70 ++++++++----------- 1 file changed, 29 insertions(+), 41 deletions(-) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java index d4c0d3561c82..82e7c17a97f1 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java @@ -249,7 +249,7 @@ public void printStart(String imageName, NativeImageKind imageKind) { stagePrinter.progressBarStart += outputPrefix.length(); } l().printHeadlineSeparator(); - recordJsonMetrics(GeneralInfo.IMAGE_NAME, imageName); + recordJsonMetric(GeneralInfo.IMAGE_NAME, imageName); String imageKindName = imageKind.name().toLowerCase().replace('_', ' '); l().blueBold().link("GraalVM Native Image", "https://www.graalvm.org/native-image/").reset() .a(": Generating '").bold().a(imageName).reset().a("' (").doclink(imageKindName, "#glossary-imagekind").a(")...").println(); @@ -266,10 +266,10 @@ public void printUnsuccessfulInitializeEnd() { public void printInitializeEnd() { stagePrinter.end(getTimer(TimerCollection.Registry.CLASSLIST).getTotalTime() + getTimer(TimerCollection.Registry.SETUP).getTotalTime()); String version = ImageSingletons.lookup(VM.class).version; - recordJsonMetrics(GeneralInfo.GRAALVM_VERSION, version); + recordJsonMetric(GeneralInfo.GRAALVM_VERSION, version); l().a(" ").doclink("Version info", "#glossary-version-info").a(": '").a(version).a("'").println(); String javaVersion = System.getProperty("java.runtime.version"); - recordJsonMetrics(GeneralInfo.JAVA_VERSION, javaVersion); + recordJsonMetric(GeneralInfo.JAVA_VERSION, javaVersion); if (javaVersion != null) { l().a(" ").doclink("Java version info", "#glossary-java-version-info").a(": '").a(javaVersion).a("'").println(); } @@ -278,9 +278,9 @@ public void printInitializeEnd() { cCompilerShort = ImageSingletons.lookup(CCompilerInvoker.class).compilerInfo.getShortDescription(); l().a(" ").doclink("C compiler", "#glossary-ccompiler").a(": ").a(cCompilerShort).println(); } - recordJsonMetrics(GeneralInfo.CC, cCompilerShort); + recordJsonMetric(GeneralInfo.CC, cCompilerShort); String gcName = Heap.getHeap().getGC().getName(); - recordJsonMetrics(GeneralInfo.GC, gcName); + recordJsonMetric(GeneralInfo.GC, gcName); l().a(" ").doclink("Garbage collector", "#glossary-gc").a(": ").a(gcName).println(); } @@ -329,22 +329,22 @@ private void printAnalysisStatistics(AnalysisUniverse universe, Collection t.isReachable()).count(); long totalClasses = universe.getTypes().size(); - recordJsonMetrics(AnalysisResults.CLASS_TOTAL, totalClasses); - recordJsonMetrics(AnalysisResults.CLASS_REACHABLE, reachableClasses); + recordJsonMetric(AnalysisResults.CLASS_TOTAL, totalClasses); + recordJsonMetric(AnalysisResults.CLASS_REACHABLE, reachableClasses); l().a(actualVsTotalFormat, reachableClasses, reachableClasses / (double) totalClasses * 100, totalClasses) .a(" classes ").doclink("reachable", "#glossary-reachability").println(); Collection fields = universe.getFields(); long reachableFields = fields.stream().filter(f -> f.isAccessed()).count(); int totalFields = fields.size(); - recordJsonMetrics(AnalysisResults.FIELD_TOTAL, totalFields); - recordJsonMetrics(AnalysisResults.FIELD_REACHABLE, reachableFields); + recordJsonMetric(AnalysisResults.FIELD_TOTAL, totalFields); + recordJsonMetric(AnalysisResults.FIELD_REACHABLE, reachableFields); l().a(actualVsTotalFormat, reachableFields, reachableFields / (double) totalFields * 100, totalFields) .a(" fields ").doclink("reachable", "#glossary-reachability").println(); Collection methods = universe.getMethods(); long reachableMethods = methods.stream().filter(m -> m.isReachable()).count(); int totalMethods = methods.size(); - recordJsonMetrics(AnalysisResults.METHOD_TOTAL, totalMethods); - recordJsonMetrics(AnalysisResults.METHOD_REACHABLE, reachableMethods); + recordJsonMetric(AnalysisResults.METHOD_TOTAL, totalMethods); + recordJsonMetric(AnalysisResults.METHOD_REACHABLE, reachableMethods); l().a(actualVsTotalFormat, reachableMethods, reachableMethods / (double) totalMethods * 100, totalMethods) .a(" methods ").doclink("reachable", "#glossary-reachability").println(); if (numRuntimeCompiledMethods >= 0) { @@ -356,14 +356,14 @@ private void printAnalysisStatistics(AnalysisUniverse universe, Collection= 0 ? numJNIMethods : UNAVAILABLE_METRIC)); - recordJsonMetrics(AnalysisResults.CLASS_JNI, (numJNIClasses >= 0 ? numJNIClasses : UNAVAILABLE_METRIC)); - recordJsonMetrics(AnalysisResults.FIELD_JNI, (numJNIFields >= 0 ? numJNIFields : UNAVAILABLE_METRIC)); + recordJsonMetric(AnalysisResults.METHOD_JNI, (numJNIMethods >= 0 ? numJNIMethods : UNAVAILABLE_METRIC)); + recordJsonMetric(AnalysisResults.CLASS_JNI, (numJNIClasses >= 0 ? numJNIClasses : UNAVAILABLE_METRIC)); + recordJsonMetric(AnalysisResults.FIELD_JNI, (numJNIFields >= 0 ? numJNIFields : UNAVAILABLE_METRIC)); if (numJNIClasses >= 0) { l().a(classesFieldsMethodFormat, numJNIClasses, numJNIFields, numJNIMethods) .doclink("registered for JNI access", "#glossary-jni-access-registrations").println(); @@ -447,12 +447,11 @@ public void printCreationEnd(int imageSize, int numHeapObjects, long imageHeapSi .doclink("code area", "#glossary-code-area").a(":%,10d compilation units", numCompilations).println(); EconomicMap, ResourceStorageEntry> resources = Resources.singleton().resources(); int numResources = resources.size(); - recordJsonMetrics(ImageDetailKey.IMAGE_HEAP_RESOURCE_COUNT, numResources); - recordJsonResourceMetrics(resources); + recordJsonMetric(ImageDetailKey.IMAGE_HEAP_RESOURCE_COUNT, numResources); l().a(format, Utils.bytesToHuman(imageHeapSize), imageHeapSize / (double) imageSize * 100) .doclink("image heap", "#glossary-image-heap").a(":%,9d objects and %,d resources", numHeapObjects, numResources).println(); if (debugInfoSize > 0) { - recordJsonMetrics(ImageDetailKey.DEBUG_INFO_SIZE, debugInfoSize); // Optional metric + recordJsonMetric(ImageDetailKey.DEBUG_INFO_SIZE, debugInfoSize); // Optional metric DirectPrinter l = l().a(format, Utils.bytesToHuman(debugInfoSize), debugInfoSize / (double) imageSize * 100) .doclink("debug info", "#glossary-debug-info"); if (debugInfoTimer != null) { @@ -461,10 +460,10 @@ public void printCreationEnd(int imageSize, int numHeapObjects, long imageHeapSi l.println(); } long otherBytes = imageSize - codeAreaSize - imageHeapSize - debugInfoSize; - recordJsonMetrics(ImageDetailKey.IMAGE_HEAP_SIZE, imageHeapSize); - recordJsonMetrics(ImageDetailKey.TOTAL_SIZE, imageSize); - recordJsonMetrics(ImageDetailKey.CODE_AREA_SIZE, codeAreaSize); - recordJsonMetrics(ImageDetailKey.NUM_COMP_UNITS, numCompilations); + recordJsonMetric(ImageDetailKey.IMAGE_HEAP_SIZE, imageHeapSize); + recordJsonMetric(ImageDetailKey.TOTAL_SIZE, imageSize); + recordJsonMetric(ImageDetailKey.CODE_AREA_SIZE, codeAreaSize); + recordJsonMetric(ImageDetailKey.NUM_COMP_UNITS, numCompilations); l().a(format, Utils.bytesToHuman(otherBytes), otherBytes / (double) imageSize * 100) .doclink("other data", "#glossary-other-data").println(); l().a("%9s in total", Utils.bytesToHuman(imageSize)).println(); @@ -529,6 +528,7 @@ private void calculateHeapBreakdown(Collection heapObjects) { resourcesByteLength += resource.length; } } + recordJsonMetric(ImageDetailKey.RESOURCE_SIZE_BYTES, resourcesByteLength); if (resourcesByteLength > 0) { heapBreakdown.put(BREAKDOWN_BYTE_ARRAY_PREFIX + linkStrategy.asDocLink("embedded resources", "#glossary-embedded-resources"), resourcesByteLength); remainingBytes -= resourcesByteLength; @@ -677,15 +677,15 @@ private void printResourceStatistics() { GCStats gcStats = GCStats.getCurrent(); double gcSeconds = Utils.millisToSeconds(gcStats.totalTimeMillis); CenteredTextPrinter p = new CenteredTextPrinter(); - recordJsonMetrics(ResourceUsageKey.GC_COUNT, gcStats.totalCount); - recordJsonMetrics(ResourceUsageKey.GC_SECS, gcSeconds); + recordJsonMetric(ResourceUsageKey.GC_COUNT, gcStats.totalCount); + recordJsonMetric(ResourceUsageKey.GC_SECS, gcSeconds); p.a("%.1fs (%.1f%% of total time) in %d ", gcSeconds, gcSeconds / totalProcessTimeSeconds * 100, gcStats.totalCount) .doclink("GCs", "#glossary-garbage-collections"); long peakRSS = ProgressReporterCHelper.getPeakRSS(); if (peakRSS >= 0) { p.a(" | ").doclink("Peak RSS", "#glossary-peak-rss").a(": ").a("%.2fGB", Utils.bytesToGiB(peakRSS)); } - recordJsonMetrics(ResourceUsageKey.PEAK_RSS, (peakRSS >= 0 ? peakRSS : UNAVAILABLE_METRIC)); + recordJsonMetric(ResourceUsageKey.PEAK_RSS, (peakRSS >= 0 ? peakRSS : UNAVAILABLE_METRIC)); OperatingSystemMXBean osMXBean = ManagementFactory.getOperatingSystemMXBean(); long processCPUTime = ((com.sun.management.OperatingSystemMXBean) osMXBean).getProcessCpuTime(); double cpuLoad = UNAVAILABLE_METRIC; @@ -693,7 +693,7 @@ private void printResourceStatistics() { cpuLoad = Utils.nanosToSeconds(processCPUTime) / totalProcessTimeSeconds; p.a(" | ").doclink("CPU load", "#glossary-cpu-load").a(": ").a("%.2f", cpuLoad); } - recordJsonMetrics(ResourceUsageKey.CPU_LOAD, cpuLoad); + recordJsonMetric(ResourceUsageKey.CPU_LOAD, cpuLoad); p.flushln(); } @@ -715,24 +715,12 @@ private void checkForExcessiveGarbageCollection() { lastGCStats = currentGCStats; } - private void recordJsonMetrics(JsonMetric metric, Object value) { + private void recordJsonMetric(JsonMetric metric, Object value) { if (jsonHelper != null) { metric.record(jsonHelper, value); } } - private void recordJsonResourceMetrics(EconomicMap, ResourceStorageEntry> resources) { - if (jsonHelper != null) { - long totalResourceBytes = 0; - for (ResourceStorageEntry entry : resources.getValues()) { - for (byte[] data : entry.getData()) { - totalResourceBytes += data.length; - } - } - ImageDetailKey.RESOURCE_SIZE_BYTES.record(jsonHelper, totalResourceBytes); - } - } - /* * HELPERS */ From 9acc9313eaa64452052142bddf93827dd631ab04 Mon Sep 17 00:00:00 2001 From: Fabio Niephaus Date: Wed, 29 Jun 2022 10:12:31 +0200 Subject: [PATCH 4/8] Document the new `-H:BuildOutputJSONFile=` option. --- .../native-image/BuildOutput.md | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docs/reference-manual/native-image/BuildOutput.md b/docs/reference-manual/native-image/BuildOutput.md index 2a6456290337..a4997412e916 100644 --- a/docs/reference-manual/native-image/BuildOutput.md +++ b/docs/reference-manual/native-image/BuildOutput.md @@ -182,6 +182,26 @@ If the [GC statistics](#glossary-garbage-collection) do not show any problems, t The CPU time used by the process divided by the total process time. Increase the number of CPU threads to reduce the time to build the image. + +## Machine-readable Build Output + +The build output produced by the `native-image` builder is designed for humans, can evolve with new releases, and should thus not be parsed in any way by tools. +Instead, use the `-H:BuildOutputJSONFile=` option to instruct the builder to produce machine-readable build output in the JSON format that can be used, for example, for building monitoring tools. +These JSON files validate against the JSON schema defined in [`build-output-schema-v1.json`][json_schema], which can be considered public API. +Note that a JSON file is produced if and only if a build succeeds. + +The following example illustrates how this could be used in a CI/CD build pipeline to check that the number of reachable methods does not exceed a certain threshold: + +```bash +native-image -H:BuildOutputJSONFile=build.json HelloWorld +# ... +cat build.json | python3 -c "import json,sys;c = json.load(sys.stdin)['analysis_results']['methods']['reachable']; assert c < 12000, f'Too many reachable methods: {c}'" +Traceback (most recent call last): + File "", line 1, in +AssertionError: Too many reachable methods: 12128 +``` + + ## Build Output Options Run `native-image --expert-options-all | grep "BuildOutput"` to see all build output options: @@ -199,6 +219,7 @@ Run `native-image --expert-options-all | grep "BuildOutput"` to see all build ou [jdoc_feature]: https://www.graalvm.org/sdk/javadoc/org/graalvm/nativeimage/hosted/Feature.html +[json_schema]: https://github.com/oracle/graal/tree/master/docs/reference-manual/native-image/assets/build-output-schema-v1.json [doc_jni]: https://github.com/oracle/graal/blob/master/docs/reference-manual/native-image/JNI.md [doc_mem_mgmt]: https://github.com/oracle/graal/blob/master/docs/reference-manual/native-image/MemoryManagement.md [doc_shared_library]: https://github.com/oracle/graal/tree/master/docs/reference-manual/native-image#build-a-shared-library From 8d23e3d01dc34ea9d5f52372a364b80b87c24509 Mon Sep 17 00:00:00 2001 From: Fabio Niephaus Date: Wed, 29 Jun 2022 10:38:04 +0200 Subject: [PATCH 5/8] Revise and simplify titles in JSON schema. --- .../assets/build-output-schema-v1.json | 68 +++++++++---------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/docs/reference-manual/native-image/assets/build-output-schema-v1.json b/docs/reference-manual/native-image/assets/build-output-schema-v1.json index 53ffc28d657e..c256d981c4ea 100644 --- a/docs/reference-manual/native-image/assets/build-output-schema-v1.json +++ b/docs/reference-manual/native-image/assets/build-output-schema-v1.json @@ -14,7 +14,7 @@ "general_info": { "type": "object", "default": {}, - "title": "General information about the image build process", + "title": "General information about the build process", "required": [ "name", "graalvm_version", @@ -26,27 +26,27 @@ "name": { "type": "string", "default": "", - "title": "The native image name" + "title": "The name of the native executable" }, "graalvm_version": { "type": "string", "default": "", - "title": "The GraalVM Version" + "title": "The GraalVM Native Image version. This value is also used for the 'java.vm.version' property within the generated image" }, "java_version": { "type": ["string", "null"], "default": null, - "title": "The precise Java version used for running the image builder (or null if not available)" + "title": "The Java version of the Native Image build process (or null if not available)" }, "c_compiler": { "type": ["string", "null"], "default": null, - "title": "The C compiler used for native compilation and linkage (or null if not available)" + "title": "The C compiler used by the Native Image build process (or null if not available)" }, "garbage_collector": { "type": "string", "default": "", - "title": "The garbage collection algorithm in use for the generated native image" + "title": "The garbage collector used within the generated image" } }, "examples": [ @@ -62,7 +62,7 @@ "analysis_results": { "type": "object", "default": {}, - "title": "Schema for the image generator statistics of the analysis phase", + "title": "Information from the analysis", "required": [ "classes", "fields", @@ -72,7 +72,7 @@ "classes": { "type": "object", "default": {}, - "title": "Number of classes detected during analysis phase", + "title": "Class information from the analysis", "required": [ "total", "reachable", @@ -88,24 +88,24 @@ "reachable": { "type": "integer", "default": 0, - "title": "The reachable reachable classes" + "title": "The number of reachable classes" }, "reflection": { "type": "integer", "default": 0, - "title": "The classes registered for reflective access" + "title": "The number of classes registered for reflection" }, "jni": { "type": "integer", "default": -1, - "title": "The classes registered for JNI access (or -1 if unset)" + "title": "The number of classes registered for JNI access (or -1 if unset)" } } }, "fields": { "type": "object", "default": {}, - "title": "Number of fields detected during analysis phase", + "title": "Field information from the analysis", "required": [ "total", "reachable", @@ -121,12 +121,12 @@ "reachable": { "type": "integer", "default": 0, - "title": "The reachable number of fields" + "title": "The number of reachable fields" }, "reflection": { "type": "integer", "default": 0, - "title": "The number of fields registered for reflective access" + "title": "The number of fields registered for reflection" }, "jni": { "type": "integer", @@ -138,7 +138,7 @@ "methods": { "type": "object", "default": {}, - "title": "Number of methods detected during analysis phase", + "title": "Method information from the analysis", "required": [ "total", "reachable", @@ -154,12 +154,12 @@ "reachable": { "type": "integer", "default": 0, - "title": "The reachable number of methods" + "title": "The number of reachable methods" }, "reflection": { "type": "integer", "default": 0, - "title": "The number of methods registered for reflective access" + "title": "The number of methods registered for reflection" }, "jni": { "type": "integer", @@ -195,7 +195,7 @@ "image_details": { "type": "object", "default": {}, - "title": "Schema for statistics of the image creation phase of the image generator", + "title": "Statistics about the generated native image", "required": [ "total_bytes", "code_area", @@ -205,12 +205,12 @@ "total_bytes": { "type": "integer", "default": 0, - "title": "The total image size (in bytes)" + "title": "The total number of bytes of the image" }, "code_area": { "type": "object", "default": {}, - "title": "The code area statistics", + "title": "Code area statistics", "required": [ "bytes", "compilation_units" @@ -219,7 +219,7 @@ "bytes": { "type": "integer", "default": 0, - "title": "Bytes used for the code area" + "title": "The number of bytes used for the code area" }, "compilation_units": { "type": "integer", @@ -231,7 +231,7 @@ "image_heap": { "type": "object", "default": {}, - "title": "The image heap statistics", + "title": "Image heap statistics", "required": [ "bytes", "resources" @@ -240,12 +240,12 @@ "bytes": { "type": "integer", "default": 0, - "title": "Number of bytes used for the image heap" + "title": "The number of bytes used for image heap" }, "resources": { "type": "object", "default": {}, - "title": "Resource statistics of the image", + "title": "Resource statistics", "required": [ "count", "bytes" @@ -254,7 +254,7 @@ "count": { "type": "integer", "default": 0, - "title": "Number of resources included in the image" + "title": "Number of resources embedded in the image" }, "bytes": { "type": "integer", @@ -268,7 +268,7 @@ "debug_info": { "type": "object", "default": {}, - "title": "The debug info statistics", + "title": "Debug info statistics", "required": [ "bytes" ], @@ -304,7 +304,7 @@ "resource_usage": { "type": "object", "default": {}, - "title": "Schema for image generator ressource usage statistics", + "title": "Resource usage statistics", "required": [ "cpu", "garbage_collection", @@ -314,7 +314,7 @@ "cpu": { "type": "object", "default": {}, - "title": "The cpu resource schema", + "title": "CPU usage statistics", "required": [ "load", "total_cores" @@ -323,19 +323,19 @@ "load": { "type": "number", "default": -1, - "title": "The cpu load of the build system after image building (or -1 if unavailable)" + "title": "The CPU load of the build process before terminating (or -1 if unavailable)" }, "total_cores": { "type": "integer", "default": 0, - "title": "The total number of cores on the build system" + "title": "The total number of cores of the build machine" } } }, "garbage_collection": { "type": "object", "default": {}, - "title": "The garbage collection resource usage schema", + "title": "Garbage collection usage statistics", "required": [ "count", "total_secs" @@ -349,14 +349,14 @@ "total_secs": { "type": "number", "default": 0.0, - "title": "The total time spent in GC (in seconds)" + "title": "The total number of seconds spent in GC" } } }, "memory": { "type": "object", "default": {}, - "title": "The memory statistics schema", + "title": "Memory usage statistics", "required": [ "system_total", "peak_rss_bytes" @@ -365,7 +365,7 @@ "system_total": { "type": "integer", "default": 0, - "title": "The total system memory" + "title": "The total number of bytes of available system memory" }, "peak_rss_bytes": { "type": "integer", From 840907375b808dd73fd53100901cc1cff4082fb7 Mon Sep 17 00:00:00 2001 From: Fabio Niephaus Date: Thu, 30 Jun 2022 11:23:08 +0200 Subject: [PATCH 6/8] Incorporate reviewer feedback. --- docs/reference-manual/native-image/BuildOutput.md | 6 +++--- ...utput-schema-v1.json => build-output-schema-v0.9.0.json} | 2 +- .../src/com/oracle/svm/core/SubstrateOptions.java | 2 +- .../src/com/oracle/svm/hosted/ProgressReporter.java | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) rename docs/reference-manual/native-image/assets/{build-output-schema-v1.json => build-output-schema-v0.9.0.json} (99%) diff --git a/docs/reference-manual/native-image/BuildOutput.md b/docs/reference-manual/native-image/BuildOutput.md index a4997412e916..486f8e3b38f3 100644 --- a/docs/reference-manual/native-image/BuildOutput.md +++ b/docs/reference-manual/native-image/BuildOutput.md @@ -187,7 +187,7 @@ Increase the number of CPU threads to reduce the time to build the image. The build output produced by the `native-image` builder is designed for humans, can evolve with new releases, and should thus not be parsed in any way by tools. Instead, use the `-H:BuildOutputJSONFile=` option to instruct the builder to produce machine-readable build output in the JSON format that can be used, for example, for building monitoring tools. -These JSON files validate against the JSON schema defined in [`build-output-schema-v1.json`][json_schema], which can be considered public API. +These JSON files validate against the JSON schema defined in [`build-output-schema-v0.9.0.json`][json_schema]. Note that a JSON file is produced if and only if a build succeeds. The following example illustrates how this could be used in a CI/CD build pipeline to check that the number of reachable methods does not exceed a certain threshold: @@ -211,7 +211,7 @@ Run `native-image --expert-options-all | grep "BuildOutput"` to see all build ou -H:±BuildOutputColorful Colorize build output. Default: + (enabled). -H:±BuildOutputGCWarnings Print GC warnings as part of build output. Default: + (enabled). -H:BuildOutputJSONFile="" Print build output statistics as JSON to the specified file. The output is according to the JSON schema located at: - docs/reference-manual/native-image/assets/build-output-schema-v1.json. + docs/reference-manual/native-image/assets/build-output-schema-v0.9.0.json. -H:±BuildOutputLinks Show links in build output. Default: + (enabled). -H:±BuildOutputPrefix Prefix build output with ':'. Default: - (disabled). -H:±BuildOutputProgress Report progress in build output. Default: + (enabled). @@ -219,7 +219,7 @@ Run `native-image --expert-options-all | grep "BuildOutput"` to see all build ou [jdoc_feature]: https://www.graalvm.org/sdk/javadoc/org/graalvm/nativeimage/hosted/Feature.html -[json_schema]: https://github.com/oracle/graal/tree/master/docs/reference-manual/native-image/assets/build-output-schema-v1.json +[json_schema]: https://github.com/oracle/graal/tree/master/docs/reference-manual/native-image/assets/build-output-schema-v0.9.0.json [doc_jni]: https://github.com/oracle/graal/blob/master/docs/reference-manual/native-image/JNI.md [doc_mem_mgmt]: https://github.com/oracle/graal/blob/master/docs/reference-manual/native-image/MemoryManagement.md [doc_shared_library]: https://github.com/oracle/graal/tree/master/docs/reference-manual/native-image#build-a-shared-library diff --git a/docs/reference-manual/native-image/assets/build-output-schema-v1.json b/docs/reference-manual/native-image/assets/build-output-schema-v0.9.0.json similarity index 99% rename from docs/reference-manual/native-image/assets/build-output-schema-v1.json rename to docs/reference-manual/native-image/assets/build-output-schema-v0.9.0.json index c256d981c4ea..765d36de7f0e 100644 --- a/docs/reference-manual/native-image/assets/build-output-schema-v1.json +++ b/docs/reference-manual/native-image/assets/build-output-schema-v0.9.0.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2019-09/schema", - "$id": "https://github.com/oracle/graal/blob/master/docs/reference-manual/native-image/assets/build-output-schema-v1.json", + "$id": "https://github.com/oracle/graal/blob/master/docs/reference-manual/native-image/assets/build-output-schema-v0.9.0.json", "type": "object", "default": {}, "title": "Schema for the JSON build output of GraalVM Native Image", diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java index 3f7a82b5bcc9..170bc969d5d7 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java @@ -414,7 +414,7 @@ public Boolean getValue(OptionValues values) { @Option(help = "Print build output statistics as JSON to the specified file. " + "The output is according to the JSON schema located at: " + - "docs/reference-manual/native-image/assets/build-output-schema-v1.json", type = OptionType.User)// + "docs/reference-manual/native-image/assets/build-output-schema-v0.9.0.json", type = OptionType.User)// public static final HostedOptionKey BuildOutputJSONFile = new HostedOptionKey<>(""); /* diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java index 82e7c17a97f1..02284d0531ae 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java @@ -631,7 +631,7 @@ private void printArtifacts(String imageName, NativeImageGenerator generator, Op }); if (jsonHelper != null && wasSuccessfulBuild) { Path jsonMetric = jsonHelper.printToFile(); - pathToTypes.computeIfAbsent(jsonMetric, p -> new ArrayList<>()).add("raw"); + pathToTypes.computeIfAbsent(jsonMetric, p -> new ArrayList<>()).add("json"); } if (generator.getBigbang() != null && ImageBuildStatistics.Options.CollectImageBuildStatistics.getValue(parsedHostedOptions)) { Path buildStatisticsPath = reportImageBuildStatistics(imageName, generator.getBigbang()); From 439a0fe4dd2cb77a94f814ca2f25561d76a8fa6b Mon Sep 17 00:00:00 2001 From: Fabio Niephaus Date: Thu, 30 Jun 2022 11:23:21 +0200 Subject: [PATCH 7/8] Add changelog entry. --- substratevm/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/substratevm/CHANGELOG.md b/substratevm/CHANGELOG.md index ca625b4aba96..8ce034ac694d 100644 --- a/substratevm/CHANGELOG.md +++ b/substratevm/CHANGELOG.md @@ -5,6 +5,7 @@ This changelog summarizes major changes to GraalVM Native Image. ## Version 22.3.0 * (GR-35721) Remove old build output style and the `-H:±BuildOutputUseNewStyle` option. * (GR-39390) Red Hat added support for the JFR event `JavaMonitorEnter`. +* (GR-39497) Add `-H:BuildOutputJSONFile=` option for [JSON build output](https://github.com/oracle/graal/edit/master/docs/reference-manual/native-image/BuildOutput.md#machine-readable-build-output). Please feel free to provide feedback so that we can stabilize the schema/API. ## Version 22.2.0 * (GR-20653) Re-enable the usage of all CPU features for JIT compilation on AMD64. From 13dbf2af0d80c485a1bde8adc9d654e1d3595fd6 Mon Sep 17 00:00:00 2001 From: Fabio Niephaus Date: Thu, 30 Jun 2022 14:24:23 +0200 Subject: [PATCH 8/8] Add optional `runtime_compiled_methods` info. --- .../assets/build-output-schema-v0.9.0.json | 25 +++++++++++++++++++ .../oracle/svm/hosted/ProgressReporter.java | 4 ++- .../hosted/ProgressReporterJsonHelper.java | 4 ++- 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/docs/reference-manual/native-image/assets/build-output-schema-v0.9.0.json b/docs/reference-manual/native-image/assets/build-output-schema-v0.9.0.json index 765d36de7f0e..510eb4226f04 100644 --- a/docs/reference-manual/native-image/assets/build-output-schema-v0.9.0.json +++ b/docs/reference-manual/native-image/assets/build-output-schema-v0.9.0.json @@ -279,6 +279,27 @@ "title": "The number of bytes used for debug info" } } + }, + "runtime_compiled_methods": { + "type": "object", + "default": {}, + "title": "Statistics on runtime compiled methods (optional)", + "required": [ + "count", + "graph_encoding_bytes" + ], + "properties": { + "count": { + "type": "integer", + "default": 0, + "title": "Number of runtime compiled methods" + }, + "graph_encoding_bytes": { + "type": "integer", + "default": 0, + "title": "The number of bytes used for graph encodings bytes" + } + } } }, "examples": [ @@ -494,6 +515,10 @@ "count": 134, "bytes": 10200 } + }, + "runtime_compiled_methods": { + "count": 12744, + "graph_encoding_bytes": 1440000 } }, "resource_usage": { diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java index 02284d0531ae..7b99816c112d 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java @@ -348,6 +348,7 @@ private void printAnalysisStatistics(AnalysisUniverse universe, Collection= 0) { + recordJsonMetric(ImageDetailKey.RUNTIME_COMPILED_METHODS_COUNT, numRuntimeCompiledMethods); l().a(actualVsTotalFormat, numRuntimeCompiledMethods, numRuntimeCompiledMethods / (double) totalMethods * 100, totalMethods) .a(" methods included for ").doclink("runtime compilation", "#glossary-runtime-methods").println(); } @@ -533,7 +534,8 @@ private void calculateHeapBreakdown(Collection heapObjects) { heapBreakdown.put(BREAKDOWN_BYTE_ARRAY_PREFIX + linkStrategy.asDocLink("embedded resources", "#glossary-embedded-resources"), resourcesByteLength); remainingBytes -= resourcesByteLength; } - if (graphEncodingByteLength > 0) { + if (graphEncodingByteLength >= 0) { + recordJsonMetric(ImageDetailKey.GRAPH_ENCODING_SIZE, graphEncodingByteLength); heapBreakdown.put(BREAKDOWN_BYTE_ARRAY_PREFIX + linkStrategy.asDocLink("graph encodings", "#glossary-graph-encodings"), graphEncodingByteLength); remainingBytes -= graphEncodingByteLength; } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporterJsonHelper.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporterJsonHelper.java index 26a0a7ca49c4..53ccc9855236 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporterJsonHelper.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporterJsonHelper.java @@ -162,7 +162,9 @@ enum ImageDetailKey implements JsonMetric { IMAGE_HEAP_SIZE("image_heap", null, "bytes"), DEBUG_INFO_SIZE("debug_info", null, "bytes"), IMAGE_HEAP_RESOURCE_COUNT("image_heap", "resources", "count"), - RESOURCE_SIZE_BYTES("image_heap", "resources", "bytes"); + RESOURCE_SIZE_BYTES("image_heap", "resources", "bytes"), + RUNTIME_COMPILED_METHODS_COUNT("runtime_compiled_methods", null, "count"), + GRAPH_ENCODING_SIZE("runtime_compiled_methods", null, "graph_encoding_bytes"); private String bucket; private String key;