diff --git a/docs/reference-manual/native-image/BuildOutput.md b/docs/reference-manual/native-image/BuildOutput.md index 9888c405fdaf..486f8e3b38f3 100644 --- a/docs/reference-manual/native-image/BuildOutput.md +++ b/docs/reference-manual/native-image/BuildOutput.md @@ -182,6 +182,26 @@ If the [GC statistics](#glossary-garbage-collection) do not show any problems, t The CPU time used by the process divided by the total process time. Increase the number of CPU threads to reduce the time to build the image. + +## Machine-readable Build Output + +The build output produced by the `native-image` builder is designed for humans, can evolve with new releases, and should thus not be parsed in any way by tools. +Instead, use the `-H:BuildOutputJSONFile=` option to instruct the builder to produce machine-readable build output in the JSON format that can be used, for example, for building monitoring tools. +These JSON files validate against the JSON schema defined in [`build-output-schema-v0.9.0.json`][json_schema]. +Note that a JSON file is produced if and only if a build succeeds. + +The following example illustrates how this could be used in a CI/CD build pipeline to check that the number of reachable methods does not exceed a certain threshold: + +```bash +native-image -H:BuildOutputJSONFile=build.json HelloWorld +# ... +cat build.json | python3 -c "import json,sys;c = json.load(sys.stdin)['analysis_results']['methods']['reachable']; assert c < 12000, f'Too many reachable methods: {c}'" +Traceback (most recent call last): + File "", line 1, in +AssertionError: Too many reachable methods: 12128 +``` + + ## Build Output Options Run `native-image --expert-options-all | grep "BuildOutput"` to see all build output options: @@ -190,6 +210,8 @@ Run `native-image --expert-options-all | grep "BuildOutput"` to see all build ou -H:±BuildOutputBreakdowns Show code and heap breakdowns as part of the build output. Default: + (enabled). -H:±BuildOutputColorful Colorize build output. Default: + (enabled). -H:±BuildOutputGCWarnings Print GC warnings as part of build output. Default: + (enabled). +-H:BuildOutputJSONFile="" Print build output statistics as JSON to the specified file. The output is according to the JSON schema located at: + docs/reference-manual/native-image/assets/build-output-schema-v0.9.0.json. -H:±BuildOutputLinks Show links in build output. Default: + (enabled). -H:±BuildOutputPrefix Prefix build output with ':'. Default: - (disabled). -H:±BuildOutputProgress Report progress in build output. Default: + (enabled). @@ -197,6 +219,7 @@ Run `native-image --expert-options-all | grep "BuildOutput"` to see all build ou [jdoc_feature]: https://www.graalvm.org/sdk/javadoc/org/graalvm/nativeimage/hosted/Feature.html +[json_schema]: https://github.com/oracle/graal/tree/master/docs/reference-manual/native-image/assets/build-output-schema-v0.9.0.json [doc_jni]: https://github.com/oracle/graal/blob/master/docs/reference-manual/native-image/JNI.md [doc_mem_mgmt]: https://github.com/oracle/graal/blob/master/docs/reference-manual/native-image/MemoryManagement.md [doc_shared_library]: https://github.com/oracle/graal/tree/master/docs/reference-manual/native-image#build-a-shared-library diff --git a/docs/reference-manual/native-image/assets/build-output-schema-v0.9.0.json b/docs/reference-manual/native-image/assets/build-output-schema-v0.9.0.json new file mode 100644 index 000000000000..510eb4226f04 --- /dev/null +++ b/docs/reference-manual/native-image/assets/build-output-schema-v0.9.0.json @@ -0,0 +1,540 @@ +{ + "$schema": "https://json-schema.org/draft/2019-09/schema", + "$id": "https://github.com/oracle/graal/blob/master/docs/reference-manual/native-image/assets/build-output-schema-v0.9.0.json", + "type": "object", + "default": {}, + "title": "Schema for the JSON build output of GraalVM Native Image", + "required": [ + "general_info", + "analysis_results", + "image_details", + "resource_usage" + ], + "properties": { + "general_info": { + "type": "object", + "default": {}, + "title": "General information about the build process", + "required": [ + "name", + "graalvm_version", + "java_version", + "c_compiler", + "garbage_collector" + ], + "properties": { + "name": { + "type": "string", + "default": "", + "title": "The name of the native executable" + }, + "graalvm_version": { + "type": "string", + "default": "", + "title": "The GraalVM Native Image version. This value is also used for the 'java.vm.version' property within the generated image" + }, + "java_version": { + "type": ["string", "null"], + "default": null, + "title": "The Java version of the Native Image build process (or null if not available)" + }, + "c_compiler": { + "type": ["string", "null"], + "default": null, + "title": "The C compiler used by the Native Image build process (or null if not available)" + }, + "garbage_collector": { + "type": "string", + "default": "", + "title": "The garbage collector used within the generated image" + } + }, + "examples": [ + { + "name": "helloworld", + "graalvm_version": "GraalVM 22.2.0-dev Java 17 CE", + "java_version": "17.0.4+5-jvmci-22.2-b03", + "c_compiler": "gcc (linux, x86_64, 9.3.0)", + "garbage_collector": "Serial GC" + } + ] + }, + "analysis_results": { + "type": "object", + "default": {}, + "title": "Information from the analysis", + "required": [ + "classes", + "fields", + "methods" + ], + "properties": { + "classes": { + "type": "object", + "default": {}, + "title": "Class information from the analysis", + "required": [ + "total", + "reachable", + "reflection", + "jni" + ], + "properties": { + "total": { + "type": "integer", + "default": 0, + "title": "The total number of classes" + }, + "reachable": { + "type": "integer", + "default": 0, + "title": "The number of reachable classes" + }, + "reflection": { + "type": "integer", + "default": 0, + "title": "The number of classes registered for reflection" + }, + "jni": { + "type": "integer", + "default": -1, + "title": "The number of classes registered for JNI access (or -1 if unset)" + } + } + }, + "fields": { + "type": "object", + "default": {}, + "title": "Field information from the analysis", + "required": [ + "total", + "reachable", + "reflection", + "jni" + ], + "properties": { + "total": { + "type": "integer", + "default": 0, + "title": "The total number of fields" + }, + "reachable": { + "type": "integer", + "default": 0, + "title": "The number of reachable fields" + }, + "reflection": { + "type": "integer", + "default": 0, + "title": "The number of fields registered for reflection" + }, + "jni": { + "type": "integer", + "default": -1, + "title": "The number of fields registered for JNI access (or -1 if unset)" + } + } + }, + "methods": { + "type": "object", + "default": {}, + "title": "Method information from the analysis", + "required": [ + "total", + "reachable", + "reflection", + "jni" + ], + "properties": { + "total": { + "type": "integer", + "default": 0, + "title": "The total number of methods" + }, + "reachable": { + "type": "integer", + "default": 0, + "title": "The number of reachable methods" + }, + "reflection": { + "type": "integer", + "default": 0, + "title": "The number of methods registered for reflection" + }, + "jni": { + "type": "integer", + "default": -1, + "title": "The number of methods registered for JNI access (or -1 if unset)" + } + } + } + }, + "examples": [ + { + "classes": { + "total": 3850, + "reachable": 2839, + "reflection": 28, + "jni": 58 + }, + "fields": { + "total": 6665, + "reachable": 3400, + "reflection": 0, + "jni": 58 + }, + "methods": { + "total": 29038, + "reachable": 12916, + "reflection": 332, + "jni": 52 + } + } + ] + }, + "image_details": { + "type": "object", + "default": {}, + "title": "Statistics about the generated native image", + "required": [ + "total_bytes", + "code_area", + "image_heap" + ], + "properties": { + "total_bytes": { + "type": "integer", + "default": 0, + "title": "The total number of bytes of the image" + }, + "code_area": { + "type": "object", + "default": {}, + "title": "Code area statistics", + "required": [ + "bytes", + "compilation_units" + ], + "properties": { + "bytes": { + "type": "integer", + "default": 0, + "title": "The number of bytes used for the code area" + }, + "compilation_units": { + "type": "integer", + "default": 0, + "title": "The number of compilation units in the image" + } + } + }, + "image_heap": { + "type": "object", + "default": {}, + "title": "Image heap statistics", + "required": [ + "bytes", + "resources" + ], + "properties": { + "bytes": { + "type": "integer", + "default": 0, + "title": "The number of bytes used for image heap" + }, + "resources": { + "type": "object", + "default": {}, + "title": "Resource statistics", + "required": [ + "count", + "bytes" + ], + "properties": { + "count": { + "type": "integer", + "default": 0, + "title": "Number of resources embedded in the image" + }, + "bytes": { + "type": "integer", + "default": 0, + "title": "The number of bytes used for resource data" + } + } + } + } + }, + "debug_info": { + "type": "object", + "default": {}, + "title": "Debug info statistics", + "required": [ + "bytes" + ], + "properties": { + "bytes": { + "type": "integer", + "default": 0, + "title": "The number of bytes used for debug info" + } + } + }, + "runtime_compiled_methods": { + "type": "object", + "default": {}, + "title": "Statistics on runtime compiled methods (optional)", + "required": [ + "count", + "graph_encoding_bytes" + ], + "properties": { + "count": { + "type": "integer", + "default": 0, + "title": "Number of runtime compiled methods" + }, + "graph_encoding_bytes": { + "type": "integer", + "default": 0, + "title": "The number of bytes used for graph encodings bytes" + } + } + } + }, + "examples": [ + { + "total_bytes": 13057934, + "code_area": { + "bytes": 4610048, + "compilation_units": 67007 + }, + "image_heap": { + "bytes": 7307264, + "resources": { + "count": 134, + "bytes": 10200 + } + }, + "debug_info": { + "bytes": 1140622 + } + } + ] + }, + "resource_usage": { + "type": "object", + "default": {}, + "title": "Resource usage statistics", + "required": [ + "cpu", + "garbage_collection", + "memory" + ], + "properties": { + "cpu": { + "type": "object", + "default": {}, + "title": "CPU usage statistics", + "required": [ + "load", + "total_cores" + ], + "properties": { + "load": { + "type": "number", + "default": -1, + "title": "The CPU load of the build process before terminating (or -1 if unavailable)" + }, + "total_cores": { + "type": "integer", + "default": 0, + "title": "The total number of cores of the build machine" + } + } + }, + "garbage_collection": { + "type": "object", + "default": {}, + "title": "Garbage collection usage statistics", + "required": [ + "count", + "total_secs" + ], + "properties": { + "count": { + "type": "integer", + "default": 0, + "title": "The number of GC cycles performed during image generation" + }, + "total_secs": { + "type": "number", + "default": 0.0, + "title": "The total number of seconds spent in GC" + } + } + }, + "memory": { + "type": "object", + "default": {}, + "title": "Memory usage statistics", + "required": [ + "system_total", + "peak_rss_bytes" + ], + "properties": { + "system_total": { + "type": "integer", + "default": 0, + "title": "The total number of bytes of available system memory" + }, + "peak_rss_bytes": { + "type": "integer", + "default": -1, + "title": "Peak RSS value of the image builder process in bytes (or -1 if unavailable)" + } + } + } + }, + "examples": [ + { + "cpu": { + "load": 8.38, + "total_cores": 10 + }, + "garbage_collection": { + "count": 17, + "total_secs": 0.9245 + }, + "memory": { + "system_total": 33254146048, + "peak_rss_bytes": 3506065408 + } + } + ] + } + }, + "examples": [ + { + "general_info": { + "name": "helloworld", + "graalvm_version": "GraalVM 22.2.0-dev Java 17 CE", + "java_version": "17.0.4+5-jvmci-22.2-b03", + "c_compiler": "gcc (linux, x86_64, 9.3.0)", + "garbage_collector": "Serial GC" + }, + "analysis_results": { + "classes": { + "total": 3850, + "reachable": 2839, + "reflection": 28, + "jni": 58 + }, + "fields": { + "total": 6665, + "reachable": 3400, + "reflection": 0, + "jni": 58 + }, + "methods": { + "total": 29038, + "reachable": 12916, + "reflection": 332, + "jni": 52 + } + }, + "image_details": { + "total_bytes": 13057934, + "code_area": { + "bytes": 4610048, + "compilation_units": 67007 + }, + "image_heap": { + "bytes": 7307264, + "resources": { + "count": 134, + "bytes": 10200 + } + }, + "debug_info": { + "bytes": 1140622 + } + }, + "resource_usage": { + "cpu": { + "load": 8.38, + "total_cores": 10 + }, + "garbage_collection": { + "count": 17, + "total_secs": 0.9245 + }, + "memory": { + "system_total": 33254146048, + "peak_rss_bytes": 3506065408 + } + } + }, + { + "general_info": { + "name": "helloworld", + "graalvm_version": "GraalVM 22.2.0-dev Java 17 CE", + "java_version": null, + "c_compiler": null, + "garbage_collector": "Serial GC" + }, + "analysis_results": { + "classes": { + "total": 3850, + "reachable": 2839, + "reflection": 28, + "jni": -1 + }, + "fields": { + "total": 6665, + "reachable": 3400, + "reflection": 0, + "jni": -1 + }, + "methods": { + "total": 29038, + "reachable": 12916, + "reflection": 332, + "jni": -1 + } + }, + "image_details": { + "total_bytes": 13057934, + "code_area": { + "bytes": 4610048, + "compilation_units": 67007 + }, + "image_heap": { + "bytes": 7307264, + "resources": { + "count": 134, + "bytes": 10200 + } + }, + "runtime_compiled_methods": { + "count": 12744, + "graph_encoding_bytes": 1440000 + } + }, + "resource_usage": { + "cpu": { + "load": -1, + "total_cores": 10 + }, + "garbage_collection": { + "count": 17, + "total_secs": 0.9245 + }, + "memory": { + "system_total": 33254146048, + "peak_rss_bytes": -1 + } + } + } + ] +} diff --git a/substratevm/CHANGELOG.md b/substratevm/CHANGELOG.md index ca625b4aba96..8ce034ac694d 100644 --- a/substratevm/CHANGELOG.md +++ b/substratevm/CHANGELOG.md @@ -5,6 +5,7 @@ This changelog summarizes major changes to GraalVM Native Image. ## Version 22.3.0 * (GR-35721) Remove old build output style and the `-H:±BuildOutputUseNewStyle` option. * (GR-39390) Red Hat added support for the JFR event `JavaMonitorEnter`. +* (GR-39497) Add `-H:BuildOutputJSONFile=` option for [JSON build output](https://github.com/oracle/graal/edit/master/docs/reference-manual/native-image/BuildOutput.md#machine-readable-build-output). Please feel free to provide feedback so that we can stabilize the schema/API. ## Version 22.2.0 * (GR-20653) Re-enable the usage of all CPU features for JIT compilation on AMD64. diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java index 3e587205b031..170bc969d5d7 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java @@ -412,6 +412,11 @@ public Boolean getValue(OptionValues values) { @Option(help = "Print GC warnings as part of build output", type = OptionType.User)// public static final HostedOptionKey BuildOutputGCWarnings = new HostedOptionKey<>(true); + @Option(help = "Print build output statistics as JSON to the specified file. " + + "The output is according to the JSON schema located at: " + + "docs/reference-manual/native-image/assets/build-output-schema-v0.9.0.json", type = OptionType.User)// + public static final HostedOptionKey BuildOutputJSONFile = new HostedOptionKey<>(""); + /* * Object and array allocation options. */ diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java index f0100ecfb051..7b99816c112d 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java @@ -24,6 +24,8 @@ */ package com.oracle.svm.hosted; +import static com.oracle.svm.hosted.ProgressReporterJsonHelper.UNAVAILABLE_METRIC; + import java.io.File; import java.io.PrintWriter; import java.lang.management.GarbageCollectorMXBean; @@ -49,7 +51,8 @@ import java.util.function.Consumer; import java.util.stream.Collectors; -import com.oracle.graal.pointsto.api.PointstoOptions; +import org.graalvm.collections.EconomicMap; +import org.graalvm.collections.Pair; import org.graalvm.compiler.debug.DebugOptions; import org.graalvm.compiler.options.OptionValues; import org.graalvm.compiler.serviceprovider.GraalServices; @@ -58,6 +61,7 @@ import org.graalvm.nativeimage.impl.ImageSingletonsSupport; import com.oracle.graal.pointsto.BigBang; +import com.oracle.graal.pointsto.api.PointstoOptions; import com.oracle.graal.pointsto.meta.AnalysisField; import com.oracle.graal.pointsto.meta.AnalysisMethod; import com.oracle.graal.pointsto.meta.AnalysisUniverse; @@ -77,6 +81,11 @@ import com.oracle.svm.core.option.HostedOptionValues; import com.oracle.svm.core.reflect.ReflectionMetadataDecoder; import com.oracle.svm.core.util.VMError; +import com.oracle.svm.hosted.ProgressReporterJsonHelper.AnalysisResults; +import com.oracle.svm.hosted.ProgressReporterJsonHelper.GeneralInfo; +import com.oracle.svm.hosted.ProgressReporterJsonHelper.ImageDetailKey; +import com.oracle.svm.hosted.ProgressReporterJsonHelper.JsonMetric; +import com.oracle.svm.hosted.ProgressReporterJsonHelper.ResourceUsageKey; import com.oracle.svm.hosted.c.codegen.CCompilerInvoker; import com.oracle.svm.hosted.code.CompileQueue.CompileTask; import com.oracle.svm.hosted.image.AbstractImage.NativeImageKind; @@ -101,6 +110,7 @@ public class ProgressReporter { private final NativeImageSystemIOWrappers builderIO; + private final ProgressReporterJsonHelper jsonHelper; private final DirectPrinter linePrinter = new DirectPrinter(); private final StagePrinter stagePrinter; private final ColorStrategy colorStrategy; @@ -166,6 +176,11 @@ public static ProgressReporter singleton() { public ProgressReporter(OptionValues options) { builderIO = NativeImageSystemIOWrappers.singleton(); + if (SubstrateOptions.BuildOutputJSONFile.hasBeenSet(options)) { + jsonHelper = new ProgressReporterJsonHelper(SubstrateOptions.BuildOutputJSONFile.getValue(options)); + } else { + jsonHelper = null; + } usePrefix = SubstrateOptions.BuildOutputPrefix.getValue(options); boolean enableColors = !IS_DUMB_TERM && !IS_CI && OS.getCurrent() != OS.WINDOWS && System.getenv("NO_COLOR") == null /* https://no-color.org/ */; @@ -234,6 +249,7 @@ public void printStart(String imageName, NativeImageKind imageKind) { stagePrinter.progressBarStart += outputPrefix.length(); } l().printHeadlineSeparator(); + recordJsonMetric(GeneralInfo.IMAGE_NAME, imageName); String imageKindName = imageKind.name().toLowerCase().replace('_', ' '); l().blueBold().link("GraalVM Native Image", "https://www.graalvm.org/native-image/").reset() .a(": Generating '").bold().a(imageName).reset().a("' (").doclink(imageKindName, "#glossary-imagekind").a(")...").println(); @@ -249,15 +265,23 @@ public void printUnsuccessfulInitializeEnd() { public void printInitializeEnd() { stagePrinter.end(getTimer(TimerCollection.Registry.CLASSLIST).getTotalTime() + getTimer(TimerCollection.Registry.SETUP).getTotalTime()); - l().a(" ").doclink("Version info", "#glossary-version-info").a(": '").a(ImageSingletons.lookup(VM.class).version).a("'").println(); + String version = ImageSingletons.lookup(VM.class).version; + recordJsonMetric(GeneralInfo.GRAALVM_VERSION, version); + l().a(" ").doclink("Version info", "#glossary-version-info").a(": '").a(version).a("'").println(); String javaVersion = System.getProperty("java.runtime.version"); + recordJsonMetric(GeneralInfo.JAVA_VERSION, javaVersion); if (javaVersion != null) { l().a(" ").doclink("Java version info", "#glossary-java-version-info").a(": '").a(javaVersion).a("'").println(); } + String cCompilerShort = null; if (ImageSingletons.contains(CCompilerInvoker.class)) { - l().a(" ").doclink("C compiler", "#glossary-ccompiler").a(": ").a(ImageSingletons.lookup(CCompilerInvoker.class).compilerInfo.getShortDescription()).println(); + cCompilerShort = ImageSingletons.lookup(CCompilerInvoker.class).compilerInfo.getShortDescription(); + l().a(" ").doclink("C compiler", "#glossary-ccompiler").a(": ").a(cCompilerShort).println(); } - l().a(" ").doclink("Garbage collector", "#glossary-gc").a(": ").a(Heap.getHeap().getGC().getName()).println(); + recordJsonMetric(GeneralInfo.CC, cCompilerShort); + String gcName = Heap.getHeap().getGC().getName(); + recordJsonMetric(GeneralInfo.GC, gcName); + l().a(" ").doclink("Garbage collector", "#glossary-gc").a(": ").a(gcName).println(); } public void printFeatures(List features) { @@ -305,27 +329,43 @@ private void printAnalysisStatistics(AnalysisUniverse universe, Collection t.isReachable()).count(); long totalClasses = universe.getTypes().size(); + recordJsonMetric(AnalysisResults.CLASS_TOTAL, totalClasses); + recordJsonMetric(AnalysisResults.CLASS_REACHABLE, reachableClasses); l().a(actualVsTotalFormat, reachableClasses, reachableClasses / (double) totalClasses * 100, totalClasses) .a(" classes ").doclink("reachable", "#glossary-reachability").println(); Collection fields = universe.getFields(); long reachableFields = fields.stream().filter(f -> f.isAccessed()).count(); int totalFields = fields.size(); + recordJsonMetric(AnalysisResults.FIELD_TOTAL, totalFields); + recordJsonMetric(AnalysisResults.FIELD_REACHABLE, reachableFields); l().a(actualVsTotalFormat, reachableFields, reachableFields / (double) totalFields * 100, totalFields) .a(" fields ").doclink("reachable", "#glossary-reachability").println(); Collection methods = universe.getMethods(); long reachableMethods = methods.stream().filter(m -> m.isReachable()).count(); int totalMethods = methods.size(); + recordJsonMetric(AnalysisResults.METHOD_TOTAL, totalMethods); + recordJsonMetric(AnalysisResults.METHOD_REACHABLE, reachableMethods); l().a(actualVsTotalFormat, reachableMethods, reachableMethods / (double) totalMethods * 100, totalMethods) .a(" methods ").doclink("reachable", "#glossary-reachability").println(); if (numRuntimeCompiledMethods >= 0) { + recordJsonMetric(ImageDetailKey.RUNTIME_COMPILED_METHODS_COUNT, numRuntimeCompiledMethods); l().a(actualVsTotalFormat, numRuntimeCompiledMethods, numRuntimeCompiledMethods / (double) totalMethods * 100, totalMethods) .a(" methods included for ").doclink("runtime compilation", "#glossary-runtime-methods").println(); } String classesFieldsMethodFormat = "%,8d classes, %,5d fields, and %,5d methods "; InternalRuntimeReflectionSupport rs = ImageSingletons.lookup(InternalRuntimeReflectionSupport.class); - l().a(classesFieldsMethodFormat, rs.getReflectionClassesCount(), rs.getReflectionFieldsCount(), rs.getReflectionMethodsCount()) + int reflectClassesCount = rs.getReflectionClassesCount(); + int reflectFieldsCount = rs.getReflectionFieldsCount(); + int reflectMethodsCount = rs.getReflectionMethodsCount(); + recordJsonMetric(AnalysisResults.METHOD_REFLECT, reflectMethodsCount); + recordJsonMetric(AnalysisResults.CLASS_REFLECT, reflectClassesCount); + recordJsonMetric(AnalysisResults.FIELD_REFLECT, reflectFieldsCount); + l().a(classesFieldsMethodFormat, reflectClassesCount, reflectFieldsCount, reflectMethodsCount) .doclink("registered for reflection", "#glossary-reflection-registrations").println(); - if (numJNIClasses > 0) { + recordJsonMetric(AnalysisResults.METHOD_JNI, (numJNIMethods >= 0 ? numJNIMethods : UNAVAILABLE_METRIC)); + recordJsonMetric(AnalysisResults.CLASS_JNI, (numJNIClasses >= 0 ? numJNIClasses : UNAVAILABLE_METRIC)); + recordJsonMetric(AnalysisResults.FIELD_JNI, (numJNIFields >= 0 ? numJNIFields : UNAVAILABLE_METRIC)); + if (numJNIClasses >= 0) { l().a(classesFieldsMethodFormat, numJNIClasses, numJNIFields, numJNIMethods) .doclink("registered for JNI access", "#glossary-jni-access-registrations").println(); } @@ -406,10 +446,13 @@ public void printCreationEnd(int imageSize, int numHeapObjects, long imageHeapSi String format = "%9s (%5.2f%%) for "; l().a(format, Utils.bytesToHuman(codeAreaSize), codeAreaSize / (double) imageSize * 100) .doclink("code area", "#glossary-code-area").a(":%,10d compilation units", numCompilations).println(); - int numResources = Resources.singleton().resources().size(); + EconomicMap, ResourceStorageEntry> resources = Resources.singleton().resources(); + int numResources = resources.size(); + recordJsonMetric(ImageDetailKey.IMAGE_HEAP_RESOURCE_COUNT, numResources); l().a(format, Utils.bytesToHuman(imageHeapSize), imageHeapSize / (double) imageSize * 100) .doclink("image heap", "#glossary-image-heap").a(":%,9d objects and %,d resources", numHeapObjects, numResources).println(); if (debugInfoSize > 0) { + recordJsonMetric(ImageDetailKey.DEBUG_INFO_SIZE, debugInfoSize); // Optional metric DirectPrinter l = l().a(format, Utils.bytesToHuman(debugInfoSize), debugInfoSize / (double) imageSize * 100) .doclink("debug info", "#glossary-debug-info"); if (debugInfoTimer != null) { @@ -418,6 +461,10 @@ public void printCreationEnd(int imageSize, int numHeapObjects, long imageHeapSi l.println(); } long otherBytes = imageSize - codeAreaSize - imageHeapSize - debugInfoSize; + recordJsonMetric(ImageDetailKey.IMAGE_HEAP_SIZE, imageHeapSize); + recordJsonMetric(ImageDetailKey.TOTAL_SIZE, imageSize); + recordJsonMetric(ImageDetailKey.CODE_AREA_SIZE, codeAreaSize); + recordJsonMetric(ImageDetailKey.NUM_COMP_UNITS, numCompilations); l().a(format, Utils.bytesToHuman(otherBytes), otherBytes / (double) imageSize * 100) .doclink("other data", "#glossary-other-data").println(); l().a("%9s in total", Utils.bytesToHuman(imageSize)).println(); @@ -482,11 +529,13 @@ private void calculateHeapBreakdown(Collection heapObjects) { resourcesByteLength += resource.length; } } + recordJsonMetric(ImageDetailKey.RESOURCE_SIZE_BYTES, resourcesByteLength); if (resourcesByteLength > 0) { heapBreakdown.put(BREAKDOWN_BYTE_ARRAY_PREFIX + linkStrategy.asDocLink("embedded resources", "#glossary-embedded-resources"), resourcesByteLength); remainingBytes -= resourcesByteLength; } - if (graphEncodingByteLength > 0) { + if (graphEncodingByteLength >= 0) { + recordJsonMetric(ImageDetailKey.GRAPH_ENCODING_SIZE, graphEncodingByteLength); heapBreakdown.put(BREAKDOWN_BYTE_ARRAY_PREFIX + linkStrategy.asDocLink("graph encodings", "#glossary-graph-encodings"), graphEncodingByteLength); remainingBytes -= graphEncodingByteLength; } @@ -556,7 +605,7 @@ public void printEpilog(String imageName, NativeImageGenerator generator, boolea Map> artifacts = generator.getBuildArtifacts(); if (!artifacts.isEmpty()) { l().printLineSeparator(); - printArtifacts(imageName, generator, parsedHostedOptions, artifacts); + printArtifacts(imageName, generator, parsedHostedOptions, artifacts, wasSuccessfulBuild); } l().printHeadlineSeparator(); @@ -573,7 +622,7 @@ public void printEpilog(String imageName, NativeImageGenerator generator, boolea executor.shutdown(); } - private void printArtifacts(String imageName, NativeImageGenerator generator, OptionValues parsedHostedOptions, Map> artifacts) { + private void printArtifacts(String imageName, NativeImageGenerator generator, OptionValues parsedHostedOptions, Map> artifacts, boolean wasSuccessfulBuild) { l().yellowBold().a("Produced artifacts:").reset().println(); // Use TreeMap to sort paths alphabetically. Map> pathToTypes = new TreeMap<>(); @@ -582,6 +631,10 @@ private void printArtifacts(String imageName, NativeImageGenerator generator, Op pathToTypes.computeIfAbsent(path, p -> new ArrayList<>()).add(artifactType.name().toLowerCase()); } }); + if (jsonHelper != null && wasSuccessfulBuild) { + Path jsonMetric = jsonHelper.printToFile(); + pathToTypes.computeIfAbsent(jsonMetric, p -> new ArrayList<>()).add("json"); + } if (generator.getBigbang() != null && ImageBuildStatistics.Options.CollectImageBuildStatistics.getValue(parsedHostedOptions)) { Path buildStatisticsPath = reportImageBuildStatistics(imageName, generator.getBigbang()); pathToTypes.computeIfAbsent(buildStatisticsPath, p -> new ArrayList<>()).add("raw"); @@ -626,17 +679,23 @@ private void printResourceStatistics() { GCStats gcStats = GCStats.getCurrent(); double gcSeconds = Utils.millisToSeconds(gcStats.totalTimeMillis); CenteredTextPrinter p = new CenteredTextPrinter(); + recordJsonMetric(ResourceUsageKey.GC_COUNT, gcStats.totalCount); + recordJsonMetric(ResourceUsageKey.GC_SECS, gcSeconds); p.a("%.1fs (%.1f%% of total time) in %d ", gcSeconds, gcSeconds / totalProcessTimeSeconds * 100, gcStats.totalCount) .doclink("GCs", "#glossary-garbage-collections"); long peakRSS = ProgressReporterCHelper.getPeakRSS(); if (peakRSS >= 0) { p.a(" | ").doclink("Peak RSS", "#glossary-peak-rss").a(": ").a("%.2fGB", Utils.bytesToGiB(peakRSS)); } + recordJsonMetric(ResourceUsageKey.PEAK_RSS, (peakRSS >= 0 ? peakRSS : UNAVAILABLE_METRIC)); OperatingSystemMXBean osMXBean = ManagementFactory.getOperatingSystemMXBean(); long processCPUTime = ((com.sun.management.OperatingSystemMXBean) osMXBean).getProcessCpuTime(); + double cpuLoad = UNAVAILABLE_METRIC; if (processCPUTime > 0) { - p.a(" | ").doclink("CPU load", "#glossary-cpu-load").a(": ").a("%.2f", Utils.nanosToSeconds(processCPUTime) / totalProcessTimeSeconds); + cpuLoad = Utils.nanosToSeconds(processCPUTime) / totalProcessTimeSeconds; + p.a(" | ").doclink("CPU load", "#glossary-cpu-load").a(": ").a("%.2f", cpuLoad); } + recordJsonMetric(ResourceUsageKey.CPU_LOAD, cpuLoad); p.flushln(); } @@ -658,6 +717,12 @@ private void checkForExcessiveGarbageCollection() { lastGCStats = currentGCStats; } + private void recordJsonMetric(JsonMetric metric, Object value) { + if (jsonHelper != null) { + metric.record(jsonHelper, value); + } + } + /* * HELPERS */ diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporterJsonHelper.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporterJsonHelper.java new file mode 100644 index 000000000000..53ccc9855236 --- /dev/null +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporterJsonHelper.java @@ -0,0 +1,296 @@ +/* + * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.hosted; + +import java.io.File; +import java.io.PrintWriter; +import java.lang.management.ManagementFactory; +import java.lang.management.OperatingSystemMXBean; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.function.Consumer; + +import com.oracle.graal.pointsto.reports.ReportUtils; +import com.oracle.svm.core.util.VMError; + +class ProgressReporterJsonHelper { + protected static final long UNAVAILABLE_METRIC = -1; + private static final String ANALYSIS_RESULTS_KEY = "analysis_results"; + private static final String GENERAL_INFO_KEY = "general_info"; + private static final String IMAGE_DETAILS_KEY = "image_details"; + private static final String RESSOURCE_USAGE_KEY = "resource_usage"; + + private final Map statsHolder = new HashMap<>(); + private final String jsonOutputFile; + + ProgressReporterJsonHelper(String outFile) { + this.jsonOutputFile = outFile; + } + + private void recordSystemFixedValues() { + putResourceUsage(ResourceUsageKey.CPU_CORES_TOTAL, Runtime.getRuntime().availableProcessors()); + putResourceUsage(ResourceUsageKey.MEMORY_TOTAL, getTotalSystemMemory()); + } + + @SuppressWarnings("deprecation") + private static long getTotalSystemMemory() { + OperatingSystemMXBean osMXBean = ManagementFactory.getOperatingSystemMXBean(); + return ((com.sun.management.OperatingSystemMXBean) osMXBean).getTotalPhysicalMemorySize(); + } + + @SuppressWarnings("unchecked") + public void putAnalysisResults(AnalysisResults key, long value) { + Map analysisMap = (Map) statsHolder.computeIfAbsent(ANALYSIS_RESULTS_KEY, k -> new HashMap<>()); + Map bucketMap = (Map) analysisMap.computeIfAbsent(key.bucket(), bk -> new HashMap<>()); + bucketMap.put(key.jsonKey(), value); + } + + @SuppressWarnings("unchecked") + public void putGeneralInfo(GeneralInfo info, String value) { + Map generalInfoMap = (Map) statsHolder.computeIfAbsent(GENERAL_INFO_KEY, gi -> new HashMap<>()); + generalInfoMap.put(info.jsonKey(), value); + } + + @SuppressWarnings("unchecked") + public void putImageDetails(ImageDetailKey key, Object value) { + Map imageDetailsMap = (Map) statsHolder.computeIfAbsent(IMAGE_DETAILS_KEY, id -> new HashMap<>()); + if (!key.hasBucket() && !key.hasSubBucket()) { + imageDetailsMap.put(key.jsonKey(), value); + } else if (!key.hasSubBucket()) { + assert key.hasBucket(); + Map bucketMap = (Map) imageDetailsMap.computeIfAbsent(key.bucket(), sb -> new HashMap<>()); + bucketMap.put(key.jsonKey(), value); + } else { + assert key.hasSubBucket(); + Map bucketMap = (Map) imageDetailsMap.computeIfAbsent(key.bucket(), sb -> new HashMap<>()); + Map subbucketMap = (Map) bucketMap.computeIfAbsent(key.subBucket(), sb -> new HashMap<>()); + subbucketMap.put(key.jsonKey(), value); + } + } + + @SuppressWarnings("unchecked") + public void putResourceUsage(ResourceUsageKey key, Object value) { + Map resUsageMap = (Map) statsHolder.computeIfAbsent(RESSOURCE_USAGE_KEY, ru -> new HashMap<>()); + Map subMap = (Map) resUsageMap.computeIfAbsent(key.bucket, k -> new HashMap<>()); + subMap.put(key.key, value); + } + + public Path printToFile() { + recordSystemFixedValues(); + final File file = new File(jsonOutputFile); + String description = "image statistics in json"; + return ReportUtils.report(description, file.getAbsoluteFile().toPath(), getReporter(), false); + } + + private Consumer getReporter() { + return out -> { + out.println(toJson()); + }; + } + + private String toJson() { + return mapToJson(statsHolder); + } + + private String mapToJson(Map map) { + // base case + if (map.isEmpty()) { + return "{}"; + } + StringBuilder builder = new StringBuilder(); + builder.append("{"); + Iterator keySetIter = map.keySet().iterator(); + while (keySetIter.hasNext()) { + String key = keySetIter.next(); + Object value = map.get(key); + builder.append("\"" + key + "\":"); + if (value == null) { + builder.append("null"); // null string + } else if (value instanceof Map) { + // Always a map + @SuppressWarnings("unchecked") + Map subMap = (Map) value; + builder.append(mapToJson(subMap)); + } else if (value instanceof String) { + builder.append("\"" + value + "\""); + } else { + assert value instanceof Number; + // Numeric value + builder.append(value); + } + if (keySetIter.hasNext()) { + builder.append(","); + } + } + builder.append("}"); + return builder.toString(); + } + + interface JsonMetric { + void record(ProgressReporterJsonHelper helper, Object value); + } + + enum ImageDetailKey implements JsonMetric { + TOTAL_SIZE(null, null, "total_bytes"), + CODE_AREA_SIZE("code_area", null, "bytes"), + NUM_COMP_UNITS("code_area", null, "compilation_units"), + IMAGE_HEAP_SIZE("image_heap", null, "bytes"), + DEBUG_INFO_SIZE("debug_info", null, "bytes"), + IMAGE_HEAP_RESOURCE_COUNT("image_heap", "resources", "count"), + RESOURCE_SIZE_BYTES("image_heap", "resources", "bytes"), + RUNTIME_COMPILED_METHODS_COUNT("runtime_compiled_methods", null, "count"), + GRAPH_ENCODING_SIZE("runtime_compiled_methods", null, "graph_encoding_bytes"); + + private String bucket; + private String key; + private String subBucket; + + ImageDetailKey(String bucket, String subBucket, String key) { + this.bucket = bucket; + this.key = key; + this.subBucket = subBucket; + } + + /** + * + * @return true iff the json is represented via a sub object. + */ + public boolean hasBucket() { + return bucket != null; + } + + public boolean hasSubBucket() { + return subBucket != null; + } + + public String bucket() { + return bucket; + } + + public String subBucket() { + return subBucket; + } + + public String jsonKey() { + return key; + } + + @Override + public void record(ProgressReporterJsonHelper helper, Object value) { + helper.putImageDetails(this, value); + } + } + + enum ResourceUsageKey implements JsonMetric { + CPU_LOAD("cpu", "load"), + CPU_CORES_TOTAL("cpu", "total_cores"), + GC_COUNT("garbage_collection", "count"), + GC_SECS("garbage_collection", "total_secs"), + PEAK_RSS("memory", "peak_rss_bytes"), + MEMORY_TOTAL("memory", "system_total"); + + private String bucket; + private String key; + + ResourceUsageKey(String bucket, String key) { + this.key = key; + this.bucket = bucket; + } + + @Override + public void record(ProgressReporterJsonHelper helper, Object value) { + helper.putResourceUsage(this, value); + } + + } + + enum AnalysisResults implements JsonMetric { + CLASS_TOTAL("classes", "total"), + CLASS_REACHABLE("classes", "reachable"), + CLASS_JNI("classes", "jni"), + CLASS_REFLECT("classes", "reflection"), + METHOD_TOTAL("methods", "total"), + METHOD_REACHABLE("methods", "reachable"), + METHOD_JNI("methods", "jni"), + METHOD_REFLECT("methods", "reflection"), + FIELD_TOTAL("fields", "total"), + FIELD_REACHABLE("fields", "reachable"), + FIELD_JNI("fields", "jni"), + FIELD_REFLECT("fields", "reflection"); + + private String key; + private String bucket; + + AnalysisResults(String bucket, String key) { + this.key = key; + this.bucket = bucket; + } + + public String jsonKey() { + return key; + } + + public String bucket() { + return bucket; + } + + @Override + public void record(ProgressReporterJsonHelper helper, Object value) { + if (value instanceof Integer) { + helper.putAnalysisResults(this, (Integer) value); + } else if (value instanceof Long) { + helper.putAnalysisResults(this, (Long) value); + } else { + VMError.shouldNotReachHere("Imcompatible type of 'value': " + value.getClass()); + } + } + } + + enum GeneralInfo implements JsonMetric { + IMAGE_NAME("name"), + JAVA_VERSION("java_version"), + GRAALVM_VERSION("graalvm_version"), + GC("garbage_collector"), + CC("c_compiler"); + + private String key; + + GeneralInfo(String key) { + this.key = key; + } + + public String jsonKey() { + return key; + } + + @Override + public void record(ProgressReporterJsonHelper helper, Object value) { + helper.putGeneralInfo(this, (String) value); + } + } +}