diff --git a/docs/reference-manual/native-image/BuildOutput.md b/docs/reference-manual/native-image/BuildOutput.md index 9888c405fdaf..2a6456290337 100644 --- a/docs/reference-manual/native-image/BuildOutput.md +++ b/docs/reference-manual/native-image/BuildOutput.md @@ -190,6 +190,8 @@ Run `native-image --expert-options-all | grep "BuildOutput"` to see all build ou -H:±BuildOutputBreakdowns Show code and heap breakdowns as part of the build output. Default: + (enabled). -H:±BuildOutputColorful Colorize build output. Default: + (enabled). -H:±BuildOutputGCWarnings Print GC warnings as part of build output. Default: + (enabled). +-H:BuildOutputJSONFile="" Print build output statistics as JSON to the specified file. The output is according to the JSON schema located at: + docs/reference-manual/native-image/assets/build-output-schema-v1.json. -H:±BuildOutputLinks Show links in build output. Default: + (enabled). -H:±BuildOutputPrefix Prefix build output with ':'. Default: - (disabled). -H:±BuildOutputProgress Report progress in build output. Default: + (enabled). diff --git a/docs/reference-manual/native-image/assets/build-output-schema-v1.json b/docs/reference-manual/native-image/assets/build-output-schema-v1.json new file mode 100644 index 000000000000..53ffc28d657e --- /dev/null +++ b/docs/reference-manual/native-image/assets/build-output-schema-v1.json @@ -0,0 +1,515 @@ +{ + "$schema": "https://json-schema.org/draft/2019-09/schema", + "$id": "https://github.com/oracle/graal/blob/master/docs/reference-manual/native-image/assets/build-output-schema-v1.json", + "type": "object", + "default": {}, + "title": "Schema for the JSON build output of GraalVM Native Image", + "required": [ + "general_info", + "analysis_results", + "image_details", + "resource_usage" + ], + "properties": { + "general_info": { + "type": "object", + "default": {}, + "title": "General information about the image build process", + "required": [ + "name", + "graalvm_version", + "java_version", + "c_compiler", + "garbage_collector" + ], + "properties": { + "name": { + "type": "string", + "default": "", + "title": "The native image name" + }, + "graalvm_version": { + "type": "string", + "default": "", + "title": "The GraalVM Version" + }, + "java_version": { + "type": ["string", "null"], + "default": null, + "title": "The precise Java version used for running the image builder (or null if not available)" + }, + "c_compiler": { + "type": ["string", "null"], + "default": null, + "title": "The C compiler used for native compilation and linkage (or null if not available)" + }, + "garbage_collector": { + "type": "string", + "default": "", + "title": "The garbage collection algorithm in use for the generated native image" + } + }, + "examples": [ + { + "name": "helloworld", + "graalvm_version": "GraalVM 22.2.0-dev Java 17 CE", + "java_version": "17.0.4+5-jvmci-22.2-b03", + "c_compiler": "gcc (linux, x86_64, 9.3.0)", + "garbage_collector": "Serial GC" + } + ] + }, + "analysis_results": { + "type": "object", + "default": {}, + "title": "Schema for the image generator statistics of the analysis phase", + "required": [ + "classes", + "fields", + "methods" + ], + "properties": { + "classes": { + "type": "object", + "default": {}, + "title": "Number of classes detected during analysis phase", + "required": [ + "total", + "reachable", + "reflection", + "jni" + ], + "properties": { + "total": { + "type": "integer", + "default": 0, + "title": "The total number of classes" + }, + "reachable": { + "type": "integer", + "default": 0, + "title": "The reachable reachable classes" + }, + "reflection": { + "type": "integer", + "default": 0, + "title": "The classes registered for reflective access" + }, + "jni": { + "type": "integer", + "default": -1, + "title": "The classes registered for JNI access (or -1 if unset)" + } + } + }, + "fields": { + "type": "object", + "default": {}, + "title": "Number of fields detected during analysis phase", + "required": [ + "total", + "reachable", + "reflection", + "jni" + ], + "properties": { + "total": { + "type": "integer", + "default": 0, + "title": "The total number of fields" + }, + "reachable": { + "type": "integer", + "default": 0, + "title": "The reachable number of fields" + }, + "reflection": { + "type": "integer", + "default": 0, + "title": "The number of fields registered for reflective access" + }, + "jni": { + "type": "integer", + "default": -1, + "title": "The number of fields registered for JNI access (or -1 if unset)" + } + } + }, + "methods": { + "type": "object", + "default": {}, + "title": "Number of methods detected during analysis phase", + "required": [ + "total", + "reachable", + "reflection", + "jni" + ], + "properties": { + "total": { + "type": "integer", + "default": 0, + "title": "The total number of methods" + }, + "reachable": { + "type": "integer", + "default": 0, + "title": "The reachable number of methods" + }, + "reflection": { + "type": "integer", + "default": 0, + "title": "The number of methods registered for reflective access" + }, + "jni": { + "type": "integer", + "default": -1, + "title": "The number of methods registered for JNI access (or -1 if unset)" + } + } + } + }, + "examples": [ + { + "classes": { + "total": 3850, + "reachable": 2839, + "reflection": 28, + "jni": 58 + }, + "fields": { + "total": 6665, + "reachable": 3400, + "reflection": 0, + "jni": 58 + }, + "methods": { + "total": 29038, + "reachable": 12916, + "reflection": 332, + "jni": 52 + } + } + ] + }, + "image_details": { + "type": "object", + "default": {}, + "title": "Schema for statistics of the image creation phase of the image generator", + "required": [ + "total_bytes", + "code_area", + "image_heap" + ], + "properties": { + "total_bytes": { + "type": "integer", + "default": 0, + "title": "The total image size (in bytes)" + }, + "code_area": { + "type": "object", + "default": {}, + "title": "The code area statistics", + "required": [ + "bytes", + "compilation_units" + ], + "properties": { + "bytes": { + "type": "integer", + "default": 0, + "title": "Bytes used for the code area" + }, + "compilation_units": { + "type": "integer", + "default": 0, + "title": "The number of compilation units in the image" + } + } + }, + "image_heap": { + "type": "object", + "default": {}, + "title": "The image heap statistics", + "required": [ + "bytes", + "resources" + ], + "properties": { + "bytes": { + "type": "integer", + "default": 0, + "title": "Number of bytes used for the image heap" + }, + "resources": { + "type": "object", + "default": {}, + "title": "Resource statistics of the image", + "required": [ + "count", + "bytes" + ], + "properties": { + "count": { + "type": "integer", + "default": 0, + "title": "Number of resources included in the image" + }, + "bytes": { + "type": "integer", + "default": 0, + "title": "The number of bytes used for resource data" + } + } + } + } + }, + "debug_info": { + "type": "object", + "default": {}, + "title": "The debug info statistics", + "required": [ + "bytes" + ], + "properties": { + "bytes": { + "type": "integer", + "default": 0, + "title": "The number of bytes used for debug info" + } + } + } + }, + "examples": [ + { + "total_bytes": 13057934, + "code_area": { + "bytes": 4610048, + "compilation_units": 67007 + }, + "image_heap": { + "bytes": 7307264, + "resources": { + "count": 134, + "bytes": 10200 + } + }, + "debug_info": { + "bytes": 1140622 + } + } + ] + }, + "resource_usage": { + "type": "object", + "default": {}, + "title": "Schema for image generator ressource usage statistics", + "required": [ + "cpu", + "garbage_collection", + "memory" + ], + "properties": { + "cpu": { + "type": "object", + "default": {}, + "title": "The cpu resource schema", + "required": [ + "load", + "total_cores" + ], + "properties": { + "load": { + "type": "number", + "default": -1, + "title": "The cpu load of the build system after image building (or -1 if unavailable)" + }, + "total_cores": { + "type": "integer", + "default": 0, + "title": "The total number of cores on the build system" + } + } + }, + "garbage_collection": { + "type": "object", + "default": {}, + "title": "The garbage collection resource usage schema", + "required": [ + "count", + "total_secs" + ], + "properties": { + "count": { + "type": "integer", + "default": 0, + "title": "The number of GC cycles performed during image generation" + }, + "total_secs": { + "type": "number", + "default": 0.0, + "title": "The total time spent in GC (in seconds)" + } + } + }, + "memory": { + "type": "object", + "default": {}, + "title": "The memory statistics schema", + "required": [ + "system_total", + "peak_rss_bytes" + ], + "properties": { + "system_total": { + "type": "integer", + "default": 0, + "title": "The total system memory" + }, + "peak_rss_bytes": { + "type": "integer", + "default": -1, + "title": "Peak RSS value of the image builder process in bytes (or -1 if unavailable)" + } + } + } + }, + "examples": [ + { + "cpu": { + "load": 8.38, + "total_cores": 10 + }, + "garbage_collection": { + "count": 17, + "total_secs": 0.9245 + }, + "memory": { + "system_total": 33254146048, + "peak_rss_bytes": 3506065408 + } + } + ] + } + }, + "examples": [ + { + "general_info": { + "name": "helloworld", + "graalvm_version": "GraalVM 22.2.0-dev Java 17 CE", + "java_version": "17.0.4+5-jvmci-22.2-b03", + "c_compiler": "gcc (linux, x86_64, 9.3.0)", + "garbage_collector": "Serial GC" + }, + "analysis_results": { + "classes": { + "total": 3850, + "reachable": 2839, + "reflection": 28, + "jni": 58 + }, + "fields": { + "total": 6665, + "reachable": 3400, + "reflection": 0, + "jni": 58 + }, + "methods": { + "total": 29038, + "reachable": 12916, + "reflection": 332, + "jni": 52 + } + }, + "image_details": { + "total_bytes": 13057934, + "code_area": { + "bytes": 4610048, + "compilation_units": 67007 + }, + "image_heap": { + "bytes": 7307264, + "resources": { + "count": 134, + "bytes": 10200 + } + }, + "debug_info": { + "bytes": 1140622 + } + }, + "resource_usage": { + "cpu": { + "load": 8.38, + "total_cores": 10 + }, + "garbage_collection": { + "count": 17, + "total_secs": 0.9245 + }, + "memory": { + "system_total": 33254146048, + "peak_rss_bytes": 3506065408 + } + } + }, + { + "general_info": { + "name": "helloworld", + "graalvm_version": "GraalVM 22.2.0-dev Java 17 CE", + "java_version": null, + "c_compiler": null, + "garbage_collector": "Serial GC" + }, + "analysis_results": { + "classes": { + "total": 3850, + "reachable": 2839, + "reflection": 28, + "jni": -1 + }, + "fields": { + "total": 6665, + "reachable": 3400, + "reflection": 0, + "jni": -1 + }, + "methods": { + "total": 29038, + "reachable": 12916, + "reflection": 332, + "jni": -1 + } + }, + "image_details": { + "total_bytes": 13057934, + "code_area": { + "bytes": 4610048, + "compilation_units": 67007 + }, + "image_heap": { + "bytes": 7307264, + "resources": { + "count": 134, + "bytes": 10200 + } + } + }, + "resource_usage": { + "cpu": { + "load": -1, + "total_cores": 10 + }, + "garbage_collection": { + "count": 17, + "total_secs": 0.9245 + }, + "memory": { + "system_total": 33254146048, + "peak_rss_bytes": -1 + } + } + } + ] +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java index 4c0995e00032..600f9ee81f2d 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java @@ -415,6 +415,11 @@ public Boolean getValue(OptionValues values) { @Option(help = "Print GC warnings as part of build output", type = OptionType.User)// public static final HostedOptionKey BuildOutputGCWarnings = new HostedOptionKey<>(true); + @Option(help = "Print build output statistics as JSON to the specified file. " + + "The output is according to the JSON schema located at: " + + "docs/reference-manual/native-image/assets/build-output-schema-v1.json", type = OptionType.User)// + public static final HostedOptionKey BuildOutputJSONFile = new HostedOptionKey<>(""); + /* * Object and array allocation options. */ diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java index 46dbd56c9d50..017e0230667b 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java @@ -49,7 +49,8 @@ import java.util.function.Consumer; import java.util.stream.Collectors; -import com.oracle.graal.pointsto.api.PointstoOptions; +import org.graalvm.collections.EconomicMap; +import org.graalvm.collections.Pair; import org.graalvm.compiler.debug.DebugOptions; import org.graalvm.compiler.options.OptionValues; import org.graalvm.compiler.serviceprovider.GraalServices; @@ -58,6 +59,7 @@ import org.graalvm.nativeimage.impl.ImageSingletonsSupport; import com.oracle.graal.pointsto.BigBang; +import com.oracle.graal.pointsto.api.PointstoOptions; import com.oracle.graal.pointsto.meta.AnalysisField; import com.oracle.graal.pointsto.meta.AnalysisMethod; import com.oracle.graal.pointsto.meta.AnalysisUniverse; @@ -77,6 +79,11 @@ import com.oracle.svm.core.option.HostedOptionValues; import com.oracle.svm.core.reflect.ReflectionMetadataDecoder; import com.oracle.svm.core.util.VMError; +import com.oracle.svm.hosted.ProgressReporterJsonStatsHelper.AnalysisResults; +import com.oracle.svm.hosted.ProgressReporterJsonStatsHelper.GeneralInfo; +import com.oracle.svm.hosted.ProgressReporterJsonStatsHelper.ImageDetailKey; +import com.oracle.svm.hosted.ProgressReporterJsonStatsHelper.JsonMetric; +import com.oracle.svm.hosted.ProgressReporterJsonStatsHelper.ResourceUsageKey; import com.oracle.svm.hosted.c.codegen.CCompilerInvoker; import com.oracle.svm.hosted.code.CompileQueue.CompileTask; import com.oracle.svm.hosted.image.AbstractImage.NativeImageKind; @@ -86,6 +93,7 @@ import com.oracle.svm.util.ReflectionUtil; public class ProgressReporter { + private static final long UNAVAILABLE_METRIC = -1; private static final int CHARACTERS_PER_LINE; private static final String HEADLINE_SEPARATOR; private static final String LINE_SEPARATOR; @@ -101,6 +109,7 @@ public class ProgressReporter { private final NativeImageSystemIOWrappers builderIO; + private final ProgressReporterJsonStatsHelper jsonHelper; private final boolean isEnabled; // TODO: clean up when deprecating old output (GR-35721). private final boolean reachabilityAnalysis; private final DirectPrinter linePrinter = new DirectPrinter(); @@ -172,6 +181,11 @@ public ProgressReporter(OptionValues options) { if (isEnabled) { Timer.disablePrinting(); } + if (SubstrateOptions.BuildOutputJSONFile.hasBeenSet(options)) { + jsonHelper = new ProgressReporterJsonStatsHelper(SubstrateOptions.BuildOutputJSONFile.getValue(options)); + } else { + jsonHelper = null; + } reachabilityAnalysis = PointstoOptions.UseExperimentalReachabilityAnalysis.getValue(options); usePrefix = SubstrateOptions.BuildOutputPrefix.getValue(options); boolean enableColors = !IS_DUMB_TERM && !IS_CI && OS.getCurrent() != OS.WINDOWS && @@ -238,6 +252,7 @@ public void printStart(String imageName, NativeImageKind imageKind) { stagePrinter.progressBarStart += outputPrefix.length(); } l().printHeadlineSeparator(); + recordJsonMetrics(GeneralInfo.IMAGE_NAME, imageName); String imageKindName = imageKind.name().toLowerCase().replace('_', ' '); l().blueBold().link("GraalVM Native Image", "https://www.graalvm.org/native-image/").reset() .a(": Generating '").bold().a(imageName).reset().a("' (").doclink(imageKindName, "#glossary-imagekind").a(")...").println(); @@ -253,15 +268,23 @@ public void printUnsuccessfulInitializeEnd() { public void printInitializeEnd() { stagePrinter.end(getTimer(TimerCollection.Registry.CLASSLIST).getTotalTime() + getTimer(TimerCollection.Registry.SETUP).getTotalTime()); - l().a(" ").doclink("Version info", "#glossary-version-info").a(": '").a(ImageSingletons.lookup(VM.class).version).a("'").println(); + String version = ImageSingletons.lookup(VM.class).version; + recordJsonMetrics(GeneralInfo.GRAALVM_VERSION, version); + l().a(" ").doclink("Version info", "#glossary-version-info").a(": '").a(version).a("'").println(); String javaVersion = System.getProperty("java.runtime.version"); + recordJsonMetrics(GeneralInfo.JAVA_VERSION, javaVersion); if (javaVersion != null) { l().a(" ").doclink("Java version info", "#glossary-java-version-info").a(": '").a(javaVersion).a("'").println(); } + String cCompilerShort = null; if (ImageSingletons.contains(CCompilerInvoker.class)) { - l().a(" ").doclink("C compiler", "#glossary-ccompiler").a(": ").a(ImageSingletons.lookup(CCompilerInvoker.class).compilerInfo.getShortDescription()).println(); + cCompilerShort = ImageSingletons.lookup(CCompilerInvoker.class).compilerInfo.getShortDescription(); + l().a(" ").doclink("C compiler", "#glossary-ccompiler").a(": ").a(cCompilerShort).println(); } - l().a(" ").doclink("Garbage collector", "#glossary-gc").a(": ").a(Heap.getHeap().getGC().getName()).println(); + recordJsonMetrics(GeneralInfo.CC, cCompilerShort); + String gcAlgo = Heap.getHeap().getGC().getName(); + recordJsonMetrics(GeneralInfo.GC, gcAlgo); + l().a(" ").doclink("Garbage collector", "#glossary-gc").a(": ").a(gcAlgo).println(); l().a(" ").doclink("Analysis", "#glossary-analysis").a(": ").a(reachabilityAnalysis ? "Reachability" : "Points-To").println(); } @@ -310,16 +333,22 @@ private void printAnalysisStatistics(AnalysisUniverse universe, Collection t.isReachable()).count(); long totalClasses = universe.getTypes().size(); + recordJsonMetrics(AnalysisResults.CLASS_TOTAL, totalClasses); + recordJsonMetrics(AnalysisResults.CLASS_REACHABLE, reachableClasses); l().a(actualVsTotalFormat, reachableClasses, reachableClasses / (double) totalClasses * 100, totalClasses) .a(" classes ").doclink("reachable", "#glossary-reachability").println(); Collection fields = universe.getFields(); long reachableFields = fields.stream().filter(f -> f.isAccessed()).count(); int totalFields = fields.size(); + recordJsonMetrics(AnalysisResults.FIELD_TOTAL, totalFields); + recordJsonMetrics(AnalysisResults.FIELD_REACHABLE, reachableFields); l().a(actualVsTotalFormat, reachableFields, reachableFields / (double) totalFields * 100, totalFields) .a(" fields ").doclink("reachable", "#glossary-reachability").println(); Collection methods = universe.getMethods(); long reachableMethods = methods.stream().filter(m -> m.isReachable()).count(); int totalMethods = methods.size(); + recordJsonMetrics(AnalysisResults.METHOD_TOTAL, totalMethods); + recordJsonMetrics(AnalysisResults.METHOD_REACHABLE, reachableMethods); l().a(actualVsTotalFormat, reachableMethods, reachableMethods / (double) totalMethods * 100, totalMethods) .a(" methods ").doclink("reachable", "#glossary-reachability").println(); if (numRuntimeCompiledMethods >= 0) { @@ -328,8 +357,17 @@ private void printAnalysisStatistics(AnalysisUniverse universe, Collection 0 ? numJNIMethods : UNAVAILABLE_METRIC)); + recordJsonMetrics(AnalysisResults.CLASS_JNI, (numJNIClasses >= 0 ? numJNIClasses : UNAVAILABLE_METRIC)); + recordJsonMetrics(AnalysisResults.FIELD_JNI, (numJNIClasses > 0 ? numJNIFields : UNAVAILABLE_METRIC)); if (numJNIClasses > 0) { l().a(classesFieldsMethodFormat, numJNIClasses, numJNIFields, numJNIMethods) .doclink("registered for JNI access", "#glossary-jni-access-registrations").println(); @@ -411,10 +449,14 @@ public void printCreationEnd(int imageSize, int numHeapObjects, long imageHeapSi String format = "%9s (%5.2f%%) for "; l().a(format, Utils.bytesToHuman(codeAreaSize), codeAreaSize / (double) imageSize * 100) .doclink("code area", "#glossary-code-area").a(":%,10d compilation units", numCompilations).println(); - int numResources = Resources.singleton().resources().size(); + EconomicMap, ResourceStorageEntry> resources = Resources.singleton().resources(); + int numResources = resources.size(); + recordJsonMetrics(ImageDetailKey.IMAGE_HEAP_RESOURCE_COUNT, numResources); + recordJsonResourceMetrics(resources); l().a(format, Utils.bytesToHuman(imageHeapSize), imageHeapSize / (double) imageSize * 100) .doclink("image heap", "#glossary-image-heap").a(":%,9d objects and %,d resources", numHeapObjects, numResources).println(); if (debugInfoSize > 0) { + recordJsonMetrics(ImageDetailKey.DEBUG_INFO_SIZE, debugInfoSize); // Optional metric DirectPrinter l = l().a(format, Utils.bytesToHuman(debugInfoSize), debugInfoSize / (double) imageSize * 100) .doclink("debug info", "#glossary-debug-info"); if (debugInfoTimer != null) { @@ -423,6 +465,10 @@ public void printCreationEnd(int imageSize, int numHeapObjects, long imageHeapSi l.println(); } long otherBytes = imageSize - codeAreaSize - imageHeapSize - debugInfoSize; + recordJsonMetrics(ImageDetailKey.IMAGE_HEAP_SIZE, imageHeapSize); + recordJsonMetrics(ImageDetailKey.TOTAL_SIZE, imageSize); + recordJsonMetrics(ImageDetailKey.CODE_AREA_SIZE, codeAreaSize); + recordJsonMetrics(ImageDetailKey.NUM_COMP_UNITS, numCompilations); l().a(format, Utils.bytesToHuman(otherBytes), otherBytes / (double) imageSize * 100) .doclink("other data", "#glossary-other-data").println(); l().a("%9s in total", Utils.bytesToHuman(imageSize)).println(); @@ -561,7 +607,7 @@ public void printEpilog(String imageName, NativeImageGenerator generator, boolea Map> artifacts = generator.getBuildArtifacts(); if (!artifacts.isEmpty()) { l().printLineSeparator(); - printArtifacts(imageName, generator, parsedHostedOptions, artifacts); + printArtifacts(imageName, generator, parsedHostedOptions, artifacts, wasSuccessfulBuild); } l().printHeadlineSeparator(); @@ -578,7 +624,7 @@ public void printEpilog(String imageName, NativeImageGenerator generator, boolea executor.shutdown(); } - private void printArtifacts(String imageName, NativeImageGenerator generator, OptionValues parsedHostedOptions, Map> artifacts) { + private void printArtifacts(String imageName, NativeImageGenerator generator, OptionValues parsedHostedOptions, Map> artifacts, boolean wasSuccessfulBuild) { l().yellowBold().a("Produced artifacts:").reset().println(); // Use TreeMap to sort paths alphabetically. Map> pathToTypes = new TreeMap<>(); @@ -587,6 +633,10 @@ private void printArtifacts(String imageName, NativeImageGenerator generator, Op pathToTypes.computeIfAbsent(path, p -> new ArrayList<>()).add(artifactType.name().toLowerCase()); } }); + if (jsonHelper != null && wasSuccessfulBuild) { + Path jsonMetric = jsonHelper.printToFile(); + pathToTypes.computeIfAbsent(jsonMetric, p -> new ArrayList<>()).add("raw"); + } if (generator.getBigbang() != null && ImageBuildStatistics.Options.CollectImageBuildStatistics.getValue(parsedHostedOptions)) { Path buildStatisticsPath = reportImageBuildStatistics(imageName, generator.getBigbang()); pathToTypes.computeIfAbsent(buildStatisticsPath, p -> new ArrayList<>()).add("raw"); @@ -631,17 +681,23 @@ private void printResourceStatistics() { GCStats gcStats = GCStats.getCurrent(); double gcSeconds = Utils.millisToSeconds(gcStats.totalTimeMillis); CenteredTextPrinter p = new CenteredTextPrinter(); + recordJsonMetrics(ResourceUsageKey.GC_COUNT, gcStats.totalCount); + recordJsonMetrics(ResourceUsageKey.GC_SECS, gcSeconds); p.a("%.1fs (%.1f%% of total time) in %d ", gcSeconds, gcSeconds / totalProcessTimeSeconds * 100, gcStats.totalCount) .doclink("GCs", "#glossary-garbage-collections"); long peakRSS = ProgressReporterCHelper.getPeakRSS(); if (peakRSS >= 0) { p.a(" | ").doclink("Peak RSS", "#glossary-peak-rss").a(": ").a("%.2fGB", Utils.bytesToGiB(peakRSS)); } + recordJsonMetrics(ResourceUsageKey.PEAK_RSS, (peakRSS >= 0 ? peakRSS : UNAVAILABLE_METRIC)); OperatingSystemMXBean osMXBean = ManagementFactory.getOperatingSystemMXBean(); long processCPUTime = ((com.sun.management.OperatingSystemMXBean) osMXBean).getProcessCpuTime(); + double cpuLoad = UNAVAILABLE_METRIC; if (processCPUTime > 0) { - p.a(" | ").doclink("CPU load", "#glossary-cpu-load").a(": ").a("%.2f", Utils.nanosToSeconds(processCPUTime) / totalProcessTimeSeconds); + cpuLoad = Utils.nanosToSeconds(processCPUTime) / totalProcessTimeSeconds; + p.a(" | ").doclink("CPU load", "#glossary-cpu-load").a(": ").a("%.2f", cpuLoad); } + recordJsonMetrics(ResourceUsageKey.CPU_LOAD, cpuLoad); p.flushln(); } @@ -663,6 +719,24 @@ private void checkForExcessiveGarbageCollection() { lastGCStats = currentGCStats; } + private void recordJsonMetrics(JsonMetric metric, Object value) { + if (jsonHelper != null) { + metric.record(jsonHelper, value); + } + } + + private void recordJsonResourceMetrics(EconomicMap, ResourceStorageEntry> resources) { + if (jsonHelper != null) { + long totalResourceBytes = 0; + for (ResourceStorageEntry entry : resources.getValues()) { + for (byte[] data : entry.getData()) { + totalResourceBytes += data.length; + } + } + ImageDetailKey.RESOURCE_SIZE_BYTES.record(jsonHelper, totalResourceBytes); + } + } + /* * HELPERS */ diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporterJsonStatsHelper.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporterJsonStatsHelper.java new file mode 100644 index 000000000000..d95684a7d59c --- /dev/null +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporterJsonStatsHelper.java @@ -0,0 +1,293 @@ +/* + * Copyright (c) 2021, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.hosted; + +import java.io.File; +import java.io.PrintWriter; +import java.lang.management.ManagementFactory; +import java.lang.management.OperatingSystemMXBean; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.function.Consumer; + +import com.oracle.graal.pointsto.reports.ReportUtils; +import com.oracle.svm.core.util.VMError; + +class ProgressReporterJsonStatsHelper { + private static final String ANALYSIS_RESULTS_KEY = "analysis_results"; + private static final String GENERAL_INFO_KEY = "general_info"; + private static final String IMAGE_DETAILS_KEY = "image_details"; + private static final String RESSOURCE_USAGE_KEY = "resource_usage"; + + private final Map statsHolder = new HashMap<>(); + private final String jsonOutputFile; + + ProgressReporterJsonStatsHelper(String outFile) { + this.jsonOutputFile = outFile; + } + + private void recordSystemFixedValues() { + putResourceUsage(ResourceUsageKey.CPU_CORES_TOTAL, Runtime.getRuntime().availableProcessors()); + putResourceUsage(ResourceUsageKey.MEMORY_TOTAL, getTotalSystemMemory()); + } + + @SuppressWarnings("deprecation") + private static long getTotalSystemMemory() { + OperatingSystemMXBean osMXBean = ManagementFactory.getOperatingSystemMXBean(); + return ((com.sun.management.OperatingSystemMXBean) osMXBean).getTotalPhysicalMemorySize(); + } + + @SuppressWarnings("unchecked") + public void putAnalysisResults(AnalysisResults key, long value) { + Map analysisMap = (Map) statsHolder.computeIfAbsent(ANALYSIS_RESULTS_KEY, k -> new HashMap<>()); + Map bucketMap = (Map) analysisMap.computeIfAbsent(key.bucket(), bk -> new HashMap<>()); + bucketMap.put(key.jsonKey(), value); + } + + @SuppressWarnings("unchecked") + public void putGeneralInfo(GeneralInfo info, String value) { + Map generalInfoMap = (Map) statsHolder.computeIfAbsent(GENERAL_INFO_KEY, gi -> new HashMap<>()); + generalInfoMap.put(info.jsonKey(), value); + } + + @SuppressWarnings("unchecked") + public void putImageDetails(ImageDetailKey key, Object value) { + Map imageDetailsMap = (Map) statsHolder.computeIfAbsent(IMAGE_DETAILS_KEY, id -> new HashMap<>()); + if (!key.hasBucket() && !key.hasSubBucket()) { + imageDetailsMap.put(key.jsonKey(), value); + } else if (!key.hasSubBucket()) { + assert key.hasBucket(); + Map bucketMap = (Map) imageDetailsMap.computeIfAbsent(key.bucket(), sb -> new HashMap<>()); + bucketMap.put(key.jsonKey(), value); + } else { + assert key.hasSubBucket(); + Map bucketMap = (Map) imageDetailsMap.computeIfAbsent(key.bucket(), sb -> new HashMap<>()); + Map subbucketMap = (Map) bucketMap.computeIfAbsent(key.subBucket(), sb -> new HashMap<>()); + subbucketMap.put(key.jsonKey(), value); + } + } + + @SuppressWarnings("unchecked") + public void putResourceUsage(ResourceUsageKey key, Object value) { + Map resUsageMap = (Map) statsHolder.computeIfAbsent(RESSOURCE_USAGE_KEY, ru -> new HashMap<>()); + Map subMap = (Map) resUsageMap.computeIfAbsent(key.bucket, k -> new HashMap<>()); + subMap.put(key.key, value); + } + + public Path printToFile() { + recordSystemFixedValues(); + final File file = new File(jsonOutputFile); + String description = "image statistics in json"; + return ReportUtils.report(description, file.getAbsoluteFile().toPath(), getReporter(), false); + } + + private Consumer getReporter() { + return out -> { + out.println(toJson()); + }; + } + + private String toJson() { + return mapToJson(statsHolder); + } + + private String mapToJson(Map map) { + // base case + if (map.isEmpty()) { + return "{}"; + } + StringBuilder builder = new StringBuilder(); + builder.append("{"); + Iterator keySetIter = map.keySet().iterator(); + while (keySetIter.hasNext()) { + String key = keySetIter.next(); + Object value = map.get(key); + builder.append("\"" + key + "\":"); + if (value == null) { + builder.append("null"); // null string + } else if (value instanceof Map) { + // Always a map + @SuppressWarnings("unchecked") + Map subMap = (Map) value; + builder.append(mapToJson(subMap)); + } else if (value instanceof String) { + builder.append("\"" + value + "\""); + } else { + assert value instanceof Number; + // Numeric value + builder.append(value); + } + if (keySetIter.hasNext()) { + builder.append(","); + } + } + builder.append("}"); + return builder.toString(); + } + + interface JsonMetric { + void record(ProgressReporterJsonStatsHelper helper, Object value); + } + + enum ImageDetailKey implements JsonMetric { + TOTAL_SIZE(null, null, "total_bytes"), + CODE_AREA_SIZE("code_area", null, "bytes"), + NUM_COMP_UNITS("code_area", null, "compilation_units"), + IMAGE_HEAP_SIZE("image_heap", null, "bytes"), + DEBUG_INFO_SIZE("debug_info", null, "bytes"), + IMAGE_HEAP_RESOURCE_COUNT("image_heap", "resources", "count"), + RESOURCE_SIZE_BYTES("image_heap", "resources", "bytes"); + + private String bucket; + private String key; + private String subBucket; + + ImageDetailKey(String bucket, String subBucket, String key) { + this.bucket = bucket; + this.key = key; + this.subBucket = subBucket; + } + + /** + * + * @return true iff the json is represented via a sub object. + */ + public boolean hasBucket() { + return bucket != null; + } + + public boolean hasSubBucket() { + return subBucket != null; + } + + public String bucket() { + return bucket; + } + + public String subBucket() { + return subBucket; + } + + public String jsonKey() { + return key; + } + + @Override + public void record(ProgressReporterJsonStatsHelper helper, Object value) { + helper.putImageDetails(this, value); + } + } + + enum ResourceUsageKey implements JsonMetric { + CPU_LOAD("cpu", "load"), + CPU_CORES_TOTAL("cpu", "total_cores"), + GC_COUNT("garbage_collection", "count"), + GC_SECS("garbage_collection", "total_secs"), + PEAK_RSS("memory", "peak_rss_bytes"), + MEMORY_TOTAL("memory", "system_total"); + + private String bucket; + private String key; + + ResourceUsageKey(String bucket, String key) { + this.key = key; + this.bucket = bucket; + } + + @Override + public void record(ProgressReporterJsonStatsHelper helper, Object value) { + helper.putResourceUsage(this, value); + } + + } + + enum AnalysisResults implements JsonMetric { + CLASS_TOTAL("classes", "total"), + CLASS_REACHABLE("classes", "reachable"), + CLASS_JNI("classes", "jni"), + CLASS_REFLECT("classes", "reflection"), + METHOD_TOTAL("methods", "total"), + METHOD_REACHABLE("methods", "reachable"), + METHOD_JNI("methods", "jni"), + METHOD_REFLECT("methods", "reflection"), + FIELD_TOTAL("fields", "total"), + FIELD_REACHABLE("fields", "reachable"), + FIELD_JNI("fields", "jni"), + FIELD_REFLECT("fields", "reflection"); + + private String key; + private String bucket; + + AnalysisResults(String bucket, String key) { + this.key = key; + this.bucket = bucket; + } + + public String jsonKey() { + return key; + } + + public String bucket() { + return bucket; + } + + @Override + public void record(ProgressReporterJsonStatsHelper helper, Object value) { + if (value instanceof Integer) { + helper.putAnalysisResults(this, (Integer) value); + } else if (value instanceof Long) { + helper.putAnalysisResults(this, (Long) value); + } else { + VMError.shouldNotReachHere("Imcompatible type of 'value': " + value.getClass()); + } + } + } + + enum GeneralInfo implements JsonMetric { + IMAGE_NAME("name"), + JAVA_VERSION("java_version"), + GRAALVM_VERSION("graalvm_version"), + GC("garbage_collector"), + CC("c_compiler"); + + private String key; + + GeneralInfo(String key) { + this.key = key; + } + + public String jsonKey() { + return key; + } + + @Override + public void record(ProgressReporterJsonStatsHelper helper, Object value) { + helper.putGeneralInfo(this, (String) value); + } + } +}