diff --git a/components/environment/src/main/java/datadog/environment/JvmOptions.java b/components/environment/src/main/java/datadog/environment/JvmOptions.java index a2b7a0ec133..6e5315f5a0b 100644 --- a/components/environment/src/main/java/datadog/environment/JvmOptions.java +++ b/components/environment/src/main/java/datadog/environment/JvmOptions.java @@ -15,7 +15,6 @@ import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; -import java.util.ListIterator; import java.util.StringTokenizer; /** Fetches and captures the JVM options. */ @@ -41,28 +40,44 @@ private String[] readProcFsCmdLine() { return null; } - @SuppressForbidden // Class.forName() as backup private List findVmOptions() { + return findVmOptions(PROCFS_CMDLINE); + } + + @SuppressForbidden // Class.forName() as backup + // Visible for testing + List findVmOptions(String[] procfsCmdline) { // Try ProcFS on Linux - if (PROCFS_CMDLINE != null) { + // Be aware that when running a native image, the command line in /proc/self/cmdline is just the + // executable + if (procfsCmdline != null) { + // Create list of VM options + List vmOptions = new ArrayList<>(); // Start at 1 to skip "java" command itself int index = 1; - // Look for main class or "-jar", end of VM options - for (; index < PROCFS_CMDLINE.length; index++) { - if (!PROCFS_CMDLINE[index].startsWith("-") || "-jar".equals(PROCFS_CMDLINE[index])) { - break; - } - } - // Create list of VM options - List vmOptions = new ArrayList<>(asList(PROCFS_CMDLINE).subList(1, index + 1)); - ListIterator iterator = vmOptions.listIterator(); - while (iterator.hasNext()) { - String vmOption = iterator.next(); - if (vmOption.startsWith("@")) { - iterator.remove(); - for (String argument : getArgumentsFromFile(vmOption)) { - iterator.add(argument); + // Look for first self-standing argument that is not prefixed with "-" or end of VM options + // Skip "-jar" and the jar file + // Simultaneously, collect all arguments in the VM options + for (; index < procfsCmdline.length; index++) { + String argument = procfsCmdline[index]; + if (argument.startsWith("@")) { + vmOptions.addAll(getArgumentsFromFile(argument)); + } else { + if ("-jar".equals(argument)) { + // skip "-jar" and the jar file + index++; + continue; + } else if ("-cp".equals(argument)) { + // slurp '-cp' and the classpath + vmOptions.add(argument); + if (index + 1 < procfsCmdline.length) { + argument = procfsCmdline[++index]; + } + } else if (!argument.startsWith("-")) { + // end of VM options + break; } + vmOptions.add(argument); } } // Insert JDK_JAVA_OPTIONS at the start if present and supported diff --git a/components/environment/src/test/java/datadog/environment/JvmOptionsTest.java b/components/environment/src/test/java/datadog/environment/JvmOptionsTest.java index 8260a14f11f..3e07fc46ba5 100644 --- a/components/environment/src/test/java/datadog/environment/JvmOptionsTest.java +++ b/components/environment/src/test/java/datadog/environment/JvmOptionsTest.java @@ -8,15 +8,23 @@ import static java.util.Collections.emptyList; import static java.util.Collections.emptyMap; import static java.util.Collections.singletonList; +import static java.util.Objects.requireNonNull; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assumptions.assumeTrue; import static org.junit.jupiter.params.provider.Arguments.arguments; import datadog.environment.CommandLineHelper.Result; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Stream; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; @@ -105,6 +113,67 @@ void testFindVmOptions( assertEquals(expectedArguments.jvmOptions, result.jvmOptions, "Failed to get JVM options"); } + @MethodSource + private static Stream procFsCmdLine() { + // spotless:off + return Stream.of( + arguments( + "No arguments", + new String[0], + emptyList() + ), + arguments( + "Native image launcher", + new String[]{"native-image-launcher", "-Xmx512m"}, + singletonList("-Xmx512m") + ), + arguments( + "Java with JAR and options", + new String[]{"java", "-Xmx512m", "-Xms256m", "-jar", "app.jar"}, + asList("-Xmx512m", "-Xms256m") + ), + arguments( + "Java from class and options", + new String[]{"java", "-Xmx512m", "-Xms256m", "-cp", "app.jar", "Main"}, + asList("-Xmx512m", "-Xms256m", "-cp", "app.jar") + ), + arguments( + "Java from class and options, mixed", + new String[]{"java", "-Xms256m", "-cp", "app.jar", "-Xmx512m", "Main"}, + asList("-Xms256m", "-cp", "app.jar", "-Xmx512m") + ), + arguments( + "Args from file", + new String[]{"java", "-Dargfile.prop=test", "-verbose:class", argFile("carriage-return-separated"), "-jar", "app.jar"}, + flatten("-Dargfile.prop=test", "-verbose:class", expectedArsFromArgFile("carriage-return-separated")) + ), + arguments( + "Args from file", + new String[]{"java", "-Dargfile.prop=test", "-verbose:class", argFile("new-line-separated"), "-jar", "app.jar"}, + flatten("-Dargfile.prop=test", "-verbose:class", expectedArsFromArgFile("new-line-separated")) + ), + arguments( + "Args from file", + new String[]{"java", "-Dargfile.prop=test", "-verbose:class", argFile("space-separated"), "-jar", "app.jar"}, + flatten("-Dargfile.prop=test", "-verbose:class", expectedArsFromArgFile("space-separated")) + ), + arguments( + "Args from file", + new String[]{"java", "-Dargfile.prop=test", "-verbose:class", argFile("tab-separated"), "-jar", "app.jar"}, + flatten("-Dargfile.prop=test", "-verbose:class", expectedArsFromArgFile("tab-separated")) + )); + // spotless:on + } + + @ParameterizedTest(name = "[{index}] {0}") + @MethodSource("procFsCmdLine") + void testFindVmOptionsWithProcFsCmdLine( + String useCase, String[] procfsCmdline, List expected) throws Exception { + JvmOptions vmOptions = new JvmOptions(); + List found = vmOptions.findVmOptions(procfsCmdline); + assertEquals(expected, found); + } + private void skipJdkJavaOptionsOnJava8(Map environmentVariables) { assumeTrue( JavaVirtualMachine.isJavaVersionAtLeast(9) @@ -121,4 +190,36 @@ private static Map env(String... keysAndValues) { } return env; } + + private static String argFile(String name) { + return "@src/test/resources/argfiles/" + name + ".txt"; + } + + private static List expectedArsFromArgFile(String name) { + List arguments = new ArrayList<>(); + try (InputStream stream = + requireNonNull( + CommandLineTest.class.getResourceAsStream("/argfiles/" + name + "-expected.txt")); + BufferedReader reader = new BufferedReader(new InputStreamReader(stream))) { + String line; + while ((line = reader.readLine()) != null) { + arguments.add(line); + } + } catch (IOException e) { + Assertions.fail("Failed to read expected args from " + name + "argfile", e); + } + return arguments; + } + + private static List flatten(Object... values) { + List result = new ArrayList<>(); + for (Object value : values) { + if (value instanceof Collection) { + result.addAll((Collection) value); + } else { + result.add(value.toString()); + } + } + return result; + } } diff --git a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java index f29fc91bc9c..d8f09c75089 100644 --- a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java +++ b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java @@ -64,7 +64,6 @@ import java.net.URISyntaxException; import java.net.URL; import java.security.CodeSource; -import java.util.Arrays; import java.util.EnumSet; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -268,19 +267,20 @@ public static void start( } } + boolean retryProfilerStart = false; if (profilingEnabled) { if (!isOracleJDK8()) { // Profiling agent startup code is written in a way to allow `startProfilingAgent` be called // multiple times // If early profiling is enabled then this call will start profiling. // If early profiling is disabled then later call will do this. - startProfilingAgent(true, inst); + retryProfilerStart = startProfilingAgent(true, true, inst); } else { log.debug("Oracle JDK 8 detected. Delaying profiler initialization."); // Profiling can not run early on Oracle JDK 8 because it will cause JFR initialization // deadlock. // Oracle JDK 8 JFR controller requires JMX so register an 'after-jmx-initialized' callback. - PROFILER_INIT_AFTER_JMX = () -> startProfilingAgent(false, inst); + PROFILER_INIT_AFTER_JMX = () -> startProfilingAgent(false, true, inst); } } @@ -376,7 +376,7 @@ public static void start( log.debug("Custom logger detected. Delaying Profiling initialization."); registerLogManagerCallback(new StartProfilingAgentCallback(inst)); } else { - startProfilingAgent(false, inst); + startProfilingAgent(false, retryProfilerStart, inst); // only enable instrumentation based profilers when we know JFR is ready InstrumentationBasedProfiling.enableInstrumentationBasedProfiling(); } @@ -671,7 +671,7 @@ public AgentThread agentThread() { @Override public void execute() { - startProfilingAgent(false, inst); + startProfilingAgent(false, true, inst); // only enable instrumentation based profilers when we know JFR is ready InstrumentationBasedProfiling.enableInstrumentationBasedProfiling(); } @@ -1229,48 +1229,51 @@ private static ProfilingContextIntegration createProfilingContextIntegration() { return ProfilingContextIntegration.NoOp.INSTANCE; } - private static void startProfilingAgent(final boolean isStartingFirst, Instrumentation inst) { - StaticEventLogger.begin("ProfilingAgent"); - + private static boolean startProfilingAgent( + final boolean earlyStart, final boolean firstAttempt, Instrumentation inst) { if (isAwsLambdaRuntime()) { - log.info("Profiling not supported in AWS Lambda runtimes"); - return; + if (firstAttempt) { + log.info("Profiling not supported in AWS Lambda runtimes"); + } + return false; } - final ClassLoader contextLoader = Thread.currentThread().getContextClassLoader(); - try { - Thread.currentThread().setContextClassLoader(AGENT_CLASSLOADER); - final Class profilingAgentClass = - AGENT_CLASSLOADER.loadClass("com.datadog.profiling.agent.ProfilingAgent"); - final Method profilingInstallerMethod = - profilingAgentClass.getMethod( - "run", Boolean.TYPE, ClassLoader.class, Instrumentation.class); - profilingInstallerMethod.invoke(null, isStartingFirst, AGENT_CLASSLOADER, inst); + boolean requestRetry = false; + + if (firstAttempt) { + StaticEventLogger.begin("ProfilingAgent"); + final ClassLoader contextLoader = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(AGENT_CLASSLOADER); + final Class profilingAgentClass = + AGENT_CLASSLOADER.loadClass("com.datadog.profiling.agent.ProfilingAgent"); + final Method profilingInstallerMethod = + profilingAgentClass.getMethod("run", Boolean.TYPE, Instrumentation.class); + requestRetry = (boolean) profilingInstallerMethod.invoke(null, earlyStart, inst); + } catch (final Throwable ex) { + log.error(SEND_TELEMETRY, "Throwable thrown while starting profiling agent", ex); + } finally { + Thread.currentThread().setContextClassLoader(contextLoader); + } + StaticEventLogger.end("ProfilingAgent"); + } + if (!earlyStart) { /* * Install the tracer hooks only when not using 'early start'. * The 'early start' is happening so early that most of the infrastructure has not been set up yet. */ - if (!isStartingFirst) { - log.debug("Scheduling scope event factory registration"); - WithGlobalTracer.registerOrExecute( - new WithGlobalTracer.Callback() { - @Override - public void withTracer(TracerAPI tracer) { - log.debug("Initializing profiler tracer integrations"); - tracer.getProfilingContext().onStart(); - } - }); - } - } catch (final Throwable t) { - log.error( - SEND_TELEMETRY, - "Throwable thrown while starting profiling agent " - + Arrays.toString(t.getCause().getStackTrace())); - } finally { - Thread.currentThread().setContextClassLoader(contextLoader); + initProfilerContext(); } + return requestRetry; + } - StaticEventLogger.end("ProfilingAgent"); + private static void initProfilerContext() { + log.debug("Scheduling profiler context initialization"); + WithGlobalTracer.registerOrExecute( + tracer -> { + log.debug("Initializing profiler context integration"); + tracer.getProfilingContext().onStart(); + }); } private static boolean isAwsLambdaRuntime() { diff --git a/dd-java-agent/agent-profiling/profiling-controller-ddprof/build.gradle b/dd-java-agent/agent-profiling/profiling-controller-ddprof/build.gradle index 721c858d565..9731a7ae8f3 100644 --- a/dd-java-agent/agent-profiling/profiling-controller-ddprof/build.gradle +++ b/dd-java-agent/agent-profiling/profiling-controller-ddprof/build.gradle @@ -23,6 +23,7 @@ excludedClassesCoverage += [ dependencies { api libs.slf4j api project(':internal-api') + api project(':utils:version-utils') api project(':dd-java-agent:agent-profiling:profiling-ddprof') api project(':dd-java-agent:agent-profiling:profiling-controller') api project(':dd-java-agent:agent-profiling:profiling-utils') diff --git a/dd-java-agent/agent-profiling/profiling-controller-ddprof/src/main/java/com/datadog/profiling/controller/ddprof/DatadogProfilerSettings.java b/dd-java-agent/agent-profiling/profiling-controller-ddprof/src/main/java/com/datadog/profiling/controller/ddprof/DatadogProfilerSettings.java index fd543ef40ae..b2746b82e5e 100644 --- a/dd-java-agent/agent-profiling/profiling-controller-ddprof/src/main/java/com/datadog/profiling/controller/ddprof/DatadogProfilerSettings.java +++ b/dd-java-agent/agent-profiling/profiling-controller-ddprof/src/main/java/com/datadog/profiling/controller/ddprof/DatadogProfilerSettings.java @@ -2,6 +2,7 @@ import com.datadog.profiling.controller.ProfilerSettingsSupport; import com.datadog.profiling.ddprof.DatadogProfiler; +import datadog.common.version.VersionInfo; import datadog.trace.bootstrap.config.provider.ConfigProvider; public class DatadogProfilerSettings extends ProfilerSettingsSupport { @@ -14,6 +15,7 @@ public DatadogProfilerSettings(DatadogProfiler datadogProfiler) { } public void publish() { + datadogProfiler.recordSetting(VERSION_KEY, VersionInfo.VERSION); datadogProfiler.recordSetting(UPLOAD_PERIOD_KEY, String.valueOf(uploadPeriod), "seconds"); datadogProfiler.recordSetting(UPLOAD_TIMEOUT_KEY, String.valueOf(uploadTimeout), "seconds"); datadogProfiler.recordSetting(UPLOAD_COMPRESSION_KEY, uploadCompression); @@ -27,7 +29,9 @@ public void publish() { datadogProfiler.recordSetting(PERF_EVENTS_PARANOID_KEY, perfEventsParanoid); datadogProfiler.recordSetting(NATIVE_STACKS_KEY, String.valueOf(hasNativeStacks)); datadogProfiler.recordSetting(JFR_IMPLEMENTATION_KEY, "ddprof"); - datadogProfiler.recordSetting(STACK_DEPTH_KEY, String.valueOf(stackDepth)); + datadogProfiler.recordSetting( + "ddprof " + STACK_DEPTH_KEY, + String.valueOf(requestedStackDepth)); // ddprof-java will accept the requested stack depth datadogProfiler.recordSetting(SELINUX_STATUS_KEY, seLinuxStatus); if (serviceInstrumentationType != null) { datadogProfiler.recordSetting(SERVICE_INSTRUMENTATION_TYPE, serviceInstrumentationType); diff --git a/dd-java-agent/agent-profiling/profiling-controller-jfr/implementation/build.gradle b/dd-java-agent/agent-profiling/profiling-controller-jfr/implementation/build.gradle index ae068359356..389d6674cfd 100644 --- a/dd-java-agent/agent-profiling/profiling-controller-jfr/implementation/build.gradle +++ b/dd-java-agent/agent-profiling/profiling-controller-jfr/implementation/build.gradle @@ -53,3 +53,13 @@ idea { jdkName = '11' } } + +project.afterEvaluate { + tasks.withType(Test).configureEach { + if (javaLauncher.get().metadata.languageVersion.asInt() >= 9) { + jvmArgs += [ + '--add-opens', + 'jdk.jfr/jdk.jfr.internal=ALL-UNNAMED'] // JPMSJFRAccess needs access to jdk.jfr.internal package + } + } +} diff --git a/dd-java-agent/agent-profiling/profiling-controller-jfr/implementation/src/main/java11/com/datadog/profiling/controller/jfr/JPMSJFRAccess.java b/dd-java-agent/agent-profiling/profiling-controller-jfr/implementation/src/main/java11/com/datadog/profiling/controller/jfr/JPMSJFRAccess.java index 54bd992e0f8..927a337b667 100644 --- a/dd-java-agent/agent-profiling/profiling-controller-jfr/implementation/src/main/java11/com/datadog/profiling/controller/jfr/JPMSJFRAccess.java +++ b/dd-java-agent/agent-profiling/profiling-controller-jfr/implementation/src/main/java11/com/datadog/profiling/controller/jfr/JPMSJFRAccess.java @@ -51,8 +51,7 @@ public JPMSJFRAccess(Instrumentation inst) throws Exception { jvmClass = JFRAccess.class.getClassLoader().loadClass("jdk.jfr.internal.JVM"); repositoryClass = JFRAccess.class.getClassLoader().loadClass("jdk.jfr.internal.Repository"); - safePathClass = - JFRAccess.class.getClassLoader().loadClass("jdk.jfr.internal.SecuritySupport$SafePath"); + safePathClass = safePathClass(); Object jvm = getJvm(); setStackDepthMH = getJvmMethodHandle(jvm, "setStackDepth", int.class); setRepositoryBaseMH = setRepositoryBaseMethodHandle(); @@ -60,9 +59,22 @@ public JPMSJFRAccess(Instrumentation inst) throws Exception { getTimeConversionFactorMH = getJvmMethodHandle(jvm, "getTimeConversionFactor"); } - private Object getJvm() - throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { - return jvmClass.getMethod("getJVM").invoke(null); + private static Class safePathClass() { + try { + return JFRAccess.class + .getClassLoader() + .loadClass("jdk.jfr.internal.SecuritySupport$SafePath"); + } catch (ClassNotFoundException e) { + return Path.class; // no SafePath with SecurityManager gone + } + } + + private Object getJvm() { + try { + return jvmClass.getMethod("getJVM").invoke(null); + } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException ignored) { + } + return null; } private MethodHandle getJvmMethodHandle(Object jvm, String method, Class... args) @@ -97,6 +109,11 @@ private MethodHandle setRepositoryBaseMethodHandle() } private static void patchModuleAccess(Instrumentation inst) { + if (inst == null) { + // used in testing; we don't have instrumentation and will patch the module access in the test + // task + return; + } Module unnamedModule = JFRAccess.class.getClassLoader().getUnnamedModule(); Module targetModule = Event.class.getModule(); @@ -126,7 +143,10 @@ public boolean setStackDepth(int depth) { @Override public boolean setBaseLocation(String location) { try { - Object safePath = safePathClass.getConstructor(Path.class).newInstance(Paths.get(location)); + Object safePath = + Path.class.isAssignableFrom(safePathClass) + ? Paths.get(location) + : safePathClass.getConstructor(Path.class).newInstance(Paths.get(location)); setRepositoryBaseMH.invoke(safePath); return true; } catch (Throwable throwable) { diff --git a/dd-java-agent/agent-profiling/profiling-controller-jfr/implementation/src/test/java/com/datadog/profiling/controller/jfr/JFRAccessTest.java b/dd-java-agent/agent-profiling/profiling-controller-jfr/implementation/src/test/java/com/datadog/profiling/controller/jfr/JFRAccessTest.java index 95d20bcda23..f15af6fe0db 100644 --- a/dd-java-agent/agent-profiling/profiling-controller-jfr/implementation/src/test/java/com/datadog/profiling/controller/jfr/JFRAccessTest.java +++ b/dd-java-agent/agent-profiling/profiling-controller-jfr/implementation/src/test/java/com/datadog/profiling/controller/jfr/JFRAccessTest.java @@ -2,6 +2,7 @@ import static datadog.environment.JavaVirtualMachine.isJ9; import static datadog.environment.JavaVirtualMachine.isJavaVersion; +import static datadog.environment.JavaVirtualMachine.isJavaVersionAtLeast; import static datadog.environment.JavaVirtualMachine.isOracleJDK8; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -22,6 +23,18 @@ void testJava8JFRAccess() { assertTrue(jfrAccess.setStackDepth(42)); } + @Test + void testJPMSJFRAccess() throws Exception { + // For Java 9 and above, the JFR access requires instrumentation in order to patch the module + // access + assumeTrue(isJavaVersionAtLeast(9) && !isJ9()); + + // just do a sanity check that it is possible to instantiate the class and call + // 'setStackDepth()' + JPMSJFRAccess jfrAccess = new JPMSJFRAccess(null); + assertTrue(jfrAccess.setStackDepth(42)); + } + @Test void testJ9JFRAccess() { assumeTrue(isJ9()); diff --git a/dd-java-agent/agent-profiling/profiling-controller-openjdk/build.gradle b/dd-java-agent/agent-profiling/profiling-controller-openjdk/build.gradle index 2d546b84c3e..f0ef842eef7 100644 --- a/dd-java-agent/agent-profiling/profiling-controller-openjdk/build.gradle +++ b/dd-java-agent/agent-profiling/profiling-controller-openjdk/build.gradle @@ -17,6 +17,7 @@ apply plugin: 'idea' dependencies { api libs.slf4j api project(':internal-api') + api project(':utils:version-utils') api(project(':dd-java-agent:agent-bootstrap')) { exclude group: 'com.datadoghq', module: 'agent-logging' } diff --git a/dd-java-agent/agent-profiling/profiling-controller-openjdk/src/main/java/com/datadog/profiling/controller/openjdk/JfrProfilerSettings.java b/dd-java-agent/agent-profiling/profiling-controller-openjdk/src/main/java/com/datadog/profiling/controller/openjdk/JfrProfilerSettings.java index 0ffa1c2e1b8..f74d626909a 100644 --- a/dd-java-agent/agent-profiling/profiling-controller-openjdk/src/main/java/com/datadog/profiling/controller/openjdk/JfrProfilerSettings.java +++ b/dd-java-agent/agent-profiling/profiling-controller-openjdk/src/main/java/com/datadog/profiling/controller/openjdk/JfrProfilerSettings.java @@ -1,7 +1,9 @@ package com.datadog.profiling.controller.openjdk; +import com.datadog.profiling.controller.ControllerContext; import com.datadog.profiling.controller.ProfilerSettingsSupport; import com.datadog.profiling.controller.openjdk.events.ProfilerSettingEvent; +import datadog.common.version.VersionInfo; import datadog.environment.JavaVirtualMachine; import datadog.trace.api.Platform; import datadog.trace.bootstrap.config.provider.ConfigProvider; @@ -14,20 +16,23 @@ final class JfrProfilerSettings extends ProfilerSettingsSupport { private static final String EXCEPTION_HISTO_REPORT_LIMIT_KEY = "Exception Histo Report Limit"; private static final String EXCEPTION_HISTO_SIZE_LIMIT_KEY = "Exception Histo Size Limit"; private final String jfrImplementation; + private final boolean isDdprofActive; public JfrProfilerSettings( ConfigProvider configProvider, - String ddprofUnavailableReason, + ControllerContext.Snapshot context, boolean hasJfrStackDepthApplied) { - super(configProvider, ddprofUnavailableReason, hasJfrStackDepthApplied); + super(configProvider, context.getDatadogProfilerUnavailableReason(), hasJfrStackDepthApplied); this.jfrImplementation = Platform.isNativeImage() ? "native-image" : (JavaVirtualMachine.isOracleJDK8() ? "oracle" : "openjdk"); + this.isDdprofActive = context.isDatadogProfilerEnabled(); } public void publish() { if (new ProfilerSettingEvent(null, null, null).isEnabled()) { + new ProfilerSettingEvent(VERSION_KEY, VersionInfo.VERSION).commit(); new ProfilerSettingEvent(UPLOAD_PERIOD_KEY, String.valueOf(uploadPeriod), "seconds").commit(); new ProfilerSettingEvent(UPLOAD_TIMEOUT_KEY, String.valueOf(uploadTimeout), "seconds") .commit(); @@ -56,8 +61,16 @@ public void publish() { new ProfilerSettingEvent(PERF_EVENTS_PARANOID_KEY, perfEventsParanoid).commit(); new ProfilerSettingEvent(NATIVE_STACKS_KEY, String.valueOf(hasNativeStacks)).commit(); new ProfilerSettingEvent(JFR_IMPLEMENTATION_KEY, jfrImplementation).commit(); - if (hasJfrStackDepthApplied) { - new ProfilerSettingEvent(STACK_DEPTH_KEY, String.valueOf(stackDepth)).commit(); + new ProfilerSettingEvent( + "JFR " + STACK_DEPTH_KEY, + String.valueOf(hasJfrStackDepthApplied ? requestedStackDepth : jfrStackDepth)) + .commit(); + if (isDdprofActive) { + // emit this setting only if datadog profiler is also active + new ProfilerSettingEvent( + "ddprof " + STACK_DEPTH_KEY, + String.valueOf(hasJfrStackDepthApplied ? requestedStackDepth : jfrStackDepth)) + .commit(); } new ProfilerSettingEvent(SELINUX_STATUS_KEY, seLinuxStatus).commit(); if (ddprofUnavailableReason != null) { diff --git a/dd-java-agent/agent-profiling/profiling-controller-openjdk/src/main/java/com/datadog/profiling/controller/openjdk/OpenJdkOngoingRecording.java b/dd-java-agent/agent-profiling/profiling-controller-openjdk/src/main/java/com/datadog/profiling/controller/openjdk/OpenJdkOngoingRecording.java index d93832c12e8..eab6ebc1e76 100644 --- a/dd-java-agent/agent-profiling/profiling-controller-openjdk/src/main/java/com/datadog/profiling/controller/openjdk/OpenJdkOngoingRecording.java +++ b/dd-java-agent/agent-profiling/profiling-controller-openjdk/src/main/java/com/datadog/profiling/controller/openjdk/OpenJdkOngoingRecording.java @@ -60,10 +60,7 @@ public class OpenJdkOngoingRecording implements OngoingRecording { recording.start(); log.debug("Recording {} started", recordingName); this.configMemento = - new JfrProfilerSettings( - configProvider, - context.getDatadogProfilerUnavailableReason(), - jfrStackDepthSettingApplied); + new JfrProfilerSettings(configProvider, context, jfrStackDepthSettingApplied); } OpenJdkOngoingRecording( @@ -77,10 +74,7 @@ public class OpenJdkOngoingRecording implements OngoingRecording { recording.start(); log.debug("Recording {} started", recording.getName()); this.configMemento = - new JfrProfilerSettings( - ConfigProvider.getInstance(), - context.getDatadogProfilerUnavailableReason(), - jfrStackDepthSettingApplied); + new JfrProfilerSettings(ConfigProvider.getInstance(), context, jfrStackDepthSettingApplied); } private void disableOverriddenEvents(ControllerContext.Snapshot context) { diff --git a/dd-java-agent/agent-profiling/profiling-controller/build.gradle b/dd-java-agent/agent-profiling/profiling-controller/build.gradle index e880bf542c2..e255fdf668d 100644 --- a/dd-java-agent/agent-profiling/profiling-controller/build.gradle +++ b/dd-java-agent/agent-profiling/profiling-controller/build.gradle @@ -19,6 +19,7 @@ excludedClassesCoverage += [ dependencies { api libs.slf4j api project(':internal-api') + api project(':components:environment') api project(':dd-java-agent:agent-profiling:profiling-utils') testImplementation libs.bundles.junit5 diff --git a/dd-java-agent/agent-profiling/profiling-controller/src/main/java/com/datadog/profiling/controller/ProfilerSettingsSupport.java b/dd-java-agent/agent-profiling/profiling-controller/src/main/java/com/datadog/profiling/controller/ProfilerSettingsSupport.java index 7de85812bf8..ed2e8a688be 100644 --- a/dd-java-agent/agent-profiling/profiling-controller/src/main/java/com/datadog/profiling/controller/ProfilerSettingsSupport.java +++ b/dd-java-agent/agent-profiling/profiling-controller/src/main/java/com/datadog/profiling/controller/ProfilerSettingsSupport.java @@ -2,6 +2,7 @@ import static datadog.trace.api.telemetry.LogCollector.SEND_TELEMETRY; +import datadog.environment.JavaVirtualMachine; import datadog.environment.OperatingSystem; import datadog.trace.api.Config; import datadog.trace.api.config.ProfilingConfig; @@ -24,6 +25,8 @@ /** Capture the profiler config first and allow emitting the setting events per each recording. */ public abstract class ProfilerSettingsSupport { private static final Logger logger = LoggerFactory.getLogger(ProfilerSettingsSupport.class); + private static final String STACKDEPTH_KEY = "stackdepth="; + private static final int DEFAULT_JFR_STACKDEPTH = 64; protected static final class ProfilerActivationSetting { public enum Ssi { @@ -63,6 +66,7 @@ public String toString() { } } + protected static final String VERSION_KEY = "Java Agent Version"; protected static final String JFR_IMPLEMENTATION_KEY = "JFR Implementation"; protected static final String UPLOAD_PERIOD_KEY = "Upload Period"; protected static final String UPLOAD_TIMEOUT_KEY = "Upload Timeout"; @@ -108,7 +112,8 @@ public String toString() { protected final ProfilerActivationSetting profilerActivationSetting; - protected final int stackDepth; + protected final int jfrStackDepth; + protected final int requestedStackDepth; protected final boolean hasJfrStackDepthApplied; protected ProfilerSettingsSupport( @@ -170,11 +175,10 @@ protected ProfilerSettingsSupport( configProvider.getString( "profiling.async.cstack", ProfilingConfig.PROFILING_DATADOG_PROFILER_CSTACK_DEFAULT))); - stackDepth = + requestedStackDepth = configProvider.getInteger( - ProfilingConfig.PROFILING_STACKDEPTH, - ProfilingConfig.PROFILING_STACKDEPTH_DEFAULT, - ProfilingConfig.PROFILING_DATADOG_PROFILER_STACKDEPTH); + ProfilingConfig.PROFILING_STACKDEPTH, ProfilingConfig.PROFILING_STACKDEPTH_DEFAULT); + jfrStackDepth = getStackDepth(); seLinuxStatus = getSELinuxStatus(); this.ddprofUnavailableReason = ddprofUnavailableReason; @@ -191,6 +195,30 @@ protected ProfilerSettingsSupport( "Profiler settings: " + this); // telemetry receiver does not recognize formatting } + private static int getStackDepth() { + String value = + JavaVirtualMachine.getVmOptions().stream() + .filter(o -> o.startsWith("-XX:FlightRecorderOptions")) + .findFirst() + .orElse(null); + if (value != null) { + int start = value.indexOf(STACKDEPTH_KEY); + if (start != -1) { + start += STACKDEPTH_KEY.length(); + int end = value.indexOf(',', start); + if (end == -1) { + end = value.length(); + } + try { + return Integer.parseInt(value.substring(start, end)); + } catch (NumberFormatException e) { + logger.debug(SEND_TELEMETRY, "Failed to parse stack depth from JFR options: " + value, e); + } + } + } + return DEFAULT_JFR_STACKDEPTH; // default stack depth if not set in JFR options + } + private static String getServiceInjection(ConfigProvider configProvider) { // usually set via DD_INJECTION_ENABLED env var return configProvider.getString("injection.enabled"); @@ -284,7 +312,8 @@ public String toString() { + ", serviceInjection='" + serviceInjection + '\'' + ", ddprofUnavailableReason='" + ddprofUnavailableReason + '\'' + ", profilerActivationSetting=" + profilerActivationSetting - + ", stackDepth=" + stackDepth + + ", jfrStackDepth=" + jfrStackDepth + + ", requestedStackDepth=" + requestedStackDepth + ", hasJfrStackDepthApplied=" + hasJfrStackDepthApplied + '}'; // spotless:on diff --git a/dd-java-agent/agent-profiling/src/main/java/com/datadog/profiling/agent/ProfilingAgent.java b/dd-java-agent/agent-profiling/src/main/java/com/datadog/profiling/agent/ProfilingAgent.java index 73582df86c4..0be95f227aa 100644 --- a/dd-java-agent/agent-profiling/src/main/java/com/datadog/profiling/agent/ProfilingAgent.java +++ b/dd-java-agent/agent-profiling/src/main/java/com/datadog/profiling/agent/ProfilingAgent.java @@ -87,8 +87,7 @@ public void onNewData(RecordingType type, RecordingData data, boolean handleSync * Main entry point into profiling Note: this must be reentrant because we may want to start * profiling before any other tool, and then attempt to start it again at normal time */ - public static synchronized void run( - final boolean isStartingFirst, ClassLoader agentClasLoader, Instrumentation inst) + public static synchronized boolean run(final boolean earlyStart, Instrumentation inst) throws IllegalArgumentException, IOException { if (profiler == null) { final Config config = Config.get(); @@ -105,19 +104,19 @@ public static synchronized void run( startForceFirst = false; } - if (isStartingFirst && !startForceFirst) { + if (earlyStart && !startForceFirst) { log.debug("Profiling: not starting first"); // early startup is disabled; - return; + return true; } if (!config.isProfilingEnabled()) { log.debug(SEND_TELEMETRY, "Profiling: disabled"); - return; + return false; } if (config.getApiKey() != null && !API_KEY_REGEX.test(config.getApiKey())) { log.info( "Profiling: API key doesn't match expected format, expected to get a 32 character hex string. Profiling is disabled."); - return; + return false; } try { @@ -176,6 +175,7 @@ public static synchronized void run( log.debug(SEND_TELEMETRY, "Failed to initialize profiling agent!", e); } } + return false; } private static boolean isStartForceFirstSafe() { diff --git a/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/ProfilerInstaller.java b/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/ProfilerInstaller.java index 086dbae8743..3488f7615dd 100644 --- a/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/ProfilerInstaller.java +++ b/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/ProfilerInstaller.java @@ -7,7 +7,7 @@ public class ProfilerInstaller { public static boolean installProfiler() { if (Config.get().isProfilingEnabled()) { try { - ProfilingAgent.run(true, null, null); + ProfilingAgent.run(true, null); return true; } catch (Exception e) { throw new RuntimeException(e); diff --git a/dd-smoke-tests/profiling-integration-tests/build.gradle b/dd-smoke-tests/profiling-integration-tests/build.gradle index 2e1d8615265..d87a0746379 100644 --- a/dd-smoke-tests/profiling-integration-tests/build.gradle +++ b/dd-smoke-tests/profiling-integration-tests/build.gradle @@ -36,6 +36,11 @@ dependencies { tasks.withType(Test).configureEach { dependsOn "shadowJar" jvmArgs "-Ddatadog.smoketest.profiling.shadowJar.path=${tasks.shadowJar.archiveFile.get()}" + + testLogging { + events "started", "passed", "skipped", "failed" + showStandardStreams = true + } } shadowJar { diff --git a/dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/JFRBasedProfilingIntegrationTest.java b/dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/JFRBasedProfilingIntegrationTest.java index 9ed4eb6f924..9e6ae68a8d6 100644 --- a/dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/JFRBasedProfilingIntegrationTest.java +++ b/dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/JFRBasedProfilingIntegrationTest.java @@ -191,7 +191,7 @@ public void testContinuousRecording(final TestInfo testInfo) throws Exception { @Test @DisplayName("Test continuous recording - 1 sec jmx delay, zstd compression") - public void testContinuousRecording_jmethodid_cache(final TestInfo testInfo) throws Exception { + public void testContinuousRecording_zstd(final TestInfo testInfo) throws Exception { testWithRetry( () -> testContinuousRecording( @@ -461,7 +461,7 @@ void testBogusApiKey(final TestInfo testInfo) throws Exception { */ final long ts = System.nanoTime(); while (!checkLogLines( - logFilePath, line -> line.contains("Initializing profiler tracer integrations"))) { + logFilePath, line -> line.contains("Initializing profiler context integration"))) { Thread.sleep(500); // Wait at most 30 seconds if (System.nanoTime() - ts > 30_000_000_000L) { @@ -584,15 +584,6 @@ private void assertRecordingEvents( events.apply(ItemFilters.type("jdk.SystemProcess")).hasItems(), "jdk.SystemProcess events should not be collected"); - assertTrue( - events - .apply( - ItemFilters.and( - ItemFilters.type("datadog.ProfilerSetting"), - ItemFilters.equals(JdkAttributes.REC_SETTING_NAME, "Stack Depth"), - ItemFilters.equals( - JdkAttributes.REC_SETTING_VALUE, String.valueOf(STACK_DEPTH_LIMIT)))) - .hasItems()); if (expectEndpointEvents) { // Check endpoint events final IItemCollection endpointEvents = events.apply(ItemFilters.type("datadog.Endpoint")); @@ -619,6 +610,8 @@ private void assertRecordingEvents( } } } + assertEquals(asyncProfilerEnabled, hasAuxiliaryDdprof(events)); + verifyStackDepthSetting(events, asyncProfilerEnabled); if (asyncProfilerEnabled) { verifyJdkEventsDisabled(events); verifyDatadogEventsNotCorrupt(events); @@ -662,8 +655,45 @@ private void assertRecordingEvents( assertEquals(Runtime.getRuntime().availableProcessors(), val); assertTrue(events.apply(ItemFilters.type("datadog.ProfilerSetting")).hasItems()); - // FIXME - for some reason the events are disabled by JFR despite being explicitly enabled - // assertTrue(events.apply(ItemFilters.type("datadog.QueueTime")).hasItems()); + // FIXME - for some reason the events are disabled by JFR despite being explicitly enabled + // assertTrue(events.apply(ItemFilters.type("datadog.QueueTime")).hasItems()); + } + + private static void verifyStackDepthSetting( + IItemCollection events, boolean asyncProfilerEnabled) { + assertTrue( + events + .apply( + ItemFilters.and( + ItemFilters.type("datadog.ProfilerSetting"), + ItemFilters.equals( + JdkAttributes.REC_SETTING_NAME, + (asyncProfilerEnabled ? "ddprof" : "JFR") + " Stack Depth"), + ItemFilters.equals( + JdkAttributes.REC_SETTING_VALUE, String.valueOf(STACK_DEPTH_LIMIT)))) + .hasItems()); + } + + private static boolean hasAuxiliaryDdprof(IItemCollection events) { + events = + events.apply( + ItemFilters.and( + ItemFilters.type("datadog.ProfilerSetting"), + ItemFilters.equals(JdkAttributes.REC_SETTING_NAME, "Auxiliary Profiler"))); + if (!events.hasItems()) { + return false; + } + for (IItemIterable event : events) { + IMemberAccessor valueAccessor = + JdkAttributes.REC_SETTING_VALUE.getAccessor(event.getType()); + for (IItem item : event) { + String value = valueAccessor.getMember(item); + if ("ddprof".equals(value)) { + return true; + } + } + } + return false; } private static void processExecutionSamples(