Fix the profiler stackdepth setting propagation in recent (22+) Java versions

components/environment/src/main/java/datadog/environment/JvmOptions.java

@@ -15,7 +15,6 @@
 import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.ListIterator;
 import java.util.StringTokenizer;
 
 /** Fetches and captures the JVM options. */
@@ -41,28 +40,44 @@ private String[] readProcFsCmdLine() {
     return null;
   }
 
-  @SuppressForbidden // Class.forName() as backup
   private List<String> findVmOptions() {
+    return findVmOptions(PROCFS_CMDLINE);
+  }
+
+  @SuppressForbidden // Class.forName() as backup
+  // Visible for testing
+  List<String> findVmOptions(String[] procfsCmdline) {
     // Try ProcFS on Linux
-    if (PROCFS_CMDLINE != null) {
+    // Be aware that when running a native image, the command line in /proc/self/cmdline is just the
+    // executable
+    if (procfsCmdline != null) {
+      // Create list of VM options
+      List<String> vmOptions = new ArrayList<>();
       // Start at 1 to skip "java" command itself
       int index = 1;
-      // Look for main class or "-jar", end of VM options
-      for (; index < PROCFS_CMDLINE.length; index++) {
-        if (!PROCFS_CMDLINE[index].startsWith("-") || "-jar".equals(PROCFS_CMDLINE[index])) {
-          break;
-        }
-      }
-      // Create list of VM options
-      List<String> vmOptions = new ArrayList<>(asList(PROCFS_CMDLINE).subList(1, index + 1));
-      ListIterator<String> iterator = vmOptions.listIterator();
-      while (iterator.hasNext()) {
-        String vmOption = iterator.next();
-        if (vmOption.startsWith("@")) {
-          iterator.remove();
-          for (String argument : getArgumentsFromFile(vmOption)) {
-            iterator.add(argument);
+      // Look for first self-standing argument that is not prefixed with "-" or end of VM options
+      // Skip "-jar" and the jar file
+      // Simultaneously, collect all arguments in the VM options
+      for (; index < procfsCmdline.length; index++) {
+        String argument = procfsCmdline[index];
+        if (argument.startsWith("@")) {
+          vmOptions.addAll(getArgumentsFromFile(argument));
+        } else {
+          if ("-jar".equals(argument)) {
+            // skip "-jar" and the jar file
+            index++;
+            continue;
+          } else if ("-cp".equals(argument)) {
+            // slurp '-cp' and the classpath
+            vmOptions.add(argument);
+            if (index + 1 < procfsCmdline.length) {
+              argument = procfsCmdline[++index];
+            }
+          } else if (!argument.startsWith("-")) {
+            // end of VM options
+            break;
           }
+          vmOptions.add(argument);
         }
       }
       // Insert JDK_JAVA_OPTIONS at the start if present and supported

components/environment/src/test/java/datadog/environment/JvmOptionsTest.java

@@ -8,15 +8,23 @@
 import static java.util.Collections.emptyList;
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.singletonList;
+import static java.util.Objects.requireNonNull;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assumptions.assumeTrue;
 import static org.junit.jupiter.params.provider.Arguments.arguments;
 
 import datadog.environment.CommandLineHelper.Result;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Stream;
+import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.Arguments;
 import org.junit.jupiter.params.provider.MethodSource;
@@ -105,6 +113,67 @@ void testFindVmOptions(
     assertEquals(expectedArguments.jvmOptions, result.jvmOptions, "Failed to get JVM options");
   }
 
+  @MethodSource
+  private static Stream<Arguments> procFsCmdLine() {
+    // spotless:off
+    return Stream.of(
+        arguments(
+            "No arguments",
+            new String[0],
+            emptyList()
+        ),
+        arguments(
+            "Native image launcher",
+            new String[]{"native-image-launcher", "-Xmx512m"},
+            singletonList("-Xmx512m")
+        ),
+        arguments(
+            "Java with JAR and options",
+            new String[]{"java", "-Xmx512m", "-Xms256m", "-jar", "app.jar"},
+            asList("-Xmx512m", "-Xms256m")
+        ),
+        arguments(
+            "Java from class and options",
+            new String[]{"java", "-Xmx512m", "-Xms256m", "-cp", "app.jar", "Main"},
+            asList("-Xmx512m", "-Xms256m", "-cp", "app.jar")
+        ),
+        arguments(
+            "Java from class and options, mixed",
+            new String[]{"java", "-Xms256m", "-cp", "app.jar", "-Xmx512m", "Main"},
+            asList("-Xms256m", "-cp", "app.jar", "-Xmx512m")
+        ),
+        arguments(
+            "Args from file",
+            new String[]{"java", "-Dargfile.prop=test", "-verbose:class", argFile("carriage-return-separated"), "-jar", "app.jar"},
+            flatten("-Dargfile.prop=test", "-verbose:class", expectedArsFromArgFile("carriage-return-separated"))
+        ),
+        arguments(
+            "Args from file",
+            new String[]{"java", "-Dargfile.prop=test", "-verbose:class", argFile("new-line-separated"), "-jar", "app.jar"},
+            flatten("-Dargfile.prop=test", "-verbose:class", expectedArsFromArgFile("new-line-separated"))
+        ),
+        arguments(
+            "Args from file",
+            new String[]{"java", "-Dargfile.prop=test", "-verbose:class", argFile("space-separated"), "-jar", "app.jar"},
+            flatten("-Dargfile.prop=test", "-verbose:class", expectedArsFromArgFile("space-separated"))
+        ),
+        arguments(
+            "Args from file",
+            new String[]{"java", "-Dargfile.prop=test", "-verbose:class", argFile("tab-separated"), "-jar", "app.jar"},
+            flatten("-Dargfile.prop=test", "-verbose:class", expectedArsFromArgFile("tab-separated"))
+        ));
+    // spotless:on
+  }
+
+  @ParameterizedTest(name = "[{index}] {0}")
+  @MethodSource("procFsCmdLine")
+  void testFindVmOptionsWithProcFsCmdLine(
+      String useCase, String[] procfsCmdline, List<String> expected) throws Exception {
+    JvmOptions vmOptions = new JvmOptions();
+    List<String> found = vmOptions.findVmOptions(procfsCmdline);
+    assertEquals(expected, found);
+  }
+
   private void skipJdkJavaOptionsOnJava8(Map<String, String> environmentVariables) {
     assumeTrue(
         JavaVirtualMachine.isJavaVersionAtLeast(9)
@@ -121,4 +190,36 @@ private static Map<String, String> env(String... keysAndValues) {
     }
     return env;
   }
+
+  private static String argFile(String name) {
+    return "@src/test/resources/argfiles/" + name + ".txt";
+  }
+
+  private static List<String> expectedArsFromArgFile(String name) {
+    List<String> arguments = new ArrayList<>();
+    try (InputStream stream =
+            requireNonNull(
+                CommandLineTest.class.getResourceAsStream("/argfiles/" + name + "-expected.txt"));
+        BufferedReader reader = new BufferedReader(new InputStreamReader(stream))) {
+      String line;
+      while ((line = reader.readLine()) != null) {
+        arguments.add(line);
+      }
+    } catch (IOException e) {
+      Assertions.fail("Failed to read expected args from " + name + "argfile", e);
+    }
+    return arguments;
+  }
+
+  private static List<String> flatten(Object... values) {
+    List<String> result = new ArrayList<>();
+    for (Object value : values) {
+      if (value instanceof Collection) {
+        result.addAll((Collection<? extends String>) value);
+      } else {
+        result.add(value.toString());
+      }
+    }
+    return result;
+  }
 }

dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java

@@ -64,7 +64,6 @@
 import java.net.URISyntaxException;
 import java.net.URL;
 import java.security.CodeSource;
-import java.util.Arrays;
 import java.util.EnumSet;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -268,19 +267,20 @@ public static void start(
       }
     }
 
+    boolean retryProfilerStart = false;
     if (profilingEnabled) {
       if (!isOracleJDK8()) {
         // Profiling agent startup code is written in a way to allow `startProfilingAgent` be called
         // multiple times
         // If early profiling is enabled then this call will start profiling.
         // If early profiling is disabled then later call will do this.
-        startProfilingAgent(true, inst);
+        retryProfilerStart = startProfilingAgent(true, true, inst);
       } else {
         log.debug("Oracle JDK 8 detected. Delaying profiler initialization.");
         // Profiling can not run early on Oracle JDK 8 because it will cause JFR initialization
         // deadlock.
         // Oracle JDK 8 JFR controller requires JMX so register an 'after-jmx-initialized' callback.
-        PROFILER_INIT_AFTER_JMX = () -> startProfilingAgent(false, inst);
+        PROFILER_INIT_AFTER_JMX = () -> startProfilingAgent(false, true, inst);
       }
     }
 
@@ -376,7 +376,7 @@ public static void start(
         log.debug("Custom logger detected. Delaying Profiling initialization.");
         registerLogManagerCallback(new StartProfilingAgentCallback(inst));
       } else {
-        startProfilingAgent(false, inst);
+        startProfilingAgent(false, retryProfilerStart, inst);
         // only enable instrumentation based profilers when we know JFR is ready
         InstrumentationBasedProfiling.enableInstrumentationBasedProfiling();
       }
@@ -671,7 +671,7 @@ public AgentThread agentThread() {
 
     @Override
     public void execute() {
-      startProfilingAgent(false, inst);
+      startProfilingAgent(false, true, inst);
       // only enable instrumentation based profilers when we know JFR is ready
       InstrumentationBasedProfiling.enableInstrumentationBasedProfiling();
     }
@@ -1229,48 +1229,51 @@ private static ProfilingContextIntegration createProfilingContextIntegration() {
     return ProfilingContextIntegration.NoOp.INSTANCE;
   }
 
-  private static void startProfilingAgent(final boolean isStartingFirst, Instrumentation inst) {
-    StaticEventLogger.begin("ProfilingAgent");
-
+  private static boolean startProfilingAgent(
+      final boolean earlyStart, final boolean firstAttempt, Instrumentation inst) {
     if (isAwsLambdaRuntime()) {
-      log.info("Profiling not supported in AWS Lambda runtimes");
-      return;
+      if (firstAttempt) {
+        log.info("Profiling not supported in AWS Lambda runtimes");
+      }
+      return false;
     }
 
-    final ClassLoader contextLoader = Thread.currentThread().getContextClassLoader();
-    try {
-      Thread.currentThread().setContextClassLoader(AGENT_CLASSLOADER);
-      final Class<?> profilingAgentClass =
-          AGENT_CLASSLOADER.loadClass("com.datadog.profiling.agent.ProfilingAgent");
-      final Method profilingInstallerMethod =
-          profilingAgentClass.getMethod(
-              "run", Boolean.TYPE, ClassLoader.class, Instrumentation.class);
-      profilingInstallerMethod.invoke(null, isStartingFirst, AGENT_CLASSLOADER, inst);
+    boolean requestRetry = false;
+
+    if (firstAttempt) {
+      StaticEventLogger.begin("ProfilingAgent");
+      final ClassLoader contextLoader = Thread.currentThread().getContextClassLoader();
+      try {
+        Thread.currentThread().setContextClassLoader(AGENT_CLASSLOADER);
+        final Class<?> profilingAgentClass =
+            AGENT_CLASSLOADER.loadClass("com.datadog.profiling.agent.ProfilingAgent");
+        final Method profilingInstallerMethod =
+            profilingAgentClass.getMethod("run", Boolean.TYPE, Instrumentation.class);
+        requestRetry = (boolean) profilingInstallerMethod.invoke(null, earlyStart, inst);
+      } catch (final Throwable ex) {
+        log.error(SEND_TELEMETRY, "Throwable thrown while starting profiling agent", ex);
+      } finally {
+        Thread.currentThread().setContextClassLoader(contextLoader);
+      }
+      StaticEventLogger.end("ProfilingAgent");
+    }
+    if (!earlyStart) {
       /*
        * Install the tracer hooks only when not using 'early start'.
        * The 'early start' is happening so early that most of the infrastructure has not been set up yet.
        */
-      if (!isStartingFirst) {
-        log.debug("Scheduling scope event factory registration");
-        WithGlobalTracer.registerOrExecute(
-            new WithGlobalTracer.Callback() {
-              @Override
-              public void withTracer(TracerAPI tracer) {
-                log.debug("Initializing profiler tracer integrations");
-                tracer.getProfilingContext().onStart();
-              }
-            });
-      }
-    } catch (final Throwable t) {
-      log.error(
-          SEND_TELEMETRY,
-          "Throwable thrown while starting profiling agent "
-              + Arrays.toString(t.getCause().getStackTrace()));
-    } finally {
-      Thread.currentThread().setContextClassLoader(contextLoader);
+      initProfilerContext();
     }
+    return requestRetry;
+  }
 
-    StaticEventLogger.end("ProfilingAgent");
+  private static void initProfilerContext() {
+    log.debug("Scheduling profiler context initialization");
+    WithGlobalTracer.registerOrExecute(
+        tracer -> {
+          log.debug("Initializing profiler context integration");
+          tracer.getProfilingContext().onStart();
+        });
   }
 
   private static boolean isAwsLambdaRuntime() {

dd-java-agent/agent-profiling/profiling-controller-ddprof/build.gradle

@@ -23,6 +23,7 @@ excludedClassesCoverage += [
 dependencies {
   api libs.slf4j
   api project(':internal-api')
+  api project(':utils:version-utils')
   api project(':dd-java-agent:agent-profiling:profiling-ddprof')
   api project(':dd-java-agent:agent-profiling:profiling-controller')
   api project(':dd-java-agent:agent-profiling:profiling-utils')

dd-java-agent/agent-profiling/profiling-controller-ddprof/src/main/java/com/datadog/profiling/controller/ddprof/DatadogProfilerSettings.java

@@ -2,6 +2,7 @@
 
 import com.datadog.profiling.controller.ProfilerSettingsSupport;
 import com.datadog.profiling.ddprof.DatadogProfiler;
+import datadog.common.version.VersionInfo;
 import datadog.trace.bootstrap.config.provider.ConfigProvider;
 
 public class DatadogProfilerSettings extends ProfilerSettingsSupport {
@@ -14,6 +15,7 @@ public DatadogProfilerSettings(DatadogProfiler datadogProfiler) {
   }
 
   public void publish() {
+    datadogProfiler.recordSetting(VERSION_KEY, VersionInfo.VERSION);
     datadogProfiler.recordSetting(UPLOAD_PERIOD_KEY, String.valueOf(uploadPeriod), "seconds");
     datadogProfiler.recordSetting(UPLOAD_TIMEOUT_KEY, String.valueOf(uploadTimeout), "seconds");
     datadogProfiler.recordSetting(UPLOAD_COMPRESSION_KEY, uploadCompression);
@@ -27,7 +29,9 @@ public void publish() {
     datadogProfiler.recordSetting(PERF_EVENTS_PARANOID_KEY, perfEventsParanoid);
     datadogProfiler.recordSetting(NATIVE_STACKS_KEY, String.valueOf(hasNativeStacks));
     datadogProfiler.recordSetting(JFR_IMPLEMENTATION_KEY, "ddprof");
-    datadogProfiler.recordSetting(STACK_DEPTH_KEY, String.valueOf(stackDepth));
+    datadogProfiler.recordSetting(
+        "ddprof " + STACK_DEPTH_KEY,
+        String.valueOf(requestedStackDepth)); // ddprof-java will accept the requested stack depth
     datadogProfiler.recordSetting(SELINUX_STATUS_KEY, seLinuxStatus);
     if (serviceInstrumentationType != null) {
       datadogProfiler.recordSetting(SERVICE_INSTRUMENTATION_TYPE, serviceInstrumentationType);

dd-java-agent/agent-profiling/profiling-controller-jfr/implementation/build.gradle

@@ -53,3 +53,13 @@ idea {
     jdkName = '11'
   }
 }
+
+project.afterEvaluate {
+  tasks.withType(Test).configureEach {
+    if (javaLauncher.get().metadata.languageVersion.asInt() >= 9) {
+      jvmArgs += [
+        '--add-opens',
+        'jdk.jfr/jdk.jfr.internal=ALL-UNNAMED'] // JPMSJFRAccess needs access to jdk.jfr.internal package
+    }
+  }
+}