diff --git a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java index ded84aa1176..00b54848832 100644 --- a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java +++ b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java @@ -184,13 +184,11 @@ public static void start( if (Platform.isNativeImageBuilder()) { // these default services are not used during native-image builds - jmxFetchEnabled = false; remoteConfigEnabled = false; telemetryEnabled = false; - // apply trace instrumentation, but skip starting other services + // apply trace instrumentation, but skip other products at native-image build time startDatadogAgent(initTelemetry, inst); StaticEventLogger.end("Agent.start"); - return; } diff --git a/dd-java-agent/agent-jmxfetch/build.gradle b/dd-java-agent/agent-jmxfetch/build.gradle index 22774b81756..c7c26f3c04f 100644 --- a/dd-java-agent/agent-jmxfetch/build.gradle +++ b/dd-java-agent/agent-jmxfetch/build.gradle @@ -11,7 +11,7 @@ plugins { apply from: "$rootDir/gradle/java.gradle" dependencies { - api('com.datadoghq:jmxfetch:0.49.6') { + api('com.datadoghq:jmxfetch:0.49.7') { exclude group: 'org.slf4j', module: 'slf4j-api' exclude group: 'org.slf4j', module: 'slf4j-jdk14' exclude group: 'com.beust', module: 'jcommander' diff --git a/dd-java-agent/agent-jmxfetch/src/main/java/com/fasterxml/jackson/core/JsonProcessingException.java b/dd-java-agent/agent-jmxfetch/src/main/java/com/fasterxml/jackson/core/JsonProcessingException.java deleted file mode 100644 index 4ceefbc5622..00000000000 --- a/dd-java-agent/agent-jmxfetch/src/main/java/com/fasterxml/jackson/core/JsonProcessingException.java +++ /dev/null @@ -1,4 +0,0 @@ -package com.fasterxml.jackson.core; - -// empty stub; here to satisfy a catch reference in org.datadog.jmxfetch.App -public class JsonProcessingException extends java.io.IOException {} diff --git a/dd-java-agent/agent-jmxfetch/src/main/java/datadog/trace/agent/jmxfetch/JMXFetch.java b/dd-java-agent/agent-jmxfetch/src/main/java/datadog/trace/agent/jmxfetch/JMXFetch.java index d81415a7d68..319f455c85e 100644 --- a/dd-java-agent/agent-jmxfetch/src/main/java/datadog/trace/agent/jmxfetch/JMXFetch.java +++ b/dd-java-agent/agent-jmxfetch/src/main/java/datadog/trace/agent/jmxfetch/JMXFetch.java @@ -9,6 +9,7 @@ import datadog.trace.api.StatsDClient; import datadog.trace.api.StatsDClientManager; import datadog.trace.api.flare.TracerFlare; +import datadog.trace.api.telemetry.LogCollector; import de.thetaphi.forbiddenapis.SuppressForbidden; import java.io.IOException; import java.io.InputStream; @@ -174,6 +175,7 @@ private static List getInternalMetricFiles() { log.debug("metricconfigs not found. returning empty set"); return Collections.emptyList(); } + log.debug("reading found metricconfigs"); Scanner scanner = new Scanner(metricConfigsStream); scanner.useDelimiter("\n"); final List result = new ArrayList<>(); @@ -183,8 +185,19 @@ private static List getInternalMetricFiles() { integrationName.clear(); integrationName.add(config.replace(".yaml", "")); - if (Config.get().isJmxFetchIntegrationEnabled(integrationName, false)) { + if (!Config.get().isJmxFetchIntegrationEnabled(integrationName, false)) { + log.debug( + "skipping metric config `{}` because integration {} is disabled", + config, + integrationName); + } else { final URL resource = JMXFetch.class.getResource("metricconfigs/" + config); + if (resource == null) { + log.debug( + LogCollector.SEND_TELEMETRY, "metric config `{}` not found. skipping", config); + continue; + } + log.debug("adding metric config `{}`", config); // jar!/ means a file internal to a jar, only add the part after if it exists final String path = resource.getPath(); diff --git a/dd-java-agent/agent-tooling/build.gradle b/dd-java-agent/agent-tooling/build.gradle index 467202dc50e..4643e4cfadc 100644 --- a/dd-java-agent/agent-tooling/build.gradle +++ b/dd-java-agent/agent-tooling/build.gradle @@ -41,6 +41,7 @@ dependencies { api(project(':dd-java-agent:agent-bootstrap')) { exclude group: 'com.datadoghq', module: 'agent-logging' } + compileOnly project(':dd-java-agent:agent-jmxfetch') compileOnly project(':dd-java-agent:agent-profiling') api group: 'com.blogspot.mydailyjava', name: 'weak-lock-free', version: '0.17' api libs.bytebuddy diff --git a/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/nativeimage/TracerActivation.java b/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/nativeimage/TracerActivation.java index 51114d9807f..9c5f769ad19 100644 --- a/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/nativeimage/TracerActivation.java +++ b/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/nativeimage/TracerActivation.java @@ -2,8 +2,11 @@ import com.datadog.profiling.controller.openjdk.JFREventContextIntegration; import datadog.communication.ddagent.SharedCommunicationObjects; +import datadog.communication.monitor.DDAgentStatsDClientManager; +import datadog.trace.agent.jmxfetch.JMXFetch; import datadog.trace.agent.tooling.ProfilerInstaller; import datadog.trace.agent.tooling.TracerInstaller; +import datadog.trace.api.StatsDClientManager; import datadog.trace.bootstrap.instrumentation.api.ProfilingContextIntegration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -20,6 +23,9 @@ public static void activate() { withProfiler ? new JFREventContextIntegration() : ProfilingContextIntegration.NoOp.INSTANCE); + + StatsDClientManager statsDClientManager = DDAgentStatsDClientManager.statsDClientManager(); + JMXFetch.run(statsDClientManager); } catch (Throwable e) { log.warn("Problem activating datadog tracer", e); } diff --git a/dd-java-agent/instrumentation/graal/native-image/src/main/java/datadog/trace/instrumentation/graal/nativeimage/AnnotationSubstitutionProcessorInstrumentation.java b/dd-java-agent/instrumentation/graal/native-image/src/main/java/datadog/trace/instrumentation/graal/nativeimage/AnnotationSubstitutionProcessorInstrumentation.java index 616c2bb6e61..fb46599f3f8 100644 --- a/dd-java-agent/instrumentation/graal/native-image/src/main/java/datadog/trace/instrumentation/graal/nativeimage/AnnotationSubstitutionProcessorInstrumentation.java +++ b/dd-java-agent/instrumentation/graal/native-image/src/main/java/datadog/trace/instrumentation/graal/nativeimage/AnnotationSubstitutionProcessorInstrumentation.java @@ -37,7 +37,10 @@ public void methodAdvice(MethodTransformer transformer) { public String[] helperClassNames() { return new String[] { packageName + ".Target_datadog_jctools_counters_FixedSizeStripedLongCounterFields", - packageName + ".Target_datadog_jctools_util_UnsafeRefArrayAccess" + packageName + ".Target_datadog_jctools_util_UnsafeRefArrayAccess", + packageName + ".Target_org_datadog_jmxfetch_App", + packageName + ".Target_org_datadog_jmxfetch_Status", + packageName + ".Target_org_datadog_jmxfetch_reporter_JsonReporter", }; } @@ -49,7 +52,10 @@ public String[] muzzleIgnoredClassNames() { "jdk.vm.ci.meta.ResolvedJavaField", // ignore helper class names as usual packageName + ".Target_datadog_jctools_counters_FixedSizeStripedLongCounterFields", - packageName + ".Target_datadog_jctools_util_UnsafeRefArrayAccess" + packageName + ".Target_datadog_jctools_util_UnsafeRefArrayAccess", + packageName + ".Target_org_datadog_jmxfetch_App", + packageName + ".Target_org_datadog_jmxfetch_Status", + packageName + ".Target_org_datadog_jmxfetch_reporter_JsonReporter", }; } @@ -58,6 +64,9 @@ public static class FindTargetClassesAdvice { public static void onExit(@Advice.Return(readOnly = false) List> result) { result.add(Target_datadog_jctools_counters_FixedSizeStripedLongCounterFields.class); result.add(Target_datadog_jctools_util_UnsafeRefArrayAccess.class); + result.add(Target_org_datadog_jmxfetch_App.class); + result.add(Target_org_datadog_jmxfetch_Status.class); + result.add(Target_org_datadog_jmxfetch_reporter_JsonReporter.class); } } } diff --git a/dd-java-agent/instrumentation/graal/native-image/src/main/java/datadog/trace/instrumentation/graal/nativeimage/ResourcesFeatureInstrumentation.java b/dd-java-agent/instrumentation/graal/native-image/src/main/java/datadog/trace/instrumentation/graal/nativeimage/ResourcesFeatureInstrumentation.java index 8b195962e59..bf546d0b97f 100644 --- a/dd-java-agent/instrumentation/graal/native-image/src/main/java/datadog/trace/instrumentation/graal/nativeimage/ResourcesFeatureInstrumentation.java +++ b/dd-java-agent/instrumentation/graal/native-image/src/main/java/datadog/trace/instrumentation/graal/nativeimage/ResourcesFeatureInstrumentation.java @@ -7,7 +7,11 @@ import com.oracle.svm.core.jdk.Resources; import datadog.trace.agent.tooling.Instrumenter; import datadog.trace.agent.tooling.InstrumenterModule; +import java.io.BufferedReader; import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.List; import net.bytebuddy.asm.Advice; @AutoService(InstrumenterModule.class) @@ -33,19 +37,45 @@ public static void onExit() { // (drop trace/shared prefixes from embedded resources, so we can find them in native-image // as the final executable won't have our isolating class-loader to map these resources) - String[] tracerResources = { - "dd-java-agent.version", - "dd-trace-api.version", - "trace/dd-trace-core.version", - "shared/dogstatsd/version.properties", - "shared/version-utils.version", - "shared/datadog/okhttp3/internal/publicsuffix/publicsuffixes.gz", - "profiling/jfr/dd.jfp", - "profiling/jfr/safepoints.jfp", - "profiling/jfr/overrides/comprehensive.jfp", - "profiling/jfr/overrides/minimal.jfp" - }; + List tracerResources = new ArrayList<>(); + tracerResources.add("dd-java-agent.version"); + tracerResources.add("dd-trace-api.version"); + tracerResources.add("trace/dd-trace-core.version"); + tracerResources.add("shared/dogstatsd/version.properties"); + tracerResources.add("shared/version-utils.version"); + tracerResources.add("shared/datadog/okhttp3/internal/publicsuffix/publicsuffixes.gz"); + tracerResources.add("profiling/jfr/dd.jfp"); + tracerResources.add("profiling/jfr/safepoints.jfp"); + tracerResources.add("profiling/jfr/overrides/comprehensive.jfp"); + tracerResources.add("profiling/jfr/overrides/minimal.jfp"); + // jmxfetch configs + tracerResources.add( + "metrics/project.properties"); // org.datadog.jmxfetch.AppConfig reads its version + tracerResources.add("metrics/org/datadog/jmxfetch/default-jmx-metrics.yaml"); + tracerResources.add("metrics/org/datadog/jmxfetch/new-gc-default-jmx-metrics.yaml"); + tracerResources.add("metrics/org/datadog/jmxfetch/old-gc-default-jmx-metrics.yaml"); + + // tracer's jmxfetch configs + tracerResources.add("metrics/jmxfetch-config.yaml"); + tracerResources.add("metrics/jmxfetch-websphere-config.yaml"); + + // jmxfetch integrations metricconfigs + String metricConfigsPath = "metrics/datadog/trace/agent/jmxfetch/"; + String metricConfigs = metricConfigsPath + "metricconfigs.txt"; + tracerResources.add(metricConfigs); + try (InputStream is = ClassLoader.getSystemResourceAsStream(metricConfigs); + BufferedReader reader = new BufferedReader(new InputStreamReader(is))) { + String metricConfig; + while ((metricConfig = reader.readLine()) != null) { + if (!metricConfig.trim().isEmpty()) { + tracerResources.add(metricConfigsPath + "metricconfigs/" + metricConfig); + } + } + } catch (Throwable ignore) { + } + + // registering tracer resources to include in the native build for (String original : tracerResources) { String flattened = original.substring(original.indexOf('/') + 1); try (InputStream is = ClassLoader.getSystemResourceAsStream(original)) { diff --git a/dd-java-agent/instrumentation/graal/native-image/src/main/java/datadog/trace/instrumentation/graal/nativeimage/Target_org_datadog_jmxfetch_App.java b/dd-java-agent/instrumentation/graal/native-image/src/main/java/datadog/trace/instrumentation/graal/nativeimage/Target_org_datadog_jmxfetch_App.java new file mode 100644 index 00000000000..0106ca1de1b --- /dev/null +++ b/dd-java-agent/instrumentation/graal/native-image/src/main/java/datadog/trace/instrumentation/graal/nativeimage/Target_org_datadog_jmxfetch_App.java @@ -0,0 +1,17 @@ +package datadog.trace.instrumentation.graal.nativeimage; + +import com.oracle.svm.core.annotate.Substitute; +import com.oracle.svm.core.annotate.TargetClass; + +@TargetClass(className = "org.datadog.jmxfetch.App") +public final class Target_org_datadog_jmxfetch_App { + @Substitute + private boolean getJsonConfigs() { + // This method has a reference to the excluded transitive dependency jackson-jr-objects. + // GraalVM Native detects it during the reachability analysis and results in + // "Discovered unresolved method during parsing: + // org.datadog.jmxfetch.App.(org.datadog.jmxfetch.AppConfig)." + // because of the missing classes that belong to the excluded dependencies. + throw new IllegalStateException("Unreachable"); + } +} diff --git a/dd-java-agent/instrumentation/graal/native-image/src/main/java/datadog/trace/instrumentation/graal/nativeimage/Target_org_datadog_jmxfetch_Status.java b/dd-java-agent/instrumentation/graal/native-image/src/main/java/datadog/trace/instrumentation/graal/nativeimage/Target_org_datadog_jmxfetch_Status.java new file mode 100644 index 00000000000..395f3a34983 --- /dev/null +++ b/dd-java-agent/instrumentation/graal/native-image/src/main/java/datadog/trace/instrumentation/graal/nativeimage/Target_org_datadog_jmxfetch_Status.java @@ -0,0 +1,17 @@ +package datadog.trace.instrumentation.graal.nativeimage; + +import com.oracle.svm.core.annotate.Substitute; +import com.oracle.svm.core.annotate.TargetClass; +import java.io.IOException; + +@TargetClass(className = "org.datadog.jmxfetch.Status") +public final class Target_org_datadog_jmxfetch_Status { + @Substitute + private String generateJson() throws IOException { + // This method has a reference to the excluded transitive dependency jackson-jr-objects. + // GraalVM Native detects it during the reachability analysis and results in + // "Discovered unresolved type during parsing: com.fasterxml.jackson.jr.ob.JSON." + // because of the missing classes that belong to the excluded dependencies. + throw new IllegalStateException("Unreachable"); + } +} diff --git a/dd-java-agent/instrumentation/graal/native-image/src/main/java/datadog/trace/instrumentation/graal/nativeimage/Target_org_datadog_jmxfetch_reporter_JsonReporter.java b/dd-java-agent/instrumentation/graal/native-image/src/main/java/datadog/trace/instrumentation/graal/nativeimage/Target_org_datadog_jmxfetch_reporter_JsonReporter.java new file mode 100644 index 00000000000..8ff63ca180f --- /dev/null +++ b/dd-java-agent/instrumentation/graal/native-image/src/main/java/datadog/trace/instrumentation/graal/nativeimage/Target_org_datadog_jmxfetch_reporter_JsonReporter.java @@ -0,0 +1,17 @@ +package datadog.trace.instrumentation.graal.nativeimage; + +import com.oracle.svm.core.annotate.Substitute; +import com.oracle.svm.core.annotate.TargetClass; + +@TargetClass(className = "org.datadog.jmxfetch.reporter.JsonReporter") +public final class Target_org_datadog_jmxfetch_reporter_JsonReporter { + @Substitute + public void doSendServiceCheck( + String serviceCheckName, String status, String message, String[] tags) { + // This method has a reference to the excluded transitive dependency jackson-jr-objects. + // GraalVM Native detects it during the reachability analysis and results in + // "Discovered unresolved type during parsing: com.fasterxml.jackson.jr.ob.JSON." + // because of the missing classes that belong to the excluded dependencies. + throw new IllegalStateException("Unreachable"); + } +} diff --git a/dd-smoke-tests/spring-boot-3.0-native/application/build.gradle b/dd-smoke-tests/spring-boot-3.0-native/application/build.gradle index 868dcf3239e..992fa0eaa6f 100644 --- a/dd-smoke-tests/spring-boot-3.0-native/application/build.gradle +++ b/dd-smoke-tests/spring-boot-3.0-native/application/build.gradle @@ -39,6 +39,7 @@ if (hasProperty('agentPath')) { if (withProfiler && property('profiler') == 'true') { buildArgs.add("-J-Ddd.profiling.enabled=true") } + buildArgs.add("--enable-monitoring=jmxserver") } } } diff --git a/dd-smoke-tests/spring-boot-3.0-native/src/test/groovy/SpringBootNativeInstrumentationTest.groovy b/dd-smoke-tests/spring-boot-3.0-native/src/test/groovy/SpringBootNativeInstrumentationTest.groovy index c5b9edeea24..972419cbbba 100644 --- a/dd-smoke-tests/spring-boot-3.0-native/src/test/groovy/SpringBootNativeInstrumentationTest.groovy +++ b/dd-smoke-tests/spring-boot-3.0-native/src/test/groovy/SpringBootNativeInstrumentationTest.groovy @@ -1,4 +1,5 @@ import datadog.smoketest.AbstractServerSmokeTest +import datadog.trace.agent.test.utils.PortUtils import okhttp3.Request import org.openjdk.jmc.common.item.IItemCollection import org.openjdk.jmc.common.item.ItemFilters @@ -13,6 +14,10 @@ import java.nio.file.Files import java.nio.file.Path import java.nio.file.SimpleFileVisitor import java.nio.file.attribute.BasicFileAttributes +import java.util.concurrent.CompletableFuture +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit +import java.util.concurrent.TimeoutException import java.util.concurrent.atomic.AtomicInteger import java.util.concurrent.locks.LockSupport @@ -21,6 +26,9 @@ class SpringBootNativeInstrumentationTest extends AbstractServerSmokeTest { @TempDir def testJfrDir + @Shared + def statsdPort = PortUtils.randomOpenPort() + @Override ProcessBuilder createProcessBuilder() { String springNativeExecutable = System.getProperty('datadog.smoketest.spring.native.executable') @@ -39,7 +47,10 @@ class SpringBootNativeInstrumentationTest extends AbstractServerSmokeTest { '-Ddd.profiling.upload.period=1', '-Ddd.profiling.start-force-first=true', "-Ddd.profiling.debug.dump_path=${testJfrDir}", - "-Ddd.integration.spring-boot.enabled=true" + "-Ddd.integration.spring-boot.enabled=true", + "-Ddd.trace.debug=true", + "-Ddd.jmxfetch.statsd.port=${statsdPort}", + "-Ddd.jmxfetch.start-delay=0", ]) ProcessBuilder processBuilder = new ProcessBuilder(command) processBuilder.directory(new File(buildDirectory)) @@ -66,8 +77,18 @@ class SpringBootNativeInstrumentationTest extends AbstractServerSmokeTest { super.isErrorLog(log) || log.contains("ClassNotFoundException") } + def setupSpec() { + try { + processTestLogLines { it.contains("JMXFetch config: ") } + } catch (TimeoutException toe) { + throw new AssertionError("'JMXFetch config: ' not found in logs. Make sure it's enabled.", toe) + } + } + def "check native instrumentation"() { setup: + CompletableFuture udpMessage = receiveUdpMessage(statsdPort, 1000) + String url = "http://localhost:${httpPort}/hello" when: @@ -87,6 +108,8 @@ class SpringBootNativeInstrumentationTest extends AbstractServerSmokeTest { LockSupport.parkNanos(1_000_000) } countJfrs() > 0 + + udpMessage.get(1, TimeUnit.SECONDS) contains "service:smoke-test-java-app,version:99,env:smoketest" } int countJfrs() { @@ -115,4 +138,20 @@ class SpringBootNativeInstrumentationTest extends AbstractServerSmokeTest { }) return jfrCount.get() } + + CompletableFuture receiveUdpMessage(int port, int bufferSize) { + def future = new CompletableFuture() + Executors.newSingleThreadExecutor().submit { + try (DatagramSocket socket = new DatagramSocket(port)) { + byte[] buffer = new byte[bufferSize] + DatagramPacket packet = new DatagramPacket(buffer, buffer.length) + socket.receive(packet) + def received = new String(packet.data, 0, packet.length) + future.complete(received) + } catch (Exception e) { + future.completeExceptionally(e) + } + } + return future + } }