From 51343114a4d5cdfaead724f329853dce5ac0ec90 Mon Sep 17 00:00:00 2001 From: gary-huang Date: Thu, 19 Sep 2024 15:45:33 -0400 Subject: [PATCH 01/28] add APIs for llm obs --- dd-trace-api/build.gradle | 4 + .../java/datadog/trace/api/llmobs/LLMObs.java | 60 +++++++ .../datadog/trace/api/llmobs/LLMObsSpan.java | 147 ++++++++++++++++++ .../trace/api/llmobs/noop/NoOpLLMObsSpan.java | 61 ++++++++ .../llmobs/noop/NoOpLLMObsSpanFactory.java | 38 +++++ 5 files changed, 310 insertions(+) create mode 100644 dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java create mode 100644 dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsSpan.java create mode 100644 dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsSpan.java create mode 100644 dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsSpanFactory.java diff --git a/dd-trace-api/build.gradle b/dd-trace-api/build.gradle index c4b00313208..9813c47ce02 100644 --- a/dd-trace-api/build.gradle +++ b/dd-trace-api/build.gradle @@ -32,6 +32,10 @@ excludedClassesCoverage += [ 'datadog.trace.api.profiling.ProfilingScope', 'datadog.trace.api.profiling.ProfilingContext', 'datadog.trace.api.profiling.ProfilingContextAttribute.NoOp', + 'datadog.trace.api.llmobs.LLMObs', + 'datadog.trace.api.llmobs.LLMObsSpan', + 'datadog.trace.api.llmobs.noop.NoOpLLMObsSpan', + 'datadog.trace.api.llmobs.noop.NoOpLLMObsSpanFactory', 'datadog.trace.api.experimental.DataStreamsCheckpointer', 'datadog.trace.api.experimental.DataStreamsCheckpointer.NoOp', 'datadog.trace.api.experimental.DataStreamsContextCarrier', diff --git a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java new file mode 100644 index 00000000000..02eb79a7d79 --- /dev/null +++ b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java @@ -0,0 +1,60 @@ +package datadog.trace.api.llmobs; + +import datadog.trace.api.llmobs.noop.NoOpLLMObsSpanFactory; +import javax.annotation.Nullable; + +public class LLMObs { + private static LLMObsSpanFactory SPAN_FACTORY = NoOpLLMObsSpanFactory.INSTANCE; + + public static LLMObsSpan startLLMSpan( + String spanName, + String modelName, + String modelProvider, + @Nullable String mlApp, + @Nullable String sessionID) { + + return SPAN_FACTORY.startLLMSpan(spanName, modelName, modelProvider, mlApp, sessionID); + } + + public static LLMObsSpan startAgentSpan( + String spanName, @Nullable String mlApp, @Nullable String sessionID) { + + return SPAN_FACTORY.startAgentSpan(spanName, mlApp, sessionID); + } + + public static LLMObsSpan startToolSpan( + String spanName, @Nullable String mlApp, @Nullable String sessionID) { + + return SPAN_FACTORY.startToolSpan(spanName, mlApp, sessionID); + } + + public static LLMObsSpan startTaskSpan( + String spanName, @Nullable String mlApp, @Nullable String sessionID) { + + return SPAN_FACTORY.startTaskSpan(spanName, mlApp, sessionID); + } + + public static LLMObsSpan startWorkflowSpan( + String spanName, @Nullable String mlApp, @Nullable String sessionID) { + + return SPAN_FACTORY.startWorkflowSpan(spanName, mlApp, sessionID); + } + + public interface LLMObsSpanFactory { + LLMObsSpan startLLMSpan( + String spanName, + String modelName, + String modelProvider, + @Nullable String mlApp, + @Nullable String sessionID); + + LLMObsSpan startAgentSpan(String spanName, @Nullable String mlApp, @Nullable String sessionID); + + LLMObsSpan startToolSpan(String spanName, @Nullable String mlApp, @Nullable String sessionID); + + LLMObsSpan startTaskSpan(String spanName, @Nullable String mlApp, @Nullable String sessionID); + + LLMObsSpan startWorkflowSpan( + String spanName, @Nullable String mlApp, @Nullable String sessionID); + } +} diff --git a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsSpan.java b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsSpan.java new file mode 100644 index 00000000000..af5eb204937 --- /dev/null +++ b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsSpan.java @@ -0,0 +1,147 @@ +package datadog.trace.api.llmobs; + +import java.util.List; +import java.util.Map; + +/** This interface represent an individual LLM Obs span. */ +public interface LLMObsSpan { + + /** + * Annotate the span with inputs and outputs + * + * @param inputData The input data of the span in the form of a list, for example a list of input + * messages + * @param outputData The output data of the span in the form of a list, for example a list of + * output messages + */ + void annotateIO(List> inputData, List> outputData); + + /** + * Annotate the span with inputs and outputs + * + * @param inputData The input data of the span in the form of a string + * @param outputData The output data of the span in the form of a string + */ + void annotateIO(String inputData, String outputData); + + /** + * Annotate the span with metadata + * + * @param metadata A map of JSON serializable key-value pairs that contains metadata information + * relevant to the input or output operation described by the span + */ + void setMetadata(Map metadata); + + /** + * Annotate the span with metrics + * + * @param metrics A map of JSON serializable keys and numeric values that users can add as metrics + * relevant to the operation described by the span (input_tokens, output_tokens, total_tokens, + * etc.). + */ + void setMetrics(Map metrics); + + /** + * Annotate the span with a single metric key value pair for the span’s context (number of tokens + * document length, etc). + * + * @param key the name of the metric + * @param value the value of the metric + */ + void setMetric(CharSequence key, int value); + + /** + * Annotate the span with a single metric key value pair for the span’s context (number of tokens + * document length, etc). + * + * @param key the name of the metric + * @param value the value of the metric + */ + void setMetric(CharSequence key, long value); + + /** + * Annotate the span with a single metric key value pair for the span’s context (number of tokens + * document length, etc). + * + * @param key the name of the metric + * @param value the value of the metric + */ + void setMetric(CharSequence key, double value); + + /** + * Annotate the span with tags + * + * @param tags An map of JSON serializable key-value pairs that users can add as tags regarding + * the span’s context (session, environment, system, versioning, etc.). + */ + void setTags(Map tags); + + /** + * Annotate the span with a single tag key value pair as a tag regarding the span’s context + * (session, environment, system, versioning, etc.). + * + * @param key the key of the tag + * @param value the value of the tag + */ + void setTag(String key, String value); + + /** + * Annotate the span with a single tag key value pair as a tag regarding the span’s context + * (session, environment, system, versioning, etc.). + * + * @param key the key of the tag + * @param value the value of the tag + */ + void setTag(String key, boolean value); + + /** + * Annotate the span with a single tag key value pair as a tag regarding the span’s context + * (session, environment, system, versioning, etc.). + * + * @param key the key of the tag + * @param value the value of the tag + */ + void setTag(String key, int value); + + /** + * Annotate the span with a single tag key value pair as a tag regarding the span’s context + * (session, environment, system, versioning, etc.). + * + * @param key the key of the tag + * @param value the value of the tag + */ + void setTag(String key, long value); + + /** + * Annotate the span with a single tag key value pair as a tag regarding the span’s context + * (session, environment, system, versioning, etc.). + * + * @param key the key of the tag + * @param value the value of the tag + */ + void setTag(String key, double value); + + /** + * Annotate the span to indicate that an error occurred + * + * @param error whether an error occurred + */ + void setError(boolean error); + + /** + * Annotate the span with an error message + * + * @param errorMessage the message of the error + */ + void setErrorMessage(String errorMessage); + + /** + * Annotate the span with a throwable + * + * @param throwable the errored throwable + */ + void addThrowable(Throwable throwable); + + /** Finishes (closes) a span */ + void finish(); +} diff --git a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsSpan.java b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsSpan.java new file mode 100644 index 00000000000..f6752dc92fa --- /dev/null +++ b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsSpan.java @@ -0,0 +1,61 @@ +package datadog.trace.api.llmobs.noop; + +import datadog.trace.api.llmobs.LLMObsSpan; +import java.util.List; +import java.util.Map; + +public class NoOpLLMObsSpan implements LLMObsSpan { + public static final LLMObsSpan INSTANCE = new NoOpLLMObsSpan(); + + @Override + public void annotateIO( + List> inputData, List> outputData) {} + + @Override + public void annotateIO(String inputData, String outputData) {} + + @Override + public void setMetadata(Map metadata) {} + + @Override + public void setMetrics(Map metrics) {} + + @Override + public void setMetric(CharSequence key, int value) {} + + @Override + public void setMetric(CharSequence key, long value) {} + + @Override + public void setMetric(CharSequence key, double value) {} + + @Override + public void setTags(Map tags) {} + + @Override + public void setTag(String key, String value) {} + + @Override + public void setTag(String key, boolean value) {} + + @Override + public void setTag(String key, int value) {} + + @Override + public void setTag(String key, long value) {} + + @Override + public void setTag(String key, double value) {} + + @Override + public void setError(boolean error) {} + + @Override + public void setErrorMessage(String errorMessage) {} + + @Override + public void addThrowable(Throwable throwable) {} + + @Override + public void finish() {} +} diff --git a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsSpanFactory.java b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsSpanFactory.java new file mode 100644 index 00000000000..5f0071b1a3e --- /dev/null +++ b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsSpanFactory.java @@ -0,0 +1,38 @@ +package datadog.trace.api.llmobs.noop; + +import datadog.trace.api.llmobs.LLMObs; +import datadog.trace.api.llmobs.LLMObsSpan; +import javax.annotation.Nullable; + +public class NoOpLLMObsSpanFactory implements LLMObs.LLMObsSpanFactory { + public static final NoOpLLMObsSpanFactory INSTANCE = new NoOpLLMObsSpanFactory(); + + public LLMObsSpan startLLMSpan( + String spanName, + String modelName, + String modelProvider, + @Nullable String mlApp, + @Nullable String sessionID) { + return NoOpLLMObsSpan.INSTANCE; + } + + public LLMObsSpan startAgentSpan( + String spanName, @Nullable String mlApp, @Nullable String sessionID) { + return NoOpLLMObsSpan.INSTANCE; + } + + public LLMObsSpan startToolSpan( + String spanName, @Nullable String mlApp, @Nullable String sessionID) { + return NoOpLLMObsSpan.INSTANCE; + } + + public LLMObsSpan startTaskSpan( + String spanName, @Nullable String mlApp, @Nullable String sessionID) { + return NoOpLLMObsSpan.INSTANCE; + } + + public LLMObsSpan startWorkflowSpan( + String spanName, @Nullable String mlApp, @Nullable String sessionID) { + return NoOpLLMObsSpan.INSTANCE; + } +} From b5c7a0a48972e352c4e73019c521ff217afff837 Mon Sep 17 00:00:00 2001 From: gary-huang Date: Thu, 6 Mar 2025 15:25:48 -0500 Subject: [PATCH 02/28] add llm message class to support llm spans --- dd-trace-api/build.gradle | 2 + .../java/datadog/trace/api/llmobs/LLMObs.java | 78 ++++++++++++++++++- .../datadog/trace/api/llmobs/LLMObsSpan.java | 10 +-- .../trace/api/llmobs/noop/NoOpLLMObsSpan.java | 4 +- 4 files changed, 85 insertions(+), 9 deletions(-) diff --git a/dd-trace-api/build.gradle b/dd-trace-api/build.gradle index 9813c47ce02..4175700c6ce 100644 --- a/dd-trace-api/build.gradle +++ b/dd-trace-api/build.gradle @@ -33,6 +33,8 @@ excludedClassesCoverage += [ 'datadog.trace.api.profiling.ProfilingContext', 'datadog.trace.api.profiling.ProfilingContextAttribute.NoOp', 'datadog.trace.api.llmobs.LLMObs', + 'datadog.trace.api.llmobs.LLMObs.LLMMessage', + 'datadog.trace.api.llmobs.LLMObs.ToolCall', 'datadog.trace.api.llmobs.LLMObsSpan', 'datadog.trace.api.llmobs.noop.NoOpLLMObsSpan', 'datadog.trace.api.llmobs.noop.NoOpLLMObsSpanFactory', diff --git a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java index 02eb79a7d79..cd7426a24c5 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java @@ -1,10 +1,14 @@ package datadog.trace.api.llmobs; import datadog.trace.api.llmobs.noop.NoOpLLMObsSpanFactory; +import java.util.List; +import java.util.Map; import javax.annotation.Nullable; public class LLMObs { - private static LLMObsSpanFactory SPAN_FACTORY = NoOpLLMObsSpanFactory.INSTANCE; + protected LLMObs() {} + + protected static LLMObsSpanFactory SPAN_FACTORY = NoOpLLMObsSpanFactory.INSTANCE; public static LLMObsSpan startLLMSpan( String spanName, @@ -57,4 +61,76 @@ LLMObsSpan startLLMSpan( LLMObsSpan startWorkflowSpan( String spanName, @Nullable String mlApp, @Nullable String sessionID); } + + public static class ToolCall { + private String name; + private String type; + private String toolID; + private Map arguments; + + public static ToolCall from( + String name, String type, String toolID, Map arguments) { + return new ToolCall(name, type, toolID, arguments); + } + + private ToolCall(String name, String type, String toolID, Map arguments) { + this.name = name; + this.type = type; + this.toolID = toolID; + this.arguments = arguments; + } + + public String getName() { + return name; + } + + public String getType() { + return type; + } + + public String getToolID() { + return toolID; + } + + public Map getArguments() { + return arguments; + } + } + + public static class LLMMessage { + private String role; + private String content; + private List toolCalls; + + public static LLMMessage from(String role, String content, List toolCalls) { + return new LLMMessage(role, content, toolCalls); + } + + public static LLMMessage from(String role, String content) { + return new LLMMessage(role, content); + } + + private LLMMessage(String role, String content, List toolCalls) { + this.role = role; + this.content = content; + this.toolCalls = toolCalls; + } + + private LLMMessage(String role, String content) { + this.role = role; + this.content = content; + } + + public String getRole() { + return role; + } + + public String getContent() { + return content; + } + + public List getToolCalls() { + return toolCalls; + } + } } diff --git a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsSpan.java b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsSpan.java index af5eb204937..80668eabd57 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsSpan.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsSpan.java @@ -7,14 +7,12 @@ public interface LLMObsSpan { /** - * Annotate the span with inputs and outputs + * Annotate the span with inputs and outputs for LLM spans * - * @param inputData The input data of the span in the form of a list, for example a list of input - * messages - * @param outputData The output data of the span in the form of a list, for example a list of - * output messages + * @param inputMessages The input messages of the span in the form of a list + * @param outputMessages The output messages of the span in the form of a list */ - void annotateIO(List> inputData, List> outputData); + void annotateIO(List inputMessages, List outputMessages); /** * Annotate the span with inputs and outputs diff --git a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsSpan.java b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsSpan.java index f6752dc92fa..a1b160616e7 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsSpan.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsSpan.java @@ -1,5 +1,6 @@ package datadog.trace.api.llmobs.noop; +import datadog.trace.api.llmobs.LLMObs; import datadog.trace.api.llmobs.LLMObsSpan; import java.util.List; import java.util.Map; @@ -8,8 +9,7 @@ public class NoOpLLMObsSpan implements LLMObsSpan { public static final LLMObsSpan INSTANCE = new NoOpLLMObsSpan(); @Override - public void annotateIO( - List> inputData, List> outputData) {} + public void annotateIO(List inputData, List outputData) {} @Override public void annotateIO(String inputData, String outputData) {} From 7f8f5863693afaf8729e977fe4b93ee068ae608d Mon Sep 17 00:00:00 2001 From: gary-huang Date: Thu, 6 Mar 2025 15:25:48 -0500 Subject: [PATCH 03/28] add llm message class to support llm spans --- dd-java-agent/agent-jmxfetch/integrations-core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dd-java-agent/agent-jmxfetch/integrations-core b/dd-java-agent/agent-jmxfetch/integrations-core index 3189af0e0ae..5240f2a7cdc 160000 --- a/dd-java-agent/agent-jmxfetch/integrations-core +++ b/dd-java-agent/agent-jmxfetch/integrations-core @@ -1 +1 @@ -Subproject commit 3189af0e0ae840c9a4bab3131662c7fd6b0de7fb +Subproject commit 5240f2a7cdcabc6ae7787b9191b9189438671f3e From 9206dbe03d26ce551c1e35cab24dafca4fe9d961 Mon Sep 17 00:00:00 2001 From: gary-huang Date: Mon, 27 Jan 2025 14:21:50 -0500 Subject: [PATCH 04/28] impl llmobs agent and llmobs apis --- .../communication/BackendApiFactory.java | 1 + .../java/datadog/trace/bootstrap/Agent.java | 47 ++- dd-java-agent/agent-llmobs/build.gradle | 42 +++ .../datadog/trace/llmobs/LLMObsServices.java | 22 ++ .../datadog/trace/llmobs/LLMObsSystem.java | 100 ++++++ .../trace/llmobs/domain/DDLLMObsSpan.java | 300 ++++++++++++++++++ .../trace/llmobs/domain/LLMObsInternal.java | 10 + .../llmobs/domain/DDLLMObsSpanTest.groovy | 212 +++++++++++++ dd-java-agent/build.gradle | 1 + .../java/datadog/trace/api/DDSpanTypes.java | 3 + .../datadog/trace/api/llmobs/LLMObsTags.java | 15 + .../main/java/datadog/trace/api/Config.java | 28 ++ .../bootstrap/instrumentation/api/Tags.java | 6 + settings.gradle | 3 + 14 files changed, 789 insertions(+), 1 deletion(-) create mode 100644 dd-java-agent/agent-llmobs/build.gradle create mode 100644 dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsServices.java create mode 100644 dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java create mode 100644 dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java create mode 100644 dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/LLMObsInternal.java create mode 100644 dd-java-agent/agent-llmobs/src/test/groovy/datadog/trace/llmobs/domain/DDLLMObsSpanTest.groovy create mode 100644 dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsTags.java diff --git a/communication/src/main/java/datadog/communication/BackendApiFactory.java b/communication/src/main/java/datadog/communication/BackendApiFactory.java index bebb7b42828..f3382792baa 100644 --- a/communication/src/main/java/datadog/communication/BackendApiFactory.java +++ b/communication/src/main/java/datadog/communication/BackendApiFactory.java @@ -72,6 +72,7 @@ private HttpUrl getAgentlessUrl(Intake intake) { public enum Intake { API("api", "v2", Config::isCiVisibilityAgentlessEnabled, Config::getCiVisibilityAgentlessUrl), + LLMOBS_API("api", "v2", Config::isLlmObsAgentlessEnabled, Config::getLlMObsAgentlessUrl), LOGS( "http-intake.logs", "v2", diff --git a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java index ded84aa1176..95b346ac509 100644 --- a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java +++ b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java @@ -25,6 +25,7 @@ import datadog.trace.api.config.GeneralConfig; import datadog.trace.api.config.IastConfig; import datadog.trace.api.config.JmxFetchConfig; +import datadog.trace.api.config.LlmObsConfig; import datadog.trace.api.config.ProfilingConfig; import datadog.trace.api.config.RemoteConfigConfig; import datadog.trace.api.config.TraceInstrumentationConfig; @@ -41,6 +42,7 @@ import datadog.trace.bootstrap.instrumentation.api.AgentTracer; import datadog.trace.bootstrap.instrumentation.api.AgentTracer.TracerAPI; import datadog.trace.bootstrap.instrumentation.api.ProfilingContextIntegration; +import datadog.trace.bootstrap.instrumentation.api.WriterConstants; import datadog.trace.bootstrap.instrumentation.jfr.InstrumentationBasedProfiling; import datadog.trace.util.AgentTaskScheduler; import datadog.trace.util.AgentThreadFactory.AgentThread; @@ -109,7 +111,10 @@ private enum AgentFeature { EXCEPTION_REPLAY(DebuggerConfig.EXCEPTION_REPLAY_ENABLED, false), CODE_ORIGIN(TraceInstrumentationConfig.CODE_ORIGIN_FOR_SPANS_ENABLED, false), DATA_JOBS(GeneralConfig.DATA_JOBS_ENABLED, false), - AGENTLESS_LOG_SUBMISSION(GeneralConfig.AGENTLESS_LOG_SUBMISSION_ENABLED, false); + AGENTLESS_LOG_SUBMISSION(GeneralConfig.AGENTLESS_LOG_SUBMISSION_ENABLED, false), + LLMOBS(propertyNameToSystemPropertyName(LlmObsConfig.LLMOBS_ENABLED), false), + LLMOBS_AGENTLESS( + propertyNameToSystemPropertyName(LlmObsConfig.LLMOBS_AGENTLESS_ENABLED), false); private final String configKey; private final String systemProp; @@ -156,6 +161,8 @@ public boolean isEnabledByDefault() { private static boolean iastFullyDisabled; private static boolean cwsEnabled = false; private static boolean ciVisibilityEnabled = false; + private static boolean llmObsEnabled = false; + private static boolean llmObsAgentlessEnabled = false; private static boolean usmEnabled = false; private static boolean telemetryEnabled = true; private static boolean dynamicInstrumentationEnabled = false; @@ -292,6 +299,25 @@ public static void start( exceptionReplayEnabled = isFeatureEnabled(AgentFeature.EXCEPTION_REPLAY); codeOriginEnabled = isFeatureEnabled(AgentFeature.CODE_ORIGIN); agentlessLogSubmissionEnabled = isFeatureEnabled(AgentFeature.AGENTLESS_LOG_SUBMISSION); + llmObsEnabled = isFeatureEnabled(AgentFeature.LLMOBS); + + // setup writers when llmobs is enabled to accomodate apm and llmobs + if (llmObsEnabled) { + // for llm obs spans, use agent proxy by default, apm spans will use agent writer + setSystemPropertyDefault( + propertyNameToSystemPropertyName(TracerConfig.WRITER_TYPE), + WriterConstants.MULTI_WRITER_TYPE + + ":" + + WriterConstants.DD_INTAKE_WRITER_TYPE + + "," + + WriterConstants.DD_AGENT_WRITER_TYPE); + if (llmObsAgentlessEnabled) { + // use API writer only + setSystemPropertyDefault( + propertyNameToSystemPropertyName(TracerConfig.WRITER_TYPE), + WriterConstants.DD_INTAKE_WRITER_TYPE); + } + } patchJPSAccess(inst); @@ -599,6 +625,7 @@ public void execute() { maybeStartAppSec(scoClass, sco); maybeStartCiVisibility(instrumentation, scoClass, sco); + maybeStartLLMObs(instrumentation, scoClass, sco); // start debugger before remote config to subscribe to it before starting to poll maybeStartDebugger(instrumentation, scoClass, sco); maybeStartRemoteConfig(scoClass, sco); @@ -954,6 +981,24 @@ private static void maybeStartCiVisibility(Instrumentation inst, Class scoCla } } + private static void maybeStartLLMObs(Instrumentation inst, Class scoClass, Object sco) { + if (llmObsEnabled) { + StaticEventLogger.begin("LLM Observability"); + + try { + final Class llmObsSysClass = + AGENT_CLASSLOADER.loadClass("datadog.trace.llmobs.LLMObsSystem"); + final Method llmObsInstallerMethod = + llmObsSysClass.getMethod("start", Instrumentation.class, scoClass); + llmObsInstallerMethod.invoke(null, inst, sco); + } catch (final Throwable e) { + log.warn("Not starting LLM Observability subsystem", e); + } + + StaticEventLogger.end("LLM Observability"); + } + } + private static void maybeInstallLogsIntake(Class scoClass, Object sco) { if (agentlessLogSubmissionEnabled) { StaticEventLogger.begin("Logs Intake"); diff --git a/dd-java-agent/agent-llmobs/build.gradle b/dd-java-agent/agent-llmobs/build.gradle new file mode 100644 index 00000000000..071840c93ec --- /dev/null +++ b/dd-java-agent/agent-llmobs/build.gradle @@ -0,0 +1,42 @@ +buildscript { + repositories { + mavenCentral() + } + + dependencies { + classpath group: 'org.jetbrains.kotlin', name: 'kotlin-gradle-plugin', version: libs.versions.kotlin.get() + } +} + +plugins { + id 'com.github.johnrengelman.shadow' + id 'java-test-fixtures' +} + +apply from: "$rootDir/gradle/java.gradle" +apply from: "$rootDir/gradle/version.gradle" +apply from: "$rootDir/gradle/test-with-kotlin.gradle" + +minimumBranchCoverage = 0.0 +minimumInstructionCoverage = 0.0 + +dependencies { + api libs.slf4j + + implementation project(':communication') + implementation project(':components:json') + implementation project(':internal-api') + + testImplementation project(":utils:test-utils") + + testFixturesApi project(':dd-java-agent:testing') + testFixturesApi project(':utils:test-utils') +} + +shadowJar { + dependencies deps.excludeShared +} + +jar { + archiveClassifier = 'unbundled' +} diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsServices.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsServices.java new file mode 100644 index 00000000000..600ea4d545a --- /dev/null +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsServices.java @@ -0,0 +1,22 @@ +package datadog.trace.llmobs; + +import datadog.communication.BackendApi; +import datadog.communication.BackendApiFactory; +import datadog.communication.ddagent.SharedCommunicationObjects; +import datadog.trace.api.Config; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class LLMObsServices { + + private static final Logger logger = LoggerFactory.getLogger(LLMObsServices.class); + + final Config config; + final BackendApi backendApi; + + LLMObsServices(Config config, SharedCommunicationObjects sco) { + this.config = config; + this.backendApi = + new BackendApiFactory(config, sco).createBackendApi(BackendApiFactory.Intake.LLMOBS_API); + } +} diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java new file mode 100644 index 00000000000..fd9d3f3aca4 --- /dev/null +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java @@ -0,0 +1,100 @@ +package datadog.trace.llmobs; + +import datadog.communication.ddagent.SharedCommunicationObjects; +import datadog.trace.api.Config; +import datadog.trace.api.llmobs.LLMObs; +import datadog.trace.api.llmobs.LLMObsSpan; +import datadog.trace.api.llmobs.LLMObsTags; +import datadog.trace.bootstrap.instrumentation.api.Tags; +import datadog.trace.llmobs.domain.DDLLMObsSpan; +import datadog.trace.llmobs.domain.LLMObsInternal; +import java.lang.instrument.Instrumentation; +import org.jetbrains.annotations.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class LLMObsSystem { + + private static final Logger LOGGER = LoggerFactory.getLogger(LLMObsSystem.class); + + public static void start(Instrumentation inst, SharedCommunicationObjects sco) { + Config config = Config.get(); + if (!config.isLlmObsEnabled()) { + LOGGER.debug("LLM Observability is disabled"); + return; + } + + sco.createRemaining(config); + + LLMObsServices llmObsServices = new LLMObsServices(config, sco); + LLMObsInternal.setLLMObsSpanFactory( + new LLMObsManualSpanFactory( + config.getLlmObsMlApp(), config.getServiceName(), llmObsServices)); + } + + private static class LLMObsManualSpanFactory implements LLMObs.LLMObsSpanFactory { + + private final LLMObsServices llmObsServices; + private final String serviceName; + private final String defaultMLApp; + + public LLMObsManualSpanFactory( + String defaultMLApp, String serviceName, LLMObsServices llmObsServices) { + this.defaultMLApp = defaultMLApp; + this.llmObsServices = llmObsServices; + this.serviceName = serviceName; + } + + @Override + public LLMObsSpan startLLMSpan( + String spanName, + String modelName, + String modelProvider, + @Nullable String mlApp, + @Nullable String sessionID) { + + DDLLMObsSpan span = + new DDLLMObsSpan( + Tags.LLMOBS_LLM_SPAN_KIND, spanName, getMLApp(mlApp), sessionID, serviceName); + + span.setTag(LLMObsTags.MODEL_NAME, modelName); + span.setTag(LLMObsTags.MODEL_PROVIDER, modelProvider); + return span; + } + + @Override + public LLMObsSpan startAgentSpan( + String spanName, @Nullable String mlApp, @Nullable String sessionID) { + return new DDLLMObsSpan( + Tags.LLMOBS_AGENT_SPAN_KIND, spanName, getMLApp(mlApp), sessionID, serviceName); + } + + @Override + public LLMObsSpan startToolSpan( + String spanName, @Nullable String mlApp, @Nullable String sessionID) { + return new DDLLMObsSpan( + Tags.LLMOBS_TOOL_SPAN_KIND, spanName, getMLApp(mlApp), sessionID, serviceName); + } + + @Override + public LLMObsSpan startTaskSpan( + String spanName, @Nullable String mlApp, @Nullable String sessionID) { + return new DDLLMObsSpan( + Tags.LLMOBS_TASK_SPAN_KIND, spanName, getMLApp(mlApp), sessionID, serviceName); + } + + @Override + public LLMObsSpan startWorkflowSpan( + String spanName, @Nullable String mlApp, @Nullable String sessionID) { + return new DDLLMObsSpan( + Tags.LLMOBS_WORKFLOW_SPAN_KIND, spanName, getMLApp(mlApp), sessionID, serviceName); + } + + private String getMLApp(String mlApp) { + if (mlApp == null || mlApp.isEmpty()) { + return defaultMLApp; + } + return mlApp; + } + } +} diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java new file mode 100644 index 00000000000..1d9630e7b2f --- /dev/null +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java @@ -0,0 +1,300 @@ +package datadog.trace.llmobs.domain; + +import datadog.trace.api.DDSpanTypes; +import datadog.trace.api.llmobs.LLMObsSpan; +import datadog.trace.api.llmobs.LLMObsTags; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.AgentTracer; +import datadog.trace.bootstrap.instrumentation.api.Tags; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import javax.annotation.Nonnull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class DDLLMObsSpan implements LLMObsSpan { + + private enum State { + VALID, + INVALID_IO_MESSAGE_KEY + } + + private static final String MESSAGE_KEY_ROLE = "role"; + private static final String MESSAGE_KEY_CONTENT = "content"; + + private static final Set VALID_MESSAGE_KEYS = + new HashSet<>(Arrays.asList(MESSAGE_KEY_ROLE, MESSAGE_KEY_CONTENT)); + + // Well known tags for LLM obs will be prefixed with _ml_obs_(tags|metrics). + // Prefix for tags + private static final String LLMOBS_TAG_PREFIX = "_ml_obs_tag."; + // Prefix for metrics + private static final String LLMOBS_METRIC_PREFIX = "_ml_obs_metric."; + + // internal tags to be prefixed + private static final String INPUT = LLMOBS_TAG_PREFIX + "input"; + private static final String OUTPUT = LLMOBS_TAG_PREFIX + "output"; + private static final String SPAN_KIND = LLMOBS_TAG_PREFIX + Tags.SPAN_KIND; + private static final String METADATA = LLMOBS_TAG_PREFIX + LLMObsTags.METADATA; + + private static final String LLM_OBS_INSTRUMENTATION_NAME = "llmobs"; + + private static final Logger LOGGER = LoggerFactory.getLogger(DDLLMObsSpan.class); + + private final AgentSpan span; + private final String spanKind; + + private boolean finished = false; + + public DDLLMObsSpan( + @Nonnull String kind, + String spanName, + @Nonnull String mlApp, + String sessionID, + @Nonnull String serviceName) { + + if (null == spanName || spanName.isEmpty()) { + spanName = kind; + } + + AgentTracer.SpanBuilder spanBuilder = + AgentTracer.get() + .buildSpan(LLM_OBS_INSTRUMENTATION_NAME, spanName) + .withServiceName(serviceName) + .withSpanType(DDSpanTypes.LLMOBS); + + this.span = spanBuilder.start(); + this.span.setTag(SPAN_KIND, kind); + this.spanKind = kind; + this.span.setTag(LLMOBS_TAG_PREFIX + LLMObsTags.ML_APP, mlApp); + if (sessionID != null && !sessionID.isEmpty()) { + this.span.setTag(LLMOBS_TAG_PREFIX + LLMObsTags.SESSION_ID, sessionID); + } + } + + @Override + public String toString() { + return super.toString() + + ", trace_id=" + + this.span.context().getTraceId() + + ", span_id=" + + this.span.context().getSpanId() + + ", ml_app=" + + this.span.getTag(LLMObsTags.ML_APP) + + ", service=" + + this.span.getServiceName() + + ", span_kind=" + + this.span.getTag(SPAN_KIND); + } + + private static State validateIOMessages(List> messages) { + for (Map message : messages) { + for (String key : message.keySet()) { + if (!VALID_MESSAGE_KEYS.contains(key)) { + return State.INVALID_IO_MESSAGE_KEY; + } + } + } + return State.VALID; + } + + @Override + public void annotateIO( + List> inputData, List> outputData) { + if (finished) { + return; + } + if (inputData != null && !inputData.isEmpty()) { + State inputState = validateIOMessages(inputData); + if (validateIOMessages(inputData) != State.VALID) { + LOGGER.debug("malformed/unexpected input message, state={}", inputState); + } + this.span.setTag(INPUT, inputData); + } + if (outputData != null && !outputData.isEmpty()) { + State outputState = validateIOMessages(outputData); + if (validateIOMessages(outputData) != State.VALID) { + LOGGER.debug("malformed/unexpected output message, state={}", outputState); + } + this.span.setTag(OUTPUT, outputData); + } + } + + @Override + public void annotateIO(String inputData, String outputData) { + if (finished) { + return; + } + if (inputData != null && !inputData.isEmpty()) { + if (Tags.LLMOBS_LLM_SPAN_KIND.equals(this.spanKind)) { + annotateIO( + Collections.singletonList(Collections.singletonMap(MESSAGE_KEY_CONTENT, inputData)), + null); + } else { + this.span.setTag(INPUT, inputData); + } + } + if (outputData != null && !outputData.isEmpty()) { + if (Tags.LLMOBS_LLM_SPAN_KIND.equals(this.spanKind)) { + annotateIO( + null, + Collections.singletonList(Collections.singletonMap(MESSAGE_KEY_CONTENT, outputData))); + } else { + this.span.setTag(OUTPUT, outputData); + } + } + } + + @Override + public void setMetadata(Map metadata) { + if (finished) { + return; + } + Object value = span.getTag(METADATA); + if (value == null) { + this.span.setTag(METADATA, new HashMap<>(metadata)); + return; + } + + if (value instanceof Map) { + ((Map) value).putAll(metadata); + } else { + LOGGER.debug( + "unexpected instance type for metadata {}, overwriting for now", + value.getClass().getName()); + this.span.setTag(METADATA, new HashMap<>(metadata)); + } + } + + @Override + public void setMetrics(Map metrics) { + if (finished) { + return; + } + for (Map.Entry entry : metrics.entrySet()) { + this.span.setMetric(LLMOBS_METRIC_PREFIX + entry.getKey(), entry.getValue().doubleValue()); + } + } + + @Override + public void setMetric(CharSequence key, int value) { + if (finished) { + return; + } + this.span.setMetric(LLMOBS_METRIC_PREFIX + key, value); + } + + @Override + public void setMetric(CharSequence key, long value) { + if (finished) { + return; + } + this.span.setMetric(LLMOBS_METRIC_PREFIX + key, value); + } + + @Override + public void setMetric(CharSequence key, double value) { + if (finished) { + return; + } + this.span.setMetric(LLMOBS_METRIC_PREFIX + key, value); + } + + @Override + public void setTags(Map tags) { + if (finished) { + return; + } + if (tags != null && !tags.isEmpty()) { + for (Map.Entry entry : tags.entrySet()) { + this.span.setTag(LLMOBS_TAG_PREFIX + entry.getKey(), entry.getValue()); + } + } + } + + @Override + public void setTag(String key, String value) { + if (finished) { + return; + } + this.span.setTag(LLMOBS_TAG_PREFIX + key, value); + } + + @Override + public void setTag(String key, boolean value) { + if (finished) { + return; + } + this.span.setTag(LLMOBS_TAG_PREFIX + key, value); + } + + @Override + public void setTag(String key, int value) { + if (finished) { + return; + } + this.span.setTag(LLMOBS_TAG_PREFIX + key, value); + } + + @Override + public void setTag(String key, long value) { + if (finished) { + return; + } + this.span.setTag(LLMOBS_TAG_PREFIX + key, value); + } + + @Override + public void setTag(String key, double value) { + if (finished) { + return; + } + this.span.setTag(LLMOBS_TAG_PREFIX + key, value); + } + + @Override + public void setError(boolean error) { + if (finished) { + return; + } + this.span.setError(error); + } + + @Override + public void setErrorMessage(String errorMessage) { + if (finished) { + return; + } + if (errorMessage == null || errorMessage.isEmpty()) { + return; + } + this.span.setError(true); + this.span.setErrorMessage(errorMessage); + } + + @Override + public void addThrowable(Throwable throwable) { + if (finished) { + return; + } + if (throwable == null) { + return; + } + this.span.setError(true); + this.span.addThrowable(throwable); + } + + @Override + public void finish() { + if (finished) { + return; + } + this.span.finish(); + this.finished = true; + } +} diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/LLMObsInternal.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/LLMObsInternal.java new file mode 100644 index 00000000000..42b0c097e48 --- /dev/null +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/LLMObsInternal.java @@ -0,0 +1,10 @@ +package datadog.trace.llmobs.domain; + +import datadog.trace.api.llmobs.LLMObs; + +public class LLMObsInternal extends LLMObs { + + public static void setLLMObsSpanFactory(final LLMObsSpanFactory factory) { + LLMObs.SPAN_FACTORY = factory; + } +} diff --git a/dd-java-agent/agent-llmobs/src/test/groovy/datadog/trace/llmobs/domain/DDLLMObsSpanTest.groovy b/dd-java-agent/agent-llmobs/src/test/groovy/datadog/trace/llmobs/domain/DDLLMObsSpanTest.groovy new file mode 100644 index 00000000000..f8602c2c7ad --- /dev/null +++ b/dd-java-agent/agent-llmobs/src/test/groovy/datadog/trace/llmobs/domain/DDLLMObsSpanTest.groovy @@ -0,0 +1,212 @@ +package datadog.trace.llmobs.domain + +import datadog.trace.agent.tooling.TracerInstaller +import datadog.trace.api.DDTags +import datadog.trace.api.IdGenerationStrategy +import datadog.trace.api.llmobs.LLMObsSpan +import datadog.trace.api.llmobs.LLMObsTags +import datadog.trace.bootstrap.instrumentation.api.AgentSpan +import datadog.trace.bootstrap.instrumentation.api.AgentTracer +import datadog.trace.bootstrap.instrumentation.api.Tags +import datadog.trace.core.CoreTracer +import datadog.trace.test.util.DDSpecification +import org.apache.groovy.util.Maps +import spock.lang.Shared + +class DDLLMObsSpanTest extends DDSpecification{ + @SuppressWarnings('PropertyName') + @Shared + AgentTracer.TracerAPI TEST_TRACER + + void setupSpec() { + TEST_TRACER = + Spy( + CoreTracer.builder() + .idGenerationStrategy(IdGenerationStrategy.fromName("SEQUENTIAL")) + .build()) + TracerInstaller.forceInstallGlobalTracer(TEST_TRACER) + + TEST_TRACER.startSpan(*_) >> { + def agentSpan = callRealMethod() + agentSpan + } + } + + void cleanupSpec() { + TEST_TRACER?.close() + } + + void setup() { + assert TEST_TRACER.activeSpan() == null: "Span is active before test has started: " + TEST_TRACER.activeSpan() + TEST_TRACER.flush() + } + + void cleanup() { + TEST_TRACER.flush() + } + + // Prefix for tags + private static final String LLMOBS_TAG_PREFIX = "_ml_obs_tag." + // Prefix for metrics + private static final String LLMOBS_METRIC_PREFIX = "_ml_obs_metric." + + // internal tags to be prefixed + private static final String INPUT = LLMOBS_TAG_PREFIX + "input" + private static final String OUTPUT = LLMOBS_TAG_PREFIX + "output" + private static final String METADATA = LLMOBS_TAG_PREFIX + LLMObsTags.METADATA + + + def "test span simple"() { + setup: + def test = givenALLMObsSpan(Tags.LLMOBS_WORKFLOW_SPAN_KIND, "test-span") + + when: + def input = "test input" + def output = "test output" + // initial set + test.annotateIO(input, output) + test.setMetadata(Maps.of("sport", "baseball", "price_data", Maps.of("gpt4", 100))) + test.setMetrics(Maps.of("rank", 1)) + test.setMetric("likelihood", 0.1) + test.setTag("DOMAIN", "north-america") + test.setTags(Maps.of("bulk1", 1, "bulk2", "2")) + def errMsg = "mr brady" + test.setErrorMessage(errMsg) + + then: + def innerSpan = (AgentSpan)test.span + assert Tags.LLMOBS_WORKFLOW_SPAN_KIND.equals(innerSpan.getTag(LLMOBS_TAG_PREFIX + "span.kind")) + + assert null == innerSpan.getTag("input") + assert input.equals(innerSpan.getTag(INPUT)) + assert null == innerSpan.getTag("output") + assert output.equals(innerSpan.getTag(OUTPUT)) + + assert null == innerSpan.getTag("metadata") + def expectedMetadata = Maps.of("sport", "baseball", "price_data", Maps.of("gpt4", 100)) + assert expectedMetadata.equals(innerSpan.getTag(METADATA)) + + assert null == innerSpan.getTag("rank") + def rankMetric = innerSpan.getTag(LLMOBS_METRIC_PREFIX + "rank") + assert rankMetric instanceof Number && 1 == (int)rankMetric + + assert null == innerSpan.getTag("likelihood") + def likelihoodMetric = innerSpan.getTag(LLMOBS_METRIC_PREFIX + "likelihood") + assert likelihoodMetric instanceof Number + assert 0.1 == (double)likelihoodMetric + + assert null == innerSpan.getTag("DOMAIN") + def domain = innerSpan.getTag(LLMOBS_TAG_PREFIX + "DOMAIN") + assert domain instanceof String + assert "north-america".equals((String)domain) + + assert null == innerSpan.getTag("bulk1") + def tagBulk1 = innerSpan.getTag(LLMOBS_TAG_PREFIX + "bulk1") + assert tagBulk1 instanceof Number + assert 1 == ((int)tagBulk1) + + assert null == innerSpan.getTag("bulk2") + def tagBulk2 = innerSpan.getTag(LLMOBS_TAG_PREFIX + "bulk2") + assert tagBulk2 instanceof String + assert "2".equals((String)tagBulk2) + + assert innerSpan.isError() + assert innerSpan.getTag(DDTags.ERROR_MSG) instanceof String + assert errMsg.equals(innerSpan.getTag(DDTags.ERROR_MSG)) + } + + def "test span with overwrites"() { + setup: + def test = givenALLMObsSpan(Tags.LLMOBS_AGENT_SPAN_KIND, "test-span") + + when: + def input = "test input" + // initial set + test.annotateIO(input, "test output") + // this should be a no-op + test.annotateIO("", "") + // this should replace the initial output + def expectedOutput = Arrays.asList(Maps.of("role", "user", "content", "how much is gas")) + test.annotateIO(null, expectedOutput) + + // initial set + test.setMetadata(Maps.of("sport", "baseball", "price_data", Maps.of("gpt4", 100))) + // this should replace baseball with hockey + test.setMetadata(Maps.of("sport", "hockey")) + // this should add a new key + test.setMetadata(Maps.of("temperature", 30)) + + // initial set + test.setMetrics(Maps.of("rank", 1)) + // this should replace the metric + test.setMetric("rank", 10) + + // initial set + test.setTag("DOMAIN", "north-america") + // add and replace + test.setTags(Maps.of("bulk1", 1, "DOMAIN", "europe")) + + def throwableMsg = "false positive" + test.addThrowable(new Throwable(throwableMsg)) + test.setError(false) + + then: + def innerSpan = (AgentSpan)test.span + assert Tags.LLMOBS_AGENT_SPAN_KIND.equals(innerSpan.getTag(LLMOBS_TAG_PREFIX + "span.kind")) + + assert null == innerSpan.getTag("input") + assert input.equals(innerSpan.getTag(INPUT)) + assert null == innerSpan.getTag("output") + assert expectedOutput.equals(innerSpan.getTag(OUTPUT)) + + assert null == innerSpan.getTag("metadata") + def expectedMetadata = Maps.of("sport", "hockey", "price_data", Maps.of("gpt4", 100), "temperature", 30) + assert expectedMetadata.equals(innerSpan.getTag(METADATA)) + + assert null == innerSpan.getTag("rank") + def rankMetric = innerSpan.getTag(LLMOBS_METRIC_PREFIX + "rank") + assert rankMetric instanceof Number && 10 == (int)rankMetric + + assert null == innerSpan.getTag("DOMAIN") + def domain = innerSpan.getTag(LLMOBS_TAG_PREFIX + "DOMAIN") + assert domain instanceof String + assert "europe".equals((String)domain) + + assert null == innerSpan.getTag("bulk1") + def tagBulk1 = innerSpan.getTag(LLMOBS_TAG_PREFIX + "bulk1") + assert tagBulk1 instanceof Number + assert 1 == ((int)tagBulk1) + + assert !innerSpan.isError() + assert innerSpan.getTag(DDTags.ERROR_MSG) instanceof String + assert throwableMsg.equals(innerSpan.getTag(DDTags.ERROR_MSG)) + assert innerSpan.getTag(DDTags.ERROR_STACK) instanceof String + assert ((String)innerSpan.getTag(DDTags.ERROR_STACK)).contains(throwableMsg) + } + + def "test llm span string input formatted to messages"() { + setup: + def test = givenALLMObsSpan(Tags.LLMOBS_LLM_SPAN_KIND, "test-span") + + when: + def input = "test input" + def output = "test output" + // initial set + test.annotateIO(input, output) + + then: + def innerSpan = (AgentSpan)test.span + assert Tags.LLMOBS_LLM_SPAN_KIND.equals(innerSpan.getTag(LLMOBS_TAG_PREFIX + "span.kind")) + + assert null == innerSpan.getTag("input") + def expectedInput = Arrays.asList(Maps.of("content", input)) + assert expectedInput.equals(innerSpan.getTag(INPUT)) + assert null == innerSpan.getTag("output") + def expectedOutput = Arrays.asList(Maps.of("content", output)) + assert expectedOutput.equals(innerSpan.getTag(OUTPUT)) + } + + private LLMObsSpan givenALLMObsSpan(String kind, name){ + new DDLLMObsSpan(kind, name, "test-ml-app", null, "test-svc") + } +} diff --git a/dd-java-agent/build.gradle b/dd-java-agent/build.gradle index d03ea2e440e..b5c3388274c 100644 --- a/dd-java-agent/build.gradle +++ b/dd-java-agent/build.gradle @@ -120,6 +120,7 @@ includeSubprojShadowJar ':dd-java-agent:appsec', 'appsec' includeSubprojShadowJar ':dd-java-agent:agent-iast', 'iast' includeSubprojShadowJar ':dd-java-agent:agent-debugger', 'debugger' includeSubprojShadowJar ':dd-java-agent:agent-ci-visibility', 'ci-visibility' +includeSubprojShadowJar ':dd-java-agent:agent-llmobs', 'llm-obs' includeSubprojShadowJar ':dd-java-agent:agent-logs-intake', 'logs-intake' includeSubprojShadowJar ':dd-java-agent:cws-tls', 'cws-tls' diff --git a/dd-trace-api/src/main/java/datadog/trace/api/DDSpanTypes.java b/dd-trace-api/src/main/java/datadog/trace/api/DDSpanTypes.java index 335dde0effe..4e41df93738 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/DDSpanTypes.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/DDSpanTypes.java @@ -36,8 +36,11 @@ public class DDSpanTypes { public static final String PROTOBUF = "protobuf"; public static final String MULE = "mule"; + public static final String VALKEY = "valkey"; public static final String WEBSOCKET = "websocket"; public static final String SERVERLESS = "serverless"; + + public static final String LLMOBS = "llm"; } diff --git a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsTags.java b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsTags.java new file mode 100644 index 00000000000..afa4f2b241e --- /dev/null +++ b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsTags.java @@ -0,0 +1,15 @@ +package datadog.trace.api.llmobs; + +// Well known tags for llm obs +public class LLMObsTags { + public static final String ML_APP = "ml_app"; + public static final String SESSION_ID = "session_id"; + + // meta + public static final String METADATA = "metadata"; + + // LLM spans related + public static final String MODEL_NAME = "model_name"; + public static final String MODEL_VERSION = "model_version"; + public static final String MODEL_PROVIDER = "model_provider"; +} diff --git a/internal-api/src/main/java/datadog/trace/api/Config.java b/internal-api/src/main/java/datadog/trace/api/Config.java index 69cec835552..5fdc26473f4 100644 --- a/internal-api/src/main/java/datadog/trace/api/Config.java +++ b/internal-api/src/main/java/datadog/trace/api/Config.java @@ -319,6 +319,7 @@ public static String getHostName() { private final int iastDbRowsToTaint; private final boolean llmObsAgentlessEnabled; + private final String llmObsAgentlessUrl; private final String llmObsMlApp; private final boolean ciVisibilityTraceSanitationEnabled; @@ -1454,6 +1455,22 @@ PROFILING_DATADOG_PROFILER_ENABLED, isDatadogProfilerSafeInCurrentEnvironment()) configProvider.getBoolean(LLMOBS_AGENTLESS_ENABLED, DEFAULT_LLM_OBS_AGENTLESS_ENABLED); llmObsMlApp = configProvider.getString(LLMOBS_ML_APP); + final String llmObsAgentlessUrlStr = getFinalLLMObsUrl(); + URI parsedLLMObsUri = null; + if (llmObsAgentlessUrlStr != null && !llmObsAgentlessUrlStr.isEmpty()) { + try { + parsedLLMObsUri = new URL(llmObsAgentlessUrlStr).toURI(); + } catch (MalformedURLException | URISyntaxException ex) { + log.error( + "Cannot parse LLM Observability agentless URL '{}', skipping", llmObsAgentlessUrlStr); + } + } + if (parsedLLMObsUri != null) { + llmObsAgentlessUrl = llmObsAgentlessUrlStr; + } else { + llmObsAgentlessUrl = null; + } + ciVisibilityTraceSanitationEnabled = configProvider.getBoolean(CIVISIBILITY_TRACE_SANITATION_ENABLED, true); @@ -2881,6 +2898,10 @@ public boolean isLlmObsAgentlessEnabled() { return llmObsAgentlessEnabled; } + public String getLlMObsAgentlessUrl() { + return llmObsAgentlessUrl; + } + public String getLlmObsMlApp() { return llmObsMlApp; } @@ -3954,6 +3975,13 @@ public String getFinalProfilingUrl() { } } + public String getFinalLLMObsUrl() { + if (llmObsAgentlessEnabled) { + return "https://llmobs-intake." + site + "/api/v2/llmobs"; + } + return null; + } + public String getFinalCrashTrackingTelemetryUrl() { if (crashTrackingAgentless) { // when agentless crashTracking is turned on we send directly to our intake diff --git a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/Tags.java b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/Tags.java index 78c90b312a8..fd11b2bf565 100644 --- a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/Tags.java +++ b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/Tags.java @@ -152,4 +152,10 @@ public class Tags { public static final String PROPAGATED_TRACE_SOURCE = "_dd.p.ts"; public static final String PROPAGATED_DEBUG = "_dd.p.debug"; + + public static final String LLMOBS_LLM_SPAN_KIND = "llm"; + public static final String LLMOBS_WORKFLOW_SPAN_KIND = "workflow"; + public static final String LLMOBS_TASK_SPAN_KIND = "task"; + public static final String LLMOBS_AGENT_SPAN_KIND = "agent"; + public static final String LLMOBS_TOOL_SPAN_KIND = "tool"; } diff --git a/settings.gradle b/settings.gradle index 615cd539a17..aac4bd02f65 100644 --- a/settings.gradle +++ b/settings.gradle @@ -99,6 +99,9 @@ include ':dd-java-agent:appsec' // ci-visibility include ':dd-java-agent:agent-ci-visibility' +// llm-observability +include ':dd-java-agent:agent-llmobs' + // iast include ':dd-java-agent:agent-iast' From f5efd268f305a8435ece78fcb9e6ad62916d468d Mon Sep 17 00:00:00 2001 From: gary-huang Date: Thu, 6 Mar 2025 15:50:53 -0500 Subject: [PATCH 05/28] support llm messages with tool calls --- .../java/datadog/trace/bootstrap/Agent.java | 5 +- .../trace/llmobs/domain/DDLLMObsSpan.java | 50 ++++----------- .../llmobs/domain/DDLLMObsSpanTest.groovy | 63 +++++++++++++++++-- 3 files changed, 74 insertions(+), 44 deletions(-) diff --git a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java index 95b346ac509..b7bb8e9c849 100644 --- a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java +++ b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java @@ -112,9 +112,8 @@ private enum AgentFeature { CODE_ORIGIN(TraceInstrumentationConfig.CODE_ORIGIN_FOR_SPANS_ENABLED, false), DATA_JOBS(GeneralConfig.DATA_JOBS_ENABLED, false), AGENTLESS_LOG_SUBMISSION(GeneralConfig.AGENTLESS_LOG_SUBMISSION_ENABLED, false), - LLMOBS(propertyNameToSystemPropertyName(LlmObsConfig.LLMOBS_ENABLED), false), - LLMOBS_AGENTLESS( - propertyNameToSystemPropertyName(LlmObsConfig.LLMOBS_AGENTLESS_ENABLED), false); + LLMOBS(LlmObsConfig.LLMOBS_ENABLED, false), + LLMOBS_AGENTLESS(LlmObsConfig.LLMOBS_AGENTLESS_ENABLED, false); private final String configKey; private final String systemProp; diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java index 1d9630e7b2f..640a61c9426 100644 --- a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java @@ -1,34 +1,22 @@ package datadog.trace.llmobs.domain; import datadog.trace.api.DDSpanTypes; +import datadog.trace.api.llmobs.LLMObs; import datadog.trace.api.llmobs.LLMObsSpan; import datadog.trace.api.llmobs.LLMObsTags; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import datadog.trace.bootstrap.instrumentation.api.AgentTracer; import datadog.trace.bootstrap.instrumentation.api.Tags; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Set; import javax.annotation.Nonnull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class DDLLMObsSpan implements LLMObsSpan { - - private enum State { - VALID, - INVALID_IO_MESSAGE_KEY - } - - private static final String MESSAGE_KEY_ROLE = "role"; - private static final String MESSAGE_KEY_CONTENT = "content"; - - private static final Set VALID_MESSAGE_KEYS = - new HashSet<>(Arrays.asList(MESSAGE_KEY_ROLE, MESSAGE_KEY_CONTENT)); + private static final String LLM_MESSAGE_UNKNOWN_ROLE = "unknown"; // Well known tags for LLM obs will be prefixed with _ml_obs_(tags|metrics). // Prefix for tags @@ -92,35 +80,15 @@ public String toString() { + this.span.getTag(SPAN_KIND); } - private static State validateIOMessages(List> messages) { - for (Map message : messages) { - for (String key : message.keySet()) { - if (!VALID_MESSAGE_KEYS.contains(key)) { - return State.INVALID_IO_MESSAGE_KEY; - } - } - } - return State.VALID; - } - @Override - public void annotateIO( - List> inputData, List> outputData) { + public void annotateIO(List inputData, List outputData) { if (finished) { return; } if (inputData != null && !inputData.isEmpty()) { - State inputState = validateIOMessages(inputData); - if (validateIOMessages(inputData) != State.VALID) { - LOGGER.debug("malformed/unexpected input message, state={}", inputState); - } this.span.setTag(INPUT, inputData); } if (outputData != null && !outputData.isEmpty()) { - State outputState = validateIOMessages(outputData); - if (validateIOMessages(outputData) != State.VALID) { - LOGGER.debug("malformed/unexpected output message, state={}", outputState); - } this.span.setTag(OUTPUT, outputData); } } @@ -130,10 +98,12 @@ public void annotateIO(String inputData, String outputData) { if (finished) { return; } + boolean wrongSpanKind = false; if (inputData != null && !inputData.isEmpty()) { if (Tags.LLMOBS_LLM_SPAN_KIND.equals(this.spanKind)) { + wrongSpanKind = true; annotateIO( - Collections.singletonList(Collections.singletonMap(MESSAGE_KEY_CONTENT, inputData)), + Collections.singletonList(LLMObs.LLMMessage.from(LLM_MESSAGE_UNKNOWN_ROLE, inputData)), null); } else { this.span.setTag(INPUT, inputData); @@ -141,13 +111,19 @@ public void annotateIO(String inputData, String outputData) { } if (outputData != null && !outputData.isEmpty()) { if (Tags.LLMOBS_LLM_SPAN_KIND.equals(this.spanKind)) { + wrongSpanKind = true; annotateIO( null, - Collections.singletonList(Collections.singletonMap(MESSAGE_KEY_CONTENT, outputData))); + Collections.singletonList( + LLMObs.LLMMessage.from(LLM_MESSAGE_UNKNOWN_ROLE, outputData))); } else { this.span.setTag(OUTPUT, outputData); } } + if (wrongSpanKind) { + LOGGER.warn( + "the span being annotated is an LLM span, it is recommended to use the overload with List as arguments"); + } } @Override diff --git a/dd-java-agent/agent-llmobs/src/test/groovy/datadog/trace/llmobs/domain/DDLLMObsSpanTest.groovy b/dd-java-agent/agent-llmobs/src/test/groovy/datadog/trace/llmobs/domain/DDLLMObsSpanTest.groovy index f8602c2c7ad..f17a4382ded 100644 --- a/dd-java-agent/agent-llmobs/src/test/groovy/datadog/trace/llmobs/domain/DDLLMObsSpanTest.groovy +++ b/dd-java-agent/agent-llmobs/src/test/groovy/datadog/trace/llmobs/domain/DDLLMObsSpanTest.groovy @@ -3,6 +3,7 @@ package datadog.trace.llmobs.domain import datadog.trace.agent.tooling.TracerInstaller import datadog.trace.api.DDTags import datadog.trace.api.IdGenerationStrategy +import datadog.trace.api.llmobs.LLMObs import datadog.trace.api.llmobs.LLMObsSpan import datadog.trace.api.llmobs.LLMObsTags import datadog.trace.bootstrap.instrumentation.api.AgentSpan @@ -199,11 +200,65 @@ class DDLLMObsSpanTest extends DDSpecification{ assert Tags.LLMOBS_LLM_SPAN_KIND.equals(innerSpan.getTag(LLMOBS_TAG_PREFIX + "span.kind")) assert null == innerSpan.getTag("input") - def expectedInput = Arrays.asList(Maps.of("content", input)) - assert expectedInput.equals(innerSpan.getTag(INPUT)) + def spanInput = innerSpan.getTag(INPUT) + assert spanInput instanceof List + assert ((List)spanInput).size() == 1 + assert spanInput.get(0) instanceof LLMObs.LLMMessage + def expectedInputMsg = LLMObs.LLMMessage.from("unknown", input) + assert expectedInputMsg.getContent().equals(input) + assert expectedInputMsg.getRole().equals("unknown") + assert expectedInputMsg.getToolCalls().equals(null) + assert null == innerSpan.getTag("output") - def expectedOutput = Arrays.asList(Maps.of("content", output)) - assert expectedOutput.equals(innerSpan.getTag(OUTPUT)) + def spanOutput = innerSpan.getTag(OUTPUT) + assert spanOutput instanceof List + assert ((List)spanOutput).size() == 1 + assert spanOutput.get(0) instanceof LLMObs.LLMMessage + def expectedOutputMsg = LLMObs.LLMMessage.from("unknown", output) + assert expectedOutputMsg.getContent().equals(output) + assert expectedOutputMsg.getRole().equals("unknown") + assert expectedOutputMsg.getToolCalls().equals(null) + } + + def "test llm span with messages"() { + setup: + def test = givenALLMObsSpan(Tags.LLMOBS_LLM_SPAN_KIND, "test-span") + + when: + def inputMsg = LLMObs.LLMMessage.from("user", "input") + def outputMsg = LLMObs.LLMMessage.from("assistant", "output", Arrays.asList(LLMObs.ToolCall.from("weather-tool", "function", "6176241000", Maps.of("location", "paris")))) + // initial set + test.annotateIO(Arrays.asList(inputMsg), Arrays.asList(outputMsg)) + + then: + def innerSpan = (AgentSpan)test.span + assert Tags.LLMOBS_LLM_SPAN_KIND.equals(innerSpan.getTag(LLMOBS_TAG_PREFIX + "span.kind")) + + assert null == innerSpan.getTag("input") + def spanInput = innerSpan.getTag(INPUT) + assert spanInput instanceof List + assert ((List)spanInput).size() == 1 + def spanInputMsg = spanInput.get(0) + assert spanInputMsg instanceof LLMObs.LLMMessage + assert spanInputMsg.getContent().equals(inputMsg.getContent()) + assert spanInputMsg.getRole().equals("user") + assert spanInputMsg.getToolCalls().equals(null) + + assert null == innerSpan.getTag("output") + def spanOutput = innerSpan.getTag(OUTPUT) + assert spanOutput instanceof List + assert ((List)spanOutput).size() == 1 + def spanOutputMsg = spanOutput.get(0) + assert spanOutputMsg instanceof LLMObs.LLMMessage + assert spanOutputMsg.getContent().equals(outputMsg.getContent()) + assert spanOutputMsg.getRole().equals("assistant") + assert spanOutputMsg.getToolCalls().size() == 1 + def toolCall = spanOutputMsg.getToolCalls().get(0) + assert toolCall.getName().equals("weather-tool") + assert toolCall.getType().equals("function") + assert toolCall.getToolID().equals("6176241000") + def expectedToolArgs = Maps.of("location", "paris") + assert toolCall.getArguments().equals(expectedToolArgs) } private LLMObsSpan givenALLMObsSpan(String kind, name){ From 76740e92122deab71162705cad720e573ea5ff30 Mon Sep 17 00:00:00 2001 From: gary-huang Date: Wed, 26 Mar 2025 11:27:27 -0400 Subject: [PATCH 06/28] handle default model name and provider --- .../src/main/java/datadog/trace/llmobs/LLMObsSystem.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java index fd9d3f3aca4..43e3fa91c33 100644 --- a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java @@ -17,6 +17,8 @@ public class LLMObsSystem { private static final Logger LOGGER = LoggerFactory.getLogger(LLMObsSystem.class); + private static final String CUSTOM_MODEL_VAL = "custom"; + public static void start(Instrumentation inst, SharedCommunicationObjects sco) { Config config = Config.get(); if (!config.isLlmObsEnabled()) { @@ -57,7 +59,14 @@ public LLMObsSpan startLLMSpan( new DDLLMObsSpan( Tags.LLMOBS_LLM_SPAN_KIND, spanName, getMLApp(mlApp), sessionID, serviceName); + if (modelName == null || modelName.isEmpty()) { + modelName = CUSTOM_MODEL_VAL; + } span.setTag(LLMObsTags.MODEL_NAME, modelName); + + if (modelProvider == null || modelProvider.isEmpty()) { + modelProvider = CUSTOM_MODEL_VAL; + } span.setTag(LLMObsTags.MODEL_PROVIDER, modelProvider); return span; } From 5e66a2f2544c2790fcb8f4d3b2536b7f43c9b7ea Mon Sep 17 00:00:00 2001 From: gary-huang Date: Thu, 10 Apr 2025 21:53:30 -0400 Subject: [PATCH 07/28] rm unneeded file --- .../datadog/trace/llmobs/LLMObsServices.java | 22 ------------------- .../datadog/trace/llmobs/LLMObsSystem.java | 7 ++---- 2 files changed, 2 insertions(+), 27 deletions(-) delete mode 100644 dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsServices.java diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsServices.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsServices.java deleted file mode 100644 index 600ea4d545a..00000000000 --- a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsServices.java +++ /dev/null @@ -1,22 +0,0 @@ -package datadog.trace.llmobs; - -import datadog.communication.BackendApi; -import datadog.communication.BackendApiFactory; -import datadog.communication.ddagent.SharedCommunicationObjects; -import datadog.trace.api.Config; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class LLMObsServices { - - private static final Logger logger = LoggerFactory.getLogger(LLMObsServices.class); - - final Config config; - final BackendApi backendApi; - - LLMObsServices(Config config, SharedCommunicationObjects sco) { - this.config = config; - this.backendApi = - new BackendApiFactory(config, sco).createBackendApi(BackendApiFactory.Intake.LLMOBS_API); - } -} diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java index 43e3fa91c33..f1290d5b166 100644 --- a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java @@ -28,22 +28,19 @@ public static void start(Instrumentation inst, SharedCommunicationObjects sco) { sco.createRemaining(config); - LLMObsServices llmObsServices = new LLMObsServices(config, sco); LLMObsInternal.setLLMObsSpanFactory( new LLMObsManualSpanFactory( - config.getLlmObsMlApp(), config.getServiceName(), llmObsServices)); + config.getLlmObsMlApp(), config.getServiceName())); } private static class LLMObsManualSpanFactory implements LLMObs.LLMObsSpanFactory { - private final LLMObsServices llmObsServices; private final String serviceName; private final String defaultMLApp; public LLMObsManualSpanFactory( - String defaultMLApp, String serviceName, LLMObsServices llmObsServices) { + String defaultMLApp, String serviceName) { this.defaultMLApp = defaultMLApp; - this.llmObsServices = llmObsServices; this.serviceName = serviceName; } From 75290afa50d13ad6a7b2cd448ead5f76b13ede63 Mon Sep 17 00:00:00 2001 From: gary-huang Date: Mon, 27 Jan 2025 14:21:50 -0500 Subject: [PATCH 08/28] impl llmobs agent and llmobs apis From 7b2db7c73b4edf498f9ee2b7d9e78f0799b2c179 Mon Sep 17 00:00:00 2001 From: gary-huang Date: Mon, 27 Jan 2025 14:21:50 -0500 Subject: [PATCH 09/28] impl llmobs agent --- .../main/java/datadog/trace/api/llmobs/LLMObsConstants.java | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsConstants.java diff --git a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsConstants.java b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsConstants.java new file mode 100644 index 00000000000..9cfcbd821cb --- /dev/null +++ b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsConstants.java @@ -0,0 +1,5 @@ +package datadog.trace.api.llmobs; + +public interface LLMObsConstants { + String LLM_OBS_INSTRUMENTATION_NAME = "llmobs"; +} From d76c94d0167d0a79a2808c450cf43cea7398def2 Mon Sep 17 00:00:00 2001 From: gary-huang Date: Fri, 7 Feb 2025 16:37:57 -0500 Subject: [PATCH 10/28] working writer --- .../trace/api/llmobs/LLMObsConstants.java | 5 - dd-trace-core/build.gradle | 1 + .../trace/common/writer/DDAgentWriter.java | 1 + .../trace/common/writer/WriterFactory.java | 24 +- .../common/writer/ddagent/DDAgentApi.java | 1 + .../common/writer/ddintake/DDIntakeApi.java | 1 + .../ddintake/DDIntakeMapperDiscovery.java | 3 + .../writer/ddintake/LLMObsSpanMapper.java | 326 ++++++++++++++++++ .../civisibility/telemetry/tag/Endpoint.java | 3 +- .../datadog/trace/api/intake/TrackType.java | 1 + .../api/InternalSpanTypes.java | 2 + 11 files changed, 359 insertions(+), 9 deletions(-) delete mode 100644 dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsConstants.java create mode 100644 dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java diff --git a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsConstants.java b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsConstants.java deleted file mode 100644 index 9cfcbd821cb..00000000000 --- a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsConstants.java +++ /dev/null @@ -1,5 +0,0 @@ -package datadog.trace.api.llmobs; - -public interface LLMObsConstants { - String LLM_OBS_INSTRUMENTATION_NAME = "llmobs"; -} diff --git a/dd-trace-core/build.gradle b/dd-trace-core/build.gradle index cfc50ded09b..7b49cfe102c 100644 --- a/dd-trace-core/build.gradle +++ b/dd-trace-core/build.gradle @@ -68,6 +68,7 @@ dependencies { implementation project(':components:json') implementation project(':utils:container-utils') implementation project(':utils:socket-utils') + // for span exception debugging compileOnly project(':dd-java-agent:agent-debugger:debugger-bootstrap') diff --git a/dd-trace-core/src/main/java/datadog/trace/common/writer/DDAgentWriter.java b/dd-trace-core/src/main/java/datadog/trace/common/writer/DDAgentWriter.java index 50b9aba2cdb..dfd7c3fbf1d 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/writer/DDAgentWriter.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/writer/DDAgentWriter.java @@ -151,6 +151,7 @@ public DDAgentWriter build() { } final DDAgentMapperDiscovery mapperDiscovery = new DDAgentMapperDiscovery(featureDiscovery); + final PayloadDispatcher dispatcher = new PayloadDispatcherImpl(mapperDiscovery, agentApi, healthMetrics, monitoring); final TraceProcessingWorker traceProcessingWorker = diff --git a/dd-trace-core/src/main/java/datadog/trace/common/writer/WriterFactory.java b/dd-trace-core/src/main/java/datadog/trace/common/writer/WriterFactory.java index b4af03b90e7..6970ead1d83 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/writer/WriterFactory.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/writer/WriterFactory.java @@ -82,7 +82,7 @@ public static Writer createWriter( // The AgentWriter doesn't support the CI Visibility protocol. If CI Visibility is // enabled, check if we can use the IntakeWriter instead. - if (DD_AGENT_WRITER_TYPE.equals(configuredType) && config.isCiVisibilityEnabled()) { + if (DD_AGENT_WRITER_TYPE.equals(configuredType) && (config.isCiVisibilityEnabled())) { if (featuresDiscovery.supportsEvpProxy() || config.isCiVisibilityAgentlessEnabled()) { configuredType = DD_INTAKE_WRITER_TYPE; } else { @@ -116,6 +116,10 @@ public static Writer createWriter( builder.addTrack(TrackType.CITESTCOV, coverageApi); } + final RemoteApi llmobsApi = + createDDIntakeRemoteApi(config, commObjects, featuresDiscovery, TrackType.LLMOBS); + builder.addTrack(TrackType.LLMOBS, llmobsApi); + remoteWriter = builder.build(); } else { // configuredType == DDAgentWriter @@ -171,7 +175,14 @@ private static RemoteApi createDDIntakeRemoteApi( SharedCommunicationObjects commObjects, DDAgentFeaturesDiscovery featuresDiscovery, TrackType trackType) { - if (featuresDiscovery.supportsEvpProxy() && !config.isCiVisibilityAgentlessEnabled()) { + boolean evpProxySupported = featuresDiscovery.supportsEvpProxy(); + boolean useProxyApi = + (evpProxySupported && TrackType.LLMOBS == trackType && !config.isLlmObsAgentlessEnabled()) + || (evpProxySupported + && (TrackType.CITESTCOV == trackType || TrackType.CITESTCYCLE == trackType) + && !config.isCiVisibilityAgentlessEnabled()); + + if (useProxyApi) { return DDEvpProxyApi.builder() .httpClient(commObjects.okHttpClient) .agentUrl(commObjects.agentUrl) @@ -179,12 +190,19 @@ private static RemoteApi createDDIntakeRemoteApi( .trackType(trackType) .compressionEnabled(featuresDiscovery.supportsContentEncodingHeadersWithEvpProxy()) .build(); - } else { HttpUrl hostUrl = null; + String llmObsAgentlessUrl = config.getLlMObsAgentlessUrl(); + if (config.getCiVisibilityAgentlessUrl() != null) { hostUrl = HttpUrl.get(config.getCiVisibilityAgentlessUrl()); log.info("Using host URL '{}' to report CI Visibility traces in Agentless mode.", hostUrl); + } else if (config.isLlmObsEnabled() + && config.isLlmObsAgentlessEnabled() + && llmObsAgentlessUrl != null + && !llmObsAgentlessUrl.isEmpty()) { + hostUrl = HttpUrl.get(llmObsAgentlessUrl); + log.info("Using host URL '{}' to report LLM Obs traces in Agentless mode.", hostUrl); } return DDIntakeApi.builder() .hostUrl(hostUrl) diff --git a/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/DDAgentApi.java b/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/DDAgentApi.java index 645bbc4b9e9..ccdc6ccda09 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/DDAgentApi.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/DDAgentApi.java @@ -90,6 +90,7 @@ public void addResponseListener(final RemoteResponseListener listener) { public Response sendSerializedTraces(final Payload payload) { final int sizeInBytes = payload.sizeInBytes(); String tracesEndpoint = featuresDiscovery.getTraceEndpoint(); + if (null == tracesEndpoint) { featuresDiscovery.discoverIfOutdated(); tracesEndpoint = featuresDiscovery.getTraceEndpoint(); diff --git a/dd-trace-core/src/main/java/datadog/trace/common/writer/ddintake/DDIntakeApi.java b/dd-trace-core/src/main/java/datadog/trace/common/writer/ddintake/DDIntakeApi.java index 7abad42d2f1..954fc8e1cf7 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/writer/ddintake/DDIntakeApi.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/writer/ddintake/DDIntakeApi.java @@ -131,6 +131,7 @@ public Response sendSerializedTraces(Payload payload) { .post(payload.toRequest()) .tag(OkHttpUtils.CustomListener.class, telemetryListener) .build(); + totalTraces += payload.traceCount(); receivedTraces += payload.traceCount(); diff --git a/dd-trace-core/src/main/java/datadog/trace/common/writer/ddintake/DDIntakeMapperDiscovery.java b/dd-trace-core/src/main/java/datadog/trace/common/writer/ddintake/DDIntakeMapperDiscovery.java index 6cc38f8a3a6..5123e6fe06e 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/writer/ddintake/DDIntakeMapperDiscovery.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/writer/ddintake/DDIntakeMapperDiscovery.java @@ -6,6 +6,7 @@ import datadog.trace.civisibility.writer.ddintake.CiTestCycleMapperV1; import datadog.trace.common.writer.RemoteMapper; import datadog.trace.common.writer.RemoteMapperDiscovery; +import datadog.trace.llmobs.writer.ddintake.LLMObsSpanMapper; /** * Mapper discovery logic when a DDIntake is used. The mapper is discovered based on a backend @@ -40,6 +41,8 @@ public void discover() { mapper = new CiTestCycleMapperV1(wellKnownTags, compressionEnabled); } else if (TrackType.CITESTCOV.equals(trackType)) { mapper = new CiTestCovMapperV2(compressionEnabled); + } else if (TrackType.LLMOBS.equals(trackType)) { + mapper = new LLMObsSpanMapper(); } else { mapper = RemoteMapper.NO_OP; } diff --git a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java new file mode 100644 index 00000000000..97e42355d45 --- /dev/null +++ b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java @@ -0,0 +1,326 @@ +package datadog.trace.llmobs.writer.ddintake; + +import static datadog.communication.http.OkHttpUtils.gzippedMsgpackRequestBodyOf; + +import datadog.communication.serialization.Writable; +import datadog.trace.api.DDTags; +import datadog.trace.api.intake.TrackType; +import datadog.trace.api.llmobs.LLMObsTags; +import datadog.trace.bootstrap.instrumentation.api.InternalSpanTypes; +import datadog.trace.bootstrap.instrumentation.api.Tags; +import datadog.trace.common.writer.Payload; +import datadog.trace.common.writer.RemoteMapper; +import datadog.trace.core.CoreSpan; +import datadog.trace.core.Metadata; +import datadog.trace.core.MetadataConsumer; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.WritableByteChannel; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import okhttp3.RequestBody; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class LLMObsSpanMapper implements RemoteMapper { + + // Well known tags for LLM obs will be prefixed with _ml_obs_(tags|metrics). + // Prefix for tags + private static final String LLMOBS_TAG_PREFIX = "_ml_obs_tag."; + // Prefix for metrics + private static final String LLMOBS_METRIC_PREFIX = "_ml_obs_metric."; + + // internal tags to be prefixed + private static final String INPUT = "input"; + private static final String OUTPUT = "output"; + private static final String SPAN_KIND_TAG_KEY = LLMOBS_TAG_PREFIX + Tags.SPAN_KIND; + + private static final Logger LOGGER = LoggerFactory.getLogger(LLMObsSpanMapper.class); + + private static final byte[] STAGE = "_dd.stage".getBytes(StandardCharsets.UTF_8); + private static final byte[] EVENT_TYPE = "event_type".getBytes(StandardCharsets.UTF_8); + + private static final byte[] SPAN_ID = "span_id".getBytes(StandardCharsets.UTF_8); + private static final byte[] TRACE_ID = "trace_id".getBytes(StandardCharsets.UTF_8); + private static final byte[] PARENT_ID = "parent_id".getBytes(StandardCharsets.UTF_8); + private static final byte[] NAME = "name".getBytes(StandardCharsets.UTF_8); + private static final byte[] DURATION = "duration".getBytes(StandardCharsets.UTF_8); + private static final byte[] START_NS = "start_ns".getBytes(StandardCharsets.UTF_8); + private static final byte[] STATUS = "status".getBytes(StandardCharsets.UTF_8); + private static final byte[] ERROR = "error".getBytes(StandardCharsets.UTF_8); + + private static final byte[] META = "meta".getBytes(StandardCharsets.UTF_8); + private static final byte[] METADATA = "metadata".getBytes(StandardCharsets.UTF_8); + private static final byte[] SPAN_KIND = "span.kind".getBytes(StandardCharsets.UTF_8); + private static final byte[] SPANS = "spans".getBytes(StandardCharsets.UTF_8); + private static final byte[] METRICS = "metrics".getBytes(StandardCharsets.UTF_8); + private static final byte[] TAGS = "tags".getBytes(StandardCharsets.UTF_8); + + private final LLMObsSpanMapper.MetaWriter metaWriter = new MetaWriter(); + private final int size; + + public LLMObsSpanMapper() { + this(5 << 20); + } + + private LLMObsSpanMapper(int size) { + this.size = size; + } + + @Override + public void map(List> trace, Writable writable) { + List> llmobsSpans = + trace.stream().filter(LLMObsSpanMapper::isLLMObsSpan).collect(Collectors.toList()); + + writable.startMap(3); + + writable.writeUTF8(EVENT_TYPE); + writable.writeString("span", null); + + writable.writeUTF8(STAGE); + writable.writeString("raw", null); + + writable.writeUTF8(SPANS); + writable.startArray(llmobsSpans.size()); + for (CoreSpan span : llmobsSpans) { + writable.startMap(11); + // 1 + writable.writeUTF8(SPAN_ID); + writable.writeString(String.valueOf(span.getSpanId()), null); + + // 2 + writable.writeUTF8(TRACE_ID); + writable.writeString(span.getTraceId().toHexString(), null); + + // 3 + writable.writeUTF8(PARENT_ID); + // TODO fix after parent ID tracking is in place + writable.writeString("undefined", null); + + // 4 + writable.writeUTF8(NAME); + writable.writeString(span.getOperationName(), null); + + // 5 + writable.writeUTF8(START_NS); + writable.writeUnsignedLong(span.getStartTime()); + + // 6 + writable.writeUTF8(DURATION); + writable.writeFloat(span.getDurationNano()); + + // 7 + writable.writeUTF8(ERROR); + writable.writeInt(span.getError()); + + boolean errored = span.getError() == 1; + + // 8 + writable.writeUTF8(STATUS); + writable.writeString(errored ? "error" : "ok", null); + + /* 9 (metrics), 10 (tags), 11 meta */ + span.processTagsAndBaggage(metaWriter.withWritable(writable, getErrorsMap(span))); + } + } + + private static boolean isLLMObsSpan(CoreSpan span) { + CharSequence type = span.getType(); + return type != null && type.toString().contentEquals(InternalSpanTypes.LLMOBS); + } + + @Override + public Payload newPayload() { + return new PayloadV1(); + } + + @Override + public int messageBufferSize() { + return size; + } + + @Override + public void reset() {} + + @Override + public String endpoint() { + return TrackType.LLMOBS + "/v2"; + } + + private static Map getErrorsMap(CoreSpan span) { + Map errors = new HashMap<>(); + String errorMsg = span.getTag(DDTags.ERROR_MSG); + if (errorMsg != null && !errorMsg.isEmpty()) { + errors.put(DDTags.ERROR_MSG, errorMsg); + } + String errorType = span.getTag(DDTags.ERROR_TYPE); + if (errorType != null && !errorType.isEmpty()) { + errors.put(DDTags.ERROR_TYPE, errorType); + } + String errorStack = span.getTag(DDTags.ERROR_STACK); + if (errorStack != null && !errorStack.isEmpty()) { + errors.put(DDTags.ERROR_STACK, errorStack); + } + return errors; + } + + private static final class MetaWriter implements MetadataConsumer { + + private Writable writable; + private Map errorInfo; + + private static final Set TAGS_FOR_REMAPPING = + Collections.unmodifiableSet( + new HashSet<>( + Arrays.asList( + LLMOBS_TAG_PREFIX + INPUT, + LLMOBS_TAG_PREFIX + OUTPUT, + LLMOBS_TAG_PREFIX + LLMObsTags.MODEL_NAME, + LLMOBS_TAG_PREFIX + LLMObsTags.MODEL_PROVIDER, + LLMOBS_TAG_PREFIX + LLMObsTags.MODEL_VERSION, + LLMOBS_TAG_PREFIX + LLMObsTags.METADATA))); + + LLMObsSpanMapper.MetaWriter withWritable(Writable writable, Map errorInfo) { + this.writable = writable; + this.errorInfo = errorInfo; + return this; + } + + @Override + public void accept(Metadata metadata) { + Map tagsToRemapToMeta = new HashMap<>(); + int metricsSize = 0, tagsSize = 0; + String spanKind = "unknown"; + for (Map.Entry tag : metadata.getTags().entrySet()) { + String key = tag.getKey(); + if (key.equals(SPAN_KIND_TAG_KEY)) { + spanKind = String.valueOf(tag.getValue()); + } else if (TAGS_FOR_REMAPPING.contains(key)) { + tagsToRemapToMeta.put(key, tag.getValue()); + } else if (key.startsWith(LLMOBS_METRIC_PREFIX) && tag.getValue() instanceof Number) { + ++metricsSize; + } else if (key.startsWith(LLMOBS_TAG_PREFIX)) { + if (key.startsWith(LLMOBS_TAG_PREFIX)) { + key = key.substring(LLMOBS_TAG_PREFIX.length()); + } + if (TAGS_FOR_REMAPPING.contains(key)) { + tagsToRemapToMeta.put(key, tag.getValue()); + } else { + ++tagsSize; + } + } + } + + if (!spanKind.equals("unknown")) { + metadata.getTags().remove(SPAN_KIND_TAG_KEY); + } else { + LOGGER.warn("missing span kind"); + } + + // write metrics (9) + writable.writeUTF8(METRICS); + writable.startMap(metricsSize); + for (Map.Entry tag : metadata.getTags().entrySet()) { + String tagKey = tag.getKey(); + if (tagKey.startsWith(LLMOBS_METRIC_PREFIX) && tag.getValue() instanceof Number) { + writable.writeString(tagKey.substring(LLMOBS_METRIC_PREFIX.length()), null); + writable.writeDouble((double) tag.getValue()); + } + } + + // write tags (10) + writable.writeUTF8(TAGS); + writable.startArray(tagsSize + 1); + writable.writeString("language:jvm", null); + for (Map.Entry tag : metadata.getTags().entrySet()) { + String key = tag.getKey(); + Object value = tag.getValue(); + if (!tagsToRemapToMeta.containsKey(key) + && key.startsWith(LLMOBS_TAG_PREFIX)) { + writable.writeObject(key.substring(LLMOBS_TAG_PREFIX.length()) + ":" + value, null); + } + } + + // write meta (11) + int metaSize = tagsToRemapToMeta.size() + 1 + (null != errorInfo ? errorInfo.size() : 0); + writable.writeUTF8(META); + writable.startMap(metaSize); + writable.writeUTF8(SPAN_KIND); + writable.writeString(spanKind, null); + + for (Map.Entry error : errorInfo.entrySet()) { + writable.writeUTF8(error.getKey().getBytes()); + writable.writeString(error.getValue(), null); + } + + for (Map.Entry tag : tagsToRemapToMeta.entrySet()) { + String key = tag.getKey().substring(LLMOBS_TAG_PREFIX.length()); + Object val = tag.getValue(); + if (key.equals(INPUT) || key.equals(OUTPUT)) { + if (!spanKind.equals(Tags.LLMOBS_LLM_SPAN_KIND)) { + key += ".value"; + } else { + key += ".messages"; + } + } else if (key.equals(LLMObsTags.METADATA) && val instanceof Map) { + Map metadataMap = (Map) val; + writable.writeUTF8(METADATA); + writable.startMap(metadataMap.size()); + for (Map.Entry entry : metadataMap.entrySet()) { + writable.writeString(entry.getKey(), null); + writable.writeObject(entry.getValue(), null); + } + continue; + } + writable.writeString(key, null); + writable.writeObject(val, null); + } + } + } + + private static class PayloadV1 extends Payload { + + @Override + public int sizeInBytes() { + if (traceCount() == 0) { + return msgpackMapHeaderSize(0); + } + + return body.array().length; + } + + @Override + public void writeTo(WritableByteChannel channel) throws IOException { + // If traceCount is 0, we write a map with 0 elements in MsgPack format. + if (traceCount() == 0) { + ByteBuffer emptyDict = msgpackMapHeader(0); + while (emptyDict.hasRemaining()) { + channel.write(emptyDict); + } + } else { + while (body.hasRemaining()) { + channel.write(body); + } + } + } + + @Override + public RequestBody toRequest() { + List buffers; + if (traceCount() == 0) { + buffers = Collections.singletonList(msgpackMapHeader(0)); + } else { + buffers = Collections.singletonList(body); + } + + return gzippedMsgpackRequestBodyOf(buffers); + } + } +} diff --git a/internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/tag/Endpoint.java b/internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/tag/Endpoint.java index 6c2d9b6b25c..529644493b2 100644 --- a/internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/tag/Endpoint.java +++ b/internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/tag/Endpoint.java @@ -5,7 +5,8 @@ /** The type of endpoint where a request is sent */ public enum Endpoint implements TagValue { TEST_CYCLE, - CODE_COVERAGE; + CODE_COVERAGE, + LLMOBS; // TODO this is probably not right, need to probably move this enum to a common package? private final String s; diff --git a/internal-api/src/main/java/datadog/trace/api/intake/TrackType.java b/internal-api/src/main/java/datadog/trace/api/intake/TrackType.java index 4d99f0655a8..e50f46b6f13 100644 --- a/internal-api/src/main/java/datadog/trace/api/intake/TrackType.java +++ b/internal-api/src/main/java/datadog/trace/api/intake/TrackType.java @@ -6,6 +6,7 @@ public enum TrackType { CITESTCYCLE(Endpoint.TEST_CYCLE), CITESTCOV(Endpoint.CODE_COVERAGE), + LLMOBS(Endpoint.LLMOBS), NOOP(null); @Nullable public final Endpoint endpoint; diff --git a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/InternalSpanTypes.java b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/InternalSpanTypes.java index c67cfe6b6e4..6682a98f8b5 100644 --- a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/InternalSpanTypes.java +++ b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/InternalSpanTypes.java @@ -46,6 +46,8 @@ public class InternalSpanTypes { UTF8BytesString.create(DDSpanTypes.VULNERABILITY); public static final UTF8BytesString PROTOBUF = UTF8BytesString.create(DDSpanTypes.PROTOBUF); + public static final UTF8BytesString LLMOBS = UTF8BytesString.create(DDSpanTypes.LLMOBS); + public static final UTF8BytesString TIBCO_BW = UTF8BytesString.create("tibco_bw"); public static final UTF8BytesString MULE = UTF8BytesString.create(DDSpanTypes.MULE); public static final CharSequence VALKEY = UTF8BytesString.create(DDSpanTypes.VALKEY); From e0b4510bc6f73b7964bb62b7ef2d1be76580c113 Mon Sep 17 00:00:00 2001 From: gary-huang Date: Thu, 6 Mar 2025 16:39:04 -0500 Subject: [PATCH 11/28] add support for llm message and tool calls --- .../writer/ddintake/LLMObsSpanMapper.java | 65 +++++++++++++++++-- 1 file changed, 60 insertions(+), 5 deletions(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java index 97e42355d45..584b26902af 100644 --- a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java +++ b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java @@ -5,6 +5,7 @@ import datadog.communication.serialization.Writable; import datadog.trace.api.DDTags; import datadog.trace.api.intake.TrackType; +import datadog.trace.api.llmobs.LLMObs; import datadog.trace.api.llmobs.LLMObsTags; import datadog.trace.bootstrap.instrumentation.api.InternalSpanTypes; import datadog.trace.bootstrap.instrumentation.api.Tags; @@ -63,6 +64,17 @@ public class LLMObsSpanMapper implements RemoteMapper { private static final byte[] METRICS = "metrics".getBytes(StandardCharsets.UTF_8); private static final byte[] TAGS = "tags".getBytes(StandardCharsets.UTF_8); + private static final byte[] LLM_MESSAGE_ROLE = "role".getBytes(StandardCharsets.UTF_8); + private static final byte[] LLM_MESSAGE_CONTENT = "content".getBytes(StandardCharsets.UTF_8); + private static final byte[] LLM_MESSAGE_TOOL_CALLS = + "tool_calls".getBytes(StandardCharsets.UTF_8); + + private static final byte[] LLM_TOOL_CALL_NAME = "name".getBytes(StandardCharsets.UTF_8); + private static final byte[] LLM_TOOL_CALL_TYPE = "type".getBytes(StandardCharsets.UTF_8); + private static final byte[] LLM_TOOL_CALL_TOOL_ID = "tool_id".getBytes(StandardCharsets.UTF_8); + private static final byte[] LLM_TOOL_CALL_ARGUMENTS = + "arguments".getBytes(StandardCharsets.UTF_8); + private final LLMObsSpanMapper.MetaWriter metaWriter = new MetaWriter(); private final int size; @@ -242,8 +254,7 @@ public void accept(Metadata metadata) { for (Map.Entry tag : metadata.getTags().entrySet()) { String key = tag.getKey(); Object value = tag.getValue(); - if (!tagsToRemapToMeta.containsKey(key) - && key.startsWith(LLMOBS_TAG_PREFIX)) { + if (!tagsToRemapToMeta.containsKey(key) && key.startsWith(LLMOBS_TAG_PREFIX)) { writable.writeObject(key.substring(LLMOBS_TAG_PREFIX.length()) + ":" + value, null); } } @@ -266,8 +277,52 @@ public void accept(Metadata metadata) { if (key.equals(INPUT) || key.equals(OUTPUT)) { if (!spanKind.equals(Tags.LLMOBS_LLM_SPAN_KIND)) { key += ".value"; + writable.writeString(key, null); + writable.writeObject(val, null); } else { + if (!(val instanceof List)) { + LOGGER.warn( + "unexpectedly found incorrect type for LLM span IO {}, expecting list", + val.getClass().getName()); + continue; + } + // llm span kind must have llm objects + List messages = (List) val; key += ".messages"; + writable.writeString(key, null); + writable.startArray(messages.size()); + for (LLMObs.LLMMessage message : messages) { + List toolCalls = message.getToolCalls(); + boolean hasToolCalls = null != toolCalls && !toolCalls.isEmpty(); + writable.startMap(hasToolCalls ? 3 : 2); + writable.writeUTF8(LLM_MESSAGE_ROLE); + writable.writeString(message.getRole(), null); + writable.writeUTF8(LLM_MESSAGE_CONTENT); + writable.writeString(message.getContent(), null); + if (hasToolCalls) { + writable.writeUTF8(LLM_MESSAGE_TOOL_CALLS); + writable.startArray(toolCalls.size()); + for (LLMObs.ToolCall toolCall : toolCalls) { + Map arguments = toolCall.getArguments(); + boolean hasArguments = null != arguments && !arguments.isEmpty(); + writable.startMap(hasArguments ? 4 : 3); + writable.writeUTF8(LLM_TOOL_CALL_NAME); + writable.writeString(toolCall.getName(), null); + writable.writeUTF8(LLM_TOOL_CALL_TYPE); + writable.writeString(toolCall.getType(), null); + writable.writeUTF8(LLM_TOOL_CALL_TOOL_ID); + writable.writeString(toolCall.getToolID(), null); + if (hasArguments) { + writable.writeUTF8(LLM_TOOL_CALL_ARGUMENTS); + writable.startMap(arguments.size()); + for (Map.Entry argument : arguments.entrySet()) { + writable.writeString(argument.getKey(), null); + writable.writeObject(argument.getValue(), null); + } + } + } + } + } } } else if (key.equals(LLMObsTags.METADATA) && val instanceof Map) { Map metadataMap = (Map) val; @@ -277,10 +332,10 @@ public void accept(Metadata metadata) { writable.writeString(entry.getKey(), null); writable.writeObject(entry.getValue(), null); } - continue; + } else { + writable.writeString(key, null); + writable.writeObject(val, null); } - writable.writeString(key, null); - writable.writeObject(val, null); } } } From 87cdaf13230b7854c3c7fe582d5085d3e55e111c Mon Sep 17 00:00:00 2001 From: gary-huang Date: Mon, 27 Jan 2025 14:21:50 -0500 Subject: [PATCH 12/28] impl llmobs agent and llmobs apis From f78b2c53aaf358605d5fac6e2f8f7cba4afc44d9 Mon Sep 17 00:00:00 2001 From: gary-huang Date: Tue, 1 Apr 2025 07:42:35 -0400 Subject: [PATCH 13/28] use new ctx api to track parent span --- .../trace/llmobs/domain/DDLLMObsSpan.java | 19 ++++++++++ .../trace/llmobs/domain/LLMObsState.java | 38 +++++++++++++++++++ .../writer/ddintake/LLMObsSpanMapper.java | 7 +++- 3 files changed, 62 insertions(+), 2 deletions(-) create mode 100644 dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/LLMObsState.java diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java index 640a61c9426..5757885e047 100644 --- a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java @@ -1,10 +1,12 @@ package datadog.trace.llmobs.domain; +import datadog.context.ContextScope; import datadog.trace.api.DDSpanTypes; import datadog.trace.api.llmobs.LLMObs; import datadog.trace.api.llmobs.LLMObsSpan; import datadog.trace.api.llmobs.LLMObsTags; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.AgentSpanContext; import datadog.trace.bootstrap.instrumentation.api.AgentTracer; import datadog.trace.bootstrap.instrumentation.api.Tags; import java.util.Collections; @@ -29,6 +31,7 @@ public class DDLLMObsSpan implements LLMObsSpan { private static final String OUTPUT = LLMOBS_TAG_PREFIX + "output"; private static final String SPAN_KIND = LLMOBS_TAG_PREFIX + Tags.SPAN_KIND; private static final String METADATA = LLMOBS_TAG_PREFIX + LLMObsTags.METADATA; + private static final String PARENT_ID_TAG_INTERNAL = "parent_id"; private static final String LLM_OBS_INSTRUMENTATION_NAME = "llmobs"; @@ -36,6 +39,7 @@ public class DDLLMObsSpan implements LLMObsSpan { private final AgentSpan span; private final String spanKind; + private final ContextScope scope; private boolean finished = false; @@ -63,6 +67,20 @@ public DDLLMObsSpan( if (sessionID != null && !sessionID.isEmpty()) { this.span.setTag(LLMOBS_TAG_PREFIX + LLMObsTags.SESSION_ID, sessionID); } + + AgentSpanContext parent = LLMObsState.getLLMObsParentContext(); + String parentSpanID = LLMObsState.ROOT_SPAN_ID; + if (null != parent) { + if (parent.getTraceId() != this.span.getTraceId()) { + LOGGER.error("trace ID mismatch, retrieved parent from context trace_id={}, span_id={}, started span trace_id={}, span_id={}", parent.getTraceId(), parent.getSpanId(), this.span.getTraceId(), this.span.getSpanId()); + } else { + parentSpanID = String.valueOf(parent.getSpanId()); + } + } + this.span.setTag( + LLMOBS_TAG_PREFIX + PARENT_ID_TAG_INTERNAL, parentSpanID); + this.scope = LLMObsState.attach(); + LLMObsState.setLLMObsParentContext(this.span.context()); } @Override @@ -271,6 +289,7 @@ public void finish() { return; } this.span.finish(); + this.scope.close(); this.finished = true; } } diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/LLMObsState.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/LLMObsState.java new file mode 100644 index 00000000000..14afd852013 --- /dev/null +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/LLMObsState.java @@ -0,0 +1,38 @@ +package datadog.trace.llmobs.domain; + +import datadog.context.Context; +import datadog.context.ContextKey; +import datadog.context.ContextScope; +import datadog.trace.bootstrap.instrumentation.api.AgentSpanContext; + +public class LLMObsState { + public static final String ROOT_SPAN_ID = "undefined"; + + private static final ContextKey CONTEXT_KEY = ContextKey.named("llmobs_span"); + + private AgentSpanContext parentSpanID; + + public static ContextScope attach() { + return Context.current().with(CONTEXT_KEY, new LLMObsState()).attach(); + } + + private static LLMObsState fromContext() { + return Context.current().get(CONTEXT_KEY); + } + + public static AgentSpanContext getLLMObsParentContext() { + LLMObsState state = fromContext(); + if (state != null) { + return state.parentSpanID; + } + return null; + } + + public static void setLLMObsParentContext(AgentSpanContext llmObsParentContext) { + LLMObsState state = fromContext(); + if (state != null) { + state.parentSpanID = llmObsParentContext; + } + } + +} diff --git a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java index 584b26902af..8eca040c05a 100644 --- a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java +++ b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java @@ -75,6 +75,9 @@ public class LLMObsSpanMapper implements RemoteMapper { private static final byte[] LLM_TOOL_CALL_ARGUMENTS = "arguments".getBytes(StandardCharsets.UTF_8); + // TODO is there a better place for this? + private static final String PARENT_ID_TAG_INTERNAL_FULL = LLMOBS_TAG_PREFIX + "parent_id"; + private final LLMObsSpanMapper.MetaWriter metaWriter = new MetaWriter(); private final int size; @@ -113,8 +116,8 @@ public void map(List> trace, Writable writable) { // 3 writable.writeUTF8(PARENT_ID); - // TODO fix after parent ID tracking is in place - writable.writeString("undefined", null); + writable.writeString(span.getTag(PARENT_ID_TAG_INTERNAL_FULL), null); + span.removeTag(PARENT_ID_TAG_INTERNAL_FULL); // 4 writable.writeUTF8(NAME); From 5fc90690e72ec998b0f0c5f89e755c6bdfc96bd4 Mon Sep 17 00:00:00 2001 From: gary-huang Date: Wed, 5 Mar 2025 18:36:50 -0500 Subject: [PATCH 14/28] add api for evals --- .../trace/llmobs/domain/DDLLMObsSpan.java | 23 ++++++++-- .../trace/llmobs/domain/LLMObsState.java | 1 - dd-trace-api/build.gradle | 1 + .../java/datadog/trace/api/llmobs/LLMObs.java | 44 +++++++++++++++++++ .../datadog/trace/api/llmobs/LLMObsSpan.java | 15 +++++++ .../llmobs/noop/NoOpLLMObsEvalProcessor.java | 33 ++++++++++++++ .../trace/api/llmobs/noop/NoOpLLMObsSpan.java | 11 +++++ 7 files changed, 123 insertions(+), 5 deletions(-) create mode 100644 dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsEvalProcessor.java diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java index 5757885e047..786a6f983dc 100644 --- a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java @@ -2,6 +2,7 @@ import datadog.context.ContextScope; import datadog.trace.api.DDSpanTypes; +import datadog.trace.api.DDTraceId; import datadog.trace.api.llmobs.LLMObs; import datadog.trace.api.llmobs.LLMObsSpan; import datadog.trace.api.llmobs.LLMObsTags; @@ -67,18 +68,22 @@ public DDLLMObsSpan( if (sessionID != null && !sessionID.isEmpty()) { this.span.setTag(LLMOBS_TAG_PREFIX + LLMObsTags.SESSION_ID, sessionID); } - + AgentSpanContext parent = LLMObsState.getLLMObsParentContext(); String parentSpanID = LLMObsState.ROOT_SPAN_ID; if (null != parent) { if (parent.getTraceId() != this.span.getTraceId()) { - LOGGER.error("trace ID mismatch, retrieved parent from context trace_id={}, span_id={}, started span trace_id={}, span_id={}", parent.getTraceId(), parent.getSpanId(), this.span.getTraceId(), this.span.getSpanId()); + LOGGER.error( + "trace ID mismatch, retrieved parent from context trace_id={}, span_id={}, started span trace_id={}, span_id={}", + parent.getTraceId(), + parent.getSpanId(), + this.span.getTraceId(), + this.span.getSpanId()); } else { parentSpanID = String.valueOf(parent.getSpanId()); } } - this.span.setTag( - LLMOBS_TAG_PREFIX + PARENT_ID_TAG_INTERNAL, parentSpanID); + this.span.setTag(LLMOBS_TAG_PREFIX + PARENT_ID_TAG_INTERNAL, parentSpanID); this.scope = LLMObsState.attach(); LLMObsState.setLLMObsParentContext(this.span.context()); } @@ -292,4 +297,14 @@ public void finish() { this.scope.close(); this.finished = true; } + + @Override + public DDTraceId getTraceId() { + return this.span.getTraceId(); + } + + @Override + public long getSpanId() { + return this.span.getSpanId(); + } } diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/LLMObsState.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/LLMObsState.java index 14afd852013..84f05afc94a 100644 --- a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/LLMObsState.java +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/LLMObsState.java @@ -34,5 +34,4 @@ public static void setLLMObsParentContext(AgentSpanContext llmObsParentContext) state.parentSpanID = llmObsParentContext; } } - } diff --git a/dd-trace-api/build.gradle b/dd-trace-api/build.gradle index 4175700c6ce..74bb8e2988e 100644 --- a/dd-trace-api/build.gradle +++ b/dd-trace-api/build.gradle @@ -38,6 +38,7 @@ excludedClassesCoverage += [ 'datadog.trace.api.llmobs.LLMObsSpan', 'datadog.trace.api.llmobs.noop.NoOpLLMObsSpan', 'datadog.trace.api.llmobs.noop.NoOpLLMObsSpanFactory', + 'datadog.trace.api.llmobs.noop.NoOpLLMObsEvalProcessor', 'datadog.trace.api.experimental.DataStreamsCheckpointer', 'datadog.trace.api.experimental.DataStreamsCheckpointer.NoOp', 'datadog.trace.api.experimental.DataStreamsContextCarrier', diff --git a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java index cd7426a24c5..7859d901b26 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java @@ -1,5 +1,6 @@ package datadog.trace.api.llmobs; +import datadog.trace.api.llmobs.noop.NoOpLLMObsEvalProcessor; import datadog.trace.api.llmobs.noop.NoOpLLMObsSpanFactory; import java.util.List; import java.util.Map; @@ -9,6 +10,7 @@ public class LLMObs { protected LLMObs() {} protected static LLMObsSpanFactory SPAN_FACTORY = NoOpLLMObsSpanFactory.INSTANCE; + protected static LLMObsEvalProcessor EVAL_PROCESSOR = NoOpLLMObsEvalProcessor.INSTANCE; public static LLMObsSpan startLLMSpan( String spanName, @@ -44,6 +46,26 @@ public static LLMObsSpan startWorkflowSpan( return SPAN_FACTORY.startWorkflowSpan(spanName, mlApp, sessionID); } + public static void SubmitEvaluation( + LLMObsSpan llmObsSpan, String label, String categoricalValue, Map tags) { + EVAL_PROCESSOR.SubmitEvaluation(llmObsSpan, label, categoricalValue, tags); + } + + public static void SubmitEvaluation( + LLMObsSpan llmObsSpan, String label, String categoricalValue, String mlApp, Map tags) { + EVAL_PROCESSOR.SubmitEvaluation(llmObsSpan, label, categoricalValue, mlApp, tags); + } + + public static void SubmitEvaluation( + LLMObsSpan llmObsSpan, String label, double scoreValue, Map tags) { + EVAL_PROCESSOR.SubmitEvaluation(llmObsSpan, label, scoreValue, tags); + } + + public static void SubmitEvaluation( + LLMObsSpan llmObsSpan, String label, double scoreValue, String mlApp, Map tags) { + EVAL_PROCESSOR.SubmitEvaluation(llmObsSpan, label, scoreValue, mlApp, tags); + } + public interface LLMObsSpanFactory { LLMObsSpan startLLMSpan( String spanName, @@ -62,6 +84,28 @@ LLMObsSpan startWorkflowSpan( String spanName, @Nullable String mlApp, @Nullable String sessionID); } + public interface LLMObsEvalProcessor { + void SubmitEvaluation( + LLMObsSpan llmObsSpan, String label, double scoreValue, Map tags); + + void SubmitEvaluation( + LLMObsSpan llmObsSpan, + String label, + double scoreValue, + String mlApp, + Map tags); + + void SubmitEvaluation( + LLMObsSpan llmObsSpan, String label, String categoricalValue, Map tags); + + void SubmitEvaluation( + LLMObsSpan llmObsSpan, + String label, + String categoricalValue, + String mlApp, + Map tags); + } + public static class ToolCall { private String name; private String type; diff --git a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsSpan.java b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsSpan.java index 80668eabd57..e86c9717e24 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsSpan.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsSpan.java @@ -1,5 +1,6 @@ package datadog.trace.api.llmobs; +import datadog.trace.api.DDTraceId; import java.util.List; import java.util.Map; @@ -142,4 +143,18 @@ public interface LLMObsSpan { /** Finishes (closes) a span */ void finish(); + + /** + * Gets the TraceId of the span's trace. + * + * @return The TraceId of the span's trace, or {@link DDTraceId#ZERO} if not set. + */ + DDTraceId getTraceId(); + + /** + * Gets the SpanId. + * + * @return The span identifier. + */ + long getSpanId(); } diff --git a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsEvalProcessor.java b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsEvalProcessor.java new file mode 100644 index 00000000000..403c0afce02 --- /dev/null +++ b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsEvalProcessor.java @@ -0,0 +1,33 @@ +package datadog.trace.api.llmobs.noop; + +import datadog.trace.api.llmobs.LLMObs; +import datadog.trace.api.llmobs.LLMObsSpan; +import java.util.Map; + +public class NoOpLLMObsEvalProcessor implements LLMObs.LLMObsEvalProcessor { + public static final NoOpLLMObsEvalProcessor INSTANCE = new NoOpLLMObsEvalProcessor(); + + @Override + public void SubmitEvaluation( + LLMObsSpan llmObsSpan, String label, double scoreValue, Map tags) {} + + @Override + public void SubmitEvaluation( + LLMObsSpan llmObsSpan, + String label, + double scoreValue, + String mlApp, + Map tags) {} + + @Override + public void SubmitEvaluation( + LLMObsSpan llmObsSpan, String label, String categoricalValue, Map tags) {} + + @Override + public void SubmitEvaluation( + LLMObsSpan llmObsSpan, + String label, + String categoricalValue, + String mlApp, + Map tags) {} +} diff --git a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsSpan.java b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsSpan.java index a1b160616e7..17ea94101b4 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsSpan.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsSpan.java @@ -1,5 +1,6 @@ package datadog.trace.api.llmobs.noop; +import datadog.trace.api.DDTraceId; import datadog.trace.api.llmobs.LLMObs; import datadog.trace.api.llmobs.LLMObsSpan; import java.util.List; @@ -58,4 +59,14 @@ public void addThrowable(Throwable throwable) {} @Override public void finish() {} + + @Override + public DDTraceId getTraceId() { + return DDTraceId.ZERO; + } + + @Override + public long getSpanId() { + return 0; + } } From a74e4560d6a401f7de52ac8e512efaec36ab4f26 Mon Sep 17 00:00:00 2001 From: gary-huang Date: Thu, 6 Mar 2025 12:38:50 -0500 Subject: [PATCH 15/28] working impl supporting both agentless and agent --- dd-java-agent/agent-llmobs/build.gradle | 1 + .../trace/llmobs/EvalProcessingWorker.java | 205 ++++++++++++++++++ .../datadog/trace/llmobs/LLMObsSystem.java | 85 ++++++++ .../trace/llmobs/domain/LLMObsEval.java | 131 +++++++++++ .../trace/llmobs/domain/LLMObsInternal.java | 5 +- dd-trace-api/build.gradle | 1 + .../java/datadog/trace/api/llmobs/LLMObs.java | 12 +- .../trace/util/AgentThreadFactory.java | 4 +- 8 files changed, 440 insertions(+), 4 deletions(-) create mode 100644 dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/EvalProcessingWorker.java create mode 100644 dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/LLMObsEval.java diff --git a/dd-java-agent/agent-llmobs/build.gradle b/dd-java-agent/agent-llmobs/build.gradle index 071840c93ec..6903359e510 100644 --- a/dd-java-agent/agent-llmobs/build.gradle +++ b/dd-java-agent/agent-llmobs/build.gradle @@ -22,6 +22,7 @@ minimumInstructionCoverage = 0.0 dependencies { api libs.slf4j + implementation libs.jctools implementation project(':communication') implementation project(':components:json') diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/EvalProcessingWorker.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/EvalProcessingWorker.java new file mode 100644 index 00000000000..ef75000c7bf --- /dev/null +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/EvalProcessingWorker.java @@ -0,0 +1,205 @@ +package datadog.trace.llmobs; + +import static datadog.trace.util.AgentThreadFactory.AgentThread.LLMOBS_EVALS_PROCESSOR; +import static datadog.trace.util.AgentThreadFactory.THREAD_JOIN_TIMOUT_MS; +import static datadog.trace.util.AgentThreadFactory.newAgentThread; + +import com.squareup.moshi.JsonAdapter; +import com.squareup.moshi.Moshi; +import datadog.communication.ddagent.DDAgentFeaturesDiscovery; +import datadog.communication.ddagent.SharedCommunicationObjects; +import datadog.communication.http.HttpRetryPolicy; +import datadog.communication.http.OkHttpUtils; +import datadog.trace.api.Config; +import datadog.trace.llmobs.domain.LLMObsEval; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import okhttp3.Headers; +import okhttp3.HttpUrl; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.RequestBody; +import org.jctools.queues.MpscBlockingConsumerArrayQueue; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class EvalProcessingWorker implements AutoCloseable { + + private static final String EVAL_METRIC_API_DOMAIN = "api"; + private static final String EVAL_METRIC_API_PATH = "api/intake/llm-obs/v1/eval-metric"; + + private static final String EVP_SUBDOMAIN_HEADER_NAME = "X-Datadog-EVP-Subdomain"; + private static final String DD_API_KEY_HEADER_NAME = "DD-API-KEY"; + + private static final Logger log = LoggerFactory.getLogger(EvalProcessingWorker.class); + + private final MpscBlockingConsumerArrayQueue queue; + private final Thread serializerThread; + + public EvalProcessingWorker( + final int capacity, + final long flushInterval, + final TimeUnit timeUnit, + final SharedCommunicationObjects sco, + Config config) { + this.queue = new MpscBlockingConsumerArrayQueue<>(capacity); + + boolean isAgentless = config.isLlmObsAgentlessEnabled(); + if (isAgentless && (config.getApiKey() == null || config.getApiKey().isEmpty())) { + log.error("Agentless eval metric submission requires an API key"); + } + + Headers headers; + HttpUrl submissionUrl; + if (isAgentless) { + submissionUrl = + HttpUrl.get( + "https://" + + EVAL_METRIC_API_DOMAIN + + "." + + config.getSite() + + "/" + + EVAL_METRIC_API_PATH); + headers = Headers.of(DD_API_KEY_HEADER_NAME, config.getApiKey()); + } else { + submissionUrl = + HttpUrl.get( + sco.agentUrl.toString() + + DDAgentFeaturesDiscovery.V2_EVP_PROXY_ENDPOINT + + EVAL_METRIC_API_PATH); + headers = Headers.of(EVP_SUBDOMAIN_HEADER_NAME, EVAL_METRIC_API_DOMAIN); + } + + EvalSerializingHandler serializingHandler = + new EvalSerializingHandler(queue, flushInterval, timeUnit, submissionUrl, headers); + this.serializerThread = newAgentThread(LLMOBS_EVALS_PROCESSOR, serializingHandler); + } + + public void start() { + this.serializerThread.start(); + } + + public boolean addToQueue(final LLMObsEval eval) { + return queue.offer(eval); + } + + @Override + public void close() { + serializerThread.interrupt(); + try { + serializerThread.join(THREAD_JOIN_TIMOUT_MS); + } catch (InterruptedException ignored) { + } + } + + public static class EvalSerializingHandler implements Runnable { + + private static final Logger log = LoggerFactory.getLogger(EvalSerializingHandler.class); + private static final int FLUSH_THRESHOLD = 50; + + private final MpscBlockingConsumerArrayQueue queue; + private final long ticksRequiredToFlush; + private long lastTicks; + + private final Moshi moshi; + private final JsonAdapter evalJsonAdapter; + private final OkHttpClient httpClient; + private final HttpUrl submissionUrl; + private final Headers headers; + + private final List buffer = new ArrayList<>(); + + public EvalSerializingHandler( + final MpscBlockingConsumerArrayQueue queue, + final long flushInterval, + final TimeUnit timeUnit, + final HttpUrl submissionUrl, + final Headers headers) { + this.queue = queue; + this.moshi = new Moshi.Builder().add(LLMObsEval.class, new LLMObsEval.Adapter()).build(); + + this.evalJsonAdapter = moshi.adapter(LLMObsEval.Request.class); + this.httpClient = new OkHttpClient(); + this.submissionUrl = submissionUrl; + this.headers = headers; + + this.lastTicks = System.nanoTime(); + this.ticksRequiredToFlush = timeUnit.toNanos(flushInterval); + + log.debug("starting eval metric serializer, url={}", submissionUrl); + } + + @Override + public void run() { + try { + runDutyCycle(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + log.debug( + "eval processor worker exited. submitting evals stopped. unsubmitted evals left: " + + !queuesAreEmpty()); + } + + private void runDutyCycle() throws InterruptedException { + Thread thread = Thread.currentThread(); + while (!thread.isInterrupted()) { + consumeBatch(); + flushIfNecessary(); + } + } + + private void consumeBatch() { + queue.drain(buffer::add, queue.size()); + } + + protected void flushIfNecessary() { + if (buffer.isEmpty()) { + return; + } + if (shouldFlush()) { + LLMObsEval.Request llmobsEvalReq = new LLMObsEval.Request(this.buffer); + HttpRetryPolicy.Factory retryPolicyFactory = new HttpRetryPolicy.Factory(5, 100, 2.0, true); + + String reqBod = evalJsonAdapter.toJson(llmobsEvalReq); + + RequestBody requestBody = + RequestBody.create(okhttp3.MediaType.parse("application/json"), reqBod); + Request request = + new Request.Builder().headers(headers).url(submissionUrl).post(requestBody).build(); + + try (okhttp3.Response response = + OkHttpUtils.sendWithRetries(httpClient, retryPolicyFactory, request)) { + + if (response.isSuccessful()) { + log.debug("successfully flushed evaluation request with {} evals", this.buffer.size()); + this.buffer.clear(); + } else { + log.error( + "Could not submit eval metrics (HTTP code " + + response.code() + + ")" + + (response.body() != null ? ": " + response.body().string() : "")); + } + } catch (Exception e) { + log.error("Could not submit eval metrics", e); + } + } + } + + private boolean shouldFlush() { + long nanoTime = System.nanoTime(); + long ticks = nanoTime - lastTicks; + if (ticks > ticksRequiredToFlush || queue.size() >= FLUSH_THRESHOLD) { + lastTicks = nanoTime; + return true; + } + return false; + } + + protected boolean queuesAreEmpty() { + return queue.isEmpty(); + } + } +} diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java index f1290d5b166..04239a3a7f5 100644 --- a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java @@ -7,8 +7,11 @@ import datadog.trace.api.llmobs.LLMObsTags; import datadog.trace.bootstrap.instrumentation.api.Tags; import datadog.trace.llmobs.domain.DDLLMObsSpan; +import datadog.trace.llmobs.domain.LLMObsEval; import datadog.trace.llmobs.domain.LLMObsInternal; import java.lang.instrument.Instrumentation; +import java.util.Map; +import java.util.concurrent.TimeUnit; import org.jetbrains.annotations.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,6 +34,88 @@ public static void start(Instrumentation inst, SharedCommunicationObjects sco) { LLMObsInternal.setLLMObsSpanFactory( new LLMObsManualSpanFactory( config.getLlmObsMlApp(), config.getServiceName())); + + String mlApp = config.getLlmObsMlApp(); + LLMObsInternal.setLLMObsSpanFactory( + new LLMObsManualSpanFactory(mlApp, config.getServiceName())); + + LLMObsInternal.setLLMObsEvalProcessor(new LLMObsCustomEvalProcessor(mlApp, sco, config)); + } + + private static class LLMObsCustomEvalProcessor implements LLMObs.LLMObsEvalProcessor { + private final String defaultMLApp; + private final EvalProcessingWorker evalProcessingWorker; + + public LLMObsCustomEvalProcessor( + String defaultMLApp, SharedCommunicationObjects sco, Config config) { + + this.defaultMLApp = defaultMLApp; + this.evalProcessingWorker = + new EvalProcessingWorker(1024, 100, TimeUnit.MILLISECONDS, sco, config); + this.evalProcessingWorker.start(); + } + + @Override + public void SubmitEvaluation( + LLMObsSpan llmObsSpan, String label, double scoreValue, Map tags) { + SubmitEvaluation(llmObsSpan, label, scoreValue, defaultMLApp, tags); + } + + @Override + public void SubmitEvaluation( + LLMObsSpan llmObsSpan, + String label, + double scoreValue, + String mlApp, + Map tags) { + if (llmObsSpan == null) { + LOGGER.error("null llm obs span provided, eval not recorded"); + } + String traceID = llmObsSpan.getTraceId().toHexString(); + long spanID = llmObsSpan.getSpanId(); + LLMObsEval.Score score = + new LLMObsEval.Score( + traceID, spanID, System.currentTimeMillis(), mlApp, label, tags, scoreValue); + if (!this.evalProcessingWorker.addToQueue(score)) { + LOGGER.warn( + "queue full, failed to add score eval, ml_app={}, trace_id={}, span_id={}, label={}", + mlApp, + traceID, + spanID, + label); + } + } + + @Override + public void SubmitEvaluation( + LLMObsSpan llmObsSpan, String label, String categoricalValue, Map tags) { + SubmitEvaluation(llmObsSpan, label, categoricalValue, defaultMLApp, tags); + } + + @Override + public void SubmitEvaluation( + LLMObsSpan llmObsSpan, + String label, + String categoricalValue, + String mlApp, + Map tags) { + if (llmObsSpan == null) { + LOGGER.error("null llm obs span provided, eval not recorded"); + } + String traceID = llmObsSpan.getTraceId().toHexString(); + long spanID = llmObsSpan.getSpanId(); + LLMObsEval.Categorical category = + new LLMObsEval.Categorical( + traceID, spanID, System.currentTimeMillis(), mlApp, label, tags, categoricalValue); + if (!this.evalProcessingWorker.addToQueue(category)) { + LOGGER.warn( + "queue full, failed to add categorical eval, ml_app={}, trace_id={}, span_id={}, label={}", + mlApp, + traceID, + spanID, + label); + } + } } private static class LLMObsManualSpanFactory implements LLMObs.LLMObsSpanFactory { diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/LLMObsEval.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/LLMObsEval.java new file mode 100644 index 00000000000..26396065679 --- /dev/null +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/LLMObsEval.java @@ -0,0 +1,131 @@ +package datadog.trace.llmobs.domain; + +import com.squareup.moshi.JsonAdapter; +import com.squareup.moshi.JsonDataException; +import com.squareup.moshi.JsonReader; +import com.squareup.moshi.JsonWriter; +import com.squareup.moshi.Moshi; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import org.jetbrains.annotations.Nullable; + +public abstract class LLMObsEval { + private static final String METRIC_TYPE_SCORE = "score"; + private static final String METRIC_TYPE_CATEGORICAL = "categorical"; + + public final String trace_id; + public final String span_id; + public final long timestamp_ms; + public final String ml_app; + public final String metric_type; + public final String label; + public final List tags; + + public LLMObsEval( + String traceID, + String spanID, + long timestampMs, + String mlApp, + String metricType, + String label, + Map tags) { + this.trace_id = traceID; + this.span_id = spanID; + this.timestamp_ms = timestampMs; + this.ml_app = mlApp; + this.metric_type = metricType; + this.label = label; + List tagsList = new ArrayList<>(tags.size()); + for (Map.Entry entry : tags.entrySet()) { + tagsList.add(entry.getKey() + ":" + entry.getValue()); + } + this.tags = tagsList; + } + + public static final class Adapter extends JsonAdapter { + private final Moshi moshi = new Moshi.Builder().build(); + private final JsonAdapter scoreJsonAdapter = moshi.adapter(Score.class); + private final JsonAdapter categoricalJsonAdapter = + moshi.adapter(Categorical.class); + + @Nullable + @Override + public LLMObsEval fromJson(JsonReader reader) { + return null; + } + + @Override + public void toJson(JsonWriter writer, LLMObsEval value) throws IOException { + if (value == null) { + throw new JsonDataException("unexpectedly got null llm obs eval "); + } + if (value instanceof Score) { + scoreJsonAdapter.toJson(writer, (Score) value); + } else if (value instanceof Categorical) { + categoricalJsonAdapter.toJson(writer, (Categorical) value); + } else { + throw new JsonDataException("Unknown llm obs eval subclass: " + value.getClass()); + } + } + } + + public static final class Score extends LLMObsEval { + public final double score_value; + + public Score( + String traceID, + long spanID, + long timestampMS, + String mlApp, + String label, + Map tags, + double scoreValue) { + super(traceID, String.valueOf(spanID), timestampMS, mlApp, METRIC_TYPE_SCORE, label, tags); + this.score_value = scoreValue; + } + } + + public static final class Categorical extends LLMObsEval { + public final String categorical_value; + + public Categorical( + String traceID, + long spanID, + long timestampMS, + String mlApp, + String label, + Map tags, + String categoricalValue) { + super( + traceID, + String.valueOf(spanID), + timestampMS, + mlApp, + METRIC_TYPE_CATEGORICAL, + label, + tags); + this.categorical_value = categoricalValue; + } + } + + public static final class Request { + public final Data data; + + public static class Data { + public final String type = "evaluation_metric"; + public Attributes attributes; + } + + public static class Attributes { + public List metrics; + } + + public Request(List metrics) { + this.data = new Data(); + this.data.attributes = new Attributes(); + this.data.attributes.metrics = metrics; + } + } +} diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/LLMObsInternal.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/LLMObsInternal.java index 42b0c097e48..85e1482b412 100644 --- a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/LLMObsInternal.java +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/LLMObsInternal.java @@ -3,8 +3,11 @@ import datadog.trace.api.llmobs.LLMObs; public class LLMObsInternal extends LLMObs { - public static void setLLMObsSpanFactory(final LLMObsSpanFactory factory) { LLMObs.SPAN_FACTORY = factory; } + + public static void setLLMObsEvalProcessor(final LLMObsEvalProcessor evalProcessor) { + LLMObs.EVAL_PROCESSOR = evalProcessor; + } } diff --git a/dd-trace-api/build.gradle b/dd-trace-api/build.gradle index 74bb8e2988e..5764cc6770d 100644 --- a/dd-trace-api/build.gradle +++ b/dd-trace-api/build.gradle @@ -56,6 +56,7 @@ excludedClassesCoverage += [ description = 'dd-trace-api' dependencies { + api libs.slf4j testImplementation libs.guava testImplementation project(':utils:test-utils') } diff --git a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java index 7859d901b26..79fa5be852a 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java @@ -52,7 +52,11 @@ public static void SubmitEvaluation( } public static void SubmitEvaluation( - LLMObsSpan llmObsSpan, String label, String categoricalValue, String mlApp, Map tags) { + LLMObsSpan llmObsSpan, + String label, + String categoricalValue, + String mlApp, + Map tags) { EVAL_PROCESSOR.SubmitEvaluation(llmObsSpan, label, categoricalValue, mlApp, tags); } @@ -62,7 +66,11 @@ public static void SubmitEvaluation( } public static void SubmitEvaluation( - LLMObsSpan llmObsSpan, String label, double scoreValue, String mlApp, Map tags) { + LLMObsSpan llmObsSpan, + String label, + double scoreValue, + String mlApp, + Map tags) { EVAL_PROCESSOR.SubmitEvaluation(llmObsSpan, label, scoreValue, mlApp, tags); } diff --git a/internal-api/src/main/java/datadog/trace/util/AgentThreadFactory.java b/internal-api/src/main/java/datadog/trace/util/AgentThreadFactory.java index 67bdf576607..33affffa533 100644 --- a/internal-api/src/main/java/datadog/trace/util/AgentThreadFactory.java +++ b/internal-api/src/main/java/datadog/trace/util/AgentThreadFactory.java @@ -57,7 +57,9 @@ public enum AgentThread { RETRANSFORMER("dd-retransformer"), - LOGS_INTAKE("dd-logs-intake"); + LOGS_INTAKE("dd-logs-intake"), + + LLMOBS_EVALS_PROCESSOR("dd-llmobs-evals-processor"); public final String threadName; From 19d95d7a780d7b4420b0c1f65db66a88a9a06818 Mon Sep 17 00:00:00 2001 From: gary-huang Date: Mon, 21 Apr 2025 10:35:11 -0400 Subject: [PATCH 16/28] handle null tags and default to default ml app if null or empty string provided in the override --- .../main/java/datadog/trace/llmobs/LLMObsSystem.java | 10 ++++++++++ .../java/datadog/trace/llmobs/domain/LLMObsEval.java | 12 ++++++++---- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java index 04239a3a7f5..94fc931f92e 100644 --- a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java @@ -70,6 +70,11 @@ public void SubmitEvaluation( Map tags) { if (llmObsSpan == null) { LOGGER.error("null llm obs span provided, eval not recorded"); + return; + } + + if (mlApp == null || mlApp.isEmpty()) { + mlApp = defaultMLApp; } String traceID = llmObsSpan.getTraceId().toHexString(); long spanID = llmObsSpan.getSpanId(); @@ -101,6 +106,11 @@ public void SubmitEvaluation( Map tags) { if (llmObsSpan == null) { LOGGER.error("null llm obs span provided, eval not recorded"); + return; + } + + if (mlApp == null || mlApp.isEmpty()) { + mlApp = defaultMLApp; } String traceID = llmObsSpan.getTraceId().toHexString(); long spanID = llmObsSpan.getSpanId(); diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/LLMObsEval.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/LLMObsEval.java index 26396065679..8438e398e38 100644 --- a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/LLMObsEval.java +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/LLMObsEval.java @@ -37,11 +37,15 @@ public LLMObsEval( this.ml_app = mlApp; this.metric_type = metricType; this.label = label; - List tagsList = new ArrayList<>(tags.size()); - for (Map.Entry entry : tags.entrySet()) { - tagsList.add(entry.getKey() + ":" + entry.getValue()); + if (tags != null) { + List tagsList = new ArrayList<>(tags.size()); + for (Map.Entry entry : tags.entrySet()) { + tagsList.add(entry.getKey() + ":" + entry.getValue()); + } + this.tags = tagsList; + } else { + this.tags = null; } - this.tags = tagsList; } public static final class Adapter extends JsonAdapter { From f1481acf7380bf3657a1a825567aaee1b6616bd9 Mon Sep 17 00:00:00 2001 From: gary-huang Date: Wed, 23 Apr 2025 14:45:22 +0200 Subject: [PATCH 17/28] spotless --- .../src/main/java/datadog/trace/llmobs/LLMObsSystem.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java index f1290d5b166..d4b76c585e2 100644 --- a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java @@ -29,8 +29,7 @@ public static void start(Instrumentation inst, SharedCommunicationObjects sco) { sco.createRemaining(config); LLMObsInternal.setLLMObsSpanFactory( - new LLMObsManualSpanFactory( - config.getLlmObsMlApp(), config.getServiceName())); + new LLMObsManualSpanFactory(config.getLlmObsMlApp(), config.getServiceName())); } private static class LLMObsManualSpanFactory implements LLMObs.LLMObsSpanFactory { @@ -38,8 +37,7 @@ private static class LLMObsManualSpanFactory implements LLMObs.LLMObsSpanFactory private final String serviceName; private final String defaultMLApp; - public LLMObsManualSpanFactory( - String defaultMLApp, String serviceName) { + public LLMObsManualSpanFactory(String defaultMLApp, String serviceName) { this.defaultMLApp = defaultMLApp; this.serviceName = serviceName; } From de0d5a522621948beb8e3915eda0d80c601c22a2 Mon Sep 17 00:00:00 2001 From: gary-huang Date: Wed, 23 Apr 2025 14:56:34 +0200 Subject: [PATCH 18/28] spotless --- .../src/main/java/datadog/trace/llmobs/LLMObsSystem.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java index f1290d5b166..d4b76c585e2 100644 --- a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java @@ -29,8 +29,7 @@ public static void start(Instrumentation inst, SharedCommunicationObjects sco) { sco.createRemaining(config); LLMObsInternal.setLLMObsSpanFactory( - new LLMObsManualSpanFactory( - config.getLlmObsMlApp(), config.getServiceName())); + new LLMObsManualSpanFactory(config.getLlmObsMlApp(), config.getServiceName())); } private static class LLMObsManualSpanFactory implements LLMObs.LLMObsSpanFactory { @@ -38,8 +37,7 @@ private static class LLMObsManualSpanFactory implements LLMObs.LLMObsSpanFactory private final String serviceName; private final String defaultMLApp; - public LLMObsManualSpanFactory( - String defaultMLApp, String serviceName) { + public LLMObsManualSpanFactory(String defaultMLApp, String serviceName) { this.defaultMLApp = defaultMLApp; this.serviceName = serviceName; } From 82b8418c7a13eb1fc070e98511be455525662d14 Mon Sep 17 00:00:00 2001 From: Gary Huang Date: Wed, 7 May 2025 11:08:22 -0400 Subject: [PATCH 19/28] add APIs for llm obs sdk (#8135) * add APIs for llm obs * add llm message class to support llm spans * follow java convention of naming Id instead of ID * add codeowners --- .github/CODEOWNERS | 6 + dd-trace-api/build.gradle | 6 + .../java/datadog/trace/api/llmobs/LLMObs.java | 136 ++++++++++++++++ .../datadog/trace/api/llmobs/LLMObsSpan.java | 145 ++++++++++++++++++ .../trace/api/llmobs/noop/NoOpLLMObsSpan.java | 61 ++++++++ .../llmobs/noop/NoOpLLMObsSpanFactory.java | 38 +++++ 6 files changed, 392 insertions(+) create mode 100644 dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java create mode 100644 dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsSpan.java create mode 100644 dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsSpan.java create mode 100644 dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsSpanFactory.java diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 5983b5d0bd2..791377393d2 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -74,3 +74,9 @@ dd-trace-core/src/main/java/datadog/trace/core/datastreams @Dat dd-trace-core/src/test/groovy/datadog/trace/core/datastreams @DataDog/data-streams-monitoring internal-api/src/main/java/datadog/trace/api/datastreams @DataDog/data-streams-monitoring internal-api/src/test/groovy/datadog/trace/api/datastreams @DataDog/data-streams-monitoring + +# @DataDog/ml-observability +dd-trace-api/src/main/java/datadog/trace/api/llmobs/ @DataDog/ml-observability +dd-java-agent/agent-llmobs/ @DataDog/ml-observability +dd-trace-core/src/main/java/datadog/trace/llmobs/ @DataDog/ml-observability +dd-trace-core/src/test/groovy/datadog/trace/llmobs/ @DataDog/ml-observability diff --git a/dd-trace-api/build.gradle b/dd-trace-api/build.gradle index c4b00313208..4175700c6ce 100644 --- a/dd-trace-api/build.gradle +++ b/dd-trace-api/build.gradle @@ -32,6 +32,12 @@ excludedClassesCoverage += [ 'datadog.trace.api.profiling.ProfilingScope', 'datadog.trace.api.profiling.ProfilingContext', 'datadog.trace.api.profiling.ProfilingContextAttribute.NoOp', + 'datadog.trace.api.llmobs.LLMObs', + 'datadog.trace.api.llmobs.LLMObs.LLMMessage', + 'datadog.trace.api.llmobs.LLMObs.ToolCall', + 'datadog.trace.api.llmobs.LLMObsSpan', + 'datadog.trace.api.llmobs.noop.NoOpLLMObsSpan', + 'datadog.trace.api.llmobs.noop.NoOpLLMObsSpanFactory', 'datadog.trace.api.experimental.DataStreamsCheckpointer', 'datadog.trace.api.experimental.DataStreamsCheckpointer.NoOp', 'datadog.trace.api.experimental.DataStreamsContextCarrier', diff --git a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java new file mode 100644 index 00000000000..fd3e1f0a952 --- /dev/null +++ b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java @@ -0,0 +1,136 @@ +package datadog.trace.api.llmobs; + +import datadog.trace.api.llmobs.noop.NoOpLLMObsSpanFactory; +import java.util.List; +import java.util.Map; +import javax.annotation.Nullable; + +public class LLMObs { + protected LLMObs() {} + + protected static LLMObsSpanFactory SPAN_FACTORY = NoOpLLMObsSpanFactory.INSTANCE; + + public static LLMObsSpan startLLMSpan( + String spanName, + String modelName, + String modelProvider, + @Nullable String mlApp, + @Nullable String sessionId) { + + return SPAN_FACTORY.startLLMSpan(spanName, modelName, modelProvider, mlApp, sessionId); + } + + public static LLMObsSpan startAgentSpan( + String spanName, @Nullable String mlApp, @Nullable String sessionId) { + + return SPAN_FACTORY.startAgentSpan(spanName, mlApp, sessionId); + } + + public static LLMObsSpan startToolSpan( + String spanName, @Nullable String mlApp, @Nullable String sessionId) { + + return SPAN_FACTORY.startToolSpan(spanName, mlApp, sessionId); + } + + public static LLMObsSpan startTaskSpan( + String spanName, @Nullable String mlApp, @Nullable String sessionId) { + + return SPAN_FACTORY.startTaskSpan(spanName, mlApp, sessionId); + } + + public static LLMObsSpan startWorkflowSpan( + String spanName, @Nullable String mlApp, @Nullable String sessionId) { + + return SPAN_FACTORY.startWorkflowSpan(spanName, mlApp, sessionId); + } + + public interface LLMObsSpanFactory { + LLMObsSpan startLLMSpan( + String spanName, + String modelName, + String modelProvider, + @Nullable String mlApp, + @Nullable String sessionId); + + LLMObsSpan startAgentSpan(String spanName, @Nullable String mlApp, @Nullable String sessionId); + + LLMObsSpan startToolSpan(String spanName, @Nullable String mlApp, @Nullable String sessionId); + + LLMObsSpan startTaskSpan(String spanName, @Nullable String mlApp, @Nullable String sessionId); + + LLMObsSpan startWorkflowSpan( + String spanName, @Nullable String mlApp, @Nullable String sessionId); + } + + public static class ToolCall { + private String name; + private String type; + private String toolId; + private Map arguments; + + public static ToolCall from( + String name, String type, String toolId, Map arguments) { + return new ToolCall(name, type, toolId, arguments); + } + + private ToolCall(String name, String type, String toolId, Map arguments) { + this.name = name; + this.type = type; + this.toolId = toolId; + this.arguments = arguments; + } + + public String getName() { + return name; + } + + public String getType() { + return type; + } + + public String getToolId() { + return toolId; + } + + public Map getArguments() { + return arguments; + } + } + + public static class LLMMessage { + private String role; + private String content; + private List toolCalls; + + public static LLMMessage from(String role, String content, List toolCalls) { + return new LLMMessage(role, content, toolCalls); + } + + public static LLMMessage from(String role, String content) { + return new LLMMessage(role, content); + } + + private LLMMessage(String role, String content, List toolCalls) { + this.role = role; + this.content = content; + this.toolCalls = toolCalls; + } + + private LLMMessage(String role, String content) { + this.role = role; + this.content = content; + } + + public String getRole() { + return role; + } + + public String getContent() { + return content; + } + + public List getToolCalls() { + return toolCalls; + } + } +} diff --git a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsSpan.java b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsSpan.java new file mode 100644 index 00000000000..80668eabd57 --- /dev/null +++ b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsSpan.java @@ -0,0 +1,145 @@ +package datadog.trace.api.llmobs; + +import java.util.List; +import java.util.Map; + +/** This interface represent an individual LLM Obs span. */ +public interface LLMObsSpan { + + /** + * Annotate the span with inputs and outputs for LLM spans + * + * @param inputMessages The input messages of the span in the form of a list + * @param outputMessages The output messages of the span in the form of a list + */ + void annotateIO(List inputMessages, List outputMessages); + + /** + * Annotate the span with inputs and outputs + * + * @param inputData The input data of the span in the form of a string + * @param outputData The output data of the span in the form of a string + */ + void annotateIO(String inputData, String outputData); + + /** + * Annotate the span with metadata + * + * @param metadata A map of JSON serializable key-value pairs that contains metadata information + * relevant to the input or output operation described by the span + */ + void setMetadata(Map metadata); + + /** + * Annotate the span with metrics + * + * @param metrics A map of JSON serializable keys and numeric values that users can add as metrics + * relevant to the operation described by the span (input_tokens, output_tokens, total_tokens, + * etc.). + */ + void setMetrics(Map metrics); + + /** + * Annotate the span with a single metric key value pair for the span’s context (number of tokens + * document length, etc). + * + * @param key the name of the metric + * @param value the value of the metric + */ + void setMetric(CharSequence key, int value); + + /** + * Annotate the span with a single metric key value pair for the span’s context (number of tokens + * document length, etc). + * + * @param key the name of the metric + * @param value the value of the metric + */ + void setMetric(CharSequence key, long value); + + /** + * Annotate the span with a single metric key value pair for the span’s context (number of tokens + * document length, etc). + * + * @param key the name of the metric + * @param value the value of the metric + */ + void setMetric(CharSequence key, double value); + + /** + * Annotate the span with tags + * + * @param tags An map of JSON serializable key-value pairs that users can add as tags regarding + * the span’s context (session, environment, system, versioning, etc.). + */ + void setTags(Map tags); + + /** + * Annotate the span with a single tag key value pair as a tag regarding the span’s context + * (session, environment, system, versioning, etc.). + * + * @param key the key of the tag + * @param value the value of the tag + */ + void setTag(String key, String value); + + /** + * Annotate the span with a single tag key value pair as a tag regarding the span’s context + * (session, environment, system, versioning, etc.). + * + * @param key the key of the tag + * @param value the value of the tag + */ + void setTag(String key, boolean value); + + /** + * Annotate the span with a single tag key value pair as a tag regarding the span’s context + * (session, environment, system, versioning, etc.). + * + * @param key the key of the tag + * @param value the value of the tag + */ + void setTag(String key, int value); + + /** + * Annotate the span with a single tag key value pair as a tag regarding the span’s context + * (session, environment, system, versioning, etc.). + * + * @param key the key of the tag + * @param value the value of the tag + */ + void setTag(String key, long value); + + /** + * Annotate the span with a single tag key value pair as a tag regarding the span’s context + * (session, environment, system, versioning, etc.). + * + * @param key the key of the tag + * @param value the value of the tag + */ + void setTag(String key, double value); + + /** + * Annotate the span to indicate that an error occurred + * + * @param error whether an error occurred + */ + void setError(boolean error); + + /** + * Annotate the span with an error message + * + * @param errorMessage the message of the error + */ + void setErrorMessage(String errorMessage); + + /** + * Annotate the span with a throwable + * + * @param throwable the errored throwable + */ + void addThrowable(Throwable throwable); + + /** Finishes (closes) a span */ + void finish(); +} diff --git a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsSpan.java b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsSpan.java new file mode 100644 index 00000000000..a1b160616e7 --- /dev/null +++ b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsSpan.java @@ -0,0 +1,61 @@ +package datadog.trace.api.llmobs.noop; + +import datadog.trace.api.llmobs.LLMObs; +import datadog.trace.api.llmobs.LLMObsSpan; +import java.util.List; +import java.util.Map; + +public class NoOpLLMObsSpan implements LLMObsSpan { + public static final LLMObsSpan INSTANCE = new NoOpLLMObsSpan(); + + @Override + public void annotateIO(List inputData, List outputData) {} + + @Override + public void annotateIO(String inputData, String outputData) {} + + @Override + public void setMetadata(Map metadata) {} + + @Override + public void setMetrics(Map metrics) {} + + @Override + public void setMetric(CharSequence key, int value) {} + + @Override + public void setMetric(CharSequence key, long value) {} + + @Override + public void setMetric(CharSequence key, double value) {} + + @Override + public void setTags(Map tags) {} + + @Override + public void setTag(String key, String value) {} + + @Override + public void setTag(String key, boolean value) {} + + @Override + public void setTag(String key, int value) {} + + @Override + public void setTag(String key, long value) {} + + @Override + public void setTag(String key, double value) {} + + @Override + public void setError(boolean error) {} + + @Override + public void setErrorMessage(String errorMessage) {} + + @Override + public void addThrowable(Throwable throwable) {} + + @Override + public void finish() {} +} diff --git a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsSpanFactory.java b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsSpanFactory.java new file mode 100644 index 00000000000..080aa41bd82 --- /dev/null +++ b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsSpanFactory.java @@ -0,0 +1,38 @@ +package datadog.trace.api.llmobs.noop; + +import datadog.trace.api.llmobs.LLMObs; +import datadog.trace.api.llmobs.LLMObsSpan; +import javax.annotation.Nullable; + +public class NoOpLLMObsSpanFactory implements LLMObs.LLMObsSpanFactory { + public static final NoOpLLMObsSpanFactory INSTANCE = new NoOpLLMObsSpanFactory(); + + public LLMObsSpan startLLMSpan( + String spanName, + String modelName, + String modelProvider, + @Nullable String mlApp, + @Nullable String sessionId) { + return NoOpLLMObsSpan.INSTANCE; + } + + public LLMObsSpan startAgentSpan( + String spanName, @Nullable String mlApp, @Nullable String sessionId) { + return NoOpLLMObsSpan.INSTANCE; + } + + public LLMObsSpan startToolSpan( + String spanName, @Nullable String mlApp, @Nullable String sessionId) { + return NoOpLLMObsSpan.INSTANCE; + } + + public LLMObsSpan startTaskSpan( + String spanName, @Nullable String mlApp, @Nullable String sessionId) { + return NoOpLLMObsSpan.INSTANCE; + } + + public LLMObsSpan startWorkflowSpan( + String spanName, @Nullable String mlApp, @Nullable String sessionId) { + return NoOpLLMObsSpan.INSTANCE; + } +} From 0a790c3fee627038afb0841c2856ee3113967e90 Mon Sep 17 00:00:00 2001 From: gary-huang Date: Wed, 7 May 2025 11:24:45 -0400 Subject: [PATCH 20/28] rename ID to Id according to java naming conventions --- .../datadog/trace/llmobs/LLMObsSystem.java | 20 +++++++++---------- .../trace/llmobs/domain/DDLLMObsSpan.java | 6 +++--- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java index d4b76c585e2..fbfef5c771f 100644 --- a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java @@ -48,11 +48,11 @@ public LLMObsSpan startLLMSpan( String modelName, String modelProvider, @Nullable String mlApp, - @Nullable String sessionID) { + @Nullable String sessionId) { DDLLMObsSpan span = new DDLLMObsSpan( - Tags.LLMOBS_LLM_SPAN_KIND, spanName, getMLApp(mlApp), sessionID, serviceName); + Tags.LLMOBS_LLM_SPAN_KIND, spanName, getMLApp(mlApp), sessionId, serviceName); if (modelName == null || modelName.isEmpty()) { modelName = CUSTOM_MODEL_VAL; @@ -68,30 +68,30 @@ public LLMObsSpan startLLMSpan( @Override public LLMObsSpan startAgentSpan( - String spanName, @Nullable String mlApp, @Nullable String sessionID) { + String spanName, @Nullable String mlApp, @Nullable String sessionId) { return new DDLLMObsSpan( - Tags.LLMOBS_AGENT_SPAN_KIND, spanName, getMLApp(mlApp), sessionID, serviceName); + Tags.LLMOBS_AGENT_SPAN_KIND, spanName, getMLApp(mlApp), sessionId, serviceName); } @Override public LLMObsSpan startToolSpan( - String spanName, @Nullable String mlApp, @Nullable String sessionID) { + String spanName, @Nullable String mlApp, @Nullable String sessionId) { return new DDLLMObsSpan( - Tags.LLMOBS_TOOL_SPAN_KIND, spanName, getMLApp(mlApp), sessionID, serviceName); + Tags.LLMOBS_TOOL_SPAN_KIND, spanName, getMLApp(mlApp), sessionId, serviceName); } @Override public LLMObsSpan startTaskSpan( - String spanName, @Nullable String mlApp, @Nullable String sessionID) { + String spanName, @Nullable String mlApp, @Nullable String sessionId) { return new DDLLMObsSpan( - Tags.LLMOBS_TASK_SPAN_KIND, spanName, getMLApp(mlApp), sessionID, serviceName); + Tags.LLMOBS_TASK_SPAN_KIND, spanName, getMLApp(mlApp), sessionId, serviceName); } @Override public LLMObsSpan startWorkflowSpan( - String spanName, @Nullable String mlApp, @Nullable String sessionID) { + String spanName, @Nullable String mlApp, @Nullable String sessionId) { return new DDLLMObsSpan( - Tags.LLMOBS_WORKFLOW_SPAN_KIND, spanName, getMLApp(mlApp), sessionID, serviceName); + Tags.LLMOBS_WORKFLOW_SPAN_KIND, spanName, getMLApp(mlApp), sessionId, serviceName); } private String getMLApp(String mlApp) { diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java index 640a61c9426..734bf1df526 100644 --- a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java @@ -43,7 +43,7 @@ public DDLLMObsSpan( @Nonnull String kind, String spanName, @Nonnull String mlApp, - String sessionID, + String sessionId, @Nonnull String serviceName) { if (null == spanName || spanName.isEmpty()) { @@ -60,8 +60,8 @@ public DDLLMObsSpan( this.span.setTag(SPAN_KIND, kind); this.spanKind = kind; this.span.setTag(LLMOBS_TAG_PREFIX + LLMObsTags.ML_APP, mlApp); - if (sessionID != null && !sessionID.isEmpty()) { - this.span.setTag(LLMOBS_TAG_PREFIX + LLMObsTags.SESSION_ID, sessionID); + if (sessionId != null && !sessionId.isEmpty()) { + this.span.setTag(LLMOBS_TAG_PREFIX + LLMObsTags.SESSION_ID, sessionId); } } From c1c38e4a29c14a5f4775a84d5c0cd2d9d654805a Mon Sep 17 00:00:00 2001 From: gary-huang Date: Wed, 14 May 2025 14:19:03 -0400 Subject: [PATCH 21/28] Undo change to integrations-core submodule --- dd-java-agent/agent-jmxfetch/integrations-core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dd-java-agent/agent-jmxfetch/integrations-core b/dd-java-agent/agent-jmxfetch/integrations-core index 5240f2a7cdc..3189af0e0ae 160000 --- a/dd-java-agent/agent-jmxfetch/integrations-core +++ b/dd-java-agent/agent-jmxfetch/integrations-core @@ -1 +1 @@ -Subproject commit 5240f2a7cdcabc6ae7787b9191b9189438671f3e +Subproject commit 3189af0e0ae840c9a4bab3131662c7fd6b0de7fb From 2be1b73b0e7d71e4843db933b01234e0b16a0453 Mon Sep 17 00:00:00 2001 From: Gary Huang Date: Wed, 7 May 2025 11:08:22 -0400 Subject: [PATCH 22/28] add APIs for llm obs sdk (#8135) * add APIs for llm obs * add llm message class to support llm spans * follow java convention of naming Id instead of ID * add codeowners --- .github/CODEOWNERS | 6 + dd-trace-api/build.gradle | 6 + .../java/datadog/trace/api/llmobs/LLMObs.java | 136 ++++++++++++++++ .../datadog/trace/api/llmobs/LLMObsSpan.java | 145 ++++++++++++++++++ .../trace/api/llmobs/noop/NoOpLLMObsSpan.java | 61 ++++++++ .../llmobs/noop/NoOpLLMObsSpanFactory.java | 38 +++++ 6 files changed, 392 insertions(+) create mode 100644 dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java create mode 100644 dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsSpan.java create mode 100644 dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsSpan.java create mode 100644 dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsSpanFactory.java diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 5983b5d0bd2..791377393d2 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -74,3 +74,9 @@ dd-trace-core/src/main/java/datadog/trace/core/datastreams @Dat dd-trace-core/src/test/groovy/datadog/trace/core/datastreams @DataDog/data-streams-monitoring internal-api/src/main/java/datadog/trace/api/datastreams @DataDog/data-streams-monitoring internal-api/src/test/groovy/datadog/trace/api/datastreams @DataDog/data-streams-monitoring + +# @DataDog/ml-observability +dd-trace-api/src/main/java/datadog/trace/api/llmobs/ @DataDog/ml-observability +dd-java-agent/agent-llmobs/ @DataDog/ml-observability +dd-trace-core/src/main/java/datadog/trace/llmobs/ @DataDog/ml-observability +dd-trace-core/src/test/groovy/datadog/trace/llmobs/ @DataDog/ml-observability diff --git a/dd-trace-api/build.gradle b/dd-trace-api/build.gradle index c4b00313208..4175700c6ce 100644 --- a/dd-trace-api/build.gradle +++ b/dd-trace-api/build.gradle @@ -32,6 +32,12 @@ excludedClassesCoverage += [ 'datadog.trace.api.profiling.ProfilingScope', 'datadog.trace.api.profiling.ProfilingContext', 'datadog.trace.api.profiling.ProfilingContextAttribute.NoOp', + 'datadog.trace.api.llmobs.LLMObs', + 'datadog.trace.api.llmobs.LLMObs.LLMMessage', + 'datadog.trace.api.llmobs.LLMObs.ToolCall', + 'datadog.trace.api.llmobs.LLMObsSpan', + 'datadog.trace.api.llmobs.noop.NoOpLLMObsSpan', + 'datadog.trace.api.llmobs.noop.NoOpLLMObsSpanFactory', 'datadog.trace.api.experimental.DataStreamsCheckpointer', 'datadog.trace.api.experimental.DataStreamsCheckpointer.NoOp', 'datadog.trace.api.experimental.DataStreamsContextCarrier', diff --git a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java new file mode 100644 index 00000000000..fd3e1f0a952 --- /dev/null +++ b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java @@ -0,0 +1,136 @@ +package datadog.trace.api.llmobs; + +import datadog.trace.api.llmobs.noop.NoOpLLMObsSpanFactory; +import java.util.List; +import java.util.Map; +import javax.annotation.Nullable; + +public class LLMObs { + protected LLMObs() {} + + protected static LLMObsSpanFactory SPAN_FACTORY = NoOpLLMObsSpanFactory.INSTANCE; + + public static LLMObsSpan startLLMSpan( + String spanName, + String modelName, + String modelProvider, + @Nullable String mlApp, + @Nullable String sessionId) { + + return SPAN_FACTORY.startLLMSpan(spanName, modelName, modelProvider, mlApp, sessionId); + } + + public static LLMObsSpan startAgentSpan( + String spanName, @Nullable String mlApp, @Nullable String sessionId) { + + return SPAN_FACTORY.startAgentSpan(spanName, mlApp, sessionId); + } + + public static LLMObsSpan startToolSpan( + String spanName, @Nullable String mlApp, @Nullable String sessionId) { + + return SPAN_FACTORY.startToolSpan(spanName, mlApp, sessionId); + } + + public static LLMObsSpan startTaskSpan( + String spanName, @Nullable String mlApp, @Nullable String sessionId) { + + return SPAN_FACTORY.startTaskSpan(spanName, mlApp, sessionId); + } + + public static LLMObsSpan startWorkflowSpan( + String spanName, @Nullable String mlApp, @Nullable String sessionId) { + + return SPAN_FACTORY.startWorkflowSpan(spanName, mlApp, sessionId); + } + + public interface LLMObsSpanFactory { + LLMObsSpan startLLMSpan( + String spanName, + String modelName, + String modelProvider, + @Nullable String mlApp, + @Nullable String sessionId); + + LLMObsSpan startAgentSpan(String spanName, @Nullable String mlApp, @Nullable String sessionId); + + LLMObsSpan startToolSpan(String spanName, @Nullable String mlApp, @Nullable String sessionId); + + LLMObsSpan startTaskSpan(String spanName, @Nullable String mlApp, @Nullable String sessionId); + + LLMObsSpan startWorkflowSpan( + String spanName, @Nullable String mlApp, @Nullable String sessionId); + } + + public static class ToolCall { + private String name; + private String type; + private String toolId; + private Map arguments; + + public static ToolCall from( + String name, String type, String toolId, Map arguments) { + return new ToolCall(name, type, toolId, arguments); + } + + private ToolCall(String name, String type, String toolId, Map arguments) { + this.name = name; + this.type = type; + this.toolId = toolId; + this.arguments = arguments; + } + + public String getName() { + return name; + } + + public String getType() { + return type; + } + + public String getToolId() { + return toolId; + } + + public Map getArguments() { + return arguments; + } + } + + public static class LLMMessage { + private String role; + private String content; + private List toolCalls; + + public static LLMMessage from(String role, String content, List toolCalls) { + return new LLMMessage(role, content, toolCalls); + } + + public static LLMMessage from(String role, String content) { + return new LLMMessage(role, content); + } + + private LLMMessage(String role, String content, List toolCalls) { + this.role = role; + this.content = content; + this.toolCalls = toolCalls; + } + + private LLMMessage(String role, String content) { + this.role = role; + this.content = content; + } + + public String getRole() { + return role; + } + + public String getContent() { + return content; + } + + public List getToolCalls() { + return toolCalls; + } + } +} diff --git a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsSpan.java b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsSpan.java new file mode 100644 index 00000000000..80668eabd57 --- /dev/null +++ b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsSpan.java @@ -0,0 +1,145 @@ +package datadog.trace.api.llmobs; + +import java.util.List; +import java.util.Map; + +/** This interface represent an individual LLM Obs span. */ +public interface LLMObsSpan { + + /** + * Annotate the span with inputs and outputs for LLM spans + * + * @param inputMessages The input messages of the span in the form of a list + * @param outputMessages The output messages of the span in the form of a list + */ + void annotateIO(List inputMessages, List outputMessages); + + /** + * Annotate the span with inputs and outputs + * + * @param inputData The input data of the span in the form of a string + * @param outputData The output data of the span in the form of a string + */ + void annotateIO(String inputData, String outputData); + + /** + * Annotate the span with metadata + * + * @param metadata A map of JSON serializable key-value pairs that contains metadata information + * relevant to the input or output operation described by the span + */ + void setMetadata(Map metadata); + + /** + * Annotate the span with metrics + * + * @param metrics A map of JSON serializable keys and numeric values that users can add as metrics + * relevant to the operation described by the span (input_tokens, output_tokens, total_tokens, + * etc.). + */ + void setMetrics(Map metrics); + + /** + * Annotate the span with a single metric key value pair for the span’s context (number of tokens + * document length, etc). + * + * @param key the name of the metric + * @param value the value of the metric + */ + void setMetric(CharSequence key, int value); + + /** + * Annotate the span with a single metric key value pair for the span’s context (number of tokens + * document length, etc). + * + * @param key the name of the metric + * @param value the value of the metric + */ + void setMetric(CharSequence key, long value); + + /** + * Annotate the span with a single metric key value pair for the span’s context (number of tokens + * document length, etc). + * + * @param key the name of the metric + * @param value the value of the metric + */ + void setMetric(CharSequence key, double value); + + /** + * Annotate the span with tags + * + * @param tags An map of JSON serializable key-value pairs that users can add as tags regarding + * the span’s context (session, environment, system, versioning, etc.). + */ + void setTags(Map tags); + + /** + * Annotate the span with a single tag key value pair as a tag regarding the span’s context + * (session, environment, system, versioning, etc.). + * + * @param key the key of the tag + * @param value the value of the tag + */ + void setTag(String key, String value); + + /** + * Annotate the span with a single tag key value pair as a tag regarding the span’s context + * (session, environment, system, versioning, etc.). + * + * @param key the key of the tag + * @param value the value of the tag + */ + void setTag(String key, boolean value); + + /** + * Annotate the span with a single tag key value pair as a tag regarding the span’s context + * (session, environment, system, versioning, etc.). + * + * @param key the key of the tag + * @param value the value of the tag + */ + void setTag(String key, int value); + + /** + * Annotate the span with a single tag key value pair as a tag regarding the span’s context + * (session, environment, system, versioning, etc.). + * + * @param key the key of the tag + * @param value the value of the tag + */ + void setTag(String key, long value); + + /** + * Annotate the span with a single tag key value pair as a tag regarding the span’s context + * (session, environment, system, versioning, etc.). + * + * @param key the key of the tag + * @param value the value of the tag + */ + void setTag(String key, double value); + + /** + * Annotate the span to indicate that an error occurred + * + * @param error whether an error occurred + */ + void setError(boolean error); + + /** + * Annotate the span with an error message + * + * @param errorMessage the message of the error + */ + void setErrorMessage(String errorMessage); + + /** + * Annotate the span with a throwable + * + * @param throwable the errored throwable + */ + void addThrowable(Throwable throwable); + + /** Finishes (closes) a span */ + void finish(); +} diff --git a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsSpan.java b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsSpan.java new file mode 100644 index 00000000000..a1b160616e7 --- /dev/null +++ b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsSpan.java @@ -0,0 +1,61 @@ +package datadog.trace.api.llmobs.noop; + +import datadog.trace.api.llmobs.LLMObs; +import datadog.trace.api.llmobs.LLMObsSpan; +import java.util.List; +import java.util.Map; + +public class NoOpLLMObsSpan implements LLMObsSpan { + public static final LLMObsSpan INSTANCE = new NoOpLLMObsSpan(); + + @Override + public void annotateIO(List inputData, List outputData) {} + + @Override + public void annotateIO(String inputData, String outputData) {} + + @Override + public void setMetadata(Map metadata) {} + + @Override + public void setMetrics(Map metrics) {} + + @Override + public void setMetric(CharSequence key, int value) {} + + @Override + public void setMetric(CharSequence key, long value) {} + + @Override + public void setMetric(CharSequence key, double value) {} + + @Override + public void setTags(Map tags) {} + + @Override + public void setTag(String key, String value) {} + + @Override + public void setTag(String key, boolean value) {} + + @Override + public void setTag(String key, int value) {} + + @Override + public void setTag(String key, long value) {} + + @Override + public void setTag(String key, double value) {} + + @Override + public void setError(boolean error) {} + + @Override + public void setErrorMessage(String errorMessage) {} + + @Override + public void addThrowable(Throwable throwable) {} + + @Override + public void finish() {} +} diff --git a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsSpanFactory.java b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsSpanFactory.java new file mode 100644 index 00000000000..080aa41bd82 --- /dev/null +++ b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsSpanFactory.java @@ -0,0 +1,38 @@ +package datadog.trace.api.llmobs.noop; + +import datadog.trace.api.llmobs.LLMObs; +import datadog.trace.api.llmobs.LLMObsSpan; +import javax.annotation.Nullable; + +public class NoOpLLMObsSpanFactory implements LLMObs.LLMObsSpanFactory { + public static final NoOpLLMObsSpanFactory INSTANCE = new NoOpLLMObsSpanFactory(); + + public LLMObsSpan startLLMSpan( + String spanName, + String modelName, + String modelProvider, + @Nullable String mlApp, + @Nullable String sessionId) { + return NoOpLLMObsSpan.INSTANCE; + } + + public LLMObsSpan startAgentSpan( + String spanName, @Nullable String mlApp, @Nullable String sessionId) { + return NoOpLLMObsSpan.INSTANCE; + } + + public LLMObsSpan startToolSpan( + String spanName, @Nullable String mlApp, @Nullable String sessionId) { + return NoOpLLMObsSpan.INSTANCE; + } + + public LLMObsSpan startTaskSpan( + String spanName, @Nullable String mlApp, @Nullable String sessionId) { + return NoOpLLMObsSpan.INSTANCE; + } + + public LLMObsSpan startWorkflowSpan( + String spanName, @Nullable String mlApp, @Nullable String sessionId) { + return NoOpLLMObsSpan.INSTANCE; + } +} From d01034ade6e548c9ff5a222669300ec639980d65 Mon Sep 17 00:00:00 2001 From: gary-huang Date: Wed, 14 May 2025 15:54:49 -0400 Subject: [PATCH 23/28] fix build gradle --- dd-java-agent/agent-llmobs/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dd-java-agent/agent-llmobs/build.gradle b/dd-java-agent/agent-llmobs/build.gradle index 071840c93ec..5e4d7cd6a78 100644 --- a/dd-java-agent/agent-llmobs/build.gradle +++ b/dd-java-agent/agent-llmobs/build.gradle @@ -9,7 +9,7 @@ buildscript { } plugins { - id 'com.github.johnrengelman.shadow' + id 'com.gradleup.shadow' id 'java-test-fixtures' } From 23d6923f3d4fbe40eab2e7fa4bca479d772060b0 Mon Sep 17 00:00:00 2001 From: gary-huang Date: Wed, 14 May 2025 15:58:27 -0400 Subject: [PATCH 24/28] rm empty line --- dd-trace-api/src/main/java/datadog/trace/api/DDSpanTypes.java | 1 - 1 file changed, 1 deletion(-) diff --git a/dd-trace-api/src/main/java/datadog/trace/api/DDSpanTypes.java b/dd-trace-api/src/main/java/datadog/trace/api/DDSpanTypes.java index 4e41df93738..04f9738c643 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/DDSpanTypes.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/DDSpanTypes.java @@ -36,7 +36,6 @@ public class DDSpanTypes { public static final String PROTOBUF = "protobuf"; public static final String MULE = "mule"; - public static final String VALKEY = "valkey"; public static final String WEBSOCKET = "websocket"; From 9ac1ab7bb3834b4eaa7b1ad5928b9b4d134ea33e Mon Sep 17 00:00:00 2001 From: gary-huang Date: Wed, 14 May 2025 16:37:40 -0400 Subject: [PATCH 25/28] fix test --- .../groovy/datadog/trace/llmobs/domain/DDLLMObsSpanTest.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dd-java-agent/agent-llmobs/src/test/groovy/datadog/trace/llmobs/domain/DDLLMObsSpanTest.groovy b/dd-java-agent/agent-llmobs/src/test/groovy/datadog/trace/llmobs/domain/DDLLMObsSpanTest.groovy index f17a4382ded..a6c8534be2b 100644 --- a/dd-java-agent/agent-llmobs/src/test/groovy/datadog/trace/llmobs/domain/DDLLMObsSpanTest.groovy +++ b/dd-java-agent/agent-llmobs/src/test/groovy/datadog/trace/llmobs/domain/DDLLMObsSpanTest.groovy @@ -256,7 +256,7 @@ class DDLLMObsSpanTest extends DDSpecification{ def toolCall = spanOutputMsg.getToolCalls().get(0) assert toolCall.getName().equals("weather-tool") assert toolCall.getType().equals("function") - assert toolCall.getToolID().equals("6176241000") + assert toolCall.getToolId().equals("6176241000") def expectedToolArgs = Maps.of("location", "paris") assert toolCall.getArguments().equals(expectedToolArgs) } From e382f227dc00cb59badea68e584f81296e308958 Mon Sep 17 00:00:00 2001 From: gary-huang Date: Wed, 14 May 2025 16:44:07 -0400 Subject: [PATCH 26/28] wip add test --- .../ddintake/LLMObsSpanMapperTest.groovy | 588 ++++++++++++++++++ 1 file changed, 588 insertions(+) create mode 100644 dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy diff --git a/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy new file mode 100644 index 00000000000..bb96ef61085 --- /dev/null +++ b/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy @@ -0,0 +1,588 @@ +package datadog.trace.llmobs.writer.ddintake + +import datadog.communication.serialization.ByteBufferConsumer +import datadog.communication.serialization.FlushingBuffer +import datadog.communication.serialization.msgpack.MsgPackWriter +import datadog.trace.api.DDSpanId +import datadog.trace.api.DDTags +import datadog.trace.api.DDTraceId +import datadog.trace.api.llmobs.LLMObs +import datadog.trace.bootstrap.instrumentation.api.Tags +import datadog.trace.common.writer.ListWriter +import datadog.trace.core.test.DDCoreSpecification +import java.nio.ByteBuffer +import org.msgpack.core.MessagePack +import org.msgpack.core.buffer.ArrayBufferInput + +class LLMObsSpanMapperTest extends DDCoreSpecification { + + def "serialize LLM obs span with id #value as int"() { + setup: + def writer = new ListWriter() + def tracer = tracerBuilder().writer(writer).build() + def traceId = DDTraceId.from(value) + def spanId = DDSpanId.from(value) + def span = createLLMSpan("test-span", "gpt-4", "openai") + CaptureBuffer capture = new CaptureBuffer() + def packer = new MsgPackWriter(new FlushingBuffer(1024, capture)) + def mapper = new LLMObsSpanMapper() + packer.format(Collections.singletonList(span), mapper) + packer.flush() + def unpacker = MessagePack.newDefaultUnpacker(new ArrayBufferInput(capture.bytes)) + int traceCount = capture.messageCount + int spanCount = unpacker.unpackArrayHeader() + + expect: + traceCount == 1 + spanCount == 1 + + // Check the top-level map structure + int topLevelSize = unpacker.unpackMapHeader() + topLevelSize == 3 + + // Check event_type + String eventTypeKey = unpacker.unpackString() + eventTypeKey == "event_type" + String eventTypeValue = unpacker.unpackString() + eventTypeValue == "span" + + // Check stage + String stageKey = unpacker.unpackString() + stageKey == "_dd.stage" + String stageValue = unpacker.unpackString() + stageValue == "raw" + + // Check spans + String spansKey = unpacker.unpackString() + spansKey == "spans" + int spansArraySize = unpacker.unpackArrayHeader() + spansArraySize == 1 + + // Check span details + int spanMapSize = unpacker.unpackMapHeader() + spanMapSize == 11 + + // Verify span_id + String spanIdKey = unpacker.unpackString() + spanIdKey == "span_id" + String spanIdValue = unpacker.unpackString() + spanIdValue == String.valueOf(spanId) + + // Verify trace_id + String traceIdKey = unpacker.unpackString() + traceIdKey == "trace_id" + String traceIdValue = unpacker.unpackString() + traceIdValue == traceId.toHexString() + + // Skip the rest of the span fields + for (int i = 0; i < spanMapSize - 2; i++) { + unpacker.unpackString() // key + unpacker.unpackValue() // value + } + + cleanup: + tracer.close() + + where: + value | _ + "0" | _ + } + + def "serialize LLM obs span with tags and metrics"() { + setup: + def writer = new ListWriter() + def tracer = tracerBuilder().writer(writer).build() + def span = createLLMSpan("test-span", "gpt-4", "openai") + + // Add LLM obs tags + span.setTag("_ml_obs_tag.input", "test input") + span.setTag("_ml_obs_tag.output", "test output") + span.setTag("_ml_obs_tag.model.name", "gpt-4") + span.setTag("_ml_obs_tag.model.provider", "openai") + span.setTag("_ml_obs_tag.model.version", "1.0") + span.setTag("_ml_obs_tag.metadata", ["key1": "value1", "key2": "value2"]) + + // Add LLM obs metrics + span.setMetric("tokens.prompt", 10) + span.setMetric("tokens.completion", 20) + span.setMetric("latency", 100.5) + + CaptureBuffer capture = new CaptureBuffer() + def packer = new MsgPackWriter(new FlushingBuffer(1024, capture)) + def mapper = new LLMObsSpanMapper() + packer.format(Collections.singletonList(span), mapper) + packer.flush() + def unpacker = MessagePack.newDefaultUnpacker(new ArrayBufferInput(capture.bytes)) + int traceCount = capture.messageCount + int spanCount = unpacker.unpackArrayHeader() + + expect: + traceCount == 1 + spanCount == 1 + + // Skip to the span details + unpacker.unpackMapHeader() // top-level map + unpacker.unpackString() // event_type key + unpacker.unpackString() // event_type value + unpacker.unpackString() // stage key + unpacker.unpackString() // stage value + unpacker.unpackString() // spans key + unpacker.unpackArrayHeader() // spans array + unpacker.unpackMapHeader() // span map + + // Skip to metrics + for (int i = 0; i < 8; i++) { + unpacker.unpackString() // key + unpacker.unpackValue() // value + } + + // Check metrics + String metricsKey = unpacker.unpackString() + metricsKey == "metrics" + int metricsSize = unpacker.unpackMapHeader() + metricsSize == 3 + + Map metrics = [:] + for (int i = 0; i < metricsSize; i++) { + String key = unpacker.unpackString() + double value = unpacker.unpackDouble() + metrics[key] = value + } + + metrics["tokens.prompt"] == 10.0 + metrics["tokens.completion"] == 20.0 + metrics["latency"] == 100.5 + + // Check tags + String tagsKey = unpacker.unpackString() + tagsKey == "tags" + int tagsSize = unpacker.unpackArrayHeader() + tagsSize == 1 // Only language:jvm + + String languageTag = unpacker.unpackString() + languageTag == "language:jvm" + + // Check meta + String metaKey = unpacker.unpackString() + metaKey == "meta" + int metaSize = unpacker.unpackMapHeader() + metaSize == 7 // span.kind + 6 remapped tags + + // Check span.kind + String spanKindKey = unpacker.unpackString() + spanKindKey == "span.kind" + String spanKindValue = unpacker.unpackString() + spanKindValue == Tags.LLMOBS_LLM_SPAN_KIND + + // Check remapped tags + Map meta = [:] + for (int i = 0; i < metaSize - 1; i++) { + String key = unpacker.unpackString() + Object value = unpacker.unpackValue() + meta[key] = value + } + + meta["input"] == "test input" + meta["output"] == "test output" + meta["model.name"] == "gpt-4" + meta["model.provider"] == "openai" + meta["model.version"] == "1.0" + + // Check metadata + meta.containsKey("metadata") + Map metadata = meta["metadata"] as Map + metadata["key1"] == "value1" + metadata["key2"] == "value2" + + cleanup: + tracer.close() + } + + def "serialize LLM obs span with LLM messages"() { + setup: + def writer = new ListWriter() + def tracer = tracerBuilder().writer(writer).build() + def span = createLLMSpan("test-span", "gpt-4", "openai") + + // Create LLM messages + List messages = [ + LLMObs.LLMMessage.from("user", "Hello, how are you?"), + LLMObs.LLMMessage.from("system", "Answer the user truthfully") + ] + + span.annotateIO(messages, Collections.singletonList( + LLMObs.LLMMessage.from("assistant", "I'm doing well, thank you for asking!") + )) + + CaptureBuffer capture = new CaptureBuffer() + def packer = new MsgPackWriter(new FlushingBuffer(1024, capture)) + def mapper = new LLMObsSpanMapper() + packer.format(Collections.singletonList(span), mapper) + packer.flush() + def unpacker = MessagePack.newDefaultUnpacker(new ArrayBufferInput(capture.bytes)) + int traceCount = capture.messageCount + int spanCount = unpacker.unpackArrayHeader() + + expect: + traceCount == 1 + spanCount == 1 + + // Skip to the meta section + unpacker.unpackMapHeader() // top-level map + unpacker.unpackString() // event_type key + unpacker.unpackString() // event_type value + unpacker.unpackString() // stage key + unpacker.unpackString() // stage value + unpacker.unpackString() // spans key + unpacker.unpackArrayHeader() // spans array + unpacker.unpackMapHeader() // span map + + // Skip to meta + for (int i = 0; i < 10; i++) { + unpacker.unpackString() // key + unpacker.unpackValue() // value + } + + // Check meta + String metaKey = unpacker.unpackString() + metaKey == "meta" + int metaSize = unpacker.unpackMapHeader() + metaSize == 3 // span.kind + input + output + + // Check span.kind + String spanKindKey = unpacker.unpackString() + spanKindKey == "span.kind" + String spanKindValue = unpacker.unpackString() + spanKindValue == Tags.LLMOBS_LLM_SPAN_KIND + + // Check input messages + String inputKey = unpacker.unpackString() + inputKey == "input.messages" + int inputMessagesSize = unpacker.unpackArrayHeader() + inputMessagesSize == 2 + + // Check first message + int firstMessageSize = unpacker.unpackMapHeader() + firstMessageSize == 2 + + String userRoleKey = unpacker.unpackString() + userRoleKey == "role" + String userRoleValue = unpacker.unpackString() + userRoleValue == "user" + + String userContentKey = unpacker.unpackString() + userContentKey == "content" + String userContentValue = unpacker.unpackString() + userContentValue == "Hello, how are you?" + + // Check second message + int secondMessageSize = unpacker.unpackMapHeader() + secondMessageSize == 2 + + String sysRoleKey = unpacker.unpackString() + sysRoleKey == "role" + String sysRoleValue = unpacker.unpackString() + sysRoleValue == "system" + + String sysContentKey = unpacker.unpackString() + sysContentKey == "content" + String sysContentValue = unpacker.unpackString() + sysContentValue == "I'm doing well, thank you for asking!" + + + // Check output messages + String outputKey = unpacker.unpackString() + outputKey == "output.messages" + int outputMessagesSize = unpacker.unpackArrayHeader() + outputMessagesSize == 1 + + // Check output message + int outputMessageSize = unpacker.unpackMapHeader() + outputMessageSize == 2 + + String assistantRoleKey = unpacker.unpackString() + assistantRoleKey == "role" + String assistantRoleValue = unpacker.unpackString() + assistantRoleValue == "assistant" + + String assistantContentKey = unpacker.unpackString() + assistantContentKey == "content" + String assistantContentValue = unpacker.unpackString() + assistantContentValue == "I'm doing well, thank you for asking!" + + cleanup: + tracer.close() + } +// +// def "serialize LLM obs span with tool calls"() { +// setup: +// def writer = new ListWriter() +// def tracer = tracerBuilder().writer(writer).build() +// def span = createLLMSpan("test-span", "gpt-4", "openai") +// +// // Create LLM messages with tool calls +// List toolCalls = [ +// LLMObs.ToolCall.from("search", "function", "search-123", ["query": "test query"]), +// LLMObs.ToolCall.from("calculator", "function", "calc-456", ["operation": "add", "a": 5, "b": 3]) +// ] +// +// List messages = [ +// LLMObs.LLMMessage.from("assistant", "Let me search for that information.", toolCalls) +// ] +// +// // Add LLM obs tags with messages +// span.annotateIO(null, messages) +// +// CaptureBuffer capture = new CaptureBuffer() +// def packer = new MsgPackWriter(new FlushingBuffer(1024, capture)) +// def mapper = new LLMObsSpanMapper() +// packer.format(Collections.singletonList(span), mapper) +// packer.flush() +// def unpacker = MessagePack.newDefaultUnpacker(new ArrayBufferInput(capture.bytes)) +// int traceCount = capture.messageCount +// int spanCount = unpacker.unpackArrayHeader() +// +// expect: +// traceCount == 1 +// spanCount == 1 +// +// // Skip to the meta section +// unpacker.unpackMapHeader() // top-level map +// unpacker.unpackString() // event_type key +// unpacker.unpackString() // event_type value +// unpacker.unpackString() // stage key +// unpacker.unpackString() // stage value +// unpacker.unpackString() // spans key +// unpacker.unpackArrayHeader() // spans array +// unpacker.unpackMapHeader() // span map +// +// // Skip to meta +// for (int i = 0; i < 10; i++) { +// unpacker.unpackString() // key +// unpacker.unpackValue() // value +// } +// +// // Check meta +// String metaKey = unpacker.unpackString() +// metaKey == "meta" +// int metaSize = unpacker.unpackMapHeader() +// metaSize == 2 // span.kind + output +// +// // Check span.kind +// String spanKindKey = unpacker.unpackString() +// spanKindKey == "span.kind" +// String spanKindValue = unpacker.unpackString() +// spanKindValue == Tags.LLMOBS_LLM_SPAN_KIND +// +// // Check output messages +// String outputKey = unpacker.unpackString() +// outputKey == "output.messages" +// int outputMessagesSize = unpacker.unpackArrayHeader() +// outputMessagesSize == 1 +// +// // Check output message +// int outputMessageSize = unpacker.unpackMapHeader() +// outputMessageSize == 3 // role, content, tool_calls +// +// String roleKey = unpacker.unpackString() +// roleKey == "role" +// String roleValue = unpacker.unpackString() +// roleValue == "assistant" +// +// String contentKey = unpacker.unpackString() +// contentKey == "content" +// String contentValue = unpacker.unpackString() +// contentValue == "Let me search for that information." +// +// // Check tool calls +// String toolCallsKey = unpacker.unpackString() +// toolCallsKey == "tool_calls" +// int toolCallsSize = unpacker.unpackArrayHeader() +// toolCallsSize == 2 +// +// // Check first tool call +// int firstToolCallSize = unpacker.unpackMapHeader() +// firstToolCallSize == 4 // name, type, tool_id, arguments +// +// String nameKey = unpacker.unpackString() +// nameKey == "name" +// String nameValue = unpacker.unpackString() +// nameValue == "search" +// +// String typeKey = unpacker.unpackString() +// typeKey == "type" +// String typeValue = unpacker.unpackString() +// typeValue == "function" +// +// String toolIdKey = unpacker.unpackString() +// toolIdKey == "tool_id" +// String toolIdValue = unpacker.unpackString() +// toolIdValue == "search-123" +// +// String argumentsKey = unpacker.unpackString() +// argumentsKey == "arguments" +// int argumentsSize = unpacker.unpackMapHeader() +// argumentsSize == 1 +// +// String queryKey = unpacker.unpackString() +// queryKey == "query" +// String queryValue = unpacker.unpackString() +// queryValue == "test query" +// +// // Check second tool call +// int secondToolCallSize = unpacker.unpackMapHeader() +// secondToolCallSize == 4 // name, type, tool_id, arguments +// +// nameKey = unpacker.unpackString() +// nameKey == "name" +// nameValue = unpacker.unpackString() +// nameValue == "calculator" +// +// typeKey = unpacker.unpackString() +// typeKey == "type" +// typeValue = unpacker.unpackString() +// typeValue == "function" +// +// toolIdKey = unpacker.unpackString() +// toolIdKey == "tool_id" +// toolIdValue = unpacker.unpackString() +// toolIdValue == "calc-456" +// +// argumentsKey = unpacker.unpackString() +// argumentsKey == "arguments" +// argumentsSize = unpacker.unpackMapHeader() +// argumentsSize == 3 +// +// String operationKey = unpacker.unpackString() +// operationKey == "operation" +// String operationValue = unpacker.unpackString() +// operationValue == "add" +// +// String aKey = unpacker.unpackString() +// aKey == "a" +// int aValue = unpacker.unpackInt() +// aValue == 5 +// +// String bKey = unpacker.unpackString() +// bKey == "b" +// int bValue = unpacker.unpackInt() +// bValue == 3 +// +// cleanup: +// tracer.close() +// } + + def "serialize LLM obs span with error information"() { + setup: + def writer = new ListWriter() + def tracer = tracerBuilder().writer(writer).build() + def span = createLLMSpan("test-span", "gpt-4", "openai") + + // Add error information + span.setErrorMessage("Something went wrong") + span.setError(true) + + CaptureBuffer capture = new CaptureBuffer() + def packer = new MsgPackWriter(new FlushingBuffer(1024, capture)) + def mapper = new LLMObsSpanMapper() + packer.format(Collections.singletonList(span), mapper) + packer.flush() + def unpacker = MessagePack.newDefaultUnpacker(new ArrayBufferInput(capture.bytes)) + int traceCount = capture.messageCount + int spanCount = unpacker.unpackArrayHeader() + + expect: + traceCount == 1 + spanCount == 1 + + // Skip to the span details + unpacker.unpackMapHeader() // top-level map + unpacker.unpackString() // event_type key + unpacker.unpackString() // event_type value + unpacker.unpackString() // stage key + unpacker.unpackString() // stage value + unpacker.unpackString() // spans key + unpacker.unpackArrayHeader() // spans array + unpacker.unpackMapHeader() // span map + + // Check error field + for (int i = 0; i < 6; i++) { + unpacker.unpackString() // key + unpacker.unpackValue() // value + } + + String errorKey = unpacker.unpackString() + errorKey == "error" + int errorValue = unpacker.unpackInt() + errorValue == 1 + + // Check status field + String statusKey = unpacker.unpackString() + statusKey == "status" + String statusValue = unpacker.unpackString() + statusValue == "error" + + // Skip to meta + for (int i = 0; i < 2; i++) { + unpacker.unpackString() // key + unpacker.unpackValue() // value + } + + // Check meta + String metaKey = unpacker.unpackString() + metaKey == "meta" + int metaSize = unpacker.unpackMapHeader() + metaSize == 2 // span.kind + error.msg + + // Check error fields + Map errorFields = [:] + for (int i = 0; i < metaSize; i++) { + String key = unpacker.unpackString() + String value = unpacker.unpackString() + errorFields[key] = value + } + + errorFields[DDTags.ERROR_MSG] == "Something went wrong" + + cleanup: + tracer.close() + } + + private class CaptureBuffer implements ByteBufferConsumer { + + private byte[] bytes + int messageCount + + @Override + void accept(int messageCount, ByteBuffer buffer) { + this.messageCount = messageCount + this.bytes = new byte[buffer.limit() - buffer.position()] + buffer.get(bytes) + } + } + + def createLLMSpan(String spanName, String modelName, String modelProvider) { + // Use the LLMObs API to create a span + return LLMObs.startLLMSpan(spanName, modelName, modelProvider, "test-app", "test-session") + } + + def createAgentSpan(String spanName) { + // Use the LLMObs API to create an agent span + return LLMObs.startAgentSpan(spanName, "test-app", "test-session") + } + + def createToolSpan(String spanName) { + // Use the LLMObs API to create a tool span + return LLMObs.startToolSpan(spanName, "test-app", "test-session") + } + + def createTaskSpan(String spanName) { + // Use the LLMObs API to create a task span + return LLMObs.startTaskSpan(spanName, "test-app", "test-session") + } + + def createWorkflowSpan(String spanName) { + // Use the LLMObs API to create a workflow span + return LLMObs.startWorkflowSpan(spanName, "test-app", "test-session") + } +} From 77291d8b1a074a0462a64df585526eadbd514ab5 Mon Sep 17 00:00:00 2001 From: Gary Huang Date: Wed, 4 Jun 2025 15:35:47 -0400 Subject: [PATCH 27/28] implement LLM Obs SDK spans APIs (#8390) * add APIs for llm obs * add llm message class to support llm spans * add llm message class to support llm spans * impl llmobs agent and llmobs apis * support llm messages with tool calls * handle default model name and provider * rm unneeded file * spotless * add APIs for llm obs sdk (#8135) * add APIs for llm obs * add llm message class to support llm spans * follow java convention of naming Id instead of ID * add codeowners * rename ID to Id according to java naming conventions * Undo change to integrations-core submodule * fix build gradle * rm empty line * fix test --- .../communication/BackendApiFactory.java | 1 + .../java/datadog/trace/bootstrap/Agent.java | 46 ++- dd-java-agent/agent-llmobs/build.gradle | 42 +++ .../datadog/trace/llmobs/LLMObsSystem.java | 104 +++++++ .../trace/llmobs/domain/DDLLMObsSpan.java | 276 ++++++++++++++++++ .../trace/llmobs/domain/LLMObsInternal.java | 10 + .../llmobs/domain/DDLLMObsSpanTest.groovy | 267 +++++++++++++++++ dd-java-agent/build.gradle | 1 + .../java/datadog/trace/api/DDSpanTypes.java | 2 + .../datadog/trace/api/llmobs/LLMObsTags.java | 15 + .../main/java/datadog/trace/api/Config.java | 28 ++ .../bootstrap/instrumentation/api/Tags.java | 6 + settings.gradle | 3 + 13 files changed, 800 insertions(+), 1 deletion(-) create mode 100644 dd-java-agent/agent-llmobs/build.gradle create mode 100644 dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java create mode 100644 dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java create mode 100644 dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/LLMObsInternal.java create mode 100644 dd-java-agent/agent-llmobs/src/test/groovy/datadog/trace/llmobs/domain/DDLLMObsSpanTest.groovy create mode 100644 dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsTags.java diff --git a/communication/src/main/java/datadog/communication/BackendApiFactory.java b/communication/src/main/java/datadog/communication/BackendApiFactory.java index bebb7b42828..f3382792baa 100644 --- a/communication/src/main/java/datadog/communication/BackendApiFactory.java +++ b/communication/src/main/java/datadog/communication/BackendApiFactory.java @@ -72,6 +72,7 @@ private HttpUrl getAgentlessUrl(Intake intake) { public enum Intake { API("api", "v2", Config::isCiVisibilityAgentlessEnabled, Config::getCiVisibilityAgentlessUrl), + LLMOBS_API("api", "v2", Config::isLlmObsAgentlessEnabled, Config::getLlMObsAgentlessUrl), LOGS( "http-intake.logs", "v2", diff --git a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java index 00b54848832..c1246ff6f7c 100644 --- a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java +++ b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java @@ -25,6 +25,7 @@ import datadog.trace.api.config.GeneralConfig; import datadog.trace.api.config.IastConfig; import datadog.trace.api.config.JmxFetchConfig; +import datadog.trace.api.config.LlmObsConfig; import datadog.trace.api.config.ProfilingConfig; import datadog.trace.api.config.RemoteConfigConfig; import datadog.trace.api.config.TraceInstrumentationConfig; @@ -41,6 +42,7 @@ import datadog.trace.bootstrap.instrumentation.api.AgentTracer; import datadog.trace.bootstrap.instrumentation.api.AgentTracer.TracerAPI; import datadog.trace.bootstrap.instrumentation.api.ProfilingContextIntegration; +import datadog.trace.bootstrap.instrumentation.api.WriterConstants; import datadog.trace.bootstrap.instrumentation.jfr.InstrumentationBasedProfiling; import datadog.trace.util.AgentTaskScheduler; import datadog.trace.util.AgentThreadFactory.AgentThread; @@ -109,7 +111,9 @@ private enum AgentFeature { EXCEPTION_REPLAY(DebuggerConfig.EXCEPTION_REPLAY_ENABLED, false), CODE_ORIGIN(TraceInstrumentationConfig.CODE_ORIGIN_FOR_SPANS_ENABLED, false), DATA_JOBS(GeneralConfig.DATA_JOBS_ENABLED, false), - AGENTLESS_LOG_SUBMISSION(GeneralConfig.AGENTLESS_LOG_SUBMISSION_ENABLED, false); + AGENTLESS_LOG_SUBMISSION(GeneralConfig.AGENTLESS_LOG_SUBMISSION_ENABLED, false), + LLMOBS(LlmObsConfig.LLMOBS_ENABLED, false), + LLMOBS_AGENTLESS(LlmObsConfig.LLMOBS_AGENTLESS_ENABLED, false); private final String configKey; private final String systemProp; @@ -156,6 +160,8 @@ public boolean isEnabledByDefault() { private static boolean iastFullyDisabled; private static boolean cwsEnabled = false; private static boolean ciVisibilityEnabled = false; + private static boolean llmObsEnabled = false; + private static boolean llmObsAgentlessEnabled = false; private static boolean usmEnabled = false; private static boolean telemetryEnabled = true; private static boolean dynamicInstrumentationEnabled = false; @@ -290,6 +296,25 @@ public static void start( exceptionReplayEnabled = isFeatureEnabled(AgentFeature.EXCEPTION_REPLAY); codeOriginEnabled = isFeatureEnabled(AgentFeature.CODE_ORIGIN); agentlessLogSubmissionEnabled = isFeatureEnabled(AgentFeature.AGENTLESS_LOG_SUBMISSION); + llmObsEnabled = isFeatureEnabled(AgentFeature.LLMOBS); + + // setup writers when llmobs is enabled to accomodate apm and llmobs + if (llmObsEnabled) { + // for llm obs spans, use agent proxy by default, apm spans will use agent writer + setSystemPropertyDefault( + propertyNameToSystemPropertyName(TracerConfig.WRITER_TYPE), + WriterConstants.MULTI_WRITER_TYPE + + ":" + + WriterConstants.DD_INTAKE_WRITER_TYPE + + "," + + WriterConstants.DD_AGENT_WRITER_TYPE); + if (llmObsAgentlessEnabled) { + // use API writer only + setSystemPropertyDefault( + propertyNameToSystemPropertyName(TracerConfig.WRITER_TYPE), + WriterConstants.DD_INTAKE_WRITER_TYPE); + } + } patchJPSAccess(inst); @@ -597,6 +622,7 @@ public void execute() { maybeStartAppSec(scoClass, sco); maybeStartCiVisibility(instrumentation, scoClass, sco); + maybeStartLLMObs(instrumentation, scoClass, sco); // start debugger before remote config to subscribe to it before starting to poll maybeStartDebugger(instrumentation, scoClass, sco); maybeStartRemoteConfig(scoClass, sco); @@ -952,6 +978,24 @@ private static void maybeStartCiVisibility(Instrumentation inst, Class scoCla } } + private static void maybeStartLLMObs(Instrumentation inst, Class scoClass, Object sco) { + if (llmObsEnabled) { + StaticEventLogger.begin("LLM Observability"); + + try { + final Class llmObsSysClass = + AGENT_CLASSLOADER.loadClass("datadog.trace.llmobs.LLMObsSystem"); + final Method llmObsInstallerMethod = + llmObsSysClass.getMethod("start", Instrumentation.class, scoClass); + llmObsInstallerMethod.invoke(null, inst, sco); + } catch (final Throwable e) { + log.warn("Not starting LLM Observability subsystem", e); + } + + StaticEventLogger.end("LLM Observability"); + } + } + private static void maybeInstallLogsIntake(Class scoClass, Object sco) { if (agentlessLogSubmissionEnabled) { StaticEventLogger.begin("Logs Intake"); diff --git a/dd-java-agent/agent-llmobs/build.gradle b/dd-java-agent/agent-llmobs/build.gradle new file mode 100644 index 00000000000..5e4d7cd6a78 --- /dev/null +++ b/dd-java-agent/agent-llmobs/build.gradle @@ -0,0 +1,42 @@ +buildscript { + repositories { + mavenCentral() + } + + dependencies { + classpath group: 'org.jetbrains.kotlin', name: 'kotlin-gradle-plugin', version: libs.versions.kotlin.get() + } +} + +plugins { + id 'com.gradleup.shadow' + id 'java-test-fixtures' +} + +apply from: "$rootDir/gradle/java.gradle" +apply from: "$rootDir/gradle/version.gradle" +apply from: "$rootDir/gradle/test-with-kotlin.gradle" + +minimumBranchCoverage = 0.0 +minimumInstructionCoverage = 0.0 + +dependencies { + api libs.slf4j + + implementation project(':communication') + implementation project(':components:json') + implementation project(':internal-api') + + testImplementation project(":utils:test-utils") + + testFixturesApi project(':dd-java-agent:testing') + testFixturesApi project(':utils:test-utils') +} + +shadowJar { + dependencies deps.excludeShared +} + +jar { + archiveClassifier = 'unbundled' +} diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java new file mode 100644 index 00000000000..fbfef5c771f --- /dev/null +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java @@ -0,0 +1,104 @@ +package datadog.trace.llmobs; + +import datadog.communication.ddagent.SharedCommunicationObjects; +import datadog.trace.api.Config; +import datadog.trace.api.llmobs.LLMObs; +import datadog.trace.api.llmobs.LLMObsSpan; +import datadog.trace.api.llmobs.LLMObsTags; +import datadog.trace.bootstrap.instrumentation.api.Tags; +import datadog.trace.llmobs.domain.DDLLMObsSpan; +import datadog.trace.llmobs.domain.LLMObsInternal; +import java.lang.instrument.Instrumentation; +import org.jetbrains.annotations.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class LLMObsSystem { + + private static final Logger LOGGER = LoggerFactory.getLogger(LLMObsSystem.class); + + private static final String CUSTOM_MODEL_VAL = "custom"; + + public static void start(Instrumentation inst, SharedCommunicationObjects sco) { + Config config = Config.get(); + if (!config.isLlmObsEnabled()) { + LOGGER.debug("LLM Observability is disabled"); + return; + } + + sco.createRemaining(config); + + LLMObsInternal.setLLMObsSpanFactory( + new LLMObsManualSpanFactory(config.getLlmObsMlApp(), config.getServiceName())); + } + + private static class LLMObsManualSpanFactory implements LLMObs.LLMObsSpanFactory { + + private final String serviceName; + private final String defaultMLApp; + + public LLMObsManualSpanFactory(String defaultMLApp, String serviceName) { + this.defaultMLApp = defaultMLApp; + this.serviceName = serviceName; + } + + @Override + public LLMObsSpan startLLMSpan( + String spanName, + String modelName, + String modelProvider, + @Nullable String mlApp, + @Nullable String sessionId) { + + DDLLMObsSpan span = + new DDLLMObsSpan( + Tags.LLMOBS_LLM_SPAN_KIND, spanName, getMLApp(mlApp), sessionId, serviceName); + + if (modelName == null || modelName.isEmpty()) { + modelName = CUSTOM_MODEL_VAL; + } + span.setTag(LLMObsTags.MODEL_NAME, modelName); + + if (modelProvider == null || modelProvider.isEmpty()) { + modelProvider = CUSTOM_MODEL_VAL; + } + span.setTag(LLMObsTags.MODEL_PROVIDER, modelProvider); + return span; + } + + @Override + public LLMObsSpan startAgentSpan( + String spanName, @Nullable String mlApp, @Nullable String sessionId) { + return new DDLLMObsSpan( + Tags.LLMOBS_AGENT_SPAN_KIND, spanName, getMLApp(mlApp), sessionId, serviceName); + } + + @Override + public LLMObsSpan startToolSpan( + String spanName, @Nullable String mlApp, @Nullable String sessionId) { + return new DDLLMObsSpan( + Tags.LLMOBS_TOOL_SPAN_KIND, spanName, getMLApp(mlApp), sessionId, serviceName); + } + + @Override + public LLMObsSpan startTaskSpan( + String spanName, @Nullable String mlApp, @Nullable String sessionId) { + return new DDLLMObsSpan( + Tags.LLMOBS_TASK_SPAN_KIND, spanName, getMLApp(mlApp), sessionId, serviceName); + } + + @Override + public LLMObsSpan startWorkflowSpan( + String spanName, @Nullable String mlApp, @Nullable String sessionId) { + return new DDLLMObsSpan( + Tags.LLMOBS_WORKFLOW_SPAN_KIND, spanName, getMLApp(mlApp), sessionId, serviceName); + } + + private String getMLApp(String mlApp) { + if (mlApp == null || mlApp.isEmpty()) { + return defaultMLApp; + } + return mlApp; + } + } +} diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java new file mode 100644 index 00000000000..734bf1df526 --- /dev/null +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java @@ -0,0 +1,276 @@ +package datadog.trace.llmobs.domain; + +import datadog.trace.api.DDSpanTypes; +import datadog.trace.api.llmobs.LLMObs; +import datadog.trace.api.llmobs.LLMObsSpan; +import datadog.trace.api.llmobs.LLMObsTags; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.AgentTracer; +import datadog.trace.bootstrap.instrumentation.api.Tags; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import javax.annotation.Nonnull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class DDLLMObsSpan implements LLMObsSpan { + private static final String LLM_MESSAGE_UNKNOWN_ROLE = "unknown"; + + // Well known tags for LLM obs will be prefixed with _ml_obs_(tags|metrics). + // Prefix for tags + private static final String LLMOBS_TAG_PREFIX = "_ml_obs_tag."; + // Prefix for metrics + private static final String LLMOBS_METRIC_PREFIX = "_ml_obs_metric."; + + // internal tags to be prefixed + private static final String INPUT = LLMOBS_TAG_PREFIX + "input"; + private static final String OUTPUT = LLMOBS_TAG_PREFIX + "output"; + private static final String SPAN_KIND = LLMOBS_TAG_PREFIX + Tags.SPAN_KIND; + private static final String METADATA = LLMOBS_TAG_PREFIX + LLMObsTags.METADATA; + + private static final String LLM_OBS_INSTRUMENTATION_NAME = "llmobs"; + + private static final Logger LOGGER = LoggerFactory.getLogger(DDLLMObsSpan.class); + + private final AgentSpan span; + private final String spanKind; + + private boolean finished = false; + + public DDLLMObsSpan( + @Nonnull String kind, + String spanName, + @Nonnull String mlApp, + String sessionId, + @Nonnull String serviceName) { + + if (null == spanName || spanName.isEmpty()) { + spanName = kind; + } + + AgentTracer.SpanBuilder spanBuilder = + AgentTracer.get() + .buildSpan(LLM_OBS_INSTRUMENTATION_NAME, spanName) + .withServiceName(serviceName) + .withSpanType(DDSpanTypes.LLMOBS); + + this.span = spanBuilder.start(); + this.span.setTag(SPAN_KIND, kind); + this.spanKind = kind; + this.span.setTag(LLMOBS_TAG_PREFIX + LLMObsTags.ML_APP, mlApp); + if (sessionId != null && !sessionId.isEmpty()) { + this.span.setTag(LLMOBS_TAG_PREFIX + LLMObsTags.SESSION_ID, sessionId); + } + } + + @Override + public String toString() { + return super.toString() + + ", trace_id=" + + this.span.context().getTraceId() + + ", span_id=" + + this.span.context().getSpanId() + + ", ml_app=" + + this.span.getTag(LLMObsTags.ML_APP) + + ", service=" + + this.span.getServiceName() + + ", span_kind=" + + this.span.getTag(SPAN_KIND); + } + + @Override + public void annotateIO(List inputData, List outputData) { + if (finished) { + return; + } + if (inputData != null && !inputData.isEmpty()) { + this.span.setTag(INPUT, inputData); + } + if (outputData != null && !outputData.isEmpty()) { + this.span.setTag(OUTPUT, outputData); + } + } + + @Override + public void annotateIO(String inputData, String outputData) { + if (finished) { + return; + } + boolean wrongSpanKind = false; + if (inputData != null && !inputData.isEmpty()) { + if (Tags.LLMOBS_LLM_SPAN_KIND.equals(this.spanKind)) { + wrongSpanKind = true; + annotateIO( + Collections.singletonList(LLMObs.LLMMessage.from(LLM_MESSAGE_UNKNOWN_ROLE, inputData)), + null); + } else { + this.span.setTag(INPUT, inputData); + } + } + if (outputData != null && !outputData.isEmpty()) { + if (Tags.LLMOBS_LLM_SPAN_KIND.equals(this.spanKind)) { + wrongSpanKind = true; + annotateIO( + null, + Collections.singletonList( + LLMObs.LLMMessage.from(LLM_MESSAGE_UNKNOWN_ROLE, outputData))); + } else { + this.span.setTag(OUTPUT, outputData); + } + } + if (wrongSpanKind) { + LOGGER.warn( + "the span being annotated is an LLM span, it is recommended to use the overload with List as arguments"); + } + } + + @Override + public void setMetadata(Map metadata) { + if (finished) { + return; + } + Object value = span.getTag(METADATA); + if (value == null) { + this.span.setTag(METADATA, new HashMap<>(metadata)); + return; + } + + if (value instanceof Map) { + ((Map) value).putAll(metadata); + } else { + LOGGER.debug( + "unexpected instance type for metadata {}, overwriting for now", + value.getClass().getName()); + this.span.setTag(METADATA, new HashMap<>(metadata)); + } + } + + @Override + public void setMetrics(Map metrics) { + if (finished) { + return; + } + for (Map.Entry entry : metrics.entrySet()) { + this.span.setMetric(LLMOBS_METRIC_PREFIX + entry.getKey(), entry.getValue().doubleValue()); + } + } + + @Override + public void setMetric(CharSequence key, int value) { + if (finished) { + return; + } + this.span.setMetric(LLMOBS_METRIC_PREFIX + key, value); + } + + @Override + public void setMetric(CharSequence key, long value) { + if (finished) { + return; + } + this.span.setMetric(LLMOBS_METRIC_PREFIX + key, value); + } + + @Override + public void setMetric(CharSequence key, double value) { + if (finished) { + return; + } + this.span.setMetric(LLMOBS_METRIC_PREFIX + key, value); + } + + @Override + public void setTags(Map tags) { + if (finished) { + return; + } + if (tags != null && !tags.isEmpty()) { + for (Map.Entry entry : tags.entrySet()) { + this.span.setTag(LLMOBS_TAG_PREFIX + entry.getKey(), entry.getValue()); + } + } + } + + @Override + public void setTag(String key, String value) { + if (finished) { + return; + } + this.span.setTag(LLMOBS_TAG_PREFIX + key, value); + } + + @Override + public void setTag(String key, boolean value) { + if (finished) { + return; + } + this.span.setTag(LLMOBS_TAG_PREFIX + key, value); + } + + @Override + public void setTag(String key, int value) { + if (finished) { + return; + } + this.span.setTag(LLMOBS_TAG_PREFIX + key, value); + } + + @Override + public void setTag(String key, long value) { + if (finished) { + return; + } + this.span.setTag(LLMOBS_TAG_PREFIX + key, value); + } + + @Override + public void setTag(String key, double value) { + if (finished) { + return; + } + this.span.setTag(LLMOBS_TAG_PREFIX + key, value); + } + + @Override + public void setError(boolean error) { + if (finished) { + return; + } + this.span.setError(error); + } + + @Override + public void setErrorMessage(String errorMessage) { + if (finished) { + return; + } + if (errorMessage == null || errorMessage.isEmpty()) { + return; + } + this.span.setError(true); + this.span.setErrorMessage(errorMessage); + } + + @Override + public void addThrowable(Throwable throwable) { + if (finished) { + return; + } + if (throwable == null) { + return; + } + this.span.setError(true); + this.span.addThrowable(throwable); + } + + @Override + public void finish() { + if (finished) { + return; + } + this.span.finish(); + this.finished = true; + } +} diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/LLMObsInternal.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/LLMObsInternal.java new file mode 100644 index 00000000000..42b0c097e48 --- /dev/null +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/LLMObsInternal.java @@ -0,0 +1,10 @@ +package datadog.trace.llmobs.domain; + +import datadog.trace.api.llmobs.LLMObs; + +public class LLMObsInternal extends LLMObs { + + public static void setLLMObsSpanFactory(final LLMObsSpanFactory factory) { + LLMObs.SPAN_FACTORY = factory; + } +} diff --git a/dd-java-agent/agent-llmobs/src/test/groovy/datadog/trace/llmobs/domain/DDLLMObsSpanTest.groovy b/dd-java-agent/agent-llmobs/src/test/groovy/datadog/trace/llmobs/domain/DDLLMObsSpanTest.groovy new file mode 100644 index 00000000000..a6c8534be2b --- /dev/null +++ b/dd-java-agent/agent-llmobs/src/test/groovy/datadog/trace/llmobs/domain/DDLLMObsSpanTest.groovy @@ -0,0 +1,267 @@ +package datadog.trace.llmobs.domain + +import datadog.trace.agent.tooling.TracerInstaller +import datadog.trace.api.DDTags +import datadog.trace.api.IdGenerationStrategy +import datadog.trace.api.llmobs.LLMObs +import datadog.trace.api.llmobs.LLMObsSpan +import datadog.trace.api.llmobs.LLMObsTags +import datadog.trace.bootstrap.instrumentation.api.AgentSpan +import datadog.trace.bootstrap.instrumentation.api.AgentTracer +import datadog.trace.bootstrap.instrumentation.api.Tags +import datadog.trace.core.CoreTracer +import datadog.trace.test.util.DDSpecification +import org.apache.groovy.util.Maps +import spock.lang.Shared + +class DDLLMObsSpanTest extends DDSpecification{ + @SuppressWarnings('PropertyName') + @Shared + AgentTracer.TracerAPI TEST_TRACER + + void setupSpec() { + TEST_TRACER = + Spy( + CoreTracer.builder() + .idGenerationStrategy(IdGenerationStrategy.fromName("SEQUENTIAL")) + .build()) + TracerInstaller.forceInstallGlobalTracer(TEST_TRACER) + + TEST_TRACER.startSpan(*_) >> { + def agentSpan = callRealMethod() + agentSpan + } + } + + void cleanupSpec() { + TEST_TRACER?.close() + } + + void setup() { + assert TEST_TRACER.activeSpan() == null: "Span is active before test has started: " + TEST_TRACER.activeSpan() + TEST_TRACER.flush() + } + + void cleanup() { + TEST_TRACER.flush() + } + + // Prefix for tags + private static final String LLMOBS_TAG_PREFIX = "_ml_obs_tag." + // Prefix for metrics + private static final String LLMOBS_METRIC_PREFIX = "_ml_obs_metric." + + // internal tags to be prefixed + private static final String INPUT = LLMOBS_TAG_PREFIX + "input" + private static final String OUTPUT = LLMOBS_TAG_PREFIX + "output" + private static final String METADATA = LLMOBS_TAG_PREFIX + LLMObsTags.METADATA + + + def "test span simple"() { + setup: + def test = givenALLMObsSpan(Tags.LLMOBS_WORKFLOW_SPAN_KIND, "test-span") + + when: + def input = "test input" + def output = "test output" + // initial set + test.annotateIO(input, output) + test.setMetadata(Maps.of("sport", "baseball", "price_data", Maps.of("gpt4", 100))) + test.setMetrics(Maps.of("rank", 1)) + test.setMetric("likelihood", 0.1) + test.setTag("DOMAIN", "north-america") + test.setTags(Maps.of("bulk1", 1, "bulk2", "2")) + def errMsg = "mr brady" + test.setErrorMessage(errMsg) + + then: + def innerSpan = (AgentSpan)test.span + assert Tags.LLMOBS_WORKFLOW_SPAN_KIND.equals(innerSpan.getTag(LLMOBS_TAG_PREFIX + "span.kind")) + + assert null == innerSpan.getTag("input") + assert input.equals(innerSpan.getTag(INPUT)) + assert null == innerSpan.getTag("output") + assert output.equals(innerSpan.getTag(OUTPUT)) + + assert null == innerSpan.getTag("metadata") + def expectedMetadata = Maps.of("sport", "baseball", "price_data", Maps.of("gpt4", 100)) + assert expectedMetadata.equals(innerSpan.getTag(METADATA)) + + assert null == innerSpan.getTag("rank") + def rankMetric = innerSpan.getTag(LLMOBS_METRIC_PREFIX + "rank") + assert rankMetric instanceof Number && 1 == (int)rankMetric + + assert null == innerSpan.getTag("likelihood") + def likelihoodMetric = innerSpan.getTag(LLMOBS_METRIC_PREFIX + "likelihood") + assert likelihoodMetric instanceof Number + assert 0.1 == (double)likelihoodMetric + + assert null == innerSpan.getTag("DOMAIN") + def domain = innerSpan.getTag(LLMOBS_TAG_PREFIX + "DOMAIN") + assert domain instanceof String + assert "north-america".equals((String)domain) + + assert null == innerSpan.getTag("bulk1") + def tagBulk1 = innerSpan.getTag(LLMOBS_TAG_PREFIX + "bulk1") + assert tagBulk1 instanceof Number + assert 1 == ((int)tagBulk1) + + assert null == innerSpan.getTag("bulk2") + def tagBulk2 = innerSpan.getTag(LLMOBS_TAG_PREFIX + "bulk2") + assert tagBulk2 instanceof String + assert "2".equals((String)tagBulk2) + + assert innerSpan.isError() + assert innerSpan.getTag(DDTags.ERROR_MSG) instanceof String + assert errMsg.equals(innerSpan.getTag(DDTags.ERROR_MSG)) + } + + def "test span with overwrites"() { + setup: + def test = givenALLMObsSpan(Tags.LLMOBS_AGENT_SPAN_KIND, "test-span") + + when: + def input = "test input" + // initial set + test.annotateIO(input, "test output") + // this should be a no-op + test.annotateIO("", "") + // this should replace the initial output + def expectedOutput = Arrays.asList(Maps.of("role", "user", "content", "how much is gas")) + test.annotateIO(null, expectedOutput) + + // initial set + test.setMetadata(Maps.of("sport", "baseball", "price_data", Maps.of("gpt4", 100))) + // this should replace baseball with hockey + test.setMetadata(Maps.of("sport", "hockey")) + // this should add a new key + test.setMetadata(Maps.of("temperature", 30)) + + // initial set + test.setMetrics(Maps.of("rank", 1)) + // this should replace the metric + test.setMetric("rank", 10) + + // initial set + test.setTag("DOMAIN", "north-america") + // add and replace + test.setTags(Maps.of("bulk1", 1, "DOMAIN", "europe")) + + def throwableMsg = "false positive" + test.addThrowable(new Throwable(throwableMsg)) + test.setError(false) + + then: + def innerSpan = (AgentSpan)test.span + assert Tags.LLMOBS_AGENT_SPAN_KIND.equals(innerSpan.getTag(LLMOBS_TAG_PREFIX + "span.kind")) + + assert null == innerSpan.getTag("input") + assert input.equals(innerSpan.getTag(INPUT)) + assert null == innerSpan.getTag("output") + assert expectedOutput.equals(innerSpan.getTag(OUTPUT)) + + assert null == innerSpan.getTag("metadata") + def expectedMetadata = Maps.of("sport", "hockey", "price_data", Maps.of("gpt4", 100), "temperature", 30) + assert expectedMetadata.equals(innerSpan.getTag(METADATA)) + + assert null == innerSpan.getTag("rank") + def rankMetric = innerSpan.getTag(LLMOBS_METRIC_PREFIX + "rank") + assert rankMetric instanceof Number && 10 == (int)rankMetric + + assert null == innerSpan.getTag("DOMAIN") + def domain = innerSpan.getTag(LLMOBS_TAG_PREFIX + "DOMAIN") + assert domain instanceof String + assert "europe".equals((String)domain) + + assert null == innerSpan.getTag("bulk1") + def tagBulk1 = innerSpan.getTag(LLMOBS_TAG_PREFIX + "bulk1") + assert tagBulk1 instanceof Number + assert 1 == ((int)tagBulk1) + + assert !innerSpan.isError() + assert innerSpan.getTag(DDTags.ERROR_MSG) instanceof String + assert throwableMsg.equals(innerSpan.getTag(DDTags.ERROR_MSG)) + assert innerSpan.getTag(DDTags.ERROR_STACK) instanceof String + assert ((String)innerSpan.getTag(DDTags.ERROR_STACK)).contains(throwableMsg) + } + + def "test llm span string input formatted to messages"() { + setup: + def test = givenALLMObsSpan(Tags.LLMOBS_LLM_SPAN_KIND, "test-span") + + when: + def input = "test input" + def output = "test output" + // initial set + test.annotateIO(input, output) + + then: + def innerSpan = (AgentSpan)test.span + assert Tags.LLMOBS_LLM_SPAN_KIND.equals(innerSpan.getTag(LLMOBS_TAG_PREFIX + "span.kind")) + + assert null == innerSpan.getTag("input") + def spanInput = innerSpan.getTag(INPUT) + assert spanInput instanceof List + assert ((List)spanInput).size() == 1 + assert spanInput.get(0) instanceof LLMObs.LLMMessage + def expectedInputMsg = LLMObs.LLMMessage.from("unknown", input) + assert expectedInputMsg.getContent().equals(input) + assert expectedInputMsg.getRole().equals("unknown") + assert expectedInputMsg.getToolCalls().equals(null) + + assert null == innerSpan.getTag("output") + def spanOutput = innerSpan.getTag(OUTPUT) + assert spanOutput instanceof List + assert ((List)spanOutput).size() == 1 + assert spanOutput.get(0) instanceof LLMObs.LLMMessage + def expectedOutputMsg = LLMObs.LLMMessage.from("unknown", output) + assert expectedOutputMsg.getContent().equals(output) + assert expectedOutputMsg.getRole().equals("unknown") + assert expectedOutputMsg.getToolCalls().equals(null) + } + + def "test llm span with messages"() { + setup: + def test = givenALLMObsSpan(Tags.LLMOBS_LLM_SPAN_KIND, "test-span") + + when: + def inputMsg = LLMObs.LLMMessage.from("user", "input") + def outputMsg = LLMObs.LLMMessage.from("assistant", "output", Arrays.asList(LLMObs.ToolCall.from("weather-tool", "function", "6176241000", Maps.of("location", "paris")))) + // initial set + test.annotateIO(Arrays.asList(inputMsg), Arrays.asList(outputMsg)) + + then: + def innerSpan = (AgentSpan)test.span + assert Tags.LLMOBS_LLM_SPAN_KIND.equals(innerSpan.getTag(LLMOBS_TAG_PREFIX + "span.kind")) + + assert null == innerSpan.getTag("input") + def spanInput = innerSpan.getTag(INPUT) + assert spanInput instanceof List + assert ((List)spanInput).size() == 1 + def spanInputMsg = spanInput.get(0) + assert spanInputMsg instanceof LLMObs.LLMMessage + assert spanInputMsg.getContent().equals(inputMsg.getContent()) + assert spanInputMsg.getRole().equals("user") + assert spanInputMsg.getToolCalls().equals(null) + + assert null == innerSpan.getTag("output") + def spanOutput = innerSpan.getTag(OUTPUT) + assert spanOutput instanceof List + assert ((List)spanOutput).size() == 1 + def spanOutputMsg = spanOutput.get(0) + assert spanOutputMsg instanceof LLMObs.LLMMessage + assert spanOutputMsg.getContent().equals(outputMsg.getContent()) + assert spanOutputMsg.getRole().equals("assistant") + assert spanOutputMsg.getToolCalls().size() == 1 + def toolCall = spanOutputMsg.getToolCalls().get(0) + assert toolCall.getName().equals("weather-tool") + assert toolCall.getType().equals("function") + assert toolCall.getToolId().equals("6176241000") + def expectedToolArgs = Maps.of("location", "paris") + assert toolCall.getArguments().equals(expectedToolArgs) + } + + private LLMObsSpan givenALLMObsSpan(String kind, name){ + new DDLLMObsSpan(kind, name, "test-ml-app", null, "test-svc") + } +} diff --git a/dd-java-agent/build.gradle b/dd-java-agent/build.gradle index 1b2c3804acb..b75c6030059 100644 --- a/dd-java-agent/build.gradle +++ b/dd-java-agent/build.gradle @@ -135,6 +135,7 @@ includeSubprojShadowJar ':dd-java-agent:appsec', 'appsec' includeSubprojShadowJar ':dd-java-agent:agent-iast', 'iast' includeSubprojShadowJar ':dd-java-agent:agent-debugger', 'debugger' includeSubprojShadowJar ':dd-java-agent:agent-ci-visibility', 'ci-visibility' +includeSubprojShadowJar ':dd-java-agent:agent-llmobs', 'llm-obs' includeSubprojShadowJar ':dd-java-agent:agent-logs-intake', 'logs-intake' includeSubprojShadowJar ':dd-java-agent:cws-tls', 'cws-tls' diff --git a/dd-trace-api/src/main/java/datadog/trace/api/DDSpanTypes.java b/dd-trace-api/src/main/java/datadog/trace/api/DDSpanTypes.java index 335dde0effe..04f9738c643 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/DDSpanTypes.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/DDSpanTypes.java @@ -40,4 +40,6 @@ public class DDSpanTypes { public static final String WEBSOCKET = "websocket"; public static final String SERVERLESS = "serverless"; + + public static final String LLMOBS = "llm"; } diff --git a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsTags.java b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsTags.java new file mode 100644 index 00000000000..afa4f2b241e --- /dev/null +++ b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsTags.java @@ -0,0 +1,15 @@ +package datadog.trace.api.llmobs; + +// Well known tags for llm obs +public class LLMObsTags { + public static final String ML_APP = "ml_app"; + public static final String SESSION_ID = "session_id"; + + // meta + public static final String METADATA = "metadata"; + + // LLM spans related + public static final String MODEL_NAME = "model_name"; + public static final String MODEL_VERSION = "model_version"; + public static final String MODEL_PROVIDER = "model_provider"; +} diff --git a/internal-api/src/main/java/datadog/trace/api/Config.java b/internal-api/src/main/java/datadog/trace/api/Config.java index 6f3041ca7d8..07ff2d148e8 100644 --- a/internal-api/src/main/java/datadog/trace/api/Config.java +++ b/internal-api/src/main/java/datadog/trace/api/Config.java @@ -320,6 +320,7 @@ public static String getHostName() { private final int iastDbRowsToTaint; private final boolean llmObsAgentlessEnabled; + private final String llmObsAgentlessUrl; private final String llmObsMlApp; private final boolean ciVisibilityTraceSanitationEnabled; @@ -1460,6 +1461,22 @@ PROFILING_DATADOG_PROFILER_ENABLED, isDatadogProfilerSafeInCurrentEnvironment()) configProvider.getBoolean(LLMOBS_AGENTLESS_ENABLED, DEFAULT_LLM_OBS_AGENTLESS_ENABLED); llmObsMlApp = configProvider.getString(LLMOBS_ML_APP); + final String llmObsAgentlessUrlStr = getFinalLLMObsUrl(); + URI parsedLLMObsUri = null; + if (llmObsAgentlessUrlStr != null && !llmObsAgentlessUrlStr.isEmpty()) { + try { + parsedLLMObsUri = new URL(llmObsAgentlessUrlStr).toURI(); + } catch (MalformedURLException | URISyntaxException ex) { + log.error( + "Cannot parse LLM Observability agentless URL '{}', skipping", llmObsAgentlessUrlStr); + } + } + if (parsedLLMObsUri != null) { + llmObsAgentlessUrl = llmObsAgentlessUrlStr; + } else { + llmObsAgentlessUrl = null; + } + ciVisibilityTraceSanitationEnabled = configProvider.getBoolean(CIVISIBILITY_TRACE_SANITATION_ENABLED, true); @@ -2891,6 +2908,10 @@ public boolean isLlmObsAgentlessEnabled() { return llmObsAgentlessEnabled; } + public String getLlMObsAgentlessUrl() { + return llmObsAgentlessUrl; + } + public String getLlmObsMlApp() { return llmObsMlApp; } @@ -3968,6 +3989,13 @@ public String getFinalProfilingUrl() { } } + public String getFinalLLMObsUrl() { + if (llmObsAgentlessEnabled) { + return "https://llmobs-intake." + site + "/api/v2/llmobs"; + } + return null; + } + public String getFinalCrashTrackingTelemetryUrl() { if (crashTrackingAgentless) { // when agentless crashTracking is turned on we send directly to our intake diff --git a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/Tags.java b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/Tags.java index 78c90b312a8..fd11b2bf565 100644 --- a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/Tags.java +++ b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/Tags.java @@ -152,4 +152,10 @@ public class Tags { public static final String PROPAGATED_TRACE_SOURCE = "_dd.p.ts"; public static final String PROPAGATED_DEBUG = "_dd.p.debug"; + + public static final String LLMOBS_LLM_SPAN_KIND = "llm"; + public static final String LLMOBS_WORKFLOW_SPAN_KIND = "workflow"; + public static final String LLMOBS_TASK_SPAN_KIND = "task"; + public static final String LLMOBS_AGENT_SPAN_KIND = "agent"; + public static final String LLMOBS_TOOL_SPAN_KIND = "tool"; } diff --git a/settings.gradle b/settings.gradle index db584c29b13..3218b1cdb50 100644 --- a/settings.gradle +++ b/settings.gradle @@ -99,6 +99,9 @@ include ':dd-java-agent:appsec' // ci-visibility include ':dd-java-agent:agent-ci-visibility' +// llm-observability +include ':dd-java-agent:agent-llmobs' + // iast include ':dd-java-agent:agent-iast' From 473966614e470777ee578c3405d00d5aa551ad3e Mon Sep 17 00:00:00 2001 From: gary-huang Date: Tue, 10 Jun 2025 20:01:15 -0400 Subject: [PATCH 28/28] use proper convention like Id instead of ID --- .../datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java index 584b26902af..bc4647bbc2a 100644 --- a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java +++ b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java @@ -311,7 +311,7 @@ public void accept(Metadata metadata) { writable.writeUTF8(LLM_TOOL_CALL_TYPE); writable.writeString(toolCall.getType(), null); writable.writeUTF8(LLM_TOOL_CALL_TOOL_ID); - writable.writeString(toolCall.getToolID(), null); + writable.writeString(toolCall.getToolId(), null); if (hasArguments) { writable.writeUTF8(LLM_TOOL_CALL_ARGUMENTS); writable.startMap(arguments.size());