Increase otel attribute limits and make them configurable

apps/webapp/app/env.server.ts

@@ -2,6 +2,7 @@ import { z } from "zod";
 import { isValidDatabaseUrl } from "./utils/db";
 import { isValidRegex } from "./utils/regex";
 import { BoolEnv } from "./utils/boolEnv";
+import { OTEL_ATTRIBUTE_PER_LINK_COUNT_LIMIT, OTEL_LINK_COUNT_LIMIT } from "@trigger.dev/core/v3";
 
 const EnvironmentSchema = z.object({
   NODE_ENV: z.union([z.literal("development"), z.literal("production"), z.literal("test")]),
@@ -276,6 +277,15 @@ const EnvironmentSchema = z.object({
   PROD_OTEL_LOG_EXPORT_TIMEOUT_MILLIS: z.string().default("30000"),
   PROD_OTEL_LOG_MAX_QUEUE_SIZE: z.string().default("512"),
 
+  TRIGGER_OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT: z.string().default("256"),
+  TRIGGER_OTEL_LOG_ATTRIBUTE_COUNT_LIMIT: z.string().default("256"),
+  TRIGGER_OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT: z.string().default("131072"),
+  TRIGGER_OTEL_LOG_ATTRIBUTE_VALUE_LENGTH_LIMIT: z.string().default("131072"),
+  TRIGGER_OTEL_SPAN_EVENT_COUNT_LIMIT: z.string().default("10"),
+  TRIGGER_OTEL_LINK_COUNT_LIMIT: z.string().default("2"),
+  TRIGGER_OTEL_ATTRIBUTE_PER_LINK_COUNT_LIMIT: z.string().default("10"),
+  TRIGGER_OTEL_ATTRIBUTE_PER_EVENT_COUNT_LIMIT: z.string().default("10"),
+
   CHECKPOINT_THRESHOLD_IN_MS: z.coerce.number().int().default(30000),
 
   // Internal OTEL environment variables

apps/webapp/app/v3/environmentVariables/environmentVariablesRepository.server.ts

@@ -810,6 +810,7 @@ export const RuntimeEnvironmentForEnvRepoPayload = {
     apiKey: true,
     organizationId: true,
     branchName: true,
+    builtInEnvironmentVariableOverrides: true,
   },
 } as const;
 
@@ -1025,5 +1026,93 @@ async function resolveBuiltInProdVariables(
 async function resolveCommonBuiltInVariables(
   runtimeEnvironment: RuntimeEnvironmentForEnvRepo
 ): Promise<Array<EnvironmentVariable>> {
-  return [];
+  return [
+    {
+      key: "TRIGGER_OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT",
+      value: resolveBuiltInEnvironmentVariableOverrides(
+        "TRIGGER_OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT",
+        runtimeEnvironment,
+        String(env.TRIGGER_OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT)
+      ),
+    },
+    {
+      key: "TRIGGER_OTEL_LOG_ATTRIBUTE_COUNT_LIMIT",
+      value: resolveBuiltInEnvironmentVariableOverrides(
+        "TRIGGER_OTEL_LOG_ATTRIBUTE_COUNT_LIMIT",
+        runtimeEnvironment,
+        String(env.TRIGGER_OTEL_LOG_ATTRIBUTE_COUNT_LIMIT)
+      ),
+    },
+    {
+      key: "TRIGGER_OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT",
+      value: resolveBuiltInEnvironmentVariableOverrides(
+        "TRIGGER_OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT",
+        runtimeEnvironment,
+        String(env.TRIGGER_OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT)
+      ),
+    },
+    {
+      key: "TRIGGER_OTEL_LOG_ATTRIBUTE_VALUE_LENGTH_LIMIT",
+      value: resolveBuiltInEnvironmentVariableOverrides(
+        "TRIGGER_OTEL_LOG_ATTRIBUTE_VALUE_LENGTH_LIMIT",
+        runtimeEnvironment,
+        String(env.TRIGGER_OTEL_LOG_ATTRIBUTE_VALUE_LENGTH_LIMIT)
+      ),
+    },
+    {
+      key: "TRIGGER_OTEL_SPAN_EVENT_COUNT_LIMIT",
+      value: resolveBuiltInEnvironmentVariableOverrides(
+        "TRIGGER_OTEL_SPAN_EVENT_COUNT_LIMIT",
+        runtimeEnvironment,
+        String(env.TRIGGER_OTEL_SPAN_EVENT_COUNT_LIMIT)
+      ),
+    },
+    {
+      key: "TRIGGER_OTEL_LINK_COUNT_LIMIT",
+      value: resolveBuiltInEnvironmentVariableOverrides(
+        "TRIGGER_OTEL_LINK_COUNT_LIMIT",
+        runtimeEnvironment,
+        String(env.TRIGGER_OTEL_LINK_COUNT_LIMIT)
+      ),
+    },
+    {
+      key: "TRIGGER_OTEL_ATTRIBUTE_PER_LINK_COUNT_LIMIT",
+      value: resolveBuiltInEnvironmentVariableOverrides(
+        "TRIGGER_OTEL_ATTRIBUTE_PER_LINK_COUNT_LIMIT",
+        runtimeEnvironment,
+        String(env.TRIGGER_OTEL_ATTRIBUTE_PER_LINK_COUNT_LIMIT)
+      ),
+    },
+    {
+      key: "TRIGGER_OTEL_ATTRIBUTE_PER_EVENT_COUNT_LIMIT",
+      value: resolveBuiltInEnvironmentVariableOverrides(
+        "TRIGGER_OTEL_ATTRIBUTE_PER_EVENT_COUNT_LIMIT",
+        runtimeEnvironment,
+        String(env.TRIGGER_OTEL_ATTRIBUTE_PER_EVENT_COUNT_LIMIT)
+      ),
+    },
+  ];
+}
+
+function resolveBuiltInEnvironmentVariableOverrides(
+  key: string,
+  runtimeEnvironment: RuntimeEnvironmentForEnvRepo,
+  defaultValue: string
+) {
+  const overrides = runtimeEnvironment.builtInEnvironmentVariableOverrides;
+
+  if (!overrides) {
+    return defaultValue;
+  }
+
+  if (
+    !Array.isArray(overrides) &&
+    typeof overrides === "object" &&
+    key in overrides &&
+    typeof overrides[key] === "string"
+  ) {
+    return overrides[key];
+  }
+
+  return defaultValue;
 }

apps/webapp/app/v3/otlpExporter.server.ts

@@ -39,7 +39,8 @@ class OTLPExporter {
 
   constructor(
     private readonly _eventRepository: EventRepository,
-    private readonly _verbose: boolean
+    private readonly _verbose: boolean,
+    private readonly _spanAttributeValueLengthLimit: number
   ) {
     this._tracer = trace.getTracer("otlp-exporter");
   }
@@ -52,7 +53,7 @@ class OTLPExporter {
       this.#logExportTracesVerbose(request);
 
       const events = this.#filterResourceSpans(request.resourceSpans).flatMap((resourceSpan) => {
-        return convertSpansToCreateableEvents(resourceSpan);
+        return convertSpansToCreateableEvents(resourceSpan, this._spanAttributeValueLengthLimit);
       });
 
       const enrichedEvents = enrichCreatableEvents(events);
@@ -79,7 +80,7 @@ class OTLPExporter {
       this.#logExportLogsVerbose(request);
 
       const events = this.#filterResourceLogs(request.resourceLogs).flatMap((resourceLog) => {
-        return convertLogsToCreateableEvents(resourceLog);
+        return convertLogsToCreateableEvents(resourceLog, this._spanAttributeValueLengthLimit);
       });
 
       const enrichedEvents = enrichCreatableEvents(events);
@@ -180,7 +181,10 @@ class OTLPExporter {
   }
 }
 
-function convertLogsToCreateableEvents(resourceLog: ResourceLogs): Array<CreatableEvent> {
+function convertLogsToCreateableEvents(
+  resourceLog: ResourceLogs,
+  spanAttributeValueLengthLimit: number
+): Array<CreatableEvent> {
   const resourceAttributes = resourceLog.resource?.attributes ?? [];
 
   const resourceProperties = extractEventProperties(resourceAttributes);
@@ -213,10 +217,10 @@ function convertLogsToCreateableEvents(resourceLog: ResourceLogs): Array<Creatab
           status: logLevelToEventStatus(log.severityNumber),
           startTime: log.timeUnixNano,
           properties: {
-            ...convertKeyValueItemsToMap(log.attributes ?? [], [
-              SemanticInternalAttributes.SPAN_ID,
-              SemanticInternalAttributes.SPAN_PARTIAL,
-            ]),
+            ...convertKeyValueItemsToMap(
+              truncateAttributes(log.attributes ?? [], spanAttributeValueLengthLimit),
+              [SemanticInternalAttributes.SPAN_ID, SemanticInternalAttributes.SPAN_PARTIAL]
+            ),
           },
           style: convertKeyValueItemsToMap(
             pickAttributes(log.attributes ?? [], SemanticInternalAttributes.STYLE),
@@ -283,7 +287,10 @@ function convertLogsToCreateableEvents(resourceLog: ResourceLogs): Array<Creatab
   });
 }
 
-function convertSpansToCreateableEvents(resourceSpan: ResourceSpans): Array<CreatableEvent> {
+function convertSpansToCreateableEvents(
+  resourceSpan: ResourceSpans,
+  spanAttributeValueLengthLimit: number
+): Array<CreatableEvent> {
   const resourceAttributes = resourceSpan.resource?.attributes ?? [];
 
   const resourceProperties = extractEventProperties(resourceAttributes);
@@ -323,10 +330,10 @@ function convertSpansToCreateableEvents(resourceSpan: ResourceSpans): Array<Crea
           events: spanEventsToEventEvents(span.events ?? []),
           duration: span.endTimeUnixNano - span.startTimeUnixNano,
           properties: {
-            ...convertKeyValueItemsToMap(span.attributes ?? [], [
-              SemanticInternalAttributes.SPAN_ID,
-              SemanticInternalAttributes.SPAN_PARTIAL,
-            ]),
+            ...convertKeyValueItemsToMap(
+              truncateAttributes(span.attributes ?? [], spanAttributeValueLengthLimit),
+              [SemanticInternalAttributes.SPAN_ID, SemanticInternalAttributes.SPAN_PARTIAL]
+            ),
           },
           style: convertKeyValueItemsToMap(
             pickAttributes(span.attributes ?? [], SemanticInternalAttributes.STYLE),
@@ -852,7 +859,23 @@ function binaryToHex(buffer: Buffer | string | undefined): string | undefined {
   return Buffer.from(Array.from(buffer)).toString("hex");
 }
 
+function truncateAttributes(attributes: KeyValue[], maximumLength: number = 1024): KeyValue[] {
+  return attributes.map((attribute) => {
+    return isStringValue(attribute.value)
+      ? {
+          key: attribute.key,
+          value: {
+            stringValue: attribute.value.stringValue.slice(0, maximumLength),
+          },
+        }
+      : attribute;
+  });
+}
+
 export const otlpExporter = new OTLPExporter(
   eventRepository,
-  process.env.OTLP_EXPORTER_VERBOSE === "1"
+  process.env.OTLP_EXPORTER_VERBOSE === "1",
+  process.env.SERVER_OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT
+    ? parseInt(process.env.SERVER_OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT, 10)
+    : 8192
 );

apps/webapp/test/timelineSpanEvents.test.ts

@@ -76,7 +76,7 @@ describe("createTimelineSpanEventsFromSpanEvents", () => {
     expect(result.some((event) => event.name === "Dequeued")).toBe(true);
     expect(result.some((event) => event.name === "Launched")).toBe(true);
     expect(result.some((event) => event.name === "Attempt created")).toBe(true);
-    expect(result.some((event) => event.name === "Importing src/trigger/chat.ts")).toBe(true);
+    expect(result.some((event) => event.name === "Importing task file")).toBe(true);
   });
 
   test("should sort events by timestamp", () => {
@@ -86,7 +86,7 @@ describe("createTimelineSpanEventsFromSpanEvents", () => {
     expect(result[0].name).toBe("Dequeued");
     expect(result[1].name).toBe("Attempt created");
     expect(result[2].name).toBe("Launched");
-    expect(result[3].name).toBe("Importing src/trigger/chat.ts");
+    expect(result[3].name).toBe("Importing Importing task file");
   });
 
   test("should calculate offsets correctly from the first event", () => {
@@ -176,7 +176,7 @@ describe("createTimelineSpanEventsFromSpanEvents", () => {
     expect(result.find((e) => e.name === "Attempt created")?.helpText).toBe(
       "An attempt was created for the run"
     );
-    expect(result.find((e) => e.name === "Importing src/trigger/chat.ts")?.helpText).toBe(
+    expect(result.find((e) => e.name === "Importing task file")?.helpText).toBe(
       "A task file was imported"
     );
   });
@@ -187,7 +187,7 @@ describe("createTimelineSpanEventsFromSpanEvents", () => {
     expect(result.find((e) => e.name === "Dequeued")?.duration).toBe(0);
     expect(result.find((e) => e.name === "Launched")?.duration).toBe(127);
     expect(result.find((e) => e.name === "Attempt created")?.duration).toBe(56);
-    expect(result.find((e) => e.name === "Importing src/trigger/chat.ts")?.duration).toBe(67);
+    expect(result.find((e) => e.name === "Importing task file")?.duration).toBe(67);
   });
 
   test("should use fallback name for import event without file property", () => {
@@ -214,7 +214,7 @@ describe("createTimelineSpanEventsFromSpanEvents", () => {
     // Without fork event, import should also be visible for non-admins
     expect(result.length).toBe(2);
     expect(result.some((event) => event.name === "Dequeued")).toBe(true);
-    expect(result.some((event) => event.name === "Importing src/trigger/chat.ts")).toBe(true);
+    expect(result.some((event) => event.name === "Importing task file")).toBe(true);
 
     // create_attempt should still be admin-only
     expect(result.some((event) => event.name === "Attempt created")).toBe(false);

internal-packages/database/prisma/migrations/20250620144525_add_built_in_overrides_to_runtime_environment/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "RuntimeEnvironment" ADD COLUMN     "builtInEnvironmentVariableOverrides" JSONB;

internal-packages/database/prisma/schema.prisma

@@ -241,6 +241,9 @@ model RuntimeEnvironment {
   createdAt DateTime @default(now())
   updatedAt DateTime @updatedAt
 
+  /// Allows us to customize the built-in environment variables for a specific environment, like TRIGGER_OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT
+  builtInEnvironmentVariableOverrides Json?
+
   tunnelId String?
 
   backgroundWorkers          BackgroundWorker[]

packages/core/src/v3/limits.ts

@@ -1,13 +1,46 @@
 import { AttributeValue, Attributes } from "@opentelemetry/api";
+import { getEnvVar } from "./utils/getEnv.js";
+
+function getOtelEnvVarLimit(key: string, defaultValue: number) {
+  const value = getEnvVar(key);
+
+  if (!value) {
+    return defaultValue;
+  }
+
+  return parseInt(value, 10);
+}
+
+export const OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT = getOtelEnvVarLimit(
+  "TRIGGER_OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT",
+  256
+);
+export const OTEL_LOG_ATTRIBUTE_COUNT_LIMIT = getOtelEnvVarLimit(
+  "TRIGGER_OTEL_LOG_ATTRIBUTE_COUNT_LIMIT",
+  256
+);
+export const OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT = getOtelEnvVarLimit(
+  "TRIGGER_OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT",
+  131072
+);
+export const OTEL_LOG_ATTRIBUTE_VALUE_LENGTH_LIMIT = getOtelEnvVarLimit(
+  "TRIGGER_OTEL_LOG_ATTRIBUTE_VALUE_LENGTH_LIMIT",
+  131072
+);
+export const OTEL_SPAN_EVENT_COUNT_LIMIT = getOtelEnvVarLimit(
+  "TRIGGER_OTEL_SPAN_EVENT_COUNT_LIMIT",
+  10
+);
+export const OTEL_LINK_COUNT_LIMIT = getOtelEnvVarLimit("TRIGGER_OTEL_LINK_COUNT_LIMIT", 2);
+export const OTEL_ATTRIBUTE_PER_LINK_COUNT_LIMIT = getOtelEnvVarLimit(
+  "TRIGGER_OTEL_ATTRIBUTE_PER_LINK_COUNT_LIMIT",
+  10
+);
+export const OTEL_ATTRIBUTE_PER_EVENT_COUNT_LIMIT = getOtelEnvVarLimit(
+  "TRIGGER_OTEL_ATTRIBUTE_PER_EVENT_COUNT_LIMIT",
+  10
+);
 
-export const OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT = 256;
-export const OTEL_LOG_ATTRIBUTE_COUNT_LIMIT = 256;
-export const OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT = 1028;
-export const OTEL_LOG_ATTRIBUTE_VALUE_LENGTH_LIMIT = 1028;
-export const OTEL_SPAN_EVENT_COUNT_LIMIT = 10;
-export const OTEL_LINK_COUNT_LIMIT = 2;
-export const OTEL_ATTRIBUTE_PER_LINK_COUNT_LIMIT = 10;
-export const OTEL_ATTRIBUTE_PER_EVENT_COUNT_LIMIT = 10;
 export const OFFLOAD_IO_PACKET_LENGTH_LIMIT = 128 * 1024;
 
 export function imposeAttributeLimits(attributes: Attributes): Attributes {