feat(tracing): Add tracestate header to outgoing requests (#3092)

Author: lobsterkatie

packages/core/src/request.ts

@@ -1,4 +1,5 @@
 import { Event, SdkInfo, SentryRequest, Session } from '@sentry/types';
+import { base64ToUnicode, logger } from '@sentry/utils';
 
 import { API } from './api';
 
@@ -66,26 +67,28 @@ export function eventToSentryRequest(event: Event, api: API): SentryRequest {
 export function transactionToSentryRequest(event: Event, api: API): SentryRequest {
   const sdkInfo = getSdkMetadataForEnvelopeHeader(api);
 
-  const { transactionSampling, ...metadata } = event.debug_meta || {};
+  const { transactionSampling, tracestate: encodedTracestate, ...metadata } = event.debug_meta || {};
   const { method: samplingMethod, rate: sampleRate } = transactionSampling || {};
   if (Object.keys(metadata).length === 0) {
     delete event.debug_meta;
   } else {
     event.debug_meta = metadata;
   }
 
+  // the tracestate is stored in bas64-encoded JSON, but envelope header values are expected to be full JS values,
+  // so we have to decode and reinflate it
+  let tracestate;
+  try {
+    tracestate = JSON.parse(base64ToUnicode(encodedTracestate as string));
+  } catch (err) {
+    logger.warn(err);
+  }
+
   const envelopeHeaders = JSON.stringify({
     event_id: event.event_id,
     sent_at: new Date().toISOString(),
     ...(sdkInfo && { sdk: sdkInfo }),
-
-    // trace context for dynamic sampling on relay
-    trace: {
-      trace_id: event.contexts?.trace?.trace_id,
-      public_key: api.getDsn().publicKey,
-      environment: event.environment || null,
-      release: event.release || null,
-    },
+    ...(tracestate && { trace: tracestate }), // trace context for dynamic sampling on relay
   });
 
   const itemHeaders = JSON.stringify({

packages/core/test/lib/request.test.ts

@@ -58,7 +58,20 @@ describe('eventToSentryRequest', () => {
     beforeEach(() => {
       transactionEvent = {
         ...eventBase,
-        debug_meta: { transactionSampling: { method: TransactionSamplingMethod.Rate, rate: 0.1121 } },
+        debug_meta: {
+          transactionSampling: { method: TransactionSamplingMethod.Rate, rate: 0.1121 },
+          // This value is hardcoded in its base64 form to avoid a dependency on @sentry/tracing, where the method to
+          // compute the value lives. It's equivalent to
+          // computeTracestateValue({
+          //   trace_id: '1231201211212012',
+          //   environment: 'dogpark',
+          //   release: 'off.leash.park',
+          //   public_key: 'dogsarebadatkeepingsecrets',
+          // }),
+          tracestate:
+            'eyJ0cmFjZV9pZCI6IjEyMzEyMDEyMTEyMTIwMTIiLCJlbnZpcm9ubWVudCI6ImRvZ3BhcmsiLCJyZWxlYXNlIjoib2ZmLmxlYXNo' +
+            'LnBhcmsiLCJwdWJsaWNfa2V5IjoiZG9nc2FyZWJhZGF0a2VlcGluZ3NlY3JldHMifQ',
+        },
         spans: [],
         transaction: '/dogs/are/great/',
         type: 'transaction',
@@ -77,7 +90,7 @@ describe('eventToSentryRequest', () => {
     });
 
     describe('envelope header', () => {
-      it('adds correct data to envelope header', () => {
+      it('adds correct entries to envelope header', () => {
         jest.spyOn(Date.prototype, 'toISOString').mockReturnValueOnce('2012-12-31T09:08:13.000Z');
 
         const result = eventToSentryRequest(transactionEvent, api);
@@ -155,7 +168,7 @@ describe('eventToSentryRequest', () => {
     });
 
     describe('item header', () => {
-      it('adds correct data to item header', () => {
+      it('adds correct entries to item header', () => {
         const result = eventToSentryRequest(transactionEvent, api);
         const envelope = parseEnvelopeRequest(result);
 

packages/node/src/integrations/http.ts

@@ -1,5 +1,5 @@
 import { getCurrentHub } from '@sentry/core';
-import { Integration, Span } from '@sentry/types';
+import { Integration, Span, TraceHeaders } from '@sentry/types';
 import { fill, logger, parseSemver } from '@sentry/utils';
 import * as http from 'http';
 import * as https from 'https';
@@ -115,9 +115,9 @@ function _createWrappedRequestMethodFactory(
             op: 'request',
           });
 
-          const sentryTraceHeader = span.toTraceparent();
-          logger.log(`[Tracing] Adding sentry-trace header to outgoing request: ${sentryTraceHeader}`);
-          requestOptions.headers = { ...requestOptions.headers, 'sentry-trace': sentryTraceHeader };
+          const traceHeaders = span.getTraceHeaders();
+          logger.log(`[Tracing] Adding sentry-trace and tracestate headers to outgoing request.`);
+          requestOptions.headers = { ...requestOptions.headers, ...(traceHeaders as TraceHeaders) };
         }
       }
 

packages/node/test/integrations/http.test.ts

@@ -64,21 +64,23 @@ describe('tracing', () => {
     expect((spans[0] as Transaction).name).toEqual('dogpark');
   });
 
-  it('attaches the sentry-trace header to outgoing non-sentry requests', async () => {
+  it('attaches tracing headers to outgoing non-sentry requests', async () => {
     nock('http://dogs.are.great')
       .get('/')
       .reply(200);
 
     createTransactionOnScope();
 
     const request = http.get('http://dogs.are.great/');
-    const sentryTraceHeader = request.getHeader('sentry-trace') as string;
+    const sentryTraceHeader = request.getHeader('sentry-trace');
+    const tracestateHeader = request.getHeader('tracestate');
 
     expect(sentryTraceHeader).toBeDefined();
-    expect(TRACEPARENT_REGEXP.test(sentryTraceHeader)).toBe(true);
+    expect(tracestateHeader).toBeDefined();
+    expect(TRACEPARENT_REGEXP.test(sentryTraceHeader as string)).toBe(true);
   });
 
-  it("doesn't attach the sentry-trace header to outgoing sentry requests", () => {
+  it("doesn't attach tracing headers to outgoing sentry requests", () => {
     nock('http://squirrelchasers.ingest.sentry.io')
       .get('/api/12312012/store/')
       .reply(200);
@@ -87,7 +89,9 @@ describe('tracing', () => {
 
     const request = http.get('http://squirrelchasers.ingest.sentry.io/api/12312012/store/');
     const sentryTraceHeader = request.getHeader('sentry-trace');
+    const tracestateHeader = request.getHeader('tracestate');
 
     expect(sentryTraceHeader).not.toBeDefined();
+    expect(tracestateHeader).not.toBeDefined();
   });
 });

packages/tracing/src/browser/request.ts

@@ -195,14 +195,14 @@ export function fetchCallback(
       // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
       if (typeof headers.append === 'function') {
         // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
-        headers.append('sentry-trace', span.toTraceparent());
+        headers.append(Object.entries(span.getTraceHeaders()));
       } else if (Array.isArray(headers)) {
-        headers = [...headers, ['sentry-trace', span.toTraceparent()]];
+        headers = [...headers, ...Object.entries(span.getTraceHeaders())];
       } else {
-        headers = { ...headers, 'sentry-trace': span.toTraceparent() };
+        headers = { ...headers, ...span.getTraceHeaders() };
       }
     } else {
-      headers = { 'sentry-trace': span.toTraceparent() };
+      headers = span.getTraceHeaders();
     }
     options.headers = headers;
   }
@@ -261,7 +261,11 @@ export function xhrCallback(
 
     if (handlerData.xhr.setRequestHeader) {
       try {
-        handlerData.xhr.setRequestHeader('sentry-trace', span.toTraceparent());
+        const sentryHeaders = span.getTraceHeaders();
+        handlerData.xhr.setRequestHeader('sentry-trace', sentryHeaders['sentry-trace']);
+        if (sentryHeaders.tracestate) {
+          handlerData.xhr.setRequestHeader('tracestate', sentryHeaders.tracestate);
+        }
       } catch (_) {
         // Error: InvalidStateError: Failed to execute 'setRequestHeader' on 'XMLHttpRequest': The object's state must be OPENED.
       }

packages/tracing/src/span.ts

@@ -1,5 +1,5 @@
 /* eslint-disable max-lines */
-import { Primitive, Span as SpanInterface, SpanContext, Transaction } from '@sentry/types';
+import { Primitive, Span as SpanInterface, SpanContext, TraceHeaders, Transaction } from '@sentry/types';
 import { dropUndefinedKeys, timestampWithMs, uuid4 } from '@sentry/utils';
 
 import { SpanStatus } from './spanstatus';
@@ -284,6 +284,19 @@ export class Span implements SpanInterface {
     return this;
   }
 
+  /**
+   * @inheritDoc
+   */
+  public getTraceHeaders(): TraceHeaders {
+    // tracestates live on the transaction, so if this is a free-floating span, there won't be one
+    const tracestate = this.transaction && `sentry=${this.transaction.tracestate}`; // TODO kmclb
+
+    return {
+      'sentry-trace': this.toTraceparent(),
+      ...(tracestate && { tracestate }),
+    };
+  }
+
   /**
    * @inheritDoc
    */

packages/tracing/src/transaction.ts

@@ -1,17 +1,18 @@
 import { getCurrentHub, Hub } from '@sentry/hub';
-import { Event, Measurements, Transaction as TransactionInterface, TransactionContext } from '@sentry/types';
+import { DebugMeta, Event, Measurements, Transaction as TransactionInterface, TransactionContext } from '@sentry/types';
 import { dropUndefinedKeys, isInstanceOf, logger } from '@sentry/utils';
 
 import { Span as SpanClass, SpanRecorder } from './span';
+import { computeTracestateValue } from './utils';
 
-interface TransactionMetadata {
-  transactionSampling?: { [key: string]: string | number };
-}
+type TransactionMetadata = Pick<DebugMeta, 'transactionSampling' | 'tracestate'>;
 
 /** JSDoc */
 export class Transaction extends SpanClass implements TransactionInterface {
   public name: string;
 
+  public readonly tracestate: string;
+
   private _metadata: TransactionMetadata = {};
 
   private _measurements: Measurements = {};
@@ -41,6 +42,10 @@ export class Transaction extends SpanClass implements TransactionInterface {
 
     this._trimEnd = transactionContext.trimEnd;
 
+    // _getNewTracestate only returns undefined in the absence of a client or dsn, in which case it doesn't matter what
+    // the header values are - nothing can be sent anyway - so the third alternative here is just to make TS happy
+    this.tracestate = transactionContext.tracestate || this._getNewTracestate() || 'things are broken';
+
     // this is because transactions are also spans, and spans have a transaction pointer
     this.transaction = this;
   }
@@ -113,6 +118,8 @@ export class Transaction extends SpanClass implements TransactionInterface {
       }).endTimestamp;
     }
 
+    this._metadata.tracestate = this.tracestate;
+
     const transaction: Event = {
       contexts: {
         trace: this.getTraceContext(),
@@ -161,4 +168,27 @@ export class Transaction extends SpanClass implements TransactionInterface {
 
     return this;
   }
+
+  /**
+   * Create a new tracestate header value
+   *
+   * @returns The new tracestate value, or undefined if there's no client or no dsn
+   */
+  private _getNewTracestate(): string | undefined {
+    const client = this._hub.getClient();
+    const dsn = client?.getDsn();
+
+    if (!client || !dsn) {
+      return;
+    }
+
+    const { environment, release } = client.getOptions() || {};
+
+    // TODO - the only reason we need the non-null assertion on `dsn.publicKey` (below) is because `dsn.publicKey` has
+    // to be optional while we transition from `dsn.user` -> `dsn.publicKey`. Once `dsn.user` is removed, we can make
+    // `dsn.publicKey` required and remove the `!`.
+
+    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+    return computeTracestateValue({ trace_id: this.traceId, environment, release, public_key: dsn.publicKey! });
+  }
 }

packages/tracing/src/utils.ts

@@ -1,5 +1,6 @@
 import { getCurrentHub, Hub } from '@sentry/hub';
 import { Options, TraceparentData, Transaction } from '@sentry/types';
+import { SentryError, unicodeToBase64 } from '@sentry/utils';
 
 export const TRACEPARENT_REGEXP = new RegExp(
   '^[ \\t]*' + // whitespace
@@ -66,3 +67,35 @@ export function secToMs(time: number): number {
 
 // so it can be used in manual instrumentation without necessitating a hard dependency on @sentry/utils
 export { stripUrlQueryAndFragment } from '@sentry/utils';
+
+/**
+ * Compute the value of a tracestate header.
+ *
+ * @throws SentryError (because using the logger creates a circular dependency)
+ * @returns the base64-encoded header value
+ */
+// Note: this is here instead of in the tracing package since @sentry/core tests rely on it
+export function computeTracestateValue(tracestateData: {
+  trace_id: string;
+  environment: string | undefined | null;
+  release: string | undefined | null;
+  public_key: string;
+}): string {
+  // `JSON.stringify` will drop keys with undefined values, but not ones with null values
+  tracestateData.environment = tracestateData.environment || null;
+  tracestateData.release = tracestateData.release || null;
+
+  // See https://www.w3.org/TR/trace-context/#tracestate-header-field-values
+  // The spec for tracestate header values calls for a string of the form
+  //
+  //    identifier1=value1,identifier2=value2,...
+  //
+  // which means the value can't include any equals signs, since they already have meaning. Equals signs are commonly
+  // used to pad the end of base64 values though, so to avoid confusion, we strip them off. (Most languages' base64
+  // decoding functions (including those in JS) are able to function without the padding.)
+  try {
+    return unicodeToBase64(JSON.stringify(tracestateData)).replace(/={1,2}$/, '');
+  } catch (err) {
+    throw new SentryError(`[Tracing] Error creating tracestate header: ${err}`);
+  }
+}