[WIP] feat: Send transactions in envelopes

The new Envelope endpoint and format is what we want to use for
transactions going forward.

TODO: include rationale.

Author: rhcarvalho

packages/browser/src/integrations/breadcrumbs.ts

@@ -170,7 +170,7 @@ export class Breadcrumbs implements Integration {
     const client = getCurrentHub().getClient<BrowserClient>();
     const dsn = client && client.getDsn();
     if (this._options.sentry && dsn) {
-      const filterUrl = new API(dsn).getStoreEndpoint();
+      const filterUrl = new API(dsn).getBaseApiEndpoint();
       // if Sentry key appears in URL, don't capture it as a request
       // but rather as our own 'sentry' type breadcrumb
       if (

packages/browser/src/transports/base.ts

@@ -5,15 +5,19 @@ import { PromiseBuffer, SentryError } from '@sentry/utils';
 /** Base Transport class implementation */
 export abstract class BaseTransport implements Transport {
   /**
-   * @inheritDoc
+   * @deprecated
    */
   public url: string;
 
+  /** Helper to get Sentry API endpoints. */
+  protected readonly _api: API;
+
   /** A simple buffer holding all requests. */
   protected readonly _buffer: PromiseBuffer<Response> = new PromiseBuffer(30);
 
   public constructor(public options: TransportOptions) {
-    this.url = new API(this.options.dsn).getStoreEndpointWithUrlEncodedAuth();
+    this._api = new API(this.options.dsn);
+    this.url = this._api.getStoreEndpointWithUrlEncodedAuth();
   }
 
   /**

packages/browser/src/transports/fetch.ts

@@ -1,3 +1,4 @@
+import { eventToSentryRequest } from '@sentry/core';
 import { Event, Response, Status } from '@sentry/types';
 import { getGlobalObject, logger, parseRetryAfterHeader, supportsReferrerPolicy, SyncPromise } from '@sentry/utils';
 
@@ -22,8 +23,10 @@ export class FetchTransport extends BaseTransport {
       });
     }
 
-    const defaultOptions: RequestInit = {
-      body: JSON.stringify(event),
+    const sentryReq = eventToSentryRequest(event, this._api);
+
+    const options: RequestInit = {
+      body: sentryReq.body,
       method: 'POST',
       // Despite all stars in the sky saying that Edge supports old draft syntax, aka 'never', 'always', 'origin' and 'default
       // https://caniuse.com/#feat=referrer-policy
@@ -33,13 +36,13 @@ export class FetchTransport extends BaseTransport {
     };
 
     if (this.options.headers !== undefined) {
-      defaultOptions.headers = this.options.headers;
+      options.headers = this.options.headers;
     }
 
     return this._buffer.add(
       new SyncPromise<Response>((resolve, reject) => {
         global
-          .fetch(this.url, defaultOptions)
+          .fetch(sentryReq.url, options)
           .then(response => {
             const status = Status.fromHttpCode(response.status);
 

packages/browser/src/transports/xhr.ts

@@ -1,3 +1,4 @@
+import { eventToSentryRequest } from '@sentry/core';
 import { Event, Response, Status } from '@sentry/types';
 import { logger, parseRetryAfterHeader, SyncPromise } from '@sentry/utils';
 
@@ -20,6 +21,8 @@ export class XHRTransport extends BaseTransport {
       });
     }
 
+    const sentryReq = eventToSentryRequest(event, this._api);
+
     return this._buffer.add(
       new SyncPromise<Response>((resolve, reject) => {
         const request = new XMLHttpRequest();
@@ -45,13 +48,13 @@ export class XHRTransport extends BaseTransport {
           reject(request);
         };
 
-        request.open('POST', this.url);
+        request.open('POST', sentryReq.url);
         for (const header in this.options.headers) {
           if (this.options.headers.hasOwnProperty(header)) {
             request.setRequestHeader(header, this.options.headers[header]);
           }
         }
-        request.send(JSON.stringify(event));
+        request.send(sentryReq.body);
       }),
     );
   }

packages/core/src/api.ts

@@ -17,29 +17,54 @@ export class API {
     return this._dsnObject;
   }
 
-  /** Returns a string with auth headers in the url to the store endpoint. */
+  /** Returns the prefix to construct Sentry ingestion API endpoints. */
+  public getBaseApiEndpoint(): string {
+    const dsn = this._dsnObject;
+    const protocol = dsn.protocol ? `${dsn.protocol}:` : '';
+    const port = dsn.port ? `:${dsn.port}` : '';
+    return `${protocol}//${dsn.host}${port}${dsn.path ? `/${dsn.path}` : ''}/api/`;
+  }
+
+  /** Returns the store endpoint URL. */
   public getStoreEndpoint(): string {
-    return `${this._getBaseUrl()}${this.getStoreEndpointPath()}`;
+    return this._getIngestEndpoint('store');
+  }
+
+  /** Returns the envelope endpoint URL. */
+  public getEnvelopeEndpoint(): string {
+    return this._getIngestEndpoint('envelope');
+  }
+
+  private _getIngestEndpoint(target: string): string {
+    const base = this.getBaseApiEndpoint();
+    const dsn = this._dsnObject;
+    return `${base}${dsn.projectId}/${target}/`;
   }
 
-  /** Returns the store endpoint with auth added in url encoded. */
+  /** Returns the store endpoint URL with auth in the query string.
+   *
+   * Sending auth as part of the query string and not as custom HTTP headers avoids CORS preflight requests.
+   */
   public getStoreEndpointWithUrlEncodedAuth(): string {
+    return `${this.getStoreEndpoint()}?${this._encodedAuth()}`;
+  }
+
+  /** Returns the envelope endpoint URL with auth in the query string.
+   *
+   * Sending auth as part of the query string and not as custom HTTP headers avoids CORS preflight requests.
+   */
+  public getEnvelopeEndpointWithUrlEncodedAuth(): string {
+    return `${this.getEnvelopeEndpoint()}?${this._encodedAuth()}`;
+  }
+
+  private _encodedAuth(): string {
     const dsn = this._dsnObject;
     const auth = {
-      sentry_key: dsn.user, // sentry_key is currently used in tracing integration to identify internal sentry requests
+      sentry_key: dsn.user,
       sentry_version: SENTRY_API_VERSION,
+      // REVIEW: don't we send sentry_client (with SDK info) when using this?! Compare with `getRequestHeaders`.
     };
-    // Auth is intentionally sent as part of query string (NOT as custom HTTP header)
-    // to avoid preflight CORS requests
-    return `${this.getStoreEndpoint()}?${urlEncode(auth)}`;
-  }
-
-  /** Returns the base path of the url including the port. */
-  private _getBaseUrl(): string {
-    const dsn = this._dsnObject;
-    const protocol = dsn.protocol ? `${dsn.protocol}:` : '';
-    const port = dsn.port ? `:${dsn.port}` : '';
-    return `${protocol}//${dsn.host}${port}`;
+    return urlEncode(auth);
   }
 
   /** Returns only the path component for the store endpoint. */
@@ -48,7 +73,10 @@ export class API {
     return `${dsn.path ? `/${dsn.path}` : ''}/api/${dsn.projectId}/store/`;
   }
 
-  /** Returns an object that can be used in request headers. */
+  /** Returns an object that can be used in request headers.
+   *
+   * @deprecated in favor of `getStoreEndpointWithUrlEncodedAuth` and `getEnvelopeEndpointWithUrlEncodedAuth`.
+   */
   public getRequestHeaders(clientName: string, clientVersion: string): { [key: string]: string } {
     const dsn = this._dsnObject;
     const header = [`Sentry sentry_version=${SENTRY_API_VERSION}`];
@@ -71,7 +99,7 @@ export class API {
     } = {},
   ): string {
     const dsn = this._dsnObject;
-    const endpoint = `${this._getBaseUrl()}${dsn.path ? `/${dsn.path}` : ''}/api/embed/error-page/`;
+    const endpoint = `${this.getBaseApiEndpoint()}embed/error-page/`;
 
     const encodedOptions = [];
     encodedOptions.push(`dsn=${dsn.toString()}`);

packages/core/src/index.ts

@@ -16,6 +16,7 @@ export { addGlobalEventProcessor, getCurrentHub, getHubFromCarrier, Hub, Scope }
 export { API } from './api';
 export { BaseClient } from './baseclient';
 export { BackendClass, BaseBackend } from './basebackend';
+export { eventToSentryRequest } from './request';
 export { initAndBind, ClientClass } from './sdk';
 export { NoopTransport } from './transports/noop';
 

packages/core/src/request.ts

@@ -0,0 +1,60 @@
+import { Event } from '@sentry/types';
+import { API } from './api';
+
+/** A generic client request. */
+interface SentryRequest {
+  // url to be used in @sentry/browser and @sentry/node.
+  // NOTE: nodejs v10.9.0 + can use url directly, older versions need an adaptor
+  // to split url into parts.
+  url: string;
+
+  // headers would contain auth & content-type headers for @sentry/node, but
+  // since @sentry/browser avoids custom headers to prevent CORS preflight
+  // requests, we can use the same approach for @sentry/browser and @sentry/node
+  // for simplicity -- no headers involved.
+  //headers: { [key: string]: string };
+
+  // body is either json (errors) || envelope (transactions)
+  body: string;
+}
+
+export function eventToSentryRequest(event: Event, api: API): SentryRequest {
+  const useEnvelope = event.type === 'transaction';
+
+  let req: SentryRequest = {
+    url: useEnvelope ? api.getEnvelopeEndpointWithUrlEncodedAuth() : api.getStoreEndpointWithUrlEncodedAuth(),
+    body: JSON.stringify(event),
+  };
+
+  // https://docs.sentry.io/development/sdk-dev/envelopes/
+
+  // Since we don't need to manipulate envelopes nor store them, there is no
+  // exported concept of an Envelope with operations including serialization and
+  // deserialization. Instead, we only implement a minimal subset of the spec to
+  // serialize events inline here.
+  if (useEnvelope) {
+    const envelopeHeaders = JSON.stringify({
+      event_id: event.event_id,
+      sent_at: new Date().toISOString(),
+    });
+    const itemHeaders = JSON.stringify({
+      type: event.type,
+      // The content-type is assumed to be 'application/json' and not part of
+      // the current spec for transaction items, so we don't bloat the request
+      // body with it.
+      //content_type: 'application/json',
+      // The length is optional. It must be the number of bytes in req.Body
+      // encoded as UTF-8. Since the server can figure this out and would
+      // otherwise refuse events that report the length incorrectly, we decided
+      // not to send the length to avoid problems and reduce request body size.
+      //length: new TextEncoder().encode(req.body).length,
+    });
+    // The trailing newline is optional. We intentionally don't send it to avoid
+    // sending unnecessary bytes.
+    //const envelope = `${envelopeHeaders}\n${itemHeaders}\n${req.body}\n`;
+    const envelope = `${envelopeHeaders}\n${itemHeaders}\n${req.body}`;
+    req.body = envelope;
+  }
+
+  return req;
+}

packages/node/src/transports/base.ts

@@ -1,12 +1,12 @@
-import { API } from '@sentry/core';
+import { API, eventToSentryRequest } from '@sentry/core';
 import { Event, Response, Status, Transport, TransportOptions } from '@sentry/types';
-import { logger, parseRetryAfterHeader, PromiseBuffer, SentryError, isString } from '@sentry/utils';
+import { logger, parseRetryAfterHeader, PromiseBuffer, SentryError } from '@sentry/utils';
 import * as fs from 'fs';
 import * as http from 'http';
 import * as https from 'https';
 import * as url from 'url';
 
-import { SDK_NAME, SDK_VERSION } from '../version';
+// import { SDK_NAME, SDK_VERSION } from '../version';
 
 /**
  * Internal used interface for typescript.
@@ -23,7 +23,9 @@ export interface HTTPRequest {
     callback?: (res: http.IncomingMessage) => void,
   ): http.ClientRequest;
 
-  // This is the new type for versions that handle URL argument correctly, but it's most likely not needed here just yet
+  // This is the type for nodejs versions that handle the URL argument
+  // (v10.9.0+), but we do not use it just yet because we support older node
+  // versions:
 
   // request(
   //   url: string | url.URL,
@@ -55,17 +57,15 @@ export abstract class BaseTransport implements Transport {
   }
 
   /** Returns a build request option object used by request */
-  protected _getRequestOptions(address: string | url.URL): http.RequestOptions | https.RequestOptions {
-    if (!isString(address) || !(address instanceof url.URL)) {
-      throw new SentryError(`Incorrect transport url: ${address}`);
-    }
-
+  protected _getRequestOptions(url: url.URL): http.RequestOptions | https.RequestOptions {
     const headers = {
-      ...this._api.getRequestHeaders(SDK_NAME, SDK_VERSION),
+      // The auth headers are not included because auth is done via query string to match @sentry/browser.
+      //...this._api.getRequestHeaders(SDK_NAME, SDK_VERSION)
       ...this.options.headers,
     };
-    const addr = address instanceof url.URL ? address : new url.URL(address);
-    const { hostname, pathname: path, port, protocol } = addr;
+    const { hostname, port, protocol } = url;
+    // See https://github.com/nodejs/node/blob/38146e717fed2fabe3aacb6540d839475e0ce1c6/lib/internal/url.js#L1268-L1290
+    const path = `${url.pathname || ''}${url.search || ''}`;
 
     return {
       agent: this.client,
@@ -92,7 +92,10 @@ export abstract class BaseTransport implements Transport {
     }
     return this._buffer.add(
       new Promise<Response>((resolve, reject) => {
-        const req = httpModule.request(this._getRequestOptions('http://foo.com/123'), (res: http.IncomingMessage) => {
+        const sentryReq = eventToSentryRequest(event, this._api);
+        const options = this._getRequestOptions(new url.URL(sentryReq.url));
+
+        const req = httpModule.request(options, (res: http.IncomingMessage) => {
           const statusCode = res.statusCode || 500;
           const status = Status.fromHttpCode(statusCode);
 
@@ -126,7 +129,7 @@ export abstract class BaseTransport implements Transport {
           });
         });
         req.on('error', reject);
-        req.end(JSON.stringify(event));
+        req.end(sentryReq.body);
       }),
     );
   }

packages/node/test/transports/http.test.ts

@@ -6,6 +6,7 @@ import { HTTPTransport } from '../../src/transports/http';
 
 const mockSetEncoding = jest.fn();
 const dsn = 'http://[email protected]:8989/mysubpath/50622';
+const transportPath = '/mysubpath/api/50622/store/?sentry_key=9e9fd4523d784609a5fc0ebb1080592f&sentry_version=7';
 let mockReturnCode = 200;
 let mockHeaders = {};
 
@@ -27,11 +28,11 @@ function createTransport(options: TransportOptions): HTTPTransport {
 }
 
 function assertBasicOptions(options: any): void {
-  expect(options.headers['X-Sentry-Auth']).toContain('sentry_version');
-  expect(options.headers['X-Sentry-Auth']).toContain('sentry_client');
-  expect(options.headers['X-Sentry-Auth']).toContain('sentry_key');
+  // expect(options.headers['X-Sentry-Auth']).toContain('sentry_version');
+  // expect(options.headers['X-Sentry-Auth']).toContain('sentry_client');
+  // expect(options.headers['X-Sentry-Auth']).toContain('sentry_key');
   expect(options.port).toEqual('8989');
-  expect(options.path).toEqual('/mysubpath/api/50622/store/');
+  expect(options.path).toEqual(transportPath);
   expect(options.hostname).toEqual('sentry.io');
 }
 

packages/node/test/transports/https.test.ts

@@ -6,6 +6,7 @@ import { HTTPSTransport } from '../../src/transports/https';
 
 const mockSetEncoding = jest.fn();
 const dsn = 'https://[email protected]:8989/mysubpath/50622';
+const transportPath = '/mysubpath/api/50622/store/?sentry_key=9e9fd4523d784609a5fc0ebb1080592f&sentry_version=7';
 let mockReturnCode = 200;
 let mockHeaders = {};
 
@@ -33,11 +34,11 @@ function createTransport(options: TransportOptions): HTTPSTransport {
 }
 
 function assertBasicOptions(options: any): void {
-  expect(options.headers['X-Sentry-Auth']).toContain('sentry_version');
-  expect(options.headers['X-Sentry-Auth']).toContain('sentry_client');
-  expect(options.headers['X-Sentry-Auth']).toContain('sentry_key');
+  // expect(options.headers['X-Sentry-Auth']).toContain('sentry_version');
+  // expect(options.headers['X-Sentry-Auth']).toContain('sentry_client');
+  // expect(options.headers['X-Sentry-Auth']).toContain('sentry_key');
   expect(options.port).toEqual('8989');
-  expect(options.path).toEqual('/mysubpath/api/50622/store/');
+  expect(options.path).toEqual(transportPath);
   expect(options.hostname).toEqual('sentry.io');
 }