feat(browser): Add IndexedDb offline transport store (#6983)

timfish · web-flow · commit b539b36382ad · 2023-02-08T14:53:15.000+01:00
diff --git a/packages/browser/package.json b/packages/browser/package.json
@@ -27,6 +27,7 @@
     "btoa": "^1.2.1",
     "chai": "^4.1.2",
     "chokidar": "^3.0.2",
+    "fake-indexeddb": "^4.0.1",
     "karma": "^6.3.16",
     "karma-chai": "^0.1.0",
     "karma-chrome-launcher": "^2.2.0",
diff --git a/packages/browser/src/index.ts b/packages/browser/src/index.ts
@@ -21,10 +21,14 @@ const INTEGRATIONS = {
 export { INTEGRATIONS as Integrations };
 
 // DO NOT DELETE THESE COMMENTS!
-// We want to exclude Replay from CDN bundles, so we remove the block below with our
-// excludeReplay Rollup plugin when generating bundles. Everything between
-// ROLLUP_EXCLUDE_FROM_BUNDLES_BEGIN and _END__ is removed for bundles.
+// We want to exclude Replay/Offline from CDN bundles, so we remove the block below with our
+// makeExcludeBlockPlugin Rollup plugin when generating bundles. Everything between
+// ROLLUP_EXCLUDE_*_FROM_BUNDLES_BEGIN and _END__ is removed for bundles.
 
-// __ROLLUP_EXCLUDE_FROM_BUNDLES_BEGIN__
+// __ROLLUP_EXCLUDE_REPLAY_FROM_BUNDLES_BEGIN__
 export { Replay } from '@sentry/replay';
-// __ROLLUP_EXCLUDE_FROM_BUNDLES_END__
+// __ROLLUP_EXCLUDE_REPLAY_FROM_BUNDLES_END__
+
+// __ROLLUP_EXCLUDE_OFFLINE_FROM_BUNDLES_BEGIN__
+export { makeBrowserOfflineTransport } from './transports/offline';
+// __ROLLUP_EXCLUDE_OFFLINE_FROM_BUNDLES_END__
diff --git a/packages/browser/src/transports/offline.ts b/packages/browser/src/transports/offline.ts
@@ -0,0 +1,158 @@
+import type { OfflineStore, OfflineTransportOptions } from '@sentry/core';
+import { makeOfflineTransport } from '@sentry/core';
+import type { Envelope, InternalBaseTransportOptions, Transport } from '@sentry/types';
+import type { TextDecoderInternal } from '@sentry/utils';
+import { parseEnvelope, serializeEnvelope } from '@sentry/utils';
+
+// 'Store', 'promisifyRequest' and 'createStore' were originally copied from the 'idb-keyval' package before being
+// modified and simplified: https://github.com/jakearchibald/idb-keyval
+//
+// At commit: 0420a704fd6cbb4225429c536b1f61112d012fca
+// Original licence:
+
+// Copyright 2016, Jake Archibald
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+type Store = <T>(callback: (store: IDBObjectStore) => T | PromiseLike<T>) => Promise<T>;
+
+function promisifyRequest<T = undefined>(request: IDBRequest<T> | IDBTransaction): Promise<T> {
+  return new Promise<T>((resolve, reject) => {
+    // @ts-ignore - file size hacks
+    request.oncomplete = request.onsuccess = () => resolve(request.result);
+    // @ts-ignore - file size hacks
+    request.onabort = request.onerror = () => reject(request.error);
+  });
+}
+
+/** Create or open an IndexedDb store */
+export function createStore(dbName: string, storeName: string): Store {
+  const request = indexedDB.open(dbName);
+  request.onupgradeneeded = () => request.result.createObjectStore(storeName);
+  const dbp = promisifyRequest(request);
+
+  return callback => dbp.then(db => callback(db.transaction(storeName, 'readwrite').objectStore(storeName)));
+}
+
+function keys(store: IDBObjectStore): Promise<number[]> {
+  return promisifyRequest(store.getAllKeys() as IDBRequest<number[]>);
+}
+
+/** Insert into the store */
+export function insert(store: Store, value: Uint8Array | string, maxQueueSize: number): Promise<void> {
+  return store(store => {
+    return keys(store).then(keys => {
+      if (keys.length >= maxQueueSize) {
+        return;
+      }
+
+      // We insert with an incremented key so that the entries are popped in order
+      store.put(value, Math.max(...keys, 0) + 1);
+      return promisifyRequest(store.transaction);
+    });
+  });
+}
+
+/** Pop the oldest value from the store */
+export function pop(store: Store): Promise<Uint8Array | string | undefined> {
+  return store(store => {
+    return keys(store).then(keys => {
+      if (keys.length === 0) {
+        return undefined;
+      }
+
+      return promisifyRequest(store.get(keys[0])).then(value => {
+        store.delete(keys[0]);
+        return promisifyRequest(store.transaction).then(() => value);
+      });
+    });
+  });
+}
+
+interface BrowserOfflineTransportOptions extends OfflineTransportOptions {
+  /**
+   * Name of indexedDb database to store envelopes in
+   * Default: 'sentry-offline'
+   */
+  dbName?: string;
+  /**
+   * Name of indexedDb object store to store envelopes in
+   * Default: 'queue'
+   */
+  storeName?: string;
+  /**
+   * Maximum number of envelopes to store
+   * Default: 30
+   */
+  maxQueueSize?: number;
+  /**
+   * Only required for testing on node.js
+   * @ignore
+   */
+  textDecoder?: TextDecoderInternal;
+}
+
+function createIndexedDbStore(options: BrowserOfflineTransportOptions): OfflineStore {
+  let store: Store | undefined;
+
+  // Lazily create the store only when it's needed
+  function getStore(): Store {
+    if (store == undefined) {
+      store = createStore(options.dbName || 'sentry-offline', options.storeName || 'queue');
+    }
+
+    return store;
+  }
+
+  return {
+    insert: async (env: Envelope) => {
+      try {
+        const serialized = await serializeEnvelope(env, options.textEncoder);
+        await insert(getStore(), serialized, options.maxQueueSize || 30);
+      } catch (_) {
+        //
+      }
+    },
+    pop: async () => {
+      try {
+        const deserialized = await pop(getStore());
+        if (deserialized) {
+          return parseEnvelope(
+            deserialized,
+            options.textEncoder || new TextEncoder(),
+            options.textDecoder || new TextDecoder(),
+          );
+        }
+      } catch (_) {
+        //
+      }
+
+      return undefined;
+    },
+  };
+}
+
+function makeIndexedDbOfflineTransport<T>(
+  createTransport: (options: T) => Transport,
+): (options: T & BrowserOfflineTransportOptions) => Transport {
+  return options => createTransport({ ...options, createStore: createIndexedDbStore });
+}
+
+/**
+ * Creates a transport that uses IndexedDb to store events when offline.
+ */
+export function makeBrowserOfflineTransport<T extends InternalBaseTransportOptions>(
+  createTransport: (options: T) => Transport,
+): (options: T & BrowserOfflineTransportOptions) => Transport {
+  return makeIndexedDbOfflineTransport<T>(makeOfflineTransport(createTransport));
+}
diff --git a/packages/browser/test/unit/transports/offline.test.ts b/packages/browser/test/unit/transports/offline.test.ts
@@ -0,0 +1,111 @@
+import 'fake-indexeddb/auto';
+
+import { createTransport } from '@sentry/core';
+import type {
+  EventEnvelope,
+  EventItem,
+  InternalBaseTransportOptions,
+  TransportMakeRequestResponse,
+} from '@sentry/types';
+import { createEnvelope } from '@sentry/utils';
+import { TextDecoder, TextEncoder } from 'util';
+
+import { MIN_DELAY } from '../../../../core/src/transports/offline';
+import { createStore, insert, makeBrowserOfflineTransport, pop } from '../../../src/transports/offline';
+
+function deleteDatabase(name: string): Promise<void> {
+  return new Promise<void>((resolve, reject) => {
+    const request = indexedDB.deleteDatabase(name);
+    request.onsuccess = () => resolve();
+    request.onerror = () => reject(request.error);
+  });
+}
+
+const ERROR_ENVELOPE = createEnvelope<EventEnvelope>({ event_id: 'aa3ff046696b4bc6b609ce6d28fde9e2', sent_at: '123' }, [
+  [{ type: 'event' }, { event_id: 'aa3ff046696b4bc6b609ce6d28fde9e2' }] as EventItem,
+]);
+
+const transportOptions = {
+  recordDroppedEvent: () => undefined, // noop
+  textEncoder: new TextEncoder(),
+  textDecoder: new TextDecoder(),
+};
+
+type MockResult<T> = T | Error;
+
+export const createTestTransport = (...sendResults: MockResult<TransportMakeRequestResponse>[]) => {
+  let sendCount = 0;
+
+  return {
+    getSendCount: () => sendCount,
+    baseTransport: (options: InternalBaseTransportOptions) =>
+      createTransport(options, () => {
+        return new Promise((resolve, reject) => {
+          const next = sendResults.shift();
+
+          if (next instanceof Error) {
+            reject(next);
+          } else {
+            sendCount += 1;
+            resolve(next as TransportMakeRequestResponse | undefined);
+          }
+        });
+      }),
+  };
+};
+
+function delay(ms: number): Promise<void> {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+describe('makeOfflineTransport', () => {
+  beforeAll(async () => {
+    await deleteDatabase('sentry');
+  });
+
+  it('indexedDb wrappers insert and pop', async () => {
+    const store = createStore('test', 'test');
+    const found = await pop(store);
+    expect(found).toBeUndefined();
+
+    await insert(store, 'test1', 30);
+    await insert(store, new Uint8Array([1, 2, 3, 4, 5]), 30);
+
+    const found2 = await pop(store);
+    expect(found2).toEqual('test1');
+    const found3 = await pop(store);
+    expect(found3).toEqual(new Uint8Array([1, 2, 3, 4, 5]));
+
+    const found4 = await pop(store);
+    expect(found4).toBeUndefined();
+  });
+
+  it('Queues and retries envelope if wrapped transport throws error', async () => {
+    const { getSendCount, baseTransport } = createTestTransport(new Error(), { statusCode: 200 }, { statusCode: 200 });
+    let queuedCount = 0;
+    const transport = makeBrowserOfflineTransport(baseTransport)({
+      ...transportOptions,
+      shouldStore: () => {
+        queuedCount += 1;
+        return true;
+      },
+    });
+    const result = await transport.send(ERROR_ENVELOPE);
+
+    expect(result).toEqual({});
+
+    await delay(MIN_DELAY * 2);
+
+    expect(getSendCount()).toEqual(0);
+    expect(queuedCount).toEqual(1);
+
+    // Sending again will retry the queued envelope too
+    const result2 = await transport.send(ERROR_ENVELOPE);
+    expect(result2).toEqual({ statusCode: 200 });
+
+    await delay(MIN_DELAY * 2);
+
+    expect(queuedCount).toEqual(1);
+    expect(getSendCount()).toEqual(2);
+  });
+});
diff --git a/packages/integration-tests/suites/transport/offline/init.js b/packages/integration-tests/suites/transport/offline/init.js
@@ -0,0 +1,8 @@
+import * as Sentry from '@sentry/browser';
+
+window.Sentry = Sentry;
+
+Sentry.init({
+  dsn: 'https://public@dsn.ingest.sentry.io/1337',
+  transport: Sentry.makeBrowserOfflineTransport(Sentry.makeFetchTransport),
+});
diff --git a/packages/integration-tests/suites/transport/offline/queued/subject.js b/packages/integration-tests/suites/transport/offline/queued/subject.js
@@ -0,0 +1,3 @@
+setTimeout(() => {
+  Sentry.captureMessage(`foo ${Math.random()}`);
+}, 500);
diff --git a/packages/integration-tests/suites/transport/offline/queued/test.ts b/packages/integration-tests/suites/transport/offline/queued/test.ts
@@ -0,0 +1,51 @@
+import { expect } from '@playwright/test';
+import type { Event } from '@sentry/types';
+
+import { sentryTest } from '../../../../utils/fixtures';
+import { getMultipleSentryEnvelopeRequests } from '../../../../utils/helpers';
+
+function delay(ms: number) {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+sentryTest('should queue and retry events when they fail to send', async ({ getLocalTestPath, page }) => {
+  // makeBrowserOfflineTransport is not included in any CDN bundles
+  if (process.env.PW_BUNDLE && process.env.PW_BUNDLE.startsWith('bundle_')) {
+    sentryTest.skip();
+  }
+
+  const url = await getLocalTestPath({ testDir: __dirname });
+
+  // This would be the obvious way to test offline support but it doesn't appear to work!
+  // await context.setOffline(true);
+
+  let abortedCount = 0;
+
+  // Abort all envelope requests so the event gets queued
+  await page.route(/ingest\.sentry\.io/, route => {
+    abortedCount += 1;
+    return route.abort();
+  });
+  await page.goto(url);
+  await delay(1_000);
+  await page.unroute(/ingest\.sentry\.io/);
+
+  expect(abortedCount).toBe(1);
+
+  // The previous event should now be queued
+
+  // This will force the page to be reloaded and a new event to be sent
+  const eventData = await getMultipleSentryEnvelopeRequests<Event>(page, 3, { url, timeout: 10_000 });
+
+  // Filter out any client reports
+  const events = eventData.filter(e => !('discarded_events' in e)) as Event[];
+
+  expect(events).toHaveLength(2);
+
+  // The next two events will be message events starting with 'foo'
+  expect(events[0].message?.startsWith('foo'));
+  expect(events[1].message?.startsWith('foo'));
+
+  // But because these are two different events, they should have different random numbers in the message
+  expect(events[0].message !== events[1].message);
+});
diff --git a/packages/utils/src/envelope.ts b/packages/utils/src/envelope.ts
@@ -115,7 +115,7 @@ function concatBuffers(buffers: Uint8Array[]): Uint8Array {
   return merged;
 }
 
-interface TextDecoderInternal {
+export interface TextDecoderInternal {
   decode(input?: Uint8Array): string;
 }
 
diff --git a/rollup/bundleHelpers.js b/rollup/bundleHelpers.js
diff --git a/rollup/plugins/bundlePlugins.js b/rollup/plugins/bundlePlugins.js
diff --git a/yarn.lock b/yarn.lock

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+setTimeout(() => {`
	`2`	+ Sentry.captureMessage(`foo ${Math.random()}`);
	`3`	`+}, 500);`
Original file line number	Diff line number	Diff line change
`@@ -115,7 +115,7 @@ function concatBuffers(buffers: Uint8Array[]): Uint8Array {`
`115`	`115`	`return merged;`
`116`	`116`	`}`
`117`	`117`
`118`		`-interface TextDecoderInternal {`
	`118`	`+export interface TextDecoderInternal {`
`119`	`119`	`decode(input?: Uint8Array): string;`
`120`	`120`	`}`
`121`	`121`