From eee77c1f28c15d0965a792982b5aa84a8c1841d9 Mon Sep 17 00:00:00 2001 From: ziyad-elabid-nw Date: Fri, 30 Jun 2023 17:34:36 +0100 Subject: [PATCH 1/4] Add a new option NetworkDetailExcludeUrls --- .../coreHandlers/handleNetworkBreadcrumbs.ts | 10 ++- .../src/coreHandlers/util/fetchUtils.ts | 3 +- packages/replay/src/integration.ts | 2 + packages/replay/src/types/replay.ts | 15 +++- .../handleNetworkBreadcrumbs.test.ts | 83 +++++++++++++++++++ .../replay/test/utils/setupReplayContainer.ts | 1 + 6 files changed, 108 insertions(+), 6 deletions(-) diff --git a/packages/replay/src/coreHandlers/handleNetworkBreadcrumbs.ts b/packages/replay/src/coreHandlers/handleNetworkBreadcrumbs.ts index 59324031e336..6624da6adfbb 100644 --- a/packages/replay/src/coreHandlers/handleNetworkBreadcrumbs.ts +++ b/packages/replay/src/coreHandlers/handleNetworkBreadcrumbs.ts @@ -31,13 +31,19 @@ export function handleNetworkBreadcrumbs(replay: ReplayContainer): void { try { const textEncoder = new TextEncoder(); - const { networkDetailAllowUrls, networkCaptureBodies, networkRequestHeaders, networkResponseHeaders } = - replay.getOptions(); + const { + networkDetailAllowUrls, + networkDetailExcludeUrls, + networkCaptureBodies, + networkRequestHeaders, + networkResponseHeaders, + } = replay.getOptions(); const options: ExtendedNetworkBreadcrumbsOptions = { replay, textEncoder, networkDetailAllowUrls, + networkDetailExcludeUrls, networkCaptureBodies, networkRequestHeaders, networkResponseHeaders, diff --git a/packages/replay/src/coreHandlers/util/fetchUtils.ts b/packages/replay/src/coreHandlers/util/fetchUtils.ts index 9411332d0197..af6c94527e9c 100644 --- a/packages/replay/src/coreHandlers/util/fetchUtils.ts +++ b/packages/replay/src/coreHandlers/util/fetchUtils.ts @@ -85,7 +85,8 @@ async function _prepareFetchData( response_body_size: responseBodySize, } = breadcrumb.data; - const captureDetails = urlMatches(url, options.networkDetailAllowUrls); + const captureDetails = + urlMatches(url, options.networkDetailAllowUrls) && !urlMatches(url, options.networkDetailExcludeUrls); const request = captureDetails ? _getRequestInfo(options, hint.input, requestBodySize) diff --git a/packages/replay/src/integration.ts b/packages/replay/src/integration.ts index 2baf117b5c38..b8f7d38b5059 100644 --- a/packages/replay/src/integration.ts +++ b/packages/replay/src/integration.ts @@ -67,6 +67,7 @@ export class Replay implements Integration { slowClickIgnoreSelectors = [], networkDetailAllowUrls = [], + networkDetailExcludeUrls = [], networkCaptureBodies = true, networkRequestHeaders = [], networkResponseHeaders = [], @@ -138,6 +139,7 @@ export class Replay implements Integration { slowClickTimeout, slowClickIgnoreSelectors, networkDetailAllowUrls, + networkDetailExcludeUrls, networkCaptureBodies, networkRequestHeaders: _getMergedNetworkHeaders(networkRequestHeaders), networkResponseHeaders: _getMergedNetworkHeaders(networkResponseHeaders), diff --git a/packages/replay/src/types/replay.ts b/packages/replay/src/types/replay.ts index e2cfddd0f525..cb30e129e30d 100644 --- a/packages/replay/src/types/replay.ts +++ b/packages/replay/src/types/replay.ts @@ -70,23 +70,32 @@ export interface ReplayNetworkOptions { */ networkDetailAllowUrls: (string | RegExp)[]; + /** + * Exclude request/response details for XHR/Fetch requests that match the given URLs. + * The URLs can be strings or regular expressions. + * When provided a string, we will exclude any URL that contains the given string. + * You can use a Regex to handle exact matches or more complex matching. + * URLs matching these patterns will not have bodies & additional headers captured. + */ + networkDetailExcludeUrls: (string | RegExp)[]; + /** * If request & response bodies should be captured. - * Only applies to URLs matched by `networkDetailAllowUrls`. + * Only applies to URLs matched by `networkDetailAllowUrls` and not matched by `networkDetailExcludeUrls`. * Defaults to true. */ networkCaptureBodies: boolean; /** * Capture the following request headers, in addition to the default ones. - * Only applies to URLs matched by `networkDetailAllowUrls`. + * Only applies to URLs matched by `networkDetailAllowUrls` and not matched by `networkDetailExcludeUrls`. * Any headers defined here will be captured in addition to the default headers. */ networkRequestHeaders: string[]; /** * Capture the following response headers, in addition to the default ones. - * Only applies to URLs matched by `networkDetailAllowUrls`. + * Only applies to URLs matched by `networkDetailAllowUrls` and not matched by `networkDetailExcludeUrls`. * Any headers defined here will be captured in addition to the default headers. */ networkResponseHeaders: string[]; diff --git a/packages/replay/test/unit/coreHandlers/handleNetworkBreadcrumbs.test.ts b/packages/replay/test/unit/coreHandlers/handleNetworkBreadcrumbs.test.ts index 825c6e4eb452..faa35a45ff20 100644 --- a/packages/replay/test/unit/coreHandlers/handleNetworkBreadcrumbs.test.ts +++ b/packages/replay/test/unit/coreHandlers/handleNetworkBreadcrumbs.test.ts @@ -63,6 +63,7 @@ describe('Unit | coreHandlers | handleNetworkBreadcrumbs', () => { textEncoder: new TextEncoder(), replay: setupReplayContainer(), networkDetailAllowUrls: ['https://example.com'], + networkDetailExcludeUrls: ['http://localhost:8080'], networkCaptureBodies: false, networkRequestHeaders: ['content-type', 'accept', 'x-custom-header'], networkResponseHeaders: ['content-type', 'accept', 'x-custom-header'], @@ -1382,5 +1383,87 @@ other-header: test`; ]); }); }); + + describe.each([ + ['exact string match', 'https://example.com/foo'], + ['partial string match', 'https://example.com/bar/what'], + ['exact regex match', 'http://example.com/exact'], + ['partial regex match', 'http://example.com/partial/string'], + ])('matching URL %s', (_label, url) => { + it('correctly excludes URL for fetch request', async () => { + options.networkDetailExcludeUrls = [ + 'https://example.com/foo', + 'com/bar', + /^http:\/\/example.com\/exact$/, + /^http:\/\/example.com\/partial/, + ]; + + const breadcrumb: Breadcrumb = { + category: 'fetch', + data: { + method: 'GET', + url, + status_code: 200, + }, + }; + + const mockResponse = getMockResponse('13', 'test response'); + + const hint: FetchBreadcrumbHint = { + input: ['GET', { body: 'test input' }], + response: mockResponse, + startTimestamp: BASE_TIMESTAMP + 1000, + endTimestamp: BASE_TIMESTAMP + 2000, + }; + beforeAddNetworkBreadcrumb(options, breadcrumb, hint); + + expect(breadcrumb).toEqual({ + category: 'fetch', + data: { + method: 'GET', + request_body_size: 10, + response_body_size: 13, + status_code: 200, + url, + }, + }); + + await waitForReplayEventBuffer(); + + expect((options.replay.eventBuffer as EventBufferArray).events).toEqual([ + { + data: { + payload: { + data: { + method: 'GET', + request: { + _meta: { + warnings: ['URL_SKIPPED'], + }, + headers: {}, + size: 10, + }, + response: { + _meta: { + warnings: ['URL_SKIPPED'], + }, + headers: {}, + size: 13, + }, + statusCode: 200, + }, + description: url, + endTimestamp: 1580598002, + op: 'resource.fetch', + startTimestamp: 1580598001, + }, + tag: 'performanceSpan', + }, + timestamp: 1580598001, + type: 5, + }, + ]); + }); + }); }); }); diff --git a/packages/replay/test/utils/setupReplayContainer.ts b/packages/replay/test/utils/setupReplayContainer.ts index 0237afddb538..618870377540 100644 --- a/packages/replay/test/utils/setupReplayContainer.ts +++ b/packages/replay/test/utils/setupReplayContainer.ts @@ -12,6 +12,7 @@ const DEFAULT_OPTIONS = { useCompression: false, blockAllMedia: true, networkDetailAllowUrls: [], + networkDetailExcludeUrls: [], networkCaptureBodies: true, networkRequestHeaders: [], networkResponseHeaders: [], From 44fb19d8a3baae5aed9f9e4267de6585d0a06f56 Mon Sep 17 00:00:00 2001 From: ziyad-elabid-nw Date: Fri, 30 Jun 2023 23:28:05 +0100 Subject: [PATCH 2/4] Handle the case of xhr request + add a unit test for the case --- .../replay/src/coreHandlers/util/xhrUtils.ts | 2 +- .../handleNetworkBreadcrumbs.test.ts | 79 +++++++++++++++++++ 2 files changed, 80 insertions(+), 1 deletion(-) diff --git a/packages/replay/src/coreHandlers/util/xhrUtils.ts b/packages/replay/src/coreHandlers/util/xhrUtils.ts index dfb1ccaf3ffc..b77b1727dd36 100644 --- a/packages/replay/src/coreHandlers/util/xhrUtils.ts +++ b/packages/replay/src/coreHandlers/util/xhrUtils.ts @@ -78,7 +78,7 @@ function _prepareXhrData( return null; } - if (!urlMatches(url, options.networkDetailAllowUrls)) { + if (!urlMatches(url, options.networkDetailAllowUrls) || urlMatches(url, options.networkDetailExcludeUrls)) { const request = buildSkippedNetworkRequestOrResponse(requestBodySize); const response = buildSkippedNetworkRequestOrResponse(responseBodySize); return { diff --git a/packages/replay/test/unit/coreHandlers/handleNetworkBreadcrumbs.test.ts b/packages/replay/test/unit/coreHandlers/handleNetworkBreadcrumbs.test.ts index faa35a45ff20..d2c8ff6b629f 100644 --- a/packages/replay/test/unit/coreHandlers/handleNetworkBreadcrumbs.test.ts +++ b/packages/replay/test/unit/coreHandlers/handleNetworkBreadcrumbs.test.ts @@ -1464,6 +1464,85 @@ other-header: test`; }, ]); }); + + it('correctly excludes URL for xhr request', async () => { + options.networkDetailExcludeUrls = [ + 'https://example.com/foo', + 'com/bar', + /^http:\/\/example.com\/exact$/, + /^http:\/\/example.com\/partial/, + ]; + + const breadcrumb: Breadcrumb = { + category: 'xhr', + data: { + method: 'GET', + url, + status_code: 200, + }, + }; + const xhr = new XMLHttpRequest(); + Object.defineProperty(xhr, 'response', { + value: 'test response', + }); + Object.defineProperty(xhr, 'responseText', { + value: 'test response', + }); + const hint: XhrBreadcrumbHint = { + xhr, + input: 'test input', + startTimestamp: BASE_TIMESTAMP + 1000, + endTimestamp: BASE_TIMESTAMP + 2000, + }; + beforeAddNetworkBreadcrumb(options, breadcrumb, hint); + + expect(breadcrumb).toEqual({ + category: 'xhr', + data: { + method: 'GET', + request_body_size: 10, + response_body_size: 13, + status_code: 200, + url, + }, + }); + + await waitForReplayEventBuffer(); + + expect((options.replay.eventBuffer as EventBufferArray).events).toEqual([ + { + data: { + payload: { + data: { + method: 'GET', + request: { + _meta: { + warnings: ['URL_SKIPPED'], + }, + headers: {}, + size: 10, + }, + response: { + _meta: { + warnings: ['URL_SKIPPED'], + }, + headers: {}, + size: 13, + }, + statusCode: 200, + }, + description: url, + endTimestamp: 1580598002, + op: 'resource.xhr', + startTimestamp: 1580598001, + }, + tag: 'performanceSpan', + }, + timestamp: 1580598001, + type: 5, + }, + ]); + }); }); }); }); From d4eac46d598c6c6be50e79503f450b20702e9fbe Mon Sep 17 00:00:00 2001 From: ziyad-elabid-nw Date: Mon, 3 Jul 2023 09:20:43 +0100 Subject: [PATCH 3/4] fix unit tests : timestamp --- .../coreHandlers/handleNetworkBreadcrumbs.test.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/replay/test/unit/coreHandlers/handleNetworkBreadcrumbs.test.ts b/packages/replay/test/unit/coreHandlers/handleNetworkBreadcrumbs.test.ts index d2c8ff6b629f..d9e6a900346a 100644 --- a/packages/replay/test/unit/coreHandlers/handleNetworkBreadcrumbs.test.ts +++ b/packages/replay/test/unit/coreHandlers/handleNetworkBreadcrumbs.test.ts @@ -1453,13 +1453,13 @@ other-header: test`; statusCode: 200, }, description: url, - endTimestamp: 1580598002, + endTimestamp: (BASE_TIMESTAMP + 2000) / 1000, op: 'resource.fetch', - startTimestamp: 1580598001, + startTimestamp: (BASE_TIMESTAMP + 1000) / 1000, }, tag: 'performanceSpan', }, - timestamp: 1580598001, + timestamp: (BASE_TIMESTAMP + 1000) / 1000, type: 5, }, ]); @@ -1532,13 +1532,13 @@ other-header: test`; statusCode: 200, }, description: url, - endTimestamp: 1580598002, + endTimestamp: (BASE_TIMESTAMP + 2000) / 1000, op: 'resource.xhr', - startTimestamp: 1580598001, + startTimestamp: (BASE_TIMESTAMP + 1000) / 1000, }, tag: 'performanceSpan', }, - timestamp: 1580598001, + timestamp: (BASE_TIMESTAMP + 1000) / 1000, type: 5, }, ]); From 3419e90f5d78fa734e29abca2deea3fbe913c91c Mon Sep 17 00:00:00 2001 From: ziyad-elabid-nw Date: Mon, 3 Jul 2023 23:14:10 +0100 Subject: [PATCH 4/4] Change naming for networkDetailExcludeUrls to networkDetailDenyUrls --- .../src/coreHandlers/handleNetworkBreadcrumbs.ts | 4 ++-- packages/replay/src/coreHandlers/util/fetchUtils.ts | 2 +- packages/replay/src/coreHandlers/util/xhrUtils.ts | 2 +- packages/replay/src/integration.ts | 4 ++-- packages/replay/src/types/replay.ts | 12 ++++++------ .../coreHandlers/handleNetworkBreadcrumbs.test.ts | 10 +++++----- packages/replay/test/utils/setupReplayContainer.ts | 2 +- 7 files changed, 18 insertions(+), 18 deletions(-) diff --git a/packages/replay/src/coreHandlers/handleNetworkBreadcrumbs.ts b/packages/replay/src/coreHandlers/handleNetworkBreadcrumbs.ts index 6624da6adfbb..20fa00a633eb 100644 --- a/packages/replay/src/coreHandlers/handleNetworkBreadcrumbs.ts +++ b/packages/replay/src/coreHandlers/handleNetworkBreadcrumbs.ts @@ -33,7 +33,7 @@ export function handleNetworkBreadcrumbs(replay: ReplayContainer): void { const { networkDetailAllowUrls, - networkDetailExcludeUrls, + networkDetailDenyUrls, networkCaptureBodies, networkRequestHeaders, networkResponseHeaders, @@ -43,7 +43,7 @@ export function handleNetworkBreadcrumbs(replay: ReplayContainer): void { replay, textEncoder, networkDetailAllowUrls, - networkDetailExcludeUrls, + networkDetailDenyUrls, networkCaptureBodies, networkRequestHeaders, networkResponseHeaders, diff --git a/packages/replay/src/coreHandlers/util/fetchUtils.ts b/packages/replay/src/coreHandlers/util/fetchUtils.ts index af6c94527e9c..491fe727b1b8 100644 --- a/packages/replay/src/coreHandlers/util/fetchUtils.ts +++ b/packages/replay/src/coreHandlers/util/fetchUtils.ts @@ -86,7 +86,7 @@ async function _prepareFetchData( } = breadcrumb.data; const captureDetails = - urlMatches(url, options.networkDetailAllowUrls) && !urlMatches(url, options.networkDetailExcludeUrls); + urlMatches(url, options.networkDetailAllowUrls) && !urlMatches(url, options.networkDetailDenyUrls); const request = captureDetails ? _getRequestInfo(options, hint.input, requestBodySize) diff --git a/packages/replay/src/coreHandlers/util/xhrUtils.ts b/packages/replay/src/coreHandlers/util/xhrUtils.ts index b77b1727dd36..638af2cbdb5c 100644 --- a/packages/replay/src/coreHandlers/util/xhrUtils.ts +++ b/packages/replay/src/coreHandlers/util/xhrUtils.ts @@ -78,7 +78,7 @@ function _prepareXhrData( return null; } - if (!urlMatches(url, options.networkDetailAllowUrls) || urlMatches(url, options.networkDetailExcludeUrls)) { + if (!urlMatches(url, options.networkDetailAllowUrls) || urlMatches(url, options.networkDetailDenyUrls)) { const request = buildSkippedNetworkRequestOrResponse(requestBodySize); const response = buildSkippedNetworkRequestOrResponse(responseBodySize); return { diff --git a/packages/replay/src/integration.ts b/packages/replay/src/integration.ts index b8f7d38b5059..2e795829894b 100644 --- a/packages/replay/src/integration.ts +++ b/packages/replay/src/integration.ts @@ -67,7 +67,7 @@ export class Replay implements Integration { slowClickIgnoreSelectors = [], networkDetailAllowUrls = [], - networkDetailExcludeUrls = [], + networkDetailDenyUrls = [], networkCaptureBodies = true, networkRequestHeaders = [], networkResponseHeaders = [], @@ -139,7 +139,7 @@ export class Replay implements Integration { slowClickTimeout, slowClickIgnoreSelectors, networkDetailAllowUrls, - networkDetailExcludeUrls, + networkDetailDenyUrls, networkCaptureBodies, networkRequestHeaders: _getMergedNetworkHeaders(networkRequestHeaders), networkResponseHeaders: _getMergedNetworkHeaders(networkResponseHeaders), diff --git a/packages/replay/src/types/replay.ts b/packages/replay/src/types/replay.ts index cb30e129e30d..494d9408f292 100644 --- a/packages/replay/src/types/replay.ts +++ b/packages/replay/src/types/replay.ts @@ -71,31 +71,31 @@ export interface ReplayNetworkOptions { networkDetailAllowUrls: (string | RegExp)[]; /** - * Exclude request/response details for XHR/Fetch requests that match the given URLs. + * Deny request/response details for XHR/Fetch requests that match the given URLs. * The URLs can be strings or regular expressions. - * When provided a string, we will exclude any URL that contains the given string. + * When provided a string, we will deny any URL that contains the given string. * You can use a Regex to handle exact matches or more complex matching. * URLs matching these patterns will not have bodies & additional headers captured. */ - networkDetailExcludeUrls: (string | RegExp)[]; + networkDetailDenyUrls: (string | RegExp)[]; /** * If request & response bodies should be captured. - * Only applies to URLs matched by `networkDetailAllowUrls` and not matched by `networkDetailExcludeUrls`. + * Only applies to URLs matched by `networkDetailAllowUrls` and not matched by `networkDetailDenyUrls`. * Defaults to true. */ networkCaptureBodies: boolean; /** * Capture the following request headers, in addition to the default ones. - * Only applies to URLs matched by `networkDetailAllowUrls` and not matched by `networkDetailExcludeUrls`. + * Only applies to URLs matched by `networkDetailAllowUrls` and not matched by `networkDetailDenyUrls`. * Any headers defined here will be captured in addition to the default headers. */ networkRequestHeaders: string[]; /** * Capture the following response headers, in addition to the default ones. - * Only applies to URLs matched by `networkDetailAllowUrls` and not matched by `networkDetailExcludeUrls`. + * Only applies to URLs matched by `networkDetailAllowUrls` and not matched by `networkDetailDenyUrls`. * Any headers defined here will be captured in addition to the default headers. */ networkResponseHeaders: string[]; diff --git a/packages/replay/test/unit/coreHandlers/handleNetworkBreadcrumbs.test.ts b/packages/replay/test/unit/coreHandlers/handleNetworkBreadcrumbs.test.ts index d9e6a900346a..c0ad9ef59b49 100644 --- a/packages/replay/test/unit/coreHandlers/handleNetworkBreadcrumbs.test.ts +++ b/packages/replay/test/unit/coreHandlers/handleNetworkBreadcrumbs.test.ts @@ -63,7 +63,7 @@ describe('Unit | coreHandlers | handleNetworkBreadcrumbs', () => { textEncoder: new TextEncoder(), replay: setupReplayContainer(), networkDetailAllowUrls: ['https://example.com'], - networkDetailExcludeUrls: ['http://localhost:8080'], + networkDetailDenyUrls: ['http://localhost:8080'], networkCaptureBodies: false, networkRequestHeaders: ['content-type', 'accept', 'x-custom-header'], networkResponseHeaders: ['content-type', 'accept', 'x-custom-header'], @@ -1390,8 +1390,8 @@ other-header: test`; ['exact regex match', 'http://example.com/exact'], ['partial regex match', 'http://example.com/partial/string'], ])('matching URL %s', (_label, url) => { - it('correctly excludes URL for fetch request', async () => { - options.networkDetailExcludeUrls = [ + it('correctly deny URL for fetch request', async () => { + options.networkDetailDenyUrls = [ 'https://example.com/foo', 'com/bar', /^http:\/\/example.com\/exact$/, @@ -1465,8 +1465,8 @@ other-header: test`; ]); }); - it('correctly excludes URL for xhr request', async () => { - options.networkDetailExcludeUrls = [ + it('correctly deny URL for xhr request', async () => { + options.networkDetailDenyUrls = [ 'https://example.com/foo', 'com/bar', /^http:\/\/example.com\/exact$/, diff --git a/packages/replay/test/utils/setupReplayContainer.ts b/packages/replay/test/utils/setupReplayContainer.ts index 618870377540..e2a49052a799 100644 --- a/packages/replay/test/utils/setupReplayContainer.ts +++ b/packages/replay/test/utils/setupReplayContainer.ts @@ -12,7 +12,7 @@ const DEFAULT_OPTIONS = { useCompression: false, blockAllMedia: true, networkDetailAllowUrls: [], - networkDetailExcludeUrls: [], + networkDetailDenyUrls: [], networkCaptureBodies: true, networkRequestHeaders: [], networkResponseHeaders: [],