From 7184723216bc2d0114275e712f0a210ae16f40e7 Mon Sep 17 00:00:00 2001 From: Vadim Korolik Date: Mon, 8 Aug 2022 23:55:25 -0700 Subject: [PATCH 1/5] support canvas downsampling configuration canvas recording takes snapshots at full canvas resolution this is too slow to record at higher fps. allow specifying canvas manager ratio and quality so that the canvas can be imaged at lower quality and resolution to allow higher FPS recording --- packages/rrweb/package.json | 2 +- packages/rrweb/src/record/index.ts | 4 ++- .../record/observers/canvas/canvas-manager.ts | 32 +++++++++++++++---- .../workers/image-bitmap-data-url-worker.ts | 6 ++-- packages/rrweb/src/types.ts | 28 ++++++++++++---- 5 files changed, 55 insertions(+), 17 deletions(-) diff --git a/packages/rrweb/package.json b/packages/rrweb/package.json index 6195e282..5e8b9502 100644 --- a/packages/rrweb/package.json +++ b/packages/rrweb/package.json @@ -1,6 +1,6 @@ { "name": "@highlight-run/rrweb", - "version": "2.1.7", + "version": "2.1.8", "description": "record and replay the web", "scripts": { "prepare": "npm run prepack", diff --git a/packages/rrweb/src/record/index.ts b/packages/rrweb/src/record/index.ts index fc7f55f8..99fec328 100644 --- a/packages/rrweb/src/record/index.ts +++ b/packages/rrweb/src/record/index.ts @@ -229,7 +229,9 @@ function record( win: window, blockClass, mirror, - sampling: sampling.canvas, + sampling: sampling?.canvas?.fps, + resizeQuality: sampling?.canvas?.resizeQuality, + resizeFactor: sampling?.canvas?.resizeFactor, }); const shadowDomManager = new ShadowDomManager({ diff --git a/packages/rrweb/src/record/observers/canvas/canvas-manager.ts b/packages/rrweb/src/record/observers/canvas/canvas-manager.ts index 90e217f3..e9905659 100644 --- a/packages/rrweb/src/record/observers/canvas/canvas-manager.ts +++ b/packages/rrweb/src/record/observers/canvas/canvas-manager.ts @@ -61,6 +61,8 @@ export class CanvasManager { blockClass: blockClass; mirror: Mirror; sampling?: 'all' | number; + resizeQuality?: 'pixelated' | 'low' | 'medium' | 'high'; + resizeFactor?: number; }) { const { sampling = 'all', win, blockClass, recordCanvas } = options; this.mutationCb = options.mutationCb; @@ -69,7 +71,13 @@ export class CanvasManager { if (recordCanvas && sampling === 'all') this.initCanvasMutationObserver(win, blockClass); if (recordCanvas && typeof sampling === 'number') - this.initCanvasFPSObserver(sampling, win, blockClass); + this.initCanvasFPSObserver( + sampling, + win, + blockClass, + options.resizeQuality, + options.resizeFactor, + ); } private processMutation: canvasManagerMutationCallback = ( @@ -93,6 +101,8 @@ export class CanvasManager { fps: number, win: IWindow, blockClass: blockClass, + resizeQuality?: 'pixelated' | 'low' | 'medium' | 'high', + resizeFactor?: number, ) { const canvasContextReset = initCanvasContextObserver(win, blockClass); const snapshotInProgressMap: Map = new Map(); @@ -103,14 +113,14 @@ export class CanvasManager { if (!('base64' in e.data)) return; - const { base64, type, width, height } = e.data; + const { base64, type, canvasWidth, canvasHeight } = e.data; this.mutationCb({ id, type: CanvasContext['2D'], commands: [ { property: 'clearRect', // wipe canvas - args: [0, 0, width, height], + args: [0, 0, canvasWidth, canvasHeight], }, { property: 'drawImage', // draws (semi-transparent) image @@ -127,6 +137,8 @@ export class CanvasManager { } as CanvasArg, 0, 0, + canvasWidth, + canvasHeight, ], }, ], @@ -178,13 +190,21 @@ export class CanvasManager { if (canvas.width === 0 || canvas.height === 0) { return; } - const bitmap = await createImageBitmap(canvas); + const width = canvas.width * (resizeFactor || 1); + const height = canvas.height * (resizeFactor || 1); + const bitmap = await createImageBitmap(canvas, { + resizeQuality: resizeQuality || 'low', + resizeWidth: width, + resizeHeight: height, + }); worker.postMessage( { id, bitmap, - width: canvas.width, - height: canvas.height, + width, + height, + canvasWidth: canvas.width, + canvasHeight: canvas.height, }, [bitmap], ); diff --git a/packages/rrweb/src/record/workers/image-bitmap-data-url-worker.ts b/packages/rrweb/src/record/workers/image-bitmap-data-url-worker.ts index e5addc0b..c27a6b54 100644 --- a/packages/rrweb/src/record/workers/image-bitmap-data-url-worker.ts +++ b/packages/rrweb/src/record/workers/image-bitmap-data-url-worker.ts @@ -45,14 +45,14 @@ worker.onmessage = async function (e) { if (!('OffscreenCanvas' in globalThis)) return worker.postMessage({ id: e.data.id }); - const { id, bitmap, width, height } = e.data; + const { id, bitmap, width, height, canvasWidth, canvasHeight } = e.data; const transparentBase64 = getTransparentBlobFor(width, height); const offscreen = new OffscreenCanvas(width, height); const ctx = offscreen.getContext('2d')!; - ctx.drawImage(bitmap, 0, 0); + ctx.drawImage(bitmap, 0, 0, width, height); bitmap.close(); const blob = await offscreen.convertToBlob(); // takes a while const type = blob.type; @@ -73,6 +73,8 @@ worker.onmessage = async function (e) { base64, width, height, + canvasWidth, + canvasHeight, }); lastBlobMap.set(id, base64); }; diff --git a/packages/rrweb/src/types.ts b/packages/rrweb/src/types.ts index 3a24933f..ab936194 100644 --- a/packages/rrweb/src/types.ts +++ b/packages/rrweb/src/types.ts @@ -185,6 +185,20 @@ export type blockClass = string | RegExp; export type maskTextClass = string | RegExp; +export type CanvasSamplingStrategy = Partial<{ + fps: 'all' | number; + /** + * 'all' will record every single canvas call + * number between 1 and 60, will record an image snapshots in a web-worker a (maximum) number of times per second. + * Number only supported where [`OffscreenCanvas`](http://mdn.io/offscreencanvas) is supported. + */ + resizeFactor: number; + /** + * The quality of canvas snapshots + */ + resizeQuality: 'pixelated' | 'low' | 'medium' | 'high'; +}>; + export type SamplingStrategy = Partial<{ /** * false means not to record mouse/touch move events @@ -213,12 +227,8 @@ export type SamplingStrategy = Partial<{ * 'last' will only record the last input value while input a sequence of chars */ input: 'all' | 'last'; - /** - * 'all' will record every single canvas call - * number between 1 and 60, will record an image snapshots in a web-worker a (maximum) number of times per second. - * Number only supported where [`OffscreenCanvas`](http://mdn.io/offscreencanvas) is supported. - */ - canvas: 'all' | number; + + canvas: CanvasSamplingStrategy; }>; export type RecordPlugin = { @@ -299,7 +309,7 @@ export type observerParam = { stylesheetManager: StylesheetManager; shadowDomManager: ShadowDomManager; canvasManager: CanvasManager; - enableStrictPrivacy: boolean, + enableStrictPrivacy: boolean; plugins: Array<{ observer: ( cb: (...arg: Array) => void, @@ -566,6 +576,8 @@ export type ImageBitmapDataURLWorkerParams = { bitmap: ImageBitmap; width: number; height: number; + canvasWidth: number; + canvasHeight: number; }; export type ImageBitmapDataURLWorkerResponse = @@ -578,6 +590,8 @@ export type ImageBitmapDataURLWorkerResponse = base64: string; width: number; height: number; + canvasWidth: number; + canvasHeight: number; }; export type fontParam = { From 8a3d0a7cc0ad99a9c30fc48efbca231fab47ebc8 Mon Sep 17 00:00:00 2001 From: Vadim Korolik Date: Tue, 9 Aug 2022 16:02:15 -0700 Subject: [PATCH 2/5] allow setting a `maxSnapshotDimension` on canvas recording to limit image size in pixels --- packages/rrweb/src/record/index.ts | 1 + .../src/record/observers/canvas/canvas-manager.ts | 11 +++++++++++ packages/rrweb/src/types.ts | 8 ++++++++ 3 files changed, 20 insertions(+) diff --git a/packages/rrweb/src/record/index.ts b/packages/rrweb/src/record/index.ts index 99fec328..055622c5 100644 --- a/packages/rrweb/src/record/index.ts +++ b/packages/rrweb/src/record/index.ts @@ -232,6 +232,7 @@ function record( sampling: sampling?.canvas?.fps, resizeQuality: sampling?.canvas?.resizeQuality, resizeFactor: sampling?.canvas?.resizeFactor, + maxSnapshotDimension: sampling?.canvas?.maxSnapshotDimension, }); const shadowDomManager = new ShadowDomManager({ diff --git a/packages/rrweb/src/record/observers/canvas/canvas-manager.ts b/packages/rrweb/src/record/observers/canvas/canvas-manager.ts index e9905659..c7fcb233 100644 --- a/packages/rrweb/src/record/observers/canvas/canvas-manager.ts +++ b/packages/rrweb/src/record/observers/canvas/canvas-manager.ts @@ -63,6 +63,7 @@ export class CanvasManager { sampling?: 'all' | number; resizeQuality?: 'pixelated' | 'low' | 'medium' | 'high'; resizeFactor?: number; + maxSnapshotDimension?: number; }) { const { sampling = 'all', win, blockClass, recordCanvas } = options; this.mutationCb = options.mutationCb; @@ -77,6 +78,7 @@ export class CanvasManager { blockClass, options.resizeQuality, options.resizeFactor, + options.maxSnapshotDimension, ); } @@ -103,6 +105,7 @@ export class CanvasManager { blockClass: blockClass, resizeQuality?: 'pixelated' | 'low' | 'medium' | 'high', resizeFactor?: number, + maxSnapshotDimension?: number, ) { const canvasContextReset = initCanvasContextObserver(win, blockClass); const snapshotInProgressMap: Map = new Map(); @@ -190,8 +193,16 @@ export class CanvasManager { if (canvas.width === 0 || canvas.height === 0) { return; } + if (maxSnapshotDimension) { + const maxDim = Math.max(canvas.width, canvas.height); + resizeFactor = Math.min( + resizeFactor || 1, + maxSnapshotDimension / maxDim, + ); + } const width = canvas.width * (resizeFactor || 1); const height = canvas.height * (resizeFactor || 1); + const bitmap = await createImageBitmap(canvas, { resizeQuality: resizeQuality || 'low', resizeWidth: width, diff --git a/packages/rrweb/src/types.ts b/packages/rrweb/src/types.ts index ab936194..987f2ff4 100644 --- a/packages/rrweb/src/types.ts +++ b/packages/rrweb/src/types.ts @@ -197,6 +197,14 @@ export type CanvasSamplingStrategy = Partial<{ * The quality of canvas snapshots */ resizeQuality: 'pixelated' | 'low' | 'medium' | 'high'; + /** + * The maximum dimension to take canvas snapshots at. + * This setting takes precedence over resizeFactor if the resulting image size + * from the resizeFactor calculation is larger than this value. + * Eg: set to 600 to ensure that the canvas is saved with images no larger than 600px + * in either dimension (while preserving the original canvas aspect ratio). + */ + maxSnapshotDimension: number; }>; export type SamplingStrategy = Partial<{ From ef09ecc9bc1fc3b9469a96e37f232b139e303c9b Mon Sep 17 00:00:00 2001 From: Vadim Korolik Date: Tue, 9 Aug 2022 17:26:56 -0700 Subject: [PATCH 3/5] bump rrweb-player version --- packages/rrweb-player/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/rrweb-player/package.json b/packages/rrweb-player/package.json index f0738d5e..50ff9072 100644 --- a/packages/rrweb-player/package.json +++ b/packages/rrweb-player/package.json @@ -23,7 +23,7 @@ "typescript": "^4.7.3" }, "dependencies": { - "@highlight-run/rrweb": "2.1.7", + "@highlight-run/rrweb": "2.1.8", "@tsconfig/svelte": "^1.0.1" }, "scripts": { From e2a315c01c566df36ff425f661e15676bd79c4e4 Mon Sep 17 00:00:00 2001 From: Vadim Korolik Date: Wed, 10 Aug 2022 12:52:39 -0700 Subject: [PATCH 4/5] refactor to cleanup and ensure resize scaling is calculated correctly per-canvas (if recording more than 1 canvas) --- .../src/record/observers/canvas/canvas-manager.ts | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/packages/rrweb/src/record/observers/canvas/canvas-manager.ts b/packages/rrweb/src/record/observers/canvas/canvas-manager.ts index c7fcb233..62f783c2 100644 --- a/packages/rrweb/src/record/observers/canvas/canvas-manager.ts +++ b/packages/rrweb/src/record/observers/canvas/canvas-manager.ts @@ -193,15 +193,13 @@ export class CanvasManager { if (canvas.width === 0 || canvas.height === 0) { return; } + let scale = resizeFactor || 1; if (maxSnapshotDimension) { const maxDim = Math.max(canvas.width, canvas.height); - resizeFactor = Math.min( - resizeFactor || 1, - maxSnapshotDimension / maxDim, - ); + scale = Math.min(scale, maxSnapshotDimension / maxDim); } - const width = canvas.width * (resizeFactor || 1); - const height = canvas.height * (resizeFactor || 1); + const width = canvas.width * scale; + const height = canvas.height * scale; const bitmap = await createImageBitmap(canvas, { resizeQuality: resizeQuality || 'low', From c8a77db4d90a5f1cff4e2bc15b5460fc3fc5d212 Mon Sep 17 00:00:00 2001 From: Vadim Korolik Date: Wed, 10 Aug 2022 12:54:41 -0700 Subject: [PATCH 5/5] fix comment order --- packages/rrweb/src/types.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/rrweb/src/types.ts b/packages/rrweb/src/types.ts index 987f2ff4..cfacfbda 100644 --- a/packages/rrweb/src/types.ts +++ b/packages/rrweb/src/types.ts @@ -186,12 +186,16 @@ export type blockClass = string | RegExp; export type maskTextClass = string | RegExp; export type CanvasSamplingStrategy = Partial<{ - fps: 'all' | number; /** * 'all' will record every single canvas call * number between 1 and 60, will record an image snapshots in a web-worker a (maximum) number of times per second. * Number only supported where [`OffscreenCanvas`](http://mdn.io/offscreencanvas) is supported. */ + fps: 'all' | number; + /** + * A scaling to apply to canvas shapshotting. Adjusts the resolution at which + * canvases are recorded by this multiple. + */ resizeFactor: number; /** * The quality of canvas snapshots