From e9335db86fab6104a18742f21981a16cdfccdf8e Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Tue, 9 Nov 2021 13:34:07 +0100 Subject: [PATCH 01/93] Add very basic webgl support --- packages/rrweb/package.json | 2 + packages/rrweb/src/record/observer.ts | 88 ++----- .../rrweb/src/record/observers/canvas-2d.ts | 94 ++++++++ .../src/record/observers/canvas-web-gl.ts | 75 ++++++ packages/rrweb/src/replay/index.ts | 12 +- packages/rrweb/src/types.ts | 6 + .../__snapshots__/integration.test.ts.snap | 221 ++++++++++++++++++ packages/rrweb/test/events/webgl.ts | 120 ++++++++++ packages/rrweb/test/html/canvas-webgl.html | 27 +++ packages/rrweb/test/integration.test.ts | 13 ++ ...uld-doutput-simple-webgl-object-1-snap.png | Bin 0 -> 10796 bytes packages/rrweb/test/replay/webgl.test.ts | 66 ++++++ packages/rrweb/tsconfig.json | 6 +- .../typings/record/observers/canvas-2d.d.ts | 2 + .../record/observers/canvas-web-gl.d.ts | 2 + packages/rrweb/typings/types.d.ts | 5 + yarn.lock | 67 +++++- 17 files changed, 730 insertions(+), 76 deletions(-) create mode 100644 packages/rrweb/src/record/observers/canvas-2d.ts create mode 100644 packages/rrweb/src/record/observers/canvas-web-gl.ts create mode 100644 packages/rrweb/test/events/webgl.ts create mode 100644 packages/rrweb/test/html/canvas-webgl.html create mode 100644 packages/rrweb/test/replay/__image_snapshots__/webgl-test-ts-replayer-webgl-should-doutput-simple-webgl-object-1-snap.png create mode 100644 packages/rrweb/test/replay/webgl.test.ts create mode 100644 packages/rrweb/typings/record/observers/canvas-2d.d.ts create mode 100644 packages/rrweb/typings/record/observers/canvas-web-gl.d.ts diff --git a/packages/rrweb/package.json b/packages/rrweb/package.json index deb06da85d..1e6cb8cb40 100644 --- a/packages/rrweb/package.json +++ b/packages/rrweb/package.json @@ -46,6 +46,7 @@ "@types/chai": "^4.1.6", "@types/inquirer": "0.0.43", "@types/jest": "^27.0.2", + "@types/jest-image-snapshot": "^4.3.1", "@types/jsdom": "^16.2.12", "@types/node": "^12.20.16", "@types/prettier": "^2.3.2", @@ -55,6 +56,7 @@ "ignore-styles": "^5.0.1", "inquirer": "^6.2.1", "jest": "^27.2.4", + "jest-image-snapshot": "^4.5.1", "jest-snapshot": "^23.6.0", "jsdom": "^17.0.0", "jsdom-global": "^3.0.2", diff --git a/packages/rrweb/src/record/observer.ts b/packages/rrweb/src/record/observer.ts index 26cd1ef75f..31134a0ac0 100644 --- a/packages/rrweb/src/record/observer.ts +++ b/packages/rrweb/src/record/observer.ts @@ -49,6 +49,8 @@ import { import MutationBuffer from './mutation'; import { IframeManager } from './iframe-manager'; import { ShadowDomManager } from './shadow-dom-manager'; +import initCanvas2DMutationObserver from './observers/canvas-2d'; +import initCanvasWebGLMutationObserver from './observers/canvas-web-gl'; type WindowWithStoredMutationObserver = IWindow & { __rrMutationObserver?: MutationObserver; @@ -712,79 +714,23 @@ function initCanvasMutationObserver( blockClass: blockClass, mirror: Mirror, ): listenerHandler { - const props = Object.getOwnPropertyNames( - win.CanvasRenderingContext2D.prototype, + const canvas2DReset = initCanvas2DMutationObserver( + cb, + win, + blockClass, + mirror, ); - const handlers: listenerHandler[] = []; - for (const prop of props) { - try { - if ( - typeof win.CanvasRenderingContext2D.prototype[ - prop as keyof CanvasRenderingContext2D - ] !== 'function' - ) { - continue; - } - const restoreHandler = patch( - win.CanvasRenderingContext2D.prototype, - prop, - function (original) { - return function ( - this: CanvasRenderingContext2D, - ...args: Array - ) { - if (!isBlocked(this.canvas, blockClass)) { - setTimeout(() => { - const recordArgs = [...args]; - if (prop === 'drawImage') { - if ( - recordArgs[0] && - recordArgs[0] instanceof HTMLCanvasElement - ) { - const canvas = recordArgs[0]; - const ctx = canvas.getContext('2d'); - let imgd = ctx?.getImageData( - 0, - 0, - canvas.width, - canvas.height, - ); - let pix = imgd?.data; - recordArgs[0] = JSON.stringify(pix); - } - } - cb({ - id: mirror.getId((this.canvas as unknown) as INode), - property: prop, - args: recordArgs, - }); - }, 0); - } - return original.apply(this, args); - }; - }, - ); - handlers.push(restoreHandler); - } catch { - const hookHandler = hookSetter( - win.CanvasRenderingContext2D.prototype, - prop, - { - set(v) { - cb({ - id: mirror.getId((this.canvas as unknown) as INode), - property: prop, - args: [v], - setter: true, - }); - }, - }, - ); - handlers.push(hookHandler); - } - } + + const canvasWebGLReset = initCanvasWebGLMutationObserver( + cb, + win, + blockClass, + mirror, + ); + return () => { - handlers.forEach((h) => h()); + canvas2DReset(); + canvasWebGLReset(); }; } diff --git a/packages/rrweb/src/record/observers/canvas-2d.ts b/packages/rrweb/src/record/observers/canvas-2d.ts new file mode 100644 index 0000000000..0aa30af391 --- /dev/null +++ b/packages/rrweb/src/record/observers/canvas-2d.ts @@ -0,0 +1,94 @@ +import { INode } from 'rrweb-snapshot'; +import { + blockClass, + CanvasContext, + canvasMutationCallback, + IWindow, + listenerHandler, + Mirror, +} from '../../types'; +import { hookSetter, isBlocked, patch } from '../../utils'; + +export default function initCanvas2DMutationObserver( + cb: canvasMutationCallback, + win: IWindow, + blockClass: blockClass, + mirror: Mirror, +): listenerHandler { + const handlers: listenerHandler[] = []; + const props2D = Object.getOwnPropertyNames( + win.CanvasRenderingContext2D.prototype, + ); + for (const prop of props2D) { + try { + if ( + typeof win.CanvasRenderingContext2D.prototype[ + prop as keyof CanvasRenderingContext2D + ] !== 'function' + ) { + continue; + } + const restoreHandler = patch( + win.CanvasRenderingContext2D.prototype, + prop, + function (original) { + return function ( + this: CanvasRenderingContext2D, + ...args: Array + ) { + if (!isBlocked((this.canvas as unknown) as INode, blockClass)) { + setTimeout(() => { + const recordArgs = [...args]; + if (prop === 'drawImage') { + if ( + recordArgs[0] && + recordArgs[0] instanceof HTMLCanvasElement + ) { + const canvas = recordArgs[0]; + const ctx = canvas.getContext('2d'); + let imgd = ctx?.getImageData( + 0, + 0, + canvas.width, + canvas.height, + ); + let pix = imgd?.data; + recordArgs[0] = JSON.stringify(pix); + } + } + cb({ + id: mirror.getId((this.canvas as unknown) as INode), + type: CanvasContext['2D'], + property: prop, + args: recordArgs, + }); + }, 0); + } + return original.apply(this, args); + }; + }, + ); + handlers.push(restoreHandler); + } catch { + const hookHandler = hookSetter( + win.CanvasRenderingContext2D.prototype, + prop, + { + set(v) { + cb({ + id: mirror.getId((this.canvas as unknown) as INode), + type: CanvasContext['2D'], + property: prop, + args: [v], + setter: true, + }); + }, + }, + ); + handlers.push(hookHandler); + } + } + return () => { + handlers.forEach((h) => h()); + }; +} diff --git a/packages/rrweb/src/record/observers/canvas-web-gl.ts b/packages/rrweb/src/record/observers/canvas-web-gl.ts new file mode 100644 index 0000000000..498b012f55 --- /dev/null +++ b/packages/rrweb/src/record/observers/canvas-web-gl.ts @@ -0,0 +1,75 @@ +import { INode } from 'rrweb-snapshot'; +import { + blockClass, + CanvasContext, + canvasMutationCallback, + IWindow, + listenerHandler, + Mirror, +} from '../../types'; +import { hookSetter, isBlocked, patch } from '../../utils'; + +export default function initCanvasWebGLMutationObserver( + cb: canvasMutationCallback, + win: IWindow, + blockClass: blockClass, + mirror: Mirror, +): listenerHandler { + const handlers: listenerHandler[] = []; + const props = Object.getOwnPropertyNames(win.WebGLRenderingContext.prototype); + for (const prop of props) { + try { + if ( + typeof win.WebGLRenderingContext.prototype[ + prop as keyof WebGLRenderingContext + ] !== 'function' + ) { + continue; + } + const restoreHandler = patch( + win.WebGLRenderingContext.prototype, + prop, + function (original) { + return function ( + this: WebGLRenderingContext, + ...args: Array + ) { + if (!isBlocked((this.canvas as unknown) as INode, blockClass)) { + setTimeout(() => { + const recordArgs = [...args]; + cb({ + id: mirror.getId((this.canvas as unknown) as INode), + type: CanvasContext.WebGL, + property: prop, + args: recordArgs, + }); + }, 0); + } + return original.apply(this, args); + }; + }, + ); + handlers.push(restoreHandler); + } catch { + const hookHandler = hookSetter( + win.WebGLRenderingContext.prototype, + prop, + { + set(v) { + cb({ + id: mirror.getId((this.canvas as unknown) as INode), + type: CanvasContext.WebGL, + property: prop, + args: [v], + setter: true, + }); + }, + }, + ); + handlers.push(hookHandler); + } + } + return () => { + handlers.forEach((h) => h()); + }; +} diff --git a/packages/rrweb/src/replay/index.ts b/packages/rrweb/src/replay/index.ts index 59a7009c30..372e057205 100644 --- a/packages/rrweb/src/replay/index.ts +++ b/packages/rrweb/src/replay/index.ts @@ -39,6 +39,7 @@ import { styleValueWithPriority, mouseMovePos, IWindow, + CanvasContext, } from '../types'; import { createMirror, @@ -1203,9 +1204,15 @@ export class Replayer { if (!target) { return this.debugNodeNotFound(d, d.id); } + let contextType: '2d' | 'webgl' = '2d'; try { + if (d.type === CanvasContext['2D']) { + contextType = '2d'; + } else if (d.type === CanvasContext.WebGL) { + contextType = 'webgl'; + } const ctx = ((target as unknown) as HTMLCanvasElement).getContext( - '2d', + contextType, )!; if (d.setter) { // skip some read-only type checks @@ -1214,8 +1221,9 @@ export class Replayer { return; } const original = ctx[ - d.property as keyof CanvasRenderingContext2D + d.property as Exclude ] as Function; + /** * We have serialized the image source into base64 string during recording, * which has been preloaded before replay. diff --git a/packages/rrweb/src/types.ts b/packages/rrweb/src/types.ts index 4f881ecb26..866fa858c0 100644 --- a/packages/rrweb/src/types.ts +++ b/packages/rrweb/src/types.ts @@ -381,6 +381,11 @@ export enum MouseInteractions { TouchCancel, } +export enum CanvasContext { + '2D', + WebGL, +} + type mouseInteractionParam = { type: MouseInteractions; id: number; @@ -434,6 +439,7 @@ export type canvasMutationCallback = (p: canvasMutationParam) => void; export type canvasMutationParam = { id: number; + type: CanvasContext; property: string; args: Array; setter?: true; diff --git a/packages/rrweb/test/__snapshots__/integration.test.ts.snap b/packages/rrweb/test/__snapshots__/integration.test.ts.snap index df33590626..4e7c341c6b 100644 --- a/packages/rrweb/test/__snapshots__/integration.test.ts.snap +++ b/packages/rrweb/test/__snapshots__/integration.test.ts.snap @@ -7316,6 +7316,7 @@ exports[`record integration tests should record canvas mutations 1`] = ` \\"data\\": { \\"source\\": 9, \\"id\\": 16, + \\"type\\": 0, \\"property\\": \\"moveTo\\", \\"args\\": [ 0, @@ -7328,6 +7329,7 @@ exports[`record integration tests should record canvas mutations 1`] = ` \\"data\\": { \\"source\\": 9, \\"id\\": 16, + \\"type\\": 0, \\"property\\": \\"lineTo\\", \\"args\\": [ 200, @@ -7340,6 +7342,7 @@ exports[`record integration tests should record canvas mutations 1`] = ` \\"data\\": { \\"source\\": 9, \\"id\\": 16, + \\"type\\": 0, \\"property\\": \\"stroke\\", \\"args\\": [] } @@ -9523,6 +9526,224 @@ exports[`record integration tests should record shadow DOM 1`] = ` ]" `; +exports[`record integration tests should record webgl canvas mutations 1`] = ` +"[ + { + \\"type\\": 0, + \\"data\\": {} + }, + { + \\"type\\": 1, + \\"data\\": {} + }, + { + \\"type\\": 4, + \\"data\\": { + \\"href\\": \\"about:blank\\", + \\"width\\": 1920, + \\"height\\": 1080 + } + }, + { + \\"type\\": 2, + \\"data\\": { + \\"node\\": { + \\"type\\": 0, + \\"childNodes\\": [ + { + \\"type\\": 1, + \\"name\\": \\"html\\", + \\"publicId\\": \\"\\", + \\"systemId\\": \\"\\", + \\"id\\": 2 + }, + { + \\"type\\": 2, + \\"tagName\\": \\"html\\", + \\"attributes\\": { + \\"lang\\": \\"en\\" + }, + \\"childNodes\\": [ + { + \\"type\\": 2, + \\"tagName\\": \\"head\\", + \\"attributes\\": {}, + \\"childNodes\\": [ + { + \\"type\\": 3, + \\"textContent\\": \\"\\\\n \\", + \\"id\\": 5 + }, + { + \\"type\\": 2, + \\"tagName\\": \\"meta\\", + \\"attributes\\": { + \\"charset\\": \\"UTF-8\\" + }, + \\"childNodes\\": [], + \\"id\\": 6 + }, + { + \\"type\\": 3, + \\"textContent\\": \\"\\\\n \\", + \\"id\\": 7 + }, + { + \\"type\\": 2, + \\"tagName\\": \\"meta\\", + \\"attributes\\": { + \\"name\\": \\"viewport\\", + \\"content\\": \\"width=device-width, initial-scale=1.0\\" + }, + \\"childNodes\\": [], + \\"id\\": 8 + }, + { + \\"type\\": 3, + \\"textContent\\": \\"\\\\n \\", + \\"id\\": 9 + }, + { + \\"type\\": 2, + \\"tagName\\": \\"title\\", + \\"attributes\\": {}, + \\"childNodes\\": [ + { + \\"type\\": 3, + \\"textContent\\": \\"canvas\\", + \\"id\\": 11 + } + ], + \\"id\\": 10 + }, + { + \\"type\\": 3, + \\"textContent\\": \\"\\\\n \\", + \\"id\\": 12 + } + ], + \\"id\\": 4 + }, + { + \\"type\\": 3, + \\"textContent\\": \\"\\\\n \\", + \\"id\\": 13 + }, + { + \\"type\\": 2, + \\"tagName\\": \\"body\\", + \\"attributes\\": {}, + \\"childNodes\\": [ + { + \\"type\\": 3, + \\"textContent\\": \\"\\\\n \\", + \\"id\\": 15 + }, + { + \\"type\\": 2, + \\"tagName\\": \\"canvas\\", + \\"attributes\\": { + \\"id\\": \\"myCanvas\\", + \\"width\\": \\"200\\", + \\"height\\": \\"100\\", + \\"style\\": \\"border: 1px solid #000000\\", + \\"rr_dataURL\\": \\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAMgAAABkCAYAAADDhn8LAAACnklEQVR4Xu3VsRHAMAzEsHj/pTOBXbB9pFchyLycz0eAwFXgsCFA4C4gEK+DwENAIJ4HAYF4AwSagD9IczM1IiCQkUNbswkIpLmZGhEQyMihrdkEBNLcTI0ICGTk0NZsAgJpbqZGBAQycmhrNgGBNDdTIwICGTm0NZuAQJqbqREBgYwc2ppNQCDNzdSIgEBGDm3NJiCQ5mZqREAgI4e2ZhMQSHMzNSIgkJFDW7MJCKS5mRoREMjIoa3ZBATS3EyNCAhk5NDWbAICaW6mRgQEMnJoazYBgTQ3UyMCAhk5tDWbgECam6kRAYGMHNqaTUAgzc3UiIBARg5tzSYgkOZmakRAICOHtmYTEEhzMzUiIJCRQ1uzCQikuZkaERDIyKGt2QQE0txMjQgIZOTQ1mwCAmlupkYEBDJyaGs2AYE0N1MjAgIZObQ1m4BAmpupEQGBjBzamk1AIM3N1IiAQEYObc0mIJDmZmpEQCAjh7ZmExBIczM1IiCQkUNbswkIpLmZGhEQyMihrdkEBNLcTI0ICGTk0NZsAgJpbqZGBAQycmhrNgGBNDdTIwICGTm0NZuAQJqbqREBgYwc2ppNQCDNzdSIgEBGDm3NJiCQ5mZqREAgI4e2ZhMQSHMzNSIgkJFDW7MJCKS5mRoREMjIoa3ZBATS3EyNCAhk5NDWbAICaW6mRgQEMnJoazYBgTQ3UyMCAhk5tDWbgECam6kRAYGMHNqaTUAgzc3UiIBARg5tzSYgkOZmakRAICOHtmYTEEhzMzUiIJCRQ1uzCQikuZkaERDIyKGt2QQE0txMjQgIZOTQ1mwCAmlupkYEBDJyaGs2AYE0N1MjAgIZObQ1m4BAmpupEQGBjBzamk1AIM3N1IiAQEYObc0mIJDmZmpE4Af1gABlH0hlGgAAAABJRU5ErkJggg==\\" + }, + \\"childNodes\\": [ + { + \\"type\\": 3, + \\"textContent\\": \\"\\\\n \\", + \\"id\\": 17 + } + ], + \\"id\\": 16 + }, + { + \\"type\\": 3, + \\"textContent\\": \\"\\\\n \\", + \\"id\\": 18 + }, + { + \\"type\\": 2, + \\"tagName\\": \\"script\\", + \\"attributes\\": {}, + \\"childNodes\\": [ + { + \\"type\\": 3, + \\"textContent\\": \\"SCRIPT_PLACEHOLDER\\", + \\"id\\": 20 + } + ], + \\"id\\": 19 + }, + { + \\"type\\": 3, + \\"textContent\\": \\"\\\\n \\\\n \\", + \\"id\\": 21 + }, + { + \\"type\\": 2, + \\"tagName\\": \\"script\\", + \\"attributes\\": {}, + \\"childNodes\\": [ + { + \\"type\\": 3, + \\"textContent\\": \\"SCRIPT_PLACEHOLDER\\", + \\"id\\": 23 + } + ], + \\"id\\": 22 + }, + { + \\"type\\": 3, + \\"textContent\\": \\"\\\\n \\\\n \\\\n\\\\n\\", + \\"id\\": 24 + } + ], + \\"id\\": 14 + } + ], + \\"id\\": 3 + } + ], + \\"id\\": 1 + }, + \\"initialOffset\\": { + \\"left\\": 0, + \\"top\\": 0 + } + } + }, + { + \\"type\\": 3, + \\"data\\": { + \\"source\\": 9, + \\"id\\": 16, + \\"type\\": 1, + \\"property\\": \\"clearColor\\", + \\"args\\": [ + 1, + 0, + 0, + 1 + ] + } + }, + { + \\"type\\": 3, + \\"data\\": { + \\"source\\": 9, + \\"id\\": 16, + \\"type\\": 1, + \\"property\\": \\"clear\\", + \\"args\\": [ + 16384 + ] + } + } +]" +`; + exports[`record integration tests will serialize node before record 1`] = ` "[ { diff --git a/packages/rrweb/test/events/webgl.ts b/packages/rrweb/test/events/webgl.ts new file mode 100644 index 0000000000..4dcdac81f6 --- /dev/null +++ b/packages/rrweb/test/events/webgl.ts @@ -0,0 +1,120 @@ +export default [ + { + type: 4, + data: { + href: '', + width: 1600, + height: 900, + }, + timestamp: 1636379531385, + }, + { + type: 2, + data: { + node: { + type: 0, + childNodes: [ + { type: 1, name: 'html', publicId: '', systemId: '', id: 2 }, + { + type: 2, + tagName: 'html', + attributes: { lang: 'en' }, + childNodes: [ + { + type: 2, + tagName: 'head', + attributes: {}, + childNodes: [ + { type: 3, textContent: '\n ', id: 5 }, + { + type: 2, + tagName: 'meta', + attributes: { charset: 'UTF-8' }, + childNodes: [], + id: 6, + }, + { type: 3, textContent: '\n ', id: 7 }, + { + type: 2, + tagName: 'meta', + attributes: { + name: 'viewport', + content: 'width=device-width, initial-scale=1.0', + }, + childNodes: [], + id: 8, + }, + { type: 3, textContent: '\n ', id: 9 }, + { + type: 2, + tagName: 'title', + attributes: {}, + childNodes: [{ type: 3, textContent: 'canvas', id: 11 }], + id: 10, + }, + { type: 3, textContent: '\n ', id: 12 }, + ], + id: 4, + }, + { type: 3, textContent: '\n ', id: 13 }, + { + type: 2, + tagName: 'body', + attributes: {}, + childNodes: [ + { type: 3, textContent: '\n ', id: 15 }, + { + type: 2, + tagName: 'canvas', + attributes: { + id: 'myCanvas', + width: '200', + height: '100', + style: 'border: 1px solid #000000', + rr_dataURL: + 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAMgAAABkCAYAAADDhn8LAAACnklEQVR4Xu3VsRHAMAzEsHj/pTOBXbB9pFchyLycz0eAwFXgsCFA4C4gEK+DwENAIJ4HAYF4AwSagD9IczM1IiCQkUNbswkIpLmZGhEQyMihrdkEBNLcTI0ICGTk0NZsAgJpbqZGBAQycmhrNgGBNDdTIwICGTm0NZuAQJqbqREBgYwc2ppNQCDNzdSIgEBGDm3NJiCQ5mZqREAgI4e2ZhMQSHMzNSIgkJFDW7MJCKS5mRoREMjIoa3ZBATS3EyNCAhk5NDWbAICaW6mRgQEMnJoazYBgTQ3UyMCAhk5tDWbgECam6kRAYGMHNqaTUAgzc3UiIBARg5tzSYgkOZmakRAICOHtmYTEEhzMzUiIJCRQ1uzCQikuZkaERDIyKGt2QQE0txMjQgIZOTQ1mwCAmlupkYEBDJyaGs2AYE0N1MjAgIZObQ1m4BAmpupEQGBjBzamk1AIM3N1IiAQEYObc0mIJDmZmpEQCAjh7ZmExBIczM1IiCQkUNbswkIpLmZGhEQyMihrdkEBNLcTI0ICGTk0NZsAgJpbqZGBAQycmhrNgGBNDdTIwICGTm0NZuAQJqbqREBgYwc2ppNQCDNzdSIgEBGDm3NJiCQ5mZqREAgI4e2ZhMQSHMzNSIgkJFDW7MJCKS5mRoREMjIoa3ZBATS3EyNCAhk5NDWbAICaW6mRgQEMnJoazYBgTQ3UyMCAhk5tDWbgECam6kRAYGMHNqaTUAgzc3UiIBARg5tzSYgkOZmakRAICOHtmYTEEhzMzUiIJCRQ1uzCQikuZkaERDIyKGt2QQE0txMjQgIZOTQ1mwCAmlupkYEBDJyaGs2AYE0N1MjAgIZObQ1m4BAmpupEQGBjBzamk1AIM3N1IiAQEYObc0mIJDmZmpE4Af1gABlH0hlGgAAAABJRU5ErkJggg==', + }, + childNodes: [{ type: 3, textContent: '\n ', id: 17 }], + id: 16, + }, + { type: 3, textContent: '\n ', id: 18 }, + { + type: 2, + tagName: 'script', + attributes: {}, + childNodes: [ + { type: 3, textContent: 'SCRIPT_PLACEHOLDER', id: 20 }, + ], + id: 19, + }, + { type: 3, textContent: '\n \n\n', id: 21 }, + ], + id: 14, + }, + ], + id: 3, + }, + ], + id: 1, + }, + initialOffset: { left: 0, top: 0 }, + }, + timestamp: 1636379531389, + }, + { + type: 3, + data: { + source: 9, + id: 16, + type: 1, + property: 'clearColor', + args: [1, 0, 0, 1], + }, + timestamp: 1636379532355, + }, + { + type: 3, + data: { source: 9, id: 16, type: 1, property: 'clear', args: [16384] }, + timestamp: 1636379532356, + }, +]; diff --git a/packages/rrweb/test/html/canvas-webgl.html b/packages/rrweb/test/html/canvas-webgl.html new file mode 100644 index 0000000000..52aacabca6 --- /dev/null +++ b/packages/rrweb/test/html/canvas-webgl.html @@ -0,0 +1,27 @@ + + + + + + canvas + + + + + + + diff --git a/packages/rrweb/test/integration.test.ts b/packages/rrweb/test/integration.test.ts index acddfdf039..58ab0ac950 100644 --- a/packages/rrweb/test/integration.test.ts +++ b/packages/rrweb/test/integration.test.ts @@ -405,6 +405,19 @@ describe('record integration tests', function (this: ISuite) { assertSnapshot(snapshots); }); + it('should record webgl canvas mutations', async () => { + const page: puppeteer.Page = await browser.newPage(); + await page.goto('about:blank'); + await page.setContent( + getHtml.call(this, 'canvas-webgl.html', { + recordCanvas: true, + }), + ); + await page.waitForTimeout(50); + const snapshots = await page.evaluate('window.snapshots'); + assertSnapshot(snapshots); + }); + it('will serialize node before record', async () => { const page: puppeteer.Page = await browser.newPage(); await page.goto('about:blank'); diff --git a/packages/rrweb/test/replay/__image_snapshots__/webgl-test-ts-replayer-webgl-should-doutput-simple-webgl-object-1-snap.png b/packages/rrweb/test/replay/__image_snapshots__/webgl-test-ts-replayer-webgl-should-doutput-simple-webgl-object-1-snap.png new file mode 100644 index 0000000000000000000000000000000000000000..a153bbfc8b80f2bfa11fca6ce42bffe0d24f5dab GIT binary patch literal 10796 zcmeAS@N?(olHy`uVBq!ia0y~yU~gbxV6os}1B$%3e9#$4F%}28J29*~C-ahlL4m>3 z#WAE}&YL?MbD0eV7##O4%(~I^Dlk+lNcF-*XXg!$FCTvO`C}2YuioZbR5nmI2z+>6 zJ7+J1$-q#}aRAB`WHNxT99kH7AS?kz24)D$1&gvHPj9k+|NU3C8lq%Q(mBg@*I(bv zX2^NSBmoxtvHjVc}B{@wEJ${PIj=&f!nrn{hPxXZk%Ez(N=7Ziz(x^ z>Px<^aUv>v?p`K{a}sYeeUM1@i2h)^-vc7@A!}X(C?J8LL!f~X!ct;MfG`?37$J^k zP;g=xRmQ+D8X%)7WH9E1j??E7kDp6~L=MBB6WbW}T)SloDl>tgA^ro434}>(F@~)` z!BX0dDgy^A$+g62xQ&Jz{&vJ@3K>lyqbUTO21e^Fa4;~8)-vF5z}D*;EiFe&%hA$u zw6tUxEgZm!U^Fj`<^^K&Ld~{vc6FX34gxH?1tzoWx}x%8l&TW zprkZ9?g!3L45K5%;BXinDhCI{pq>%1G#DUAb);s+o}crf;(?yx=B`(4-? z>OhMp(5a%qIMO^iBLt46(Si^h45I}hI2ZGfj!ND+EI6#7d^uodJiiO%Hvzz-bf(-C<^>bP0 Hl+XkK4Zbd2 literal 0 HcmV?d00001 diff --git a/packages/rrweb/test/replay/webgl.test.ts b/packages/rrweb/test/replay/webgl.test.ts new file mode 100644 index 0000000000..e353050132 --- /dev/null +++ b/packages/rrweb/test/replay/webgl.test.ts @@ -0,0 +1,66 @@ +import * as fs from 'fs'; +import * as path from 'path'; +import { assertDomSnapshot, launchPuppeteer } from '../utils'; +import { toMatchImageSnapshot } from 'jest-image-snapshot'; +import * as puppeteer from 'puppeteer'; +import events from '../events/webgl'; + +interface ISuite { + code: string; + browser: puppeteer.Browser; + page: puppeteer.Page; +} + +expect.extend({ toMatchImageSnapshot }); + +describe('replayer', function () { + jest.setTimeout(10_000); + + let code: ISuite['code']; + let browser: ISuite['browser']; + let page: ISuite['page']; + + beforeAll(async () => { + browser = await launchPuppeteer(); + + const bundlePath = path.resolve(__dirname, '../../dist/rrweb.min.js'); + code = fs.readFileSync(bundlePath, 'utf8'); + }); + + beforeEach(async () => { + page = await browser.newPage(); + await page.goto('about:blank'); + // mouse cursor canvas is large and pushes the replayer below the fold + // lets hide it... + await page.addStyleTag({ + content: '.replayer-mouse-tail{display: none !important;}', + }); + await page.evaluate(code); + await page.evaluate(`let events = ${JSON.stringify(events)}`); + + page.on('console', (msg) => console.log('PAGE LOG:', msg.text())); + }); + + afterEach(async () => { + await page.close(); + }); + + afterAll(async () => { + await browser.close(); + }); + + describe('webgl', () => { + it('should doutput simple webgl object', async () => { + await page.evaluate(` + const { Replayer } = rrweb; + const replayer = new Replayer(events, { + UNSAFE_replayCanvas: true, + }); + replayer.play(2500); + `); + + const image = await page.screenshot(); + expect(image).toMatchImageSnapshot(); + }); + }); +}); diff --git a/packages/rrweb/tsconfig.json b/packages/rrweb/tsconfig.json index 624bc19af8..6ac48c750c 100644 --- a/packages/rrweb/tsconfig.json +++ b/packages/rrweb/tsconfig.json @@ -13,5 +13,9 @@ "downlevelIteration": true }, "exclude": ["test"], - "include": ["src", "node_modules/@types/css-font-loading-module/index.d.ts"] + "include": [ + "src", + "node_modules/@types/css-font-loading-module/index.d.ts", + "node_modules/@types/jest-image-snapshot/index.d.ts" + ] } diff --git a/packages/rrweb/typings/record/observers/canvas-2d.d.ts b/packages/rrweb/typings/record/observers/canvas-2d.d.ts new file mode 100644 index 0000000000..3912e174f4 --- /dev/null +++ b/packages/rrweb/typings/record/observers/canvas-2d.d.ts @@ -0,0 +1,2 @@ +import { blockClass, canvasMutationCallback, IWindow, listenerHandler, Mirror } from '../../types'; +export default function initCanvas2DMutationObserver(cb: canvasMutationCallback, win: IWindow, blockClass: blockClass, mirror: Mirror): listenerHandler; diff --git a/packages/rrweb/typings/record/observers/canvas-web-gl.d.ts b/packages/rrweb/typings/record/observers/canvas-web-gl.d.ts new file mode 100644 index 0000000000..9c18d9b3b0 --- /dev/null +++ b/packages/rrweb/typings/record/observers/canvas-web-gl.d.ts @@ -0,0 +1,2 @@ +import { blockClass, canvasMutationCallback, IWindow, listenerHandler, Mirror } from '../../types'; +export default function initCanvasWebGLMutationObserver(cb: canvasMutationCallback, win: IWindow, blockClass: blockClass, mirror: Mirror): listenerHandler; diff --git a/packages/rrweb/typings/types.d.ts b/packages/rrweb/typings/types.d.ts index 7d9c049435..59bd91a472 100644 --- a/packages/rrweb/typings/types.d.ts +++ b/packages/rrweb/typings/types.d.ts @@ -280,6 +280,10 @@ export declare enum MouseInteractions { TouchEnd = 9, TouchCancel = 10 } +export declare enum CanvasContext { + '2D' = 0, + WebGL = 1 +} declare type mouseInteractionParam = { type: MouseInteractions; id: number; @@ -322,6 +326,7 @@ export declare type styleDeclarationCallback = (s: styleDeclarationParam) => voi export declare type canvasMutationCallback = (p: canvasMutationParam) => void; export declare type canvasMutationParam = { id: number; + type: CanvasContext; property: string; args: Array; setter?: true; diff --git a/yarn.lock b/yarn.lock index fe25937f5e..5abff4ff0d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1566,7 +1566,16 @@ dependencies: "@types/istanbul-lib-report" "*" -"@types/jest@^27.0.2": +"@types/jest-image-snapshot@^4.3.1": + version "4.3.1" + resolved "https://registry.yarnpkg.com/@types/jest-image-snapshot/-/jest-image-snapshot-4.3.1.tgz#1382e9e155d6e29af0a81efce1056aaba92110c9" + integrity sha512-WDdUruGF14C53axe/mNDgQP2YIhtcwXrwmmVP8eOGyfNTVD+FbxWjWR7RTU+lzEy4K6V6+z7nkVDm/auI/r3xQ== + dependencies: + "@types/jest" "*" + "@types/pixelmatch" "*" + ssim.js "^3.1.1" + +"@types/jest@*", "@types/jest@^27.0.2": version "27.0.2" resolved "https://registry.yarnpkg.com/@types/jest/-/jest-27.0.2.tgz#ac383c4d4aaddd29bbf2b916d8d105c304a5fcd7" integrity sha512-4dRxkS/AFX0c5XW6IPMNOydLn2tEhNhJV7DnYK+0bjoJZ+QTmfucBlihX7aoEsh/ocYtkLC73UbnBXBXIxsULA== @@ -1628,6 +1637,13 @@ resolved "https://registry.yarnpkg.com/@types/parse5/-/parse5-6.0.1.tgz#f8ae4fbcd2b9ba4ff934698e28778961f9cb22ca" integrity sha512-ARATsLdrGPUnaBvxLhUlnltcMgn7pQG312S8ccdYlnyijabrX9RN/KN/iGj9Am96CoW8e/K9628BA7Bv4XHdrA== +"@types/pixelmatch@*": + version "5.2.4" + resolved "https://registry.yarnpkg.com/@types/pixelmatch/-/pixelmatch-5.2.4.tgz#ca145cc5ede1388c71c68edf2d1f5190e5ddd0f6" + integrity sha512-HDaSHIAv9kwpMN7zlmwfTv6gax0PiporJOipcrGsVNF3Ba+kryOZc0Pio5pn6NhisgWr7TaajlPEKTbTAypIBQ== + dependencies: + "@types/node" "*" + "@types/prettier@^2.1.5": version "2.4.1" resolved "https://registry.yarnpkg.com/@types/prettier/-/prettier-2.4.1.tgz#e1303048d5389563e130f5bdd89d37a99acb75eb" @@ -4428,6 +4444,11 @@ get-port@^5.1.1: resolved "https://registry.yarnpkg.com/get-port/-/get-port-5.1.1.tgz#0469ed07563479de6efb986baf053dcd7d4e3193" integrity sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ== +get-stdin@^5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/get-stdin/-/get-stdin-5.0.1.tgz#122e161591e21ff4c52530305693f20e6393a398" + integrity sha1-Ei4WFZHiH/TFJTAwVpPyDmOTo5g= + get-stream@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-3.0.0.tgz#8e943d1358dc37555054ecbe2edb05aa174ede14" @@ -4588,6 +4609,11 @@ globby@^6.1.0: pify "^2.0.0" pinkie-promise "^2.0.0" +glur@^1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/glur/-/glur-1.1.2.tgz#f20ea36db103bfc292343921f1f91e83c3467689" + integrity sha1-8g6jbbEDv8KSNDkh8fkeg8NGdok= + got@^6.7.1: version "6.7.1" resolved "https://registry.yarnpkg.com/got/-/got-6.7.1.tgz#240cd05785a9a18e561dc1b44b41c763ef1e8db0" @@ -5591,6 +5617,21 @@ jest-haste-map@^27.2.4: optionalDependencies: fsevents "^2.3.2" +jest-image-snapshot@^4.5.1: + version "4.5.1" + resolved "https://registry.yarnpkg.com/jest-image-snapshot/-/jest-image-snapshot-4.5.1.tgz#79fe0419c7729eb1be6c873365307a7b60f5cda0" + integrity sha512-0YkgupgkkCx0wIZkxvqs/oNiUT0X0d2WTpUhaAp+Dy6CpqBUZMRTIZo4KR1f+dqmx6WXrLCvecjnHLIsLkI+gQ== + dependencies: + chalk "^1.1.3" + get-stdin "^5.0.1" + glur "^1.1.2" + lodash "^4.17.4" + mkdirp "^0.5.1" + pixelmatch "^5.1.0" + pngjs "^3.4.0" + rimraf "^2.6.2" + ssim.js "^3.1.1" + jest-jasmine2@^27.2.4: version "27.2.4" resolved "https://registry.yarnpkg.com/jest-jasmine2/-/jest-jasmine2-27.2.4.tgz#4a1608133dbdb4d68b5929bfd785503ed9c9ba51" @@ -7487,6 +7528,13 @@ pirates@^4.0.1: dependencies: node-modules-regexp "^1.0.0" +pixelmatch@^5.1.0: + version "5.2.1" + resolved "https://registry.yarnpkg.com/pixelmatch/-/pixelmatch-5.2.1.tgz#9e4e4f4aa59648208a31310306a5bed5522b0d65" + integrity sha512-WjcAdYSnKrrdDdqTcVEY7aB7UhhwjYQKYhHiBXdJef0MOaQeYpUdQ+iVyBLa5YBKS8MPVPPMX7rpOByISLpeEQ== + dependencies: + pngjs "^4.0.1" + pkg-dir@^4.2.0: version "4.2.0" resolved "https://registry.yarnpkg.com/pkg-dir/-/pkg-dir-4.2.0.tgz#f099133df7ede422e81d1d8448270eeb3e4261f3" @@ -7494,6 +7542,16 @@ pkg-dir@^4.2.0: dependencies: find-up "^4.0.0" +pngjs@^3.4.0: + version "3.4.0" + resolved "https://registry.yarnpkg.com/pngjs/-/pngjs-3.4.0.tgz#99ca7d725965fb655814eaf65f38f12bbdbf555f" + integrity sha512-NCrCHhWmnQklfH4MtJMRjZ2a8c80qXeMlQMv2uVp9ISJMTt562SbGd6n2oq0PaPgKm7Z6pL9E2UlLIhC+SHL3w== + +pngjs@^4.0.1: + version "4.0.1" + resolved "https://registry.yarnpkg.com/pngjs/-/pngjs-4.0.1.tgz#f803869bb2fc1bfe1bf99aa4ec21c108117cfdbe" + integrity sha512-rf5+2/ioHeQxR6IxuYNYGFytUyG3lma/WW1nsmjeHlWwtb2aByla6dkVc8pmJ9nplzkTA0q2xx7mMWrOTqT4Gg== + postcss-calc@^7.0.1: version "7.0.5" resolved "https://registry.yarnpkg.com/postcss-calc/-/postcss-calc-7.0.5.tgz#f8a6e99f12e619c2ebc23cf6c486fdc15860933e" @@ -8426,7 +8484,7 @@ rgba-regex@^1.0.0: resolved "https://registry.yarnpkg.com/rgba-regex/-/rgba-regex-1.0.0.tgz#43374e2e2ca0968b0ef1523460b7d730ff22eeb3" integrity sha1-QzdOLiyglosO8VI0YLfXMP8i7rM= -rimraf@^2.6.1, rimraf@^2.6.3: +rimraf@^2.6.1, rimraf@^2.6.2, rimraf@^2.6.3: version "2.7.1" resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.7.1.tgz#35797f13a7fdadc566142c29d4f07ccad483e3ec" integrity sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w== @@ -8873,6 +8931,11 @@ sshpk@^1.7.0: safer-buffer "^2.0.2" tweetnacl "~0.14.0" +ssim.js@^3.1.1: + version "3.5.0" + resolved "https://registry.yarnpkg.com/ssim.js/-/ssim.js-3.5.0.tgz#d7276b9ee99b57a5ff0db34035f02f35197e62df" + integrity sha512-Aj6Jl2z6oDmgYFFbQqK7fght19bXdOxY7Tj03nF+03M9gCBAjeIiO8/PlEGMfKDwYpw4q6iBqVq2YuREorGg/g== + ssri@^8.0.0, ssri@^8.0.1: version "8.0.1" resolved "https://registry.yarnpkg.com/ssri/-/ssri-8.0.1.tgz#638e4e439e2ffbd2cd289776d5ca457c4f51a2af" From 866481624aea2d1382004102ac12aed48f773962 Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Tue, 9 Nov 2021 13:44:24 +0100 Subject: [PATCH 02/93] document the default --- packages/rrweb/src/replay/index.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/rrweb/src/replay/index.ts b/packages/rrweb/src/replay/index.ts index 372e057205..fe0fbe4e45 100644 --- a/packages/rrweb/src/replay/index.ts +++ b/packages/rrweb/src/replay/index.ts @@ -1204,11 +1204,10 @@ export class Replayer { if (!target) { return this.debugNodeNotFound(d, d.id); } + // default is '2d' for backwards compatibility (rrweb below 1.1.x) let contextType: '2d' | 'webgl' = '2d'; try { - if (d.type === CanvasContext['2D']) { - contextType = '2d'; - } else if (d.type === CanvasContext.WebGL) { + if (d.type === CanvasContext.WebGL) { contextType = 'webgl'; } const ctx = ((target as unknown) as HTMLCanvasElement).getContext( From 4d77c0959166e03b028d967051cbfc74cb5609c9 Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Wed, 24 Nov 2021 15:44:09 +0100 Subject: [PATCH 03/93] only capture rr_dataURL in 2d canvas contexts --- packages/rrweb-snapshot/src/snapshot.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/rrweb-snapshot/src/snapshot.ts b/packages/rrweb-snapshot/src/snapshot.ts index ca54833297..0084b1e1cb 100644 --- a/packages/rrweb-snapshot/src/snapshot.ts +++ b/packages/rrweb-snapshot/src/snapshot.ts @@ -484,8 +484,12 @@ function serializeNode( delete attributes.selected; } } - // canvas image data - if (tagName === 'canvas' && recordCanvas) { + // 2d canvas image data + if ( + tagName === 'canvas' && + recordCanvas && + (n as HTMLCanvasElement).getContext('2d') + ) { attributes.rr_dataURL = (n as HTMLCanvasElement).toDataURL(); } // media elements From 39eb35ac649703d50fcb7e3584ee65afa5b38eed Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Wed, 24 Nov 2021 15:47:00 +0100 Subject: [PATCH 04/93] rr_dataURL no longer part of webgl snapshot --- packages/rrweb/test/events/webgl.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/rrweb/test/events/webgl.ts b/packages/rrweb/test/events/webgl.ts index 4dcdac81f6..513c1ade48 100644 --- a/packages/rrweb/test/events/webgl.ts +++ b/packages/rrweb/test/events/webgl.ts @@ -71,8 +71,6 @@ export default [ width: '200', height: '100', style: 'border: 1px solid #000000', - rr_dataURL: - 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAMgAAABkCAYAAADDhn8LAAACnklEQVR4Xu3VsRHAMAzEsHj/pTOBXbB9pFchyLycz0eAwFXgsCFA4C4gEK+DwENAIJ4HAYF4AwSagD9IczM1IiCQkUNbswkIpLmZGhEQyMihrdkEBNLcTI0ICGTk0NZsAgJpbqZGBAQycmhrNgGBNDdTIwICGTm0NZuAQJqbqREBgYwc2ppNQCDNzdSIgEBGDm3NJiCQ5mZqREAgI4e2ZhMQSHMzNSIgkJFDW7MJCKS5mRoREMjIoa3ZBATS3EyNCAhk5NDWbAICaW6mRgQEMnJoazYBgTQ3UyMCAhk5tDWbgECam6kRAYGMHNqaTUAgzc3UiIBARg5tzSYgkOZmakRAICOHtmYTEEhzMzUiIJCRQ1uzCQikuZkaERDIyKGt2QQE0txMjQgIZOTQ1mwCAmlupkYEBDJyaGs2AYE0N1MjAgIZObQ1m4BAmpupEQGBjBzamk1AIM3N1IiAQEYObc0mIJDmZmpEQCAjh7ZmExBIczM1IiCQkUNbswkIpLmZGhEQyMihrdkEBNLcTI0ICGTk0NZsAgJpbqZGBAQycmhrNgGBNDdTIwICGTm0NZuAQJqbqREBgYwc2ppNQCDNzdSIgEBGDm3NJiCQ5mZqREAgI4e2ZhMQSHMzNSIgkJFDW7MJCKS5mRoREMjIoa3ZBATS3EyNCAhk5NDWbAICaW6mRgQEMnJoazYBgTQ3UyMCAhk5tDWbgECam6kRAYGMHNqaTUAgzc3UiIBARg5tzSYgkOZmakRAICOHtmYTEEhzMzUiIJCRQ1uzCQikuZkaERDIyKGt2QQE0txMjQgIZOTQ1mwCAmlupkYEBDJyaGs2AYE0N1MjAgIZObQ1m4BAmpupEQGBjBzamk1AIM3N1IiAQEYObc0mIJDmZmpE4Af1gABlH0hlGgAAAABJRU5ErkJggg==', }, childNodes: [{ type: 3, textContent: '\n ', id: 17 }], id: 16, From 6a87e79e644ed8c728aa6ef1a59f0fb366299fd9 Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Wed, 24 Nov 2021 16:47:20 +0100 Subject: [PATCH 05/93] ignore __diff_output__ from jest-image-snapshot --- packages/rrweb/.gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/rrweb/.gitignore b/packages/rrweb/.gitignore index afbda82bac..4875c32f5c 100644 --- a/packages/rrweb/.gitignore +++ b/packages/rrweb/.gitignore @@ -13,3 +13,4 @@ temp *.log .env +__diff_output__ \ No newline at end of file From 423bab190204bbd81be346693e61905adf1374a2 Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Wed, 24 Nov 2021 16:48:34 +0100 Subject: [PATCH 06/93] Rename generic "Monorepo" to "RRWeb Monorepo" --- .vscode/monorepo.code-workspace | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.vscode/monorepo.code-workspace b/.vscode/monorepo.code-workspace index 4caba94749..8b028ddde2 100644 --- a/.vscode/monorepo.code-workspace +++ b/.vscode/monorepo.code-workspace @@ -1,7 +1,7 @@ { "folders": [ { - "name": "Monorepo", + "name": "RRWeb Monorepo", "path": ".." }, { @@ -18,9 +18,6 @@ } ], "settings": { - "jest.disabledWorkspaceFolders": [ - "Monorepo", - "rrweb-player" - ] + "jest.disabledWorkspaceFolders": ["RRWeb Monorepo", "rrweb-player", "rrdom"] } } From 6a614f67218c8b74012e2911736f28b502241829 Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Thu, 25 Nov 2021 17:36:27 +0100 Subject: [PATCH 07/93] Serialize WebGL variables --- .../src/record/observers/canvas-web-gl.ts | 39 +++- packages/rrweb/test/record/webgl.test.ts | 175 ++++++++++++++++++ 2 files changed, 213 insertions(+), 1 deletion(-) create mode 100644 packages/rrweb/test/record/webgl.test.ts diff --git a/packages/rrweb/src/record/observers/canvas-web-gl.ts b/packages/rrweb/src/record/observers/canvas-web-gl.ts index 498b012f55..3966fbeb9c 100644 --- a/packages/rrweb/src/record/observers/canvas-web-gl.ts +++ b/packages/rrweb/src/record/observers/canvas-web-gl.ts @@ -9,6 +9,43 @@ import { } from '../../types'; import { hookSetter, isBlocked, patch } from '../../utils'; +// from webgl-recorder: https://github.com/evanw/webgl-recorder/blob/bef0e65596e981ee382126587e2dcbe0fc7748e2/webgl-recorder.js#L50-L77 +const webGLVars: Record> = {}; +function getVariable(value: any) { + if ( + value instanceof WebGLActiveInfo || + value instanceof WebGLBuffer || + value instanceof WebGLFramebuffer || + value instanceof WebGLProgram || + value instanceof WebGLRenderbuffer || + value instanceof WebGLShader || + value instanceof WebGLShaderPrecisionFormat || + value instanceof WebGLTexture || + value instanceof WebGLUniformLocation || + value instanceof WebGLVertexArrayObject || + // In Chrome, value won't be an instanceof WebGLVertexArrayObject. + (value && value.constructor.name == 'WebGLVertexArrayObjectOES') || + typeof value === 'object' + ) { + const name = value.constructor.name; + const list = webGLVars[name] || (webGLVars[name] = []); + let index = list.indexOf(value); + + if (index === -1) { + index = list.length; + list.push(value); + } + + return `$${name}#${index}`; + } + + return value; +} + +const serializeArgs = (args: Array) => { + return [...args].map(getVariable); +}; + export default function initCanvasWebGLMutationObserver( cb: canvasMutationCallback, win: IWindow, @@ -36,7 +73,7 @@ export default function initCanvasWebGLMutationObserver( ) { if (!isBlocked((this.canvas as unknown) as INode, blockClass)) { setTimeout(() => { - const recordArgs = [...args]; + const recordArgs = serializeArgs([...args]); cb({ id: mirror.getId((this.canvas as unknown) as INode), type: CanvasContext.WebGL, diff --git a/packages/rrweb/test/record/webgl.test.ts b/packages/rrweb/test/record/webgl.test.ts new file mode 100644 index 0000000000..12e6d5eee4 --- /dev/null +++ b/packages/rrweb/test/record/webgl.test.ts @@ -0,0 +1,175 @@ +/* tslint:disable no-console */ + +import * as fs from 'fs'; +import * as path from 'path'; +import * as puppeteer from 'puppeteer'; +import { + recordOptions, + listenerHandler, + eventWithTime, + EventType, + IncrementalSource, + styleSheetRuleData, + CanvasContext, +} from '../../src/types'; +import { assertSnapshot, launchPuppeteer } from '../utils'; + +interface ISuite { + code: string; + browser: puppeteer.Browser; + page: puppeteer.Page; + events: eventWithTime[]; +} + +interface IWindow extends Window { + rrweb: { + record: ( + options: recordOptions, + ) => listenerHandler | undefined; + addCustomEvent(tag: string, payload: T): void; + }; + emit: (e: eventWithTime) => undefined; +} + +const setup = function (this: ISuite, content: string): ISuite { + const ctx = {} as ISuite; + + beforeAll(async () => { + ctx.browser = await launchPuppeteer(); + + const bundlePath = path.resolve(__dirname, '../../dist/rrweb.min.js'); + ctx.code = fs.readFileSync(bundlePath, 'utf8'); + }); + + beforeEach(async () => { + ctx.page = await ctx.browser.newPage(); + await ctx.page.goto('about:blank'); + await ctx.page.setContent(content); + await ctx.page.evaluate(ctx.code); + ctx.events = []; + await ctx.page.exposeFunction('emit', (e: eventWithTime) => { + if (e.type === EventType.DomContentLoaded || e.type === EventType.Load) { + return; + } + ctx.events.push(e); + }); + + ctx.page.on('console', (msg) => console.log('PAGE LOG:', msg.text())); + }); + + afterEach(async () => { + await ctx.page.close(); + }); + + afterAll(async () => { + await ctx.browser.close(); + }); + + return ctx; +}; + +describe('record webgl', function (this: ISuite) { + jest.setTimeout(100_000); + + const ctx: ISuite = setup.call( + this, + ` + + + + + + + `, + ); + + it('will record changes to a canvas element', async () => { + await ctx.page.evaluate(() => { + const { record } = ((window as unknown) as IWindow).rrweb; + record({ + recordCanvas: true, + emit(event: eventWithTime) { + ((window as unknown) as IWindow).emit(event); + }, + }); + }); + await ctx.page.evaluate(() => { + var canvas = document.getElementById('canvas') as HTMLCanvasElement; + var gl = canvas.getContext('webgl')!; + + gl.clear(gl.COLOR_BUFFER_BIT); + }); + + await ctx.page.waitForTimeout(50); + + const lastEvent = ctx.events[ctx.events.length - 1]; + expect(lastEvent).toMatchObject({ + data: { + source: IncrementalSource.CanvasMutation, + args: [16384], + type: CanvasContext.WebGL, + property: 'clear', + }, + }); + }); + + it('will record changes to a canvas element before the canvas gets added', async () => { + await ctx.page.evaluate(() => { + const { record } = ((window as unknown) as IWindow).rrweb; + record({ + recordCanvas: true, + emit: ((window as unknown) as IWindow).emit, + }); + }); + await ctx.page.evaluate(() => { + var canvas = document.createElement('canvas'); + var gl = canvas.getContext('webgl')!; + var program = gl.createProgram()!; + gl.linkProgram(program); + gl.clear(gl.COLOR_BUFFER_BIT); + document.body.appendChild(canvas); + }); + + await ctx.page.waitForTimeout(50); + + const lastEvent = ctx.events[ctx.events.length - 1]; + expect(lastEvent).toMatchObject({ + data: { + source: IncrementalSource.CanvasMutation, + type: CanvasContext.WebGL, + property: 'clear', + }, + }); + // TODO: make this a jest snapshot + }); + + it('will record webgl variables', async () => { + await ctx.page.evaluate(() => { + const { record } = ((window as unknown) as IWindow).rrweb; + record({ + recordCanvas: true, + emit: ((window as unknown) as IWindow).emit, + }); + }); + await ctx.page.evaluate(() => { + var canvas = document.getElementById('canvas') as HTMLCanvasElement; + var gl = canvas.getContext('webgl')!; + var program0 = gl.createProgram()!; + gl.linkProgram(program0); + var program1 = gl.createProgram()!; + gl.linkProgram(program1); + }); + + await ctx.page.waitForTimeout(50); + + const lastEvent = ctx.events[ctx.events.length - 1]; + expect(lastEvent).toMatchObject({ + data: { + source: IncrementalSource.CanvasMutation, + property: 'linkProgram', + type: CanvasContext.WebGL, + args: ['$WebGLProgram#1'], // `program1` is WebGLProgram, this is the second WebGLProgram variable (#1) + }, + }); + }); +}); From 4c123b7319ea79501b8980e77a98384cd29d67e0 Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Thu, 25 Nov 2021 17:38:17 +0100 Subject: [PATCH 08/93] Move rrweb test port number to unique port rrweb-snapshot uses 3030, rrweb uses 3031 --- packages/rrweb/test/integration.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/rrweb/test/integration.test.ts b/packages/rrweb/test/integration.test.ts index 58ab0ac950..d412cebad1 100644 --- a/packages/rrweb/test/integration.test.ts +++ b/packages/rrweb/test/integration.test.ts @@ -45,7 +45,7 @@ const startServer = () => res.end(); } }); - s.listen(3030).on('listening', () => { + s.listen(3031).on('listening', () => { resolve(s); }); }); @@ -500,7 +500,7 @@ describe('record integration tests', function (this: ISuite) { it('should nest record iframe', async () => { const page: puppeteer.Page = await browser.newPage(); - await page.goto(`http://localhost:3030/html`); + await page.goto(`http://localhost:3031/html`); await page.setContent(getHtml.call(this, 'main.html')); await page.waitForTimeout(500); From f71a598ad2885efc345ca6e92325ade35787732b Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Thu, 25 Nov 2021 17:38:35 +0100 Subject: [PATCH 09/93] Prepare for WebGL2 --- packages/rrweb/src/types.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/rrweb/src/types.ts b/packages/rrweb/src/types.ts index 866fa858c0..6fad33b20c 100644 --- a/packages/rrweb/src/types.ts +++ b/packages/rrweb/src/types.ts @@ -384,6 +384,7 @@ export enum MouseInteractions { export enum CanvasContext { '2D', WebGL, + WebGL2, } type mouseInteractionParam = { From 59fec7103bfd13ba66e53b49dbdf29b72bbf7963 Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Thu, 25 Nov 2021 22:00:40 +0100 Subject: [PATCH 10/93] Split up canvas replay and record webgl vars --- packages/rrweb/src/replay/canvas/2d.ts | 48 ++++++++++ packages/rrweb/src/replay/canvas/index.ts | 34 +++++++ packages/rrweb/src/replay/canvas/webgl.ts | 93 +++++++++++++++++++ packages/rrweb/src/replay/index.ts | 50 +++------- packages/rrweb/typings/replay/canvas/2d.d.ts | 4 + .../rrweb/typings/replay/canvas/index.d.ts | 4 + .../rrweb/typings/replay/canvas/webgl.d.ts | 4 + 7 files changed, 198 insertions(+), 39 deletions(-) create mode 100644 packages/rrweb/src/replay/canvas/2d.ts create mode 100644 packages/rrweb/src/replay/canvas/index.ts create mode 100644 packages/rrweb/src/replay/canvas/webgl.ts create mode 100644 packages/rrweb/typings/replay/canvas/2d.d.ts create mode 100644 packages/rrweb/typings/replay/canvas/index.d.ts create mode 100644 packages/rrweb/typings/replay/canvas/webgl.d.ts diff --git a/packages/rrweb/src/replay/canvas/2d.ts b/packages/rrweb/src/replay/canvas/2d.ts new file mode 100644 index 0000000000..44b4f651d2 --- /dev/null +++ b/packages/rrweb/src/replay/canvas/2d.ts @@ -0,0 +1,48 @@ +import { Replayer } from '../../../typings/entries/all'; +import { canvasMutationData } from '../../types'; + +export default function canvasMutation({ + event, + mutation, + target, + imageMap, + errorHandler, +}: { + event: Parameters[0]; + mutation: canvasMutationData; + target: HTMLCanvasElement; + imageMap: Replayer['imageMap']; + errorHandler: Replayer['warnCanvasMutationFailed']; +}): void { + try { + const ctx = ((target as unknown) as HTMLCanvasElement).getContext('2d')!; + + if (mutation.setter) { + // skip some read-only type checks + // tslint:disable-next-line:no-any + (ctx as any)[mutation.property] = mutation.args[0]; + return; + } + const original = ctx[ + mutation.property as Exclude + ] as Function; + + /** + * We have serialized the image source into base64 string during recording, + * which has been preloaded before replay. + * So we can get call drawImage SYNCHRONOUSLY which avoid some fragile cast. + */ + if ( + mutation.property === 'drawImage' && + typeof mutation.args[0] === 'string' + ) { + const image = imageMap.get(event); + mutation.args[0] = image; + original.apply(ctx, mutation.args); + } else { + original.apply(ctx, mutation.args); + } + } catch (error) { + errorHandler(mutation, error); + } +} diff --git a/packages/rrweb/src/replay/canvas/index.ts b/packages/rrweb/src/replay/canvas/index.ts new file mode 100644 index 0000000000..6d5d3c53cc --- /dev/null +++ b/packages/rrweb/src/replay/canvas/index.ts @@ -0,0 +1,34 @@ +import { Replayer } from '..'; +import { CanvasContext, canvasMutationData } from '../../types'; +import webglMutation from './webgl'; +import canvas2DMutation from './2d'; + +export default function canvasMutation({ + event, + mutation, + target, + imageMap, + errorHandler, +}: { + event: Parameters[0]; + mutation: canvasMutationData; + target: HTMLCanvasElement; + imageMap: Replayer['imageMap']; + errorHandler: Replayer['warnCanvasMutationFailed']; +}): void { + try { + if ([CanvasContext.WebGL, CanvasContext.WebGL2].includes(mutation.type)) { + return webglMutation({ mutation, target, errorHandler }); + } + // default is '2d' for backwards compatibility (rrweb below 1.1.x) + return canvas2DMutation({ + event, + mutation, + target, + imageMap, + errorHandler, + }); + } catch (error) { + errorHandler(mutation, error); + } +} diff --git a/packages/rrweb/src/replay/canvas/webgl.ts b/packages/rrweb/src/replay/canvas/webgl.ts new file mode 100644 index 0000000000..60ae512659 --- /dev/null +++ b/packages/rrweb/src/replay/canvas/webgl.ts @@ -0,0 +1,93 @@ +import { Replayer } from '../../../typings/entries/all'; +import { CanvasContext, canvasMutationData } from '../../types'; + +const webGLVarMap: Map = new Map(); +function variableListFor(ctor: string) { + if (!webGLVarMap.has(ctor)) { + webGLVarMap.set(ctor, []); + } + return webGLVarMap.get(ctor) as any[]; +} + +function getContext( + target: HTMLCanvasElement, + type: CanvasContext, +): WebGLRenderingContext | WebGL2RenderingContext | null { + try { + if (type === CanvasContext.WebGL) { + return ( + target.getContext('webgl')! || target.getContext('experimental-webgl') + ); + } + return target.getContext('webgl2')!; + } catch (e) { + return null; + } +} + +const WebGLVariableConstructors = [ + WebGLActiveInfo, + WebGLBuffer, + WebGLFramebuffer, + WebGLProgram, + WebGLRenderbuffer, + WebGLShader, + WebGLShaderPrecisionFormat, + WebGLTexture, + WebGLUniformLocation, + WebGLVertexArrayObject, +]; +const WebGLVariableConstructorsNames = WebGLVariableConstructors.map( + (ctor) => ctor.name, +); + +function saveToWebGLVarMap(result: any) { + if (result?.constructor) return; // probably null or undefined + + const { name } = result.constructor; + if (!WebGLVariableConstructorsNames.includes(name)) return; // not a WebGL variable + + const variables = variableListFor(name); + if (!variables.includes(result)) variables.push(result); +} + +export default function webglMutation({ + mutation, + target, + errorHandler, +}: { + mutation: canvasMutationData; + target: HTMLCanvasElement; + errorHandler: Replayer['warnCanvasMutationFailed']; +}): void { + try { + const ctx = getContext(target, mutation.type); + if (!ctx) return; + + + if (mutation.setter) { + // skip some read-only type checks + // tslint:disable-next-line:no-any + (ctx as any)[mutation.property] = mutation.args[0]; + return; + } + const original = ctx[ + mutation.property as Exclude + ] as Function; + + const args = mutation.args.map((arg: any) => { + if (typeof arg === 'string') { + if (arg.startsWith('$')) { + const [name, index] = arg.slice(1).split('#'); + return variableListFor(name)[Number(index)]; + } + return arg; + } + }); + const result = original.apply(ctx, args); + + saveToWebGLVarMap(result); + } catch (error) { + errorHandler(mutation, error); + } +} diff --git a/packages/rrweb/src/replay/index.ts b/packages/rrweb/src/replay/index.ts index fe0fbe4e45..e20698a116 100644 --- a/packages/rrweb/src/replay/index.ts +++ b/packages/rrweb/src/replay/index.ts @@ -63,6 +63,7 @@ import { getNestedRule, getPositionsAndIndex, } from './virtual-styles'; +import canvasMutation from './canvas'; const SKIP_TIME_THRESHOLD = 10 * 1000; const SKIP_TIME_INTERVAL = 5 * 1000; @@ -1204,40 +1205,15 @@ export class Replayer { if (!target) { return this.debugNodeNotFound(d, d.id); } - // default is '2d' for backwards compatibility (rrweb below 1.1.x) - let contextType: '2d' | 'webgl' = '2d'; - try { - if (d.type === CanvasContext.WebGL) { - contextType = 'webgl'; - } - const ctx = ((target as unknown) as HTMLCanvasElement).getContext( - contextType, - )!; - if (d.setter) { - // skip some read-only type checks - // tslint:disable-next-line:no-any - (ctx as any)[d.property] = d.args[0]; - return; - } - const original = ctx[ - d.property as Exclude - ] as Function; - /** - * We have serialized the image source into base64 string during recording, - * which has been preloaded before replay. - * So we can get call drawImage SYNCHRONOUSLY which avoid some fragile cast. - */ - if (d.property === 'drawImage' && typeof d.args[0] === 'string') { - const image = this.imageMap.get(e); - d.args[0] = image; - original.apply(ctx, d.args); - } else { - original.apply(ctx, d.args); - } - } catch (error) { - this.warnCanvasMutationFailed(d, d.id, error); - } + canvasMutation({ + event: e, + mutation: d, + target: (target as unknown) as HTMLCanvasElement, + imageMap: this.imageMap, + errorHandler: this.warnCanvasMutationFailed, + }); + break; } case IncrementalSource.Font: { @@ -1863,12 +1839,8 @@ export class Replayer { } } - private warnCanvasMutationFailed( - d: canvasMutationData, - id: number, - error: unknown, - ) { - this.warn(`Has error on update canvas '${id}'`, d, error); + private warnCanvasMutationFailed(d: canvasMutationData, error: unknown) { + this.warn(`Has error on canvas update`, error, 'canvas mutation:', d); } private debugNodeNotFound(d: incrementalData, id: number) { diff --git a/packages/rrweb/typings/replay/canvas/2d.d.ts b/packages/rrweb/typings/replay/canvas/2d.d.ts new file mode 100644 index 0000000000..9765ec6780 --- /dev/null +++ b/packages/rrweb/typings/replay/canvas/2d.d.ts @@ -0,0 +1,4 @@ +import { INode } from 'rrweb-snapshot'; +import { Replayer } from '../../../typings/entries/all'; +import { canvasMutationData } from '../../types'; +export default function canvasMutation(event: Parameters[0], d: canvasMutationData, target: INode, imageMap: Replayer['imageMap'], warnCanvasMutationFailed: Replayer['warnCanvasMutationFailed']): void; diff --git a/packages/rrweb/typings/replay/canvas/index.d.ts b/packages/rrweb/typings/replay/canvas/index.d.ts new file mode 100644 index 0000000000..a207ad4d01 --- /dev/null +++ b/packages/rrweb/typings/replay/canvas/index.d.ts @@ -0,0 +1,4 @@ +import { INode } from 'rrweb-snapshot'; +import { Replayer } from '..'; +import { canvasMutationData } from '../../types'; +export default function canvasMutation(event: Parameters[0], d: canvasMutationData, target: INode, imageMap: Replayer['imageMap'], warnCanvasMutationFailed: Replayer['warnCanvasMutationFailed']): void; diff --git a/packages/rrweb/typings/replay/canvas/webgl.d.ts b/packages/rrweb/typings/replay/canvas/webgl.d.ts new file mode 100644 index 0000000000..89869c4754 --- /dev/null +++ b/packages/rrweb/typings/replay/canvas/webgl.d.ts @@ -0,0 +1,4 @@ +import { INode } from 'rrweb-snapshot'; +import { Replayer } from '../../../typings/entries/all'; +import { canvasMutationData } from '../../types'; +export default function webglMutation(d: canvasMutationData, target: INode, warnCanvasMutationFailed: Replayer['warnCanvasMutationFailed']): void; From 49d2b5822d7dff3965a3088133cd913f63a8cdfd Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Thu, 25 Nov 2021 22:15:13 +0100 Subject: [PATCH 11/93] fix typo --- packages/rrweb/test/replay/webgl.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/rrweb/test/replay/webgl.test.ts b/packages/rrweb/test/replay/webgl.test.ts index e353050132..f7d0498f17 100644 --- a/packages/rrweb/test/replay/webgl.test.ts +++ b/packages/rrweb/test/replay/webgl.test.ts @@ -50,7 +50,7 @@ describe('replayer', function () { }); describe('webgl', () => { - it('should doutput simple webgl object', async () => { + it('should output simple webgl object', async () => { await page.evaluate(` const { Replayer } = rrweb; const replayer = new Replayer(events, { From d9a03f259fde29dee350ba30fbc55b9013dbf304 Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Thu, 25 Nov 2021 22:17:41 +0100 Subject: [PATCH 12/93] fix typo part 2 --- ...gl-should-output-simple-webgl-object-1-snap.png} | Bin 1 file changed, 0 insertions(+), 0 deletions(-) rename packages/rrweb/test/replay/__image_snapshots__/{webgl-test-ts-replayer-webgl-should-doutput-simple-webgl-object-1-snap.png => webgl-test-ts-replayer-webgl-should-output-simple-webgl-object-1-snap.png} (100%) diff --git a/packages/rrweb/test/replay/__image_snapshots__/webgl-test-ts-replayer-webgl-should-doutput-simple-webgl-object-1-snap.png b/packages/rrweb/test/replay/__image_snapshots__/webgl-test-ts-replayer-webgl-should-output-simple-webgl-object-1-snap.png similarity index 100% rename from packages/rrweb/test/replay/__image_snapshots__/webgl-test-ts-replayer-webgl-should-doutput-simple-webgl-object-1-snap.png rename to packages/rrweb/test/replay/__image_snapshots__/webgl-test-ts-replayer-webgl-should-output-simple-webgl-object-1-snap.png From c03f53b5b1ea58adc5f4af4503f390487a1606fe Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Thu, 25 Nov 2021 22:58:20 +0100 Subject: [PATCH 13/93] fix typo --- packages/rrweb/src/replay/canvas/webgl.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/rrweb/src/replay/canvas/webgl.ts b/packages/rrweb/src/replay/canvas/webgl.ts index 60ae512659..8dcb40bc3a 100644 --- a/packages/rrweb/src/replay/canvas/webgl.ts +++ b/packages/rrweb/src/replay/canvas/webgl.ts @@ -42,7 +42,7 @@ const WebGLVariableConstructorsNames = WebGLVariableConstructors.map( ); function saveToWebGLVarMap(result: any) { - if (result?.constructor) return; // probably null or undefined + if (!result?.constructor) return; // probably null or undefined const { name } = result.constructor; if (!WebGLVariableConstructorsNames.includes(name)) return; // not a WebGL variable From 9f0e7d291a457be919cae0d67e437dac80483c86 Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Thu, 25 Nov 2021 23:19:40 +0100 Subject: [PATCH 14/93] Handle non-variables too --- packages/rrweb/src/replay/canvas/webgl.ts | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/packages/rrweb/src/replay/canvas/webgl.ts b/packages/rrweb/src/replay/canvas/webgl.ts index 8dcb40bc3a..115be223f5 100644 --- a/packages/rrweb/src/replay/canvas/webgl.ts +++ b/packages/rrweb/src/replay/canvas/webgl.ts @@ -76,13 +76,11 @@ export default function webglMutation({ ] as Function; const args = mutation.args.map((arg: any) => { - if (typeof arg === 'string') { - if (arg.startsWith('$')) { - const [name, index] = arg.slice(1).split('#'); - return variableListFor(name)[Number(index)]; - } - return arg; + if (typeof arg === 'string' && arg.startsWith('$')) { + const [name, index] = arg.slice(1).split('#'); + return variableListFor(name)[Number(index)]; } + return arg; }); const result = original.apply(ctx, args); From 3b1bf2a35edd5412af6e3376b90ef26a016f1519 Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Thu, 25 Nov 2021 23:20:29 +0100 Subject: [PATCH 15/93] provide correct context for warning --- packages/rrweb/src/replay/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/rrweb/src/replay/index.ts b/packages/rrweb/src/replay/index.ts index e20698a116..aa1d05db7b 100644 --- a/packages/rrweb/src/replay/index.ts +++ b/packages/rrweb/src/replay/index.ts @@ -1211,7 +1211,7 @@ export class Replayer { mutation: d, target: (target as unknown) as HTMLCanvasElement, imageMap: this.imageMap, - errorHandler: this.warnCanvasMutationFailed, + errorHandler: this.warnCanvasMutationFailed.bind(this), }); break; From a2c27a6b827a233d6c1c2816e2c3a5f392617e34 Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Fri, 26 Nov 2021 17:37:00 +0100 Subject: [PATCH 16/93] (De)Serialize a lot of different objects --- .../src/record/observers/canvas-web-gl.ts | 49 ++++++- packages/rrweb/src/replay/canvas/webgl.ts | 26 +++- packages/rrweb/src/types.ts | 15 ++ .../rrweb/test/record/serialize-args.test.ts | 109 ++++++++++++++ .../test/replay/deserialize-args.test.ts | 136 ++++++++++++++++++ 5 files changed, 330 insertions(+), 5 deletions(-) create mode 100644 packages/rrweb/test/record/serialize-args.test.ts create mode 100644 packages/rrweb/test/replay/deserialize-args.test.ts diff --git a/packages/rrweb/src/record/observers/canvas-web-gl.ts b/packages/rrweb/src/record/observers/canvas-web-gl.ts index 3966fbeb9c..b106a6002b 100644 --- a/packages/rrweb/src/record/observers/canvas-web-gl.ts +++ b/packages/rrweb/src/record/observers/canvas-web-gl.ts @@ -6,13 +6,51 @@ import { IWindow, listenerHandler, Mirror, + SerializedWebGlArg, } from '../../types'; import { hookSetter, isBlocked, patch } from '../../utils'; // from webgl-recorder: https://github.com/evanw/webgl-recorder/blob/bef0e65596e981ee382126587e2dcbe0fc7748e2/webgl-recorder.js#L50-L77 const webGLVars: Record> = {}; -function getVariable(value: any) { - if ( +export function serializeArg(value: any): SerializedWebGlArg { + + if (value instanceof Array) { + return value.map(serializeArg); + } else if (value === null) { + return value; + } else if ( + value instanceof Float32Array || + value instanceof Float64Array || + value instanceof Int32Array || + value instanceof Uint32Array || + value instanceof Uint8Array || + value instanceof Uint16Array || + value instanceof Int16Array || + value instanceof Int8Array + ) { + const name = value.constructor.name; + return { + rr_type: name, + args: [Object.values(value)], + }; + } else if ( + // SharedArrayBuffer disabled on most browsers due to spectre. + // More info: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/SharedArrayBuffer/SharedArrayBuffer + // value instanceof SharedArrayBuffer || + value instanceof ArrayBuffer + ) { + const name = value.constructor.name; + return { + rr_type: name, + args: [value.byteLength], + }; + } else if (value instanceof DataView) { + const name = value.constructor.name; + return { + rr_type: name, + args: [serializeArg(value.buffer), value.byteOffset, value.byteLength], + }; + } else if ( value instanceof WebGLActiveInfo || value instanceof WebGLBuffer || value instanceof WebGLFramebuffer || @@ -36,14 +74,17 @@ function getVariable(value: any) { list.push(value); } - return `$${name}#${index}`; + return { + rr_type: name, + index, + }; } return value; } const serializeArgs = (args: Array) => { - return [...args].map(getVariable); + return [...args].map(serializeArg); }; export default function initCanvasWebGLMutationObserver( diff --git a/packages/rrweb/src/replay/canvas/webgl.ts b/packages/rrweb/src/replay/canvas/webgl.ts index 115be223f5..4c15cd0f19 100644 --- a/packages/rrweb/src/replay/canvas/webgl.ts +++ b/packages/rrweb/src/replay/canvas/webgl.ts @@ -1,5 +1,9 @@ import { Replayer } from '../../../typings/entries/all'; -import { CanvasContext, canvasMutationData } from '../../types'; +import { + CanvasContext, + canvasMutationData, + SerializedWebGlArg, +} from '../../types'; const webGLVarMap: Map = new Map(); function variableListFor(ctor: string) { @@ -51,6 +55,26 @@ function saveToWebGLVarMap(result: any) { if (!variables.includes(result)) variables.push(result); } +export function deserializeArg(arg: SerializedWebGlArg): any { + if (arg && typeof arg === 'object' && 'rr_type' in arg) { + if ('index' in arg) { + const { rr_type: name, index } = arg; + return variableListFor(name)[index]; + } else if ('args' in arg) { + const { rr_type: name, args } = arg; + + // @ts-ignore + const ctor = window[name] as unknown; + + // @ts-ignore + return new ctor(...args.map(deserializeArg)); + } + } else if (Array.isArray(arg)) { + return arg.map(deserializeArg); + } + return arg; +} + export default function webglMutation({ mutation, target, diff --git a/packages/rrweb/src/types.ts b/packages/rrweb/src/types.ts index 6fad33b20c..cbfa0d44ac 100644 --- a/packages/rrweb/src/types.ts +++ b/packages/rrweb/src/types.ts @@ -387,6 +387,21 @@ export enum CanvasContext { WebGL2, } +export type SerializedWebGlArg = + | { + rr_type: string; + args: Array; + } + | { + rr_type: string; + index: number; + } + | string + | number + | boolean + | null + | SerializedWebGlArg[]; + type mouseInteractionParam = { type: MouseInteractions; id: number; diff --git a/packages/rrweb/test/record/serialize-args.test.ts b/packages/rrweb/test/record/serialize-args.test.ts new file mode 100644 index 0000000000..8b98feb7d5 --- /dev/null +++ b/packages/rrweb/test/record/serialize-args.test.ts @@ -0,0 +1,109 @@ +/** + * @jest-environment jsdom + */ +import { serializeArg } from '../../src/record/observers/canvas-web-gl'; + +// polyfill as jsdom does not have support for these classes +// consider replacing for https://www.npmjs.com/package/canvas +class FakeConstructor { + constructor() {} +} +global.WebGLActiveInfo = FakeConstructor as any; +global.WebGLBuffer = FakeConstructor as any; +global.WebGLFramebuffer = FakeConstructor as any; +class WebGLProgram { + constructor() {} +} +global.WebGLProgram = WebGLProgram as any; +global.WebGLRenderbuffer = FakeConstructor as any; +global.WebGLShader = FakeConstructor as any; +global.WebGLShaderPrecisionFormat = FakeConstructor as any; +global.WebGLTexture = FakeConstructor as any; +global.WebGLUniformLocation = FakeConstructor as any; +global.WebGLVertexArrayObject = FakeConstructor as any; + +describe('serializeArg', () => { + it('should serialize Float32Array values', async () => { + const float32Array = new Float32Array([-1, -1, 3, -1, -1, 3]); + const expected = { + rr_type: 'Float32Array', + args: [[-1, -1, 3, -1, -1, 3]], + }; + expect(serializeArg(float32Array)).toEqual(expected); + }); + + it('should serialize Float64Array values', async () => { + const float64Array = new Float64Array([-1, -1, 3, -1, -1, 3]); + const expected = { + rr_type: 'Float64Array', + args: [[-1, -1, 3, -1, -1, 3]], + }; + + expect(serializeArg(float64Array)).toEqual(expected); + }); + + it('should serialize ArrayBuffer values', async () => { + const arrayBuffer = new ArrayBuffer(16); + const expected = { + rr_type: 'ArrayBuffer', + args: [16], + }; + + expect(serializeArg(arrayBuffer)).toEqual(expected); + }); + + it('should serialize DataView values', async () => { + const dataView = new DataView(new ArrayBuffer(16), 0, 16); + const expected = { + rr_type: 'DataView', + args: [ + { + rr_type: 'ArrayBuffer', + args: [16], + }, + 0, + 16, + ], + }; + + expect(serializeArg(dataView)).toEqual(expected); + }); + + it('should leave arrays intact', async () => { + const array = [1, 2, 3, 4]; + expect(serializeArg(array)).toEqual(array); + }); + + it('should serialize complex objects', async () => { + const dataView = [new DataView(new ArrayBuffer(16), 0, 16), 5, 6]; + const expected = [ + { + rr_type: 'DataView', + args: [ + { + rr_type: 'ArrayBuffer', + args: [16], + }, + 0, + 16, + ], + }, + 5, + 6, + ]; + + expect(serializeArg(dataView)).toEqual(expected); + }); + + it('should leave null as-is', async () => { + expect(serializeArg(null)).toStrictEqual(null); + }); + + it('should support indexed variables', async () => { + const webGLProgram = new WebGLProgram(); + expect(serializeArg(webGLProgram)).toEqual({ + rr_type: 'WebGLProgram', + index: 0, + }); + }); +}); diff --git a/packages/rrweb/test/replay/deserialize-args.test.ts b/packages/rrweb/test/replay/deserialize-args.test.ts new file mode 100644 index 0000000000..cb14794ff4 --- /dev/null +++ b/packages/rrweb/test/replay/deserialize-args.test.ts @@ -0,0 +1,136 @@ +/** + * @jest-environment jsdom + */ + +// polyfill as jsdom does not have support for these classes +// consider replacing with https://www.npmjs.com/package/canvas +class WebGLActiveInfo { + constructor() {} +} + +global.WebGLActiveInfo = WebGLActiveInfo as any; +class WebGLBuffer { + constructor() {} +} + +global.WebGLBuffer = WebGLBuffer as any; +class WebGLFramebuffer { + constructor() {} +} + +global.WebGLFramebuffer = WebGLFramebuffer as any; +class WebGLProgram { + constructor() {} +} + +global.WebGLProgram = WebGLProgram as any; +class WebGLRenderbuffer { + constructor() {} +} + +global.WebGLRenderbuffer = WebGLRenderbuffer as any; +class WebGLShader { + constructor() {} +} + +global.WebGLShader = WebGLShader as any; +class WebGLShaderPrecisionFormat { + constructor() {} +} + +global.WebGLShaderPrecisionFormat = WebGLShaderPrecisionFormat as any; +class WebGLTexture { + constructor() {} +} + +global.WebGLTexture = WebGLTexture as any; +class WebGLUniformLocation { + constructor() {} +} + +global.WebGLUniformLocation = WebGLUniformLocation as any; +class WebGLVertexArrayObject { + constructor() {} +} + +global.WebGLVertexArrayObject = WebGLVertexArrayObject as any; + +import { deserializeArg } from '../../src/replay/canvas/webgl'; + +describe('deserializeArg', () => { + it('should deserialize Float32Array values', async () => { + expect( + deserializeArg({ + rr_type: 'Float32Array', + args: [[-1, -1, 3, -1, -1, 3]], + }), + ).toEqual(new Float32Array([-1, -1, 3, -1, -1, 3])); + }); + + it('should deserialize Float64Array values', async () => { + expect( + deserializeArg({ + rr_type: 'Float64Array', + args: [[-1, -1, 3, -1, -1, 3]], + }), + ).toEqual(new Float64Array([-1, -1, 3, -1, -1, 3])); + }); + + it('should deserialize ArrayBuffer values', async () => { + const arrayBuffer = new ArrayBuffer(16); + expect( + deserializeArg({ + rr_type: 'ArrayBuffer', + args: [16], + }), + ).toEqual(new ArrayBuffer(16)); + }); + + it('should deserialize DataView values', async () => { + expect( + deserializeArg({ + rr_type: 'DataView', + args: [ + { + rr_type: 'ArrayBuffer', + args: [16], + }, + 0, + 16, + ], + }), + ).toEqual(new DataView(new ArrayBuffer(16), 0, 16)); + }); + + it('should leave arrays intact', async () => { + const array = [1, 2, 3, 4]; + expect(deserializeArg(array)).toEqual(array); + }); + + it('should deserialize complex objects', async () => { + const serializedArg = [ + { + rr_type: 'DataView', + args: [ + { + rr_type: 'ArrayBuffer', + args: [16], + }, + 0, + 16, + ], + }, + 5, + 6, + ]; + expect(deserializeArg(serializedArg)).toEqual([ + new DataView(new ArrayBuffer(16), 0, 16), + 5, + 6, + ]); + }); + + it('should leave null as-is', async () => { + expect(deserializeArg(null)).toStrictEqual(null); + }); +}); From 45847b0c9f16212cd4c7eb00e10e173988042e71 Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Mon, 29 Nov 2021 14:37:48 +0100 Subject: [PATCH 17/93] monorepo root should be the first in the list --- .vscode/monorepo.code-workspace | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.vscode/monorepo.code-workspace b/.vscode/monorepo.code-workspace index 8b028ddde2..e0895cc7ff 100644 --- a/.vscode/monorepo.code-workspace +++ b/.vscode/monorepo.code-workspace @@ -1,7 +1,7 @@ { "folders": [ { - "name": "RRWeb Monorepo", + "name": "_RRWeb Monorepo", "path": ".." }, { @@ -18,6 +18,10 @@ } ], "settings": { - "jest.disabledWorkspaceFolders": ["RRWeb Monorepo", "rrweb-player", "rrdom"] + "jest.disabledWorkspaceFolders": [ + "_RRWeb Monorepo", + "rrweb-player", + "rrdom" + ] } } From d3499000fba01ad197ad99149810fff46c1deb1b Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Mon, 29 Nov 2021 15:41:04 +0100 Subject: [PATCH 18/93] Upgrade puppeteer to 11.x --- packages/rrweb/package.json | 4 +- yarn.lock | 140 ++++++++++++++++++++++-------------- 2 files changed, 87 insertions(+), 57 deletions(-) diff --git a/packages/rrweb/package.json b/packages/rrweb/package.json index 31783e8b8a..544b403498 100644 --- a/packages/rrweb/package.json +++ b/packages/rrweb/package.json @@ -50,7 +50,7 @@ "@types/jsdom": "^16.2.12", "@types/node": "^12.20.16", "@types/prettier": "^2.3.2", - "@types/puppeteer": "^5.4.3", + "@types/puppeteer": "^5.4.4", "cross-env": "^5.2.0", "fast-mhtml": "^1.1.9", "ignore-styles": "^5.0.1", @@ -61,7 +61,7 @@ "jsdom": "^17.0.0", "jsdom-global": "^3.0.2", "prettier": "2.2.1", - "puppeteer": "^9.1.1", + "puppeteer": "^11.0.0", "rollup": "^2.45.2", "rollup-plugin-postcss": "^3.1.1", "rollup-plugin-rename-node-modules": "^1.1.0", diff --git a/yarn.lock b/yarn.lock index 5abff4ff0d..48796afcea 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1666,7 +1666,7 @@ dependencies: "@types/node" "*" -"@types/puppeteer@^5.4.3": +"@types/puppeteer@^5.4.4": version "5.4.4" resolved "https://registry.yarnpkg.com/@types/puppeteer/-/puppeteer-5.4.4.tgz#e92abeccc4f46207c3e1b38934a1246be080ccd0" integrity sha512-3Nau+qi69CN55VwZb0ATtdUAlYlqOOQ3OfQfq0Hqgc4JMFXiQT/XInlwQ9g6LbicDslE6loIFsXFklGh5XmI6Q== @@ -3381,7 +3381,7 @@ debug@2.6.9, debug@^2.6.9: dependencies: ms "2.0.0" -debug@4, debug@^4.0.1, debug@^4.1.0, debug@^4.1.1: +debug@4, debug@4.3.2, debug@^4.0.1, debug@^4.1.0, debug@^4.1.1: version "4.3.2" resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.2.tgz#f0a49c18ac8779e31d4a0c6029dfb76873c7428b" integrity sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw== @@ -3497,10 +3497,10 @@ detect-newline@^3.0.0: resolved "https://registry.yarnpkg.com/detect-newline/-/detect-newline-3.1.0.tgz#576f5dfc63ae1a192ff192d8ad3af6308991b651" integrity sha512-TLz+x/vEXm/Y7P7wn1EJFNLxYpUD4TgMosxY6fAVJUnJMbupHBOncxyWUG9OpTaH9EBD7uFI5LfEgmMOc54DsA== -devtools-protocol@0.0.869402: - version "0.0.869402" - resolved "https://registry.yarnpkg.com/devtools-protocol/-/devtools-protocol-0.0.869402.tgz#03ade701761742e43ae4de5dc188bcd80f156d8d" - integrity sha512-VvlVYY+VDJe639yHs5PHISzdWTLL3Aw8rO4cvUtwvoxFd6FHbE4OpHHcde52M6096uYYazAmd4l0o5VuFRO2WA== +devtools-protocol@0.0.901419: + version "0.0.901419" + resolved "https://registry.yarnpkg.com/devtools-protocol/-/devtools-protocol-0.0.901419.tgz#79b5459c48fe7e1c5563c02bd72f8fec3e0cebcd" + integrity sha512-4INMPwNm9XRpBukhNbF7OB6fNTTCaI8pzy/fXg0xQzAy5h3zL1P8xT3QazgKqBrb/hAYwIBizqDBZ7GtJE74QQ== dezalgo@^1.0.0: version "1.0.3" @@ -4059,17 +4059,7 @@ extglob@^0.3.1: dependencies: is-extglob "^1.0.0" -extract-zip@^1.6.6: - version "1.7.0" - resolved "https://registry.yarnpkg.com/extract-zip/-/extract-zip-1.7.0.tgz#556cc3ae9df7f452c493a0cfb51cc30277940927" - integrity sha512-xoh5G1W/PB0/27lXgMQyIhP5DSY/LhoCsOyZgb+6iMmRtCwVBo55uKaMoEYrDCKQhWvqEip5ZPKAc6eFNyf/MA== - dependencies: - concat-stream "^1.6.2" - debug "^2.6.9" - mkdirp "^0.5.4" - yauzl "^2.10.0" - -extract-zip@^2.0.0: +extract-zip@2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/extract-zip/-/extract-zip-2.0.1.tgz#663dca56fe46df890d5f131ef4a06d22bb8ba13a" integrity sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg== @@ -4080,6 +4070,16 @@ extract-zip@^2.0.0: optionalDependencies: "@types/yauzl" "^2.9.1" +extract-zip@^1.6.6: + version "1.7.0" + resolved "https://registry.yarnpkg.com/extract-zip/-/extract-zip-1.7.0.tgz#556cc3ae9df7f452c493a0cfb51cc30277940927" + integrity sha512-xoh5G1W/PB0/27lXgMQyIhP5DSY/LhoCsOyZgb+6iMmRtCwVBo55uKaMoEYrDCKQhWvqEip5ZPKAc6eFNyf/MA== + dependencies: + concat-stream "^1.6.2" + debug "^2.6.9" + mkdirp "^0.5.4" + yauzl "^2.10.0" + extsprintf@1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.3.0.tgz#96918440e3041a7a414f8c52e3c574eb3c3e1e05" @@ -4809,6 +4809,14 @@ http-signature@~1.2.0: jsprim "^1.2.2" sshpk "^1.7.0" +https-proxy-agent@5.0.0, https-proxy-agent@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz#e2a90542abb68a762e0a0850f6c9edadfd8506b2" + integrity sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA== + dependencies: + agent-base "6" + debug "4" + https-proxy-agent@^2.2.1: version "2.2.4" resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-2.2.4.tgz#4ee7a737abd92678a293d9b34a1af4d0d08c787b" @@ -4817,14 +4825,6 @@ https-proxy-agent@^2.2.1: agent-base "^4.3.0" debug "^3.1.0" -https-proxy-agent@^5.0.0: - version "5.0.0" - resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz#e2a90542abb68a762e0a0850f6c9edadfd8506b2" - integrity sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA== - dependencies: - agent-base "6" - debug "4" - human-signals@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/human-signals/-/human-signals-2.1.0.tgz#dc91fcba42e4d06e4abaed33b3e7a3c02f514ea0" @@ -6833,6 +6833,13 @@ nice-try@^1.0.4: resolved "https://registry.yarnpkg.com/nice-try/-/nice-try-1.0.5.tgz#a3378a7696ce7d223e88fc9b764bd7ef1089e366" integrity sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ== +node-fetch@2.6.5: + version "2.6.5" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.5.tgz#42735537d7f080a7e5f78b6c549b7146be1742fd" + integrity sha512-mmlIVHJEu5rnIxgEgez6b9GgWXbkZj5YZ7fx+2r94a2E+Uirsp6HsPTPlomfdHtpt/B0cdKviwkoaM6pyvUOpQ== + dependencies: + whatwg-url "^5.0.0" + node-fetch@^2.6.1: version "2.6.1" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.1.tgz#045bd323631f76ed2e2b55573394416b639a0052" @@ -7535,7 +7542,7 @@ pixelmatch@^5.1.0: dependencies: pngjs "^4.0.1" -pkg-dir@^4.2.0: +pkg-dir@4.2.0, pkg-dir@^4.2.0: version "4.2.0" resolved "https://registry.yarnpkg.com/pkg-dir/-/pkg-dir-4.2.0.tgz#f099133df7ede422e81d1d8448270eeb3e4261f3" integrity sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ== @@ -7975,7 +7982,7 @@ process-nextick-args@~2.0.0: resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2" integrity sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag== -progress@^2.0.0, progress@^2.0.1: +progress@2.0.3, progress@^2.0.0, progress@^2.0.1: version "2.0.3" resolved "https://registry.yarnpkg.com/progress/-/progress-2.0.3.tgz#7e8cf8d8f5b8f239c1bc68beb4eb78567d572ef8" integrity sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA== @@ -8031,7 +8038,7 @@ proxy-addr@~2.0.5: forwarded "0.2.0" ipaddr.js "1.9.1" -proxy-from-env@^1.0.0, proxy-from-env@^1.1.0: +proxy-from-env@1.1.0, proxy-from-env@^1.0.0: version "1.1.0" resolved "https://registry.yarnpkg.com/proxy-from-env/-/proxy-from-env-1.1.0.tgz#e102f16ca355424865755d2c9e8ea4f24d58c3e2" integrity sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg== @@ -8073,23 +8080,23 @@ puppeteer@^1.15.0: rimraf "^2.6.1" ws "^6.1.0" -puppeteer@^9.1.1: - version "9.1.1" - resolved "https://registry.yarnpkg.com/puppeteer/-/puppeteer-9.1.1.tgz#f74b7facf86887efd6c6b9fabb7baae6fdce012c" - integrity sha512-W+nOulP2tYd/ZG99WuZC/I5ljjQQ7EUw/jQGcIb9eu8mDlZxNY2SgcJXTLG9h5gRvqA3uJOe4hZXYsd3EqioMw== - dependencies: - debug "^4.1.0" - devtools-protocol "0.0.869402" - extract-zip "^2.0.0" - https-proxy-agent "^5.0.0" - node-fetch "^2.6.1" - pkg-dir "^4.2.0" - progress "^2.0.1" - proxy-from-env "^1.1.0" - rimraf "^3.0.2" - tar-fs "^2.0.0" - unbzip2-stream "^1.3.3" - ws "^7.2.3" +puppeteer@^11.0.0: + version "11.0.0" + resolved "https://registry.yarnpkg.com/puppeteer/-/puppeteer-11.0.0.tgz#0808719c38e15315ecc1b1c28911f1c9054d201f" + integrity sha512-6rPFqN1ABjn4shgOICGDBITTRV09EjXVqhDERBDKwCLz0UyBxeeBH6Ay0vQUJ84VACmlxwzOIzVEJXThcF3aNg== + dependencies: + debug "4.3.2" + devtools-protocol "0.0.901419" + extract-zip "2.0.1" + https-proxy-agent "5.0.0" + node-fetch "2.6.5" + pkg-dir "4.2.0" + progress "2.0.3" + proxy-from-env "1.1.0" + rimraf "3.0.2" + tar-fs "2.1.1" + unbzip2-stream "1.4.3" + ws "8.2.3" q@^1.1.2, q@^1.5.1: version "1.5.1" @@ -8484,6 +8491,13 @@ rgba-regex@^1.0.0: resolved "https://registry.yarnpkg.com/rgba-regex/-/rgba-regex-1.0.0.tgz#43374e2e2ca0968b0ef1523460b7d730ff22eeb3" integrity sha1-QzdOLiyglosO8VI0YLfXMP8i7rM= +rimraf@3.0.2, rimraf@^3.0.0, rimraf@^3.0.2: + version "3.0.2" + resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-3.0.2.tgz#f1a5402ba6220ad52cc1282bac1ae3aa49fd061a" + integrity sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA== + dependencies: + glob "^7.1.3" + rimraf@^2.6.1, rimraf@^2.6.2, rimraf@^2.6.3: version "2.7.1" resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.7.1.tgz#35797f13a7fdadc566142c29d4f07ccad483e3ec" @@ -8491,13 +8505,6 @@ rimraf@^2.6.1, rimraf@^2.6.2, rimraf@^2.6.3: dependencies: glob "^7.1.3" -rimraf@^3.0.0, rimraf@^3.0.2: - version "3.0.2" - resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-3.0.2.tgz#f1a5402ba6220ad52cc1282bac1ae3aa49fd061a" - integrity sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA== - dependencies: - glob "^7.1.3" - rollup-plugin-css-only@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/rollup-plugin-css-only/-/rollup-plugin-css-only-3.1.0.tgz#6a701cc5b051c6b3f0961e69b108a9a118e1b1df" @@ -9250,7 +9257,7 @@ table@^6.0.9: string-width "^4.2.0" strip-ansi "^6.0.0" -tar-fs@^2.0.0: +tar-fs@2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-2.1.1.tgz#489a15ab85f1f0befabb370b7de4f9eb5cbe8784" integrity sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng== @@ -9453,6 +9460,11 @@ tr46@^2.1.0: dependencies: punycode "^2.1.1" +tr46@~0.0.3: + version "0.0.3" + resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a" + integrity sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o= + trim-newlines@^3.0.0: version "3.0.1" resolved "https://registry.yarnpkg.com/trim-newlines/-/trim-newlines-3.0.1.tgz#260a5d962d8b752425b32f3a7db0dcacd176c144" @@ -9656,7 +9668,7 @@ unbox-primitive@^1.0.1: has-symbols "^1.0.2" which-boxed-primitive "^1.0.2" -unbzip2-stream@^1.3.3: +unbzip2-stream@1.4.3: version "1.4.3" resolved "https://registry.yarnpkg.com/unbzip2-stream/-/unbzip2-stream-1.4.3.tgz#b0da04c4371311df771cdc215e87f2130991ace7" integrity sha512-mlExGW4w71ebDJviH16lQLtZS32VKqsSfk80GCfUlwT/4/hNRFsoscrF/c++9xinkMzECL1uL9DDwXqFWkruPg== @@ -9879,6 +9891,11 @@ wcwidth@^1.0.0: dependencies: defaults "^1.0.3" +webidl-conversions@^3.0.0: + version "3.0.1" + resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871" + integrity sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE= + webidl-conversions@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-5.0.0.tgz#ae59c8a00b121543a2acc65c0434f57b0fc11aff" @@ -9901,6 +9918,14 @@ whatwg-mimetype@^2.3.0: resolved "https://registry.yarnpkg.com/whatwg-mimetype/-/whatwg-mimetype-2.3.0.tgz#3d4b1e0312d2079879f826aff18dbeeca5960fbf" integrity sha512-M4yMwr6mAnQz76TbJm914+gPpB/nCwvZbJU28cUD6dR004SAxDLOOSUaB1JDRqLtaOV/vi0IC5lEAGFgrjGv/g== +whatwg-url@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d" + integrity sha1-lmRU6HZUYuN2RNNib2dCzotwll0= + dependencies: + tr46 "~0.0.3" + webidl-conversions "^3.0.0" + whatwg-url@^8.0.0, whatwg-url@^8.4.0, whatwg-url@^8.5.0: version "8.7.0" resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-8.7.0.tgz#656a78e510ff8f3937bc0bcbe9f5c0ac35941b77" @@ -10038,6 +10063,11 @@ write-pkg@^4.0.0: type-fest "^0.4.1" write-json-file "^3.2.0" +ws@8.2.3: + version "8.2.3" + resolved "https://registry.yarnpkg.com/ws/-/ws-8.2.3.tgz#63a56456db1b04367d0b721a0b80cae6d8becbba" + integrity sha512-wBuoj1BDpC6ZQ1B7DWQBYVLphPWkm8i9Y0/3YdHjHKHiohOJ1ws+3OccDWtH+PoC9DZD5WOTrJvNbWvjS6JWaA== + ws@^6.1.0: version "6.2.2" resolved "https://registry.yarnpkg.com/ws/-/ws-6.2.2.tgz#dd5cdbd57a9979916097652d78f1cc5faea0c32e" @@ -10045,7 +10075,7 @@ ws@^6.1.0: dependencies: async-limiter "~1.0.0" -ws@^7.2.3, ws@^7.4.3, ws@^7.4.5: +ws@^7.4.3, ws@^7.4.5: version "7.5.3" resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.3.tgz#160835b63c7d97bfab418fc1b8a9fced2ac01a74" integrity sha512-kQ/dHIzuLrS6Je9+uv81ueZomEwH0qVYstcAQ4/Z93K8zeko9gtAbttJWzoC5ukqXY1PpoouV3+VSOqEAFt5wg== From 5c7be1f876655fd0d6f4e326395cb939b98c9d3b Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Mon, 29 Nov 2021 15:44:53 +0100 Subject: [PATCH 19/93] Correctly de-serialize webgl variables --- packages/rrweb/src/replay/canvas/webgl.ts | 11 +--- .../rrweb/test/record/serialize-args.test.ts | 22 +------- .../rrweb/test/replay/webgl-mutation.test.ts | 47 ++++++++++++++++ packages/rrweb/test/utils.ts | 55 +++++++++++++++++++ .../record/observers/canvas-web-gl.d.ts | 3 +- packages/rrweb/typings/replay/canvas/2d.d.ts | 9 ++- .../rrweb/typings/replay/canvas/index.d.ts | 9 ++- .../rrweb/typings/replay/canvas/webgl.d.ts | 10 +++- 8 files changed, 131 insertions(+), 35 deletions(-) create mode 100644 packages/rrweb/test/replay/webgl-mutation.test.ts diff --git a/packages/rrweb/src/replay/canvas/webgl.ts b/packages/rrweb/src/replay/canvas/webgl.ts index 4c15cd0f19..b7654952a0 100644 --- a/packages/rrweb/src/replay/canvas/webgl.ts +++ b/packages/rrweb/src/replay/canvas/webgl.ts @@ -5,8 +5,9 @@ import { SerializedWebGlArg, } from '../../types'; +// TODO: add ability to wipe this list const webGLVarMap: Map = new Map(); -function variableListFor(ctor: string) { +export function variableListFor(ctor: string) { if (!webGLVarMap.has(ctor)) { webGLVarMap.set(ctor, []); } @@ -99,13 +100,7 @@ export default function webglMutation({ mutation.property as Exclude ] as Function; - const args = mutation.args.map((arg: any) => { - if (typeof arg === 'string' && arg.startsWith('$')) { - const [name, index] = arg.slice(1).split('#'); - return variableListFor(name)[Number(index)]; - } - return arg; - }); + const args = mutation.args.map(deserializeArg); const result = original.apply(ctx, args); saveToWebGLVarMap(result); diff --git a/packages/rrweb/test/record/serialize-args.test.ts b/packages/rrweb/test/record/serialize-args.test.ts index 8b98feb7d5..5f5fa5c143 100644 --- a/packages/rrweb/test/record/serialize-args.test.ts +++ b/packages/rrweb/test/record/serialize-args.test.ts @@ -1,26 +1,10 @@ /** * @jest-environment jsdom */ -import { serializeArg } from '../../src/record/observers/canvas-web-gl'; +import { polyfillWebGLGlobals } from '../utils'; +polyfillWebGLGlobals(); -// polyfill as jsdom does not have support for these classes -// consider replacing for https://www.npmjs.com/package/canvas -class FakeConstructor { - constructor() {} -} -global.WebGLActiveInfo = FakeConstructor as any; -global.WebGLBuffer = FakeConstructor as any; -global.WebGLFramebuffer = FakeConstructor as any; -class WebGLProgram { - constructor() {} -} -global.WebGLProgram = WebGLProgram as any; -global.WebGLRenderbuffer = FakeConstructor as any; -global.WebGLShader = FakeConstructor as any; -global.WebGLShaderPrecisionFormat = FakeConstructor as any; -global.WebGLTexture = FakeConstructor as any; -global.WebGLUniformLocation = FakeConstructor as any; -global.WebGLVertexArrayObject = FakeConstructor as any; +import { serializeArg } from '../../src/record/observers/canvas-web-gl'; describe('serializeArg', () => { it('should serialize Float32Array values', async () => { diff --git a/packages/rrweb/test/replay/webgl-mutation.test.ts b/packages/rrweb/test/replay/webgl-mutation.test.ts new file mode 100644 index 0000000000..dc9f1b836a --- /dev/null +++ b/packages/rrweb/test/replay/webgl-mutation.test.ts @@ -0,0 +1,47 @@ +/** + * @jest-environment jsdom + */ + +import webglMutation, { variableListFor } from '../../src/replay/canvas/webgl'; +import { CanvasContext, IncrementalSource } from '../../src/types'; +import { polyfillWebGLGlobals } from '../utils'; + +polyfillWebGLGlobals(); + +let canvas: HTMLCanvasElement; +describe('webglMutation', () => { + beforeEach(() => { + canvas = document.createElement('canvas'); + }); + afterEach(() => { + jest.clearAllMocks(); + }); + + it('should create webgl variables', async () => { + const createShaderMock = jest.fn().mockImplementation(() => { + return new WebGLShader(); + }); + jest.spyOn(canvas, 'getContext').mockImplementation(() => { + return ({ + createShader: createShaderMock, + } as unknown) as WebGLRenderingContext; + }); + + expect(variableListFor('WebGLShader')).toHaveLength(0); + + webglMutation({ + mutation: { + source: IncrementalSource.CanvasMutation, + type: CanvasContext.WebGL, + id: 1, + property: 'createShader', + args: [35633], + }, + target: canvas, + errorHandler: () => {}, + }); + + expect(createShaderMock).toHaveBeenCalledWith(35633); + expect(variableListFor('WebGLShader')).toHaveLength(1); + }); +}); diff --git a/packages/rrweb/test/utils.ts b/packages/rrweb/test/utils.ts index 45d427eecf..fbe34c8d45 100644 --- a/packages/rrweb/test/utils.ts +++ b/packages/rrweb/test/utils.ts @@ -338,3 +338,58 @@ export const sampleStyleSheetRemoveEvents: eventWithTime[] = [ timestamp: now + 2000, }, ]; + +export const polyfillWebGLGlobals = () => { + // polyfill as jsdom does not have support for these classes + // consider replacing with https://www.npmjs.com/package/canvas + class WebGLActiveInfo { + constructor() {} + } + + global.WebGLActiveInfo = WebGLActiveInfo as any; + class WebGLBuffer { + constructor() {} + } + + global.WebGLBuffer = WebGLBuffer as any; + class WebGLFramebuffer { + constructor() {} + } + + global.WebGLFramebuffer = WebGLFramebuffer as any; + class WebGLProgram { + constructor() {} + } + + global.WebGLProgram = WebGLProgram as any; + class WebGLRenderbuffer { + constructor() {} + } + + global.WebGLRenderbuffer = WebGLRenderbuffer as any; + class WebGLShader { + constructor() {} + } + + global.WebGLShader = WebGLShader as any; + class WebGLShaderPrecisionFormat { + constructor() {} + } + + global.WebGLShaderPrecisionFormat = WebGLShaderPrecisionFormat as any; + class WebGLTexture { + constructor() {} + } + + global.WebGLTexture = WebGLTexture as any; + class WebGLUniformLocation { + constructor() {} + } + + global.WebGLUniformLocation = WebGLUniformLocation as any; + class WebGLVertexArrayObject { + constructor() {} + } + + global.WebGLVertexArrayObject = WebGLVertexArrayObject as any; +}; diff --git a/packages/rrweb/typings/record/observers/canvas-web-gl.d.ts b/packages/rrweb/typings/record/observers/canvas-web-gl.d.ts index 9c18d9b3b0..d2ea3d274a 100644 --- a/packages/rrweb/typings/record/observers/canvas-web-gl.d.ts +++ b/packages/rrweb/typings/record/observers/canvas-web-gl.d.ts @@ -1,2 +1,3 @@ -import { blockClass, canvasMutationCallback, IWindow, listenerHandler, Mirror } from '../../types'; +import { blockClass, canvasMutationCallback, IWindow, listenerHandler, Mirror, SerializedWebGlArg } from '../../types'; +export declare function serializeArg(value: any): SerializedWebGlArg; export default function initCanvasWebGLMutationObserver(cb: canvasMutationCallback, win: IWindow, blockClass: blockClass, mirror: Mirror): listenerHandler; diff --git a/packages/rrweb/typings/replay/canvas/2d.d.ts b/packages/rrweb/typings/replay/canvas/2d.d.ts index 9765ec6780..85942bf827 100644 --- a/packages/rrweb/typings/replay/canvas/2d.d.ts +++ b/packages/rrweb/typings/replay/canvas/2d.d.ts @@ -1,4 +1,9 @@ -import { INode } from 'rrweb-snapshot'; import { Replayer } from '../../../typings/entries/all'; import { canvasMutationData } from '../../types'; -export default function canvasMutation(event: Parameters[0], d: canvasMutationData, target: INode, imageMap: Replayer['imageMap'], warnCanvasMutationFailed: Replayer['warnCanvasMutationFailed']): void; +export default function canvasMutation({ event, mutation, target, imageMap, errorHandler, }: { + event: Parameters[0]; + mutation: canvasMutationData; + target: HTMLCanvasElement; + imageMap: Replayer['imageMap']; + errorHandler: Replayer['warnCanvasMutationFailed']; +}): void; diff --git a/packages/rrweb/typings/replay/canvas/index.d.ts b/packages/rrweb/typings/replay/canvas/index.d.ts index a207ad4d01..72c6bdfce3 100644 --- a/packages/rrweb/typings/replay/canvas/index.d.ts +++ b/packages/rrweb/typings/replay/canvas/index.d.ts @@ -1,4 +1,9 @@ -import { INode } from 'rrweb-snapshot'; import { Replayer } from '..'; import { canvasMutationData } from '../../types'; -export default function canvasMutation(event: Parameters[0], d: canvasMutationData, target: INode, imageMap: Replayer['imageMap'], warnCanvasMutationFailed: Replayer['warnCanvasMutationFailed']): void; +export default function canvasMutation({ event, mutation, target, imageMap, errorHandler, }: { + event: Parameters[0]; + mutation: canvasMutationData; + target: HTMLCanvasElement; + imageMap: Replayer['imageMap']; + errorHandler: Replayer['warnCanvasMutationFailed']; +}): void; diff --git a/packages/rrweb/typings/replay/canvas/webgl.d.ts b/packages/rrweb/typings/replay/canvas/webgl.d.ts index 89869c4754..2abcc5afc7 100644 --- a/packages/rrweb/typings/replay/canvas/webgl.d.ts +++ b/packages/rrweb/typings/replay/canvas/webgl.d.ts @@ -1,4 +1,8 @@ -import { INode } from 'rrweb-snapshot'; import { Replayer } from '../../../typings/entries/all'; -import { canvasMutationData } from '../../types'; -export default function webglMutation(d: canvasMutationData, target: INode, warnCanvasMutationFailed: Replayer['warnCanvasMutationFailed']): void; +import { canvasMutationData, SerializedWebGlArg } from '../../types'; +export declare function deserializeArg(arg: SerializedWebGlArg): any; +export default function webglMutation({ mutation, target, errorHandler, }: { + mutation: canvasMutationData; + target: HTMLCanvasElement; + errorHandler: Replayer['warnCanvasMutationFailed']; +}): void; From 42693ca367d633c2f6de90ab3c72494d9f46279b Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Tue, 30 Nov 2021 14:27:34 +0100 Subject: [PATCH 20/93] Encode arrayBuffers contents to base64 --- packages/rrweb/package.json | 1 + .../src/record/observers/canvas-web-gl.ts | 7 +++++-- packages/rrweb/src/replay/canvas/webgl.ts | 3 +++ packages/rrweb/src/types.ts | 4 ++++ .../rrweb/test/record/serialize-args.test.ts | 18 ++++++++++++++---- .../rrweb/test/replay/deserialize-args.test.ts | 12 ++++++------ .../rrweb/typings/replay/canvas/webgl.d.ts | 1 + yarn.lock | 5 +++++ 8 files changed, 39 insertions(+), 12 deletions(-) diff --git a/packages/rrweb/package.json b/packages/rrweb/package.json index 544b403498..2638eeb08f 100644 --- a/packages/rrweb/package.json +++ b/packages/rrweb/package.json @@ -75,6 +75,7 @@ "dependencies": { "@types/css-font-loading-module": "0.0.4", "@xstate/fsm": "^1.4.0", + "base64-arraybuffer": "^1.0.1", "fflate": "^0.4.4", "mitt": "^1.1.3", "rrweb-snapshot": "^1.1.10" diff --git a/packages/rrweb/src/record/observers/canvas-web-gl.ts b/packages/rrweb/src/record/observers/canvas-web-gl.ts index b106a6002b..b695180539 100644 --- a/packages/rrweb/src/record/observers/canvas-web-gl.ts +++ b/packages/rrweb/src/record/observers/canvas-web-gl.ts @@ -1,3 +1,4 @@ +import { encode } from 'base64-arraybuffer'; import { INode } from 'rrweb-snapshot'; import { blockClass, @@ -39,10 +40,12 @@ export function serializeArg(value: any): SerializedWebGlArg { // value instanceof SharedArrayBuffer || value instanceof ArrayBuffer ) { - const name = value.constructor.name; + const name = value.constructor.name as 'ArrayBuffer'; + const base64 = encode(value); + return { rr_type: name, - args: [value.byteLength], + base64, }; } else if (value instanceof DataView) { const name = value.constructor.name; diff --git a/packages/rrweb/src/replay/canvas/webgl.ts b/packages/rrweb/src/replay/canvas/webgl.ts index b7654952a0..fa7a196981 100644 --- a/packages/rrweb/src/replay/canvas/webgl.ts +++ b/packages/rrweb/src/replay/canvas/webgl.ts @@ -1,3 +1,4 @@ +import { decode } from 'base64-arraybuffer'; import { Replayer } from '../../../typings/entries/all'; import { CanvasContext, @@ -69,6 +70,8 @@ export function deserializeArg(arg: SerializedWebGlArg): any { // @ts-ignore return new ctor(...args.map(deserializeArg)); + } else if ('contents' in arg) { + return decode(arg.contents); } } else if (Array.isArray(arg)) { return arg.map(deserializeArg); diff --git a/packages/rrweb/src/types.ts b/packages/rrweb/src/types.ts index cbfa0d44ac..6dd78b36c2 100644 --- a/packages/rrweb/src/types.ts +++ b/packages/rrweb/src/types.ts @@ -388,6 +388,10 @@ export enum CanvasContext { } export type SerializedWebGlArg = + | { + rr_type: 'ArrayBuffer'; + base64: string; // base64 + } | { rr_type: string; args: Array; diff --git a/packages/rrweb/test/record/serialize-args.test.ts b/packages/rrweb/test/record/serialize-args.test.ts index 5f5fa5c143..9f3fd43862 100644 --- a/packages/rrweb/test/record/serialize-args.test.ts +++ b/packages/rrweb/test/record/serialize-args.test.ts @@ -27,10 +27,10 @@ describe('serializeArg', () => { }); it('should serialize ArrayBuffer values', async () => { - const arrayBuffer = new ArrayBuffer(16); + const arrayBuffer = new Uint8Array([1, 2, 0, 4]).buffer; const expected = { rr_type: 'ArrayBuffer', - args: [16], + base64: 'AQIABA==', }; expect(serializeArg(arrayBuffer)).toEqual(expected); @@ -43,7 +43,7 @@ describe('serializeArg', () => { args: [ { rr_type: 'ArrayBuffer', - args: [16], + base64: 'AAAAAAAAAAAAAAAAAAAAAA==', }, 0, 16, @@ -66,7 +66,7 @@ describe('serializeArg', () => { args: [ { rr_type: 'ArrayBuffer', - args: [16], + base64: 'AAAAAAAAAAAAAAAAAAAAAA==', }, 0, 16, @@ -79,6 +79,16 @@ describe('serializeArg', () => { expect(serializeArg(dataView)).toEqual(expected); }); + it('should serialize arraybuffer contents', async () => { + const buffer = new Float32Array([1, 2, 3, 4]).buffer; + const expected = { + rr_type: 'ArrayBuffer', + base64: 'AACAPwAAAEAAAEBAAACAQA==', + }; + + expect(serializeArg(buffer)).toEqual(expected); + }); + it('should leave null as-is', async () => { expect(serializeArg(null)).toStrictEqual(null); }); diff --git a/packages/rrweb/test/replay/deserialize-args.test.ts b/packages/rrweb/test/replay/deserialize-args.test.ts index cb14794ff4..c5026867c1 100644 --- a/packages/rrweb/test/replay/deserialize-args.test.ts +++ b/packages/rrweb/test/replay/deserialize-args.test.ts @@ -77,13 +77,13 @@ describe('deserializeArg', () => { }); it('should deserialize ArrayBuffer values', async () => { - const arrayBuffer = new ArrayBuffer(16); + const contents = [1, 2, 0, 4]; expect( deserializeArg({ rr_type: 'ArrayBuffer', - args: [16], + base64: 'AQIABA==', }), - ).toEqual(new ArrayBuffer(16)); + ).toStrictEqual(new Uint8Array(contents).buffer); }); it('should deserialize DataView values', async () => { @@ -93,13 +93,13 @@ describe('deserializeArg', () => { args: [ { rr_type: 'ArrayBuffer', - args: [16], + base64: 'AAAAAAAAAAAAAAAAAAAAAA==', }, 0, 16, ], }), - ).toEqual(new DataView(new ArrayBuffer(16), 0, 16)); + ).toStrictEqual(new DataView(new ArrayBuffer(16), 0, 16)); }); it('should leave arrays intact', async () => { @@ -123,7 +123,7 @@ describe('deserializeArg', () => { 5, 6, ]; - expect(deserializeArg(serializedArg)).toEqual([ + expect(deserializeArg(serializedArg)).toStrictEqual([ new DataView(new ArrayBuffer(16), 0, 16), 5, 6, diff --git a/packages/rrweb/typings/replay/canvas/webgl.d.ts b/packages/rrweb/typings/replay/canvas/webgl.d.ts index 2abcc5afc7..3e87fe9565 100644 --- a/packages/rrweb/typings/replay/canvas/webgl.d.ts +++ b/packages/rrweb/typings/replay/canvas/webgl.d.ts @@ -1,5 +1,6 @@ import { Replayer } from '../../../typings/entries/all'; import { canvasMutationData, SerializedWebGlArg } from '../../types'; +export declare function variableListFor(ctor: string): any[]; export declare function deserializeArg(arg: SerializedWebGlArg): any; export default function webglMutation({ mutation, target, errorHandler, }: { mutation: canvasMutationData; diff --git a/yarn.lock b/yarn.lock index 48796afcea..38976cfc8d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2314,6 +2314,11 @@ balanced-match@^1.0.0: resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee" integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw== +base64-arraybuffer@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/base64-arraybuffer/-/base64-arraybuffer-1.0.1.tgz#87bd13525626db4a9838e00a508c2b73efcf348c" + integrity sha512-vFIUq7FdLtjZMhATwDul5RZWv2jpXQ09Pd6jcVEOvIsqCWTRFD/ONHNfyOS8dA/Ippi5dsIgpyKWKZaAKZltbA== + base64-js@^1.3.1: version "1.5.1" resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a" From ecb4d6bad8e26c2e574b46352812375fd48da29d Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Tue, 30 Nov 2021 14:30:42 +0100 Subject: [PATCH 21/93] rename contents to base64 --- packages/rrweb/src/replay/canvas/webgl.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/rrweb/src/replay/canvas/webgl.ts b/packages/rrweb/src/replay/canvas/webgl.ts index fa7a196981..ef436cc0fa 100644 --- a/packages/rrweb/src/replay/canvas/webgl.ts +++ b/packages/rrweb/src/replay/canvas/webgl.ts @@ -70,8 +70,8 @@ export function deserializeArg(arg: SerializedWebGlArg): any { // @ts-ignore return new ctor(...args.map(deserializeArg)); - } else if ('contents' in arg) { - return decode(arg.contents); + } else if ('base64' in arg) { + return decode(arg.base64); } } else if (Array.isArray(arg)) { return arg.map(deserializeArg); From 79e42a7bbd93ec01c3d426364db3a987f798e924 Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Tue, 30 Nov 2021 17:49:02 +0100 Subject: [PATCH 22/93] add webgl2 support and serialize HTMLImageElements --- packages/rrweb/src/record/observer.ts | 13 ++- .../observers/{canvas-2d.ts => canvas/2d.ts} | 4 +- .../serialize-args.ts} | 87 ++----------------- .../src/record/observers/canvas/webgl.ts | 77 ++++++++++++++++ .../src/record/observers/canvas/webgl2.ts | 78 +++++++++++++++++ packages/rrweb/src/replay/canvas/webgl.ts | 4 + packages/rrweb/src/types.ts | 4 + .../rrweb/test/record/serialize-args.test.ts | 37 ++++++-- .../test/replay/deserialize-args.test.ts | 11 +++ .../rrweb/test/replay/webgl-mutation.test.ts | 6 +- .../typings/record/observers/canvas/2d.d.ts | 2 + .../observers/canvas/serialize-args.d.ts | 3 + .../record/observers/canvas/webgl.d.ts | 2 + .../record/observers/canvas/webgl2.d.ts | 2 + 14 files changed, 236 insertions(+), 94 deletions(-) rename packages/rrweb/src/record/observers/{canvas-2d.ts => canvas/2d.ts} (96%) rename packages/rrweb/src/record/observers/{canvas-web-gl.ts => canvas/serialize-args.ts} (53%) create mode 100644 packages/rrweb/src/record/observers/canvas/webgl.ts create mode 100644 packages/rrweb/src/record/observers/canvas/webgl2.ts create mode 100644 packages/rrweb/typings/record/observers/canvas/2d.d.ts create mode 100644 packages/rrweb/typings/record/observers/canvas/serialize-args.d.ts create mode 100644 packages/rrweb/typings/record/observers/canvas/webgl.d.ts create mode 100644 packages/rrweb/typings/record/observers/canvas/webgl2.d.ts diff --git a/packages/rrweb/src/record/observer.ts b/packages/rrweb/src/record/observer.ts index 31134a0ac0..3efc97290d 100644 --- a/packages/rrweb/src/record/observer.ts +++ b/packages/rrweb/src/record/observer.ts @@ -49,8 +49,9 @@ import { import MutationBuffer from './mutation'; import { IframeManager } from './iframe-manager'; import { ShadowDomManager } from './shadow-dom-manager'; -import initCanvas2DMutationObserver from './observers/canvas-2d'; -import initCanvasWebGLMutationObserver from './observers/canvas-web-gl'; +import initCanvas2DMutationObserver from './observers/canvas/2d'; +import initCanvasWebGLMutationObserver from './observers/canvas/webgl'; +import initCanvasWebGL2MutationObserver from './observers/canvas/webgl2'; type WindowWithStoredMutationObserver = IWindow & { __rrMutationObserver?: MutationObserver; @@ -728,9 +729,17 @@ function initCanvasMutationObserver( mirror, ); + const canvasWebGL2Reset = initCanvasWebGL2MutationObserver( + cb, + win, + blockClass, + mirror, + ); + return () => { canvas2DReset(); canvasWebGLReset(); + canvasWebGL2Reset(); }; } diff --git a/packages/rrweb/src/record/observers/canvas-2d.ts b/packages/rrweb/src/record/observers/canvas/2d.ts similarity index 96% rename from packages/rrweb/src/record/observers/canvas-2d.ts rename to packages/rrweb/src/record/observers/canvas/2d.ts index 0aa30af391..68261f826f 100644 --- a/packages/rrweb/src/record/observers/canvas-2d.ts +++ b/packages/rrweb/src/record/observers/canvas/2d.ts @@ -6,8 +6,8 @@ import { IWindow, listenerHandler, Mirror, -} from '../../types'; -import { hookSetter, isBlocked, patch } from '../../utils'; +} from '../../../types'; +import { hookSetter, isBlocked, patch } from '../../../utils'; export default function initCanvas2DMutationObserver( cb: canvasMutationCallback, diff --git a/packages/rrweb/src/record/observers/canvas-web-gl.ts b/packages/rrweb/src/record/observers/canvas/serialize-args.ts similarity index 53% rename from packages/rrweb/src/record/observers/canvas-web-gl.ts rename to packages/rrweb/src/record/observers/canvas/serialize-args.ts index b695180539..f4a50907cd 100644 --- a/packages/rrweb/src/record/observers/canvas-web-gl.ts +++ b/packages/rrweb/src/record/observers/canvas/serialize-args.ts @@ -1,20 +1,9 @@ import { encode } from 'base64-arraybuffer'; -import { INode } from 'rrweb-snapshot'; -import { - blockClass, - CanvasContext, - canvasMutationCallback, - IWindow, - listenerHandler, - Mirror, - SerializedWebGlArg, -} from '../../types'; -import { hookSetter, isBlocked, patch } from '../../utils'; +import { SerializedWebGlArg } from '../../../types'; // from webgl-recorder: https://github.com/evanw/webgl-recorder/blob/bef0e65596e981ee382126587e2dcbe0fc7748e2/webgl-recorder.js#L50-L77 const webGLVars: Record> = {}; export function serializeArg(value: any): SerializedWebGlArg { - if (value instanceof Array) { return value.map(serializeArg); } else if (value === null) { @@ -53,6 +42,13 @@ export function serializeArg(value: any): SerializedWebGlArg { rr_type: name, args: [serializeArg(value.buffer), value.byteOffset, value.byteLength], }; + } else if (value instanceof HTMLImageElement) { + const name = value.constructor.name; + const { src } = value; + return { + rr_type: name, + src, + }; } else if ( value instanceof WebGLActiveInfo || value instanceof WebGLBuffer || @@ -86,71 +82,6 @@ export function serializeArg(value: any): SerializedWebGlArg { return value; } -const serializeArgs = (args: Array) => { +export const serializeArgs = (args: Array) => { return [...args].map(serializeArg); }; - -export default function initCanvasWebGLMutationObserver( - cb: canvasMutationCallback, - win: IWindow, - blockClass: blockClass, - mirror: Mirror, -): listenerHandler { - const handlers: listenerHandler[] = []; - const props = Object.getOwnPropertyNames(win.WebGLRenderingContext.prototype); - for (const prop of props) { - try { - if ( - typeof win.WebGLRenderingContext.prototype[ - prop as keyof WebGLRenderingContext - ] !== 'function' - ) { - continue; - } - const restoreHandler = patch( - win.WebGLRenderingContext.prototype, - prop, - function (original) { - return function ( - this: WebGLRenderingContext, - ...args: Array - ) { - if (!isBlocked((this.canvas as unknown) as INode, blockClass)) { - setTimeout(() => { - const recordArgs = serializeArgs([...args]); - cb({ - id: mirror.getId((this.canvas as unknown) as INode), - type: CanvasContext.WebGL, - property: prop, - args: recordArgs, - }); - }, 0); - } - return original.apply(this, args); - }; - }, - ); - handlers.push(restoreHandler); - } catch { - const hookHandler = hookSetter( - win.WebGLRenderingContext.prototype, - prop, - { - set(v) { - cb({ - id: mirror.getId((this.canvas as unknown) as INode), - type: CanvasContext.WebGL, - property: prop, - args: [v], - setter: true, - }); - }, - }, - ); - handlers.push(hookHandler); - } - } - return () => { - handlers.forEach((h) => h()); - }; -} diff --git a/packages/rrweb/src/record/observers/canvas/webgl.ts b/packages/rrweb/src/record/observers/canvas/webgl.ts new file mode 100644 index 0000000000..79559fccfa --- /dev/null +++ b/packages/rrweb/src/record/observers/canvas/webgl.ts @@ -0,0 +1,77 @@ +import { INode } from 'rrweb-snapshot'; +import { + blockClass, + CanvasContext, + canvasMutationCallback, + IWindow, + listenerHandler, + Mirror, + SerializedWebGlArg, +} from '../../../types'; +import { hookSetter, isBlocked, patch } from '../../../utils'; +import { serializeArgs } from './serialize-args'; + +export default function initCanvasWebGLMutationObserver( + cb: canvasMutationCallback, + win: IWindow, + blockClass: blockClass, + mirror: Mirror, +): listenerHandler { + const handlers: listenerHandler[] = []; + const props = Object.getOwnPropertyNames(win.WebGLRenderingContext.prototype); + for (const prop of props) { + try { + if ( + typeof win.WebGLRenderingContext.prototype[ + prop as keyof WebGLRenderingContext + ] !== 'function' + ) { + continue; + } + const restoreHandler = patch( + win.WebGLRenderingContext.prototype, + prop, + function (original) { + return function ( + this: WebGLRenderingContext, + ...args: Array + ) { + if (!isBlocked((this.canvas as unknown) as INode, blockClass)) { + setTimeout(() => { + const recordArgs = serializeArgs([...args]); + cb({ + id: mirror.getId((this.canvas as unknown) as INode), + type: CanvasContext.WebGL, + property: prop, + args: recordArgs, + }); + }, 0); + } + return original.apply(this, args); + }; + }, + ); + handlers.push(restoreHandler); + } catch { + const hookHandler = hookSetter( + win.WebGLRenderingContext.prototype, + prop, + { + set(v) { + cb({ + id: mirror.getId((this.canvas as unknown) as INode), + type: CanvasContext.WebGL, + property: prop, + args: [v], + setter: true, + }); + }, + }, + ); + handlers.push(hookHandler); + } + } + return () => { + handlers.forEach((h) => h()); + }; +} diff --git a/packages/rrweb/src/record/observers/canvas/webgl2.ts b/packages/rrweb/src/record/observers/canvas/webgl2.ts new file mode 100644 index 0000000000..9589c35a60 --- /dev/null +++ b/packages/rrweb/src/record/observers/canvas/webgl2.ts @@ -0,0 +1,78 @@ +import { INode } from 'rrweb-snapshot'; +import { + blockClass, + CanvasContext, + canvasMutationCallback, + IWindow, + listenerHandler, + Mirror, +} from '../../../types'; +import { hookSetter, isBlocked, patch } from '../../../utils'; +import { serializeArgs } from './serialize-args'; + +export default function initCanvasWebGLMutationObserver( + cb: canvasMutationCallback, + win: IWindow, + blockClass: blockClass, + mirror: Mirror, +): listenerHandler { + const handlers: listenerHandler[] = []; + const props = Object.getOwnPropertyNames( + win.WebGL2RenderingContext.prototype, + ); + for (const prop of props) { + try { + if ( + typeof win.WebGL2RenderingContext.prototype[ + prop as keyof WebGL2RenderingContext + ] !== 'function' + ) { + continue; + } + const restoreHandler = patch( + win.WebGL2RenderingContext.prototype, + prop, + function (original) { + return function ( + this: WebGL2RenderingContext, + ...args: Array + ) { + if (!isBlocked((this.canvas as unknown) as INode, blockClass)) { + setTimeout(() => { + const recordArgs = serializeArgs([...args]); + cb({ + id: mirror.getId((this.canvas as unknown) as INode), + type: CanvasContext.WebGL, + property: prop, + args: recordArgs, + }); + }, 0); + } + return original.apply(this, args); + }; + }, + ); + handlers.push(restoreHandler); + } catch { + const hookHandler = hookSetter( + win.WebGL2RenderingContext.prototype, + prop, + { + set(v) { + cb({ + id: mirror.getId((this.canvas as unknown) as INode), + type: CanvasContext.WebGL2, + property: prop, + args: [v], + setter: true, + }); + }, + }, + ); + handlers.push(hookHandler); + } + } + return () => { + handlers.forEach((h) => h()); + }; +} diff --git a/packages/rrweb/src/replay/canvas/webgl.ts b/packages/rrweb/src/replay/canvas/webgl.ts index ef436cc0fa..0f3dfee778 100644 --- a/packages/rrweb/src/replay/canvas/webgl.ts +++ b/packages/rrweb/src/replay/canvas/webgl.ts @@ -72,6 +72,10 @@ export function deserializeArg(arg: SerializedWebGlArg): any { return new ctor(...args.map(deserializeArg)); } else if ('base64' in arg) { return decode(arg.base64); + } else if ('src' in arg) { + const image = new Image(); + image.src = arg.src; + return image; } } else if (Array.isArray(arg)) { return arg.map(deserializeArg); diff --git a/packages/rrweb/src/types.ts b/packages/rrweb/src/types.ts index 6dd78b36c2..bed9ba50b0 100644 --- a/packages/rrweb/src/types.ts +++ b/packages/rrweb/src/types.ts @@ -392,6 +392,10 @@ export type SerializedWebGlArg = rr_type: 'ArrayBuffer'; base64: string; // base64 } + | { + rr_type: string; + src: string; // url of image + } | { rr_type: string; args: Array; diff --git a/packages/rrweb/test/record/serialize-args.test.ts b/packages/rrweb/test/record/serialize-args.test.ts index 9f3fd43862..3f362e53c3 100644 --- a/packages/rrweb/test/record/serialize-args.test.ts +++ b/packages/rrweb/test/record/serialize-args.test.ts @@ -4,7 +4,7 @@ import { polyfillWebGLGlobals } from '../utils'; polyfillWebGLGlobals(); -import { serializeArg } from '../../src/record/observers/canvas-web-gl'; +import { serializeArg } from '../../src/record/observers/canvas/serialize-args'; describe('serializeArg', () => { it('should serialize Float32Array values', async () => { @@ -13,7 +13,7 @@ describe('serializeArg', () => { rr_type: 'Float32Array', args: [[-1, -1, 3, -1, -1, 3]], }; - expect(serializeArg(float32Array)).toEqual(expected); + expect(serializeArg(float32Array)).toStrictEqual(expected); }); it('should serialize Float64Array values', async () => { @@ -23,7 +23,7 @@ describe('serializeArg', () => { args: [[-1, -1, 3, -1, -1, 3]], }; - expect(serializeArg(float64Array)).toEqual(expected); + expect(serializeArg(float64Array)).toStrictEqual(expected); }); it('should serialize ArrayBuffer values', async () => { @@ -33,7 +33,17 @@ describe('serializeArg', () => { base64: 'AQIABA==', }; - expect(serializeArg(arrayBuffer)).toEqual(expected); + expect(serializeArg(arrayBuffer)).toStrictEqual(expected); + }); + + it('should serialize Uint8Array values', async () => { + const object = new Uint8Array([1, 2, 0, 4]); + const expected = { + rr_type: 'Uint8Array', + args: [[1, 2, 0, 4]], + }; + + expect(serializeArg(object)).toStrictEqual(expected); }); it('should serialize DataView values', async () => { @@ -50,12 +60,12 @@ describe('serializeArg', () => { ], }; - expect(serializeArg(dataView)).toEqual(expected); + expect(serializeArg(dataView)).toStrictEqual(expected); }); it('should leave arrays intact', async () => { const array = [1, 2, 3, 4]; - expect(serializeArg(array)).toEqual(array); + expect(serializeArg(array)).toStrictEqual(array); }); it('should serialize complex objects', async () => { @@ -76,7 +86,7 @@ describe('serializeArg', () => { 6, ]; - expect(serializeArg(dataView)).toEqual(expected); + expect(serializeArg(dataView)).toStrictEqual(expected); }); it('should serialize arraybuffer contents', async () => { @@ -86,7 +96,7 @@ describe('serializeArg', () => { base64: 'AACAPwAAAEAAAEBAAACAQA==', }; - expect(serializeArg(buffer)).toEqual(expected); + expect(serializeArg(buffer)).toStrictEqual(expected); }); it('should leave null as-is', async () => { @@ -95,9 +105,18 @@ describe('serializeArg', () => { it('should support indexed variables', async () => { const webGLProgram = new WebGLProgram(); - expect(serializeArg(webGLProgram)).toEqual({ + expect(serializeArg(webGLProgram)).toStrictEqual({ rr_type: 'WebGLProgram', index: 0, }); }); + + it('should support HTMLImageElements', async () => { + const image = new Image(); + image.src = 'http://example.com/image.png'; + expect(serializeArg(image)).toStrictEqual({ + rr_type: 'HTMLImageElement', + src: 'http://example.com/image.png', + }); + }); }); diff --git a/packages/rrweb/test/replay/deserialize-args.test.ts b/packages/rrweb/test/replay/deserialize-args.test.ts index c5026867c1..855a9ef13f 100644 --- a/packages/rrweb/test/replay/deserialize-args.test.ts +++ b/packages/rrweb/test/replay/deserialize-args.test.ts @@ -133,4 +133,15 @@ describe('deserializeArg', () => { it('should leave null as-is', async () => { expect(deserializeArg(null)).toStrictEqual(null); }); + + it('should support HTMLImageElements', async () => { + const image = new Image(); + image.src = 'http://example.com/image.png'; + expect( + deserializeArg({ + rr_type: 'HTMLImageElement', + src: 'http://example.com/image.png', + }), + ).toStrictEqual(image); + }); }); diff --git a/packages/rrweb/test/replay/webgl-mutation.test.ts b/packages/rrweb/test/replay/webgl-mutation.test.ts index dc9f1b836a..478aee3ed3 100644 --- a/packages/rrweb/test/replay/webgl-mutation.test.ts +++ b/packages/rrweb/test/replay/webgl-mutation.test.ts @@ -2,12 +2,12 @@ * @jest-environment jsdom */ -import webglMutation, { variableListFor } from '../../src/replay/canvas/webgl'; -import { CanvasContext, IncrementalSource } from '../../src/types'; import { polyfillWebGLGlobals } from '../utils'; - polyfillWebGLGlobals(); +import webglMutation, { variableListFor } from '../../src/replay/canvas/webgl'; +import { CanvasContext, IncrementalSource } from '../../src/types'; + let canvas: HTMLCanvasElement; describe('webglMutation', () => { beforeEach(() => { diff --git a/packages/rrweb/typings/record/observers/canvas/2d.d.ts b/packages/rrweb/typings/record/observers/canvas/2d.d.ts new file mode 100644 index 0000000000..23d51acb6b --- /dev/null +++ b/packages/rrweb/typings/record/observers/canvas/2d.d.ts @@ -0,0 +1,2 @@ +import { blockClass, canvasMutationCallback, IWindow, listenerHandler, Mirror } from '../../../types'; +export default function initCanvas2DMutationObserver(cb: canvasMutationCallback, win: IWindow, blockClass: blockClass, mirror: Mirror): listenerHandler; diff --git a/packages/rrweb/typings/record/observers/canvas/serialize-args.d.ts b/packages/rrweb/typings/record/observers/canvas/serialize-args.d.ts new file mode 100644 index 0000000000..b6dc836dbc --- /dev/null +++ b/packages/rrweb/typings/record/observers/canvas/serialize-args.d.ts @@ -0,0 +1,3 @@ +import { SerializedWebGlArg } from '../../../types'; +export declare function serializeArg(value: any): SerializedWebGlArg; +export declare const serializeArgs: (args: Array) => SerializedWebGlArg[]; diff --git a/packages/rrweb/typings/record/observers/canvas/webgl.d.ts b/packages/rrweb/typings/record/observers/canvas/webgl.d.ts new file mode 100644 index 0000000000..c1263a8f58 --- /dev/null +++ b/packages/rrweb/typings/record/observers/canvas/webgl.d.ts @@ -0,0 +1,2 @@ +import { blockClass, canvasMutationCallback, IWindow, listenerHandler, Mirror } from '../../../types'; +export default function initCanvasWebGLMutationObserver(cb: canvasMutationCallback, win: IWindow, blockClass: blockClass, mirror: Mirror): listenerHandler; diff --git a/packages/rrweb/typings/record/observers/canvas/webgl2.d.ts b/packages/rrweb/typings/record/observers/canvas/webgl2.d.ts new file mode 100644 index 0000000000..c1263a8f58 --- /dev/null +++ b/packages/rrweb/typings/record/observers/canvas/webgl2.d.ts @@ -0,0 +1,2 @@ +import { blockClass, canvasMutationCallback, IWindow, listenerHandler, Mirror } from '../../../types'; +export default function initCanvasWebGLMutationObserver(cb: canvasMutationCallback, win: IWindow, blockClass: blockClass, mirror: Mirror): listenerHandler; From 766a4789e5c5e1fe195571b8efb3d7d21ff2214b Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Thu, 2 Dec 2021 16:20:45 +0100 Subject: [PATCH 23/93] Support serializing ImageData --- .../record/observers/canvas/serialize-args.ts | 9 +++++- .../rrweb/test/record/serialize-args.test.ts | 28 +++++++++++++++++++ packages/rrweb/test/utils.ts | 13 +++++++++ 3 files changed, 49 insertions(+), 1 deletion(-) diff --git a/packages/rrweb/src/record/observers/canvas/serialize-args.ts b/packages/rrweb/src/record/observers/canvas/serialize-args.ts index f4a50907cd..40e7074298 100644 --- a/packages/rrweb/src/record/observers/canvas/serialize-args.ts +++ b/packages/rrweb/src/record/observers/canvas/serialize-args.ts @@ -16,7 +16,8 @@ export function serializeArg(value: any): SerializedWebGlArg { value instanceof Uint8Array || value instanceof Uint16Array || value instanceof Int16Array || - value instanceof Int8Array + value instanceof Int8Array || + value instanceof Uint8ClampedArray ) { const name = value.constructor.name; return { @@ -49,6 +50,12 @@ export function serializeArg(value: any): SerializedWebGlArg { rr_type: name, src, }; + } else if (value instanceof ImageData) { + const name = value.constructor.name; + return { + rr_type: name, + args: [serializeArg(value.data), value.width, value.height], + }; } else if ( value instanceof WebGLActiveInfo || value instanceof WebGLBuffer || diff --git a/packages/rrweb/test/record/serialize-args.test.ts b/packages/rrweb/test/record/serialize-args.test.ts index 3f362e53c3..6a719e9e94 100644 --- a/packages/rrweb/test/record/serialize-args.test.ts +++ b/packages/rrweb/test/record/serialize-args.test.ts @@ -119,4 +119,32 @@ describe('serializeArg', () => { src: 'http://example.com/image.png', }); }); + + it('should serialize ImageData', async () => { + const arr = new Uint8ClampedArray(40000); + + // Iterate through every pixel + for (let i = 0; i < arr.length; i += 4) { + arr[i + 0] = 0; // R value + arr[i + 1] = 190; // G value + arr[i + 2] = 0; // B value + arr[i + 3] = 255; // A value + } + + // Initialize a new ImageData object + let imageData = new ImageData(arr, 200, 50); + + const contents = Array.from(arr); + expect(serializeArg(imageData)).toStrictEqual({ + rr_type: 'ImageData', + args: [ + { + rr_type: 'Uint8ClampedArray', + args: [contents], + }, + 200, + 50, + ], + }); + }); }); diff --git a/packages/rrweb/test/utils.ts b/packages/rrweb/test/utils.ts index fbe34c8d45..bab49946d4 100644 --- a/packages/rrweb/test/utils.ts +++ b/packages/rrweb/test/utils.ts @@ -392,4 +392,17 @@ export const polyfillWebGLGlobals = () => { } global.WebGLVertexArrayObject = WebGLVertexArrayObject as any; + + class ImageData { + public data: Uint8ClampedArray; + public width: number; + public height: number; + constructor(data: Uint8ClampedArray, width: number, height: number) { + this.data = data; + this.width = width; + this.height = height; + } + } + + global.ImageData = ImageData as any; }; From 35b60502017fdf10088498013cb03b36dc6b8173 Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Thu, 2 Dec 2021 16:21:18 +0100 Subject: [PATCH 24/93] Correctly classify WebGL2 events --- packages/rrweb/src/record/observers/canvas/webgl2.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/rrweb/src/record/observers/canvas/webgl2.ts b/packages/rrweb/src/record/observers/canvas/webgl2.ts index 9589c35a60..68bd0e1690 100644 --- a/packages/rrweb/src/record/observers/canvas/webgl2.ts +++ b/packages/rrweb/src/record/observers/canvas/webgl2.ts @@ -42,7 +42,7 @@ export default function initCanvasWebGLMutationObserver( const recordArgs = serializeArgs([...args]); cb({ id: mirror.getId((this.canvas as unknown) as INode), - type: CanvasContext.WebGL, + type: CanvasContext.WebGL2, property: prop, args: recordArgs, }); From af42d0f9b35a54364705016dde8ef253e5010c29 Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Thu, 2 Dec 2021 16:54:27 +0100 Subject: [PATCH 25/93] Serialize format changed --- packages/rrweb/test/record/webgl.test.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/rrweb/test/record/webgl.test.ts b/packages/rrweb/test/record/webgl.test.ts index 12e6d5eee4..be55835009 100644 --- a/packages/rrweb/test/record/webgl.test.ts +++ b/packages/rrweb/test/record/webgl.test.ts @@ -168,7 +168,12 @@ describe('record webgl', function (this: ISuite) { source: IncrementalSource.CanvasMutation, property: 'linkProgram', type: CanvasContext.WebGL, - args: ['$WebGLProgram#1'], // `program1` is WebGLProgram, this is the second WebGLProgram variable (#1) + args: [ + { + index: 1, + rr_type: 'WebGLProgram', + }, + ], // `program1` is WebGLProgram, this is the second WebGLProgram variable (index #1) }, }); }); From 637e3580d678fbf88ffc475f8ff47d7d0db30a72 Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Thu, 2 Dec 2021 20:39:24 +0100 Subject: [PATCH 26/93] check if canvas has contents before we save the dataURL --- packages/rrweb-snapshot/src/snapshot.ts | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/packages/rrweb-snapshot/src/snapshot.ts b/packages/rrweb-snapshot/src/snapshot.ts index 00b97a2620..1928649b9e 100644 --- a/packages/rrweb-snapshot/src/snapshot.ts +++ b/packages/rrweb-snapshot/src/snapshot.ts @@ -492,13 +492,20 @@ function serializeNode( delete attributes.selected; } } - // 2d canvas image data - if ( - tagName === 'canvas' && - recordCanvas && - (n as HTMLCanvasElement).getContext('2d') - ) { - attributes.rr_dataURL = (n as HTMLCanvasElement).toDataURL(); + // canvas image data + if (tagName === 'canvas' && recordCanvas) { + const canvasDataURL = (n as HTMLCanvasElement).toDataURL(); + + // create blank canvas of same dimensions + const blankCanvas = document.createElement('canvas'); + blankCanvas.width = (n as HTMLCanvasElement).width; + blankCanvas.height = (n as HTMLCanvasElement).height; + const blankCanvasDataURL = blankCanvas.toDataURL(); + + // no need to save dataURL if it's the same as blank canvas + if (canvasDataURL !== blankCanvasDataURL) { + attributes.rr_dataURL = (n as HTMLCanvasElement).toDataURL(); + } } // media elements if (tagName === 'audio' || tagName === 'video') { From e2e715000b77d7c9c71fbac0830995248e7f4cfe Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Thu, 2 Dec 2021 20:41:19 +0100 Subject: [PATCH 27/93] Remove blank dataURL --- packages/rrweb/test/__snapshots__/integration.test.ts.snap | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/rrweb/test/__snapshots__/integration.test.ts.snap b/packages/rrweb/test/__snapshots__/integration.test.ts.snap index f2b42154c3..56ea412acf 100644 --- a/packages/rrweb/test/__snapshots__/integration.test.ts.snap +++ b/packages/rrweb/test/__snapshots__/integration.test.ts.snap @@ -9646,8 +9646,7 @@ exports[`record integration tests should record webgl canvas mutations 1`] = ` \\"id\\": \\"myCanvas\\", \\"width\\": \\"200\\", \\"height\\": \\"100\\", - \\"style\\": \\"border: 1px solid #000000\\", - \\"rr_dataURL\\": \\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAMgAAABkCAYAAADDhn8LAAACnklEQVR4Xu3VsRHAMAzEsHj/pTOBXbB9pFchyLycz0eAwFXgsCFA4C4gEK+DwENAIJ4HAYF4AwSagD9IczM1IiCQkUNbswkIpLmZGhEQyMihrdkEBNLcTI0ICGTk0NZsAgJpbqZGBAQycmhrNgGBNDdTIwICGTm0NZuAQJqbqREBgYwc2ppNQCDNzdSIgEBGDm3NJiCQ5mZqREAgI4e2ZhMQSHMzNSIgkJFDW7MJCKS5mRoREMjIoa3ZBATS3EyNCAhk5NDWbAICaW6mRgQEMnJoazYBgTQ3UyMCAhk5tDWbgECam6kRAYGMHNqaTUAgzc3UiIBARg5tzSYgkOZmakRAICOHtmYTEEhzMzUiIJCRQ1uzCQikuZkaERDIyKGt2QQE0txMjQgIZOTQ1mwCAmlupkYEBDJyaGs2AYE0N1MjAgIZObQ1m4BAmpupEQGBjBzamk1AIM3N1IiAQEYObc0mIJDmZmpEQCAjh7ZmExBIczM1IiCQkUNbswkIpLmZGhEQyMihrdkEBNLcTI0ICGTk0NZsAgJpbqZGBAQycmhrNgGBNDdTIwICGTm0NZuAQJqbqREBgYwc2ppNQCDNzdSIgEBGDm3NJiCQ5mZqREAgI4e2ZhMQSHMzNSIgkJFDW7MJCKS5mRoREMjIoa3ZBATS3EyNCAhk5NDWbAICaW6mRgQEMnJoazYBgTQ3UyMCAhk5tDWbgECam6kRAYGMHNqaTUAgzc3UiIBARg5tzSYgkOZmakRAICOHtmYTEEhzMzUiIJCRQ1uzCQikuZkaERDIyKGt2QQE0txMjQgIZOTQ1mwCAmlupkYEBDJyaGs2AYE0N1MjAgIZObQ1m4BAmpupEQGBjBzamk1AIM3N1IiAQEYObc0mIJDmZmpE4Af1gABlH0hlGgAAAABJRU5ErkJggg==\\" + \\"style\\": \\"border: 1px solid #000000\\" }, \\"childNodes\\": [ { From 6910276d64cb96519301b8e46f439894a6a1413a Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Mon, 6 Dec 2021 16:30:22 +0100 Subject: [PATCH 28/93] reference original file not type defintion file --- packages/rrweb/src/replay/canvas/2d.ts | 2 +- packages/rrweb/src/replay/canvas/webgl.ts | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/rrweb/src/replay/canvas/2d.ts b/packages/rrweb/src/replay/canvas/2d.ts index 44b4f651d2..7daa7e4224 100644 --- a/packages/rrweb/src/replay/canvas/2d.ts +++ b/packages/rrweb/src/replay/canvas/2d.ts @@ -1,4 +1,4 @@ -import { Replayer } from '../../../typings/entries/all'; +import { Replayer } from '../'; import { canvasMutationData } from '../../types'; export default function canvasMutation({ diff --git a/packages/rrweb/src/replay/canvas/webgl.ts b/packages/rrweb/src/replay/canvas/webgl.ts index 0f3dfee778..e28693de6d 100644 --- a/packages/rrweb/src/replay/canvas/webgl.ts +++ b/packages/rrweb/src/replay/canvas/webgl.ts @@ -1,5 +1,5 @@ import { decode } from 'base64-arraybuffer'; -import { Replayer } from '../../../typings/entries/all'; +import { Replayer } from '../'; import { CanvasContext, canvasMutationData, @@ -96,7 +96,6 @@ export default function webglMutation({ const ctx = getContext(target, mutation.type); if (!ctx) return; - if (mutation.setter) { // skip some read-only type checks // tslint:disable-next-line:no-any From 7463e30031fd6c0a2dd555271607534429b1024c Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Mon, 6 Dec 2021 16:41:00 +0100 Subject: [PATCH 29/93] update types --- packages/rrweb/typings/replay/canvas/2d.d.ts | 2 +- packages/rrweb/typings/replay/canvas/webgl.d.ts | 2 +- packages/rrweb/typings/types.d.ts | 16 +++++++++++++++- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/packages/rrweb/typings/replay/canvas/2d.d.ts b/packages/rrweb/typings/replay/canvas/2d.d.ts index 85942bf827..9f05cc2ef9 100644 --- a/packages/rrweb/typings/replay/canvas/2d.d.ts +++ b/packages/rrweb/typings/replay/canvas/2d.d.ts @@ -1,4 +1,4 @@ -import { Replayer } from '../../../typings/entries/all'; +import { Replayer } from '../'; import { canvasMutationData } from '../../types'; export default function canvasMutation({ event, mutation, target, imageMap, errorHandler, }: { event: Parameters[0]; diff --git a/packages/rrweb/typings/replay/canvas/webgl.d.ts b/packages/rrweb/typings/replay/canvas/webgl.d.ts index 3e87fe9565..f1cf4b587b 100644 --- a/packages/rrweb/typings/replay/canvas/webgl.d.ts +++ b/packages/rrweb/typings/replay/canvas/webgl.d.ts @@ -1,4 +1,4 @@ -import { Replayer } from '../../../typings/entries/all'; +import { Replayer } from '../'; import { canvasMutationData, SerializedWebGlArg } from '../../types'; export declare function variableListFor(ctor: string): any[]; export declare function deserializeArg(arg: SerializedWebGlArg): any; diff --git a/packages/rrweb/typings/types.d.ts b/packages/rrweb/typings/types.d.ts index 59bd91a472..44eaa68e26 100644 --- a/packages/rrweb/typings/types.d.ts +++ b/packages/rrweb/typings/types.d.ts @@ -282,8 +282,22 @@ export declare enum MouseInteractions { } export declare enum CanvasContext { '2D' = 0, - WebGL = 1 + WebGL = 1, + WebGL2 = 2 } +export declare type SerializedWebGlArg = { + rr_type: 'ArrayBuffer'; + base64: string; +} | { + rr_type: string; + src: string; +} | { + rr_type: string; + args: Array; +} | { + rr_type: string; + index: number; +} | string | number | boolean | null | SerializedWebGlArg[]; declare type mouseInteractionParam = { type: MouseInteractions; id: number; From 76b8efb9889143a8ff0a88a080de9414550057da Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Mon, 6 Dec 2021 16:51:31 +0100 Subject: [PATCH 30/93] rename code worspace --- .gitignore | 2 +- .vscode/monorepo.code-workspace | 27 ---------------------- .vscode/rrweb-monorepo.code-workspace | 32 +++++++++++++++++++++++++++ 3 files changed, 33 insertions(+), 28 deletions(-) delete mode 100644 .vscode/monorepo.code-workspace create mode 100644 .vscode/rrweb-monorepo.code-workspace diff --git a/.gitignore b/.gitignore index c70cf42b7f..895e1115aa 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ .vscode/* -!/.vscode/monorepo.code-workspace +!/.vscode/rrweb-monorepo.code-workspace .idea node_modules package-lock.json diff --git a/.vscode/monorepo.code-workspace b/.vscode/monorepo.code-workspace deleted file mode 100644 index e0895cc7ff..0000000000 --- a/.vscode/monorepo.code-workspace +++ /dev/null @@ -1,27 +0,0 @@ -{ - "folders": [ - { - "name": "_RRWeb Monorepo", - "path": ".." - }, - { - "path": "../packages/rrdom" - }, - { - "path": "../packages/rrweb" - }, - { - "path": "../packages/rrweb-player" - }, - { - "path": "../packages/rrweb-snapshot" - } - ], - "settings": { - "jest.disabledWorkspaceFolders": [ - "_RRWeb Monorepo", - "rrweb-player", - "rrdom" - ] - } -} diff --git a/.vscode/rrweb-monorepo.code-workspace b/.vscode/rrweb-monorepo.code-workspace new file mode 100644 index 0000000000..37d44396ae --- /dev/null +++ b/.vscode/rrweb-monorepo.code-workspace @@ -0,0 +1,32 @@ +{ + "folders": [ + { + "name": " rrweb monorepo", // added a space to bump it to the top + "path": ".." + }, + { + "name": "rrdom (package)", + "path": "../packages/rrdom" + }, + { + "name": "rrweb (package)", + "path": "../packages/rrweb" + }, + { + "name": "rrweb-player (package)", + "path": "../packages/rrweb-player" + }, + { + "name": "rrweb-snapshot (package)", + "path": "../packages/rrweb-snapshot" + } + ], + "settings": { + "jest.disabledWorkspaceFolders": [ + " rrweb monorepo", + "rrweb-player (package)", + "rrdom (package)" + ], + "liveServer.settings.multiRootWorkspaceName": "rrweb" + } +} From e2cfdbb0c229c8a3f7bc07dffd615af82cb3bef5 Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Mon, 6 Dec 2021 16:57:49 +0100 Subject: [PATCH 31/93] update dependencies --- yarn.lock | 135 +++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 109 insertions(+), 26 deletions(-) diff --git a/yarn.lock b/yarn.lock index 38976cfc8d..634560a808 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2846,9 +2846,9 @@ color-name@^1.0.0, color-name@~1.1.4: integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA== color-string@^1.6.0: - version "1.6.0" - resolved "https://registry.yarnpkg.com/color-string/-/color-string-1.6.0.tgz#c3915f61fe267672cb7e1e064c9d692219f6c312" - integrity sha512-c/hGS+kRWJutUBEngKKmk4iH3sD59MBkoxVapS/0wgpCz2u7XsNloxknyvBhzwEs1IbV36D9PwqLPJ2DTu3vMA== + version "1.8.2" + resolved "https://registry.yarnpkg.com/color-string/-/color-string-1.8.2.tgz#08bd49fa5f3889c27b0c670052ed746dd7a671de" + integrity sha512-w5ZkKRdLsc5NOYsmnpS2DpyRW71npwZGwbRpLrJTuqjfTs2Bhrba7UiV59IX9siBlCPl2pne5NtiwnVWUzvYFA== dependencies: color-name "^1.0.0" simple-swizzle "^0.2.2" @@ -3719,7 +3719,33 @@ error-ex@^1.3.1: dependencies: is-arrayish "^0.2.1" -es-abstract@^1.17.2, es-abstract@^1.18.0-next.2, es-abstract@^1.18.2: +es-abstract@^1.17.2, es-abstract@^1.19.1: + version "1.19.1" + resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.19.1.tgz#d4885796876916959de78edaa0df456627115ec3" + integrity sha512-2vJ6tjA/UfqLm2MPs7jxVybLoB8i1t1Jd9R3kISld20sIxPcTbLuggQOUxeWeAvIUkduv/CfMjuh4WmiXr2v9w== + dependencies: + call-bind "^1.0.2" + es-to-primitive "^1.2.1" + function-bind "^1.1.1" + get-intrinsic "^1.1.1" + get-symbol-description "^1.0.0" + has "^1.0.3" + has-symbols "^1.0.2" + internal-slot "^1.0.3" + is-callable "^1.2.4" + is-negative-zero "^2.0.1" + is-regex "^1.1.4" + is-shared-array-buffer "^1.0.1" + is-string "^1.0.7" + is-weakref "^1.0.1" + object-inspect "^1.11.0" + object-keys "^1.1.1" + object.assign "^4.1.2" + string.prototype.trimend "^1.0.4" + string.prototype.trimstart "^1.0.4" + unbox-primitive "^1.0.1" + +es-abstract@^1.18.0-next.2: version "1.18.3" resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.18.3.tgz#25c4c3380a27aa203c44b2b685bba94da31b63e0" integrity sha512-nQIr12dxV7SSxE6r6f1l3DtAeEYdsGpps13dR0TwJg1S8gyp4ZPgy3FZcHBgbiQqnoqSTb+oC+kO4UQ0C/J8vw== @@ -4415,7 +4441,7 @@ get-caller-file@^2.0.5: resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-2.0.5.tgz#4f94412a82db32f36e3b0b9741f8a97feb031f7e" integrity sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg== -get-intrinsic@^1.0.2, get-intrinsic@^1.1.1: +get-intrinsic@^1.0.2, get-intrinsic@^1.1.0, get-intrinsic@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.1.1.tgz#15f59f376f855c446963948f0d24cd3637b4abc6" integrity sha512-kWZrnVM42QCiEA2Ig1bG8zjoIMOgxWwYCEeNdwY6Tv/cOSeGpcoX4pXHfKUxNKVoArnrEr2e9srnAxxGIraS9Q== @@ -4471,6 +4497,14 @@ get-stream@^6.0.0: resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-6.0.1.tgz#a262d8eef67aced57c2852ad6167526a43cbf7b7" integrity sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg== +get-symbol-description@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/get-symbol-description/-/get-symbol-description-1.0.0.tgz#7fdb81c900101fbd564dd5f1a30af5aadc1e58d6" + integrity sha512-2EmdH1YvIQiZpltCNgkuiUnyukzxM/R6NDJX31Ke3BG1Nq5b0S2PhX59UKi9vZpPDQVdqn+1IcaAwnzTT5vCjw== + dependencies: + call-bind "^1.0.2" + get-intrinsic "^1.1.1" + getpass@^0.1.1: version "0.1.7" resolved "https://registry.yarnpkg.com/getpass/-/getpass-0.1.7.tgz#5eff8e3e684d569ae4cb2b1282604e8ba62149fa" @@ -4708,6 +4742,13 @@ has-symbols@^1.0.1, has-symbols@^1.0.2: resolved "https://registry.yarnpkg.com/has-symbols/-/has-symbols-1.0.2.tgz#165d3070c00309752a1236a479331e3ac56f1423" integrity sha512-chXa79rL/UC2KlX17jo3vRGz0azaWEx5tGqZg5pO3NUyEJVB17dMruQlzCCOfUvElghKcm5194+BCRvi2Rv/Gw== +has-tostringtag@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/has-tostringtag/-/has-tostringtag-1.0.0.tgz#7e133818a7d394734f941e73c3d3f9291e658b25" + integrity sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ== + dependencies: + has-symbols "^1.0.2" + has-unicode@^2.0.0, has-unicode@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/has-unicode/-/has-unicode-2.0.1.tgz#e0e6fe6a28cf51138855e086d1691e771de2a8b9" @@ -5040,6 +5081,15 @@ inquirer@^7.3.3: strip-ansi "^6.0.0" through "^2.3.6" +internal-slot@^1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/internal-slot/-/internal-slot-1.0.3.tgz#7347e307deeea2faac2ac6205d4bc7d34967f59c" + integrity sha512-O0DB1JC/sPyZl7cIo78n5dR7eUSwwpYPiXRhTzNxZVAMUuB8vlnRFyLxdrVToks6XPLVnFfbzaVd5WLjhgg+vA== + dependencies: + get-intrinsic "^1.1.0" + has "^1.0.3" + side-channel "^1.0.4" + ip@^1.1.5: version "1.1.5" resolved "https://registry.yarnpkg.com/ip/-/ip-1.1.5.tgz#bdded70114290828c0a039e72ef25f5aaec4354a" @@ -5094,6 +5144,11 @@ is-callable@^1.1.3, is-callable@^1.1.4, is-callable@^1.2.3: resolved "https://registry.yarnpkg.com/is-callable/-/is-callable-1.2.3.tgz#8b1e0500b73a1d76c70487636f368e519de8db8e" integrity sha512-J1DcMe8UYTBSrKezuIUTUwjXsho29693unXM2YhJUTR2txK/eG47bvNa/wipPFmZFgr/N6f1GA66dv0mEyTIyQ== +is-callable@^1.2.4: + version "1.2.4" + resolved "https://registry.yarnpkg.com/is-callable/-/is-callable-1.2.4.tgz#47301d58dd0259407865547853df6d61fe471945" + integrity sha512-nsuwtxZfMX67Oryl9LCQ+upnC0Z0BgpwntpS89m1H/TLF0zNfzfLMV/9Wa/6MZsj0acpEjAO0KF1xT6ZdLl95w== + is-ci@^1.0.10: version "1.2.1" resolved "https://registry.yarnpkg.com/is-ci/-/is-ci-1.2.1.tgz#e3779c8ee17fccf428488f6e281187f2e632841c" @@ -5331,6 +5386,14 @@ is-regex@^1.1.3: call-bind "^1.0.2" has-symbols "^1.0.2" +is-regex@^1.1.4: + version "1.1.4" + resolved "https://registry.yarnpkg.com/is-regex/-/is-regex-1.1.4.tgz#eef5663cd59fa4c0ae339505323df6854bb15958" + integrity sha512-kvRdxDsxZjhzUX07ZnLydzS1TU/TJlTUHHY4YLL87e37oUA49DfkLqgy+VjFocowy29cKvcSiu+kIv728jTTVg== + dependencies: + call-bind "^1.0.2" + has-tostringtag "^1.0.0" + is-resolvable@^1.0.0: version "1.1.0" resolved "https://registry.yarnpkg.com/is-resolvable/-/is-resolvable-1.1.0.tgz#fb18f87ce1feb925169c9a407c19318a3206ed88" @@ -5341,6 +5404,11 @@ is-retry-allowed@^1.0.0: resolved "https://registry.yarnpkg.com/is-retry-allowed/-/is-retry-allowed-1.2.0.tgz#d778488bd0a4666a3be8a1482b9f2baafedea8b4" integrity sha512-RUbUeKwvm3XG2VYamhJL1xFktgjvPzL0Hq8C+6yrWIswDy3BIXGqCxhxkc30N9jqK311gVU137K8Ei55/zVJRg== +is-shared-array-buffer@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/is-shared-array-buffer/-/is-shared-array-buffer-1.0.1.tgz#97b0c85fbdacb59c9c446fe653b82cf2b5b7cfe6" + integrity sha512-IU0NmyknYZN0rChcKhRO1X8LYz5Isj/Fsqh8NJOSf+N/hCOTwy29F32Ik7a+QszE63IdvmwdTPDd6cZ5pg4cwA== + is-ssh@^1.3.0: version "1.3.3" resolved "https://registry.yarnpkg.com/is-ssh/-/is-ssh-1.3.3.tgz#7f133285ccd7f2c2c7fc897b771b53d95a2b2c7e" @@ -5363,6 +5431,13 @@ is-string@^1.0.5, is-string@^1.0.6: resolved "https://registry.yarnpkg.com/is-string/-/is-string-1.0.6.tgz#3fe5d5992fb0d93404f32584d4b0179a71b54a5f" integrity sha512-2gdzbKUuqtQ3lYNrUTQYoClPhm7oQu4UdpSZMp1/DGgkHBT8E2Z1l0yMdb6D4zNAxwDiMv8MdulKROJGNl0Q0w== +is-string@^1.0.7: + version "1.0.7" + resolved "https://registry.yarnpkg.com/is-string/-/is-string-1.0.7.tgz#0dd12bf2006f255bb58f695110eff7491eebc0fd" + integrity sha512-tE2UXzivje6ofPW7l23cjDOMa09gb7xlAqG6jG5ej6uPV32TlWP3NKPigtaGeHNu9fohccRYvIiZMfOOnOYUtg== + dependencies: + has-tostringtag "^1.0.0" + is-symbol@^1.0.2, is-symbol@^1.0.3: version "1.0.4" resolved "https://registry.yarnpkg.com/is-symbol/-/is-symbol-1.0.4.tgz#a6dac93b635b063ca6872236de88910a57af139c" @@ -5382,6 +5457,13 @@ is-typedarray@^1.0.0, is-typedarray@~1.0.0: resolved "https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a" integrity sha1-5HnICFjfDBsR3dppQPlgEfzaSpo= +is-weakref@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/is-weakref/-/is-weakref-1.0.1.tgz#842dba4ec17fa9ac9850df2d6efbc1737274f2a2" + integrity sha512-b2jKc2pQZjaeFYWEf7ScFj+Be1I+PXmlu572Q8coTXZ+LD/QQZ7ShPMst8h16riVgyXTQwUsFEl74mDvc/3MHQ== + dependencies: + call-bind "^1.0.0" + isarray@1.0.0, isarray@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" @@ -7106,7 +7188,7 @@ object-assign@^4.0.1, object-assign@^4.1.0: resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" integrity sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM= -object-inspect@^1.10.3, object-inspect@^1.9.0: +object-inspect@^1.10.3, object-inspect@^1.11.0, object-inspect@^1.9.0: version "1.11.0" resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.11.0.tgz#9dceb146cedd4148a0d9e51ab88d34cf509922b1" integrity sha512-jp7ikS6Sd3GxQfZJPyH3cjcbJF6GZPClgdV+EFygjFLQ5FmW/dRUnTd9PQ9k0JhoNDabWFbpF1yCdSWCC6gexg== @@ -7126,7 +7208,7 @@ object.assign@^4.1.2: has-symbols "^1.0.1" object-keys "^1.1.1" -object.getownpropertydescriptors@^2.0.3, object.getownpropertydescriptors@^2.1.0, object.getownpropertydescriptors@^2.1.1: +object.getownpropertydescriptors@^2.0.3, object.getownpropertydescriptors@^2.1.1: version "2.1.2" resolved "https://registry.yarnpkg.com/object.getownpropertydescriptors/-/object.getownpropertydescriptors-2.1.2.tgz#1bd63aeacf0d5d2d2f31b5e393b03a7c601a23f7" integrity sha512-WtxeKSzfBjlzL+F9b7M7hewDzMwy+C8NRssHd1YrNlzHzIDrXcXiNOMrezdAEM4UXixgV+vvnyBeN7Rygl2ttQ== @@ -7135,6 +7217,15 @@ object.getownpropertydescriptors@^2.0.3, object.getownpropertydescriptors@^2.1.0 define-properties "^1.1.3" es-abstract "^1.18.0-next.2" +object.getownpropertydescriptors@^2.1.0: + version "2.1.3" + resolved "https://registry.yarnpkg.com/object.getownpropertydescriptors/-/object.getownpropertydescriptors-2.1.3.tgz#b223cf38e17fefb97a63c10c91df72ccb386df9e" + integrity sha512-VdDoCwvJI4QdC6ndjpqFmoL3/+HxffFBbcJzKi5hwLLqqx3mdbedRpfZDdK0SrOSauj8X4GzBvnDZl4vTN7dOw== + dependencies: + call-bind "^1.0.2" + define-properties "^1.1.3" + es-abstract "^1.19.1" + object.omit@^2.0.0: version "2.0.1" resolved "https://registry.yarnpkg.com/object.omit/-/object.omit-2.0.1.tgz#1a9c744829f39dbb858c76ca3579ae2a54ebd1fa" @@ -7144,13 +7235,13 @@ object.omit@^2.0.0: is-extendable "^0.1.1" object.values@^1.1.0: - version "1.1.4" - resolved "https://registry.yarnpkg.com/object.values/-/object.values-1.1.4.tgz#0d273762833e816b693a637d30073e7051535b30" - integrity sha512-TnGo7j4XSnKQoK3MfvkzqKCi0nVe/D9I9IjwTNYdb/fxYHpjrluHVOgw0AF6jrRFGMPHdfuidR09tIDiIvnaSg== + version "1.1.5" + resolved "https://registry.yarnpkg.com/object.values/-/object.values-1.1.5.tgz#959f63e3ce9ef108720333082131e4a459b716ac" + integrity sha512-QUZRW0ilQ3PnPpbNtgdNV1PDbEqLIiSFB3l+EnGtBQ/8SUTLj1PZwtQHABZtLgwpJZTSZhuGLOGk57Drx2IvYg== dependencies: call-bind "^1.0.2" define-properties "^1.1.3" - es-abstract "^1.18.2" + es-abstract "^1.19.1" on-finished@~2.3.0: version "2.3.0" @@ -7931,13 +8022,12 @@ postcss@^6.0.1, postcss@^6.0.11: supports-color "^5.4.0" postcss@^7.0.0, postcss@^7.0.1, postcss@^7.0.27: - version "7.0.36" - resolved "https://registry.yarnpkg.com/postcss/-/postcss-7.0.36.tgz#056f8cffa939662a8f5905950c07d5285644dfcb" - integrity sha512-BebJSIUMwJHRH0HAQoxN4u1CN86glsrwsW0q7T+/m44eXOUAxSNdHRkNZPYz5vVUbg17hFgOQDE7fZk7li3pZw== + version "7.0.39" + resolved "https://registry.yarnpkg.com/postcss/-/postcss-7.0.39.tgz#9624375d965630e2e1f2c02a935c82a59cb48309" + integrity sha512-yioayjNbHn6z1/Bywyb2Y4s3yvDAeXGOyxqD+LnVOinq6Mdmd++SW2wUNVzavyyHxd6+DxzWGIuosg6P1Rj8uA== dependencies: - chalk "^2.4.2" + picocolors "^0.2.1" source-map "^0.6.1" - supports-color "^6.1.0" prelude-ls@^1.2.1: version "1.2.1" @@ -8545,9 +8635,9 @@ rollup-plugin-postcss@^3.1.1: style-inject "^0.3.0" rollup-plugin-rename-node-modules@^1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/rollup-plugin-rename-node-modules/-/rollup-plugin-rename-node-modules-1.1.0.tgz#c73de5fed61b997857993813a7053285e2cca2dd" - integrity sha512-JpfsJ7NYI/4OdqWvZ/BY6fgjZb5j7sRFvHMv8EU0zrFiNUcW4ke9tw7WXImsHnjq7Bp3xv+UILRPpA7plOa38Q== + version "1.2.0" + resolved "https://registry.yarnpkg.com/rollup-plugin-rename-node-modules/-/rollup-plugin-rename-node-modules-1.2.0.tgz#f1f1bb2192d1bbec258569bf6bda097002d7dbdf" + integrity sha512-IKsS3eJXHLAMXIndzNso9ijWJw1V3mqubRc2gb67v7VuLX9t41LObXqciM0JC3j7/WrHeptG47cejFU0qxXUJA== dependencies: estree-walker "^2.0.1" magic-string "^0.25.7" @@ -9167,13 +9257,6 @@ supports-color@^5.3.0, supports-color@^5.4.0: dependencies: has-flag "^3.0.0" -supports-color@^6.1.0: - version "6.1.0" - resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-6.1.0.tgz#0764abc69c63d5ac842dd4867e8d025e880df8f3" - integrity sha512-qe1jfm1Mg7Nq/NSh6XE24gPXROEVsWHxC1LIx//XNlD9iw7YZQGjZNjYN7xGaEG6iKdA8EtNFW6R0gjnVXp+wQ== - dependencies: - has-flag "^3.0.0" - supports-color@^7.0.0, supports-color@^7.1.0: version "7.2.0" resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-7.2.0.tgz#1b7dcdcb32b8138801b3e478ba6a51caa89648da" From 5d14caad8fcb55a3e754eaee02108f0110635849 Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Mon, 6 Dec 2021 18:12:04 +0100 Subject: [PATCH 32/93] add spector to inspect webgl --- packages/rrweb/package.json | 1 + yarn.lock | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/packages/rrweb/package.json b/packages/rrweb/package.json index 2638eeb08f..00634a6015 100644 --- a/packages/rrweb/package.json +++ b/packages/rrweb/package.json @@ -66,6 +66,7 @@ "rollup-plugin-postcss": "^3.1.1", "rollup-plugin-rename-node-modules": "^1.1.0", "rollup-plugin-terser": "^7.0.2", + "spector": "^0.0.0", "ts-jest": "^27.0.5", "ts-node": "^7.0.1", "tslib": "^1.9.3", diff --git a/yarn.lock b/yarn.lock index 634560a808..bda38821f2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2757,6 +2757,13 @@ cli-cursor@^3.1.0: dependencies: restore-cursor "^3.1.0" +cli-table@*: + version "0.3.11" + resolved "https://registry.yarnpkg.com/cli-table/-/cli-table-0.3.11.tgz#ac69cdecbe81dccdba4889b9a18b7da312a9d3ee" + integrity sha512-IqLQi4lO0nIB4tcdTpN4LCB9FI3uqrJZK7RC515EnhZ6qBaglkIgICb1wjeAqpdoOabm1+SuQtkXIPdYC93jhQ== + dependencies: + colors "1.0.3" + cli-width@^2.0.0: version "2.2.1" resolved "https://registry.yarnpkg.com/cli-width/-/cli-width-2.2.1.tgz#b0433d0b4e9c847ef18868a4ef16fd5fc8271c48" @@ -2866,6 +2873,11 @@ colorette@^1.2.2: resolved "https://registry.yarnpkg.com/colorette/-/colorette-1.2.2.tgz#cbcc79d5e99caea2dbf10eb3a26fd8b3e6acfa94" integrity sha512-MKGMzyfeuutC/ZJ1cba9NqcNpfeqMUcYmyF1ZFY6/Cn7CNSAKx6a+s48sqLqyAiZuaP2TcqMhoo+dlwFnVxT9w== +colors@1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/colors/-/colors-1.0.3.tgz#0433f44d809680fdeb60ed260f1b0c262e82a40b" + integrity sha1-BDP0TYCWgP3rYO0mDxsMJi6CpAs= + colors@^1.1.2: version "1.4.0" resolved "https://registry.yarnpkg.com/colors/-/colors-1.4.0.tgz#c50491479d4c1bdaed2c9ced32cf7c7dc2360f78" @@ -8994,6 +9006,13 @@ spdx-license-ids@^3.0.0: resolved "https://registry.yarnpkg.com/spdx-license-ids/-/spdx-license-ids-3.0.10.tgz#0d9becccde7003d6c658d487dd48a32f0bf3014b" integrity sha512-oie3/+gKf7QtpitB0LYLETe+k8SifzsX4KixvpOsbI6S0kRiRQ5MKOio8eMSAKQ17N06+wdEOXRiId+zOxo0hA== +spector@^0.0.0: + version "0.0.0" + resolved "https://registry.yarnpkg.com/spector/-/spector-0.0.0.tgz#3be8197aa70416d76940ea17f74457871fc7b900" + integrity sha1-O+gZeqcEFtdpQOoX90RXhx/HuQA= + dependencies: + cli-table "*" + split-on-first@^1.0.0: version "1.1.0" resolved "https://registry.yarnpkg.com/split-on-first/-/split-on-first-1.1.0.tgz#f610afeee3b12bce1d0c30425e76398b78249a5f" From fabb033d95ea83e0de9929a96f2bd4336e5a9c41 Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Mon, 6 Dec 2021 18:12:30 +0100 Subject: [PATCH 33/93] remove live server settings from code workspace --- .vscode/rrweb-monorepo.code-workspace | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.vscode/rrweb-monorepo.code-workspace b/.vscode/rrweb-monorepo.code-workspace index 37d44396ae..79849c8fe9 100644 --- a/.vscode/rrweb-monorepo.code-workspace +++ b/.vscode/rrweb-monorepo.code-workspace @@ -26,7 +26,6 @@ " rrweb monorepo", "rrweb-player (package)", "rrdom (package)" - ], - "liveServer.settings.multiRootWorkspaceName": "rrweb" + ] } } From 2641e29c7e1bd8df027647a5616020da336bd092 Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Wed, 8 Dec 2021 12:09:16 +0100 Subject: [PATCH 34/93] Save canvas context in the node Prevents from saving webgl canvases as 2d dataUrls --- packages/rrweb-snapshot/src/snapshot.ts | 7 +++- packages/rrweb-snapshot/src/types.ts | 4 ++ packages/rrweb/src/record/observer.ts | 3 ++ .../src/record/observers/canvas/canvas.ts | 38 +++++++++++++++++++ packages/rrweb/test/record/webgl.test.ts | 37 ++++++++++++++++++ 5 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 packages/rrweb/src/record/observers/canvas/canvas.ts diff --git a/packages/rrweb-snapshot/src/snapshot.ts b/packages/rrweb-snapshot/src/snapshot.ts index 9d85f01d5c..4a34ed0eae 100644 --- a/packages/rrweb-snapshot/src/snapshot.ts +++ b/packages/rrweb-snapshot/src/snapshot.ts @@ -10,6 +10,7 @@ import { MaskTextFn, MaskInputFn, KeepIframeSrcFn, + ICanvas, documentNode, } from './types'; import { isElement, isShadowRoot, maskInputValue } from './utils'; @@ -493,7 +494,11 @@ function serializeNode( } } // canvas image data - if (tagName === 'canvas' && recordCanvas) { + if ( + tagName === 'canvas' && + recordCanvas && + (!('__context' in n) || (n as ICanvas).__context === '2d') // only record this on 2d canvas + ) { const canvasDataURL = (n as HTMLCanvasElement).toDataURL(); // create blank canvas of same dimensions diff --git a/packages/rrweb-snapshot/src/types.ts b/packages/rrweb-snapshot/src/types.ts index 8720783ee5..f239213d32 100644 --- a/packages/rrweb-snapshot/src/types.ts +++ b/packages/rrweb-snapshot/src/types.ts @@ -71,6 +71,10 @@ export interface INode extends Node { __sn: serializedNodeWithId; } +export interface ICanvas extends HTMLCanvasElement { + __context: string; +} + export type idNodeMap = { [key: number]: INode; }; diff --git a/packages/rrweb/src/record/observer.ts b/packages/rrweb/src/record/observer.ts index 3efc97290d..6ea1940a8f 100644 --- a/packages/rrweb/src/record/observer.ts +++ b/packages/rrweb/src/record/observer.ts @@ -49,6 +49,7 @@ import { import MutationBuffer from './mutation'; import { IframeManager } from './iframe-manager'; import { ShadowDomManager } from './shadow-dom-manager'; +import initCanvasContextObserver from './observers/canvas/canvas'; import initCanvas2DMutationObserver from './observers/canvas/2d'; import initCanvasWebGLMutationObserver from './observers/canvas/webgl'; import initCanvasWebGL2MutationObserver from './observers/canvas/webgl2'; @@ -715,6 +716,7 @@ function initCanvasMutationObserver( blockClass: blockClass, mirror: Mirror, ): listenerHandler { + const canvasContextReset = initCanvasContextObserver(win, blockClass); const canvas2DReset = initCanvas2DMutationObserver( cb, win, @@ -737,6 +739,7 @@ function initCanvasMutationObserver( ); return () => { + canvasContextReset(); canvas2DReset(); canvasWebGLReset(); canvasWebGL2Reset(); diff --git a/packages/rrweb/src/record/observers/canvas/canvas.ts b/packages/rrweb/src/record/observers/canvas/canvas.ts new file mode 100644 index 0000000000..8808c8acb6 --- /dev/null +++ b/packages/rrweb/src/record/observers/canvas/canvas.ts @@ -0,0 +1,38 @@ +import { INode } from 'rrweb-snapshot'; +import { blockClass, IWindow, listenerHandler } from '../../../types'; +import { isBlocked, patch } from '../../../utils'; + +// TODO: replace me for ICanvas from rrweb-snapshot +export type OgmentedCanvas = HTMLCanvasElement & { __context: string }; + +export default function initCanvasContextObserver( + win: IWindow, + blockClass: blockClass, +): listenerHandler { + const handlers: listenerHandler[] = []; + try { + const restoreHandler = patch( + win.HTMLCanvasElement.prototype, + 'getContext', + function (original) { + return function ( + this: OgmentedCanvas, + contextType: string, + ...args: Array + ) { + if (!isBlocked((this as unknown) as INode, blockClass)) { + if (!('__context' in this)) + (this as OgmentedCanvas).__context = contextType; + } + return original.apply(this, [contextType, ...args]); + }; + }, + ); + handlers.push(restoreHandler); + } catch { + console.error('failed to patch HTMLCanvasElement.prototype.getContext'); + } + return () => { + handlers.forEach((h) => h()); + }; +} diff --git a/packages/rrweb/test/record/webgl.test.ts b/packages/rrweb/test/record/webgl.test.ts index be55835009..8e9d6b5903 100644 --- a/packages/rrweb/test/record/webgl.test.ts +++ b/packages/rrweb/test/record/webgl.test.ts @@ -13,6 +13,7 @@ import { CanvasContext, } from '../../src/types'; import { assertSnapshot, launchPuppeteer } from '../utils'; +import { OgmentedCanvas } from '../../src/record/observers/canvas/canvas'; interface ISuite { code: string; @@ -177,4 +178,40 @@ describe('record webgl', function (this: ISuite) { }, }); }); + + it('sets _context on canvas.getContext()', async () => { + await ctx.page.evaluate(() => { + const { record } = ((window as unknown) as IWindow).rrweb; + record({ + recordCanvas: true, + emit: ((window as unknown) as IWindow).emit, + }); + }); + const context = await ctx.page.evaluate(() => { + var canvas = document.getElementById('canvas') as HTMLCanvasElement; + canvas.getContext('webgl')!; + return (canvas as OgmentedCanvas).__context; + }); + + expect(context).toBe('webgl'); + }); + + it('only sets _context on first canvas.getContext() call', async () => { + await ctx.page.evaluate(() => { + const { record } = ((window as unknown) as IWindow).rrweb; + record({ + recordCanvas: true, + emit: ((window as unknown) as IWindow).emit, + }); + }); + const context = await ctx.page.evaluate(() => { + var canvas = document.getElementById('canvas') as HTMLCanvasElement; + canvas.getContext('webgl'); + canvas.getContext('2d'); // returns null + return (canvas as OgmentedCanvas).__context; + }); + + expect(context).toBe('webgl'); + }); +}); }); From bc7f2baf11b0941b34be06b85775884e3bd52308 Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Wed, 8 Dec 2021 12:46:22 +0100 Subject: [PATCH 35/93] remove extra braces --- packages/rrweb/test/record/webgl.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/rrweb/test/record/webgl.test.ts b/packages/rrweb/test/record/webgl.test.ts index 8e9d6b5903..4ac6daf3af 100644 --- a/packages/rrweb/test/record/webgl.test.ts +++ b/packages/rrweb/test/record/webgl.test.ts @@ -214,4 +214,3 @@ describe('record webgl', function (this: ISuite) { expect(context).toBe('webgl'); }); }); -}); From fbf67a0ee6158cbb988f9c7b8cd797d737af35e9 Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Wed, 8 Dec 2021 12:49:15 +0100 Subject: [PATCH 36/93] add ICanvas type --- packages/rrweb-snapshot/typings/types.d.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/rrweb-snapshot/typings/types.d.ts b/packages/rrweb-snapshot/typings/types.d.ts index 5831c7b5fa..262d3f9aef 100644 --- a/packages/rrweb-snapshot/typings/types.d.ts +++ b/packages/rrweb-snapshot/typings/types.d.ts @@ -55,6 +55,9 @@ export declare type tagMap = { export interface INode extends Node { __sn: serializedNodeWithId; } +export interface ICanvas extends HTMLCanvasElement { + __context: string; +} export declare type idNodeMap = { [key: number]: INode; }; From 63adef7bd47a271c2c88f80003774b6170b6397d Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Wed, 8 Dec 2021 12:53:03 +0100 Subject: [PATCH 37/93] use ICanvas from rrweb-snapshot in rrweb instead of OgmentedCanvas --- packages/rrweb/src/record/observers/canvas/canvas.ts | 9 +++------ packages/rrweb/test/record/webgl.test.ts | 6 +++--- .../rrweb/typings/record/observers/canvas/canvas.d.ts | 2 ++ 3 files changed, 8 insertions(+), 9 deletions(-) create mode 100644 packages/rrweb/typings/record/observers/canvas/canvas.d.ts diff --git a/packages/rrweb/src/record/observers/canvas/canvas.ts b/packages/rrweb/src/record/observers/canvas/canvas.ts index 8808c8acb6..437af7d5f3 100644 --- a/packages/rrweb/src/record/observers/canvas/canvas.ts +++ b/packages/rrweb/src/record/observers/canvas/canvas.ts @@ -1,10 +1,7 @@ -import { INode } from 'rrweb-snapshot'; +import { INode, ICanvas } from 'rrweb-snapshot'; import { blockClass, IWindow, listenerHandler } from '../../../types'; import { isBlocked, patch } from '../../../utils'; -// TODO: replace me for ICanvas from rrweb-snapshot -export type OgmentedCanvas = HTMLCanvasElement & { __context: string }; - export default function initCanvasContextObserver( win: IWindow, blockClass: blockClass, @@ -16,13 +13,13 @@ export default function initCanvasContextObserver( 'getContext', function (original) { return function ( - this: OgmentedCanvas, + this: ICanvas, contextType: string, ...args: Array ) { if (!isBlocked((this as unknown) as INode, blockClass)) { if (!('__context' in this)) - (this as OgmentedCanvas).__context = contextType; + (this as ICanvas).__context = contextType; } return original.apply(this, [contextType, ...args]); }; diff --git a/packages/rrweb/test/record/webgl.test.ts b/packages/rrweb/test/record/webgl.test.ts index 4ac6daf3af..8c46e5c69e 100644 --- a/packages/rrweb/test/record/webgl.test.ts +++ b/packages/rrweb/test/record/webgl.test.ts @@ -13,7 +13,7 @@ import { CanvasContext, } from '../../src/types'; import { assertSnapshot, launchPuppeteer } from '../utils'; -import { OgmentedCanvas } from '../../src/record/observers/canvas/canvas'; +import { ICanvas } from 'rrweb-snapshot'; interface ISuite { code: string; @@ -190,7 +190,7 @@ describe('record webgl', function (this: ISuite) { const context = await ctx.page.evaluate(() => { var canvas = document.getElementById('canvas') as HTMLCanvasElement; canvas.getContext('webgl')!; - return (canvas as OgmentedCanvas).__context; + return (canvas as ICanvas).__context; }); expect(context).toBe('webgl'); @@ -208,7 +208,7 @@ describe('record webgl', function (this: ISuite) { var canvas = document.getElementById('canvas') as HTMLCanvasElement; canvas.getContext('webgl'); canvas.getContext('2d'); // returns null - return (canvas as OgmentedCanvas).__context; + return (canvas as ICanvas).__context; }); expect(context).toBe('webgl'); diff --git a/packages/rrweb/typings/record/observers/canvas/canvas.d.ts b/packages/rrweb/typings/record/observers/canvas/canvas.d.ts new file mode 100644 index 0000000000..359d95928d --- /dev/null +++ b/packages/rrweb/typings/record/observers/canvas/canvas.d.ts @@ -0,0 +1,2 @@ +import { blockClass, IWindow, listenerHandler } from '../../../types'; +export default function initCanvasContextObserver(win: IWindow, blockClass: blockClass): listenerHandler; From d958d5e5917d92b4a94a6407d5aa320b99128e0b Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Wed, 8 Dec 2021 16:32:44 +0100 Subject: [PATCH 38/93] add snapshots and webgl 2 tests --- .../record/__snapshots__/webgl.test.ts.snap | 451 ++++++++++++++++++ packages/rrweb/test/record/webgl.test.ts | 64 ++- 2 files changed, 514 insertions(+), 1 deletion(-) create mode 100644 packages/rrweb/test/record/__snapshots__/webgl.test.ts.snap diff --git a/packages/rrweb/test/record/__snapshots__/webgl.test.ts.snap b/packages/rrweb/test/record/__snapshots__/webgl.test.ts.snap new file mode 100644 index 0000000000..5638ec5d7c --- /dev/null +++ b/packages/rrweb/test/record/__snapshots__/webgl.test.ts.snap @@ -0,0 +1,451 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`record webgl will record changes to a canvas element 1`] = ` +"[ + { + \\"type\\": 4, + \\"data\\": { + \\"href\\": \\"about:blank\\", + \\"width\\": 1920, + \\"height\\": 1080 + } + }, + { + \\"type\\": 2, + \\"data\\": { + \\"node\\": { + \\"type\\": 0, + \\"childNodes\\": [ + { + \\"type\\": 1, + \\"name\\": \\"html\\", + \\"publicId\\": \\"\\", + \\"systemId\\": \\"\\", + \\"id\\": 2 + }, + { + \\"type\\": 2, + \\"tagName\\": \\"html\\", + \\"attributes\\": {}, + \\"childNodes\\": [ + { + \\"type\\": 2, + \\"tagName\\": \\"head\\", + \\"attributes\\": {}, + \\"childNodes\\": [], + \\"id\\": 4 + }, + { + \\"type\\": 2, + \\"tagName\\": \\"body\\", + \\"attributes\\": {}, + \\"childNodes\\": [ + { + \\"type\\": 3, + \\"textContent\\": \\"\\\\n \\", + \\"id\\": 6 + }, + { + \\"type\\": 2, + \\"tagName\\": \\"canvas\\", + \\"attributes\\": { + \\"id\\": \\"canvas\\" + }, + \\"childNodes\\": [], + \\"id\\": 7 + }, + { + \\"type\\": 3, + \\"textContent\\": \\"\\\\n \\\\n \\\\n \\", + \\"id\\": 8 + } + ], + \\"id\\": 5 + } + ], + \\"id\\": 3 + } + ], + \\"id\\": 1 + }, + \\"initialOffset\\": { + \\"left\\": 0, + \\"top\\": 0 + } + } + }, + { + \\"type\\": 3, + \\"data\\": { + \\"source\\": 9, + \\"id\\": 7, + \\"type\\": 1, + \\"property\\": \\"clear\\", + \\"args\\": [ + 16384 + ] + } + } +]" +`; + +exports[`record webgl will record changes to a canvas element before the canvas gets added (webgl2) 1`] = ` +"[ + { + \\"type\\": 4, + \\"data\\": { + \\"href\\": \\"about:blank\\", + \\"width\\": 1920, + \\"height\\": 1080 + } + }, + { + \\"type\\": 2, + \\"data\\": { + \\"node\\": { + \\"type\\": 0, + \\"childNodes\\": [ + { + \\"type\\": 1, + \\"name\\": \\"html\\", + \\"publicId\\": \\"\\", + \\"systemId\\": \\"\\", + \\"id\\": 2 + }, + { + \\"type\\": 2, + \\"tagName\\": \\"html\\", + \\"attributes\\": {}, + \\"childNodes\\": [ + { + \\"type\\": 2, + \\"tagName\\": \\"head\\", + \\"attributes\\": {}, + \\"childNodes\\": [], + \\"id\\": 4 + }, + { + \\"type\\": 2, + \\"tagName\\": \\"body\\", + \\"attributes\\": {}, + \\"childNodes\\": [ + { + \\"type\\": 3, + \\"textContent\\": \\"\\\\n \\", + \\"id\\": 6 + }, + { + \\"type\\": 2, + \\"tagName\\": \\"canvas\\", + \\"attributes\\": { + \\"id\\": \\"canvas\\" + }, + \\"childNodes\\": [], + \\"id\\": 7 + }, + { + \\"type\\": 3, + \\"textContent\\": \\"\\\\n \\\\n \\\\n \\", + \\"id\\": 8 + } + ], + \\"id\\": 5 + } + ], + \\"id\\": 3 + } + ], + \\"id\\": 1 + }, + \\"initialOffset\\": { + \\"left\\": 0, + \\"top\\": 0 + } + } + }, + { + \\"type\\": 3, + \\"data\\": { + \\"source\\": 0, + \\"texts\\": [], + \\"attributes\\": [], + \\"removes\\": [], + \\"adds\\": [ + { + \\"parentId\\": 5, + \\"nextId\\": null, + \\"node\\": { + \\"type\\": 2, + \\"tagName\\": \\"canvas\\", + \\"attributes\\": {}, + \\"childNodes\\": [], + \\"id\\": 9 + } + } + ] + } + }, + { + \\"type\\": 3, + \\"data\\": { + \\"source\\": 9, + \\"id\\": 9, + \\"type\\": 2, + \\"property\\": \\"createProgram\\", + \\"args\\": [] + } + }, + { + \\"type\\": 3, + \\"data\\": { + \\"source\\": 9, + \\"id\\": 9, + \\"type\\": 2, + \\"property\\": \\"linkProgram\\", + \\"args\\": [ + { + \\"rr_type\\": \\"WebGLProgram\\", + \\"index\\": 0 + } + ] + } + }, + { + \\"type\\": 3, + \\"data\\": { + \\"source\\": 9, + \\"id\\": 9, + \\"type\\": 2, + \\"property\\": \\"clear\\", + \\"args\\": [ + 16384 + ] + } + } +]" +`; + +exports[`record webgl will record changes to a canvas element before the canvas gets added 1`] = ` +"[ + { + \\"type\\": 4, + \\"data\\": { + \\"href\\": \\"about:blank\\", + \\"width\\": 1920, + \\"height\\": 1080 + } + }, + { + \\"type\\": 2, + \\"data\\": { + \\"node\\": { + \\"type\\": 0, + \\"childNodes\\": [ + { + \\"type\\": 1, + \\"name\\": \\"html\\", + \\"publicId\\": \\"\\", + \\"systemId\\": \\"\\", + \\"id\\": 2 + }, + { + \\"type\\": 2, + \\"tagName\\": \\"html\\", + \\"attributes\\": {}, + \\"childNodes\\": [ + { + \\"type\\": 2, + \\"tagName\\": \\"head\\", + \\"attributes\\": {}, + \\"childNodes\\": [], + \\"id\\": 4 + }, + { + \\"type\\": 2, + \\"tagName\\": \\"body\\", + \\"attributes\\": {}, + \\"childNodes\\": [ + { + \\"type\\": 3, + \\"textContent\\": \\"\\\\n \\", + \\"id\\": 6 + }, + { + \\"type\\": 2, + \\"tagName\\": \\"canvas\\", + \\"attributes\\": { + \\"id\\": \\"canvas\\" + }, + \\"childNodes\\": [], + \\"id\\": 7 + }, + { + \\"type\\": 3, + \\"textContent\\": \\"\\\\n \\\\n \\\\n \\", + \\"id\\": 8 + } + ], + \\"id\\": 5 + } + ], + \\"id\\": 3 + } + ], + \\"id\\": 1 + }, + \\"initialOffset\\": { + \\"left\\": 0, + \\"top\\": 0 + } + } + }, + { + \\"type\\": 3, + \\"data\\": { + \\"source\\": 0, + \\"texts\\": [], + \\"attributes\\": [], + \\"removes\\": [], + \\"adds\\": [ + { + \\"parentId\\": 5, + \\"nextId\\": null, + \\"node\\": { + \\"type\\": 2, + \\"tagName\\": \\"canvas\\", + \\"attributes\\": {}, + \\"childNodes\\": [], + \\"id\\": 9 + } + } + ] + } + }, + { + \\"type\\": 3, + \\"data\\": { + \\"source\\": 9, + \\"id\\": 9, + \\"type\\": 1, + \\"property\\": \\"createProgram\\", + \\"args\\": [] + } + }, + { + \\"type\\": 3, + \\"data\\": { + \\"source\\": 9, + \\"id\\": 9, + \\"type\\": 1, + \\"property\\": \\"linkProgram\\", + \\"args\\": [ + { + \\"rr_type\\": \\"WebGLProgram\\", + \\"index\\": 0 + } + ] + } + }, + { + \\"type\\": 3, + \\"data\\": { + \\"source\\": 9, + \\"id\\": 9, + \\"type\\": 1, + \\"property\\": \\"clear\\", + \\"args\\": [ + 16384 + ] + } + } +]" +`; + +exports[`record webgl will record changes to a webgl2 canvas element 1`] = ` +"[ + { + \\"type\\": 4, + \\"data\\": { + \\"href\\": \\"about:blank\\", + \\"width\\": 1920, + \\"height\\": 1080 + } + }, + { + \\"type\\": 2, + \\"data\\": { + \\"node\\": { + \\"type\\": 0, + \\"childNodes\\": [ + { + \\"type\\": 1, + \\"name\\": \\"html\\", + \\"publicId\\": \\"\\", + \\"systemId\\": \\"\\", + \\"id\\": 2 + }, + { + \\"type\\": 2, + \\"tagName\\": \\"html\\", + \\"attributes\\": {}, + \\"childNodes\\": [ + { + \\"type\\": 2, + \\"tagName\\": \\"head\\", + \\"attributes\\": {}, + \\"childNodes\\": [], + \\"id\\": 4 + }, + { + \\"type\\": 2, + \\"tagName\\": \\"body\\", + \\"attributes\\": {}, + \\"childNodes\\": [ + { + \\"type\\": 3, + \\"textContent\\": \\"\\\\n \\", + \\"id\\": 6 + }, + { + \\"type\\": 2, + \\"tagName\\": \\"canvas\\", + \\"attributes\\": { + \\"id\\": \\"canvas\\" + }, + \\"childNodes\\": [], + \\"id\\": 7 + }, + { + \\"type\\": 3, + \\"textContent\\": \\"\\\\n \\\\n \\\\n \\", + \\"id\\": 8 + } + ], + \\"id\\": 5 + } + ], + \\"id\\": 3 + } + ], + \\"id\\": 1 + }, + \\"initialOffset\\": { + \\"left\\": 0, + \\"top\\": 0 + } + } + }, + { + \\"type\\": 3, + \\"data\\": { + \\"source\\": 9, + \\"id\\": 7, + \\"type\\": 2, + \\"property\\": \\"clear\\", + \\"args\\": [ + 16384 + ] + } + } +]" +`; diff --git a/packages/rrweb/test/record/webgl.test.ts b/packages/rrweb/test/record/webgl.test.ts index 8c46e5c69e..848cb8ac5f 100644 --- a/packages/rrweb/test/record/webgl.test.ts +++ b/packages/rrweb/test/record/webgl.test.ts @@ -112,6 +112,38 @@ describe('record webgl', function (this: ISuite) { property: 'clear', }, }); + assertSnapshot(ctx.events); + }); + + it('will record changes to a webgl2 canvas element', async () => { + await ctx.page.evaluate(() => { + const { record } = ((window as unknown) as IWindow).rrweb; + record({ + recordCanvas: true, + emit(event: eventWithTime) { + ((window as unknown) as IWindow).emit(event); + }, + }); + }); + await ctx.page.evaluate(() => { + var canvas = document.getElementById('canvas') as HTMLCanvasElement; + var gl = canvas.getContext('webgl2')!; + + gl.clear(gl.COLOR_BUFFER_BIT); + }); + + await ctx.page.waitForTimeout(50); + + const lastEvent = ctx.events[ctx.events.length - 1]; + expect(lastEvent).toMatchObject({ + data: { + source: IncrementalSource.CanvasMutation, + args: [16384], + type: CanvasContext.WebGL2, + property: 'clear', + }, + }); + assertSnapshot(ctx.events); }); it('will record changes to a canvas element before the canvas gets added', async () => { @@ -141,7 +173,37 @@ describe('record webgl', function (this: ISuite) { property: 'clear', }, }); - // TODO: make this a jest snapshot + assertSnapshot(ctx.events); + }); + + it('will record changes to a canvas element before the canvas gets added (webgl2)', async () => { + await ctx.page.evaluate(() => { + const { record } = ((window as unknown) as IWindow).rrweb; + record({ + recordCanvas: true, + emit: ((window as unknown) as IWindow).emit, + }); + }); + await ctx.page.evaluate(() => { + var canvas = document.createElement('canvas'); + var gl = canvas.getContext('webgl2')!; + var program = gl.createProgram()!; + gl.linkProgram(program); + gl.clear(gl.COLOR_BUFFER_BIT); + document.body.appendChild(canvas); + }); + + await ctx.page.waitForTimeout(50); + + const lastEvent = ctx.events[ctx.events.length - 1]; + expect(lastEvent).toMatchObject({ + data: { + source: IncrementalSource.CanvasMutation, + type: CanvasContext.WebGL2, + property: 'clear', + }, + }); + assertSnapshot(ctx.events); }); it('will record webgl variables', async () => { From f730f578047d166a2ce06012ce35940673b7263b Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Thu, 9 Dec 2021 11:57:58 +0100 Subject: [PATCH 39/93] Upgrade to puppeteer 12.0.1 --- packages/rrweb/package.json | 2 +- yarn.lock | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/rrweb/package.json b/packages/rrweb/package.json index 00634a6015..667b848a49 100644 --- a/packages/rrweb/package.json +++ b/packages/rrweb/package.json @@ -61,7 +61,7 @@ "jsdom": "^17.0.0", "jsdom-global": "^3.0.2", "prettier": "2.2.1", - "puppeteer": "^11.0.0", + "puppeteer": "^12.0.1", "rollup": "^2.45.2", "rollup-plugin-postcss": "^3.1.1", "rollup-plugin-rename-node-modules": "^1.1.0", diff --git a/yarn.lock b/yarn.lock index bda38821f2..a9d7e1ae57 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3514,10 +3514,10 @@ detect-newline@^3.0.0: resolved "https://registry.yarnpkg.com/detect-newline/-/detect-newline-3.1.0.tgz#576f5dfc63ae1a192ff192d8ad3af6308991b651" integrity sha512-TLz+x/vEXm/Y7P7wn1EJFNLxYpUD4TgMosxY6fAVJUnJMbupHBOncxyWUG9OpTaH9EBD7uFI5LfEgmMOc54DsA== -devtools-protocol@0.0.901419: - version "0.0.901419" - resolved "https://registry.yarnpkg.com/devtools-protocol/-/devtools-protocol-0.0.901419.tgz#79b5459c48fe7e1c5563c02bd72f8fec3e0cebcd" - integrity sha512-4INMPwNm9XRpBukhNbF7OB6fNTTCaI8pzy/fXg0xQzAy5h3zL1P8xT3QazgKqBrb/hAYwIBizqDBZ7GtJE74QQ== +devtools-protocol@0.0.937139: + version "0.0.937139" + resolved "https://registry.yarnpkg.com/devtools-protocol/-/devtools-protocol-0.0.937139.tgz#bdee3751fdfdb81cb701fd3afa94b1065dafafcf" + integrity sha512-daj+rzR3QSxsPRy5vjjthn58axO8c11j58uY0lG5vvlJk/EiOdCWOptGdkXDjtuRHr78emKq0udHCXM4trhoDQ== dezalgo@^1.0.0: version "1.0.3" @@ -8187,13 +8187,13 @@ puppeteer@^1.15.0: rimraf "^2.6.1" ws "^6.1.0" -puppeteer@^11.0.0: - version "11.0.0" - resolved "https://registry.yarnpkg.com/puppeteer/-/puppeteer-11.0.0.tgz#0808719c38e15315ecc1b1c28911f1c9054d201f" - integrity sha512-6rPFqN1ABjn4shgOICGDBITTRV09EjXVqhDERBDKwCLz0UyBxeeBH6Ay0vQUJ84VACmlxwzOIzVEJXThcF3aNg== +puppeteer@^12.0.1: + version "12.0.1" + resolved "https://registry.yarnpkg.com/puppeteer/-/puppeteer-12.0.1.tgz#ae79d0e174a07563e0bf2e05c94ccafce3e70033" + integrity sha512-YQ3GRiyZW0ddxTW+iiQcv2/8TT5c3+FcRUCg7F8q2gHqxd5akZN400VRXr9cHQKLWGukmJLDiE72MrcLK9tFHQ== dependencies: debug "4.3.2" - devtools-protocol "0.0.901419" + devtools-protocol "0.0.937139" extract-zip "2.0.1" https-proxy-agent "5.0.0" node-fetch "2.6.5" From cc7c0552768cd82d18bfe2c029b0af488f40ac7a Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Thu, 9 Dec 2021 12:18:43 +0100 Subject: [PATCH 40/93] Revert back to puppeteer 9.1.1 --- packages/rrweb/package.json | 2 +- yarn.lock | 141 +++++++++++++++--------------------- 2 files changed, 59 insertions(+), 84 deletions(-) diff --git a/packages/rrweb/package.json b/packages/rrweb/package.json index 667b848a49..63602f8f9a 100644 --- a/packages/rrweb/package.json +++ b/packages/rrweb/package.json @@ -61,7 +61,7 @@ "jsdom": "^17.0.0", "jsdom-global": "^3.0.2", "prettier": "2.2.1", - "puppeteer": "^12.0.1", + "puppeteer": "^9.1.1", "rollup": "^2.45.2", "rollup-plugin-postcss": "^3.1.1", "rollup-plugin-rename-node-modules": "^1.1.0", diff --git a/yarn.lock b/yarn.lock index a9d7e1ae57..11bb99098c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3398,7 +3398,7 @@ debug@2.6.9, debug@^2.6.9: dependencies: ms "2.0.0" -debug@4, debug@4.3.2, debug@^4.0.1, debug@^4.1.0, debug@^4.1.1: +debug@4, debug@^4.0.1, debug@^4.1.0, debug@^4.1.1: version "4.3.2" resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.2.tgz#f0a49c18ac8779e31d4a0c6029dfb76873c7428b" integrity sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw== @@ -3514,10 +3514,10 @@ detect-newline@^3.0.0: resolved "https://registry.yarnpkg.com/detect-newline/-/detect-newline-3.1.0.tgz#576f5dfc63ae1a192ff192d8ad3af6308991b651" integrity sha512-TLz+x/vEXm/Y7P7wn1EJFNLxYpUD4TgMosxY6fAVJUnJMbupHBOncxyWUG9OpTaH9EBD7uFI5LfEgmMOc54DsA== -devtools-protocol@0.0.937139: - version "0.0.937139" - resolved "https://registry.yarnpkg.com/devtools-protocol/-/devtools-protocol-0.0.937139.tgz#bdee3751fdfdb81cb701fd3afa94b1065dafafcf" - integrity sha512-daj+rzR3QSxsPRy5vjjthn58axO8c11j58uY0lG5vvlJk/EiOdCWOptGdkXDjtuRHr78emKq0udHCXM4trhoDQ== +devtools-protocol@0.0.869402: + version "0.0.869402" + resolved "https://registry.yarnpkg.com/devtools-protocol/-/devtools-protocol-0.0.869402.tgz#03ade701761742e43ae4de5dc188bcd80f156d8d" + integrity sha512-VvlVYY+VDJe639yHs5PHISzdWTLL3Aw8rO4cvUtwvoxFd6FHbE4OpHHcde52M6096uYYazAmd4l0o5VuFRO2WA== dezalgo@^1.0.0: version "1.0.3" @@ -4102,17 +4102,6 @@ extglob@^0.3.1: dependencies: is-extglob "^1.0.0" -extract-zip@2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/extract-zip/-/extract-zip-2.0.1.tgz#663dca56fe46df890d5f131ef4a06d22bb8ba13a" - integrity sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg== - dependencies: - debug "^4.1.1" - get-stream "^5.1.0" - yauzl "^2.10.0" - optionalDependencies: - "@types/yauzl" "^2.9.1" - extract-zip@^1.6.6: version "1.7.0" resolved "https://registry.yarnpkg.com/extract-zip/-/extract-zip-1.7.0.tgz#556cc3ae9df7f452c493a0cfb51cc30277940927" @@ -4123,6 +4112,17 @@ extract-zip@^1.6.6: mkdirp "^0.5.4" yauzl "^2.10.0" +extract-zip@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/extract-zip/-/extract-zip-2.0.1.tgz#663dca56fe46df890d5f131ef4a06d22bb8ba13a" + integrity sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg== + dependencies: + debug "^4.1.1" + get-stream "^5.1.0" + yauzl "^2.10.0" + optionalDependencies: + "@types/yauzl" "^2.9.1" + extsprintf@1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.3.0.tgz#96918440e3041a7a414f8c52e3c574eb3c3e1e05" @@ -4867,14 +4867,6 @@ http-signature@~1.2.0: jsprim "^1.2.2" sshpk "^1.7.0" -https-proxy-agent@5.0.0, https-proxy-agent@^5.0.0: - version "5.0.0" - resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz#e2a90542abb68a762e0a0850f6c9edadfd8506b2" - integrity sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA== - dependencies: - agent-base "6" - debug "4" - https-proxy-agent@^2.2.1: version "2.2.4" resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-2.2.4.tgz#4ee7a737abd92678a293d9b34a1af4d0d08c787b" @@ -4883,6 +4875,14 @@ https-proxy-agent@^2.2.1: agent-base "^4.3.0" debug "^3.1.0" +https-proxy-agent@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz#e2a90542abb68a762e0a0850f6c9edadfd8506b2" + integrity sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA== + dependencies: + agent-base "6" + debug "4" + human-signals@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/human-signals/-/human-signals-2.1.0.tgz#dc91fcba42e4d06e4abaed33b3e7a3c02f514ea0" @@ -6932,13 +6932,6 @@ nice-try@^1.0.4: resolved "https://registry.yarnpkg.com/nice-try/-/nice-try-1.0.5.tgz#a3378a7696ce7d223e88fc9b764bd7ef1089e366" integrity sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ== -node-fetch@2.6.5: - version "2.6.5" - resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.5.tgz#42735537d7f080a7e5f78b6c549b7146be1742fd" - integrity sha512-mmlIVHJEu5rnIxgEgez6b9GgWXbkZj5YZ7fx+2r94a2E+Uirsp6HsPTPlomfdHtpt/B0cdKviwkoaM6pyvUOpQ== - dependencies: - whatwg-url "^5.0.0" - node-fetch@^2.6.1: version "2.6.1" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.1.tgz#045bd323631f76ed2e2b55573394416b639a0052" @@ -7650,7 +7643,7 @@ pixelmatch@^5.1.0: dependencies: pngjs "^4.0.1" -pkg-dir@4.2.0, pkg-dir@^4.2.0: +pkg-dir@^4.2.0: version "4.2.0" resolved "https://registry.yarnpkg.com/pkg-dir/-/pkg-dir-4.2.0.tgz#f099133df7ede422e81d1d8448270eeb3e4261f3" integrity sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ== @@ -8089,7 +8082,7 @@ process-nextick-args@~2.0.0: resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2" integrity sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag== -progress@2.0.3, progress@^2.0.0, progress@^2.0.1: +progress@^2.0.0, progress@^2.0.1: version "2.0.3" resolved "https://registry.yarnpkg.com/progress/-/progress-2.0.3.tgz#7e8cf8d8f5b8f239c1bc68beb4eb78567d572ef8" integrity sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA== @@ -8145,7 +8138,7 @@ proxy-addr@~2.0.5: forwarded "0.2.0" ipaddr.js "1.9.1" -proxy-from-env@1.1.0, proxy-from-env@^1.0.0: +proxy-from-env@^1.0.0, proxy-from-env@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/proxy-from-env/-/proxy-from-env-1.1.0.tgz#e102f16ca355424865755d2c9e8ea4f24d58c3e2" integrity sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg== @@ -8187,23 +8180,23 @@ puppeteer@^1.15.0: rimraf "^2.6.1" ws "^6.1.0" -puppeteer@^12.0.1: - version "12.0.1" - resolved "https://registry.yarnpkg.com/puppeteer/-/puppeteer-12.0.1.tgz#ae79d0e174a07563e0bf2e05c94ccafce3e70033" - integrity sha512-YQ3GRiyZW0ddxTW+iiQcv2/8TT5c3+FcRUCg7F8q2gHqxd5akZN400VRXr9cHQKLWGukmJLDiE72MrcLK9tFHQ== - dependencies: - debug "4.3.2" - devtools-protocol "0.0.937139" - extract-zip "2.0.1" - https-proxy-agent "5.0.0" - node-fetch "2.6.5" - pkg-dir "4.2.0" - progress "2.0.3" - proxy-from-env "1.1.0" - rimraf "3.0.2" - tar-fs "2.1.1" - unbzip2-stream "1.4.3" - ws "8.2.3" +puppeteer@^9.1.1: + version "9.1.1" + resolved "https://registry.yarnpkg.com/puppeteer/-/puppeteer-9.1.1.tgz#f74b7facf86887efd6c6b9fabb7baae6fdce012c" + integrity sha512-W+nOulP2tYd/ZG99WuZC/I5ljjQQ7EUw/jQGcIb9eu8mDlZxNY2SgcJXTLG9h5gRvqA3uJOe4hZXYsd3EqioMw== + dependencies: + debug "^4.1.0" + devtools-protocol "0.0.869402" + extract-zip "^2.0.0" + https-proxy-agent "^5.0.0" + node-fetch "^2.6.1" + pkg-dir "^4.2.0" + progress "^2.0.1" + proxy-from-env "^1.1.0" + rimraf "^3.0.2" + tar-fs "^2.0.0" + unbzip2-stream "^1.3.3" + ws "^7.2.3" q@^1.1.2, q@^1.5.1: version "1.5.1" @@ -8598,13 +8591,6 @@ rgba-regex@^1.0.0: resolved "https://registry.yarnpkg.com/rgba-regex/-/rgba-regex-1.0.0.tgz#43374e2e2ca0968b0ef1523460b7d730ff22eeb3" integrity sha1-QzdOLiyglosO8VI0YLfXMP8i7rM= -rimraf@3.0.2, rimraf@^3.0.0, rimraf@^3.0.2: - version "3.0.2" - resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-3.0.2.tgz#f1a5402ba6220ad52cc1282bac1ae3aa49fd061a" - integrity sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA== - dependencies: - glob "^7.1.3" - rimraf@^2.6.1, rimraf@^2.6.2, rimraf@^2.6.3: version "2.7.1" resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.7.1.tgz#35797f13a7fdadc566142c29d4f07ccad483e3ec" @@ -8612,6 +8598,13 @@ rimraf@^2.6.1, rimraf@^2.6.2, rimraf@^2.6.3: dependencies: glob "^7.1.3" +rimraf@^3.0.0, rimraf@^3.0.2: + version "3.0.2" + resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-3.0.2.tgz#f1a5402ba6220ad52cc1282bac1ae3aa49fd061a" + integrity sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA== + dependencies: + glob "^7.1.3" + rollup-plugin-css-only@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/rollup-plugin-css-only/-/rollup-plugin-css-only-3.1.0.tgz#6a701cc5b051c6b3f0961e69b108a9a118e1b1df" @@ -9364,7 +9357,7 @@ table@^6.0.9: string-width "^4.2.0" strip-ansi "^6.0.0" -tar-fs@2.1.1: +tar-fs@^2.0.0: version "2.1.1" resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-2.1.1.tgz#489a15ab85f1f0befabb370b7de4f9eb5cbe8784" integrity sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng== @@ -9567,11 +9560,6 @@ tr46@^2.1.0: dependencies: punycode "^2.1.1" -tr46@~0.0.3: - version "0.0.3" - resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a" - integrity sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o= - trim-newlines@^3.0.0: version "3.0.1" resolved "https://registry.yarnpkg.com/trim-newlines/-/trim-newlines-3.0.1.tgz#260a5d962d8b752425b32f3a7db0dcacd176c144" @@ -9775,7 +9763,7 @@ unbox-primitive@^1.0.1: has-symbols "^1.0.2" which-boxed-primitive "^1.0.2" -unbzip2-stream@1.4.3: +unbzip2-stream@^1.3.3: version "1.4.3" resolved "https://registry.yarnpkg.com/unbzip2-stream/-/unbzip2-stream-1.4.3.tgz#b0da04c4371311df771cdc215e87f2130991ace7" integrity sha512-mlExGW4w71ebDJviH16lQLtZS32VKqsSfk80GCfUlwT/4/hNRFsoscrF/c++9xinkMzECL1uL9DDwXqFWkruPg== @@ -9998,11 +9986,6 @@ wcwidth@^1.0.0: dependencies: defaults "^1.0.3" -webidl-conversions@^3.0.0: - version "3.0.1" - resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871" - integrity sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE= - webidl-conversions@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-5.0.0.tgz#ae59c8a00b121543a2acc65c0434f57b0fc11aff" @@ -10025,14 +10008,6 @@ whatwg-mimetype@^2.3.0: resolved "https://registry.yarnpkg.com/whatwg-mimetype/-/whatwg-mimetype-2.3.0.tgz#3d4b1e0312d2079879f826aff18dbeeca5960fbf" integrity sha512-M4yMwr6mAnQz76TbJm914+gPpB/nCwvZbJU28cUD6dR004SAxDLOOSUaB1JDRqLtaOV/vi0IC5lEAGFgrjGv/g== -whatwg-url@^5.0.0: - version "5.0.0" - resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d" - integrity sha1-lmRU6HZUYuN2RNNib2dCzotwll0= - dependencies: - tr46 "~0.0.3" - webidl-conversions "^3.0.0" - whatwg-url@^8.0.0, whatwg-url@^8.4.0, whatwg-url@^8.5.0: version "8.7.0" resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-8.7.0.tgz#656a78e510ff8f3937bc0bcbe9f5c0ac35941b77" @@ -10170,11 +10145,6 @@ write-pkg@^4.0.0: type-fest "^0.4.1" write-json-file "^3.2.0" -ws@8.2.3: - version "8.2.3" - resolved "https://registry.yarnpkg.com/ws/-/ws-8.2.3.tgz#63a56456db1b04367d0b721a0b80cae6d8becbba" - integrity sha512-wBuoj1BDpC6ZQ1B7DWQBYVLphPWkm8i9Y0/3YdHjHKHiohOJ1ws+3OccDWtH+PoC9DZD5WOTrJvNbWvjS6JWaA== - ws@^6.1.0: version "6.2.2" resolved "https://registry.yarnpkg.com/ws/-/ws-6.2.2.tgz#dd5cdbd57a9979916097652d78f1cc5faea0c32e" @@ -10182,6 +10152,11 @@ ws@^6.1.0: dependencies: async-limiter "~1.0.0" +ws@^7.2.3: + version "7.5.6" + resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.6.tgz#e59fc509fb15ddfb65487ee9765c5a51dec5fe7b" + integrity sha512-6GLgCqo2cy2A2rjCNFlxQS6ZljG/coZfZXclldI8FB/1G3CCI36Zd8xy2HrFVACi8tfk5XrgLQEk+P0Tnz9UcA== + ws@^7.4.3, ws@^7.4.5: version "7.5.3" resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.3.tgz#160835b63c7d97bfab418fc1b8a9fced2ac01a74" From 313adb42d2c8d11700e010f2b30da41a905f36ba Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Tue, 14 Dec 2021 17:18:38 +0100 Subject: [PATCH 41/93] Keep index order consistent between replay and record --- .../record/observers/canvas/serialize-args.ts | 31 ++++++++++++++++ .../src/record/observers/canvas/webgl.ts | 7 ++-- packages/rrweb/test/record/webgl.test.ts | 35 +++++++++++++++++++ 3 files changed, 71 insertions(+), 2 deletions(-) diff --git a/packages/rrweb/src/record/observers/canvas/serialize-args.ts b/packages/rrweb/src/record/observers/canvas/serialize-args.ts index 40e7074298..29d4dba44d 100644 --- a/packages/rrweb/src/record/observers/canvas/serialize-args.ts +++ b/packages/rrweb/src/record/observers/canvas/serialize-args.ts @@ -92,3 +92,34 @@ export function serializeArg(value: any): SerializedWebGlArg { export const serializeArgs = (args: Array) => { return [...args].map(serializeArg); }; + +export const saveWebGLVar = (value: any): number | void => { + if ( + !( + value instanceof WebGLActiveInfo || + value instanceof WebGLBuffer || + value instanceof WebGLFramebuffer || + value instanceof WebGLProgram || + value instanceof WebGLRenderbuffer || + value instanceof WebGLShader || + value instanceof WebGLShaderPrecisionFormat || + value instanceof WebGLTexture || + value instanceof WebGLUniformLocation || + value instanceof WebGLVertexArrayObject || + // In Chrome, value won't be an instanceof WebGLVertexArrayObject. + (value && value.constructor.name == 'WebGLVertexArrayObjectOES') || + typeof value === 'object' + ) + ) + return; + + const name = value.constructor.name; + const list = webGLVars[name] || (webGLVars[name] = []); + let index = list.indexOf(value); + + if (index === -1) { + index = list.length; + list.push(value); + } + return index; +}; diff --git a/packages/rrweb/src/record/observers/canvas/webgl.ts b/packages/rrweb/src/record/observers/canvas/webgl.ts index 79559fccfa..0e2adf680d 100644 --- a/packages/rrweb/src/record/observers/canvas/webgl.ts +++ b/packages/rrweb/src/record/observers/canvas/webgl.ts @@ -9,7 +9,7 @@ import { SerializedWebGlArg, } from '../../../types'; import { hookSetter, isBlocked, patch } from '../../../utils'; -import { serializeArgs } from './serialize-args'; +import { saveWebGLVar, serializeArgs } from './serialize-args'; export default function initCanvasWebGLMutationObserver( cb: canvasMutationCallback, @@ -47,7 +47,10 @@ export default function initCanvasWebGLMutationObserver( }); }, 0); } - return original.apply(this, args); + + const result = original.apply(this, args); + saveWebGLVar(result); + return result; }; }, ); diff --git a/packages/rrweb/test/record/webgl.test.ts b/packages/rrweb/test/record/webgl.test.ts index 848cb8ac5f..f65319cc22 100644 --- a/packages/rrweb/test/record/webgl.test.ts +++ b/packages/rrweb/test/record/webgl.test.ts @@ -241,6 +241,41 @@ describe('record webgl', function (this: ISuite) { }); }); + it('will record webgl variables in reverse order', async () => { + await ctx.page.evaluate(() => { + const { record } = ((window as unknown) as IWindow).rrweb; + record({ + recordCanvas: true, + emit: ((window as unknown) as IWindow).emit, + }); + }); + await ctx.page.evaluate(() => { + var canvas = document.getElementById('canvas') as HTMLCanvasElement; + var gl = canvas.getContext('webgl')!; + var program0 = gl.createProgram()!; + var program1 = gl.createProgram()!; + gl.linkProgram(program1); + gl.linkProgram(program0); + }); + + await ctx.page.waitForTimeout(50); + + const lastEvent = ctx.events[ctx.events.length - 1]; + expect(lastEvent).toMatchObject({ + data: { + source: IncrementalSource.CanvasMutation, + property: 'linkProgram', + type: CanvasContext.WebGL, + args: [ + { + index: 0, + rr_type: 'WebGLProgram', + }, + ], // `program0` is WebGLProgram, this is the first WebGLProgram variable (index #0) + }, + }); + }); + it('sets _context on canvas.getContext()', async () => { await ctx.page.evaluate(() => { const { record } = ((window as unknown) as IWindow).rrweb; From 723acf81bb863232a33be0699deefdd81fea5dcf Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Tue, 14 Dec 2021 17:21:14 +0100 Subject: [PATCH 42/93] keep correct index order in webgl2 --- packages/rrweb/src/record/observers/canvas/webgl2.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/rrweb/src/record/observers/canvas/webgl2.ts b/packages/rrweb/src/record/observers/canvas/webgl2.ts index 68bd0e1690..a2c3c42cc8 100644 --- a/packages/rrweb/src/record/observers/canvas/webgl2.ts +++ b/packages/rrweb/src/record/observers/canvas/webgl2.ts @@ -37,6 +37,8 @@ export default function initCanvasWebGLMutationObserver( this: WebGL2RenderingContext, ...args: Array ) { + const result = original.apply(this, args); + saveWebGLVar(result); if (!isBlocked((this.canvas as unknown) as INode, blockClass)) { setTimeout(() => { const recordArgs = serializeArgs([...args]); @@ -48,7 +50,7 @@ export default function initCanvasWebGLMutationObserver( }); }, 0); } - return original.apply(this, args); + return result; }; }, ); From 3f9eebd1f9e85007289c0bf2f77bfb72b401a69e Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Tue, 14 Dec 2021 17:21:46 +0100 Subject: [PATCH 43/93] fixed forgotten import --- packages/rrweb/src/record/observers/canvas/webgl2.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/rrweb/src/record/observers/canvas/webgl2.ts b/packages/rrweb/src/record/observers/canvas/webgl2.ts index a2c3c42cc8..99f7e38dae 100644 --- a/packages/rrweb/src/record/observers/canvas/webgl2.ts +++ b/packages/rrweb/src/record/observers/canvas/webgl2.ts @@ -8,7 +8,7 @@ import { Mirror, } from '../../../types'; import { hookSetter, isBlocked, patch } from '../../../utils'; -import { serializeArgs } from './serialize-args'; +import { saveWebGLVar, serializeArgs } from './serialize-args'; export default function initCanvasWebGLMutationObserver( cb: canvasMutationCallback, From 4a6b556470f41431903a860921083825cc6f5817 Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Tue, 14 Dec 2021 17:28:28 +0100 Subject: [PATCH 44/93] buffer up pending canvas mutations --- .../src/record/observers/canvas/webgl2.ts | 68 ++++++++++++++++--- packages/rrweb/test/record/webgl.test.ts | 4 +- 2 files changed, 62 insertions(+), 10 deletions(-) diff --git a/packages/rrweb/src/record/observers/canvas/webgl2.ts b/packages/rrweb/src/record/observers/canvas/webgl2.ts index 99f7e38dae..4fce44d729 100644 --- a/packages/rrweb/src/record/observers/canvas/webgl2.ts +++ b/packages/rrweb/src/record/observers/canvas/webgl2.ts @@ -3,6 +3,7 @@ import { blockClass, CanvasContext, canvasMutationCallback, + canvasMutationParam, IWindow, listenerHandler, Mirror, @@ -10,6 +11,37 @@ import { import { hookSetter, isBlocked, patch } from '../../../utils'; import { saveWebGLVar, serializeArgs } from './serialize-args'; +const pendingCanvasMutations = new Map< + HTMLCanvasElement, + canvasMutationParam[] +>(); + +// FIXME: total hack here, we need to find a better way to do this +function flushPendingCanvasMutations( + cb: canvasMutationCallback, + mirror: Mirror, +) { + pendingCanvasMutations.forEach( + (values: canvasMutationParam[], canvas: HTMLCanvasElement) => { + const id = mirror.getId((canvas as unknown) as INode); + flushPendingCanvasMutationFor(canvas, id, cb); + }, + ); + requestAnimationFrame(() => flushPendingCanvasMutations(cb, mirror)); +} + +function flushPendingCanvasMutationFor( + canvas: HTMLCanvasElement, + id: number, + cb: canvasMutationCallback, +) { + const values = pendingCanvasMutations.get(canvas); + if (!values || id === -1) return; + + values.forEach((p) => cb({ ...p, id })); + pendingCanvasMutations.delete(canvas); +} + export default function initCanvasWebGLMutationObserver( cb: canvasMutationCallback, win: IWindow, @@ -20,6 +52,10 @@ export default function initCanvasWebGLMutationObserver( const props = Object.getOwnPropertyNames( win.WebGL2RenderingContext.prototype, ); + + // TODO: replace me + requestAnimationFrame(() => flushPendingCanvasMutations(cb, mirror)); + for (const prop of props) { try { if ( @@ -40,16 +76,30 @@ export default function initCanvasWebGLMutationObserver( const result = original.apply(this, args); saveWebGLVar(result); if (!isBlocked((this.canvas as unknown) as INode, blockClass)) { - setTimeout(() => { - const recordArgs = serializeArgs([...args]); - cb({ - id: mirror.getId((this.canvas as unknown) as INode), - type: CanvasContext.WebGL2, - property: prop, - args: recordArgs, - }); - }, 0); + const id = mirror.getId((this.canvas as unknown) as INode); + + const recordArgs = serializeArgs([...args]); + const mutation = { + id, + type: CanvasContext.WebGL2, + property: prop, + args: recordArgs, + }; + + if (id === -1) { + if (!pendingCanvasMutations.has(this.canvas)) + pendingCanvasMutations.set(this.canvas, []); + + pendingCanvasMutations + .get(this.canvas as HTMLCanvasElement)! + .push(mutation); + } else { + // flush all pending mutations + flushPendingCanvasMutationFor(this.canvas, id, cb); + cb(mutation); + } } + return result; }; }, diff --git a/packages/rrweb/test/record/webgl.test.ts b/packages/rrweb/test/record/webgl.test.ts index f65319cc22..6abfa6c71c 100644 --- a/packages/rrweb/test/record/webgl.test.ts +++ b/packages/rrweb/test/record/webgl.test.ts @@ -190,7 +190,9 @@ describe('record webgl', function (this: ISuite) { var program = gl.createProgram()!; gl.linkProgram(program); gl.clear(gl.COLOR_BUFFER_BIT); - document.body.appendChild(canvas); + setTimeout(() => { + document.body.appendChild(canvas); + }, 10); }); await ctx.page.waitForTimeout(50); From 970ee44bbb224ebe20bc48e01b1c415e354df92e Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Wed, 15 Dec 2021 15:44:29 +0100 Subject: [PATCH 45/93] unify the way webgl and webgl2 get patched --- packages/rrweb/src/record/observer.ts | 13 +- .../src/record/observers/canvas/webgl.ts | 165 +++++++++++++----- .../src/record/observers/canvas/webgl2.ts | 130 -------------- 3 files changed, 119 insertions(+), 189 deletions(-) delete mode 100644 packages/rrweb/src/record/observers/canvas/webgl2.ts diff --git a/packages/rrweb/src/record/observer.ts b/packages/rrweb/src/record/observer.ts index 6ea1940a8f..4ec75f4e3c 100644 --- a/packages/rrweb/src/record/observer.ts +++ b/packages/rrweb/src/record/observer.ts @@ -52,7 +52,6 @@ import { ShadowDomManager } from './shadow-dom-manager'; import initCanvasContextObserver from './observers/canvas/canvas'; import initCanvas2DMutationObserver from './observers/canvas/2d'; import initCanvasWebGLMutationObserver from './observers/canvas/webgl'; -import initCanvasWebGL2MutationObserver from './observers/canvas/webgl2'; type WindowWithStoredMutationObserver = IWindow & { __rrMutationObserver?: MutationObserver; @@ -724,14 +723,7 @@ function initCanvasMutationObserver( mirror, ); - const canvasWebGLReset = initCanvasWebGLMutationObserver( - cb, - win, - blockClass, - mirror, - ); - - const canvasWebGL2Reset = initCanvasWebGL2MutationObserver( + const canvasWebGL1and2Reset = initCanvasWebGLMutationObserver( cb, win, blockClass, @@ -741,8 +733,7 @@ function initCanvasMutationObserver( return () => { canvasContextReset(); canvas2DReset(); - canvasWebGLReset(); - canvasWebGL2Reset(); + canvasWebGL1and2Reset(); }; } diff --git a/packages/rrweb/src/record/observers/canvas/webgl.ts b/packages/rrweb/src/record/observers/canvas/webgl.ts index 0e2adf680d..eb3aaae4bf 100644 --- a/packages/rrweb/src/record/observers/canvas/webgl.ts +++ b/packages/rrweb/src/record/observers/canvas/webgl.ts @@ -3,77 +3,146 @@ import { blockClass, CanvasContext, canvasMutationCallback, + canvasMutationParam, IWindow, listenerHandler, Mirror, - SerializedWebGlArg, } from '../../../types'; import { hookSetter, isBlocked, patch } from '../../../utils'; import { saveWebGLVar, serializeArgs } from './serialize-args'; -export default function initCanvasWebGLMutationObserver( +const pendingCanvasMutations = new Map< + HTMLCanvasElement, + canvasMutationParam[] +>(); + +// FIXME: total hack here, we need to find a better way to do this +function flushPendingCanvasMutations( + cb: canvasMutationCallback, + mirror: Mirror, +) { + pendingCanvasMutations.forEach( + (values: canvasMutationParam[], canvas: HTMLCanvasElement) => { + const id = mirror.getId((canvas as unknown) as INode); + flushPendingCanvasMutationFor(canvas, id, cb); + }, + ); + requestAnimationFrame(() => flushPendingCanvasMutations(cb, mirror)); +} + +function flushPendingCanvasMutationFor( + canvas: HTMLCanvasElement, + id: number, + cb: canvasMutationCallback, +) { + const values = pendingCanvasMutations.get(canvas); + if (!values || id === -1) return; + + values.forEach((p) => cb({ ...p, id })); + pendingCanvasMutations.delete(canvas); +} + +function patchGLPrototype( + prototype: WebGLRenderingContext | WebGL2RenderingContext, + type: CanvasContext, cb: canvasMutationCallback, - win: IWindow, blockClass: blockClass, mirror: Mirror, -): listenerHandler { +): listenerHandler[] { const handlers: listenerHandler[] = []; - const props = Object.getOwnPropertyNames(win.WebGLRenderingContext.prototype); + + const props = Object.getOwnPropertyNames(prototype); + for (const prop of props) { try { - if ( - typeof win.WebGLRenderingContext.prototype[ - prop as keyof WebGLRenderingContext - ] !== 'function' - ) { + if (typeof prototype[prop as keyof typeof prototype] !== 'function') { continue; } - const restoreHandler = patch( - win.WebGLRenderingContext.prototype, - prop, - function (original) { - return function ( - this: WebGLRenderingContext, - ...args: Array - ) { - if (!isBlocked((this.canvas as unknown) as INode, blockClass)) { - setTimeout(() => { - const recordArgs = serializeArgs([...args]); - cb({ - id: mirror.getId((this.canvas as unknown) as INode), - type: CanvasContext.WebGL, - property: prop, - args: recordArgs, - }); - }, 0); + const restoreHandler = patch(prototype, prop, function (original) { + return function (this: typeof prototype, ...args: Array) { + const result = original.apply(this, args); + saveWebGLVar(result); + if (!isBlocked((this.canvas as unknown) as INode, blockClass)) { + const id = mirror.getId((this.canvas as unknown) as INode); + + const recordArgs = serializeArgs([...args]); + const mutation = { + id, + type, + property: prop, + args: recordArgs, + }; + + if (id === -1) { + if (!pendingCanvasMutations.has(this.canvas)) + pendingCanvasMutations.set(this.canvas, []); + + pendingCanvasMutations + .get(this.canvas as HTMLCanvasElement)! + .push(mutation); + } else { + // flush all pending mutations + flushPendingCanvasMutationFor(this.canvas, id, cb); + cb(mutation); } + } - const result = original.apply(this, args); - saveWebGLVar(result); - return result; - }; - }, - ); + return result; + }; + }); handlers.push(restoreHandler); } catch { - const hookHandler = hookSetter( - win.WebGLRenderingContext.prototype, - prop, - { - set(v) { - cb({ - id: mirror.getId((this.canvas as unknown) as INode), - type: CanvasContext.WebGL, - property: prop, - args: [v], - setter: true, - }); - }, + const hookHandler = hookSetter(prototype, prop, { + set(v) { + cb({ + id: mirror.getId((this.canvas as unknown) as INode), + type, + property: prop, + args: [v], + setter: true, + }); }, - ); + }); handlers.push(hookHandler); } } + + return handlers; +} + +export default function initCanvasWebGLMutationObserver( + cb: canvasMutationCallback, + win: IWindow, + blockClass: blockClass, + mirror: Mirror, +): listenerHandler { + const handlers: listenerHandler[] = []; + + // TODO: replace me + requestAnimationFrame(() => flushPendingCanvasMutations(cb, mirror)); + + handlers.push( + ...patchGLPrototype( + win.WebGLRenderingContext.prototype, + CanvasContext.WebGL, + cb, + blockClass, + mirror, + ), + ); + + if (typeof win.WebGL2RenderingContext !== 'undefined') { + handlers.push( + ...patchGLPrototype( + win.WebGL2RenderingContext.prototype, + CanvasContext.WebGL2, + cb, + blockClass, + mirror, + ), + ); + } + return () => { handlers.forEach((h) => h()); }; diff --git a/packages/rrweb/src/record/observers/canvas/webgl2.ts b/packages/rrweb/src/record/observers/canvas/webgl2.ts deleted file mode 100644 index 4fce44d729..0000000000 --- a/packages/rrweb/src/record/observers/canvas/webgl2.ts +++ /dev/null @@ -1,130 +0,0 @@ -import { INode } from 'rrweb-snapshot'; -import { - blockClass, - CanvasContext, - canvasMutationCallback, - canvasMutationParam, - IWindow, - listenerHandler, - Mirror, -} from '../../../types'; -import { hookSetter, isBlocked, patch } from '../../../utils'; -import { saveWebGLVar, serializeArgs } from './serialize-args'; - -const pendingCanvasMutations = new Map< - HTMLCanvasElement, - canvasMutationParam[] ->(); - -// FIXME: total hack here, we need to find a better way to do this -function flushPendingCanvasMutations( - cb: canvasMutationCallback, - mirror: Mirror, -) { - pendingCanvasMutations.forEach( - (values: canvasMutationParam[], canvas: HTMLCanvasElement) => { - const id = mirror.getId((canvas as unknown) as INode); - flushPendingCanvasMutationFor(canvas, id, cb); - }, - ); - requestAnimationFrame(() => flushPendingCanvasMutations(cb, mirror)); -} - -function flushPendingCanvasMutationFor( - canvas: HTMLCanvasElement, - id: number, - cb: canvasMutationCallback, -) { - const values = pendingCanvasMutations.get(canvas); - if (!values || id === -1) return; - - values.forEach((p) => cb({ ...p, id })); - pendingCanvasMutations.delete(canvas); -} - -export default function initCanvasWebGLMutationObserver( - cb: canvasMutationCallback, - win: IWindow, - blockClass: blockClass, - mirror: Mirror, -): listenerHandler { - const handlers: listenerHandler[] = []; - const props = Object.getOwnPropertyNames( - win.WebGL2RenderingContext.prototype, - ); - - // TODO: replace me - requestAnimationFrame(() => flushPendingCanvasMutations(cb, mirror)); - - for (const prop of props) { - try { - if ( - typeof win.WebGL2RenderingContext.prototype[ - prop as keyof WebGL2RenderingContext - ] !== 'function' - ) { - continue; - } - const restoreHandler = patch( - win.WebGL2RenderingContext.prototype, - prop, - function (original) { - return function ( - this: WebGL2RenderingContext, - ...args: Array - ) { - const result = original.apply(this, args); - saveWebGLVar(result); - if (!isBlocked((this.canvas as unknown) as INode, blockClass)) { - const id = mirror.getId((this.canvas as unknown) as INode); - - const recordArgs = serializeArgs([...args]); - const mutation = { - id, - type: CanvasContext.WebGL2, - property: prop, - args: recordArgs, - }; - - if (id === -1) { - if (!pendingCanvasMutations.has(this.canvas)) - pendingCanvasMutations.set(this.canvas, []); - - pendingCanvasMutations - .get(this.canvas as HTMLCanvasElement)! - .push(mutation); - } else { - // flush all pending mutations - flushPendingCanvasMutationFor(this.canvas, id, cb); - cb(mutation); - } - } - - return result; - }; - }, - ); - handlers.push(restoreHandler); - } catch { - const hookHandler = hookSetter( - win.WebGL2RenderingContext.prototype, - prop, - { - set(v) { - cb({ - id: mirror.getId((this.canvas as unknown) as INode), - type: CanvasContext.WebGL2, - property: prop, - args: [v], - setter: true, - }); - }, - }, - ); - handlers.push(hookHandler); - } - } - return () => { - handlers.forEach((h) => h()); - }; -} From 0d42d832bc96f63c605a4d3ddc752109353954e0 Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Wed, 15 Dec 2021 17:13:41 +0100 Subject: [PATCH 46/93] fix parsing error --- .../src/record/observers/canvas/webgl.ts | 31 +++++++++++++------ 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/packages/rrweb/src/record/observers/canvas/webgl.ts b/packages/rrweb/src/record/observers/canvas/webgl.ts index eb3aaae4bf..1e5533704b 100644 --- a/packages/rrweb/src/record/observers/canvas/webgl.ts +++ b/packages/rrweb/src/record/observers/canvas/webgl.ts @@ -74,16 +74,27 @@ function patchGLPrototype( }; if (id === -1) { - if (!pendingCanvasMutations.has(this.canvas)) - pendingCanvasMutations.set(this.canvas, []); - - pendingCanvasMutations - .get(this.canvas as HTMLCanvasElement)! - .push(mutation); - } else { - // flush all pending mutations - flushPendingCanvasMutationFor(this.canvas, id, cb); - cb(mutation); + // FIXME! THIS COULD MAYBE BE AN OFFSCREEN CANVAS + if ( + !pendingCanvasMutations.has(this.canvas as HTMLCanvasElement) + ) { + pendingCanvasMutations.set( + this.canvas as HTMLCanvasElement, + [], + ); + + pendingCanvasMutations + .get(this.canvas as HTMLCanvasElement)! + .push(mutation); + } else { + // flush all pending mutations + flushPendingCanvasMutationFor( + this.canvas as HTMLCanvasElement, + id, + cb, + ); + cb(mutation); + } } } From d26676e3c96f1635f0d53b7ff21903a6e72f320f Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Thu, 16 Dec 2021 11:24:16 +0100 Subject: [PATCH 47/93] Add types for serialize-args --- .../rrweb/typings/record/observers/canvas/serialize-args.d.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/rrweb/typings/record/observers/canvas/serialize-args.d.ts b/packages/rrweb/typings/record/observers/canvas/serialize-args.d.ts index b6dc836dbc..0f625d7a32 100644 --- a/packages/rrweb/typings/record/observers/canvas/serialize-args.d.ts +++ b/packages/rrweb/typings/record/observers/canvas/serialize-args.d.ts @@ -1,3 +1,4 @@ import { SerializedWebGlArg } from '../../../types'; export declare function serializeArg(value: any): SerializedWebGlArg; export declare const serializeArgs: (args: Array) => SerializedWebGlArg[]; +export declare const saveWebGLVar: (value: any) => number | void; From 2b2e11f257f7dcb82b77ce7f97891d29de0c7802 Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Thu, 16 Dec 2021 11:31:22 +0100 Subject: [PATCH 48/93] Add debugging for webgl replay --- packages/rrweb/src/replay/canvas/webgl.ts | 28 +++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/packages/rrweb/src/replay/canvas/webgl.ts b/packages/rrweb/src/replay/canvas/webgl.ts index e28693de6d..5540de40c6 100644 --- a/packages/rrweb/src/replay/canvas/webgl.ts +++ b/packages/rrweb/src/replay/canvas/webgl.ts @@ -108,6 +108,34 @@ export default function webglMutation({ const args = mutation.args.map(deserializeArg); const result = original.apply(ctx, args); + // const debugMode = false; + const debugMode = true; + if (debugMode) { + if (mutation.property === 'compileShader') { + if (!ctx.getShaderParameter(args[0], ctx.COMPILE_STATUS)) + console.warn( + 'something went wrong in replay', + ctx.getShaderInfoLog(args[0]), + ); + } else if (mutation.property === 'linkProgram') { + ctx.validateProgram(args[0]); + if (!ctx.getProgramParameter(args[0], ctx.LINK_STATUS)) + console.warn( + 'something went wrong in replay', + ctx.getProgramInfoLog(args[0]), + ); + } + const webglError = ctx.getError(); + if (webglError !== ctx.NO_ERROR) { + console.warn( + 'WEBGL ERROR', + webglError, + 'on command:', + mutation.property, + ...args, + ); + } + } saveToWebGLVarMap(result); } catch (error) { From 8ca984eb3880f3fc4095087e9cdb57dfe297db8f Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Thu, 16 Dec 2021 15:55:44 +0100 Subject: [PATCH 49/93] Move start-server to utils --- packages/rrweb/test/integration.test.ts | 46 +++++-------------- packages/rrweb/test/utils.ts | 59 +++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 36 deletions(-) diff --git a/packages/rrweb/test/integration.test.ts b/packages/rrweb/test/integration.test.ts index d412cebad1..a50bda2831 100644 --- a/packages/rrweb/test/integration.test.ts +++ b/packages/rrweb/test/integration.test.ts @@ -1,14 +1,19 @@ import * as fs from 'fs'; import * as path from 'path'; import * as http from 'http'; -import * as url from 'url'; import * as puppeteer from 'puppeteer'; -import { assertSnapshot, launchPuppeteer } from './utils'; +import { + assertSnapshot, + startServer, + getServerURL, + launchPuppeteer, +} from './utils'; import { recordOptions, eventWithTime, EventType } from '../src/types'; import { visitSnapshot, NodeType } from 'rrweb-snapshot'; interface ISuite { server: http.Server; + serverURL: string; code: string; browser: puppeteer.Browser; } @@ -17,39 +22,6 @@ interface IMimeType { [key: string]: string; } -const startServer = () => - new Promise((resolve) => { - const mimeType: IMimeType = { - '.html': 'text/html', - '.js': 'text/javascript', - '.css': 'text/css', - }; - const s = http.createServer((req, res) => { - const parsedUrl = url.parse(req.url!); - const sanitizePath = path - .normalize(parsedUrl.pathname!) - .replace(/^(\.\.[\/\\])+/, ''); - let pathname = path.join(__dirname, sanitizePath); - try { - const data = fs.readFileSync(pathname); - const ext = path.parse(pathname).ext; - res.setHeader('Content-type', mimeType[ext] || 'text/plain'); - res.setHeader('Access-Control-Allow-Origin', '*'); - res.setHeader('Access-Control-Allow-Methods', 'GET'); - res.setHeader('Access-Control-Allow-Headers', 'Content-type'); - setTimeout(() => { - res.end(data); - // mock delay - }, 100); - } catch (error) { - res.end(); - } - }); - s.listen(3031).on('listening', () => { - resolve(s); - }); - }); - describe('record integration tests', function (this: ISuite) { jest.setTimeout(10_000); @@ -85,11 +57,13 @@ describe('record integration tests', function (this: ISuite) { }; let server: ISuite['server']; + let serverURL: string; let code: ISuite['code']; let browser: ISuite['browser']; beforeAll(async () => { server = await startServer(); + serverURL = getServerURL(server); browser = await launchPuppeteer(); const bundlePath = path.resolve(__dirname, '../dist/rrweb.min.js'); @@ -500,7 +474,7 @@ describe('record integration tests', function (this: ISuite) { it('should nest record iframe', async () => { const page: puppeteer.Page = await browser.newPage(); - await page.goto(`http://localhost:3031/html`); + await page.goto(`${serverURL}/html`); await page.setContent(getHtml.call(this, 'main.html')); await page.waitForTimeout(500); diff --git a/packages/rrweb/test/utils.ts b/packages/rrweb/test/utils.ts index bab49946d4..8f08920723 100644 --- a/packages/rrweb/test/utils.ts +++ b/packages/rrweb/test/utils.ts @@ -7,6 +7,10 @@ import { } from '../src/types'; import * as puppeteer from 'puppeteer'; import { format } from 'prettier'; +import * as path from 'path'; +import * as http from 'http'; +import * as url from 'url'; +import * as fs from 'fs'; export async function launchPuppeteer() { return await puppeteer.launch({ @@ -15,10 +19,65 @@ export async function launchPuppeteer() { width: 1920, height: 1080, }, + devtools: true, args: ['--no-sandbox'], }); } +interface IMimeType { + [key: string]: string; +} + +export const startServer = (defaultPort: number = 3030) => + new Promise((resolve) => { + const mimeType: IMimeType = { + '.html': 'text/html', + '.js': 'text/javascript', + '.css': 'text/css', + }; + const s = http.createServer((req, res) => { + const parsedUrl = url.parse(req.url!); + const sanitizePath = path + .normalize(parsedUrl.pathname!) + .replace(/^(\.\.[\/\\])+/, ''); + let pathname = path.join(__dirname, sanitizePath); + + try { + const data = fs.readFileSync(pathname); + const ext = path.parse(pathname).ext; + res.setHeader('Content-type', mimeType[ext] || 'text/plain'); + res.setHeader('Access-Control-Allow-Origin', '*'); + res.setHeader('Access-Control-Allow-Methods', 'GET'); + res.setHeader('Access-Control-Allow-Headers', 'Content-type'); + setTimeout(() => { + res.end(data); + // mock delay + }, 100); + } catch (error) { + res.end(); + } + }); + s.listen() + .on('listening', () => { + resolve(s); + }) + .on('error', (e) => { + console.log('port in use, trying next one'); + s.listen().on('listening', () => { + resolve(s); + }); + }); + }); + +export function getServerURL(server: http.Server): string { + const address = server.address(); + if (address && typeof address !== 'string') { + return `http://localhost:${address.port}`; + } else { + return `${address}`; + } +} + /** * Puppeteer may cast random mouse move which make our tests flaky. * So we only do snapshot test with filtered events. From 88de0f018e39f7cd608b339ffd76ed8ac059ce9c Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Thu, 16 Dec 2021 16:22:16 +0100 Subject: [PATCH 50/93] turn off debug mode by default --- packages/rrweb/src/replay/canvas/webgl.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/rrweb/src/replay/canvas/webgl.ts b/packages/rrweb/src/replay/canvas/webgl.ts index 5540de40c6..ab094fad35 100644 --- a/packages/rrweb/src/replay/canvas/webgl.ts +++ b/packages/rrweb/src/replay/canvas/webgl.ts @@ -108,8 +108,10 @@ export default function webglMutation({ const args = mutation.args.map(deserializeArg); const result = original.apply(ctx, args); - // const debugMode = false; - const debugMode = true; + saveToWebGLVarMap(result); + + const debugMode = false; + // const debugMode = true; if (debugMode) { if (mutation.property === 'compileShader') { if (!ctx.getShaderParameter(args[0], ctx.COMPILE_STATUS)) @@ -136,8 +138,6 @@ export default function webglMutation({ ); } } - - saveToWebGLVarMap(result); } catch (error) { errorHandler(mutation, error); } From 317d2ab21b61efdd5b903a4a51ca5288082bcad6 Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Thu, 16 Dec 2021 17:10:49 +0100 Subject: [PATCH 51/93] Move pendingCanvasMutations to local object and fix if/else statement --- .../src/record/observers/canvas/webgl.ts | 40 ++++++++++++------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/packages/rrweb/src/record/observers/canvas/webgl.ts b/packages/rrweb/src/record/observers/canvas/webgl.ts index 1e5533704b..03fdc505ac 100644 --- a/packages/rrweb/src/record/observers/canvas/webgl.ts +++ b/packages/rrweb/src/record/observers/canvas/webgl.ts @@ -11,27 +11,28 @@ import { import { hookSetter, isBlocked, patch } from '../../../utils'; import { saveWebGLVar, serializeArgs } from './serialize-args'; -const pendingCanvasMutations = new Map< - HTMLCanvasElement, - canvasMutationParam[] ->(); +type pendingCanvasMutationsMap = Map; // FIXME: total hack here, we need to find a better way to do this function flushPendingCanvasMutations( + pendingCanvasMutations: pendingCanvasMutationsMap, cb: canvasMutationCallback, mirror: Mirror, ) { pendingCanvasMutations.forEach( (values: canvasMutationParam[], canvas: HTMLCanvasElement) => { const id = mirror.getId((canvas as unknown) as INode); - flushPendingCanvasMutationFor(canvas, id, cb); + flushPendingCanvasMutationFor(canvas, pendingCanvasMutations, id, cb); }, ); - requestAnimationFrame(() => flushPendingCanvasMutations(cb, mirror)); + requestAnimationFrame(() => + flushPendingCanvasMutations(pendingCanvasMutations, cb, mirror), + ); } function flushPendingCanvasMutationFor( canvas: HTMLCanvasElement, + pendingCanvasMutations: pendingCanvasMutationsMap, id: number, cb: canvasMutationCallback, ) { @@ -48,6 +49,7 @@ function patchGLPrototype( cb: canvasMutationCallback, blockClass: blockClass, mirror: Mirror, + pendingCanvasMutations: pendingCanvasMutationsMap, ): listenerHandler[] { const handlers: listenerHandler[] = []; @@ -86,15 +88,16 @@ function patchGLPrototype( pendingCanvasMutations .get(this.canvas as HTMLCanvasElement)! .push(mutation); - } else { - // flush all pending mutations - flushPendingCanvasMutationFor( - this.canvas as HTMLCanvasElement, - id, - cb, - ); - cb(mutation); } + } else { + // flush all pending mutations + flushPendingCanvasMutationFor( + this.canvas as HTMLCanvasElement, + pendingCanvasMutations, + id, + cb, + ); + cb(mutation); } } @@ -128,9 +131,13 @@ export default function initCanvasWebGLMutationObserver( mirror: Mirror, ): listenerHandler { const handlers: listenerHandler[] = []; + const pendingCanvasMutations: pendingCanvasMutationsMap = new Map(); + // TODO: replace me - requestAnimationFrame(() => flushPendingCanvasMutations(cb, mirror)); + requestAnimationFrame(() => + flushPendingCanvasMutations(pendingCanvasMutations, cb, mirror), + ); handlers.push( ...patchGLPrototype( @@ -139,6 +146,7 @@ export default function initCanvasWebGLMutationObserver( cb, blockClass, mirror, + pendingCanvasMutations, ), ); @@ -150,11 +158,13 @@ export default function initCanvasWebGLMutationObserver( cb, blockClass, mirror, + pendingCanvasMutations, ), ); } return () => { + pendingCanvasMutations.clear(); handlers.forEach((h) => h()); }; } From 8f862e78f17f835a72dded9bb1390bf749408340 Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Fri, 17 Dec 2021 12:49:21 +0100 Subject: [PATCH 52/93] Always save pending mutations --- packages/rrweb/src/record/observers/canvas/webgl.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/rrweb/src/record/observers/canvas/webgl.ts b/packages/rrweb/src/record/observers/canvas/webgl.ts index 03fdc505ac..7ce4101df8 100644 --- a/packages/rrweb/src/record/observers/canvas/webgl.ts +++ b/packages/rrweb/src/record/observers/canvas/webgl.ts @@ -84,11 +84,11 @@ function patchGLPrototype( this.canvas as HTMLCanvasElement, [], ); - - pendingCanvasMutations - .get(this.canvas as HTMLCanvasElement)! - .push(mutation); } + + pendingCanvasMutations + .get(this.canvas as HTMLCanvasElement)! + .push(mutation); } else { // flush all pending mutations flushPendingCanvasMutationFor( From 8c72988b9d846b9810398e66f1ed5498ea96ff81 Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Fri, 17 Dec 2021 13:46:54 +0100 Subject: [PATCH 53/93] only use assert snapshot as it's clearer whats going on --- packages/rrweb/test/record/webgl.test.ts | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/packages/rrweb/test/record/webgl.test.ts b/packages/rrweb/test/record/webgl.test.ts index 6abfa6c71c..b33b3eff85 100644 --- a/packages/rrweb/test/record/webgl.test.ts +++ b/packages/rrweb/test/record/webgl.test.ts @@ -164,15 +164,6 @@ describe('record webgl', function (this: ISuite) { }); await ctx.page.waitForTimeout(50); - - const lastEvent = ctx.events[ctx.events.length - 1]; - expect(lastEvent).toMatchObject({ - data: { - source: IncrementalSource.CanvasMutation, - type: CanvasContext.WebGL, - property: 'clear', - }, - }); assertSnapshot(ctx.events); }); @@ -197,14 +188,6 @@ describe('record webgl', function (this: ISuite) { await ctx.page.waitForTimeout(50); - const lastEvent = ctx.events[ctx.events.length - 1]; - expect(lastEvent).toMatchObject({ - data: { - source: IncrementalSource.CanvasMutation, - type: CanvasContext.WebGL2, - property: 'clear', - }, - }); assertSnapshot(ctx.events); }); From 67648cda87af5365b5ae24d4fd60ae06e1b6b15f Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Fri, 17 Dec 2021 14:50:44 +0100 Subject: [PATCH 54/93] Ugly fix for now --- packages/rrweb/test/record/webgl.test.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/rrweb/test/record/webgl.test.ts b/packages/rrweb/test/record/webgl.test.ts index b33b3eff85..32199da4ca 100644 --- a/packages/rrweb/test/record/webgl.test.ts +++ b/packages/rrweb/test/record/webgl.test.ts @@ -186,7 +186,11 @@ describe('record webgl', function (this: ISuite) { }, 10); }); - await ctx.page.waitForTimeout(50); + // FIXME: this is a terrible way of getting this test to pass. + // But I want to step over this for now. + // When `pendingCanvasMutations` isn't run on requestAnimationFrame, + // this should work again without timeout 500. + await ctx.page.waitForTimeout(500); assertSnapshot(ctx.events); }); From ceb97c6f3021c98002a0eab75a4c1c698cb165b4 Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Mon, 20 Dec 2021 15:01:20 +0100 Subject: [PATCH 55/93] Making the tests more DRY --- packages/rrweb/test/record/webgl.test.ts | 85 +++++------------------- 1 file changed, 18 insertions(+), 67 deletions(-) diff --git a/packages/rrweb/test/record/webgl.test.ts b/packages/rrweb/test/record/webgl.test.ts index 32199da4ca..ee0a04111b 100644 --- a/packages/rrweb/test/record/webgl.test.ts +++ b/packages/rrweb/test/record/webgl.test.ts @@ -56,6 +56,14 @@ const setup = function (this: ISuite, content: string): ISuite { }); ctx.page.on('console', (msg) => console.log('PAGE LOG:', msg.text())); + + await ctx.page.evaluate(() => { + const { record } = ((window as unknown) as IWindow).rrweb; + record({ + recordCanvas: true, + emit: ((window as unknown) as IWindow).emit, + }); + }); }); afterEach(async () => { @@ -85,15 +93,6 @@ describe('record webgl', function (this: ISuite) { ); it('will record changes to a canvas element', async () => { - await ctx.page.evaluate(() => { - const { record } = ((window as unknown) as IWindow).rrweb; - record({ - recordCanvas: true, - emit(event: eventWithTime) { - ((window as unknown) as IWindow).emit(event); - }, - }); - }); await ctx.page.evaluate(() => { var canvas = document.getElementById('canvas') as HTMLCanvasElement; var gl = canvas.getContext('webgl')!; @@ -116,15 +115,6 @@ describe('record webgl', function (this: ISuite) { }); it('will record changes to a webgl2 canvas element', async () => { - await ctx.page.evaluate(() => { - const { record } = ((window as unknown) as IWindow).rrweb; - record({ - recordCanvas: true, - emit(event: eventWithTime) { - ((window as unknown) as IWindow).emit(event); - }, - }); - }); await ctx.page.evaluate(() => { var canvas = document.getElementById('canvas') as HTMLCanvasElement; var gl = canvas.getContext('webgl2')!; @@ -147,13 +137,6 @@ describe('record webgl', function (this: ISuite) { }); it('will record changes to a canvas element before the canvas gets added', async () => { - await ctx.page.evaluate(() => { - const { record } = ((window as unknown) as IWindow).rrweb; - record({ - recordCanvas: true, - emit: ((window as unknown) as IWindow).emit, - }); - }); await ctx.page.evaluate(() => { var canvas = document.createElement('canvas'); var gl = canvas.getContext('webgl')!; @@ -169,22 +152,18 @@ describe('record webgl', function (this: ISuite) { it('will record changes to a canvas element before the canvas gets added (webgl2)', async () => { await ctx.page.evaluate(() => { - const { record } = ((window as unknown) as IWindow).rrweb; - record({ - recordCanvas: true, - emit: ((window as unknown) as IWindow).emit, + return new Promise((resolve) => { + var canvas = document.createElement('canvas'); + var gl = canvas.getContext('webgl2')!; + var program = gl.createProgram()!; + gl.linkProgram(program); + gl.clear(gl.COLOR_BUFFER_BIT); + setTimeout(() => { + document.body.appendChild(canvas); + resolve(); + }, 10); }); }); - await ctx.page.evaluate(() => { - var canvas = document.createElement('canvas'); - var gl = canvas.getContext('webgl2')!; - var program = gl.createProgram()!; - gl.linkProgram(program); - gl.clear(gl.COLOR_BUFFER_BIT); - setTimeout(() => { - document.body.appendChild(canvas); - }, 10); - }); // FIXME: this is a terrible way of getting this test to pass. // But I want to step over this for now. @@ -196,13 +175,6 @@ describe('record webgl', function (this: ISuite) { }); it('will record webgl variables', async () => { - await ctx.page.evaluate(() => { - const { record } = ((window as unknown) as IWindow).rrweb; - record({ - recordCanvas: true, - emit: ((window as unknown) as IWindow).emit, - }); - }); await ctx.page.evaluate(() => { var canvas = document.getElementById('canvas') as HTMLCanvasElement; var gl = canvas.getContext('webgl')!; @@ -231,13 +203,6 @@ describe('record webgl', function (this: ISuite) { }); it('will record webgl variables in reverse order', async () => { - await ctx.page.evaluate(() => { - const { record } = ((window as unknown) as IWindow).rrweb; - record({ - recordCanvas: true, - emit: ((window as unknown) as IWindow).emit, - }); - }); await ctx.page.evaluate(() => { var canvas = document.getElementById('canvas') as HTMLCanvasElement; var gl = canvas.getContext('webgl')!; @@ -266,13 +231,6 @@ describe('record webgl', function (this: ISuite) { }); it('sets _context on canvas.getContext()', async () => { - await ctx.page.evaluate(() => { - const { record } = ((window as unknown) as IWindow).rrweb; - record({ - recordCanvas: true, - emit: ((window as unknown) as IWindow).emit, - }); - }); const context = await ctx.page.evaluate(() => { var canvas = document.getElementById('canvas') as HTMLCanvasElement; canvas.getContext('webgl')!; @@ -283,13 +241,6 @@ describe('record webgl', function (this: ISuite) { }); it('only sets _context on first canvas.getContext() call', async () => { - await ctx.page.evaluate(() => { - const { record } = ((window as unknown) as IWindow).rrweb; - record({ - recordCanvas: true, - emit: ((window as unknown) as IWindow).emit, - }); - }); const context = await ctx.page.evaluate(() => { var canvas = document.getElementById('canvas') as HTMLCanvasElement; canvas.getContext('webgl'); From 5b37d2eabf9990fba041cc4d817db2952c478288 Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Mon, 20 Dec 2021 15:42:31 +0100 Subject: [PATCH 56/93] flush at the end of each request animation frame --- .../src/record/observers/canvas/webgl.ts | 22 ++- packages/rrweb/src/replay/canvas/webgl.ts | 2 + packages/rrweb/src/types.ts | 1 + .../record/__snapshots__/webgl.test.ts.snap | 153 ++++++++++++++++++ packages/rrweb/test/record/webgl.test.ts | 24 +++ packages/rrweb/typings/types.d.ts | 1 + 6 files changed, 202 insertions(+), 1 deletion(-) diff --git a/packages/rrweb/src/record/observers/canvas/webgl.ts b/packages/rrweb/src/record/observers/canvas/webgl.ts index 7ce4101df8..fb85c58731 100644 --- a/packages/rrweb/src/record/observers/canvas/webgl.ts +++ b/packages/rrweb/src/record/observers/canvas/webgl.ts @@ -12,6 +12,7 @@ import { hookSetter, isBlocked, patch } from '../../../utils'; import { saveWebGLVar, serializeArgs } from './serialize-args'; type pendingCanvasMutationsMap = Map; +type rafStampsType = { latestId: number; invokeId: number | null }; // FIXME: total hack here, we need to find a better way to do this function flushPendingCanvasMutations( @@ -50,6 +51,7 @@ function patchGLPrototype( blockClass: blockClass, mirror: Mirror, pendingCanvasMutations: pendingCanvasMutationsMap, + rafStamps: rafStampsType, ): listenerHandler[] { const handlers: listenerHandler[] = []; @@ -62,18 +64,24 @@ function patchGLPrototype( } const restoreHandler = patch(prototype, prop, function (original) { return function (this: typeof prototype, ...args: Array) { + const newFrame = + rafStamps.invokeId && rafStamps.latestId !== rafStamps.invokeId; + if (newFrame || !rafStamps.invokeId) + rafStamps.invokeId = rafStamps.latestId; + const result = original.apply(this, args); saveWebGLVar(result); if (!isBlocked((this.canvas as unknown) as INode, blockClass)) { const id = mirror.getId((this.canvas as unknown) as INode); const recordArgs = serializeArgs([...args]); - const mutation = { + const mutation: canvasMutationParam = { id, type, property: prop, args: recordArgs, }; + if (newFrame) mutation.newFrame = true; if (id === -1) { // FIXME! THIS COULD MAYBE BE AN OFFSCREEN CANVAS @@ -133,6 +141,16 @@ export default function initCanvasWebGLMutationObserver( const handlers: listenerHandler[] = []; const pendingCanvasMutations: pendingCanvasMutationsMap = new Map(); + const rafStamps: rafStampsType = { + latestId: 0, + invokeId: null, + }; + + const setLatestRAFTimestamp = (timestamp: DOMHighResTimeStamp) => { + rafStamps.latestId = timestamp; + requestAnimationFrame(setLatestRAFTimestamp); + }; + requestAnimationFrame(setLatestRAFTimestamp); // TODO: replace me requestAnimationFrame(() => @@ -147,6 +165,7 @@ export default function initCanvasWebGLMutationObserver( blockClass, mirror, pendingCanvasMutations, + rafStamps, ), ); @@ -159,6 +178,7 @@ export default function initCanvasWebGLMutationObserver( blockClass, mirror, pendingCanvasMutations, + rafStamps, ), ); } diff --git a/packages/rrweb/src/replay/canvas/webgl.ts b/packages/rrweb/src/replay/canvas/webgl.ts index ab094fad35..3465e0811c 100644 --- a/packages/rrweb/src/replay/canvas/webgl.ts +++ b/packages/rrweb/src/replay/canvas/webgl.ts @@ -96,6 +96,8 @@ export default function webglMutation({ const ctx = getContext(target, mutation.type); if (!ctx) return; + if (mutation.newFrame) ctx.flush(); // flush to emulate the ending of the last request animation frame + if (mutation.setter) { // skip some read-only type checks // tslint:disable-next-line:no-any diff --git a/packages/rrweb/src/types.ts b/packages/rrweb/src/types.ts index bed9ba50b0..3ecd81c595 100644 --- a/packages/rrweb/src/types.ts +++ b/packages/rrweb/src/types.ts @@ -467,6 +467,7 @@ export type canvasMutationParam = { property: string; args: Array; setter?: true; + newFrame?: true; }; export type fontParam = { diff --git a/packages/rrweb/test/record/__snapshots__/webgl.test.ts.snap b/packages/rrweb/test/record/__snapshots__/webgl.test.ts.snap index 5638ec5d7c..b0ce47df14 100644 --- a/packages/rrweb/test/record/__snapshots__/webgl.test.ts.snap +++ b/packages/rrweb/test/record/__snapshots__/webgl.test.ts.snap @@ -1,5 +1,158 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP +exports[`record webgl should mark every RAF as \`newFrame: true\` 1`] = ` +"[ + { + \\"type\\": 4, + \\"data\\": { + \\"href\\": \\"about:blank\\", + \\"width\\": 1920, + \\"height\\": 1080 + } + }, + { + \\"type\\": 2, + \\"data\\": { + \\"node\\": { + \\"type\\": 0, + \\"childNodes\\": [ + { + \\"type\\": 1, + \\"name\\": \\"html\\", + \\"publicId\\": \\"\\", + \\"systemId\\": \\"\\", + \\"id\\": 2 + }, + { + \\"type\\": 2, + \\"tagName\\": \\"html\\", + \\"attributes\\": {}, + \\"childNodes\\": [ + { + \\"type\\": 2, + \\"tagName\\": \\"head\\", + \\"attributes\\": {}, + \\"childNodes\\": [], + \\"id\\": 4 + }, + { + \\"type\\": 2, + \\"tagName\\": \\"body\\", + \\"attributes\\": {}, + \\"childNodes\\": [ + { + \\"type\\": 3, + \\"textContent\\": \\"\\\\n \\", + \\"id\\": 6 + }, + { + \\"type\\": 2, + \\"tagName\\": \\"canvas\\", + \\"attributes\\": { + \\"id\\": \\"canvas\\" + }, + \\"childNodes\\": [], + \\"id\\": 7 + }, + { + \\"type\\": 3, + \\"textContent\\": \\"\\\\n \\\\n \\\\n \\", + \\"id\\": 8 + } + ], + \\"id\\": 5 + } + ], + \\"id\\": 3 + } + ], + \\"id\\": 1 + }, + \\"initialOffset\\": { + \\"left\\": 0, + \\"top\\": 0 + } + } + }, + { + \\"type\\": 3, + \\"data\\": { + \\"source\\": 9, + \\"id\\": 7, + \\"type\\": 1, + \\"property\\": \\"createProgram\\", + \\"args\\": [] + } + }, + { + \\"type\\": 3, + \\"data\\": { + \\"source\\": 9, + \\"id\\": 7, + \\"type\\": 1, + \\"property\\": \\"linkProgram\\", + \\"args\\": [ + { + \\"rr_type\\": \\"WebGLProgram\\", + \\"index\\": 0 + } + ] + } + }, + { + \\"type\\": 3, + \\"data\\": { + \\"source\\": 9, + \\"id\\": 7, + \\"type\\": 1, + \\"property\\": \\"createProgram\\", + \\"args\\": [], + \\"newFrame\\": true + } + }, + { + \\"type\\": 3, + \\"data\\": { + \\"source\\": 9, + \\"id\\": 7, + \\"type\\": 1, + \\"property\\": \\"linkProgram\\", + \\"args\\": [ + { + \\"rr_type\\": \\"WebGLProgram\\", + \\"index\\": 1 + } + ] + } + }, + { + \\"type\\": 3, + \\"data\\": { + \\"source\\": 9, + \\"id\\": 7, + \\"type\\": 1, + \\"property\\": \\"clear\\", + \\"args\\": [ + 16384 + ] + } + }, + { + \\"type\\": 3, + \\"data\\": { + \\"source\\": 9, + \\"id\\": 7, + \\"type\\": 1, + \\"property\\": \\"clear\\", + \\"args\\": [ + 16384 + ], + \\"newFrame\\": true + } + } +]" +`; + exports[`record webgl will record changes to a canvas element 1`] = ` "[ { diff --git a/packages/rrweb/test/record/webgl.test.ts b/packages/rrweb/test/record/webgl.test.ts index ee0a04111b..bf22566c6e 100644 --- a/packages/rrweb/test/record/webgl.test.ts +++ b/packages/rrweb/test/record/webgl.test.ts @@ -250,4 +250,28 @@ describe('record webgl', function (this: ISuite) { expect(context).toBe('webgl'); }); + + it('should mark every RAF as `newFrame: true`', async () => { + await ctx.page.evaluate(() => { + return new Promise((resolve) => { + const canvas = document.getElementById('canvas') as HTMLCanvasElement; + const gl = canvas.getContext('webgl') as WebGLRenderingContext; + const program = gl.createProgram()!; + gl.linkProgram(program); + requestAnimationFrame(() => { + const program2 = gl.createProgram()!; + gl.linkProgram(program2); + gl.clear(gl.COLOR_BUFFER_BIT); + requestAnimationFrame(() => { + gl.clear(gl.COLOR_BUFFER_BIT); + resolve(); + }); + }); + }); + }); + + await ctx.page.waitForTimeout(50); + + assertSnapshot(ctx.events); + }); }); diff --git a/packages/rrweb/typings/types.d.ts b/packages/rrweb/typings/types.d.ts index 44eaa68e26..508af81702 100644 --- a/packages/rrweb/typings/types.d.ts +++ b/packages/rrweb/typings/types.d.ts @@ -344,6 +344,7 @@ export declare type canvasMutationParam = { property: string; args: Array; setter?: true; + newFrame?: true; }; export declare type fontParam = { family: string; From abf7064d1e38b7c38004a41bf2a0e6ecb148159c Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Mon, 20 Dec 2021 15:49:25 +0100 Subject: [PATCH 57/93] Looks like the promise made this test more predictable --- packages/rrweb/test/record/webgl.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/rrweb/test/record/webgl.test.ts b/packages/rrweb/test/record/webgl.test.ts index bf22566c6e..7a5d5af1e6 100644 --- a/packages/rrweb/test/record/webgl.test.ts +++ b/packages/rrweb/test/record/webgl.test.ts @@ -168,8 +168,8 @@ describe('record webgl', function (this: ISuite) { // FIXME: this is a terrible way of getting this test to pass. // But I want to step over this for now. // When `pendingCanvasMutations` isn't run on requestAnimationFrame, - // this should work again without timeout 500. - await ctx.page.waitForTimeout(500); + // this should work again without timeout 100. + await ctx.page.waitForTimeout(100); assertSnapshot(ctx.events); }); From ee6ec1fc9690731cf178d86046774dc93dce26a0 Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Tue, 21 Dec 2021 16:20:40 +0100 Subject: [PATCH 58/93] add waitForRAF --- packages/rrweb/test/record/webgl.test.ts | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/packages/rrweb/test/record/webgl.test.ts b/packages/rrweb/test/record/webgl.test.ts index 7a5d5af1e6..c3e5aaf876 100644 --- a/packages/rrweb/test/record/webgl.test.ts +++ b/packages/rrweb/test/record/webgl.test.ts @@ -77,6 +77,16 @@ const setup = function (this: ISuite, content: string): ISuite { return ctx; }; +async function waitForRAF(page: puppeteer.Page) { + return await page.evaluate(() => { + return new Promise((resolve) => { + requestAnimationFrame(() => { + requestAnimationFrame(resolve); + }); + }); + }); +} + describe('record webgl', function (this: ISuite) { jest.setTimeout(100_000); @@ -146,7 +156,8 @@ describe('record webgl', function (this: ISuite) { document.body.appendChild(canvas); }); - await ctx.page.waitForTimeout(50); + await waitForRAF(ctx.page); + assertSnapshot(ctx.events); }); @@ -165,11 +176,10 @@ describe('record webgl', function (this: ISuite) { }); }); - // FIXME: this is a terrible way of getting this test to pass. - // But I want to step over this for now. + // FIXME: this wait deeply couples the test to the implementation // When `pendingCanvasMutations` isn't run on requestAnimationFrame, - // this should work again without timeout 100. - await ctx.page.waitForTimeout(100); + // we need to change this + await waitForRAF(ctx.page); assertSnapshot(ctx.events); }); From 21e55bea7aae57b79592286467f536c7dd77a782 Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Tue, 21 Dec 2021 16:50:06 +0100 Subject: [PATCH 59/93] Make nested iframe recording robust no matter the test speed --- packages/rrweb/test/integration.test.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/rrweb/test/integration.test.ts b/packages/rrweb/test/integration.test.ts index a50bda2831..b83bf09a4a 100644 --- a/packages/rrweb/test/integration.test.ts +++ b/packages/rrweb/test/integration.test.ts @@ -477,7 +477,14 @@ describe('record integration tests', function (this: ISuite) { await page.goto(`${serverURL}/html`); await page.setContent(getHtml.call(this, 'main.html')); - await page.waitForTimeout(500); + await page.waitForSelector('#two'); + const frameIdTwo = await page.frames()[2]; + await frameIdTwo.waitForSelector('#four'); + const frameIdFour = frameIdTwo.childFrames()[1]; + await frameIdFour.waitForSelector('#five'); + + await page.waitForTimeout(50); + const snapshots = await page.evaluate('window.snapshots'); assertSnapshot(snapshots); }); From 662fae59661c85d8ab815eabdeff9da3ace0a5fe Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Tue, 21 Dec 2021 16:55:54 +0100 Subject: [PATCH 60/93] mute noisy error in test --- packages/rrweb/test/packer.test.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packages/rrweb/test/packer.test.ts b/packages/rrweb/test/packer.test.ts index f6ffaca92f..08b35c6cca 100644 --- a/packages/rrweb/test/packer.test.ts +++ b/packages/rrweb/test/packer.test.ts @@ -27,7 +27,14 @@ describe('unpack', () => { }); it('stop on unknown data format', () => { + const consoleSpy = jest + .spyOn(console, 'error') + .mockImplementation(() => {}); + expect(() => unpack('[""]')).toThrow(''); + + expect(consoleSpy).toHaveBeenCalled(); + jest.resetAllMocks(); }); it('can unpack packed data', () => { From 1844311391de435173881f78859e6af8f80e3494 Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Tue, 21 Dec 2021 17:39:52 +0100 Subject: [PATCH 61/93] force a requestAnimationFrame --- packages/rrweb/src/replay/index.ts | 2 ++ packages/rrweb/src/replay/machine.ts | 4 ++++ packages/rrweb/src/replay/timer.ts | 5 ++++- packages/rrweb/src/types.ts | 1 + packages/rrweb/typings/types.d.ts | 1 + 5 files changed, 12 insertions(+), 1 deletion(-) diff --git a/packages/rrweb/src/replay/index.ts b/packages/rrweb/src/replay/index.ts index aa1d05db7b..94521b7068 100644 --- a/packages/rrweb/src/replay/index.ts +++ b/packages/rrweb/src/replay/index.ts @@ -880,6 +880,7 @@ export class Replayer { p.timeOffset + e.timestamp - this.service.state.context.baselineTime, + newFrame: false, }; this.timer.addAction(action); }); @@ -887,6 +888,7 @@ export class Replayer { this.timer.addAction({ doAction() {}, delay: e.delay! - d.positions[0]?.timeOffset, + newFrame: false, }); } break; diff --git a/packages/rrweb/src/replay/machine.ts b/packages/rrweb/src/replay/machine.ts index 2dbc512b9c..86e0dce152 100644 --- a/packages/rrweb/src/replay/machine.ts +++ b/packages/rrweb/src/replay/machine.ts @@ -207,6 +207,8 @@ export function createPlayerService( emitter.emit(ReplayerEvents.EventCast, event); }, delay: event.delay!, + newFrame: + ('newFrame' in event.data && event.data.newFrame) || false, }); } } @@ -272,6 +274,8 @@ export function createPlayerService( emitter.emit(ReplayerEvents.EventCast, event); }, delay: event.delay!, + newFrame: + ('newFrame' in event.data && event.data.newFrame) || false, }); } } diff --git a/packages/rrweb/src/replay/timer.ts b/packages/rrweb/src/replay/timer.ts index 684c3ff06c..cec6652ece 100644 --- a/packages/rrweb/src/replay/timer.ts +++ b/packages/rrweb/src/replay/timer.ts @@ -44,7 +44,10 @@ export class Timer { lastTimestamp = time; while (actions.length) { const action = actions[0]; - if (self.timeOffset >= action.delay) { + if (action.newFrame) { + action.newFrame = false; + break; + } else if (self.timeOffset >= action.delay) { actions.shift(); action.doAction(); } else { diff --git a/packages/rrweb/src/types.ts b/packages/rrweb/src/types.ts index 3ecd81c595..a0700ba645 100644 --- a/packages/rrweb/src/types.ts +++ b/packages/rrweb/src/types.ts @@ -589,6 +589,7 @@ export type missingNodeMap = { export type actionWithDelay = { doAction: () => void; delay: number; + newFrame: boolean; }; export type Handler = (event?: unknown) => void; diff --git a/packages/rrweb/typings/types.d.ts b/packages/rrweb/typings/types.d.ts index 508af81702..fe7c8fe23c 100644 --- a/packages/rrweb/typings/types.d.ts +++ b/packages/rrweb/typings/types.d.ts @@ -440,6 +440,7 @@ export declare type missingNodeMap = { export declare type actionWithDelay = { doAction: () => void; delay: number; + newFrame: boolean; }; export declare type Handler = (event?: unknown) => void; export declare type Emitter = { From 114a00d44ceb44306426dc9ce96e600ad08e9c5c Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Wed, 22 Dec 2021 12:56:11 +0100 Subject: [PATCH 62/93] Bundle events within one frame together as much as possible WebGL events need to be bundled together as much as possible so they don't accidentally get split over multiple animation frames. `newFrame: true` is used to indicate the start of an new animation frame in the recording, and that the event shouldn't be bundled with the previous events. --- packages/rrweb/src/replay/machine.ts | 6 ++++-- packages/rrweb/src/replay/timer.ts | 26 +++++++++++++++++++++++++- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/packages/rrweb/src/replay/machine.ts b/packages/rrweb/src/replay/machine.ts index 86e0dce152..49dd96ef41 100644 --- a/packages/rrweb/src/replay/machine.ts +++ b/packages/rrweb/src/replay/machine.ts @@ -8,7 +8,7 @@ import { Emitter, IncrementalSource, } from '../types'; -import { Timer, addDelay } from './timer'; +import { Timer, addDelay, LastDelay } from './timer'; export type PlayerContext = { events: eventWithTime[]; @@ -167,9 +167,11 @@ export function createPlayerService( play(ctx) { const { timer, events, baselineTime, lastPlayedEvent } = ctx; timer.clear(); + + const lastDelay: LastDelay = { at: null }; for (const event of events) { // TODO: improve this API - addDelay(event, baselineTime); + addDelay(event, baselineTime, lastDelay); } const neededEvents = discardPriorSnapshots(events, baselineTime); diff --git a/packages/rrweb/src/replay/timer.ts b/packages/rrweb/src/replay/timer.ts index cec6652ece..3638351263 100644 --- a/packages/rrweb/src/replay/timer.ts +++ b/packages/rrweb/src/replay/timer.ts @@ -100,8 +100,14 @@ export class Timer { } } +export type LastDelay = { at: number | null }; + // TODO: add speed to mouse move timestamp calculation -export function addDelay(event: eventWithTime, baselineTime: number): number { +export function addDelay( + event: eventWithTime, + baselineTime: number, + lastDelay?: LastDelay, +): number { // Mouse move events was recorded in a throttle function, // so we need to find the real timestamp by traverse the time offsets. if ( @@ -114,6 +120,24 @@ export function addDelay(event: eventWithTime, baselineTime: number): number { event.delay = firstTimestamp - baselineTime; return firstTimestamp - baselineTime; } + + if (lastDelay) { + // WebGL events need to be bundled together as much as possible so they don't + // accidentally get split over multiple animation frames. + if ( + event.type === EventType.IncrementalSnapshot && + event.data.source === IncrementalSource.CanvasMutation && + // `newFrame: true` is used to indicate the start of an new animation frame in the recording, + // and that the event shouldn't be bundled with the previous events. + !event.data.newFrame && + lastDelay.at + ) { + event.delay = lastDelay.at; + } else { + lastDelay.at = event.delay!; + } + } + event.delay = event.timestamp - baselineTime; return event.delay; } From 8058c3ae4b08de2d75019cf09ffce03a7ae2fb41 Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Wed, 22 Dec 2021 12:58:15 +0100 Subject: [PATCH 63/93] Rename RafStamps --- packages/rrweb/src/record/observers/canvas/webgl.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/rrweb/src/record/observers/canvas/webgl.ts b/packages/rrweb/src/record/observers/canvas/webgl.ts index fb85c58731..d93630676a 100644 --- a/packages/rrweb/src/record/observers/canvas/webgl.ts +++ b/packages/rrweb/src/record/observers/canvas/webgl.ts @@ -12,7 +12,7 @@ import { hookSetter, isBlocked, patch } from '../../../utils'; import { saveWebGLVar, serializeArgs } from './serialize-args'; type pendingCanvasMutationsMap = Map; -type rafStampsType = { latestId: number; invokeId: number | null }; +type RafStamps = { latestId: number; invokeId: number | null }; // FIXME: total hack here, we need to find a better way to do this function flushPendingCanvasMutations( @@ -51,7 +51,7 @@ function patchGLPrototype( blockClass: blockClass, mirror: Mirror, pendingCanvasMutations: pendingCanvasMutationsMap, - rafStamps: rafStampsType, + rafStamps: RafStamps, ): listenerHandler[] { const handlers: listenerHandler[] = []; @@ -141,7 +141,7 @@ export default function initCanvasWebGLMutationObserver( const handlers: listenerHandler[] = []; const pendingCanvasMutations: pendingCanvasMutationsMap = new Map(); - const rafStamps: rafStampsType = { + const rafStamps: RafStamps = { latestId: 0, invokeId: null, }; From fbaf1880cd8096f05a323d81c7293ce638dd4faf Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Wed, 22 Dec 2021 14:44:08 +0100 Subject: [PATCH 64/93] Override event.delay --- packages/rrweb/src/replay/timer.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/rrweb/src/replay/timer.ts b/packages/rrweb/src/replay/timer.ts index 3638351263..9d79a41be0 100644 --- a/packages/rrweb/src/replay/timer.ts +++ b/packages/rrweb/src/replay/timer.ts @@ -133,6 +133,7 @@ export function addDelay( lastDelay.at ) { event.delay = lastDelay.at; + return event.delay; } else { lastDelay.at = event.delay!; } From 791715824012e367c87ec9b7c7cce24df6fa371a Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Wed, 22 Dec 2021 15:23:17 +0100 Subject: [PATCH 65/93] cleanup --- packages/rrweb/src/replay/canvas/webgl.ts | 7 ++++++- packages/rrweb/src/replay/timer.ts | 4 +++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/rrweb/src/replay/canvas/webgl.ts b/packages/rrweb/src/replay/canvas/webgl.ts index 3465e0811c..68937a9008 100644 --- a/packages/rrweb/src/replay/canvas/webgl.ts +++ b/packages/rrweb/src/replay/canvas/webgl.ts @@ -19,6 +19,9 @@ function getContext( target: HTMLCanvasElement, type: CanvasContext, ): WebGLRenderingContext | WebGL2RenderingContext | null { + // Note to whomever is going to implement support for `contextAttributes`: + // if `preserveDrawingBuffer` is set to true, + // you have to do `ctx.flush()` before every `newFrame: true` try { if (type === CanvasContext.WebGL) { return ( @@ -96,7 +99,9 @@ export default function webglMutation({ const ctx = getContext(target, mutation.type); if (!ctx) return; - if (mutation.newFrame) ctx.flush(); // flush to emulate the ending of the last request animation frame + // NOTE: if `preserveDrawingBuffer` is set to true, + // we must flush the buffers on every newFrame: true + // if (mutation.newFrame) ctx.flush(); if (mutation.setter) { // skip some read-only type checks diff --git a/packages/rrweb/src/replay/timer.ts b/packages/rrweb/src/replay/timer.ts index 9d79a41be0..885aa92a28 100644 --- a/packages/rrweb/src/replay/timer.ts +++ b/packages/rrweb/src/replay/timer.ts @@ -47,7 +47,9 @@ export class Timer { if (action.newFrame) { action.newFrame = false; break; - } else if (self.timeOffset >= action.delay) { + } + + if (self.timeOffset >= action.delay) { actions.shift(); action.doAction(); } else { From 3d1ff8125c26ff9b57651f6cbb53024f5b6b5492 Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Wed, 22 Dec 2021 16:40:41 +0100 Subject: [PATCH 66/93] Add tests for addDelay --- packages/rrweb/src/replay/timer.ts | 5 +- packages/rrweb/test/replay/timer.test.ts | 61 ++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 2 deletions(-) create mode 100644 packages/rrweb/test/replay/timer.test.ts diff --git a/packages/rrweb/src/replay/timer.ts b/packages/rrweb/src/replay/timer.ts index 885aa92a28..1fdbdb0208 100644 --- a/packages/rrweb/src/replay/timer.ts +++ b/packages/rrweb/src/replay/timer.ts @@ -123,6 +123,8 @@ export function addDelay( return firstTimestamp - baselineTime; } + event.delay = event.timestamp - baselineTime; + if (lastDelay) { // WebGL events need to be bundled together as much as possible so they don't // accidentally get split over multiple animation frames. @@ -134,13 +136,12 @@ export function addDelay( !event.data.newFrame && lastDelay.at ) { + // Override the current delay with the last delay event.delay = lastDelay.at; - return event.delay; } else { lastDelay.at = event.delay!; } } - event.delay = event.timestamp - baselineTime; return event.delay; } diff --git a/packages/rrweb/test/replay/timer.test.ts b/packages/rrweb/test/replay/timer.test.ts new file mode 100644 index 0000000000..304f0e7b31 --- /dev/null +++ b/packages/rrweb/test/replay/timer.test.ts @@ -0,0 +1,61 @@ +import { addDelay, LastDelay } from '../../src/replay/timer'; +import { + CanvasContext, + EventType, + eventWithTime, + IncrementalSource, +} from '../../src/types'; + +const canvasMutationEventWithTime = ( + timestamp: number, + newFrame?: boolean, +): eventWithTime => { + const newFrameObj: { newFrame: true } | {} = newFrame + ? { newFrame: true } + : {}; + + return { + timestamp, + type: EventType.IncrementalSnapshot, + data: { + source: IncrementalSource.CanvasMutation, + property: 'x', + args: [], + id: 1, + type: CanvasContext.WebGL, + ...newFrameObj, + }, + }; +}; + +describe('addDelay', () => { + let baselineTime: number; + let lastDelay: LastDelay; + beforeEach(() => { + baselineTime = 0; + lastDelay = { + at: null, + }; + }); + it('should bundle canvas mutations together', () => { + const event1 = canvasMutationEventWithTime(1000); + const event2 = canvasMutationEventWithTime(1001); + addDelay(event1, baselineTime, lastDelay); + addDelay(event2, baselineTime, lastDelay); + expect(event2.delay).toBe(1000); + }); + + it('should bundle canvas mutations on the same frame together', () => { + const event1 = canvasMutationEventWithTime(1000); + const event2 = canvasMutationEventWithTime(1001); + const event3 = canvasMutationEventWithTime(1002, true); + const event4 = canvasMutationEventWithTime(1003); + addDelay(event1, baselineTime, lastDelay); + addDelay(event2, baselineTime, lastDelay); + addDelay(event3, baselineTime, lastDelay); + addDelay(event4, baselineTime, lastDelay); + expect(event2.delay).toBe(1000); + expect(event3.delay).toBe(1002); + expect(event4.delay).toBe(1002); + }); +}); From 0a2690e4055d9eebe2ac9fd92f9b784ff8c99eb5 Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Wed, 22 Dec 2021 17:10:02 +0100 Subject: [PATCH 67/93] Add webgl e2e test --- ...ecord-and-replay-a-webgl-square-1-snap.png | Bin 0 -> 10812 bytes packages/rrweb/test/e2e/webgl.test.ts | 133 ++++++++++++++++++ .../rrweb/test/html/canvas-webgl-square.html | 115 +++++++++++++++ 3 files changed, 248 insertions(+) create mode 100644 packages/rrweb/test/e2e/__image_snapshots__/webgl-test-ts-e-2-e-webgl-will-record-and-replay-a-webgl-square-1-snap.png create mode 100644 packages/rrweb/test/e2e/webgl.test.ts create mode 100644 packages/rrweb/test/html/canvas-webgl-square.html diff --git a/packages/rrweb/test/e2e/__image_snapshots__/webgl-test-ts-e-2-e-webgl-will-record-and-replay-a-webgl-square-1-snap.png b/packages/rrweb/test/e2e/__image_snapshots__/webgl-test-ts-e-2-e-webgl-will-record-and-replay-a-webgl-square-1-snap.png new file mode 100644 index 0000000000000000000000000000000000000000..e22bc48831104120ac0c4a19cd3b9f41dc62b426 GIT binary patch literal 10812 zcmeAS@N?(olHy`uVBq!ia0y~yU~geyV6ov~1Bz6wRMrPljKx9jP7LeL$-HD>P+;(M zaSW-L^XAUR+`|S8u8vwCOxA3k^!vd>vQoAdb7--lIu=j~*Nh&If%k?S`+Ch<=I$!Ol9Sr{Z><@wFZ*LC!v-SLD_78`D z-()Te+9?l_PN^`GBF1PUT`;N)9GE1>+-SIsh8zC8jX#Cx?YFA6&S3q2`}j>}NOoTE z@~Og}YgR?jVur!`bBYN>#$k#A4}>M4$^ljF!g&ZH#1PnIFsh7!VKhL%sbDmPfP-N) z%RtJ6(ZT^74x@#`XyGtgQ9_D@(Y!F47e@2KXkK6#Z6iSv!Dt%^91f%P!bq(bK3ttA zJB5LVg^?%5>*|{N`1FNpT7C00}Tl=sjP@{zdC_#)C4&ZDtTJ3^^VYIam z4hBkw0$$CXdGoI2^E-Laa^`{i`FH2u-FpWynBhUXylc0-o#vIc*B}v3S3j3^P6 { + let code: ISuite['code']; + let page: ISuite['page']; + let browser: ISuite['browser']; + let server: ISuite['server']; + let serverURL: ISuite['serverURL']; + + beforeAll(async () => { + server = await startServer(); + serverURL = getServerURL(server); + browser = await launchPuppeteer(); + + const bundlePath = path.resolve(__dirname, '../../dist/rrweb.min.js'); + code = fs.readFileSync(bundlePath, 'utf8'); + + }); + + afterEach(async () => { + await page.close(); + }); + + afterAll(async () => { + await browser.close(); + }); + + const getHtml = ( + fileName: string, + options: recordOptions = {}, + ): string => { + const filePath = path.resolve(__dirname, `../html/${fileName}`); + const html = fs.readFileSync(filePath, 'utf8'); + return html.replace( + '', + ` + + + `, + ); + }; + + const fakeGoto = async (page: puppeteer.Page, url: string) => { + const intercept = async (request: puppeteer.HTTPRequest) => { + await request.respond({ + status: 200, + contentType: 'text/html', + body: ' ', // non-empty string or page will load indefinitely + }); + }; + page.setRequestInterception(true); + page.on('request', intercept); + await page.goto(url); + page.setRequestInterception(false); + page.off('request', intercept); + }; + + const hideMouseAnimation = async (page: puppeteer.Page) => { + await page.addStyleTag({ + content: '.replayer-mouse-tail{display: none !important;}', + }); + }; + + it('will record and replay a webgl square', async () => { + page = await browser.newPage(); + await fakeGoto(page, `${serverURL}/html/canvas-webgl-square.html`); + + await page.setContent( + getHtml.call(this, 'canvas-webgl-square.html', { recordCanvas: true }), + ); + + await page.waitForTimeout(100); + const snapshots: eventWithTime[] = await page.evaluate('window.snapshots'); + + page = await browser.newPage(); + + await page.goto('about:blank'); + await page.evaluate(code); + + await hideMouseAnimation(page); + await page.evaluate(`let events = ${JSON.stringify(snapshots)}`); + await page.evaluate(` + const { Replayer } = rrweb; + const replayer = new Replayer(events, { + UNSAFE_replayCanvas: true, + }); + replayer.play(500); + `); + await page.waitForTimeout(50); + + const element = await page.$('iframe'); + const frameImage = await element!.screenshot(); + + expect(frameImage).toMatchImageSnapshot(); + }); +}); diff --git a/packages/rrweb/test/html/canvas-webgl-square.html b/packages/rrweb/test/html/canvas-webgl-square.html new file mode 100644 index 0000000000..272ecd4d36 --- /dev/null +++ b/packages/rrweb/test/html/canvas-webgl-square.html @@ -0,0 +1,115 @@ + + + + + + canvas webgl square + + + + + + + + + From dd8054a7bcc9e1fedb50a2de87d21a28a512e1fe Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Wed, 22 Dec 2021 17:12:32 +0100 Subject: [PATCH 68/93] Remove settimeout --- .../rrweb/test/html/canvas-webgl-square.html | 151 +++++++++--------- 1 file changed, 73 insertions(+), 78 deletions(-) diff --git a/packages/rrweb/test/html/canvas-webgl-square.html b/packages/rrweb/test/html/canvas-webgl-square.html index 272ecd4d36..cbdc7ec62e 100644 --- a/packages/rrweb/test/html/canvas-webgl-square.html +++ b/packages/rrweb/test/html/canvas-webgl-square.html @@ -32,84 +32,79 @@ } From 9b0d849ced21be1c426a9f25e27b7232108820ea Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Thu, 23 Dec 2021 15:08:16 +0100 Subject: [PATCH 69/93] DRY-up test --- .../test/replay/deserialize-args.test.ts | 54 +------------------ 1 file changed, 2 insertions(+), 52 deletions(-) diff --git a/packages/rrweb/test/replay/deserialize-args.test.ts b/packages/rrweb/test/replay/deserialize-args.test.ts index 855a9ef13f..d178d4dcd2 100644 --- a/packages/rrweb/test/replay/deserialize-args.test.ts +++ b/packages/rrweb/test/replay/deserialize-args.test.ts @@ -2,58 +2,8 @@ * @jest-environment jsdom */ -// polyfill as jsdom does not have support for these classes -// consider replacing with https://www.npmjs.com/package/canvas -class WebGLActiveInfo { - constructor() {} -} - -global.WebGLActiveInfo = WebGLActiveInfo as any; -class WebGLBuffer { - constructor() {} -} - -global.WebGLBuffer = WebGLBuffer as any; -class WebGLFramebuffer { - constructor() {} -} - -global.WebGLFramebuffer = WebGLFramebuffer as any; -class WebGLProgram { - constructor() {} -} - -global.WebGLProgram = WebGLProgram as any; -class WebGLRenderbuffer { - constructor() {} -} - -global.WebGLRenderbuffer = WebGLRenderbuffer as any; -class WebGLShader { - constructor() {} -} - -global.WebGLShader = WebGLShader as any; -class WebGLShaderPrecisionFormat { - constructor() {} -} - -global.WebGLShaderPrecisionFormat = WebGLShaderPrecisionFormat as any; -class WebGLTexture { - constructor() {} -} - -global.WebGLTexture = WebGLTexture as any; -class WebGLUniformLocation { - constructor() {} -} - -global.WebGLUniformLocation = WebGLUniformLocation as any; -class WebGLVertexArrayObject { - constructor() {} -} - -global.WebGLVertexArrayObject = WebGLVertexArrayObject as any; +import { polyfillWebGLGlobals } from '../utils'; +polyfillWebGLGlobals(); import { deserializeArg } from '../../src/replay/canvas/webgl'; From 47d7166c7ef9081226256f238dfe314752fd6ddf Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Thu, 23 Dec 2021 15:22:34 +0100 Subject: [PATCH 70/93] Preload images in webgl --- packages/rrweb/jest.config.js | 3 + packages/rrweb/package.json | 1 + packages/rrweb/src/replay/canvas/index.ts | 2 +- packages/rrweb/src/replay/canvas/webgl.ts | 56 +++++--- packages/rrweb/src/replay/index.ts | 43 +++++- .../test/replay/deserialize-args.test.ts | 30 +++-- .../test/replay/preload-all-images.test.ts | 125 ++++++++++++++++++ .../rrweb/typings/replay/canvas/webgl.d.ts | 5 +- packages/rrweb/typings/replay/index.d.ts | 2 + packages/rrweb/typings/replay/timer.d.ts | 5 +- yarn.lock | 12 ++ 11 files changed, 249 insertions(+), 35 deletions(-) create mode 100644 packages/rrweb/test/replay/preload-all-images.test.ts diff --git a/packages/rrweb/jest.config.js b/packages/rrweb/jest.config.js index 46cb05c37d..29db4e7fa0 100644 --- a/packages/rrweb/jest.config.js +++ b/packages/rrweb/jest.config.js @@ -3,4 +3,7 @@ module.exports = { preset: 'ts-jest', testEnvironment: 'node', testMatch: ['**/**.test.ts'], + moduleNameMapper: { + '\\.css$': 'identity-obj-proxy', + }, }; diff --git a/packages/rrweb/package.json b/packages/rrweb/package.json index 63602f8f9a..ef5d283650 100644 --- a/packages/rrweb/package.json +++ b/packages/rrweb/package.json @@ -53,6 +53,7 @@ "@types/puppeteer": "^5.4.4", "cross-env": "^5.2.0", "fast-mhtml": "^1.1.9", + "identity-obj-proxy": "^3.0.0", "ignore-styles": "^5.0.1", "inquirer": "^6.2.1", "jest": "^27.2.4", diff --git a/packages/rrweb/src/replay/canvas/index.ts b/packages/rrweb/src/replay/canvas/index.ts index 6d5d3c53cc..783f0bde23 100644 --- a/packages/rrweb/src/replay/canvas/index.ts +++ b/packages/rrweb/src/replay/canvas/index.ts @@ -18,7 +18,7 @@ export default function canvasMutation({ }): void { try { if ([CanvasContext.WebGL, CanvasContext.WebGL2].includes(mutation.type)) { - return webglMutation({ mutation, target, errorHandler }); + return webglMutation({ mutation, target, imageMap, errorHandler }); } // default is '2d' for backwards compatibility (rrweb below 1.1.x) return canvas2DMutation({ diff --git a/packages/rrweb/src/replay/canvas/webgl.ts b/packages/rrweb/src/replay/canvas/webgl.ts index 68937a9008..68b4babfc8 100644 --- a/packages/rrweb/src/replay/canvas/webgl.ts +++ b/packages/rrweb/src/replay/canvas/webgl.ts @@ -60,39 +60,51 @@ function saveToWebGLVarMap(result: any) { if (!variables.includes(result)) variables.push(result); } -export function deserializeArg(arg: SerializedWebGlArg): any { - if (arg && typeof arg === 'object' && 'rr_type' in arg) { - if ('index' in arg) { - const { rr_type: name, index } = arg; - return variableListFor(name)[index]; - } else if ('args' in arg) { - const { rr_type: name, args } = arg; +export function deserializeArg( + imageMap: Replayer['imageMap'], +): (arg: SerializedWebGlArg) => any { + return (arg: SerializedWebGlArg): any => { + if (arg && typeof arg === 'object' && 'rr_type' in arg) { + if ('index' in arg) { + const { rr_type: name, index } = arg; + return variableListFor(name)[index]; + } else if ('args' in arg) { + const { rr_type: name, args } = arg; - // @ts-ignore - const ctor = window[name] as unknown; + // @ts-ignore + const ctor = window[name] as unknown; - // @ts-ignore - return new ctor(...args.map(deserializeArg)); - } else if ('base64' in arg) { - return decode(arg.base64); - } else if ('src' in arg) { - const image = new Image(); - image.src = arg.src; - return image; + // @ts-ignore + return new ctor(...args.map(deserializeArg(imageMap))); + } else if ('base64' in arg) { + return decode(arg.base64); + } else if ('src' in arg) { + const image = imageMap.get(arg.src); + if (image) { + return image; + } else { + const image = new Image(); + image.src = arg.src; + imageMap.set(arg.src, image); + return image; + } + } + } else if (Array.isArray(arg)) { + return arg.map(deserializeArg(imageMap)); } - } else if (Array.isArray(arg)) { - return arg.map(deserializeArg); - } - return arg; + return arg; + }; } export default function webglMutation({ mutation, target, + imageMap, errorHandler, }: { mutation: canvasMutationData; target: HTMLCanvasElement; + imageMap: Replayer['imageMap']; errorHandler: Replayer['warnCanvasMutationFailed']; }): void { try { @@ -113,7 +125,7 @@ export default function webglMutation({ mutation.property as Exclude ] as Function; - const args = mutation.args.map(deserializeArg); + const args = mutation.args.map(deserializeArg(imageMap)); const result = original.apply(ctx, args); saveToWebGLVarMap(result); diff --git a/packages/rrweb/src/replay/index.ts b/packages/rrweb/src/replay/index.ts index 94521b7068..7c6a13ccd0 100644 --- a/packages/rrweb/src/replay/index.ts +++ b/packages/rrweb/src/replay/index.ts @@ -122,7 +122,7 @@ export class Replayer { // The replayer uses the cache to speed up replay and scrubbing. private cache: BuildCache = createCache(); - private imageMap: Map = new Map(); + private imageMap: Map = new Map(); private mirror: Mirror = createMirror(); @@ -811,6 +811,37 @@ export class Replayer { } } + private hasImageArg(args: any[]): boolean { + for (const arg of args) { + if (!arg || typeof arg !== 'object') { + // do nothing + } else if ('rr_type' in arg && 'args' in arg) { + if (this.hasImageArg(arg.args)) return true; + } else if ('rr_type' in arg && arg.rr_type === 'HTMLImageElement') { + return true; // has image! + } else if (arg instanceof Array) { + if (this.hasImageArg(arg)) return true; + } + } + return false; + } + + private getImageArgs(args: any[]): string[] { + const images: string[] = []; + for (const arg of args) { + if (!arg || typeof arg !== 'object') { + // do nothing + } else if ('rr_type' in arg && 'args' in arg) { + images.push(...this.getImageArgs(arg.args)); + } else if ('rr_type' in arg && arg.rr_type === 'HTMLImageElement') { + images.push(arg.src); + } else if (arg instanceof Array) { + images.push(...this.getImageArgs(arg)); + } + } + return images; + } + /** * pause when there are some canvas drawImage args need to be loaded */ @@ -835,6 +866,16 @@ export class Replayer { let d = imgd?.data; d = JSON.parse(event.data.args[0]); ctx?.putImageData(imgd!, 0, 0); + } else if ( + event.type === EventType.IncrementalSnapshot && + event.data.source === IncrementalSource.CanvasMutation && + this.hasImageArg(event.data.args) + ) { + this.getImageArgs(event.data.args).forEach((url) => { + const image = new Image(); + image.src = url; // this preloads the image + this.imageMap.set(url, image); + }); } } } diff --git a/packages/rrweb/test/replay/deserialize-args.test.ts b/packages/rrweb/test/replay/deserialize-args.test.ts index d178d4dcd2..ec6bfdef1f 100644 --- a/packages/rrweb/test/replay/deserialize-args.test.ts +++ b/packages/rrweb/test/replay/deserialize-args.test.ts @@ -10,7 +10,7 @@ import { deserializeArg } from '../../src/replay/canvas/webgl'; describe('deserializeArg', () => { it('should deserialize Float32Array values', async () => { expect( - deserializeArg({ + deserializeArg(new Map())({ rr_type: 'Float32Array', args: [[-1, -1, 3, -1, -1, 3]], }), @@ -19,7 +19,7 @@ describe('deserializeArg', () => { it('should deserialize Float64Array values', async () => { expect( - deserializeArg({ + deserializeArg(new Map())({ rr_type: 'Float64Array', args: [[-1, -1, 3, -1, -1, 3]], }), @@ -29,7 +29,7 @@ describe('deserializeArg', () => { it('should deserialize ArrayBuffer values', async () => { const contents = [1, 2, 0, 4]; expect( - deserializeArg({ + deserializeArg(new Map())({ rr_type: 'ArrayBuffer', base64: 'AQIABA==', }), @@ -38,7 +38,7 @@ describe('deserializeArg', () => { it('should deserialize DataView values', async () => { expect( - deserializeArg({ + deserializeArg(new Map())({ rr_type: 'DataView', args: [ { @@ -54,7 +54,7 @@ describe('deserializeArg', () => { it('should leave arrays intact', async () => { const array = [1, 2, 3, 4]; - expect(deserializeArg(array)).toEqual(array); + expect(deserializeArg(new Map())(array)).toEqual(array); }); it('should deserialize complex objects', async () => { @@ -73,7 +73,7 @@ describe('deserializeArg', () => { 5, 6, ]; - expect(deserializeArg(serializedArg)).toStrictEqual([ + expect(deserializeArg(new Map())(serializedArg)).toStrictEqual([ new DataView(new ArrayBuffer(16), 0, 16), 5, 6, @@ -81,17 +81,31 @@ describe('deserializeArg', () => { }); it('should leave null as-is', async () => { - expect(deserializeArg(null)).toStrictEqual(null); + expect(deserializeArg(new Map())(null)).toStrictEqual(null); }); it('should support HTMLImageElements', async () => { const image = new Image(); image.src = 'http://example.com/image.png'; expect( - deserializeArg({ + deserializeArg(new Map())({ rr_type: 'HTMLImageElement', src: 'http://example.com/image.png', }), ).toStrictEqual(image); }); + + it('should return image from imageMap for HTMLImageElements', async () => { + const image = new Image(); + image.src = 'http://example.com/image.png'; + const imageMap = new Map(); + imageMap.set(image.src, image); + + expect( + deserializeArg(imageMap)({ + rr_type: 'HTMLImageElement', + src: 'http://example.com/image.png', + }), + ).toBe(image); + }); }); diff --git a/packages/rrweb/test/replay/preload-all-images.test.ts b/packages/rrweb/test/replay/preload-all-images.test.ts new file mode 100644 index 0000000000..4b4e0a4c71 --- /dev/null +++ b/packages/rrweb/test/replay/preload-all-images.test.ts @@ -0,0 +1,125 @@ +/** + * @jest-environment jsdom + */ +import { polyfillWebGLGlobals } from '../utils'; +polyfillWebGLGlobals(); + +import { Replayer } from '../../src/replay'; +import {} from '../../src/types'; +import { + CanvasContext, + SerializedWebGlArg, + IncrementalSource, + EventType, + eventWithTime, +} from '../../src/types'; + +let replayer: Replayer; + +const canvasMutationEventWithArgs = ( + args: SerializedWebGlArg[], +): eventWithTime => { + return { + timestamp: 100, + type: EventType.IncrementalSnapshot, + data: { + source: IncrementalSource.CanvasMutation, + property: 'x', + args, + id: 1, + type: CanvasContext.WebGL, + }, + }; +}; + +const event = (): eventWithTime => { + return { + timestamp: 1, + type: EventType.DomContentLoaded, + data: {}, + }; +}; + +describe('preloadAllImages', () => { + beforeEach(() => { + replayer = new Replayer( + // Get around the error "Replayer need at least 2 events." + [event(), event()], + ); + }); + + it('should preload image', () => { + replayer.service.state.context.events = [ + canvasMutationEventWithArgs([ + { + rr_type: 'HTMLImageElement', + src: 'http://example.com', + }, + ]), + ]; + + (replayer as any).preloadAllImages(); + + const expectedImage = new Image(); + expectedImage.src = 'http://example.com'; + expect((replayer as any).imageMap.get('http://example.com')).toEqual( + expectedImage, + ); + }); + + it('should preload nested image', () => { + replayer.service.state.context.events = [ + canvasMutationEventWithArgs([ + { + rr_type: 'something', + args: [ + { + rr_type: 'HTMLImageElement', + src: 'http://example.com', + }, + ], + }, + ]), + ]; + + (replayer as any).preloadAllImages(); + + const expectedImage = new Image(); + expectedImage.src = 'http://example.com'; + + expect((replayer as any).imageMap.get('http://example.com')).toEqual( + expectedImage, + ); + }); + + it('should preload multiple images', () => { + replayer.service.state.context.events = [ + canvasMutationEventWithArgs([ + { + rr_type: 'HTMLImageElement', + src: 'http://example.com/img1.png', + }, + { + rr_type: 'HTMLImageElement', + src: 'http://example.com/img2.png', + }, + ]), + ]; + + (replayer as any).preloadAllImages(); + + const expectedImage1 = new Image(); + expectedImage1.src = 'http://example.com/img1.png'; + + expect( + (replayer as any).imageMap.get('http://example.com/img1.png'), + ).toEqual(expectedImage1); + + const expectedImage2 = new Image(); + expectedImage1.src = 'http://example.com/img2.png'; + + expect( + (replayer as any).imageMap.get('http://example.com/img2.png'), + ).toEqual(expectedImage1); + }); +}); diff --git a/packages/rrweb/typings/replay/canvas/webgl.d.ts b/packages/rrweb/typings/replay/canvas/webgl.d.ts index f1cf4b587b..12d703d345 100644 --- a/packages/rrweb/typings/replay/canvas/webgl.d.ts +++ b/packages/rrweb/typings/replay/canvas/webgl.d.ts @@ -1,9 +1,10 @@ import { Replayer } from '../'; import { canvasMutationData, SerializedWebGlArg } from '../../types'; export declare function variableListFor(ctor: string): any[]; -export declare function deserializeArg(arg: SerializedWebGlArg): any; -export default function webglMutation({ mutation, target, errorHandler, }: { +export declare function deserializeArg(imageMap: Replayer['imageMap']): (arg: SerializedWebGlArg) => any; +export default function webglMutation({ mutation, target, imageMap, errorHandler, }: { mutation: canvasMutationData; target: HTMLCanvasElement; + imageMap: Replayer['imageMap']; errorHandler: Replayer['warnCanvasMutationFailed']; }): void; diff --git a/packages/rrweb/typings/replay/index.d.ts b/packages/rrweb/typings/replay/index.d.ts index 2c49091b4b..b5d03cd736 100644 --- a/packages/rrweb/typings/replay/index.d.ts +++ b/packages/rrweb/typings/replay/index.d.ts @@ -51,6 +51,8 @@ export declare class Replayer { private attachDocumentToIframe; private collectIframeAndAttachDocument; private waitForStylesheetLoad; + private hasImageArg; + private getImageArgs; private preloadAllImages; private applyIncremental; private applyMutation; diff --git a/packages/rrweb/typings/replay/timer.d.ts b/packages/rrweb/typings/replay/timer.d.ts index 376c92495b..842b1cbe20 100644 --- a/packages/rrweb/typings/replay/timer.d.ts +++ b/packages/rrweb/typings/replay/timer.d.ts @@ -15,4 +15,7 @@ export declare class Timer { isActive(): boolean; private findActionIndex; } -export declare function addDelay(event: eventWithTime, baselineTime: number): number; +export declare type LastDelay = { + at: number | null; +}; +export declare function addDelay(event: eventWithTime, baselineTime: number, lastDelay?: LastDelay): number; diff --git a/yarn.lock b/yarn.lock index 11bb99098c..79feba9fea 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4722,6 +4722,11 @@ hard-rejection@^2.1.0: resolved "https://registry.yarnpkg.com/hard-rejection/-/hard-rejection-2.1.0.tgz#1c6eda5c1685c63942766d79bb40ae773cecd883" integrity sha512-VIZB+ibDhx7ObhAe7OVtoEbuP4h/MuOTHJ+J8h/eBXotJYl0fBgR72xDFCKgIh22OJZIOVNxBMWuhAr10r8HdA== +harmony-reflect@^1.4.6: + version "1.6.2" + resolved "https://registry.yarnpkg.com/harmony-reflect/-/harmony-reflect-1.6.2.tgz#31ecbd32e648a34d030d86adb67d4d47547fe710" + integrity sha512-HIp/n38R9kQjDEziXyDTuW3vvoxxyxjxFzXLrBr18uB47GnSt+G9D29fqrpM5ZkspMcPICud3XsBJQ4Y2URg8g== + has-ansi@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/has-ansi/-/has-ansi-2.0.0.tgz#34f5049ce1ecdf2b0649af3ef24e45ed35416d91" @@ -4914,6 +4919,13 @@ icss-replace-symbols@1.1.0, icss-replace-symbols@^1.1.0: resolved "https://registry.yarnpkg.com/icss-replace-symbols/-/icss-replace-symbols-1.1.0.tgz#06ea6f83679a7749e386cfe1fe812ae5db223ded" integrity sha1-Bupvg2ead0njhs/h/oEq5dsiPe0= +identity-obj-proxy@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/identity-obj-proxy/-/identity-obj-proxy-3.0.0.tgz#94d2bda96084453ef36fbc5aaec37e0f79f1fc14" + integrity sha1-lNK9qWCERT7zb7xarsN+D3nx/BQ= + dependencies: + harmony-reflect "^1.4.6" + ieee754@^1.1.13: version "1.2.1" resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352" From dcd97e25f50aba9d110a3a677ca02ddc0033a722 Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Thu, 23 Dec 2021 15:34:05 +0100 Subject: [PATCH 71/93] Add e2e test for webgl image preloading --- ...record-and-replay-a-webgl-image-1-snap.png | Bin 0 -> 10913 bytes packages/rrweb/test/e2e/webgl.test.ts | 37 +- .../rrweb/test/html/assets/webgl-utils.js | 1496 +++++++++++++++++ .../rrweb/test/html/canvas-webgl-image.html | 149 ++ 4 files changed, 1680 insertions(+), 2 deletions(-) create mode 100644 packages/rrweb/test/e2e/__image_snapshots__/webgl-test-ts-e-2-e-webgl-will-record-and-replay-a-webgl-image-1-snap.png create mode 100644 packages/rrweb/test/html/assets/webgl-utils.js create mode 100644 packages/rrweb/test/html/canvas-webgl-image.html diff --git a/packages/rrweb/test/e2e/__image_snapshots__/webgl-test-ts-e-2-e-webgl-will-record-and-replay-a-webgl-image-1-snap.png b/packages/rrweb/test/e2e/__image_snapshots__/webgl-test-ts-e-2-e-webgl-will-record-and-replay-a-webgl-image-1-snap.png new file mode 100644 index 0000000000000000000000000000000000000000..731898701b69eeca46729aa8eef2af872597d9de GIT binary patch literal 10913 zcmeAS@N?(olHy`uVBq!ia0y~yU~geyV6ov~1Bz6wRMrPljKx9jP7LeL$-HD>P+;(M zaSW-L^XB%(TqQ?_w!mjwuI5PQ9ho@0fKlXOk6Xq`)ibW*b9sf>p1r$Vt*Bh~zt{fk zj~_o~umcSOfuL`-^Y@4KLpTf%_I<0hfAhQU#=k!wzJ1$P#19oO|Mc@`rG*S1yLk&E zL}(F9B81VP;V1!NDR>EhTnPjnLZixn!9YoXY`e;wXL>B+*FVeWm%cNDLzN-sahm~z zbG_t>0NuXEpEt z%jeKQu$d)ITqqz6!V*|p*UA;FzWnh|yuCcc%?~b8CtSWZxL4NJ|GfAg7&LRIL8EKU zL+V6poXzt+6?VsVfI_6cRNH~lZ4&ZPY@hyjvr5__07`XgFYXL?JiqT3FwjyA( zMGeXgqb+K1nJ`*uLW;)GN)sFoBfipvq=wPLVYF}mdKI;Vst0Q@>>_5c6? literal 0 HcmV?d00001 diff --git a/packages/rrweb/test/e2e/webgl.test.ts b/packages/rrweb/test/e2e/webgl.test.ts index 267e1254ca..44a5653fbb 100644 --- a/packages/rrweb/test/e2e/webgl.test.ts +++ b/packages/rrweb/test/e2e/webgl.test.ts @@ -85,11 +85,11 @@ describe('e2e webgl', () => { body: ' ', // non-empty string or page will load indefinitely }); }; - page.setRequestInterception(true); + await page.setRequestInterception(true); page.on('request', intercept); await page.goto(url); - page.setRequestInterception(false); page.off('request', intercept); + await page.setRequestInterception(false); }; const hideMouseAnimation = async (page: puppeteer.Page) => { @@ -130,4 +130,37 @@ describe('e2e webgl', () => { expect(frameImage).toMatchImageSnapshot(); }); + + it('will record and replay a webgl image', async () => { + page = await browser.newPage(); + await fakeGoto(page, `${serverURL}/html/canvas-webgl-image.html`); + + await page.setContent( + getHtml.call(this, 'canvas-webgl-image.html', { recordCanvas: true }), + ); + + await page.waitForTimeout(100); + const snapshots: eventWithTime[] = await page.evaluate('window.snapshots'); + + page = await browser.newPage(); + + await page.goto('about:blank'); + await page.evaluate(code); + + await hideMouseAnimation(page); + await page.evaluate(`let events = ${JSON.stringify(snapshots)}`); + await page.evaluate(` + const { Replayer } = rrweb; + const replayer = new Replayer(events, { + UNSAFE_replayCanvas: true, + }); + replayer.play(500); + `); + await page.waitForTimeout(50); + + const element = await page.$('iframe'); + const frameImage = await element!.screenshot(); + + expect(frameImage).toMatchImageSnapshot(); + }); }); diff --git a/packages/rrweb/test/html/assets/webgl-utils.js b/packages/rrweb/test/html/assets/webgl-utils.js new file mode 100644 index 0000000000..183a5acf41 --- /dev/null +++ b/packages/rrweb/test/html/assets/webgl-utils.js @@ -0,0 +1,1496 @@ +/* + * Copyright 2021 GFXFundamentals. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of GFXFundamentals. nor the names of his + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +(function (root, factory) { + // eslint-disable-line + if (typeof define === 'function' && define.amd) { + // AMD. Register as an anonymous module. + define([], function () { + return factory.call(root); + }); + } else { + // Browser globals + root.webglUtils = factory.call(root); + } +})(this, function () { + 'use strict'; + + const topWindow = this; + + /** @module webgl-utils */ + + function isInIFrame(w) { + w = w || topWindow; + return w !== w.top; + } + + if (!isInIFrame()) { + console.log( + '%c%s', + 'color:blue;font-weight:bold;', + 'for more about webgl-utils.js see:', + ); // eslint-disable-line + console.log( + '%c%s', + 'color:blue;font-weight:bold;', + 'https://webglfundamentals.org/webgl/lessons/webgl-boilerplate.html', + ); // eslint-disable-line + } + + /** + * Wrapped logging function. + * @param {string} msg The message to log. + */ + function error(msg) { + if (topWindow.console) { + if (topWindow.console.error) { + topWindow.console.error(msg); + } else if (topWindow.console.log) { + topWindow.console.log(msg); + } + } + } + + /** + * Error Callback + * @callback ErrorCallback + * @param {string} msg error message. + * @memberOf module:webgl-utils + */ + + /** + * Loads a shader. + * @param {WebGLRenderingContext} gl The WebGLRenderingContext to use. + * @param {string} shaderSource The shader source. + * @param {number} shaderType The type of shader. + * @param {module:webgl-utils.ErrorCallback} opt_errorCallback callback for errors. + * @return {WebGLShader} The created shader. + */ + function loadShader(gl, shaderSource, shaderType, opt_errorCallback) { + const errFn = opt_errorCallback || error; + // Create the shader object + const shader = gl.createShader(shaderType); + + // Load the shader source + gl.shaderSource(shader, shaderSource); + + // Compile the shader + gl.compileShader(shader); + + // Check the compile status + const compiled = gl.getShaderParameter(shader, gl.COMPILE_STATUS); + if (!compiled) { + // Something went wrong during compilation; get the error + const lastError = gl.getShaderInfoLog(shader); + errFn( + "*** Error compiling shader '" + + shader + + "':" + + lastError + + `\n` + + shaderSource + .split('\n') + .map((l, i) => `${i + 1}: ${l}`) + .join('\n'), + ); + gl.deleteShader(shader); + return null; + } + + return shader; + } + + /** + * Creates a program, attaches shaders, binds attrib locations, links the + * program and calls useProgram. + * @param {WebGLShader[]} shaders The shaders to attach + * @param {string[]} [opt_attribs] An array of attribs names. Locations will be assigned by index if not passed in + * @param {number[]} [opt_locations] The locations for the. A parallel array to opt_attribs letting you assign locations. + * @param {module:webgl-utils.ErrorCallback} opt_errorCallback callback for errors. By default it just prints an error to the console + * on error. If you want something else pass an callback. It's passed an error message. + * @memberOf module:webgl-utils + */ + function createProgram( + gl, + shaders, + opt_attribs, + opt_locations, + opt_errorCallback, + ) { + const errFn = opt_errorCallback || error; + const program = gl.createProgram(); + shaders.forEach(function (shader) { + gl.attachShader(program, shader); + }); + if (opt_attribs) { + opt_attribs.forEach(function (attrib, ndx) { + gl.bindAttribLocation( + program, + opt_locations ? opt_locations[ndx] : ndx, + attrib, + ); + }); + } + gl.linkProgram(program); + + // Check the link status + const linked = gl.getProgramParameter(program, gl.LINK_STATUS); + if (!linked) { + // something went wrong with the link + const lastError = gl.getProgramInfoLog(program); + errFn('Error in program linking:' + lastError); + + gl.deleteProgram(program); + return null; + } + return program; + } + + /** + * Loads a shader from a script tag. + * @param {WebGLRenderingContext} gl The WebGLRenderingContext to use. + * @param {string} scriptId The id of the script tag. + * @param {number} opt_shaderType The type of shader. If not passed in it will + * be derived from the type of the script tag. + * @param {module:webgl-utils.ErrorCallback} opt_errorCallback callback for errors. + * @return {WebGLShader} The created shader. + */ + function createShaderFromScript( + gl, + scriptId, + opt_shaderType, + opt_errorCallback, + ) { + let shaderSource = ''; + let shaderType; + const shaderScript = document.getElementById(scriptId); + if (!shaderScript) { + throw '*** Error: unknown script element' + scriptId; + } + shaderSource = shaderScript.text; + + if (!opt_shaderType) { + if (shaderScript.type === 'x-shader/x-vertex') { + shaderType = gl.VERTEX_SHADER; + } else if (shaderScript.type === 'x-shader/x-fragment') { + shaderType = gl.FRAGMENT_SHADER; + } else if ( + shaderType !== gl.VERTEX_SHADER && + shaderType !== gl.FRAGMENT_SHADER + ) { + throw '*** Error: unknown shader type'; + } + } + + return loadShader( + gl, + shaderSource, + opt_shaderType ? opt_shaderType : shaderType, + opt_errorCallback, + ); + } + + const defaultShaderType = ['VERTEX_SHADER', 'FRAGMENT_SHADER']; + + /** + * Creates a program from 2 script tags. + * + * @param {WebGLRenderingContext} gl The WebGLRenderingContext + * to use. + * @param {string[]} shaderScriptIds Array of ids of the script + * tags for the shaders. The first is assumed to be the + * vertex shader, the second the fragment shader. + * @param {string[]} [opt_attribs] An array of attribs names. Locations will be assigned by index if not passed in + * @param {number[]} [opt_locations] The locations for the. A parallel array to opt_attribs letting you assign locations. + * @param {module:webgl-utils.ErrorCallback} opt_errorCallback callback for errors. By default it just prints an error to the console + * on error. If you want something else pass an callback. It's passed an error message. + * @return {WebGLProgram} The created program. + * @memberOf module:webgl-utils + */ + function createProgramFromScripts( + gl, + shaderScriptIds, + opt_attribs, + opt_locations, + opt_errorCallback, + ) { + const shaders = []; + for (let ii = 0; ii < shaderScriptIds.length; ++ii) { + shaders.push( + createShaderFromScript( + gl, + shaderScriptIds[ii], + gl[defaultShaderType[ii]], + opt_errorCallback, + ), + ); + } + return createProgram( + gl, + shaders, + opt_attribs, + opt_locations, + opt_errorCallback, + ); + } + + /** + * Creates a program from 2 sources. + * + * @param {WebGLRenderingContext} gl The WebGLRenderingContext + * to use. + * @param {string[]} shaderSourcess Array of sources for the + * shaders. The first is assumed to be the vertex shader, + * the second the fragment shader. + * @param {string[]} [opt_attribs] An array of attribs names. Locations will be assigned by index if not passed in + * @param {number[]} [opt_locations] The locations for the. A parallel array to opt_attribs letting you assign locations. + * @param {module:webgl-utils.ErrorCallback} opt_errorCallback callback for errors. By default it just prints an error to the console + * on error. If you want something else pass an callback. It's passed an error message. + * @return {WebGLProgram} The created program. + * @memberOf module:webgl-utils + */ + function createProgramFromSources( + gl, + shaderSources, + opt_attribs, + opt_locations, + opt_errorCallback, + ) { + const shaders = []; + for (let ii = 0; ii < shaderSources.length; ++ii) { + shaders.push( + loadShader( + gl, + shaderSources[ii], + gl[defaultShaderType[ii]], + opt_errorCallback, + ), + ); + } + return createProgram( + gl, + shaders, + opt_attribs, + opt_locations, + opt_errorCallback, + ); + } + + /** + * Returns the corresponding bind point for a given sampler type + */ + function getBindPointForSamplerType(gl, type) { + if (type === gl.SAMPLER_2D) return gl.TEXTURE_2D; // eslint-disable-line + if (type === gl.SAMPLER_CUBE) return gl.TEXTURE_CUBE_MAP; // eslint-disable-line + return undefined; + } + + /** + * @typedef {Object.} Setters + */ + + /** + * Creates setter functions for all uniforms of a shader + * program. + * + * @see {@link module:webgl-utils.setUniforms} + * + * @param {WebGLProgram} program the program to create setters for. + * @returns {Object.} an object with a setter by name for each uniform + * @memberOf module:webgl-utils + */ + function createUniformSetters(gl, program) { + let textureUnit = 0; + + /** + * Creates a setter for a uniform of the given program with it's + * location embedded in the setter. + * @param {WebGLProgram} program + * @param {WebGLUniformInfo} uniformInfo + * @returns {function} the created setter. + */ + function createUniformSetter(program, uniformInfo) { + const location = gl.getUniformLocation(program, uniformInfo.name); + const type = uniformInfo.type; + // Check if this uniform is an array + const isArray = + uniformInfo.size > 1 && uniformInfo.name.substr(-3) === '[0]'; + if (type === gl.FLOAT && isArray) { + return function (v) { + gl.uniform1fv(location, v); + }; + } + if (type === gl.FLOAT) { + return function (v) { + gl.uniform1f(location, v); + }; + } + if (type === gl.FLOAT_VEC2) { + return function (v) { + gl.uniform2fv(location, v); + }; + } + if (type === gl.FLOAT_VEC3) { + return function (v) { + gl.uniform3fv(location, v); + }; + } + if (type === gl.FLOAT_VEC4) { + return function (v) { + gl.uniform4fv(location, v); + }; + } + if (type === gl.INT && isArray) { + return function (v) { + gl.uniform1iv(location, v); + }; + } + if (type === gl.INT) { + return function (v) { + gl.uniform1i(location, v); + }; + } + if (type === gl.INT_VEC2) { + return function (v) { + gl.uniform2iv(location, v); + }; + } + if (type === gl.INT_VEC3) { + return function (v) { + gl.uniform3iv(location, v); + }; + } + if (type === gl.INT_VEC4) { + return function (v) { + gl.uniform4iv(location, v); + }; + } + if (type === gl.BOOL) { + return function (v) { + gl.uniform1iv(location, v); + }; + } + if (type === gl.BOOL_VEC2) { + return function (v) { + gl.uniform2iv(location, v); + }; + } + if (type === gl.BOOL_VEC3) { + return function (v) { + gl.uniform3iv(location, v); + }; + } + if (type === gl.BOOL_VEC4) { + return function (v) { + gl.uniform4iv(location, v); + }; + } + if (type === gl.FLOAT_MAT2) { + return function (v) { + gl.uniformMatrix2fv(location, false, v); + }; + } + if (type === gl.FLOAT_MAT3) { + return function (v) { + gl.uniformMatrix3fv(location, false, v); + }; + } + if (type === gl.FLOAT_MAT4) { + return function (v) { + gl.uniformMatrix4fv(location, false, v); + }; + } + if ((type === gl.SAMPLER_2D || type === gl.SAMPLER_CUBE) && isArray) { + const units = []; + for (let ii = 0; ii < info.size; ++ii) { + units.push(textureUnit++); + } + return (function (bindPoint, units) { + return function (textures) { + gl.uniform1iv(location, units); + textures.forEach(function (texture, index) { + gl.activeTexture(gl.TEXTURE0 + units[index]); + gl.bindTexture(bindPoint, texture); + }); + }; + })(getBindPointForSamplerType(gl, type), units); + } + if (type === gl.SAMPLER_2D || type === gl.SAMPLER_CUBE) { + return (function (bindPoint, unit) { + return function (texture) { + gl.uniform1i(location, unit); + gl.activeTexture(gl.TEXTURE0 + unit); + gl.bindTexture(bindPoint, texture); + }; + })(getBindPointForSamplerType(gl, type), textureUnit++); + } + throw 'unknown type: 0x' + type.toString(16); // we should never get here. + } + + const uniformSetters = {}; + const numUniforms = gl.getProgramParameter(program, gl.ACTIVE_UNIFORMS); + + for (let ii = 0; ii < numUniforms; ++ii) { + const uniformInfo = gl.getActiveUniform(program, ii); + if (!uniformInfo) { + break; + } + let name = uniformInfo.name; + // remove the array suffix. + if (name.substr(-3) === '[0]') { + name = name.substr(0, name.length - 3); + } + const setter = createUniformSetter(program, uniformInfo); + uniformSetters[name] = setter; + } + return uniformSetters; + } + + /** + * Set uniforms and binds related textures. + * + * Example: + * + * let programInfo = createProgramInfo( + * gl, ["some-vs", "some-fs"]); + * + * let tex1 = gl.createTexture(); + * let tex2 = gl.createTexture(); + * + * ... assume we setup the textures with data ... + * + * let uniforms = { + * u_someSampler: tex1, + * u_someOtherSampler: tex2, + * u_someColor: [1,0,0,1], + * u_somePosition: [0,1,1], + * u_someMatrix: [ + * 1,0,0,0, + * 0,1,0,0, + * 0,0,1,0, + * 0,0,0,0, + * ], + * }; + * + * gl.useProgram(program); + * + * This will automatically bind the textures AND set the + * uniforms. + * + * setUniforms(programInfo.uniformSetters, uniforms); + * + * For the example above it is equivalent to + * + * let texUnit = 0; + * gl.activeTexture(gl.TEXTURE0 + texUnit); + * gl.bindTexture(gl.TEXTURE_2D, tex1); + * gl.uniform1i(u_someSamplerLocation, texUnit++); + * gl.activeTexture(gl.TEXTURE0 + texUnit); + * gl.bindTexture(gl.TEXTURE_2D, tex2); + * gl.uniform1i(u_someSamplerLocation, texUnit++); + * gl.uniform4fv(u_someColorLocation, [1, 0, 0, 1]); + * gl.uniform3fv(u_somePositionLocation, [0, 1, 1]); + * gl.uniformMatrix4fv(u_someMatrix, false, [ + * 1,0,0,0, + * 0,1,0,0, + * 0,0,1,0, + * 0,0,0,0, + * ]); + * + * Note it is perfectly reasonable to call `setUniforms` multiple times. For example + * + * let uniforms = { + * u_someSampler: tex1, + * u_someOtherSampler: tex2, + * }; + * + * let moreUniforms { + * u_someColor: [1,0,0,1], + * u_somePosition: [0,1,1], + * u_someMatrix: [ + * 1,0,0,0, + * 0,1,0,0, + * 0,0,1,0, + * 0,0,0,0, + * ], + * }; + * + * setUniforms(programInfo.uniformSetters, uniforms); + * setUniforms(programInfo.uniformSetters, moreUniforms); + * + * @param {Object.|module:webgl-utils.ProgramInfo} setters the setters returned from + * `createUniformSetters` or a ProgramInfo from {@link module:webgl-utils.createProgramInfo}. + * @param {Object.} an object with values for the + * uniforms. + * @memberOf module:webgl-utils + */ + function setUniforms(setters, ...values) { + setters = setters.uniformSetters || setters; + for (const uniforms of values) { + Object.keys(uniforms).forEach(function (name) { + const setter = setters[name]; + if (setter) { + setter(uniforms[name]); + } + }); + } + } + + /** + * Creates setter functions for all attributes of a shader + * program. You can pass this to {@link module:webgl-utils.setBuffersAndAttributes} to set all your buffers and attributes. + * + * @see {@link module:webgl-utils.setAttributes} for example + * @param {WebGLProgram} program the program to create setters for. + * @return {Object.} an object with a setter for each attribute by name. + * @memberOf module:webgl-utils + */ + function createAttributeSetters(gl, program) { + const attribSetters = {}; + + function createAttribSetter(index) { + return function (b) { + if (b.value) { + gl.disableVertexAttribArray(index); + switch (b.value.length) { + case 4: + gl.vertexAttrib4fv(index, b.value); + break; + case 3: + gl.vertexAttrib3fv(index, b.value); + break; + case 2: + gl.vertexAttrib2fv(index, b.value); + break; + case 1: + gl.vertexAttrib1fv(index, b.value); + break; + default: + throw new Error( + 'the length of a float constant value must be between 1 and 4!', + ); + } + } else { + gl.bindBuffer(gl.ARRAY_BUFFER, b.buffer); + gl.enableVertexAttribArray(index); + gl.vertexAttribPointer( + index, + b.numComponents || b.size, + b.type || gl.FLOAT, + b.normalize || false, + b.stride || 0, + b.offset || 0, + ); + } + }; + } + + const numAttribs = gl.getProgramParameter(program, gl.ACTIVE_ATTRIBUTES); + for (let ii = 0; ii < numAttribs; ++ii) { + const attribInfo = gl.getActiveAttrib(program, ii); + if (!attribInfo) { + break; + } + const index = gl.getAttribLocation(program, attribInfo.name); + attribSetters[attribInfo.name] = createAttribSetter(index); + } + + return attribSetters; + } + + /** + * Sets attributes and binds buffers (deprecated... use {@link module:webgl-utils.setBuffersAndAttributes}) + * + * Example: + * + * let program = createProgramFromScripts( + * gl, ["some-vs", "some-fs"]); + * + * let attribSetters = createAttributeSetters(program); + * + * let positionBuffer = gl.createBuffer(); + * let texcoordBuffer = gl.createBuffer(); + * + * let attribs = { + * a_position: {buffer: positionBuffer, numComponents: 3}, + * a_texcoord: {buffer: texcoordBuffer, numComponents: 2}, + * }; + * + * gl.useProgram(program); + * + * This will automatically bind the buffers AND set the + * attributes. + * + * setAttributes(attribSetters, attribs); + * + * Properties of attribs. For each attrib you can add + * properties: + * + * * type: the type of data in the buffer. Default = gl.FLOAT + * * normalize: whether or not to normalize the data. Default = false + * * stride: the stride. Default = 0 + * * offset: offset into the buffer. Default = 0 + * + * For example if you had 3 value float positions, 2 value + * float texcoord and 4 value uint8 colors you'd setup your + * attribs like this + * + * let attribs = { + * a_position: {buffer: positionBuffer, numComponents: 3}, + * a_texcoord: {buffer: texcoordBuffer, numComponents: 2}, + * a_color: { + * buffer: colorBuffer, + * numComponents: 4, + * type: gl.UNSIGNED_BYTE, + * normalize: true, + * }, + * }; + * + * @param {Object.|model:webgl-utils.ProgramInfo} setters Attribute setters as returned from createAttributeSetters or a ProgramInfo as returned {@link module:webgl-utils.createProgramInfo} + * @param {Object.} attribs AttribInfos mapped by attribute name. + * @memberOf module:webgl-utils + * @deprecated use {@link module:webgl-utils.setBuffersAndAttributes} + */ + function setAttributes(setters, attribs) { + setters = setters.attribSetters || setters; + Object.keys(attribs).forEach(function (name) { + const setter = setters[name]; + if (setter) { + setter(attribs[name]); + } + }); + } + + /** + * Creates a vertex array object and then sets the attributes + * on it + * + * @param {WebGLRenderingContext} gl The WebGLRenderingContext + * to use. + * @param {Object.} setters Attribute setters as returned from createAttributeSetters + * @param {Object.} attribs AttribInfos mapped by attribute name. + * @param {WebGLBuffer} [indices] an optional ELEMENT_ARRAY_BUFFER of indices + */ + function createVAOAndSetAttributes(gl, setters, attribs, indices) { + const vao = gl.createVertexArray(); + gl.bindVertexArray(vao); + setAttributes(setters, attribs); + if (indices) { + gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, indices); + } + // We unbind this because otherwise any change to ELEMENT_ARRAY_BUFFER + // like when creating buffers for other stuff will mess up this VAO's binding + gl.bindVertexArray(null); + return vao; + } + + /** + * Creates a vertex array object and then sets the attributes + * on it + * + * @param {WebGLRenderingContext} gl The WebGLRenderingContext + * to use. + * @param {Object.| module:webgl-utils.ProgramInfo} programInfo as returned from createProgramInfo or Attribute setters as returned from createAttributeSetters + * @param {module:webgl-utils:BufferInfo} bufferInfo BufferInfo as returned from createBufferInfoFromArrays etc... + * @param {WebGLBuffer} [indices] an optional ELEMENT_ARRAY_BUFFER of indices + */ + function createVAOFromBufferInfo(gl, programInfo, bufferInfo) { + return createVAOAndSetAttributes( + gl, + programInfo.attribSetters || programInfo, + bufferInfo.attribs, + bufferInfo.indices, + ); + } + + /** + * @typedef {Object} ProgramInfo + * @property {WebGLProgram} program A shader program + * @property {Object} uniformSetters: object of setters as returned from createUniformSetters, + * @property {Object} attribSetters: object of setters as returned from createAttribSetters, + * @memberOf module:webgl-utils + */ + + /** + * Creates a ProgramInfo from 2 sources. + * + * A ProgramInfo contains + * + * programInfo = { + * program: WebGLProgram, + * uniformSetters: object of setters as returned from createUniformSetters, + * attribSetters: object of setters as returned from createAttribSetters, + * } + * + * @param {WebGLRenderingContext} gl The WebGLRenderingContext + * to use. + * @param {string[]} shaderSourcess Array of sources for the + * shaders or ids. The first is assumed to be the vertex shader, + * the second the fragment shader. + * @param {string[]} [opt_attribs] An array of attribs names. Locations will be assigned by index if not passed in + * @param {number[]} [opt_locations] The locations for the. A parallel array to opt_attribs letting you assign locations. + * @param {module:webgl-utils.ErrorCallback} opt_errorCallback callback for errors. By default it just prints an error to the console + * on error. If you want something else pass an callback. It's passed an error message. + * @return {module:webgl-utils.ProgramInfo} The created program. + * @memberOf module:webgl-utils + */ + function createProgramInfo( + gl, + shaderSources, + opt_attribs, + opt_locations, + opt_errorCallback, + ) { + shaderSources = shaderSources.map(function (source) { + const script = document.getElementById(source); + return script ? script.text : source; + }); + const program = webglUtils.createProgramFromSources( + gl, + shaderSources, + opt_attribs, + opt_locations, + opt_errorCallback, + ); + if (!program) { + return null; + } + const uniformSetters = createUniformSetters(gl, program); + const attribSetters = createAttributeSetters(gl, program); + return { + program: program, + uniformSetters: uniformSetters, + attribSetters: attribSetters, + }; + } + + /** + * Sets attributes and buffers including the `ELEMENT_ARRAY_BUFFER` if appropriate + * + * Example: + * + * let programInfo = createProgramInfo( + * gl, ["some-vs", "some-fs"]); + * + * let arrays = { + * position: { numComponents: 3, data: [0, 0, 0, 10, 0, 0, 0, 10, 0, 10, 10, 0], }, + * texcoord: { numComponents: 2, data: [0, 0, 0, 1, 1, 0, 1, 1], }, + * }; + * + * let bufferInfo = createBufferInfoFromArrays(gl, arrays); + * + * gl.useProgram(programInfo.program); + * + * This will automatically bind the buffers AND set the + * attributes. + * + * setBuffersAndAttributes(programInfo.attribSetters, bufferInfo); + * + * For the example above it is equivilent to + * + * gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer); + * gl.enableVertexAttribArray(a_positionLocation); + * gl.vertexAttribPointer(a_positionLocation, 3, gl.FLOAT, false, 0, 0); + * gl.bindBuffer(gl.ARRAY_BUFFER, texcoordBuffer); + * gl.enableVertexAttribArray(a_texcoordLocation); + * gl.vertexAttribPointer(a_texcoordLocation, 4, gl.FLOAT, false, 0, 0); + * + * @param {WebGLRenderingContext} gl A WebGLRenderingContext. + * @param {Object.} setters Attribute setters as returned from `createAttributeSetters` + * @param {module:webgl-utils.BufferInfo} buffers a BufferInfo as returned from `createBufferInfoFromArrays`. + * @memberOf module:webgl-utils + */ + function setBuffersAndAttributes(gl, setters, buffers) { + setAttributes(setters, buffers.attribs); + if (buffers.indices) { + gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, buffers.indices); + } + } + + // Add your prefix here. + const browserPrefixes = ['', 'MOZ_', 'OP_', 'WEBKIT_']; + + /** + * Given an extension name like WEBGL_compressed_texture_s3tc + * returns the supported version extension, like + * WEBKIT_WEBGL_compressed_teture_s3tc + * @param {string} name Name of extension to look for + * @return {WebGLExtension} The extension or undefined if not + * found. + * @memberOf module:webgl-utils + */ + function getExtensionWithKnownPrefixes(gl, name) { + for (let ii = 0; ii < browserPrefixes.length; ++ii) { + const prefixedName = browserPrefixes[ii] + name; + const ext = gl.getExtension(prefixedName); + if (ext) { + return ext; + } + } + return undefined; + } + + /** + * Resize a canvas to match the size its displayed. + * @param {HTMLCanvasElement} canvas The canvas to resize. + * @param {number} [multiplier] amount to multiply by. + * Pass in window.devicePixelRatio for native pixels. + * @return {boolean} true if the canvas was resized. + * @memberOf module:webgl-utils + */ + function resizeCanvasToDisplaySize(canvas, multiplier) { + multiplier = multiplier || 1; + const width = (canvas.clientWidth * multiplier) | 0; + const height = (canvas.clientHeight * multiplier) | 0; + if (canvas.width !== width || canvas.height !== height) { + canvas.width = width; + canvas.height = height; + return true; + } + return false; + } + + // Add `push` to a typed array. It just keeps a 'cursor' + // and allows use to `push` values into the array so we + // don't have to manually compute offsets + function augmentTypedArray(typedArray, numComponents) { + let cursor = 0; + typedArray.push = function () { + for (let ii = 0; ii < arguments.length; ++ii) { + const value = arguments[ii]; + if ( + value instanceof Array || + (value.buffer && value.buffer instanceof ArrayBuffer) + ) { + for (let jj = 0; jj < value.length; ++jj) { + typedArray[cursor++] = value[jj]; + } + } else { + typedArray[cursor++] = value; + } + } + }; + typedArray.reset = function (opt_index) { + cursor = opt_index || 0; + }; + typedArray.numComponents = numComponents; + Object.defineProperty(typedArray, 'numElements', { + get: function () { + return (this.length / this.numComponents) | 0; + }, + }); + return typedArray; + } + + /** + * creates a typed array with a `push` function attached + * so that you can easily *push* values. + * + * `push` can take multiple arguments. If an argument is an array each element + * of the array will be added to the typed array. + * + * Example: + * + * let array = createAugmentedTypedArray(3, 2); // creates a Float32Array with 6 values + * array.push(1, 2, 3); + * array.push([4, 5, 6]); + * // array now contains [1, 2, 3, 4, 5, 6] + * + * Also has `numComponents` and `numElements` properties. + * + * @param {number} numComponents number of components + * @param {number} numElements number of elements. The total size of the array will be `numComponents * numElements`. + * @param {constructor} opt_type A constructor for the type. Default = `Float32Array`. + * @return {ArrayBuffer} A typed array. + * @memberOf module:webgl-utils + */ + function createAugmentedTypedArray(numComponents, numElements, opt_type) { + const Type = opt_type || Float32Array; + return augmentTypedArray( + new Type(numComponents * numElements), + numComponents, + ); + } + + function createBufferFromTypedArray(gl, array, type, drawType) { + type = type || gl.ARRAY_BUFFER; + const buffer = gl.createBuffer(); + gl.bindBuffer(type, buffer); + gl.bufferData(type, array, drawType || gl.STATIC_DRAW); + return buffer; + } + + function allButIndices(name) { + return name !== 'indices'; + } + + function createMapping(obj) { + const mapping = {}; + Object.keys(obj) + .filter(allButIndices) + .forEach(function (key) { + mapping['a_' + key] = key; + }); + return mapping; + } + + function getGLTypeForTypedArray(gl, typedArray) { + if (typedArray instanceof Int8Array) { + return gl.BYTE; + } // eslint-disable-line + if (typedArray instanceof Uint8Array) { + return gl.UNSIGNED_BYTE; + } // eslint-disable-line + if (typedArray instanceof Int16Array) { + return gl.SHORT; + } // eslint-disable-line + if (typedArray instanceof Uint16Array) { + return gl.UNSIGNED_SHORT; + } // eslint-disable-line + if (typedArray instanceof Int32Array) { + return gl.INT; + } // eslint-disable-line + if (typedArray instanceof Uint32Array) { + return gl.UNSIGNED_INT; + } // eslint-disable-line + if (typedArray instanceof Float32Array) { + return gl.FLOAT; + } // eslint-disable-line + throw 'unsupported typed array type'; + } + + // This is really just a guess. Though I can't really imagine using + // anything else? Maybe for some compression? + function getNormalizationForTypedArray(typedArray) { + if (typedArray instanceof Int8Array) { + return true; + } // eslint-disable-line + if (typedArray instanceof Uint8Array) { + return true; + } // eslint-disable-line + return false; + } + + function isArrayBuffer(a) { + return a.buffer && a.buffer instanceof ArrayBuffer; + } + + function guessNumComponentsFromName(name, length) { + let numComponents; + if (name.indexOf('coord') >= 0) { + numComponents = 2; + } else if (name.indexOf('color') >= 0) { + numComponents = 4; + } else { + numComponents = 3; // position, normals, indices ... + } + + if (length % numComponents > 0) { + throw 'can not guess numComponents. You should specify it.'; + } + + return numComponents; + } + + function makeTypedArray(array, name) { + if (isArrayBuffer(array)) { + return array; + } + + if (array.data && isArrayBuffer(array.data)) { + return array.data; + } + + if (Array.isArray(array)) { + array = { + data: array, + }; + } + + if (!array.numComponents) { + array.numComponents = guessNumComponentsFromName(name, array.length); + } + + let type = array.type; + if (!type) { + if (name === 'indices') { + type = Uint16Array; + } + } + const typedArray = createAugmentedTypedArray( + array.numComponents, + (array.data.length / array.numComponents) | 0, + type, + ); + typedArray.push(array.data); + return typedArray; + } + + /** + * @typedef {Object} AttribInfo + * @property {number} [numComponents] the number of components for this attribute. + * @property {number} [size] the number of components for this attribute. + * @property {number} [type] the type of the attribute (eg. `gl.FLOAT`, `gl.UNSIGNED_BYTE`, etc...) Default = `gl.FLOAT` + * @property {boolean} [normalized] whether or not to normalize the data. Default = false + * @property {number} [offset] offset into buffer in bytes. Default = 0 + * @property {number} [stride] the stride in bytes per element. Default = 0 + * @property {WebGLBuffer} buffer the buffer that contains the data for this attribute + * @memberOf module:webgl-utils + */ + + /** + * Creates a set of attribute data and WebGLBuffers from set of arrays + * + * Given + * + * let arrays = { + * position: { numComponents: 3, data: [0, 0, 0, 10, 0, 0, 0, 10, 0, 10, 10, 0], }, + * texcoord: { numComponents: 2, data: [0, 0, 0, 1, 1, 0, 1, 1], }, + * normal: { numComponents: 3, data: [0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1], }, + * color: { numComponents: 4, data: [255, 255, 255, 255, 255, 0, 0, 255, 0, 0, 255, 255], type: Uint8Array, }, + * indices: { numComponents: 3, data: [0, 1, 2, 1, 2, 3], }, + * }; + * + * returns something like + * + * let attribs = { + * a_position: { numComponents: 3, type: gl.FLOAT, normalize: false, buffer: WebGLBuffer, }, + * a_texcoord: { numComponents: 2, type: gl.FLOAT, normalize: false, buffer: WebGLBuffer, }, + * a_normal: { numComponents: 3, type: gl.FLOAT, normalize: false, buffer: WebGLBuffer, }, + * a_color: { numComponents: 4, type: gl.UNSIGNED_BYTE, normalize: true, buffer: WebGLBuffer, }, + * }; + * + * @param {WebGLRenderingContext} gl The webgl rendering context. + * @param {Object.} arrays The arrays + * @param {Object.} [opt_mapping] mapping from attribute name to array name. + * if not specified defaults to "a_name" -> "name". + * @return {Object.} the attribs + * @memberOf module:webgl-utils + */ + function createAttribsFromArrays(gl, arrays, opt_mapping) { + const mapping = opt_mapping || createMapping(arrays); + const attribs = {}; + Object.keys(mapping).forEach(function (attribName) { + const bufferName = mapping[attribName]; + const origArray = arrays[bufferName]; + if (origArray.value) { + attribs[attribName] = { + value: origArray.value, + }; + } else { + const array = makeTypedArray(origArray, bufferName); + attribs[attribName] = { + buffer: createBufferFromTypedArray(gl, array), + numComponents: + origArray.numComponents || + array.numComponents || + guessNumComponentsFromName(bufferName), + type: getGLTypeForTypedArray(gl, array), + normalize: getNormalizationForTypedArray(array), + }; + } + }); + return attribs; + } + + function getArray(array) { + return array.length ? array : array.data; + } + + const texcoordRE = /coord|texture/i; + const colorRE = /color|colour/i; + + function guessNumComponentsFromName(name, length) { + let numComponents; + if (texcoordRE.test(name)) { + numComponents = 2; + } else if (colorRE.test(name)) { + numComponents = 4; + } else { + numComponents = 3; // position, normals, indices ... + } + + if (length % numComponents > 0) { + throw new Error( + `Can not guess numComponents for attribute '${name}'. Tried ${numComponents} but ${length} values is not evenly divisible by ${numComponents}. You should specify it.`, + ); + } + + return numComponents; + } + + function getNumComponents(array, arrayName) { + return ( + array.numComponents || + array.size || + guessNumComponentsFromName(arrayName, getArray(array).length) + ); + } + + /** + * tries to get the number of elements from a set of arrays. + */ + const positionKeys = ['position', 'positions', 'a_position']; + function getNumElementsFromNonIndexedArrays(arrays) { + let key; + for (const k of positionKeys) { + if (k in arrays) { + key = k; + break; + } + } + key = key || Object.keys(arrays)[0]; + const array = arrays[key]; + const length = getArray(array).length; + const numComponents = getNumComponents(array, key); + const numElements = length / numComponents; + if (length % numComponents > 0) { + throw new Error( + `numComponents ${numComponents} not correct for length ${length}`, + ); + } + return numElements; + } + + /** + * @typedef {Object} BufferInfo + * @property {number} numElements The number of elements to pass to `gl.drawArrays` or `gl.drawElements`. + * @property {WebGLBuffer} [indices] The indices `ELEMENT_ARRAY_BUFFER` if any indices exist. + * @property {Object.} attribs The attribs approriate to call `setAttributes` + * @memberOf module:webgl-utils + */ + + /** + * Creates a BufferInfo from an object of arrays. + * + * This can be passed to {@link module:webgl-utils.setBuffersAndAttributes} and to + * {@link module:webgl-utils:drawBufferInfo}. + * + * Given an object like + * + * let arrays = { + * position: { numComponents: 3, data: [0, 0, 0, 10, 0, 0, 0, 10, 0, 10, 10, 0], }, + * texcoord: { numComponents: 2, data: [0, 0, 0, 1, 1, 0, 1, 1], }, + * normal: { numComponents: 3, data: [0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1], }, + * indices: { numComponents: 3, data: [0, 1, 2, 1, 2, 3], }, + * }; + * + * Creates an BufferInfo like this + * + * bufferInfo = { + * numElements: 4, // or whatever the number of elements is + * indices: WebGLBuffer, // this property will not exist if there are no indices + * attribs: { + * a_position: { buffer: WebGLBuffer, numComponents: 3, }, + * a_normal: { buffer: WebGLBuffer, numComponents: 3, }, + * a_texcoord: { buffer: WebGLBuffer, numComponents: 2, }, + * }, + * }; + * + * The properties of arrays can be JavaScript arrays in which case the number of components + * will be guessed. + * + * let arrays = { + * position: [0, 0, 0, 10, 0, 0, 0, 10, 0, 10, 10, 0], + * texcoord: [0, 0, 0, 1, 1, 0, 1, 1], + * normal: [0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1], + * indices: [0, 1, 2, 1, 2, 3], + * }; + * + * They can also by TypedArrays + * + * let arrays = { + * position: new Float32Array([0, 0, 0, 10, 0, 0, 0, 10, 0, 10, 10, 0]), + * texcoord: new Float32Array([0, 0, 0, 1, 1, 0, 1, 1]), + * normal: new Float32Array([0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1]), + * indices: new Uint16Array([0, 1, 2, 1, 2, 3]), + * }; + * + * Or augmentedTypedArrays + * + * let positions = createAugmentedTypedArray(3, 4); + * let texcoords = createAugmentedTypedArray(2, 4); + * let normals = createAugmentedTypedArray(3, 4); + * let indices = createAugmentedTypedArray(3, 2, Uint16Array); + * + * positions.push([0, 0, 0, 10, 0, 0, 0, 10, 0, 10, 10, 0]); + * texcoords.push([0, 0, 0, 1, 1, 0, 1, 1]); + * normals.push([0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1]); + * indices.push([0, 1, 2, 1, 2, 3]); + * + * let arrays = { + * position: positions, + * texcoord: texcoords, + * normal: normals, + * indices: indices, + * }; + * + * For the last example it is equivalent to + * + * let bufferInfo = { + * attribs: { + * a_position: { numComponents: 3, buffer: gl.createBuffer(), }, + * a_texcoods: { numComponents: 2, buffer: gl.createBuffer(), }, + * a_normals: { numComponents: 3, buffer: gl.createBuffer(), }, + * }, + * indices: gl.createBuffer(), + * numElements: 6, + * }; + * + * gl.bindBuffer(gl.ARRAY_BUFFER, bufferInfo.attribs.a_position.buffer); + * gl.bufferData(gl.ARRAY_BUFFER, arrays.position, gl.STATIC_DRAW); + * gl.bindBuffer(gl.ARRAY_BUFFER, bufferInfo.attribs.a_texcoord.buffer); + * gl.bufferData(gl.ARRAY_BUFFER, arrays.texcoord, gl.STATIC_DRAW); + * gl.bindBuffer(gl.ARRAY_BUFFER, bufferInfo.attribs.a_normal.buffer); + * gl.bufferData(gl.ARRAY_BUFFER, arrays.normal, gl.STATIC_DRAW); + * gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, bufferInfo.indices); + * gl.bufferData(gl.ELEMENT_ARRAY_BUFFER, arrays.indices, gl.STATIC_DRAW); + * + * @param {WebGLRenderingContext} gl A WebGLRenderingContext + * @param {Object.} arrays Your data + * @param {Object.} [opt_mapping] an optional mapping of attribute to array name. + * If not passed in it's assumed the array names will be mapped to an attribute + * of the same name with "a_" prefixed to it. An other words. + * + * let arrays = { + * position: ..., + * texcoord: ..., + * normal: ..., + * indices: ..., + * }; + * + * bufferInfo = createBufferInfoFromArrays(gl, arrays); + * + * Is the same as + * + * let arrays = { + * position: ..., + * texcoord: ..., + * normal: ..., + * indices: ..., + * }; + * + * let mapping = { + * a_position: "position", + * a_texcoord: "texcoord", + * a_normal: "normal", + * }; + * + * bufferInfo = createBufferInfoFromArrays(gl, arrays, mapping); + * + * @return {module:webgl-utils.BufferInfo} A BufferInfo + * @memberOf module:webgl-utils + */ + function createBufferInfoFromArrays(gl, arrays, opt_mapping) { + const bufferInfo = { + attribs: createAttribsFromArrays(gl, arrays, opt_mapping), + }; + let indices = arrays.indices; + if (indices) { + indices = makeTypedArray(indices, 'indices'); + bufferInfo.indices = createBufferFromTypedArray( + gl, + indices, + gl.ELEMENT_ARRAY_BUFFER, + ); + bufferInfo.numElements = indices.length; + } else { + bufferInfo.numElements = getNumElementsFromNonIndexedArrays(arrays); + } + + return bufferInfo; + } + + /** + * Creates buffers from typed arrays + * + * Given something like this + * + * let arrays = { + * positions: [1, 2, 3], + * normals: [0, 0, 1], + * } + * + * returns something like + * + * buffers = { + * positions: WebGLBuffer, + * normals: WebGLBuffer, + * } + * + * If the buffer is named 'indices' it will be made an ELEMENT_ARRAY_BUFFER. + * + * @param {WebGLRenderingContext} gl A WebGLRenderingContext. + * @param {Object} arrays + * @return {Object} returns an object with one WebGLBuffer per array + * @memberOf module:webgl-utils + */ + function createBuffersFromArrays(gl, arrays) { + const buffers = {}; + Object.keys(arrays).forEach(function (key) { + const type = + key === 'indices' ? gl.ELEMENT_ARRAY_BUFFER : gl.ARRAY_BUFFER; + const array = makeTypedArray(arrays[key], name); + buffers[key] = createBufferFromTypedArray(gl, array, type); + }); + + // hrm + if (arrays.indices) { + buffers.numElements = arrays.indices.length; + } else if (arrays.position) { + buffers.numElements = arrays.position.length / 3; + } + + return buffers; + } + + /** + * Calls `gl.drawElements` or `gl.drawArrays`, whichever is appropriate + * + * normally you'd call `gl.drawElements` or `gl.drawArrays` yourself + * but calling this means if you switch from indexed data to non-indexed + * data you don't have to remember to update your draw call. + * + * @param {WebGLRenderingContext} gl A WebGLRenderingContext + * @param {module:webgl-utils.BufferInfo} bufferInfo as returned from createBufferInfoFromArrays + * @param {enum} [primitiveType] eg (gl.TRIANGLES, gl.LINES, gl.POINTS, gl.TRIANGLE_STRIP, ...) + * @param {number} [count] An optional count. Defaults to bufferInfo.numElements + * @param {number} [offset] An optional offset. Defaults to 0. + * @memberOf module:webgl-utils + */ + function drawBufferInfo(gl, bufferInfo, primitiveType, count, offset) { + const indices = bufferInfo.indices; + primitiveType = primitiveType === undefined ? gl.TRIANGLES : primitiveType; + const numElements = count === undefined ? bufferInfo.numElements : count; + offset = offset === undefined ? 0 : offset; + if (indices) { + gl.drawElements(primitiveType, numElements, gl.UNSIGNED_SHORT, offset); + } else { + gl.drawArrays(primitiveType, offset, numElements); + } + } + + /** + * @typedef {Object} DrawObject + * @property {module:webgl-utils.ProgramInfo} programInfo A ProgramInfo as returned from createProgramInfo + * @property {module:webgl-utils.BufferInfo} bufferInfo A BufferInfo as returned from createBufferInfoFromArrays + * @property {Object} uniforms The values for the uniforms + * @memberOf module:webgl-utils + */ + + /** + * Draws a list of objects + * @param {WebGLRenderingContext} gl A WebGLRenderingContext + * @param {DrawObject[]} objectsToDraw an array of objects to draw. + * @memberOf module:webgl-utils + */ + function drawObjectList(gl, objectsToDraw) { + let lastUsedProgramInfo = null; + let lastUsedBufferInfo = null; + + objectsToDraw.forEach(function (object) { + const programInfo = object.programInfo; + const bufferInfo = object.bufferInfo; + let bindBuffers = false; + + if (programInfo !== lastUsedProgramInfo) { + lastUsedProgramInfo = programInfo; + gl.useProgram(programInfo.program); + bindBuffers = true; + } + + // Setup all the needed attributes. + if (bindBuffers || bufferInfo !== lastUsedBufferInfo) { + lastUsedBufferInfo = bufferInfo; + setBuffersAndAttributes(gl, programInfo.attribSetters, bufferInfo); + } + + // Set the uniforms. + setUniforms(programInfo.uniformSetters, object.uniforms); + + // Draw + drawBufferInfo(gl, bufferInfo); + }); + } + + function glEnumToString(gl, v) { + const results = []; + for (const key in gl) { + if (gl[key] === v) { + results.push(key); + } + } + return results.length ? results.join(' | ') : `0x${v.toString(16)}`; + } + + const isIE = /*@cc_on!@*/ false || !!document.documentMode; + // Edge 20+ + const isEdge = !isIE && !!window.StyleMedia; + if (isEdge) { + // Hack for Edge. Edge's WebGL implmentation is crap still and so they + // only respond to "experimental-webgl". I don't want to clutter the + // examples with that so his hack works around it + HTMLCanvasElement.prototype.getContext = (function (origFn) { + return function () { + let args = arguments; + const type = args[0]; + if (type === 'webgl') { + args = [].slice.call(arguments); + args[0] = 'experimental-webgl'; + } + return origFn.apply(this, args); + }; + })(HTMLCanvasElement.prototype.getContext); + } + + return { + createAugmentedTypedArray: createAugmentedTypedArray, + createAttribsFromArrays: createAttribsFromArrays, + createBuffersFromArrays: createBuffersFromArrays, + createBufferInfoFromArrays: createBufferInfoFromArrays, + createAttributeSetters: createAttributeSetters, + createProgram: createProgram, + createProgramFromScripts: createProgramFromScripts, + createProgramFromSources: createProgramFromSources, + createProgramInfo: createProgramInfo, + createUniformSetters: createUniformSetters, + createVAOAndSetAttributes: createVAOAndSetAttributes, + createVAOFromBufferInfo: createVAOFromBufferInfo, + drawBufferInfo: drawBufferInfo, + drawObjectList: drawObjectList, + glEnumToString: glEnumToString, + getExtensionWithKnownPrefixes: getExtensionWithKnownPrefixes, + resizeCanvasToDisplaySize: resizeCanvasToDisplaySize, + setAttributes: setAttributes, + setBuffersAndAttributes: setBuffersAndAttributes, + setUniforms: setUniforms, + }; +}); diff --git a/packages/rrweb/test/html/canvas-webgl-image.html b/packages/rrweb/test/html/canvas-webgl-image.html new file mode 100644 index 0000000000..96bc31031e --- /dev/null +++ b/packages/rrweb/test/html/canvas-webgl-image.html @@ -0,0 +1,149 @@ + + + + + + + Document + + + + + + + + + + + + From f2841778a7cc3e143027989eef2e84549a439b07 Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Thu, 23 Dec 2021 15:34:29 +0100 Subject: [PATCH 72/93] don't turn on devtools by default! --- packages/rrweb/test/utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/rrweb/test/utils.ts b/packages/rrweb/test/utils.ts index 8f08920723..cae95b0420 100644 --- a/packages/rrweb/test/utils.ts +++ b/packages/rrweb/test/utils.ts @@ -19,7 +19,7 @@ export async function launchPuppeteer() { width: 1920, height: 1080, }, - devtools: true, + // devtools: true, args: ['--no-sandbox'], }); } From dd94aa91d266b54fbed6ddda56a1ff57e2ef8107 Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Thu, 23 Dec 2021 15:47:23 +0100 Subject: [PATCH 73/93] Remove spector --- packages/rrweb/package.json | 1 - yarn.lock | 19 ------------------- 2 files changed, 20 deletions(-) diff --git a/packages/rrweb/package.json b/packages/rrweb/package.json index ef5d283650..6df993a401 100644 --- a/packages/rrweb/package.json +++ b/packages/rrweb/package.json @@ -67,7 +67,6 @@ "rollup-plugin-postcss": "^3.1.1", "rollup-plugin-rename-node-modules": "^1.1.0", "rollup-plugin-terser": "^7.0.2", - "spector": "^0.0.0", "ts-jest": "^27.0.5", "ts-node": "^7.0.1", "tslib": "^1.9.3", diff --git a/yarn.lock b/yarn.lock index 79feba9fea..60f89aa6ad 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2757,13 +2757,6 @@ cli-cursor@^3.1.0: dependencies: restore-cursor "^3.1.0" -cli-table@*: - version "0.3.11" - resolved "https://registry.yarnpkg.com/cli-table/-/cli-table-0.3.11.tgz#ac69cdecbe81dccdba4889b9a18b7da312a9d3ee" - integrity sha512-IqLQi4lO0nIB4tcdTpN4LCB9FI3uqrJZK7RC515EnhZ6qBaglkIgICb1wjeAqpdoOabm1+SuQtkXIPdYC93jhQ== - dependencies: - colors "1.0.3" - cli-width@^2.0.0: version "2.2.1" resolved "https://registry.yarnpkg.com/cli-width/-/cli-width-2.2.1.tgz#b0433d0b4e9c847ef18868a4ef16fd5fc8271c48" @@ -2873,11 +2866,6 @@ colorette@^1.2.2: resolved "https://registry.yarnpkg.com/colorette/-/colorette-1.2.2.tgz#cbcc79d5e99caea2dbf10eb3a26fd8b3e6acfa94" integrity sha512-MKGMzyfeuutC/ZJ1cba9NqcNpfeqMUcYmyF1ZFY6/Cn7CNSAKx6a+s48sqLqyAiZuaP2TcqMhoo+dlwFnVxT9w== -colors@1.0.3: - version "1.0.3" - resolved "https://registry.yarnpkg.com/colors/-/colors-1.0.3.tgz#0433f44d809680fdeb60ed260f1b0c262e82a40b" - integrity sha1-BDP0TYCWgP3rYO0mDxsMJi6CpAs= - colors@^1.1.2: version "1.4.0" resolved "https://registry.yarnpkg.com/colors/-/colors-1.4.0.tgz#c50491479d4c1bdaed2c9ced32cf7c7dc2360f78" @@ -9011,13 +8999,6 @@ spdx-license-ids@^3.0.0: resolved "https://registry.yarnpkg.com/spdx-license-ids/-/spdx-license-ids-3.0.10.tgz#0d9becccde7003d6c658d487dd48a32f0bf3014b" integrity sha512-oie3/+gKf7QtpitB0LYLETe+k8SifzsX4KixvpOsbI6S0kRiRQ5MKOio8eMSAKQ17N06+wdEOXRiId+zOxo0hA== -spector@^0.0.0: - version "0.0.0" - resolved "https://registry.yarnpkg.com/spector/-/spector-0.0.0.tgz#3be8197aa70416d76940ea17f74457871fc7b900" - integrity sha1-O+gZeqcEFtdpQOoX90RXhx/HuQA= - dependencies: - cli-table "*" - split-on-first@^1.0.0: version "1.1.0" resolved "https://registry.yarnpkg.com/split-on-first/-/split-on-first-1.1.0.tgz#f610afeee3b12bce1d0c30425e76398b78249a5f" From 6cd6cc84d446fafb038f635cc57476447a105672 Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Thu, 23 Dec 2021 16:03:45 +0100 Subject: [PATCH 74/93] close server after use --- packages/rrweb/test/e2e/webgl.test.ts | 1 + packages/rrweb/test/utils.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/rrweb/test/e2e/webgl.test.ts b/packages/rrweb/test/e2e/webgl.test.ts index 44a5653fbb..f718c75ef1 100644 --- a/packages/rrweb/test/e2e/webgl.test.ts +++ b/packages/rrweb/test/e2e/webgl.test.ts @@ -44,6 +44,7 @@ describe('e2e webgl', () => { }); afterAll(async () => { + await server.close(); await browser.close(); }); diff --git a/packages/rrweb/test/utils.ts b/packages/rrweb/test/utils.ts index cae95b0420..1b51d005de 100644 --- a/packages/rrweb/test/utils.ts +++ b/packages/rrweb/test/utils.ts @@ -57,7 +57,7 @@ export const startServer = (defaultPort: number = 3030) => res.end(); } }); - s.listen() + s.listen(defaultPort) .on('listening', () => { resolve(s); }) From 4abdfffa413cae05fb2d936bdd063ece56057af7 Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Thu, 23 Dec 2021 16:20:21 +0100 Subject: [PATCH 75/93] Add imageMap parameter --- packages/rrweb/test/replay/webgl-mutation.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/rrweb/test/replay/webgl-mutation.test.ts b/packages/rrweb/test/replay/webgl-mutation.test.ts index 478aee3ed3..bae750c785 100644 --- a/packages/rrweb/test/replay/webgl-mutation.test.ts +++ b/packages/rrweb/test/replay/webgl-mutation.test.ts @@ -38,6 +38,7 @@ describe('webglMutation', () => { args: [35633], }, target: canvas, + imageMap: new Map(), errorHandler: () => {}, }); From 354754e7906774696dd28d45da77144792027344 Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Thu, 23 Dec 2021 16:31:25 +0100 Subject: [PATCH 76/93] Make e2e image test more robust --- packages/rrweb/test/e2e/webgl.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/rrweb/test/e2e/webgl.test.ts b/packages/rrweb/test/e2e/webgl.test.ts index f718c75ef1..cf6fe9616a 100644 --- a/packages/rrweb/test/e2e/webgl.test.ts +++ b/packages/rrweb/test/e2e/webgl.test.ts @@ -155,8 +155,10 @@ describe('e2e webgl', () => { const replayer = new Replayer(events, { UNSAFE_replayCanvas: true, }); - replayer.play(500); `); + // wait for iframe to get added and `preloadAllImages` to ge called + await page.waitForSelector('iframe'); + await page.evaluate(`replayer.play(500);`); await page.waitForTimeout(50); const element = await page.$('iframe'); From ad0f9f66b3797d3cd26c882854047c99dc2a2e02 Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Wed, 5 Jan 2022 16:13:47 +0100 Subject: [PATCH 77/93] document debug mode --- packages/rrweb/src/replay/canvas/webgl.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/rrweb/src/replay/canvas/webgl.ts b/packages/rrweb/src/replay/canvas/webgl.ts index 68b4babfc8..bae9e22113 100644 --- a/packages/rrweb/src/replay/canvas/webgl.ts +++ b/packages/rrweb/src/replay/canvas/webgl.ts @@ -129,8 +129,8 @@ export default function webglMutation({ const result = original.apply(ctx, args); saveToWebGLVarMap(result); + // Slows down replay considerably, only use for debugging const debugMode = false; - // const debugMode = true; if (debugMode) { if (mutation.property === 'compileShader') { if (!ctx.getShaderParameter(args[0], ctx.COMPILE_STATUS)) From d42e7d407e821ba7cc9823c01af0860bfe3d6626 Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Wed, 5 Jan 2022 16:14:20 +0100 Subject: [PATCH 78/93] cleanup --- packages/rrweb/src/replay/canvas/webgl.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/rrweb/src/replay/canvas/webgl.ts b/packages/rrweb/src/replay/canvas/webgl.ts index bae9e22113..63b00e9675 100644 --- a/packages/rrweb/src/replay/canvas/webgl.ts +++ b/packages/rrweb/src/replay/canvas/webgl.ts @@ -70,11 +70,8 @@ export function deserializeArg( return variableListFor(name)[index]; } else if ('args' in arg) { const { rr_type: name, args } = arg; + const ctor = window[name as keyof Window]; - // @ts-ignore - const ctor = window[name] as unknown; - - // @ts-ignore return new ctor(...args.map(deserializeArg(imageMap))); } else if ('base64' in arg) { return decode(arg.base64); From 5c2e1bd74889f4d0ece99a12e15720e5e61bca70 Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Wed, 5 Jan 2022 16:17:43 +0100 Subject: [PATCH 79/93] WebGL recording in iframes & Safari 14 support --- .../record/observers/canvas/serialize-args.ts | 92 +++++++++++-------- .../src/record/observers/canvas/webgl.ts | 7 +- packages/rrweb/src/replay/canvas/webgl.ts | 25 +++-- .../observers/canvas/serialize-args.d.ts | 9 +- 4 files changed, 73 insertions(+), 60 deletions(-) diff --git a/packages/rrweb/src/record/observers/canvas/serialize-args.ts b/packages/rrweb/src/record/observers/canvas/serialize-args.ts index 29d4dba44d..1209a820db 100644 --- a/packages/rrweb/src/record/observers/canvas/serialize-args.ts +++ b/packages/rrweb/src/record/observers/canvas/serialize-args.ts @@ -1,11 +1,11 @@ import { encode } from 'base64-arraybuffer'; -import { SerializedWebGlArg } from '../../../types'; +import { IWindow, SerializedWebGlArg } from '../../../types'; // from webgl-recorder: https://github.com/evanw/webgl-recorder/blob/bef0e65596e981ee382126587e2dcbe0fc7748e2/webgl-recorder.js#L50-L77 const webGLVars: Record> = {}; -export function serializeArg(value: any): SerializedWebGlArg { +export function serializeArg(value: any, win: IWindow): SerializedWebGlArg { if (value instanceof Array) { - return value.map(serializeArg); + return value.map((arg) => serializeArg(arg, win)); } else if (value === null) { return value; } else if ( @@ -41,7 +41,11 @@ export function serializeArg(value: any): SerializedWebGlArg { const name = value.constructor.name; return { rr_type: name, - args: [serializeArg(value.buffer), value.byteOffset, value.byteLength], + args: [ + serializeArg(value.buffer, win), + value.byteOffset, + value.byteLength, + ], }; } else if (value instanceof HTMLImageElement) { const name = value.constructor.name; @@ -54,23 +58,9 @@ export function serializeArg(value: any): SerializedWebGlArg { const name = value.constructor.name; return { rr_type: name, - args: [serializeArg(value.data), value.width, value.height], + args: [serializeArg(value.data, win), value.width, value.height], }; - } else if ( - value instanceof WebGLActiveInfo || - value instanceof WebGLBuffer || - value instanceof WebGLFramebuffer || - value instanceof WebGLProgram || - value instanceof WebGLRenderbuffer || - value instanceof WebGLShader || - value instanceof WebGLShaderPrecisionFormat || - value instanceof WebGLTexture || - value instanceof WebGLUniformLocation || - value instanceof WebGLVertexArrayObject || - // In Chrome, value won't be an instanceof WebGLVertexArrayObject. - (value && value.constructor.name == 'WebGLVertexArrayObjectOES') || - typeof value === 'object' - ) { + } else if (isInstanceOfWebGLObject(value, win) || typeof value === 'object') { const name = value.constructor.name; const list = webGLVars[name] || (webGLVars[name] = []); let index = list.indexOf(value); @@ -89,28 +79,50 @@ export function serializeArg(value: any): SerializedWebGlArg { return value; } -export const serializeArgs = (args: Array) => { - return [...args].map(serializeArg); +export const serializeArgs = (args: Array, win: IWindow) => { + return [...args].map((arg) => serializeArg(arg, win)); +}; + +export const isInstanceOfWebGLObject = ( + value: any, + win: IWindow, +): value is + | WebGLActiveInfo + | WebGLBuffer + | WebGLFramebuffer + | WebGLProgram + | WebGLRenderbuffer + | WebGLShader + | WebGLShaderPrecisionFormat + | WebGLTexture + | WebGLUniformLocation + | WebGLVertexArrayObject => { + const webGLConstructorNames: string[] = [ + 'WebGLActiveInfo', + 'WebGLBuffer', + 'WebGLFramebuffer', + 'WebGLProgram', + 'WebGLRenderbuffer', + 'WebGLShader', + 'WebGLShaderPrecisionFormat', + 'WebGLTexture', + 'WebGLUniformLocation', + 'WebGLVertexArrayObject', + // In old Chrome versions, value won't be an instanceof WebGLVertexArrayObject. + 'WebGLVertexArrayObjectOES', + ]; + const supportedWebGLConstructorNames = webGLConstructorNames.filter( + (name: string) => typeof win[name as keyof Window] === 'function', + ); + return Boolean( + supportedWebGLConstructorNames.find( + (name: string) => value instanceof win[name as keyof Window], + ), + ); }; -export const saveWebGLVar = (value: any): number | void => { - if ( - !( - value instanceof WebGLActiveInfo || - value instanceof WebGLBuffer || - value instanceof WebGLFramebuffer || - value instanceof WebGLProgram || - value instanceof WebGLRenderbuffer || - value instanceof WebGLShader || - value instanceof WebGLShaderPrecisionFormat || - value instanceof WebGLTexture || - value instanceof WebGLUniformLocation || - value instanceof WebGLVertexArrayObject || - // In Chrome, value won't be an instanceof WebGLVertexArrayObject. - (value && value.constructor.name == 'WebGLVertexArrayObjectOES') || - typeof value === 'object' - ) - ) +export const saveWebGLVar = (value: any, win: IWindow): number | void => { + if (!(isInstanceOfWebGLObject(value, win) || typeof value === 'object')) return; const name = value.constructor.name; diff --git a/packages/rrweb/src/record/observers/canvas/webgl.ts b/packages/rrweb/src/record/observers/canvas/webgl.ts index d93630676a..8fb205009b 100644 --- a/packages/rrweb/src/record/observers/canvas/webgl.ts +++ b/packages/rrweb/src/record/observers/canvas/webgl.ts @@ -52,6 +52,7 @@ function patchGLPrototype( mirror: Mirror, pendingCanvasMutations: pendingCanvasMutationsMap, rafStamps: RafStamps, + win: IWindow, ): listenerHandler[] { const handlers: listenerHandler[] = []; @@ -70,11 +71,11 @@ function patchGLPrototype( rafStamps.invokeId = rafStamps.latestId; const result = original.apply(this, args); - saveWebGLVar(result); + saveWebGLVar(result, win); if (!isBlocked((this.canvas as unknown) as INode, blockClass)) { const id = mirror.getId((this.canvas as unknown) as INode); - const recordArgs = serializeArgs([...args]); + const recordArgs = serializeArgs([...args], win); const mutation: canvasMutationParam = { id, type, @@ -166,6 +167,7 @@ export default function initCanvasWebGLMutationObserver( mirror, pendingCanvasMutations, rafStamps, + win, ), ); @@ -179,6 +181,7 @@ export default function initCanvasWebGLMutationObserver( mirror, pendingCanvasMutations, rafStamps, + win, ), ); } diff --git a/packages/rrweb/src/replay/canvas/webgl.ts b/packages/rrweb/src/replay/canvas/webgl.ts index 63b00e9675..45f7edafdd 100644 --- a/packages/rrweb/src/replay/canvas/webgl.ts +++ b/packages/rrweb/src/replay/canvas/webgl.ts @@ -34,21 +34,18 @@ function getContext( } } -const WebGLVariableConstructors = [ - WebGLActiveInfo, - WebGLBuffer, - WebGLFramebuffer, - WebGLProgram, - WebGLRenderbuffer, - WebGLShader, - WebGLShaderPrecisionFormat, - WebGLTexture, - WebGLUniformLocation, - WebGLVertexArrayObject, +const WebGLVariableConstructorsNames = [ + 'WebGLActiveInfo', + 'WebGLBuffer', + 'WebGLFramebuffer', + 'WebGLProgram', + 'WebGLRenderbuffer', + 'WebGLShader', + 'WebGLShaderPrecisionFormat', + 'WebGLTexture', + 'WebGLUniformLocation', + 'WebGLVertexArrayObject', ]; -const WebGLVariableConstructorsNames = WebGLVariableConstructors.map( - (ctor) => ctor.name, -); function saveToWebGLVarMap(result: any) { if (!result?.constructor) return; // probably null or undefined diff --git a/packages/rrweb/typings/record/observers/canvas/serialize-args.d.ts b/packages/rrweb/typings/record/observers/canvas/serialize-args.d.ts index 0f625d7a32..599e2a6bb7 100644 --- a/packages/rrweb/typings/record/observers/canvas/serialize-args.d.ts +++ b/packages/rrweb/typings/record/observers/canvas/serialize-args.d.ts @@ -1,4 +1,5 @@ -import { SerializedWebGlArg } from '../../../types'; -export declare function serializeArg(value: any): SerializedWebGlArg; -export declare const serializeArgs: (args: Array) => SerializedWebGlArg[]; -export declare const saveWebGLVar: (value: any) => number | void; +import { IWindow, SerializedWebGlArg } from '../../../types'; +export declare function serializeArg(value: any, win: IWindow): SerializedWebGlArg; +export declare const serializeArgs: (args: Array, win: IWindow) => SerializedWebGlArg[]; +export declare const isInstanceOfWebGLObject: (value: any, win: IWindow) => value is WebGLShader | WebGLBuffer | WebGLVertexArrayObject | WebGLTexture | WebGLProgram | WebGLActiveInfo | WebGLUniformLocation | WebGLFramebuffer | WebGLRenderbuffer | WebGLShaderPrecisionFormat; +export declare const saveWebGLVar: (value: any, win: IWindow) => number | void; From 1e9589125587eba20459d8eadb55c6ac694b5229 Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Thu, 6 Jan 2022 11:39:11 +0100 Subject: [PATCH 80/93] fix tests --- .../rrweb/test/record/serialize-args.test.ts | 24 +++++++++---------- .../typings/record/observers/canvas-2d.d.ts | 2 -- .../record/observers/canvas-web-gl.d.ts | 3 --- 3 files changed, 12 insertions(+), 17 deletions(-) delete mode 100644 packages/rrweb/typings/record/observers/canvas-2d.d.ts delete mode 100644 packages/rrweb/typings/record/observers/canvas-web-gl.d.ts diff --git a/packages/rrweb/test/record/serialize-args.test.ts b/packages/rrweb/test/record/serialize-args.test.ts index 6a719e9e94..0a547ce32e 100644 --- a/packages/rrweb/test/record/serialize-args.test.ts +++ b/packages/rrweb/test/record/serialize-args.test.ts @@ -13,7 +13,7 @@ describe('serializeArg', () => { rr_type: 'Float32Array', args: [[-1, -1, 3, -1, -1, 3]], }; - expect(serializeArg(float32Array)).toStrictEqual(expected); + expect(serializeArg(float32Array, window)).toStrictEqual(expected); }); it('should serialize Float64Array values', async () => { @@ -23,7 +23,7 @@ describe('serializeArg', () => { args: [[-1, -1, 3, -1, -1, 3]], }; - expect(serializeArg(float64Array)).toStrictEqual(expected); + expect(serializeArg(float64Array, window)).toStrictEqual(expected); }); it('should serialize ArrayBuffer values', async () => { @@ -33,7 +33,7 @@ describe('serializeArg', () => { base64: 'AQIABA==', }; - expect(serializeArg(arrayBuffer)).toStrictEqual(expected); + expect(serializeArg(arrayBuffer, window)).toStrictEqual(expected); }); it('should serialize Uint8Array values', async () => { @@ -43,7 +43,7 @@ describe('serializeArg', () => { args: [[1, 2, 0, 4]], }; - expect(serializeArg(object)).toStrictEqual(expected); + expect(serializeArg(object, window)).toStrictEqual(expected); }); it('should serialize DataView values', async () => { @@ -60,12 +60,12 @@ describe('serializeArg', () => { ], }; - expect(serializeArg(dataView)).toStrictEqual(expected); + expect(serializeArg(dataView, window)).toStrictEqual(expected); }); it('should leave arrays intact', async () => { const array = [1, 2, 3, 4]; - expect(serializeArg(array)).toStrictEqual(array); + expect(serializeArg(array, window)).toStrictEqual(array); }); it('should serialize complex objects', async () => { @@ -86,7 +86,7 @@ describe('serializeArg', () => { 6, ]; - expect(serializeArg(dataView)).toStrictEqual(expected); + expect(serializeArg(dataView, window)).toStrictEqual(expected); }); it('should serialize arraybuffer contents', async () => { @@ -96,16 +96,16 @@ describe('serializeArg', () => { base64: 'AACAPwAAAEAAAEBAAACAQA==', }; - expect(serializeArg(buffer)).toStrictEqual(expected); + expect(serializeArg(buffer, window)).toStrictEqual(expected); }); it('should leave null as-is', async () => { - expect(serializeArg(null)).toStrictEqual(null); + expect(serializeArg(null, window)).toStrictEqual(null); }); it('should support indexed variables', async () => { const webGLProgram = new WebGLProgram(); - expect(serializeArg(webGLProgram)).toStrictEqual({ + expect(serializeArg(webGLProgram, window)).toStrictEqual({ rr_type: 'WebGLProgram', index: 0, }); @@ -114,7 +114,7 @@ describe('serializeArg', () => { it('should support HTMLImageElements', async () => { const image = new Image(); image.src = 'http://example.com/image.png'; - expect(serializeArg(image)).toStrictEqual({ + expect(serializeArg(image, window)).toStrictEqual({ rr_type: 'HTMLImageElement', src: 'http://example.com/image.png', }); @@ -135,7 +135,7 @@ describe('serializeArg', () => { let imageData = new ImageData(arr, 200, 50); const contents = Array.from(arr); - expect(serializeArg(imageData)).toStrictEqual({ + expect(serializeArg(imageData, window)).toStrictEqual({ rr_type: 'ImageData', args: [ { diff --git a/packages/rrweb/typings/record/observers/canvas-2d.d.ts b/packages/rrweb/typings/record/observers/canvas-2d.d.ts deleted file mode 100644 index 3912e174f4..0000000000 --- a/packages/rrweb/typings/record/observers/canvas-2d.d.ts +++ /dev/null @@ -1,2 +0,0 @@ -import { blockClass, canvasMutationCallback, IWindow, listenerHandler, Mirror } from '../../types'; -export default function initCanvas2DMutationObserver(cb: canvasMutationCallback, win: IWindow, blockClass: blockClass, mirror: Mirror): listenerHandler; diff --git a/packages/rrweb/typings/record/observers/canvas-web-gl.d.ts b/packages/rrweb/typings/record/observers/canvas-web-gl.d.ts deleted file mode 100644 index d2ea3d274a..0000000000 --- a/packages/rrweb/typings/record/observers/canvas-web-gl.d.ts +++ /dev/null @@ -1,3 +0,0 @@ -import { blockClass, canvasMutationCallback, IWindow, listenerHandler, Mirror, SerializedWebGlArg } from '../../types'; -export declare function serializeArg(value: any): SerializedWebGlArg; -export default function initCanvasWebGLMutationObserver(cb: canvasMutationCallback, win: IWindow, blockClass: blockClass, mirror: Mirror): listenerHandler; From caa577d6549fac3201962e29cc828e93b2bd45ca Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Thu, 6 Jan 2022 11:54:59 +0100 Subject: [PATCH 81/93] don't save null objects as WebGLVar --- packages/rrweb/src/record/observers/canvas/serialize-args.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/rrweb/src/record/observers/canvas/serialize-args.ts b/packages/rrweb/src/record/observers/canvas/serialize-args.ts index 1209a820db..aa83c5f4f0 100644 --- a/packages/rrweb/src/record/observers/canvas/serialize-args.ts +++ b/packages/rrweb/src/record/observers/canvas/serialize-args.ts @@ -122,7 +122,10 @@ export const isInstanceOfWebGLObject = ( }; export const saveWebGLVar = (value: any, win: IWindow): number | void => { - if (!(isInstanceOfWebGLObject(value, win) || typeof value === 'object')) + if ( + !value || + !(isInstanceOfWebGLObject(value, win) || typeof value === 'object') + ) return; const name = value.constructor.name; From 117506341939e7dc4f2f2232f784b548c4d62879 Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Thu, 6 Jan 2022 23:34:12 +0100 Subject: [PATCH 82/93] group (de)serialized webgl variables by context --- .../record/observers/canvas/serialize-args.ts | 92 ++++++++++++------- .../src/record/observers/canvas/webgl.ts | 4 +- packages/rrweb/src/replay/canvas/webgl.ts | 40 +++++--- .../rrweb/test/record/serialize-args.test.ts | 53 ++++++++--- packages/rrweb/test/utils.ts | 5 + 5 files changed, 135 insertions(+), 59 deletions(-) diff --git a/packages/rrweb/src/record/observers/canvas/serialize-args.ts b/packages/rrweb/src/record/observers/canvas/serialize-args.ts index aa83c5f4f0..e245ee4123 100644 --- a/packages/rrweb/src/record/observers/canvas/serialize-args.ts +++ b/packages/rrweb/src/record/observers/canvas/serialize-args.ts @@ -1,11 +1,57 @@ import { encode } from 'base64-arraybuffer'; import { IWindow, SerializedWebGlArg } from '../../../types'; +// TODO: unify with `replay/webgl.ts` +type GLVarMap = Map; +const webGLVarMap: Map< + WebGLRenderingContext | WebGL2RenderingContext, + GLVarMap +> = new Map(); +export function variableListFor( + ctx: WebGLRenderingContext | WebGL2RenderingContext, + ctor: string, +) { + let contextMap = webGLVarMap.get(ctx); + if (!contextMap) { + contextMap = new Map(); + webGLVarMap.set(ctx, contextMap); + } + if (!contextMap.has(ctor)) { + contextMap.set(ctor, []); + } + return contextMap.get(ctor) as any[]; +} + +export const saveWebGLVar = ( + value: any, + win: IWindow, + ctx: WebGL2RenderingContext | WebGLRenderingContext, +): number | void => { + if ( + !value || + !(isInstanceOfWebGLObject(value, win) || typeof value === 'object') + ) + return; + + const name = value.constructor.name; + const list = variableListFor(ctx, name); + let index = list.indexOf(value); + + if (index === -1) { + index = list.length; + list.push(value); + } + return index; +}; + // from webgl-recorder: https://github.com/evanw/webgl-recorder/blob/bef0e65596e981ee382126587e2dcbe0fc7748e2/webgl-recorder.js#L50-L77 -const webGLVars: Record> = {}; -export function serializeArg(value: any, win: IWindow): SerializedWebGlArg { +export function serializeArg( + value: any, + win: IWindow, + ctx: WebGL2RenderingContext | WebGLRenderingContext, +): SerializedWebGlArg { if (value instanceof Array) { - return value.map((arg) => serializeArg(arg, win)); + return value.map((arg) => serializeArg(arg, win, ctx)); } else if (value === null) { return value; } else if ( @@ -42,7 +88,7 @@ export function serializeArg(value: any, win: IWindow): SerializedWebGlArg { return { rr_type: name, args: [ - serializeArg(value.buffer, win), + serializeArg(value.buffer, win, ctx), value.byteOffset, value.byteLength, ], @@ -58,29 +104,27 @@ export function serializeArg(value: any, win: IWindow): SerializedWebGlArg { const name = value.constructor.name; return { rr_type: name, - args: [serializeArg(value.data, win), value.width, value.height], + args: [serializeArg(value.data, win, ctx), value.width, value.height], }; } else if (isInstanceOfWebGLObject(value, win) || typeof value === 'object') { const name = value.constructor.name; - const list = webGLVars[name] || (webGLVars[name] = []); - let index = list.indexOf(value); - - if (index === -1) { - index = list.length; - list.push(value); - } + const index = saveWebGLVar(value, win, ctx) as number; return { rr_type: name, - index, + index: index, }; } return value; } -export const serializeArgs = (args: Array, win: IWindow) => { - return [...args].map((arg) => serializeArg(arg, win)); +export const serializeArgs = ( + args: Array, + win: IWindow, + ctx: WebGLRenderingContext | WebGL2RenderingContext, +) => { + return [...args].map((arg) => serializeArg(arg, win, ctx)); }; export const isInstanceOfWebGLObject = ( @@ -120,21 +164,3 @@ export const isInstanceOfWebGLObject = ( ), ); }; - -export const saveWebGLVar = (value: any, win: IWindow): number | void => { - if ( - !value || - !(isInstanceOfWebGLObject(value, win) || typeof value === 'object') - ) - return; - - const name = value.constructor.name; - const list = webGLVars[name] || (webGLVars[name] = []); - let index = list.indexOf(value); - - if (index === -1) { - index = list.length; - list.push(value); - } - return index; -}; diff --git a/packages/rrweb/src/record/observers/canvas/webgl.ts b/packages/rrweb/src/record/observers/canvas/webgl.ts index 8fb205009b..f1335bba13 100644 --- a/packages/rrweb/src/record/observers/canvas/webgl.ts +++ b/packages/rrweb/src/record/observers/canvas/webgl.ts @@ -71,11 +71,11 @@ function patchGLPrototype( rafStamps.invokeId = rafStamps.latestId; const result = original.apply(this, args); - saveWebGLVar(result, win); + saveWebGLVar(result, win, prototype); if (!isBlocked((this.canvas as unknown) as INode, blockClass)) { const id = mirror.getId((this.canvas as unknown) as INode); - const recordArgs = serializeArgs([...args], win); + const recordArgs = serializeArgs([...args], win, prototype); const mutation: canvasMutationParam = { id, type, diff --git a/packages/rrweb/src/replay/canvas/webgl.ts b/packages/rrweb/src/replay/canvas/webgl.ts index 45f7edafdd..ab97332598 100644 --- a/packages/rrweb/src/replay/canvas/webgl.ts +++ b/packages/rrweb/src/replay/canvas/webgl.ts @@ -7,12 +7,24 @@ import { } from '../../types'; // TODO: add ability to wipe this list -const webGLVarMap: Map = new Map(); -export function variableListFor(ctor: string) { - if (!webGLVarMap.has(ctor)) { - webGLVarMap.set(ctor, []); +type GLVarMap = Map; +const webGLVarMap: Map< + WebGLRenderingContext | WebGL2RenderingContext, + GLVarMap +> = new Map(); +export function variableListFor( + ctx: WebGLRenderingContext | WebGL2RenderingContext, + ctor: string, +) { + let contextMap = webGLVarMap.get(ctx); + if (!contextMap) { + contextMap = new Map(); + webGLVarMap.set(ctx, contextMap); } - return webGLVarMap.get(ctor) as any[]; + if (!contextMap.has(ctor)) { + contextMap.set(ctor, []); + } + return contextMap.get(ctor) as any[]; } function getContext( @@ -47,29 +59,33 @@ const WebGLVariableConstructorsNames = [ 'WebGLVertexArrayObject', ]; -function saveToWebGLVarMap(result: any) { +function saveToWebGLVarMap( + ctx: WebGLRenderingContext | WebGL2RenderingContext, + result: any, +) { if (!result?.constructor) return; // probably null or undefined const { name } = result.constructor; if (!WebGLVariableConstructorsNames.includes(name)) return; // not a WebGL variable - const variables = variableListFor(name); + const variables = variableListFor(ctx, name); if (!variables.includes(result)) variables.push(result); } export function deserializeArg( imageMap: Replayer['imageMap'], + ctx: WebGLRenderingContext | WebGL2RenderingContext, ): (arg: SerializedWebGlArg) => any { return (arg: SerializedWebGlArg): any => { if (arg && typeof arg === 'object' && 'rr_type' in arg) { if ('index' in arg) { const { rr_type: name, index } = arg; - return variableListFor(name)[index]; + return variableListFor(ctx, name)[index]; } else if ('args' in arg) { const { rr_type: name, args } = arg; const ctor = window[name as keyof Window]; - return new ctor(...args.map(deserializeArg(imageMap))); + return new ctor(...args.map(deserializeArg(imageMap, ctx))); } else if ('base64' in arg) { return decode(arg.base64); } else if ('src' in arg) { @@ -84,7 +100,7 @@ export function deserializeArg( } } } else if (Array.isArray(arg)) { - return arg.map(deserializeArg(imageMap)); + return arg.map(deserializeArg(imageMap, ctx)); } return arg; }; @@ -119,9 +135,9 @@ export default function webglMutation({ mutation.property as Exclude ] as Function; - const args = mutation.args.map(deserializeArg(imageMap)); + const args = mutation.args.map(deserializeArg(imageMap, ctx)); const result = original.apply(ctx, args); - saveToWebGLVarMap(result); + saveToWebGLVarMap(ctx, result); // Slows down replay considerably, only use for debugging const debugMode = false; diff --git a/packages/rrweb/test/record/serialize-args.test.ts b/packages/rrweb/test/record/serialize-args.test.ts index 0a547ce32e..8ba72e6cd0 100644 --- a/packages/rrweb/test/record/serialize-args.test.ts +++ b/packages/rrweb/test/record/serialize-args.test.ts @@ -6,14 +6,23 @@ polyfillWebGLGlobals(); import { serializeArg } from '../../src/record/observers/canvas/serialize-args'; +const createContext = () => { + const ctx = new WebGL2RenderingContext(); + return ctx; +}; + +let context: WebGL2RenderingContext; describe('serializeArg', () => { + beforeEach(() => { + context = createContext(); + }); it('should serialize Float32Array values', async () => { const float32Array = new Float32Array([-1, -1, 3, -1, -1, 3]); const expected = { rr_type: 'Float32Array', args: [[-1, -1, 3, -1, -1, 3]], }; - expect(serializeArg(float32Array, window)).toStrictEqual(expected); + expect(serializeArg(float32Array, window, context)).toStrictEqual(expected); }); it('should serialize Float64Array values', async () => { @@ -23,7 +32,7 @@ describe('serializeArg', () => { args: [[-1, -1, 3, -1, -1, 3]], }; - expect(serializeArg(float64Array, window)).toStrictEqual(expected); + expect(serializeArg(float64Array, window, context)).toStrictEqual(expected); }); it('should serialize ArrayBuffer values', async () => { @@ -33,7 +42,7 @@ describe('serializeArg', () => { base64: 'AQIABA==', }; - expect(serializeArg(arrayBuffer, window)).toStrictEqual(expected); + expect(serializeArg(arrayBuffer, window, context)).toStrictEqual(expected); }); it('should serialize Uint8Array values', async () => { @@ -43,7 +52,7 @@ describe('serializeArg', () => { args: [[1, 2, 0, 4]], }; - expect(serializeArg(object, window)).toStrictEqual(expected); + expect(serializeArg(object, window, context)).toStrictEqual(expected); }); it('should serialize DataView values', async () => { @@ -60,12 +69,12 @@ describe('serializeArg', () => { ], }; - expect(serializeArg(dataView, window)).toStrictEqual(expected); + expect(serializeArg(dataView, window, context)).toStrictEqual(expected); }); it('should leave arrays intact', async () => { const array = [1, 2, 3, 4]; - expect(serializeArg(array, window)).toStrictEqual(array); + expect(serializeArg(array, window, context)).toStrictEqual(array); }); it('should serialize complex objects', async () => { @@ -86,7 +95,7 @@ describe('serializeArg', () => { 6, ]; - expect(serializeArg(dataView, window)).toStrictEqual(expected); + expect(serializeArg(dataView, window, context)).toStrictEqual(expected); }); it('should serialize arraybuffer contents', async () => { @@ -96,16 +105,36 @@ describe('serializeArg', () => { base64: 'AACAPwAAAEAAAEBAAACAQA==', }; - expect(serializeArg(buffer, window)).toStrictEqual(expected); + expect(serializeArg(buffer, window, context)).toStrictEqual(expected); }); it('should leave null as-is', async () => { - expect(serializeArg(null, window)).toStrictEqual(null); + expect(serializeArg(null, window, context)).toStrictEqual(null); }); it('should support indexed variables', async () => { const webGLProgram = new WebGLProgram(); - expect(serializeArg(webGLProgram, window)).toStrictEqual({ + expect(serializeArg(webGLProgram, window, context)).toStrictEqual({ + rr_type: 'WebGLProgram', + index: 0, + }); + const webGLProgram2 = new WebGLProgram(); + expect(serializeArg(webGLProgram2, window, context)).toStrictEqual({ + rr_type: 'WebGLProgram', + index: 1, + }); + }); + + it('should support indexed variables grouped by context', async () => { + const context1 = createContext(); + const webGLProgram1 = new WebGLProgram(); + expect(serializeArg(webGLProgram1, window, context1)).toStrictEqual({ + rr_type: 'WebGLProgram', + index: 0, + }); + const context2 = createContext(); + const webGLProgram2 = new WebGLProgram(); + expect(serializeArg(webGLProgram2, window, context2)).toStrictEqual({ rr_type: 'WebGLProgram', index: 0, }); @@ -114,7 +143,7 @@ describe('serializeArg', () => { it('should support HTMLImageElements', async () => { const image = new Image(); image.src = 'http://example.com/image.png'; - expect(serializeArg(image, window)).toStrictEqual({ + expect(serializeArg(image, window, context)).toStrictEqual({ rr_type: 'HTMLImageElement', src: 'http://example.com/image.png', }); @@ -135,7 +164,7 @@ describe('serializeArg', () => { let imageData = new ImageData(arr, 200, 50); const contents = Array.from(arr); - expect(serializeArg(imageData, window)).toStrictEqual({ + expect(serializeArg(imageData, window, context)).toStrictEqual({ rr_type: 'ImageData', args: [ { diff --git a/packages/rrweb/test/utils.ts b/packages/rrweb/test/utils.ts index 1b51d005de..0890d60902 100644 --- a/packages/rrweb/test/utils.ts +++ b/packages/rrweb/test/utils.ts @@ -464,4 +464,9 @@ export const polyfillWebGLGlobals = () => { } global.ImageData = ImageData as any; + + class WebGL2RenderingContext { + constructor() {} + } + global.WebGL2RenderingContext = WebGL2RenderingContext as any; }; From d0ac3dade232d924f890f4f4e00af2dd8a394149 Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Fri, 7 Jan 2022 19:14:23 +0100 Subject: [PATCH 83/93] Fix test --- .../rrweb/test/replay/webgl-mutation.test.ts | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/packages/rrweb/test/replay/webgl-mutation.test.ts b/packages/rrweb/test/replay/webgl-mutation.test.ts index bae750c785..a8d6392c13 100644 --- a/packages/rrweb/test/replay/webgl-mutation.test.ts +++ b/packages/rrweb/test/replay/webgl-mutation.test.ts @@ -6,7 +6,7 @@ import { polyfillWebGLGlobals } from '../utils'; polyfillWebGLGlobals(); import webglMutation, { variableListFor } from '../../src/replay/canvas/webgl'; -import { CanvasContext, IncrementalSource } from '../../src/types'; +import { CanvasContext } from '../../src/types'; let canvas: HTMLCanvasElement; describe('webglMutation', () => { @@ -21,28 +21,27 @@ describe('webglMutation', () => { const createShaderMock = jest.fn().mockImplementation(() => { return new WebGLShader(); }); + const context = ({ + createShader: createShaderMock, + } as unknown) as WebGLRenderingContext; jest.spyOn(canvas, 'getContext').mockImplementation(() => { - return ({ - createShader: createShaderMock, - } as unknown) as WebGLRenderingContext; + return context; }); - expect(variableListFor('WebGLShader')).toHaveLength(0); + expect(variableListFor(context, 'WebGLShader')).toHaveLength(0); webglMutation({ mutation: { - source: IncrementalSource.CanvasMutation, - type: CanvasContext.WebGL, - id: 1, property: 'createShader', args: [35633], }, + type: CanvasContext.WebGL, target: canvas, imageMap: new Map(), errorHandler: () => {}, }); expect(createShaderMock).toHaveBeenCalledWith(35633); - expect(variableListFor('WebGLShader')).toHaveLength(1); + expect(variableListFor(context, 'WebGLShader')).toHaveLength(1); }); }); From 602eced1cf7b2f2540002cbfea706aa0eea6fd67 Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Fri, 7 Jan 2022 19:20:40 +0100 Subject: [PATCH 84/93] fix tests --- .../test/replay/deserialize-args.test.ts | 40 ++++++++++++++----- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/packages/rrweb/test/replay/deserialize-args.test.ts b/packages/rrweb/test/replay/deserialize-args.test.ts index ec6bfdef1f..a39c439d1d 100644 --- a/packages/rrweb/test/replay/deserialize-args.test.ts +++ b/packages/rrweb/test/replay/deserialize-args.test.ts @@ -7,10 +7,17 @@ polyfillWebGLGlobals(); import { deserializeArg } from '../../src/replay/canvas/webgl'; +let context: WebGLRenderingContext | WebGL2RenderingContext; describe('deserializeArg', () => { + beforeEach(() => { + context = new WebGL2RenderingContext(); + }); it('should deserialize Float32Array values', async () => { expect( - deserializeArg(new Map())({ + deserializeArg( + new Map(), + context, + )({ rr_type: 'Float32Array', args: [[-1, -1, 3, -1, -1, 3]], }), @@ -19,7 +26,10 @@ describe('deserializeArg', () => { it('should deserialize Float64Array values', async () => { expect( - deserializeArg(new Map())({ + deserializeArg( + new Map(), + context, + )({ rr_type: 'Float64Array', args: [[-1, -1, 3, -1, -1, 3]], }), @@ -29,7 +39,10 @@ describe('deserializeArg', () => { it('should deserialize ArrayBuffer values', async () => { const contents = [1, 2, 0, 4]; expect( - deserializeArg(new Map())({ + deserializeArg( + new Map(), + context, + )({ rr_type: 'ArrayBuffer', base64: 'AQIABA==', }), @@ -38,7 +51,10 @@ describe('deserializeArg', () => { it('should deserialize DataView values', async () => { expect( - deserializeArg(new Map())({ + deserializeArg( + new Map(), + context, + )({ rr_type: 'DataView', args: [ { @@ -54,7 +70,7 @@ describe('deserializeArg', () => { it('should leave arrays intact', async () => { const array = [1, 2, 3, 4]; - expect(deserializeArg(new Map())(array)).toEqual(array); + expect(deserializeArg(new Map(), context)(array)).toEqual(array); }); it('should deserialize complex objects', async () => { @@ -73,7 +89,7 @@ describe('deserializeArg', () => { 5, 6, ]; - expect(deserializeArg(new Map())(serializedArg)).toStrictEqual([ + expect(deserializeArg(new Map(), context)(serializedArg)).toStrictEqual([ new DataView(new ArrayBuffer(16), 0, 16), 5, 6, @@ -81,14 +97,17 @@ describe('deserializeArg', () => { }); it('should leave null as-is', async () => { - expect(deserializeArg(new Map())(null)).toStrictEqual(null); + expect(deserializeArg(new Map(), context)(null)).toStrictEqual(null); }); it('should support HTMLImageElements', async () => { const image = new Image(); image.src = 'http://example.com/image.png'; expect( - deserializeArg(new Map())({ + deserializeArg( + new Map(), + context, + )({ rr_type: 'HTMLImageElement', src: 'http://example.com/image.png', }), @@ -102,7 +121,10 @@ describe('deserializeArg', () => { imageMap.set(image.src, image); expect( - deserializeArg(imageMap)({ + deserializeArg( + imageMap, + context, + )({ rr_type: 'HTMLImageElement', src: 'http://example.com/image.png', }), From a3b6e0c2c0dcb3d9968fe2eb0856f087dbd3c5a3 Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Fri, 7 Jan 2022 19:30:43 +0100 Subject: [PATCH 85/93] bundle webgl mutations on request animation frame Instead of fixing it on the replay side we buffer up webgl canvas mutations and wait for a new RAF to flush them. This allows us to remove `newFrame` from the events and simplify things a little --- .../src/record/observers/canvas/webgl.ts | 58 ++- packages/rrweb/src/replay/canvas/2d.ts | 4 +- packages/rrweb/src/replay/canvas/index.ts | 33 +- packages/rrweb/src/replay/canvas/webgl.ts | 12 +- packages/rrweb/src/replay/index.ts | 59 +-- packages/rrweb/src/replay/machine.ts | 9 +- packages/rrweb/src/replay/timer.ts | 31 +- packages/rrweb/src/types.ts | 17 +- .../__snapshots__/integration.test.ts.snap | 34 +- .../record/__snapshots__/webgl.test.ts.snap | 394 ++++++++++++++---- packages/rrweb/test/record/webgl.test.ts | 51 +-- packages/rrweb/test/replay/timer.test.ts | 61 --- 12 files changed, 442 insertions(+), 321 deletions(-) delete mode 100644 packages/rrweb/test/replay/timer.test.ts diff --git a/packages/rrweb/src/record/observers/canvas/webgl.ts b/packages/rrweb/src/record/observers/canvas/webgl.ts index f1335bba13..873a97bc2f 100644 --- a/packages/rrweb/src/record/observers/canvas/webgl.ts +++ b/packages/rrweb/src/record/observers/canvas/webgl.ts @@ -3,7 +3,7 @@ import { blockClass, CanvasContext, canvasMutationCallback, - canvasMutationParam, + canvasMutationCommand, IWindow, listenerHandler, Mirror, @@ -11,17 +11,20 @@ import { import { hookSetter, isBlocked, patch } from '../../../utils'; import { saveWebGLVar, serializeArgs } from './serialize-args'; -type pendingCanvasMutationsMap = Map; +type canvasMutationWithType = { type: CanvasContext } & canvasMutationCommand; +type pendingCanvasMutationsMap = Map< + HTMLCanvasElement, + canvasMutationWithType[] +>; type RafStamps = { latestId: number; invokeId: number | null }; -// FIXME: total hack here, we need to find a better way to do this function flushPendingCanvasMutations( pendingCanvasMutations: pendingCanvasMutationsMap, cb: canvasMutationCallback, mirror: Mirror, ) { pendingCanvasMutations.forEach( - (values: canvasMutationParam[], canvas: HTMLCanvasElement) => { + (values: canvasMutationCommand[], canvas: HTMLCanvasElement) => { const id = mirror.getId((canvas as unknown) as INode); flushPendingCanvasMutationFor(canvas, pendingCanvasMutations, id, cb); }, @@ -37,10 +40,17 @@ function flushPendingCanvasMutationFor( id: number, cb: canvasMutationCallback, ) { - const values = pendingCanvasMutations.get(canvas); - if (!values || id === -1) return; + const valuesWithType = pendingCanvasMutations.get(canvas); + if (!valuesWithType || id === -1) return; + + const values = valuesWithType.map((value) => { + const { type, ...rest } = value; + return rest; + }); + const { type } = valuesWithType[0]; + + cb({ id, type, commands: values }); - values.forEach((p) => cb({ ...p, id })); pendingCanvasMutations.delete(canvas); } @@ -76,38 +86,20 @@ function patchGLPrototype( const id = mirror.getId((this.canvas as unknown) as INode); const recordArgs = serializeArgs([...args], win, prototype); - const mutation: canvasMutationParam = { - id, + const mutation: canvasMutationWithType = { type, property: prop, args: recordArgs, }; - if (newFrame) mutation.newFrame = true; - if (id === -1) { - // FIXME! THIS COULD MAYBE BE AN OFFSCREEN CANVAS - if ( - !pendingCanvasMutations.has(this.canvas as HTMLCanvasElement) - ) { - pendingCanvasMutations.set( - this.canvas as HTMLCanvasElement, - [], - ); - } - - pendingCanvasMutations - .get(this.canvas as HTMLCanvasElement)! - .push(mutation); - } else { - // flush all pending mutations - flushPendingCanvasMutationFor( - this.canvas as HTMLCanvasElement, - pendingCanvasMutations, - id, - cb, - ); - cb(mutation); + if (!pendingCanvasMutations.has(this.canvas as HTMLCanvasElement)) { + pendingCanvasMutations.set(this.canvas as HTMLCanvasElement, []); } + + pendingCanvasMutations + // FIXME! THIS COULD MAYBE BE AN OFFSCREEN CANVAS + .get(this.canvas as HTMLCanvasElement)! + .push(mutation); } return result; diff --git a/packages/rrweb/src/replay/canvas/2d.ts b/packages/rrweb/src/replay/canvas/2d.ts index 7daa7e4224..b9fde639b3 100644 --- a/packages/rrweb/src/replay/canvas/2d.ts +++ b/packages/rrweb/src/replay/canvas/2d.ts @@ -1,5 +1,5 @@ import { Replayer } from '../'; -import { canvasMutationData } from '../../types'; +import { canvasMutationCommand } from '../../types'; export default function canvasMutation({ event, @@ -9,7 +9,7 @@ export default function canvasMutation({ errorHandler, }: { event: Parameters[0]; - mutation: canvasMutationData; + mutation: canvasMutationCommand; target: HTMLCanvasElement; imageMap: Replayer['imageMap']; errorHandler: Replayer['warnCanvasMutationFailed']; diff --git a/packages/rrweb/src/replay/canvas/index.ts b/packages/rrweb/src/replay/canvas/index.ts index 783f0bde23..73411a2b10 100644 --- a/packages/rrweb/src/replay/canvas/index.ts +++ b/packages/rrweb/src/replay/canvas/index.ts @@ -1,5 +1,9 @@ import { Replayer } from '..'; -import { CanvasContext, canvasMutationData } from '../../types'; +import { + CanvasContext, + canvasMutationCommand, + canvasMutationData, +} from '../../types'; import webglMutation from './webgl'; import canvas2DMutation from './2d'; @@ -17,16 +21,29 @@ export default function canvasMutation({ errorHandler: Replayer['warnCanvasMutationFailed']; }): void { try { + const mutations: canvasMutationCommand[] = + 'commands' in mutation ? mutation.commands : [mutation]; + if ([CanvasContext.WebGL, CanvasContext.WebGL2].includes(mutation.type)) { - return webglMutation({ mutation, target, imageMap, errorHandler }); + return mutations.forEach((command) => { + webglMutation({ + mutation: command, + type: mutation.type, + target, + imageMap, + errorHandler, + }); + }); } // default is '2d' for backwards compatibility (rrweb below 1.1.x) - return canvas2DMutation({ - event, - mutation, - target, - imageMap, - errorHandler, + return mutations.forEach((command) => { + canvas2DMutation({ + event, + mutation: command, + target, + imageMap, + errorHandler, + }); }); } catch (error) { errorHandler(mutation, error); diff --git a/packages/rrweb/src/replay/canvas/webgl.ts b/packages/rrweb/src/replay/canvas/webgl.ts index ab97332598..58d323dc3d 100644 --- a/packages/rrweb/src/replay/canvas/webgl.ts +++ b/packages/rrweb/src/replay/canvas/webgl.ts @@ -2,7 +2,7 @@ import { decode } from 'base64-arraybuffer'; import { Replayer } from '../'; import { CanvasContext, - canvasMutationData, + canvasMutationCommand, SerializedWebGlArg, } from '../../types'; @@ -33,7 +33,7 @@ function getContext( ): WebGLRenderingContext | WebGL2RenderingContext | null { // Note to whomever is going to implement support for `contextAttributes`: // if `preserveDrawingBuffer` is set to true, - // you have to do `ctx.flush()` before every `newFrame: true` + // you might have to do `ctx.flush()` before every webgl canvas event try { if (type === CanvasContext.WebGL) { return ( @@ -109,20 +109,22 @@ export function deserializeArg( export default function webglMutation({ mutation, target, + type, imageMap, errorHandler, }: { - mutation: canvasMutationData; + mutation: canvasMutationCommand; target: HTMLCanvasElement; + type: CanvasContext; imageMap: Replayer['imageMap']; errorHandler: Replayer['warnCanvasMutationFailed']; }): void { try { - const ctx = getContext(target, mutation.type); + const ctx = getContext(target, type); if (!ctx) return; // NOTE: if `preserveDrawingBuffer` is set to true, - // we must flush the buffers on every newFrame: true + // we must flush the buffers on every new canvas event // if (mutation.newFrame) ctx.flush(); if (mutation.setter) { diff --git a/packages/rrweb/src/replay/index.ts b/packages/rrweb/src/replay/index.ts index 85ab0819c9..f0146433d0 100644 --- a/packages/rrweb/src/replay/index.ts +++ b/packages/rrweb/src/replay/index.ts @@ -39,7 +39,7 @@ import { styleValueWithPriority, mouseMovePos, IWindow, - CanvasContext, + canvasMutationCommand, } from '../types'; import { createMirror, @@ -855,28 +855,34 @@ export class Replayer { for (const event of this.service.state.context.events) { if ( event.type === EventType.IncrementalSnapshot && - event.data.source === IncrementalSource.CanvasMutation && - event.data.property === 'drawImage' && - typeof event.data.args[0] === 'string' && - !this.imageMap.has(event) - ) { - const canvas = document.createElement('canvas'); - const ctx = canvas.getContext('2d'); - const imgd = ctx?.createImageData(canvas.width, canvas.height); - let d = imgd?.data; - d = JSON.parse(event.data.args[0]); - ctx?.putImageData(imgd!, 0, 0); - } else if ( - event.type === EventType.IncrementalSnapshot && - event.data.source === IncrementalSource.CanvasMutation && - this.hasImageArg(event.data.args) - ) { - this.getImageArgs(event.data.args).forEach((url) => { - const image = new Image(); - image.src = url; // this preloads the image - this.imageMap.set(url, image); - }); - } + event.data.source === IncrementalSource.CanvasMutation + ) + if ('commands' in event.data) { + event.data.commands.forEach((c) => this.preloadImages(c, event)); + } else { + this.preloadImages(event.data, event); + } + } + } + + private preloadImages(data: canvasMutationCommand, event: eventWithTime) { + if ( + data.property === 'drawImage' && + typeof data.args[0] === 'string' && + !this.imageMap.has(event) + ) { + const canvas = document.createElement('canvas'); + const ctx = canvas.getContext('2d'); + const imgd = ctx?.createImageData(canvas.width, canvas.height); + let d = imgd?.data; + d = JSON.parse(data.args[0]); + ctx?.putImageData(imgd!, 0, 0); + } else if (this.hasImageArg(data.args)) { + this.getImageArgs(data.args).forEach((url) => { + const image = new Image(); + image.src = url; // this preloads the image + this.imageMap.set(url, image); + }); } } @@ -921,7 +927,6 @@ export class Replayer { p.timeOffset + e.timestamp - this.service.state.context.baselineTime, - newFrame: false, }; this.timer.addAction(action); }); @@ -929,7 +934,6 @@ export class Replayer { this.timer.addAction({ doAction() {}, delay: e.delay! - d.positions[0]?.timeOffset, - newFrame: false, }); } break; @@ -1882,7 +1886,10 @@ export class Replayer { } } - private warnCanvasMutationFailed(d: canvasMutationData, error: unknown) { + private warnCanvasMutationFailed( + d: canvasMutationData | canvasMutationCommand, + error: unknown, + ) { this.warn(`Has error on canvas update`, error, 'canvas mutation:', d); } diff --git a/packages/rrweb/src/replay/machine.ts b/packages/rrweb/src/replay/machine.ts index 49dd96ef41..4816fcbb99 100644 --- a/packages/rrweb/src/replay/machine.ts +++ b/packages/rrweb/src/replay/machine.ts @@ -8,7 +8,7 @@ import { Emitter, IncrementalSource, } from '../types'; -import { Timer, addDelay, LastDelay } from './timer'; +import { Timer, addDelay } from './timer'; export type PlayerContext = { events: eventWithTime[]; @@ -168,10 +168,9 @@ export function createPlayerService( const { timer, events, baselineTime, lastPlayedEvent } = ctx; timer.clear(); - const lastDelay: LastDelay = { at: null }; for (const event of events) { // TODO: improve this API - addDelay(event, baselineTime, lastDelay); + addDelay(event, baselineTime); } const neededEvents = discardPriorSnapshots(events, baselineTime); @@ -209,8 +208,6 @@ export function createPlayerService( emitter.emit(ReplayerEvents.EventCast, event); }, delay: event.delay!, - newFrame: - ('newFrame' in event.data && event.data.newFrame) || false, }); } } @@ -276,8 +273,6 @@ export function createPlayerService( emitter.emit(ReplayerEvents.EventCast, event); }, delay: event.delay!, - newFrame: - ('newFrame' in event.data && event.data.newFrame) || false, }); } } diff --git a/packages/rrweb/src/replay/timer.ts b/packages/rrweb/src/replay/timer.ts index 1fdbdb0208..f6e7f82628 100644 --- a/packages/rrweb/src/replay/timer.ts +++ b/packages/rrweb/src/replay/timer.ts @@ -44,10 +44,6 @@ export class Timer { lastTimestamp = time; while (actions.length) { const action = actions[0]; - if (action.newFrame) { - action.newFrame = false; - break; - } if (self.timeOffset >= action.delay) { actions.shift(); @@ -102,14 +98,8 @@ export class Timer { } } -export type LastDelay = { at: number | null }; - // TODO: add speed to mouse move timestamp calculation -export function addDelay( - event: eventWithTime, - baselineTime: number, - lastDelay?: LastDelay, -): number { +export function addDelay(event: eventWithTime, baselineTime: number): number { // Mouse move events was recorded in a throttle function, // so we need to find the real timestamp by traverse the time offsets. if ( @@ -124,24 +114,5 @@ export function addDelay( } event.delay = event.timestamp - baselineTime; - - if (lastDelay) { - // WebGL events need to be bundled together as much as possible so they don't - // accidentally get split over multiple animation frames. - if ( - event.type === EventType.IncrementalSnapshot && - event.data.source === IncrementalSource.CanvasMutation && - // `newFrame: true` is used to indicate the start of an new animation frame in the recording, - // and that the event shouldn't be bundled with the previous events. - !event.data.newFrame && - lastDelay.at - ) { - // Override the current delay with the last delay - event.delay = lastDelay.at; - } else { - lastDelay.at = event.delay!; - } - } - return event.delay; } diff --git a/packages/rrweb/src/types.ts b/packages/rrweb/src/types.ts index a0700ba645..84d4e0b0eb 100644 --- a/packages/rrweb/src/types.ts +++ b/packages/rrweb/src/types.ts @@ -461,15 +461,23 @@ export type styleDeclarationCallback = (s: styleDeclarationParam) => void; export type canvasMutationCallback = (p: canvasMutationParam) => void; -export type canvasMutationParam = { - id: number; - type: CanvasContext; +export type canvasMutationCommand = { property: string; args: Array; setter?: true; - newFrame?: true; }; +export type canvasMutationParam = + | { + id: number; + type: CanvasContext; + commands: canvasMutationCommand[]; + } + | ({ + id: number; + type: CanvasContext; + } & canvasMutationCommand); + export type fontParam = { family: string; fontSource: string; @@ -589,7 +597,6 @@ export type missingNodeMap = { export type actionWithDelay = { doAction: () => void; delay: number; - newFrame: boolean; }; export type Handler = (event?: unknown) => void; diff --git a/packages/rrweb/test/__snapshots__/integration.test.ts.snap b/packages/rrweb/test/__snapshots__/integration.test.ts.snap index 56ea412acf..6bc34487a5 100644 --- a/packages/rrweb/test/__snapshots__/integration.test.ts.snap +++ b/packages/rrweb/test/__snapshots__/integration.test.ts.snap @@ -9719,24 +9719,22 @@ exports[`record integration tests should record webgl canvas mutations 1`] = ` \\"source\\": 9, \\"id\\": 16, \\"type\\": 1, - \\"property\\": \\"clearColor\\", - \\"args\\": [ - 1, - 0, - 0, - 1 - ] - } - }, - { - \\"type\\": 3, - \\"data\\": { - \\"source\\": 9, - \\"id\\": 16, - \\"type\\": 1, - \\"property\\": \\"clear\\", - \\"args\\": [ - 16384 + \\"commands\\": [ + { + \\"property\\": \\"clearColor\\", + \\"args\\": [ + 1, + 0, + 0, + 1 + ] + }, + { + \\"property\\": \\"clear\\", + \\"args\\": [ + 16384 + ] + } ] } } diff --git a/packages/rrweb/test/record/__snapshots__/webgl.test.ts.snap b/packages/rrweb/test/record/__snapshots__/webgl.test.ts.snap index b0ce47df14..03a208dcd5 100644 --- a/packages/rrweb/test/record/__snapshots__/webgl.test.ts.snap +++ b/packages/rrweb/test/record/__snapshots__/webgl.test.ts.snap @@ -1,6 +1,6 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`record webgl should mark every RAF as \`newFrame: true\` 1`] = ` +exports[`record webgl should batch events by RAF 1`] = ` "[ { \\"type\\": 4, @@ -80,21 +80,19 @@ exports[`record webgl should mark every RAF as \`newFrame: true\` 1`] = ` \\"source\\": 9, \\"id\\": 7, \\"type\\": 1, - \\"property\\": \\"createProgram\\", - \\"args\\": [] - } - }, - { - \\"type\\": 3, - \\"data\\": { - \\"source\\": 9, - \\"id\\": 7, - \\"type\\": 1, - \\"property\\": \\"linkProgram\\", - \\"args\\": [ + \\"commands\\": [ { - \\"rr_type\\": \\"WebGLProgram\\", - \\"index\\": 0 + \\"property\\": \\"createProgram\\", + \\"args\\": [] + }, + { + \\"property\\": \\"linkProgram\\", + \\"args\\": [ + { + \\"rr_type\\": \\"WebGLProgram\\", + \\"index\\": 0 + } + ] } ] } @@ -105,22 +103,25 @@ exports[`record webgl should mark every RAF as \`newFrame: true\` 1`] = ` \\"source\\": 9, \\"id\\": 7, \\"type\\": 1, - \\"property\\": \\"createProgram\\", - \\"args\\": [], - \\"newFrame\\": true - } - }, - { - \\"type\\": 3, - \\"data\\": { - \\"source\\": 9, - \\"id\\": 7, - \\"type\\": 1, - \\"property\\": \\"linkProgram\\", - \\"args\\": [ + \\"commands\\": [ + { + \\"property\\": \\"createProgram\\", + \\"args\\": [] + }, + { + \\"property\\": \\"linkProgram\\", + \\"args\\": [ + { + \\"rr_type\\": \\"WebGLProgram\\", + \\"index\\": 1 + } + ] + }, { - \\"rr_type\\": \\"WebGLProgram\\", - \\"index\\": 1 + \\"property\\": \\"clear\\", + \\"args\\": [ + 16384 + ] } ] } @@ -131,24 +132,15 @@ exports[`record webgl should mark every RAF as \`newFrame: true\` 1`] = ` \\"source\\": 9, \\"id\\": 7, \\"type\\": 1, - \\"property\\": \\"clear\\", - \\"args\\": [ - 16384 + \\"commands\\": [ + { + \\"property\\": \\"clear\\", + \\"args\\": [ + 16384 + ] + } ] } - }, - { - \\"type\\": 3, - \\"data\\": { - \\"source\\": 9, - \\"id\\": 7, - \\"type\\": 1, - \\"property\\": \\"clear\\", - \\"args\\": [ - 16384 - ], - \\"newFrame\\": true - } } ]" `; @@ -233,9 +225,13 @@ exports[`record webgl will record changes to a canvas element 1`] = ` \\"source\\": 9, \\"id\\": 7, \\"type\\": 1, - \\"property\\": \\"clear\\", - \\"args\\": [ - 16384 + \\"commands\\": [ + { + \\"property\\": \\"clear\\", + \\"args\\": [ + 16384 + ] + } ] } } @@ -344,36 +340,28 @@ exports[`record webgl will record changes to a canvas element before the canvas \\"source\\": 9, \\"id\\": 9, \\"type\\": 2, - \\"property\\": \\"createProgram\\", - \\"args\\": [] - } - }, - { - \\"type\\": 3, - \\"data\\": { - \\"source\\": 9, - \\"id\\": 9, - \\"type\\": 2, - \\"property\\": \\"linkProgram\\", - \\"args\\": [ + \\"commands\\": [ + { + \\"property\\": \\"createProgram\\", + \\"args\\": [] + }, { - \\"rr_type\\": \\"WebGLProgram\\", - \\"index\\": 0 + \\"property\\": \\"linkProgram\\", + \\"args\\": [ + { + \\"rr_type\\": \\"WebGLProgram\\", + \\"index\\": 0 + } + ] + }, + { + \\"property\\": \\"clear\\", + \\"args\\": [ + 16384 + ] } ] } - }, - { - \\"type\\": 3, - \\"data\\": { - \\"source\\": 9, - \\"id\\": 9, - \\"type\\": 2, - \\"property\\": \\"clear\\", - \\"args\\": [ - 16384 - ] - } } ]" `; @@ -480,41 +468,239 @@ exports[`record webgl will record changes to a canvas element before the canvas \\"source\\": 9, \\"id\\": 9, \\"type\\": 1, - \\"property\\": \\"createProgram\\", - \\"args\\": [] + \\"commands\\": [ + { + \\"property\\": \\"createProgram\\", + \\"args\\": [] + }, + { + \\"property\\": \\"linkProgram\\", + \\"args\\": [ + { + \\"rr_type\\": \\"WebGLProgram\\", + \\"index\\": 0 + } + ] + }, + { + \\"property\\": \\"clear\\", + \\"args\\": [ + 16384 + ] + } + ] + } + } +]" +`; + +exports[`record webgl will record changes to a webgl2 canvas element 1`] = ` +"[ + { + \\"type\\": 4, + \\"data\\": { + \\"href\\": \\"about:blank\\", + \\"width\\": 1920, + \\"height\\": 1080 + } + }, + { + \\"type\\": 2, + \\"data\\": { + \\"node\\": { + \\"type\\": 0, + \\"childNodes\\": [ + { + \\"type\\": 1, + \\"name\\": \\"html\\", + \\"publicId\\": \\"\\", + \\"systemId\\": \\"\\", + \\"id\\": 2 + }, + { + \\"type\\": 2, + \\"tagName\\": \\"html\\", + \\"attributes\\": {}, + \\"childNodes\\": [ + { + \\"type\\": 2, + \\"tagName\\": \\"head\\", + \\"attributes\\": {}, + \\"childNodes\\": [], + \\"id\\": 4 + }, + { + \\"type\\": 2, + \\"tagName\\": \\"body\\", + \\"attributes\\": {}, + \\"childNodes\\": [ + { + \\"type\\": 3, + \\"textContent\\": \\"\\\\n \\", + \\"id\\": 6 + }, + { + \\"type\\": 2, + \\"tagName\\": \\"canvas\\", + \\"attributes\\": { + \\"id\\": \\"canvas\\" + }, + \\"childNodes\\": [], + \\"id\\": 7 + }, + { + \\"type\\": 3, + \\"textContent\\": \\"\\\\n \\\\n \\\\n \\", + \\"id\\": 8 + } + ], + \\"id\\": 5 + } + ], + \\"id\\": 3 + } + ], + \\"id\\": 1 + }, + \\"initialOffset\\": { + \\"left\\": 0, + \\"top\\": 0 + } } }, { \\"type\\": 3, \\"data\\": { \\"source\\": 9, - \\"id\\": 9, - \\"type\\": 1, - \\"property\\": \\"linkProgram\\", - \\"args\\": [ + \\"id\\": 7, + \\"type\\": 2, + \\"commands\\": [ { - \\"rr_type\\": \\"WebGLProgram\\", - \\"index\\": 0 + \\"property\\": \\"clear\\", + \\"args\\": [ + 16384 + ] } ] } + } +]" +`; + +exports[`record webgl will record webgl variables 1`] = ` +"[ + { + \\"type\\": 4, + \\"data\\": { + \\"href\\": \\"about:blank\\", + \\"width\\": 1920, + \\"height\\": 1080 + } + }, + { + \\"type\\": 2, + \\"data\\": { + \\"node\\": { + \\"type\\": 0, + \\"childNodes\\": [ + { + \\"type\\": 1, + \\"name\\": \\"html\\", + \\"publicId\\": \\"\\", + \\"systemId\\": \\"\\", + \\"id\\": 2 + }, + { + \\"type\\": 2, + \\"tagName\\": \\"html\\", + \\"attributes\\": {}, + \\"childNodes\\": [ + { + \\"type\\": 2, + \\"tagName\\": \\"head\\", + \\"attributes\\": {}, + \\"childNodes\\": [], + \\"id\\": 4 + }, + { + \\"type\\": 2, + \\"tagName\\": \\"body\\", + \\"attributes\\": {}, + \\"childNodes\\": [ + { + \\"type\\": 3, + \\"textContent\\": \\"\\\\n \\", + \\"id\\": 6 + }, + { + \\"type\\": 2, + \\"tagName\\": \\"canvas\\", + \\"attributes\\": { + \\"id\\": \\"canvas\\" + }, + \\"childNodes\\": [], + \\"id\\": 7 + }, + { + \\"type\\": 3, + \\"textContent\\": \\"\\\\n \\\\n \\\\n \\", + \\"id\\": 8 + } + ], + \\"id\\": 5 + } + ], + \\"id\\": 3 + } + ], + \\"id\\": 1 + }, + \\"initialOffset\\": { + \\"left\\": 0, + \\"top\\": 0 + } + } }, { \\"type\\": 3, \\"data\\": { \\"source\\": 9, - \\"id\\": 9, + \\"id\\": 7, \\"type\\": 1, - \\"property\\": \\"clear\\", - \\"args\\": [ - 16384 + \\"commands\\": [ + { + \\"property\\": \\"createProgram\\", + \\"args\\": [] + }, + { + \\"property\\": \\"linkProgram\\", + \\"args\\": [ + { + \\"rr_type\\": \\"WebGLProgram\\", + \\"index\\": 0 + } + ] + }, + { + \\"property\\": \\"createProgram\\", + \\"args\\": [] + }, + { + \\"property\\": \\"linkProgram\\", + \\"args\\": [ + { + \\"rr_type\\": \\"WebGLProgram\\", + \\"index\\": 1 + } + ] + } ] } } ]" `; -exports[`record webgl will record changes to a webgl2 canvas element 1`] = ` +exports[`record webgl will record webgl variables in reverse order 1`] = ` "[ { \\"type\\": 4, @@ -593,10 +779,34 @@ exports[`record webgl will record changes to a webgl2 canvas element 1`] = ` \\"data\\": { \\"source\\": 9, \\"id\\": 7, - \\"type\\": 2, - \\"property\\": \\"clear\\", - \\"args\\": [ - 16384 + \\"type\\": 1, + \\"commands\\": [ + { + \\"property\\": \\"createProgram\\", + \\"args\\": [] + }, + { + \\"property\\": \\"createProgram\\", + \\"args\\": [] + }, + { + \\"property\\": \\"linkProgram\\", + \\"args\\": [ + { + \\"rr_type\\": \\"WebGLProgram\\", + \\"index\\": 1 + } + ] + }, + { + \\"property\\": \\"linkProgram\\", + \\"args\\": [ + { + \\"rr_type\\": \\"WebGLProgram\\", + \\"index\\": 0 + } + ] + } ] } } diff --git a/packages/rrweb/test/record/webgl.test.ts b/packages/rrweb/test/record/webgl.test.ts index c3e5aaf876..31a2383944 100644 --- a/packages/rrweb/test/record/webgl.test.ts +++ b/packages/rrweb/test/record/webgl.test.ts @@ -9,7 +9,6 @@ import { eventWithTime, EventType, IncrementalSource, - styleSheetRuleData, CanvasContext, } from '../../src/types'; import { assertSnapshot, launchPuppeteer } from '../utils'; @@ -116,9 +115,13 @@ describe('record webgl', function (this: ISuite) { expect(lastEvent).toMatchObject({ data: { source: IncrementalSource.CanvasMutation, - args: [16384], type: CanvasContext.WebGL, - property: 'clear', + commands: [ + { + args: [16384], + property: 'clear', + }, + ], }, }); assertSnapshot(ctx.events); @@ -138,9 +141,13 @@ describe('record webgl', function (this: ISuite) { expect(lastEvent).toMatchObject({ data: { source: IncrementalSource.CanvasMutation, - args: [16384], type: CanvasContext.WebGL2, - property: 'clear', + commands: [ + { + args: [16384], + property: 'clear', + }, + ], }, }); assertSnapshot(ctx.events); @@ -196,20 +203,7 @@ describe('record webgl', function (this: ISuite) { await ctx.page.waitForTimeout(50); - const lastEvent = ctx.events[ctx.events.length - 1]; - expect(lastEvent).toMatchObject({ - data: { - source: IncrementalSource.CanvasMutation, - property: 'linkProgram', - type: CanvasContext.WebGL, - args: [ - { - index: 1, - rr_type: 'WebGLProgram', - }, - ], // `program1` is WebGLProgram, this is the second WebGLProgram variable (index #1) - }, - }); + assertSnapshot(ctx.events); }); it('will record webgl variables in reverse order', async () => { @@ -218,26 +212,14 @@ describe('record webgl', function (this: ISuite) { var gl = canvas.getContext('webgl')!; var program0 = gl.createProgram()!; var program1 = gl.createProgram()!; + // attach them in reverse order gl.linkProgram(program1); gl.linkProgram(program0); }); await ctx.page.waitForTimeout(50); - const lastEvent = ctx.events[ctx.events.length - 1]; - expect(lastEvent).toMatchObject({ - data: { - source: IncrementalSource.CanvasMutation, - property: 'linkProgram', - type: CanvasContext.WebGL, - args: [ - { - index: 0, - rr_type: 'WebGLProgram', - }, - ], // `program0` is WebGLProgram, this is the first WebGLProgram variable (index #0) - }, - }); + assertSnapshot(ctx.events); }); it('sets _context on canvas.getContext()', async () => { @@ -261,7 +243,7 @@ describe('record webgl', function (this: ISuite) { expect(context).toBe('webgl'); }); - it('should mark every RAF as `newFrame: true`', async () => { + it('should batch events by RAF', async () => { await ctx.page.evaluate(() => { return new Promise((resolve) => { const canvas = document.getElementById('canvas') as HTMLCanvasElement; @@ -283,5 +265,6 @@ describe('record webgl', function (this: ISuite) { await ctx.page.waitForTimeout(50); assertSnapshot(ctx.events); + expect(ctx.events.length).toEqual(5); }); }); diff --git a/packages/rrweb/test/replay/timer.test.ts b/packages/rrweb/test/replay/timer.test.ts deleted file mode 100644 index 304f0e7b31..0000000000 --- a/packages/rrweb/test/replay/timer.test.ts +++ /dev/null @@ -1,61 +0,0 @@ -import { addDelay, LastDelay } from '../../src/replay/timer'; -import { - CanvasContext, - EventType, - eventWithTime, - IncrementalSource, -} from '../../src/types'; - -const canvasMutationEventWithTime = ( - timestamp: number, - newFrame?: boolean, -): eventWithTime => { - const newFrameObj: { newFrame: true } | {} = newFrame - ? { newFrame: true } - : {}; - - return { - timestamp, - type: EventType.IncrementalSnapshot, - data: { - source: IncrementalSource.CanvasMutation, - property: 'x', - args: [], - id: 1, - type: CanvasContext.WebGL, - ...newFrameObj, - }, - }; -}; - -describe('addDelay', () => { - let baselineTime: number; - let lastDelay: LastDelay; - beforeEach(() => { - baselineTime = 0; - lastDelay = { - at: null, - }; - }); - it('should bundle canvas mutations together', () => { - const event1 = canvasMutationEventWithTime(1000); - const event2 = canvasMutationEventWithTime(1001); - addDelay(event1, baselineTime, lastDelay); - addDelay(event2, baselineTime, lastDelay); - expect(event2.delay).toBe(1000); - }); - - it('should bundle canvas mutations on the same frame together', () => { - const event1 = canvasMutationEventWithTime(1000); - const event2 = canvasMutationEventWithTime(1001); - const event3 = canvasMutationEventWithTime(1002, true); - const event4 = canvasMutationEventWithTime(1003); - addDelay(event1, baselineTime, lastDelay); - addDelay(event2, baselineTime, lastDelay); - addDelay(event3, baselineTime, lastDelay); - addDelay(event4, baselineTime, lastDelay); - expect(event2.delay).toBe(1000); - expect(event3.delay).toBe(1002); - expect(event4.delay).toBe(1002); - }); -}); From f999d496316f81d1fe48cdee560450e7bfde1aa8 Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Mon, 31 Jan 2022 10:53:22 +0100 Subject: [PATCH 86/93] Add canvas element to mutation observer file --- .../__snapshots__/integration.test.ts.snap | 158 +++++++++++++----- .../rrweb/test/html/mutation-observer.html | 3 +- 2 files changed, 117 insertions(+), 44 deletions(-) diff --git a/packages/rrweb/test/__snapshots__/integration.test.ts.snap b/packages/rrweb/test/__snapshots__/integration.test.ts.snap index 6bc34487a5..5fde3b0844 100644 --- a/packages/rrweb/test/__snapshots__/integration.test.ts.snap +++ b/packages/rrweb/test/__snapshots__/integration.test.ts.snap @@ -98,9 +98,21 @@ exports[`record integration tests can freeze mutations 1`] = ` }, { \\"type\\": 3, - \\"textContent\\": \\"\\\\n\\\\n \\", + \\"textContent\\": \\"\\\\n \\", \\"id\\": 14 }, + { + \\"type\\": 2, + \\"tagName\\": \\"canvas\\", + \\"attributes\\": {}, + \\"childNodes\\": [], + \\"id\\": 15 + }, + { + \\"type\\": 3, + \\"textContent\\": \\"\\\\n\\\\n \\", + \\"id\\": 16 + }, { \\"type\\": 2, \\"tagName\\": \\"script\\", @@ -109,15 +121,15 @@ exports[`record integration tests can freeze mutations 1`] = ` { \\"type\\": 3, \\"textContent\\": \\"SCRIPT_PLACEHOLDER\\", - \\"id\\": 16 + \\"id\\": 18 } ], - \\"id\\": 15 + \\"id\\": 17 }, { \\"type\\": 3, - \\"textContent\\": \\"\\\\n \\\\n \\", - \\"id\\": 17 + \\"textContent\\": \\"\\\\n \\\\n \\\\n\\", + \\"id\\": 19 } ], \\"id\\": 5 @@ -141,7 +153,7 @@ exports[`record integration tests can freeze mutations 1`] = ` \\"texts\\": [], \\"attributes\\": [ { - \\"id\\": 18, + \\"id\\": 20, \\"attributes\\": { \\"foo\\": \\"bar\\" } @@ -165,7 +177,7 @@ exports[`record integration tests can freeze mutations 1`] = ` \\"foo\\": \\"bar\\" }, \\"childNodes\\": [], - \\"id\\": 18 + \\"id\\": 20 } } ] @@ -272,9 +284,21 @@ exports[`record integration tests can mask character data mutations 1`] = ` }, { \\"type\\": 3, - \\"textContent\\": \\"\\\\n\\\\n \\", + \\"textContent\\": \\"\\\\n \\", \\"id\\": 14 }, + { + \\"type\\": 2, + \\"tagName\\": \\"canvas\\", + \\"attributes\\": {}, + \\"childNodes\\": [], + \\"id\\": 15 + }, + { + \\"type\\": 3, + \\"textContent\\": \\"\\\\n\\\\n \\", + \\"id\\": 16 + }, { \\"type\\": 2, \\"tagName\\": \\"script\\", @@ -283,15 +307,15 @@ exports[`record integration tests can mask character data mutations 1`] = ` { \\"type\\": 3, \\"textContent\\": \\"SCRIPT_PLACEHOLDER\\", - \\"id\\": 16 + \\"id\\": 18 } ], - \\"id\\": 15 + \\"id\\": 17 }, { \\"type\\": 3, - \\"textContent\\": \\"\\\\n \\\\n \\", - \\"id\\": 17 + \\"textContent\\": \\"\\\\n \\\\n \\\\n\\", + \\"id\\": 19 } ], \\"id\\": 5 @@ -338,16 +362,16 @@ exports[`record integration tests can mask character data mutations 1`] = ` \\"class\\": \\"rr-mask\\" }, \\"childNodes\\": [], - \\"id\\": 18 + \\"id\\": 20 } }, { - \\"parentId\\": 18, + \\"parentId\\": 20, \\"nextId\\": null, \\"node\\": { \\"type\\": 3, \\"textContent\\": \\"*** **** ****\\", - \\"id\\": 19 + \\"id\\": 21 } }, { @@ -356,7 +380,7 @@ exports[`record integration tests can mask character data mutations 1`] = ` \\"node\\": { \\"type\\": 3, \\"textContent\\": \\"*******\\", - \\"id\\": 20 + \\"id\\": 22 } } ] @@ -463,9 +487,21 @@ exports[`record integration tests can record attribute mutation 1`] = ` }, { \\"type\\": 3, - \\"textContent\\": \\"\\\\n\\\\n \\", + \\"textContent\\": \\"\\\\n \\", \\"id\\": 14 }, + { + \\"type\\": 2, + \\"tagName\\": \\"canvas\\", + \\"attributes\\": {}, + \\"childNodes\\": [], + \\"id\\": 15 + }, + { + \\"type\\": 3, + \\"textContent\\": \\"\\\\n\\\\n \\", + \\"id\\": 16 + }, { \\"type\\": 2, \\"tagName\\": \\"script\\", @@ -474,15 +510,15 @@ exports[`record integration tests can record attribute mutation 1`] = ` { \\"type\\": 3, \\"textContent\\": \\"SCRIPT_PLACEHOLDER\\", - \\"id\\": 16 + \\"id\\": 18 } ], - \\"id\\": 15 + \\"id\\": 17 }, { \\"type\\": 3, - \\"textContent\\": \\"\\\\n \\\\n \\", - \\"id\\": 17 + \\"textContent\\": \\"\\\\n \\\\n \\\\n\\", + \\"id\\": 19 } ], \\"id\\": 5 @@ -622,9 +658,21 @@ exports[`record integration tests can record character data muatations 1`] = ` }, { \\"type\\": 3, - \\"textContent\\": \\"\\\\n\\\\n \\", + \\"textContent\\": \\"\\\\n \\", \\"id\\": 14 }, + { + \\"type\\": 2, + \\"tagName\\": \\"canvas\\", + \\"attributes\\": {}, + \\"childNodes\\": [], + \\"id\\": 15 + }, + { + \\"type\\": 3, + \\"textContent\\": \\"\\\\n\\\\n \\", + \\"id\\": 16 + }, { \\"type\\": 2, \\"tagName\\": \\"script\\", @@ -633,15 +681,15 @@ exports[`record integration tests can record character data muatations 1`] = ` { \\"type\\": 3, \\"textContent\\": \\"SCRIPT_PLACEHOLDER\\", - \\"id\\": 16 + \\"id\\": 18 } ], - \\"id\\": 15 + \\"id\\": 17 }, { \\"type\\": 3, - \\"textContent\\": \\"\\\\n \\\\n \\", - \\"id\\": 17 + \\"textContent\\": \\"\\\\n \\\\n \\\\n\\", + \\"id\\": 19 } ], \\"id\\": 5 @@ -681,7 +729,7 @@ exports[`record integration tests can record character data muatations 1`] = ` \\"node\\": { \\"type\\": 3, \\"textContent\\": \\"mutated\\", - \\"id\\": 18 + \\"id\\": 20 } } ] @@ -788,9 +836,21 @@ exports[`record integration tests can record childList mutations 1`] = ` }, { \\"type\\": 3, - \\"textContent\\": \\"\\\\n\\\\n \\", + \\"textContent\\": \\"\\\\n \\", \\"id\\": 14 }, + { + \\"type\\": 2, + \\"tagName\\": \\"canvas\\", + \\"attributes\\": {}, + \\"childNodes\\": [], + \\"id\\": 15 + }, + { + \\"type\\": 3, + \\"textContent\\": \\"\\\\n\\\\n \\", + \\"id\\": 16 + }, { \\"type\\": 2, \\"tagName\\": \\"script\\", @@ -799,15 +859,15 @@ exports[`record integration tests can record childList mutations 1`] = ` { \\"type\\": 3, \\"textContent\\": \\"SCRIPT_PLACEHOLDER\\", - \\"id\\": 16 + \\"id\\": 18 } ], - \\"id\\": 15 + \\"id\\": 17 }, { \\"type\\": 3, - \\"textContent\\": \\"\\\\n \\\\n \\", - \\"id\\": 17 + \\"textContent\\": \\"\\\\n \\\\n \\\\n\\", + \\"id\\": 19 } ], \\"id\\": 5 @@ -845,7 +905,7 @@ exports[`record integration tests can record childList mutations 1`] = ` \\"tagName\\": \\"span\\", \\"attributes\\": {}, \\"childNodes\\": [], - \\"id\\": 18 + \\"id\\": 20 } } ] @@ -9839,9 +9899,21 @@ exports[`record integration tests will serialize node before record 1`] = ` }, { \\"type\\": 3, - \\"textContent\\": \\"\\\\n\\\\n \\", + \\"textContent\\": \\"\\\\n \\", \\"id\\": 14 }, + { + \\"type\\": 2, + \\"tagName\\": \\"canvas\\", + \\"attributes\\": {}, + \\"childNodes\\": [], + \\"id\\": 15 + }, + { + \\"type\\": 3, + \\"textContent\\": \\"\\\\n\\\\n \\", + \\"id\\": 16 + }, { \\"type\\": 2, \\"tagName\\": \\"script\\", @@ -9850,15 +9922,15 @@ exports[`record integration tests will serialize node before record 1`] = ` { \\"type\\": 3, \\"textContent\\": \\"SCRIPT_PLACEHOLDER\\", - \\"id\\": 16 + \\"id\\": 18 } ], - \\"id\\": 15 + \\"id\\": 17 }, { \\"type\\": 3, - \\"textContent\\": \\"\\\\n \\\\n \\", - \\"id\\": 17 + \\"textContent\\": \\"\\\\n \\\\n \\\\n\\", + \\"id\\": 19 } ], \\"id\\": 5 @@ -9891,29 +9963,29 @@ exports[`record integration tests will serialize node before record 1`] = ` \\"tagName\\": \\"li\\", \\"attributes\\": {}, \\"childNodes\\": [], - \\"id\\": 18 + \\"id\\": 20 } }, { \\"parentId\\": 10, - \\"nextId\\": 18, + \\"nextId\\": 20, \\"node\\": { \\"type\\": 2, \\"tagName\\": \\"li\\", \\"attributes\\": {}, \\"childNodes\\": [], - \\"id\\": 19 + \\"id\\": 21 } }, { \\"parentId\\": 10, - \\"nextId\\": 19, + \\"nextId\\": 21, \\"node\\": { \\"type\\": 2, \\"tagName\\": \\"li\\", \\"attributes\\": {}, \\"childNodes\\": [], - \\"id\\": 20 + \\"id\\": 22 } } ] diff --git a/packages/rrweb/test/html/mutation-observer.html b/packages/rrweb/test/html/mutation-observer.html index 0175a31901..9d149c5a22 100644 --- a/packages/rrweb/test/html/mutation-observer.html +++ b/packages/rrweb/test/html/mutation-observer.html @@ -4,4 +4,5 @@
- \ No newline at end of file + + From 35da32d9f9f80439c239b77d73a53b37cc9a57a4 Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Tue, 1 Feb 2022 12:36:33 +0100 Subject: [PATCH 87/93] Add Canvas (Mutation) Manager Allows you to do `record.freezePage()` and canvas events will get paused. Based on https://github.com/rrweb-io/rrweb/pull/756#issuecomment-1007566907 --- packages/rrweb/src/record/index.ts | 32 ++-- packages/rrweb/src/record/mutation.ts | 19 ++- packages/rrweb/src/record/observer.ts | 46 +----- .../rrweb/src/record/observers/canvas/2d.ts | 12 +- .../record/observers/canvas/canvas-manager.ts | 152 ++++++++++++++++++ .../src/record/observers/canvas/webgl.ts | 95 +---------- .../rrweb/src/record/shadow-dom-manager.ts | 3 + packages/rrweb/src/types.ts | 15 +- .../__snapshots__/integration.test.ts.snap | 46 +++--- packages/rrweb/test/integration.test.ts | 13 +- packages/rrweb/test/record/webgl.test.ts | 12 +- packages/rrweb/test/utils.ts | 10 ++ packages/rrweb/typings/record/mutation.d.ts | 5 +- packages/rrweb/typings/record/observer.d.ts | 3 +- .../typings/record/observers/canvas/2d.d.ts | 4 +- .../observers/canvas/canvas-manager.d.ts | 31 ++++ .../observers/canvas/serialize-args.d.ts | 7 +- .../record/observers/canvas/webgl.d.ts | 4 +- .../typings/record/shadow-dom-manager.d.ts | 2 + packages/rrweb/typings/replay/canvas/2d.d.ts | 4 +- .../rrweb/typings/replay/canvas/webgl.d.ts | 11 +- packages/rrweb/typings/replay/index.d.ts | 1 + packages/rrweb/typings/replay/timer.d.ts | 5 +- packages/rrweb/typings/types.d.ts | 28 +++- yarn.lock | 8 + 25 files changed, 353 insertions(+), 215 deletions(-) create mode 100644 packages/rrweb/src/record/observers/canvas/canvas-manager.ts create mode 100644 packages/rrweb/typings/record/observers/canvas/canvas-manager.d.ts diff --git a/packages/rrweb/src/record/index.ts b/packages/rrweb/src/record/index.ts index 5930913a8a..6279ed315b 100644 --- a/packages/rrweb/src/record/index.ts +++ b/packages/rrweb/src/record/index.ts @@ -18,9 +18,11 @@ import { listenerHandler, mutationCallbackParam, scrollCallback, + canvasMutationParam, } from '../types'; import { IframeManager } from './iframe-manager'; import { ShadowDomManager } from './shadow-dom-manager'; +import { CanvasManager } from './observers/canvas/canvas-manager'; function wrapEvent(e: event): eventWithTime { return { @@ -179,11 +181,28 @@ function record( }, }), ); + const wrappedCanvasMutationEmit = (p: canvasMutationParam) => + wrappedEmit( + wrapEvent({ + type: EventType.IncrementalSnapshot, + data: { + source: IncrementalSource.CanvasMutation, + ...p, + }, + }), + ); const iframeManager = new IframeManager({ mutationCb: wrappedMutationEmit, }); + const canvasManager = new CanvasManager({ + mutationCb: wrappedCanvasMutationEmit, + win: window, + blockClass, + mirror, + }); + const shadowDomManager = new ShadowDomManager({ mutationCb: wrappedMutationEmit, scrollCb: wrappedScrollEmit, @@ -200,6 +219,7 @@ function record( sampling, slimDOMOptions, iframeManager, + canvasManager, }, mirror, }); @@ -362,16 +382,7 @@ function record( }, }), ), - canvasMutationCb: (p) => - wrappedEmit( - wrapEvent({ - type: EventType.IncrementalSnapshot, - data: { - source: IncrementalSource.CanvasMutation, - ...p, - }, - }), - ), + canvasMutationCb: wrappedCanvasMutationEmit, fontCb: (p) => wrappedEmit( wrapEvent({ @@ -400,6 +411,7 @@ function record( mirror, iframeManager, shadowDomManager, + canvasManager, plugins: plugins?.map((p) => ({ observer: p.observer, diff --git a/packages/rrweb/src/record/mutation.ts b/packages/rrweb/src/record/mutation.ts index 924c58cd2c..d675e88971 100644 --- a/packages/rrweb/src/record/mutation.ts +++ b/packages/rrweb/src/record/mutation.ts @@ -31,6 +31,7 @@ import { hasShadowRoot, } from '../utils'; import { IframeManager } from './iframe-manager'; +import { CanvasManager } from './observers/canvas/canvas-manager'; import { ShadowDomManager } from './shadow-dom-manager'; type DoubleLinkedListNode = { @@ -178,6 +179,7 @@ export default class MutationBuffer { private mirror: Mirror; private iframeManager: IframeManager; private shadowDomManager: ShadowDomManager; + private canvasManager: CanvasManager; public init( cb: mutationCallBack, @@ -195,6 +197,7 @@ export default class MutationBuffer { mirror: Mirror, iframeManager: IframeManager, shadowDomManager: ShadowDomManager, + canvasManager: CanvasManager, ) { this.blockClass = blockClass; this.blockSelector = blockSelector; @@ -211,14 +214,17 @@ export default class MutationBuffer { this.mirror = mirror; this.iframeManager = iframeManager; this.shadowDomManager = shadowDomManager; + this.canvasManager = canvasManager; } public freeze() { this.frozen = true; + this.canvasManager.freeze(); } public unfreeze() { this.frozen = false; + this.canvasManager.unfreeze(); this.emit(); } @@ -228,16 +234,22 @@ export default class MutationBuffer { public lock() { this.locked = true; + this.canvasManager.lock(); } public unlock() { this.locked = false; + this.canvasManager.unlock(); this.emit(); } + public reset() { + this.canvasManager.reset(); + } + public processMutations = (mutations: mutationRecord[]) => { - mutations.forEach(this.processMutation); - this.emit(); + mutations.forEach(this.processMutation); // adds mutations to the buffer + this.emit(); // clears buffer if not locked/frozen }; public emit = () => { @@ -504,7 +516,8 @@ export default class MutationBuffer { } } for (const pname of Array.from(old.style)) { - if (target.style.getPropertyValue(pname) === '') { // "if not set, returns the empty string" + if (target.style.getPropertyValue(pname) === '') { + // "if not set, returns the empty string" styleObj[pname] = false; // delete } } diff --git a/packages/rrweb/src/record/observer.ts b/packages/rrweb/src/record/observer.ts index ce92e9446d..00c944c0cb 100644 --- a/packages/rrweb/src/record/observer.ts +++ b/packages/rrweb/src/record/observer.ts @@ -49,9 +49,7 @@ import { import MutationBuffer from './mutation'; import { IframeManager } from './iframe-manager'; import { ShadowDomManager } from './shadow-dom-manager'; -import initCanvasContextObserver from './observers/canvas/canvas'; -import initCanvas2DMutationObserver from './observers/canvas/2d'; -import initCanvasWebGLMutationObserver from './observers/canvas/webgl'; +import { CanvasManager } from './observers/canvas/canvas-manager'; type WindowWithStoredMutationObserver = IWindow & { __rrMutationObserver?: MutationObserver; @@ -104,6 +102,8 @@ export function initMutationObserver( mirror: Mirror, iframeManager: IframeManager, shadowDomManager: ShadowDomManager, + canvasManager: CanvasManager, + // canvasCb: canvasMutationCallback, rootEl: Node, ): MutationObserver { const mutationBuffer = new MutationBuffer(); @@ -125,6 +125,7 @@ export function initMutationObserver( mirror, iframeManager, shadowDomManager, + canvasManager, ); let mutationObserverCtor = window.MutationObserver || @@ -715,34 +716,6 @@ function initMediaInteractionObserver( }; } -function initCanvasMutationObserver( - cb: canvasMutationCallback, - win: IWindow, - blockClass: blockClass, - mirror: Mirror, -): listenerHandler { - const canvasContextReset = initCanvasContextObserver(win, blockClass); - const canvas2DReset = initCanvas2DMutationObserver( - cb, - win, - blockClass, - mirror, - ); - - const canvasWebGL1and2Reset = initCanvasWebGLMutationObserver( - cb, - win, - blockClass, - mirror, - ); - - return () => { - canvasContextReset(); - canvas2DReset(); - canvasWebGL1and2Reset(); - }; -} - function initFontObserver(cb: fontCallback, doc: Document): listenerHandler { const win = doc.defaultView as IWindow; if (!win) { @@ -904,6 +877,7 @@ export function initObservers( o.mirror, o.iframeManager, o.shadowDomManager, + o.canvasManager, o.doc, ); const mousemoveHandler = initMoveObserver( @@ -955,14 +929,6 @@ export function initObservers( currentWindow, o.mirror, ); - const canvasMutationObserver = o.recordCanvas - ? initCanvasMutationObserver( - o.canvasMutationCb, - currentWindow, - o.blockClass, - o.mirror, - ) - : () => {}; const fontObserver = o.collectFonts ? initFontObserver(o.fontCb, o.doc) : () => {}; @@ -975,6 +941,7 @@ export function initObservers( } return () => { + mutationBuffers.forEach((b) => b.reset()); mutationObserver.disconnect(); mousemoveHandler(); mouseInteractionHandler(); @@ -984,7 +951,6 @@ export function initObservers( mediaInteractionHandler(); styleSheetObserver(); styleDeclarationObserver(); - canvasMutationObserver(); fontObserver(); pluginHandlers.forEach((h) => h()); }; diff --git a/packages/rrweb/src/record/observers/canvas/2d.ts b/packages/rrweb/src/record/observers/canvas/2d.ts index 68261f826f..dd63469b1e 100644 --- a/packages/rrweb/src/record/observers/canvas/2d.ts +++ b/packages/rrweb/src/record/observers/canvas/2d.ts @@ -2,7 +2,7 @@ import { INode } from 'rrweb-snapshot'; import { blockClass, CanvasContext, - canvasMutationCallback, + canvasManagerMutationCallback, IWindow, listenerHandler, Mirror, @@ -10,7 +10,7 @@ import { import { hookSetter, isBlocked, patch } from '../../../utils'; export default function initCanvas2DMutationObserver( - cb: canvasMutationCallback, + cb: canvasManagerMutationCallback, win: IWindow, blockClass: blockClass, mirror: Mirror, @@ -37,6 +37,8 @@ export default function initCanvas2DMutationObserver( ...args: Array ) { if (!isBlocked((this.canvas as unknown) as INode, blockClass)) { + // Using setTimeout as getImageData + JSON.stringify can be heavy + // and we'd rather not block the main thread setTimeout(() => { const recordArgs = [...args]; if (prop === 'drawImage') { @@ -56,8 +58,7 @@ export default function initCanvas2DMutationObserver( recordArgs[0] = JSON.stringify(pix); } } - cb({ - id: mirror.getId((this.canvas as unknown) as INode), + cb(this.canvas, { type: CanvasContext['2D'], property: prop, args: recordArgs, @@ -75,8 +76,7 @@ export default function initCanvas2DMutationObserver( prop, { set(v) { - cb({ - id: mirror.getId((this.canvas as unknown) as INode), + cb(this.canvas, { type: CanvasContext['2D'], property: prop, args: [v], diff --git a/packages/rrweb/src/record/observers/canvas/canvas-manager.ts b/packages/rrweb/src/record/observers/canvas/canvas-manager.ts new file mode 100644 index 0000000000..28baaa8505 --- /dev/null +++ b/packages/rrweb/src/record/observers/canvas/canvas-manager.ts @@ -0,0 +1,152 @@ +import { INode } from 'rrweb-snapshot'; +import { + blockClass, + canvasManagerMutationCallback, + canvasMutationCallback, + canvasMutationCommand, + canvasMutationWithType, + IWindow, + listenerHandler, + Mirror, +} from '../../../types'; +import initCanvas2DMutationObserver from './2d'; +import initCanvasContextObserver from './canvas'; +import initCanvasWebGLMutationObserver from './webgl'; + +export type RafStamps = { latestId: number; invokeId: number | null }; + +type pendingCanvasMutationsMap = Map< + HTMLCanvasElement, + canvasMutationWithType[] +>; + +export class CanvasManager { + private pendingCanvasMutations: pendingCanvasMutationsMap = new Map(); + private rafStamps: RafStamps = { latestId: 0, invokeId: null }; + private mirror: Mirror; + + private mutationCb: canvasMutationCallback; + private resetObservers: listenerHandler; + private frozen: boolean = false; + private locked: boolean = false; + + public reset() { + this.pendingCanvasMutations.clear(); + this.resetObservers(); + } + + public freeze() { + this.frozen = true; + } + + public unfreeze() { + this.frozen = false; + } + + public lock() { + this.locked = true; + } + + public unlock() { + this.locked = false; + } + + constructor(options: { + mutationCb: canvasMutationCallback; + win: IWindow; + blockClass: blockClass; + mirror: Mirror; + }) { + this.mutationCb = options.mutationCb; + this.mirror = options.mirror; + + this.initCanvasMutationObserver(options.win, options.blockClass); + } + + private processMutation: canvasManagerMutationCallback = function ( + target, + mutation, + ) { + const newFrame = + this.rafStamps.invokeId && + this.rafStamps.latestId !== this.rafStamps.invokeId; + if (newFrame || !this.rafStamps.invokeId) + this.rafStamps.invokeId = this.rafStamps.latestId; + + if (!this.pendingCanvasMutations.has(target)) { + this.pendingCanvasMutations.set(target, []); + } + + this.pendingCanvasMutations.get(target)!.push(mutation); + }; + + private initCanvasMutationObserver( + win: IWindow, + blockClass: blockClass, + ): void { + this.startRAFTimestamping(); + this.startPendingCanvasMutationFlusher(); + + const canvasContextReset = initCanvasContextObserver(win, blockClass); + const canvas2DReset = initCanvas2DMutationObserver( + this.processMutation.bind(this), + win, + blockClass, + this.mirror, + ); + + const canvasWebGL1and2Reset = initCanvasWebGLMutationObserver( + this.processMutation.bind(this), + win, + blockClass, + this.mirror, + ); + + this.resetObservers = () => { + canvasContextReset(); + canvas2DReset(); + canvasWebGL1and2Reset(); + }; + } + + private startPendingCanvasMutationFlusher() { + requestAnimationFrame(() => this.flushPendingCanvasMutations()); + } + + private startRAFTimestamping() { + const setLatestRAFTimestamp = (timestamp: DOMHighResTimeStamp) => { + this.rafStamps.latestId = timestamp; + requestAnimationFrame(setLatestRAFTimestamp); + }; + requestAnimationFrame(setLatestRAFTimestamp); + } + + flushPendingCanvasMutations() { + this.pendingCanvasMutations.forEach( + (values: canvasMutationCommand[], canvas: HTMLCanvasElement) => { + const id = this.mirror.getId((canvas as unknown) as INode); + this.flushPendingCanvasMutationFor(canvas, id); + }, + ); + requestAnimationFrame(() => this.flushPendingCanvasMutations()); + } + + flushPendingCanvasMutationFor(canvas: HTMLCanvasElement, id: number) { + if (this.frozen || this.locked) { + return; + } + + const valuesWithType = this.pendingCanvasMutations.get(canvas); + if (!valuesWithType || id === -1) return; + + const values = valuesWithType.map((value) => { + const { type, ...rest } = value; + return rest; + }); + const { type } = valuesWithType[0]; + + this.mutationCb({ id, type, commands: values }); + + this.pendingCanvasMutations.delete(canvas); + } +} diff --git a/packages/rrweb/src/record/observers/canvas/webgl.ts b/packages/rrweb/src/record/observers/canvas/webgl.ts index 873a97bc2f..45b03928f3 100644 --- a/packages/rrweb/src/record/observers/canvas/webgl.ts +++ b/packages/rrweb/src/record/observers/canvas/webgl.ts @@ -2,8 +2,8 @@ import { INode } from 'rrweb-snapshot'; import { blockClass, CanvasContext, - canvasMutationCallback, - canvasMutationCommand, + canvasManagerMutationCallback, + canvasMutationWithType, IWindow, listenerHandler, Mirror, @@ -11,57 +11,12 @@ import { import { hookSetter, isBlocked, patch } from '../../../utils'; import { saveWebGLVar, serializeArgs } from './serialize-args'; -type canvasMutationWithType = { type: CanvasContext } & canvasMutationCommand; -type pendingCanvasMutationsMap = Map< - HTMLCanvasElement, - canvasMutationWithType[] ->; -type RafStamps = { latestId: number; invokeId: number | null }; - -function flushPendingCanvasMutations( - pendingCanvasMutations: pendingCanvasMutationsMap, - cb: canvasMutationCallback, - mirror: Mirror, -) { - pendingCanvasMutations.forEach( - (values: canvasMutationCommand[], canvas: HTMLCanvasElement) => { - const id = mirror.getId((canvas as unknown) as INode); - flushPendingCanvasMutationFor(canvas, pendingCanvasMutations, id, cb); - }, - ); - requestAnimationFrame(() => - flushPendingCanvasMutations(pendingCanvasMutations, cb, mirror), - ); -} - -function flushPendingCanvasMutationFor( - canvas: HTMLCanvasElement, - pendingCanvasMutations: pendingCanvasMutationsMap, - id: number, - cb: canvasMutationCallback, -) { - const valuesWithType = pendingCanvasMutations.get(canvas); - if (!valuesWithType || id === -1) return; - - const values = valuesWithType.map((value) => { - const { type, ...rest } = value; - return rest; - }); - const { type } = valuesWithType[0]; - - cb({ id, type, commands: values }); - - pendingCanvasMutations.delete(canvas); -} - function patchGLPrototype( prototype: WebGLRenderingContext | WebGL2RenderingContext, type: CanvasContext, - cb: canvasMutationCallback, + cb: canvasManagerMutationCallback, blockClass: blockClass, mirror: Mirror, - pendingCanvasMutations: pendingCanvasMutationsMap, - rafStamps: RafStamps, win: IWindow, ): listenerHandler[] { const handlers: listenerHandler[] = []; @@ -75,11 +30,6 @@ function patchGLPrototype( } const restoreHandler = patch(prototype, prop, function (original) { return function (this: typeof prototype, ...args: Array) { - const newFrame = - rafStamps.invokeId && rafStamps.latestId !== rafStamps.invokeId; - if (newFrame || !rafStamps.invokeId) - rafStamps.invokeId = rafStamps.latestId; - const result = original.apply(this, args); saveWebGLVar(result, win, prototype); if (!isBlocked((this.canvas as unknown) as INode, blockClass)) { @@ -91,15 +41,8 @@ function patchGLPrototype( property: prop, args: recordArgs, }; - - if (!pendingCanvasMutations.has(this.canvas as HTMLCanvasElement)) { - pendingCanvasMutations.set(this.canvas as HTMLCanvasElement, []); - } - - pendingCanvasMutations - // FIXME! THIS COULD MAYBE BE AN OFFSCREEN CANVAS - .get(this.canvas as HTMLCanvasElement)! - .push(mutation); + // TODO: this could potentially also be an OffscreenCanvas as well as HTMLCanvasElement + cb(this.canvas as HTMLCanvasElement, mutation); } return result; @@ -109,8 +52,8 @@ function patchGLPrototype( } catch { const hookHandler = hookSetter(prototype, prop, { set(v) { - cb({ - id: mirror.getId((this.canvas as unknown) as INode), + // TODO: this could potentially also be an OffscreenCanvas as well as HTMLCanvasElement + cb(this.canvas as HTMLCanvasElement, { type, property: prop, args: [v], @@ -126,29 +69,12 @@ function patchGLPrototype( } export default function initCanvasWebGLMutationObserver( - cb: canvasMutationCallback, + cb: canvasManagerMutationCallback, win: IWindow, blockClass: blockClass, mirror: Mirror, ): listenerHandler { const handlers: listenerHandler[] = []; - const pendingCanvasMutations: pendingCanvasMutationsMap = new Map(); - - const rafStamps: RafStamps = { - latestId: 0, - invokeId: null, - }; - - const setLatestRAFTimestamp = (timestamp: DOMHighResTimeStamp) => { - rafStamps.latestId = timestamp; - requestAnimationFrame(setLatestRAFTimestamp); - }; - requestAnimationFrame(setLatestRAFTimestamp); - - // TODO: replace me - requestAnimationFrame(() => - flushPendingCanvasMutations(pendingCanvasMutations, cb, mirror), - ); handlers.push( ...patchGLPrototype( @@ -157,8 +83,6 @@ export default function initCanvasWebGLMutationObserver( cb, blockClass, mirror, - pendingCanvasMutations, - rafStamps, win, ), ); @@ -171,15 +95,12 @@ export default function initCanvasWebGLMutationObserver( cb, blockClass, mirror, - pendingCanvasMutations, - rafStamps, win, ), ); } return () => { - pendingCanvasMutations.clear(); handlers.forEach((h) => h()); }; } diff --git a/packages/rrweb/src/record/shadow-dom-manager.ts b/packages/rrweb/src/record/shadow-dom-manager.ts index e068bb645d..230c2de516 100644 --- a/packages/rrweb/src/record/shadow-dom-manager.ts +++ b/packages/rrweb/src/record/shadow-dom-manager.ts @@ -14,6 +14,7 @@ import { } from 'rrweb-snapshot'; import { IframeManager } from './iframe-manager'; import { initMutationObserver, initScrollObserver } from './observer'; +import { CanvasManager } from './observers/canvas/canvas-manager'; type BypassOptions = { blockClass: blockClass; @@ -28,6 +29,7 @@ type BypassOptions = { sampling: SamplingStrategy; slimDOMOptions: SlimDOMOptions; iframeManager: IframeManager; + canvasManager: CanvasManager; }; export class ShadowDomManager { @@ -65,6 +67,7 @@ export class ShadowDomManager { this.mirror, this.bypassOptions.iframeManager, this, + this.bypassOptions.canvasManager, shadowRoot, ); initScrollObserver( diff --git a/packages/rrweb/src/types.ts b/packages/rrweb/src/types.ts index b7a4372f3b..0167f0284f 100644 --- a/packages/rrweb/src/types.ts +++ b/packages/rrweb/src/types.ts @@ -12,6 +12,7 @@ import { FontFaceDescriptors } from 'css-font-loading-module'; import { IframeManager } from './record/iframe-manager'; import { ShadowDomManager } from './record/shadow-dom-manager'; import type { Replayer } from './replay'; +import { CanvasManager } from './record/observers/canvas/canvas-manager'; export enum EventType { DomContentLoaded, @@ -266,6 +267,7 @@ export type observerParam = { mirror: Mirror; iframeManager: IframeManager; shadowDomManager: ShadowDomManager; + canvasManager: CanvasManager; plugins: Array<{ observer: Function; callback: Function; @@ -463,8 +465,6 @@ export type styleDeclarationParam = { export type styleDeclarationCallback = (s: styleDeclarationParam) => void; -export type canvasMutationCallback = (p: canvasMutationParam) => void; - export type canvasMutationCommand = { property: string; args: Array; @@ -482,6 +482,17 @@ export type canvasMutationParam = type: CanvasContext; } & canvasMutationCommand); +export type canvasMutationWithType = { + type: CanvasContext; +} & canvasMutationCommand; + +export type canvasMutationCallback = (p: canvasMutationParam) => void; + +export type canvasManagerMutationCallback = ( + target: HTMLCanvasElement, + p: canvasMutationWithType, +) => void; + export type fontParam = { family: string; fontSource: string; diff --git a/packages/rrweb/test/__snapshots__/integration.test.ts.snap b/packages/rrweb/test/__snapshots__/integration.test.ts.snap index 5fde3b0844..417e3e91bb 100644 --- a/packages/rrweb/test/__snapshots__/integration.test.ts.snap +++ b/packages/rrweb/test/__snapshots__/integration.test.ts.snap @@ -7377,35 +7377,27 @@ exports[`record integration tests should record canvas mutations 1`] = ` \\"source\\": 9, \\"id\\": 16, \\"type\\": 0, - \\"property\\": \\"moveTo\\", - \\"args\\": [ - 0, - 0 - ] - } - }, - { - \\"type\\": 3, - \\"data\\": { - \\"source\\": 9, - \\"id\\": 16, - \\"type\\": 0, - \\"property\\": \\"lineTo\\", - \\"args\\": [ - 200, - 100 + \\"commands\\": [ + { + \\"property\\": \\"moveTo\\", + \\"args\\": [ + 0, + 0 + ] + }, + { + \\"property\\": \\"lineTo\\", + \\"args\\": [ + 200, + 100 + ] + }, + { + \\"property\\": \\"stroke\\", + \\"args\\": [] + } ] } - }, - { - \\"type\\": 3, - \\"data\\": { - \\"source\\": 9, - \\"id\\": 16, - \\"type\\": 0, - \\"property\\": \\"stroke\\", - \\"args\\": [] - } } ]" `; diff --git a/packages/rrweb/test/integration.test.ts b/packages/rrweb/test/integration.test.ts index b83bf09a4a..90f688dce7 100644 --- a/packages/rrweb/test/integration.test.ts +++ b/packages/rrweb/test/integration.test.ts @@ -7,6 +7,7 @@ import { startServer, getServerURL, launchPuppeteer, + waitForRAF, } from './utils'; import { recordOptions, eventWithTime, EventType } from '../src/types'; import { visitSnapshot, NodeType } from 'rrweb-snapshot'; @@ -171,7 +172,9 @@ describe('record integration tests', function (this: ISuite) { it('can freeze mutations', async () => { const page: puppeteer.Page = await browser.newPage(); await page.goto('about:blank'); - await page.setContent(getHtml.call(this, 'mutation-observer.html')); + await page.setContent( + getHtml.call(this, 'mutation-observer.html', { recordCanvas: true }), + ); await page.evaluate(() => { const li = document.createElement('li'); @@ -183,6 +186,9 @@ describe('record integration tests', function (this: ISuite) { await page.evaluate('rrweb.freezePage()'); await page.evaluate(() => { document.body.setAttribute('test', 'bad'); + const canvas = document.querySelector('canvas') as HTMLCanvasElement; + const gl = canvas.getContext('webgl') as WebGLRenderingContext; + gl.getExtension('bad'); const ul = document.querySelector('ul') as HTMLUListElement; const li = document.createElement('li'); li.setAttribute('bad-attr', 'bad'); @@ -190,6 +196,9 @@ describe('record integration tests', function (this: ISuite) { ul.appendChild(li); document.body.removeChild(ul); }); + + await waitForRAF(page); + const snapshots = await page.evaluate('window.snapshots'); assertSnapshot(snapshots); }); @@ -365,7 +374,7 @@ describe('record integration tests', function (this: ISuite) { recordCanvas: true, }), ); - await page.waitForTimeout(50); + await waitForRAF(page); const snapshots = await page.evaluate('window.snapshots'); for (const event of snapshots) { if (event.type === EventType.FullSnapshot) { diff --git a/packages/rrweb/test/record/webgl.test.ts b/packages/rrweb/test/record/webgl.test.ts index 31a2383944..17442669d4 100644 --- a/packages/rrweb/test/record/webgl.test.ts +++ b/packages/rrweb/test/record/webgl.test.ts @@ -11,7 +11,7 @@ import { IncrementalSource, CanvasContext, } from '../../src/types'; -import { assertSnapshot, launchPuppeteer } from '../utils'; +import { assertSnapshot, launchPuppeteer, waitForRAF } from '../utils'; import { ICanvas } from 'rrweb-snapshot'; interface ISuite { @@ -76,16 +76,6 @@ const setup = function (this: ISuite, content: string): ISuite { return ctx; }; -async function waitForRAF(page: puppeteer.Page) { - return await page.evaluate(() => { - return new Promise((resolve) => { - requestAnimationFrame(() => { - requestAnimationFrame(resolve); - }); - }); - }); -} - describe('record webgl', function (this: ISuite) { jest.setTimeout(100_000); diff --git a/packages/rrweb/test/utils.ts b/packages/rrweb/test/utils.ts index 0890d60902..ea904c0cc8 100644 --- a/packages/rrweb/test/utils.ts +++ b/packages/rrweb/test/utils.ts @@ -470,3 +470,13 @@ export const polyfillWebGLGlobals = () => { } global.WebGL2RenderingContext = WebGL2RenderingContext as any; }; + +export async function waitForRAF(page: puppeteer.Page) { + return await page.evaluate(() => { + return new Promise((resolve) => { + requestAnimationFrame(() => { + requestAnimationFrame(resolve); + }); + }); + }); +} diff --git a/packages/rrweb/typings/record/mutation.d.ts b/packages/rrweb/typings/record/mutation.d.ts index b67548e30b..4ded7668e1 100644 --- a/packages/rrweb/typings/record/mutation.d.ts +++ b/packages/rrweb/typings/record/mutation.d.ts @@ -1,6 +1,7 @@ import { MaskInputOptions, SlimDOMOptions, MaskTextFn, MaskInputFn } from 'rrweb-snapshot'; import { mutationRecord, blockClass, maskTextClass, mutationCallBack, Mirror } from '../types'; import { IframeManager } from './iframe-manager'; +import { CanvasManager } from './observers/canvas/canvas-manager'; import { ShadowDomManager } from './shadow-dom-manager'; export default class MutationBuffer { private frozen; @@ -28,12 +29,14 @@ export default class MutationBuffer { private mirror; private iframeManager; private shadowDomManager; - init(cb: mutationCallBack, blockClass: blockClass, blockSelector: string | null, maskTextClass: maskTextClass, maskTextSelector: string | null, inlineStylesheet: boolean, maskInputOptions: MaskInputOptions, maskTextFn: MaskTextFn | undefined, maskInputFn: MaskInputFn | undefined, recordCanvas: boolean, slimDOMOptions: SlimDOMOptions, doc: Document, mirror: Mirror, iframeManager: IframeManager, shadowDomManager: ShadowDomManager): void; + private canvasManager; + init(cb: mutationCallBack, blockClass: blockClass, blockSelector: string | null, maskTextClass: maskTextClass, maskTextSelector: string | null, inlineStylesheet: boolean, maskInputOptions: MaskInputOptions, maskTextFn: MaskTextFn | undefined, maskInputFn: MaskInputFn | undefined, recordCanvas: boolean, slimDOMOptions: SlimDOMOptions, doc: Document, mirror: Mirror, iframeManager: IframeManager, shadowDomManager: ShadowDomManager, canvasManager: CanvasManager): void; freeze(): void; unfreeze(): void; isFrozen(): boolean; lock(): void; unlock(): void; + reset(): void; processMutations: (mutations: mutationRecord[]) => void; emit: () => void; private processMutation; diff --git a/packages/rrweb/typings/record/observer.d.ts b/packages/rrweb/typings/record/observer.d.ts index ffca5918e6..ac3e2f4891 100644 --- a/packages/rrweb/typings/record/observer.d.ts +++ b/packages/rrweb/typings/record/observer.d.ts @@ -3,8 +3,9 @@ import { mutationCallBack, observerParam, listenerHandler, scrollCallback, block import MutationBuffer from './mutation'; import { IframeManager } from './iframe-manager'; import { ShadowDomManager } from './shadow-dom-manager'; +import { CanvasManager } from './observers/canvas/canvas-manager'; export declare const mutationBuffers: MutationBuffer[]; -export declare function initMutationObserver(cb: mutationCallBack, doc: Document, blockClass: blockClass, blockSelector: string | null, maskTextClass: maskTextClass, maskTextSelector: string | null, inlineStylesheet: boolean, maskInputOptions: MaskInputOptions, maskTextFn: MaskTextFn | undefined, maskInputFn: MaskInputFn | undefined, recordCanvas: boolean, slimDOMOptions: SlimDOMOptions, mirror: Mirror, iframeManager: IframeManager, shadowDomManager: ShadowDomManager, rootEl: Node): MutationObserver; +export declare function initMutationObserver(cb: mutationCallBack, doc: Document, blockClass: blockClass, blockSelector: string | null, maskTextClass: maskTextClass, maskTextSelector: string | null, inlineStylesheet: boolean, maskInputOptions: MaskInputOptions, maskTextFn: MaskTextFn | undefined, maskInputFn: MaskInputFn | undefined, recordCanvas: boolean, slimDOMOptions: SlimDOMOptions, mirror: Mirror, iframeManager: IframeManager, shadowDomManager: ShadowDomManager, canvasManager: CanvasManager, rootEl: Node): MutationObserver; export declare function initScrollObserver(cb: scrollCallback, doc: Document, mirror: Mirror, blockClass: blockClass, sampling: SamplingStrategy): listenerHandler; export declare const INPUT_TAGS: string[]; export declare function initObservers(o: observerParam, hooks?: hooksParam): listenerHandler; diff --git a/packages/rrweb/typings/record/observers/canvas/2d.d.ts b/packages/rrweb/typings/record/observers/canvas/2d.d.ts index 23d51acb6b..cb7b7f2e94 100644 --- a/packages/rrweb/typings/record/observers/canvas/2d.d.ts +++ b/packages/rrweb/typings/record/observers/canvas/2d.d.ts @@ -1,2 +1,2 @@ -import { blockClass, canvasMutationCallback, IWindow, listenerHandler, Mirror } from '../../../types'; -export default function initCanvas2DMutationObserver(cb: canvasMutationCallback, win: IWindow, blockClass: blockClass, mirror: Mirror): listenerHandler; +import { blockClass, canvasManagerMutationCallback, IWindow, listenerHandler, Mirror } from '../../../types'; +export default function initCanvas2DMutationObserver(cb: canvasManagerMutationCallback, win: IWindow, blockClass: blockClass, mirror: Mirror): listenerHandler; diff --git a/packages/rrweb/typings/record/observers/canvas/canvas-manager.d.ts b/packages/rrweb/typings/record/observers/canvas/canvas-manager.d.ts new file mode 100644 index 0000000000..6004d787ca --- /dev/null +++ b/packages/rrweb/typings/record/observers/canvas/canvas-manager.d.ts @@ -0,0 +1,31 @@ +import { blockClass, canvasMutationCallback, IWindow, Mirror } from '../../../types'; +export declare type RafStamps = { + latestId: number; + invokeId: number | null; +}; +export declare class CanvasManager { + private pendingCanvasMutations; + private rafStamps; + private mirror; + private mutationCb; + private resetObservers; + private frozen; + private locked; + reset(): void; + freeze(): void; + unfreeze(): void; + lock(): void; + unlock(): void; + constructor(options: { + mutationCb: canvasMutationCallback; + win: IWindow; + blockClass: blockClass; + mirror: Mirror; + }); + private processMutation; + private initCanvasMutationObserver; + private startPendingCanvasMutationFlusher; + private startRAFTimestamping; + flushPendingCanvasMutations(): void; + flushPendingCanvasMutationFor(canvas: HTMLCanvasElement, id: number): void; +} diff --git a/packages/rrweb/typings/record/observers/canvas/serialize-args.d.ts b/packages/rrweb/typings/record/observers/canvas/serialize-args.d.ts index 599e2a6bb7..5a1c90ce77 100644 --- a/packages/rrweb/typings/record/observers/canvas/serialize-args.d.ts +++ b/packages/rrweb/typings/record/observers/canvas/serialize-args.d.ts @@ -1,5 +1,6 @@ import { IWindow, SerializedWebGlArg } from '../../../types'; -export declare function serializeArg(value: any, win: IWindow): SerializedWebGlArg; -export declare const serializeArgs: (args: Array, win: IWindow) => SerializedWebGlArg[]; +export declare function variableListFor(ctx: WebGLRenderingContext | WebGL2RenderingContext, ctor: string): any[]; +export declare const saveWebGLVar: (value: any, win: IWindow, ctx: WebGL2RenderingContext | WebGLRenderingContext) => number | void; +export declare function serializeArg(value: any, win: IWindow, ctx: WebGL2RenderingContext | WebGLRenderingContext): SerializedWebGlArg; +export declare const serializeArgs: (args: Array, win: IWindow, ctx: WebGLRenderingContext | WebGL2RenderingContext) => SerializedWebGlArg[]; export declare const isInstanceOfWebGLObject: (value: any, win: IWindow) => value is WebGLShader | WebGLBuffer | WebGLVertexArrayObject | WebGLTexture | WebGLProgram | WebGLActiveInfo | WebGLUniformLocation | WebGLFramebuffer | WebGLRenderbuffer | WebGLShaderPrecisionFormat; -export declare const saveWebGLVar: (value: any, win: IWindow) => number | void; diff --git a/packages/rrweb/typings/record/observers/canvas/webgl.d.ts b/packages/rrweb/typings/record/observers/canvas/webgl.d.ts index c1263a8f58..0f446770a5 100644 --- a/packages/rrweb/typings/record/observers/canvas/webgl.d.ts +++ b/packages/rrweb/typings/record/observers/canvas/webgl.d.ts @@ -1,2 +1,2 @@ -import { blockClass, canvasMutationCallback, IWindow, listenerHandler, Mirror } from '../../../types'; -export default function initCanvasWebGLMutationObserver(cb: canvasMutationCallback, win: IWindow, blockClass: blockClass, mirror: Mirror): listenerHandler; +import { blockClass, canvasManagerMutationCallback, IWindow, listenerHandler, Mirror } from '../../../types'; +export default function initCanvasWebGLMutationObserver(cb: canvasManagerMutationCallback, win: IWindow, blockClass: blockClass, mirror: Mirror): listenerHandler; diff --git a/packages/rrweb/typings/record/shadow-dom-manager.d.ts b/packages/rrweb/typings/record/shadow-dom-manager.d.ts index c3bbaef717..e853b73cd5 100644 --- a/packages/rrweb/typings/record/shadow-dom-manager.d.ts +++ b/packages/rrweb/typings/record/shadow-dom-manager.d.ts @@ -1,6 +1,7 @@ import { mutationCallBack, blockClass, maskTextClass, Mirror, scrollCallback, SamplingStrategy } from '../types'; import { MaskInputOptions, SlimDOMOptions, MaskTextFn, MaskInputFn } from 'rrweb-snapshot'; import { IframeManager } from './iframe-manager'; +import { CanvasManager } from './observers/canvas/canvas-manager'; declare type BypassOptions = { blockClass: blockClass; blockSelector: string | null; @@ -14,6 +15,7 @@ declare type BypassOptions = { sampling: SamplingStrategy; slimDOMOptions: SlimDOMOptions; iframeManager: IframeManager; + canvasManager: CanvasManager; }; export declare class ShadowDomManager { private mutationCb; diff --git a/packages/rrweb/typings/replay/canvas/2d.d.ts b/packages/rrweb/typings/replay/canvas/2d.d.ts index 9f05cc2ef9..338cbf28f9 100644 --- a/packages/rrweb/typings/replay/canvas/2d.d.ts +++ b/packages/rrweb/typings/replay/canvas/2d.d.ts @@ -1,8 +1,8 @@ import { Replayer } from '../'; -import { canvasMutationData } from '../../types'; +import { canvasMutationCommand } from '../../types'; export default function canvasMutation({ event, mutation, target, imageMap, errorHandler, }: { event: Parameters[0]; - mutation: canvasMutationData; + mutation: canvasMutationCommand; target: HTMLCanvasElement; imageMap: Replayer['imageMap']; errorHandler: Replayer['warnCanvasMutationFailed']; diff --git a/packages/rrweb/typings/replay/canvas/webgl.d.ts b/packages/rrweb/typings/replay/canvas/webgl.d.ts index 12d703d345..6e51941047 100644 --- a/packages/rrweb/typings/replay/canvas/webgl.d.ts +++ b/packages/rrweb/typings/replay/canvas/webgl.d.ts @@ -1,10 +1,11 @@ import { Replayer } from '../'; -import { canvasMutationData, SerializedWebGlArg } from '../../types'; -export declare function variableListFor(ctor: string): any[]; -export declare function deserializeArg(imageMap: Replayer['imageMap']): (arg: SerializedWebGlArg) => any; -export default function webglMutation({ mutation, target, imageMap, errorHandler, }: { - mutation: canvasMutationData; +import { CanvasContext, canvasMutationCommand, SerializedWebGlArg } from '../../types'; +export declare function variableListFor(ctx: WebGLRenderingContext | WebGL2RenderingContext, ctor: string): any[]; +export declare function deserializeArg(imageMap: Replayer['imageMap'], ctx: WebGLRenderingContext | WebGL2RenderingContext): (arg: SerializedWebGlArg) => any; +export default function webglMutation({ mutation, target, type, imageMap, errorHandler, }: { + mutation: canvasMutationCommand; target: HTMLCanvasElement; + type: CanvasContext; imageMap: Replayer['imageMap']; errorHandler: Replayer['warnCanvasMutationFailed']; }): void; diff --git a/packages/rrweb/typings/replay/index.d.ts b/packages/rrweb/typings/replay/index.d.ts index b5d03cd736..05e89b38be 100644 --- a/packages/rrweb/typings/replay/index.d.ts +++ b/packages/rrweb/typings/replay/index.d.ts @@ -54,6 +54,7 @@ export declare class Replayer { private hasImageArg; private getImageArgs; private preloadAllImages; + private preloadImages; private applyIncremental; private applyMutation; private applyScroll; diff --git a/packages/rrweb/typings/replay/timer.d.ts b/packages/rrweb/typings/replay/timer.d.ts index 842b1cbe20..376c92495b 100644 --- a/packages/rrweb/typings/replay/timer.d.ts +++ b/packages/rrweb/typings/replay/timer.d.ts @@ -15,7 +15,4 @@ export declare class Timer { isActive(): boolean; private findActionIndex; } -export declare type LastDelay = { - at: number | null; -}; -export declare function addDelay(event: eventWithTime, baselineTime: number, lastDelay?: LastDelay): number; +export declare function addDelay(event: eventWithTime, baselineTime: number): number; diff --git a/packages/rrweb/typings/types.d.ts b/packages/rrweb/typings/types.d.ts index fe7c8fe23c..2cbf98abba 100644 --- a/packages/rrweb/typings/types.d.ts +++ b/packages/rrweb/typings/types.d.ts @@ -4,6 +4,7 @@ import { FontFaceDescriptors } from 'css-font-loading-module'; import { IframeManager } from './record/iframe-manager'; import { ShadowDomManager } from './record/shadow-dom-manager'; import type { Replayer } from './replay'; +import { CanvasManager } from './record/observers/canvas/canvas-manager'; export declare enum EventType { DomContentLoaded = 0, Load = 1, @@ -122,6 +123,7 @@ export declare type SamplingStrategy = Partial<{ mousemoveCallback: number; mouseInteraction: boolean | Record; scroll: number; + media: number; input: 'all' | 'last'; }>; export declare type RecordPlugin = { @@ -184,6 +186,7 @@ export declare type observerParam = { mirror: Mirror; iframeManager: IframeManager; shadowDomManager: ShadowDomManager; + canvasManager: CanvasManager; plugins: Array<{ observer: Function; callback: Function; @@ -337,15 +340,24 @@ export declare type styleDeclarationParam = { }; }; export declare type styleDeclarationCallback = (s: styleDeclarationParam) => void; -export declare type canvasMutationCallback = (p: canvasMutationParam) => void; -export declare type canvasMutationParam = { - id: number; - type: CanvasContext; +export declare type canvasMutationCommand = { property: string; args: Array; setter?: true; - newFrame?: true; }; +export declare type canvasMutationParam = { + id: number; + type: CanvasContext; + commands: canvasMutationCommand[]; +} | ({ + id: number; + type: CanvasContext; +} & canvasMutationCommand); +export declare type canvasMutationWithType = { + type: CanvasContext; +} & canvasMutationCommand; +export declare type canvasMutationCallback = (p: canvasMutationParam) => void; +export declare type canvasManagerMutationCallback = (target: HTMLCanvasElement, p: canvasMutationWithType) => void; export declare type fontParam = { family: string; fontSource: string; @@ -369,12 +381,15 @@ export declare type inputCallback = (v: inputValue & { export declare const enum MediaInteractions { Play = 0, Pause = 1, - Seeked = 2 + Seeked = 2, + VolumeChange = 3 } export declare type mediaInteractionParam = { type: MediaInteractions; id: number; currentTime?: number; + volume?: number; + muted?: boolean; }; export declare type mediaInteractionCallback = (p: mediaInteractionParam) => void; export declare type DocumentDimension = { @@ -440,7 +455,6 @@ export declare type missingNodeMap = { export declare type actionWithDelay = { doAction: () => void; delay: number; - newFrame: boolean; }; export declare type Handler = (event?: unknown) => void; export declare type Emitter = { diff --git a/yarn.lock b/yarn.lock index b27235cdf1..aefde5b5bd 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1965,6 +1965,14 @@ jest-diff "^27.0.0" pretty-format "^27.0.0" +"@types/jest@^27.0.1": + version "27.4.0" + resolved "https://registry.yarnpkg.com/@types/jest/-/jest-27.4.0.tgz#037ab8b872067cae842a320841693080f9cb84ed" + integrity sha512-gHl8XuC1RZ8H2j5sHv/JqsaxXkDDM9iDOgu0Wp8sjs4u/snb2PVehyWXJPr+ORA0RPpgw231mnutWI1+0hgjIQ== + dependencies: + jest-diff "^27.0.0" + pretty-format "^27.0.0" + "@types/jsdom@^16.2.12", "@types/jsdom@^16.2.4": version "16.2.13" resolved "https://registry.yarnpkg.com/@types/jsdom/-/jsdom-16.2.13.tgz#126c8b7441b159d6234610a48de77b6066f1823f" From 33171299dbcbea89bc911e4e2f200185c23075b6 Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Tue, 1 Feb 2022 12:45:22 +0100 Subject: [PATCH 88/93] cleanup --- packages/rrweb/src/record/observer.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/rrweb/src/record/observer.ts b/packages/rrweb/src/record/observer.ts index 714352c770..7a08416fc5 100644 --- a/packages/rrweb/src/record/observer.ts +++ b/packages/rrweb/src/record/observer.ts @@ -104,7 +104,6 @@ export function initMutationObserver( iframeManager: IframeManager, shadowDomManager: ShadowDomManager, canvasManager: CanvasManager, - // canvasCb: canvasMutationCallback, rootEl: Node, ): MutationObserver { const mutationBuffer = new MutationBuffer(); From 9b0c63fd5c54d711cb2814c0962ba38c9248f4be Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Tue, 1 Feb 2022 15:45:27 +0100 Subject: [PATCH 89/93] Make sure the correct gets replaced --- packages/rrweb/test/e2e/webgl.test.ts | 10 ++++++++-- packages/rrweb/test/integration.test.ts | 4 +++- packages/rrweb/test/utils.ts | 9 +++++++++ 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/packages/rrweb/test/e2e/webgl.test.ts b/packages/rrweb/test/e2e/webgl.test.ts index cf6fe9616a..25f5d60e97 100644 --- a/packages/rrweb/test/e2e/webgl.test.ts +++ b/packages/rrweb/test/e2e/webgl.test.ts @@ -2,7 +2,12 @@ import * as http from 'http'; import * as fs from 'fs'; import * as path from 'path'; import * as puppeteer from 'puppeteer'; -import { startServer, launchPuppeteer, getServerURL } from '../utils'; +import { + startServer, + launchPuppeteer, + getServerURL, + replaceLast, +} from '../utils'; import { recordOptions, eventWithTime, @@ -54,7 +59,8 @@ describe('e2e webgl', () => { ): string => { const filePath = path.resolve(__dirname, `../html/${fileName}`); const html = fs.readFileSync(filePath, 'utf8'); - return html.replace( + return replaceLast( + html, '', `