Add webgl recording and playback

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@highlight-run/rrweb",
-  "version": "1.0.8",
+  "version": "1.1.0",
   "description": "record and replay the web",
   "scripts": {
     "test": "npm run bundle:browser && cross-env TS_NODE_CACHE=false TS_NODE_FILES=true mocha -r ts-node/register -r ignore-styles -r jsdom-global/register test/**.test.ts",
@@ -43,13 +43,13 @@
     "chai": "^4.2.0",
     "cross-env": "^5.2.0",
     "css-loader": "^5.0.1",
-    "mini-css-extract-plugin": "^1.3.8",
     "fast-mhtml": "^1.1.9",
     "ignore-styles": "^5.0.1",
     "inquirer": "^6.2.1",
     "jest-snapshot": "^23.6.0",
     "jsdom": "^16.6.0",
     "jsdom-global": "^3.0.2",
+    "mini-css-extract-plugin": "^1.3.8",
     "mocha": "^5.2.0",
     "node-libtidy": "^0.4.0",
     "prettier": "2.2.1",
@@ -76,7 +76,9 @@
   "dependencies": {
     "@types/css-font-loading-module": "0.0.4",
     "@xstate/fsm": "^1.4.0",
+    "base64-arraybuffer": "^1.0.1",
     "fflate": "^0.4.4",
+    "identity-obj-proxy": "^3.0.0",
     "mitt": "^1.1.3"
   }
 }

src/record/observer.ts

@@ -50,6 +50,9 @@ import {
 import MutationBuffer from './mutation';
 import { IframeManager } from './iframe-manager';
 import { ShadowDomManager } from './shadow-dom-manager';
+import initCanvasContextObserver from './observers/canvas/canvas';
+import initCanvas2DMutationObserver from './observers/canvas/2d';
+import initCanvasWebGLMutationObserver from './observers/canvas/webgl';
 
 type WindowWithStoredMutationObserver = IWindow & {
   __rrMutationObserver?: MutationObserver;
@@ -715,79 +718,25 @@ function initCanvasMutationObserver(
   blockClass: blockClass,
   mirror: Mirror,
 ): listenerHandler {
-  const props = Object.getOwnPropertyNames(
-    (win as any).CanvasRenderingContext2D.prototype,
+  const canvasContextReset = initCanvasContextObserver(win, blockClass);
+  const canvas2DReset = initCanvas2DMutationObserver(
+    cb,
+    win,
+    blockClass,
+    mirror,
   );
-  const handlers: listenerHandler[] = [];
-  for (const prop of props) {
-    try {
-      if (
-        typeof (win as any).CanvasRenderingContext2D.prototype[
-          prop as keyof CanvasRenderingContext2D
-        ] !== 'function'
-      ) {
-        continue;
-      }
-      const restoreHandler = patch(
-        (win as any).CanvasRenderingContext2D.prototype,
-        prop,
-        function (original) {
-          return function (
-            this: CanvasRenderingContext2D,
-            ...args: Array<unknown>
-          ) {
-            if (!isBlocked(this.canvas, blockClass)) {
-              setTimeout(() => {
-                const recordArgs = [...args];
-                if (prop === 'drawImage') {
-                  if (
-                    recordArgs[0] &&
-                    recordArgs[0] instanceof HTMLCanvasElement
-                  ) {
-                    const canvas = recordArgs[0];
-                    const ctx = canvas.getContext('2d');
-                    let imgd = ctx?.getImageData(
-                      0,
-                      0,
-                      canvas.width,
-                      canvas.height,
-                    );
-                    let pix = imgd?.data;
-                    recordArgs[0] = JSON.stringify(pix);
-                  }
-                }
-                cb({
-                  id: mirror.getId((this.canvas as unknown) as INode),
-                  property: prop,
-                  args: recordArgs,
-                });
-              }, 0);
-            }
-            return original.apply(this, args);
-          };
-        },
-      );
-      handlers.push(restoreHandler);
-    } catch {
-      const hookHandler = hookSetter<CanvasRenderingContext2D>(
-        (win as any).CanvasRenderingContext2D.prototype,
-        prop,
-        {
-          set(v) {
-            cb({
-              id: mirror.getId((this.canvas as unknown) as INode),
-              property: prop,
-              args: [v],
-              setter: true,
-            });
-          },
-        },
-      );
-      handlers.push(hookHandler);
-    }
-  }
+
+  const canvasWebGL1and2Reset = initCanvasWebGLMutationObserver(
+    cb,
+    win,
+    blockClass,
+    mirror,
+  );
+
   return () => {
-    handlers.forEach((h) => h());
+    canvasContextReset();
+    canvas2DReset();
+    canvasWebGL1and2Reset();
   };
 }
 

src/record/observers/canvas/2d.ts

@@ -0,0 +1,94 @@
+import { INode } from '../../../snapshot';
+import {
+  blockClass,
+  CanvasContext,
+  canvasMutationCallback,
+  IWindow,
+  listenerHandler,
+  Mirror,
+} from '../../../types';
+import { hookSetter, isBlocked, patch } from '../../../utils';
+
+export default function initCanvas2DMutationObserver(
+  cb: canvasMutationCallback,
+  win: IWindow,
+  blockClass: blockClass,
+  mirror: Mirror,
+): listenerHandler {
+  const handlers: listenerHandler[] = [];
+  const props2D = Object.getOwnPropertyNames(
+    win.CanvasRenderingContext2D.prototype,
+  );
+  for (const prop of props2D) {
+    try {
+      if (
+        typeof win.CanvasRenderingContext2D.prototype[
+          prop as keyof CanvasRenderingContext2D
+        ] !== 'function'
+      ) {
+        continue;
+      }
+      const restoreHandler = patch(
+        win.CanvasRenderingContext2D.prototype,
+        prop,
+        function (original) {
+          return function (
+            this: CanvasRenderingContext2D,
+            ...args: Array<unknown>
+          ) {
+            if (!isBlocked((this.canvas as unknown) as INode, blockClass)) {
+              setTimeout(() => {
+                const recordArgs = [...args];
+                if (prop === 'drawImage') {
+                  if (
+                    recordArgs[0] &&
+                    recordArgs[0] instanceof HTMLCanvasElement
+                  ) {
+                    const canvas = recordArgs[0];
+                    const ctx = canvas.getContext('2d');
+                    let imgd = ctx?.getImageData(
+                      0,
+                      0,
+                      canvas.width,
+                      canvas.height,
+                    );
+                    let pix = imgd?.data;
+                    recordArgs[0] = JSON.stringify(pix);
+                  }
+                }
+                cb({
+                  id: mirror.getId((this.canvas as unknown) as INode),
+                  type: CanvasContext['2D'],
+                  property: prop,
+                  args: recordArgs,
+                });
+              }, 0);
+            }
+            return original.apply(this, args);
+          };
+        },
+      );
+      handlers.push(restoreHandler);
+    } catch {
+      const hookHandler = hookSetter<CanvasRenderingContext2D>(
+        win.CanvasRenderingContext2D.prototype,
+        prop,
+        {
+          set(v) {
+            cb({
+              id: mirror.getId((this.canvas as unknown) as INode),
+              type: CanvasContext['2D'],
+              property: prop,
+              args: [v],
+              setter: true,
+            });
+          },
+        },
+      );
+      handlers.push(hookHandler);
+    }
+  }
+  return () => {
+    handlers.forEach((h) => h());
+  };
+}

src/record/observers/canvas/canvas.ts

@@ -0,0 +1,35 @@
+import { ICanvas, INode } from '../../../snapshot';
+import { blockClass, IWindow, listenerHandler } from '../../../types';
+import { isBlocked, patch } from '../../../utils';
+
+export default function initCanvasContextObserver(
+  win: IWindow,
+  blockClass: blockClass,
+): listenerHandler {
+  const handlers: listenerHandler[] = [];
+  try {
+    const restoreHandler = patch(
+      win.HTMLCanvasElement.prototype,
+      'getContext',
+      function (original) {
+        return function (
+          this: ICanvas,
+          contextType: string,
+          ...args: Array<unknown>
+        ) {
+          if (!isBlocked((this as unknown) as INode, blockClass)) {
+            if (!('__context' in this))
+              (this as ICanvas).__context = contextType;
+          }
+          return original.apply(this, [contextType, ...args]);
+        };
+      },
+    );
+    handlers.push(restoreHandler);
+  } catch {
+    console.error('failed to patch HTMLCanvasElement.prototype.getContext');
+  }
+  return () => {
+    handlers.forEach((h) => h());
+  };
+}

src/record/observers/canvas/serialize-args.ts

@@ -0,0 +1,125 @@
+import { encode } from 'base64-arraybuffer';
+import { SerializedWebGlArg } from '../../../types';
+
+// from webgl-recorder: https://github.com/evanw/webgl-recorder/blob/bef0e65596e981ee382126587e2dcbe0fc7748e2/webgl-recorder.js#L50-L77
+const webGLVars: Record<string, Array<any>> = {};
+export function serializeArg(value: any): SerializedWebGlArg {
+  if (value instanceof Array) {
+    return value.map(serializeArg);
+  } else if (value === null) {
+    return value;
+  } else if (
+    value instanceof Float32Array ||
+    value instanceof Float64Array ||
+    value instanceof Int32Array ||
+    value instanceof Uint32Array ||
+    value instanceof Uint8Array ||
+    value instanceof Uint16Array ||
+    value instanceof Int16Array ||
+    value instanceof Int8Array ||
+    value instanceof Uint8ClampedArray
+  ) {
+    const name = value.constructor.name;
+    return {
+      rr_type: name,
+      args: [Object.values(value)],
+    };
+  } else if (
+    // SharedArrayBuffer disabled on most browsers due to spectre.
+    // More info: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/SharedArrayBuffer/SharedArrayBuffer
+    // value instanceof SharedArrayBuffer ||
+    value instanceof ArrayBuffer
+  ) {
+    const name = value.constructor.name as 'ArrayBuffer';
+    const base64 = encode(value);
+
+    return {
+      rr_type: name,
+      base64,
+    };
+  } else if (value instanceof DataView) {
+    const name = value.constructor.name;
+    return {
+      rr_type: name,
+      args: [serializeArg(value.buffer), value.byteOffset, value.byteLength],
+    };
+  } else if (value instanceof HTMLImageElement) {
+    const name = value.constructor.name;
+    const { src } = value;
+    return {
+      rr_type: name,
+      src,
+    };
+  } else if (value instanceof ImageData) {
+    const name = value.constructor.name;
+    return {
+      rr_type: name,
+      args: [serializeArg(value.data), value.width, value.height],
+    };
+  } else if (
+    value instanceof WebGLActiveInfo ||
+    value instanceof WebGLBuffer ||
+    value instanceof WebGLFramebuffer ||
+    value instanceof WebGLProgram ||
+    value instanceof WebGLRenderbuffer ||
+    value instanceof WebGLShader ||
+    value instanceof WebGLShaderPrecisionFormat ||
+    value instanceof WebGLTexture ||
+    value instanceof WebGLUniformLocation ||
+    value instanceof WebGLVertexArrayObject ||
+    // In Chrome, value won't be an instanceof WebGLVertexArrayObject.
+    (value && value.constructor.name == 'WebGLVertexArrayObjectOES') ||
+    typeof value === 'object'
+  ) {
+    const name = value.constructor.name;
+    const list = webGLVars[name] || (webGLVars[name] = []);
+    let index = list.indexOf(value);
+
+    if (index === -1) {
+      index = list.length;
+      list.push(value);
+    }
+
+    return {
+      rr_type: name,
+      index,
+    };
+  }
+
+  return value;
+}
+
+export const serializeArgs = (args: Array<any>) => {
+  return [...args].map(serializeArg);
+};
+
+export const saveWebGLVar = (value: any): number | void => {
+  if (
+    !(
+      value instanceof WebGLActiveInfo ||
+      value instanceof WebGLBuffer ||
+      value instanceof WebGLFramebuffer ||
+      value instanceof WebGLProgram ||
+      value instanceof WebGLRenderbuffer ||
+      value instanceof WebGLShader ||
+      value instanceof WebGLShaderPrecisionFormat ||
+      value instanceof WebGLTexture ||
+      value instanceof WebGLUniformLocation ||
+      value instanceof WebGLVertexArrayObject ||
+      // In Chrome, value won't be an instanceof WebGLVertexArrayObject.
+      (value && value.constructor.name == 'WebGLVertexArrayObjectOES') ||
+      typeof value === 'object'
+    )
+  )
+    return;
+
+  const name = value.constructor.name;
+  const list = webGLVars[name] || (webGLVars[name] = []);
+  let index = list.indexOf(value);
+
+  if (index === -1) {
+    index = list.length;
+    list.push(value);
+  }
+  return index;
+};