Skip to content

fix recording of webgl2 canvases #106

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Jul 21, 2023
Merged

fix recording of webgl2 canvases #106

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
51791d7
fix recording of webgl2 canvases
Vadman97 Jul 13, 2023
b04ddfc
improve canvas worker logging
Vadman97 Jul 14, 2023
36da0d2
store canvas context only on successfull getContext
Vadman97 Jul 14, 2023
eaf469b
WIP try to fix transparent bitmaps
Vadman97 Jul 14, 2023
b539d79
add setting
Vadman97 Jul 19, 2023
2cc726b
remove debug toBlob
Vadman97 Jul 19, 2023
137e02c
add snapshot delay setting
Vadman97 Jul 19, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions packages/rrweb/src/record/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -320,6 +320,8 @@ function record<T = eventWithTime>(
blockSelector,
mirror,
sampling: sampling?.canvas?.fps,
clearWebGLBuffer: sampling?.canvas?.clearWebGLBuffer,
initialSnapshotDelay: sampling?.canvas?.initialSnapshotDelay,
dataURLOptions,
resizeFactor: sampling?.canvas?.resizeFactor,
maxSnapshotDimension: sampling?.canvas?.maxSnapshotDimension,
Expand Down
303 changes: 170 additions & 133 deletions packages/rrweb/src/record/observers/canvas/canvas-manager.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,8 @@ export class CanvasManager {
blockSelector: string | null;
mirror: Mirror;
sampling?: 'all' | number;
clearWebGLBuffer?: boolean;
initialSnapshotDelay?: number;
dataURLOptions: DataURLOptions;
resizeFactor?: number;
maxSnapshotDimension?: number;
Expand All @@ -87,6 +89,8 @@ export class CanvasManager {
blockSelector,
recordCanvas,
recordVideos,
clearWebGLBuffer,
initialSnapshotDelay,
dataURLOptions,
} = options;
this.mutationCb = options.mutationCb;
Expand All @@ -103,6 +107,8 @@ export class CanvasManager {
blockClass,
blockSelector,
{
clearWebGLBuffer,
initialSnapshotDelay,
dataURLOptions,
},
options.resizeFactor,
Expand All @@ -111,15 +117,19 @@ export class CanvasManager {
}

private debug(
element: HTMLCanvasElement | HTMLVideoElement,
element: HTMLCanvasElement | HTMLVideoElement | null,
...args: Parameters<typeof console.log>
) {
if (!this.logger) return;
let prefix = `[highlight-${element.tagName.toLowerCase()}]`;
if (element.tagName.toLowerCase() === 'canvas') {
prefix += ` [ctx:${(element as ICanvas).__context}]`;
const id = this.mirror.getId(element);
let prefix = '[highlight-canvas-manager]';
if (element) {
prefix = `[highlight-${element.tagName.toLowerCase()}] [id:${id}]`;
if (element.tagName.toLowerCase() === 'canvas') {
prefix += ` [ctx:${(element as ICanvas).__context}]`;
}
}
this.logger.debug(prefix, element, ...args);
this.logger.debug(prefix, ...args);
}

private processMutation: canvasManagerMutationCallback = (
Expand All @@ -146,6 +156,8 @@ export class CanvasManager {
blockClass: blockClass,
blockSelector: string | null,
options: {
clearWebGLBuffer?: boolean;
initialSnapshotDelay?: number;
dataURLOptions: DataURLOptions;
},
resizeFactor?: number,
Expand All @@ -163,7 +175,12 @@ export class CanvasManager {
const { id } = e.data;
snapshotInProgressMap.set(id, false);

if (!('base64' in e.data)) return;
if (!('base64' in e.data)) {
this.debug(null, 'canvas worker received empty message', {
status: e.data.status,
});
return;
}

const { base64, type, dx, dy, dw, dh } = e.data;
this.mutationCb({
Expand Down Expand Up @@ -235,58 +252,70 @@ export class CanvasManager {
}
lastSnapshotTime = timestamp;

const promises: Promise<void>[] = []
promises.push(...getCanvas().map(async (canvas: HTMLCanvasElement) => {
this.debug(canvas, 'starting snapshotting');
const id = this.mirror.getId(canvas);
if (snapshotInProgressMap.get(id)) {
this.debug(canvas, 'snapshotting already in progress for', id);
return;
}
snapshotInProgressMap.set(id, true);
try {
if (['webgl', 'webgl2'].includes((canvas as ICanvas).__context)) {
// if the canvas hasn't been modified recently,
// its contents won't be in memory and `createImageBitmap`
// will return a transparent imageBitmap

const context = canvas.getContext((canvas as ICanvas).__context) as
| WebGLRenderingContext
| WebGL2RenderingContext
| null;
const promises: Promise<void>[] = [];
promises.push(
...getCanvas().map(async (canvas: HTMLCanvasElement) => {
this.debug(canvas, 'starting snapshotting');
const id = this.mirror.getId(canvas);
if (snapshotInProgressMap.get(id)) {
this.debug(canvas, 'snapshotting already in progress for', id);
return;
}
snapshotInProgressMap.set(id, true);
try {
if (
context?.getContextAttributes()?.preserveDrawingBuffer === false
options.clearWebGLBuffer !== false &&
['webgl', 'webgl2'].includes((canvas as ICanvas).__context)
) {
// Hack to load canvas back into memory so `createImageBitmap` can grab it's contents.
// Context: https://twitter.com/Juice10/status/1499775271758704643
// This hack might change the background color of the canvas in the unlikely event that
// the canvas background was changed but clear was not called directly afterwards.
context?.clear(context.COLOR_BUFFER_BIT);
// if the canvas hasn't been modified recently,
// its contents won't be in memory and `createImageBitmap`
// will return a transparent imageBitmap

const context = canvas.getContext(
(canvas as ICanvas).__context,
) as WebGLRenderingContext | WebGL2RenderingContext | null;
if (
context?.getContextAttributes()?.preserveDrawingBuffer === false
) {
// Hack to load canvas back into memory so `createImageBitmap` can grab it's contents.
// Context: https://twitter.com/Juice10/status/1499775271758704643
// This hack might change the background color of the canvas in the unlikely event that
// the canvas background was changed but clear was not called directly afterwards.
context?.clear(context?.COLOR_BUFFER_BIT);
this.debug(
canvas,
'cleared webgl canvas to load it into memory',
{ attributes: context?.getContextAttributes() },
);
}
}
}
// canvas is not yet ready... this retry on the next sampling iteration.
// we don't want to crash the worker if the canvas is not yet rendered.
if (canvas.width === 0 || canvas.height === 0) {
this.debug(canvas, 'not yet ready', {
width: canvas.width,
height: canvas.height,
// canvas is not yet ready... this retry on the next sampling iteration.
// we don't want to crash the worker by sending an undefined bitmap
// if the canvas is not yet rendered.
if (canvas.width === 0 || canvas.height === 0) {
this.debug(canvas, 'not yet ready', {
width: canvas.width,
height: canvas.height,
});
return;
}
let scale = resizeFactor || 1;
if (maxSnapshotDimension) {
const maxDim = Math.max(canvas.width, canvas.height);
scale = Math.min(scale, maxSnapshotDimension / maxDim);
}
const width = canvas.width * scale;
const height = canvas.height * scale;

const bitmap = await createImageBitmap(canvas, {
resizeWidth: width,
resizeHeight: height,
});
return;
}
let scale = resizeFactor || 1;
if (maxSnapshotDimension) {
const maxDim = Math.max(canvas.width, canvas.height);
scale = Math.min(scale, maxSnapshotDimension / maxDim);
}
const width = canvas.width * scale;
const height = canvas.height * scale;

const bitmap = await createImageBitmap(canvas, {
resizeWidth: width,
resizeHeight: height,
});
this.debug(canvas, 'created image bitmap');
worker.postMessage(
this.debug(canvas, 'created image bitmap', {
width: bitmap.width,
height: bitmap.height,
});
worker.postMessage(
{
id,
bitmap,
Expand All @@ -299,93 +328,101 @@ export class CanvasManager {
dataURLOptions: options.dataURLOptions,
},
[bitmap],
);
this.debug(canvas, 'sent message');
} catch (e) {
this.debug(canvas, 'failed to snapshot', e);
} finally {
snapshotInProgressMap.set(id, false);
}
}))
promises.push(...getVideos().map(async (video: HTMLVideoElement) => {
this.debug(video, 'starting video snapshotting');
const id = this.mirror.getId(video);
if (snapshotInProgressMap.get(id)) {
this.debug(video, 'video snapshotting already in progress for', id);
return;
}
snapshotInProgressMap.set(id, true);
try {
const { width: boxWidth, height: boxHeight } =
video.getBoundingClientRect();
const { actualWidth, actualHeight } = {
actualWidth: video.videoWidth,
actualHeight: video.videoHeight,
};
const maxDim = Math.max(actualWidth, actualHeight);
let scale = resizeFactor || 1;
if (maxSnapshotDimension) {
scale = Math.min(scale, maxSnapshotDimension / maxDim);
);
this.debug(canvas, 'sent message');
} catch (e) {
this.debug(canvas, 'failed to snapshot', e);
} finally {
snapshotInProgressMap.set(id, false);
}
const width = actualWidth * scale;
const height = actualHeight * scale;

const bitmap = await createImageBitmap(video, {
resizeWidth: width,
resizeHeight: height,
});

let outputScale = Math.max(boxWidth, boxHeight) / maxDim;
const outputWidth = actualWidth * outputScale;
const outputHeight = actualHeight * outputScale;
const offsetX = (boxWidth - outputWidth) / 2;
const offsetY = (boxHeight - outputHeight) / 2;
this.debug(video, 'created image bitmap', {
actualWidth,
actualHeight,
boxWidth,
boxHeight,
outputWidth,
outputHeight,
resizeWidth: width,
resizeHeight: height,
scale,
outputScale,
offsetX,
offsetY,
});

worker.postMessage(
{
id,
bitmap,
width,
height,
dx: offsetX,
dy: offsetY,
dw: outputWidth,
dh: outputHeight,
dataURLOptions: options.dataURLOptions,
},
[bitmap],
);
this.debug(video, 'send message');
} catch (e) {
this.debug(video, 'failed to snapshot', e);
} finally {
snapshotInProgressMap.set(id, false);
}
}))
await Promise.all(promises)
}),
);
promises.push(
...getVideos().map(async (video: HTMLVideoElement) => {
this.debug(video, 'starting video snapshotting');
const id = this.mirror.getId(video);
if (snapshotInProgressMap.get(id)) {
this.debug(video, 'video snapshotting already in progress for', id);
return;
}
snapshotInProgressMap.set(id, true);
try {
const { width: boxWidth, height: boxHeight } =
video.getBoundingClientRect();
const { actualWidth, actualHeight } = {
actualWidth: video.videoWidth,
actualHeight: video.videoHeight,
};
const maxDim = Math.max(actualWidth, actualHeight);
let scale = resizeFactor || 1;
if (maxSnapshotDimension) {
scale = Math.min(scale, maxSnapshotDimension / maxDim);
}
const width = actualWidth * scale;
const height = actualHeight * scale;

const bitmap = await createImageBitmap(video, {
resizeWidth: width,
resizeHeight: height,
});

let outputScale = Math.max(boxWidth, boxHeight) / maxDim;
const outputWidth = actualWidth * outputScale;
const outputHeight = actualHeight * outputScale;
const offsetX = (boxWidth - outputWidth) / 2;
const offsetY = (boxHeight - outputHeight) / 2;
this.debug(video, 'created image bitmap', {
actualWidth,
actualHeight,
boxWidth,
boxHeight,
outputWidth,
outputHeight,
resizeWidth: width,
resizeHeight: height,
scale,
outputScale,
offsetX,
offsetY,
});

worker.postMessage(
{
id,
bitmap,
width,
height,
dx: offsetX,
dy: offsetY,
dw: outputWidth,
dh: outputHeight,
dataURLOptions: options.dataURLOptions,
},
[bitmap],
);
this.debug(video, 'send message');
} catch (e) {
this.debug(video, 'failed to snapshot', e);
} finally {
snapshotInProgressMap.set(id, false);
}
}),
);
Promise.all(promises).catch(console.error);

rafId = requestAnimationFrame(takeSnapshots);
};

rafId = requestAnimationFrame(takeSnapshots);
const delay = setTimeout(() => {
rafId = requestAnimationFrame(takeSnapshots);
}, options.initialSnapshotDelay);

this.resetObservers = () => {
canvasContextReset();
cancelAnimationFrame(rafId);
clearTimeout(delay);
if (rafId) {
cancelAnimationFrame(rafId);
}
};
}

Expand Down
Loading