From e389741c5739c2b06bca1038b3b5b30633cf3c86 Mon Sep 17 00:00:00 2001 From: Jackson Gardner Date: Tue, 29 Aug 2023 16:18:01 -0700 Subject: [PATCH 1/4] Add an API in `ui_web` to create a `ui.Image` from an `ImageBitmap` --- .../src/engine/canvaskit/canvaskit_api.dart | 23 +++++++++-- .../engine/canvaskit/image_web_codecs.dart | 2 +- .../lib/src/engine/canvaskit/renderer.dart | 12 ++++++ lib/web_ui/lib/src/engine/dom.dart | 41 ++++++++++++++++++- lib/web_ui/lib/src/engine/html/renderer.dart | 28 +++++++++++++ lib/web_ui/lib/src/engine/renderer.dart | 10 ++--- .../lib/src/engine/safe_browser_api.dart | 6 +++ .../src/engine/skwasm/skwasm_impl/codecs.dart | 6 ++- .../skwasm/skwasm_impl/raw/raw_image.dart | 19 ++++----- .../engine/skwasm/skwasm_impl/renderer.dart | 10 +++++ .../engine/skwasm/skwasm_stub/renderer.dart | 5 +++ lib/web_ui/lib/ui_web/src/ui_web/images.dart | 18 ++++++++ lib/web_ui/skwasm/image.cpp | 32 ++++++++------- lib/web_ui/skwasm/library_skwasm_support.js | 8 ++-- lib/web_ui/skwasm/skwasm_support.h | 4 +- lib/web_ui/skwasm/surface.cpp | 8 ++-- lib/web_ui/skwasm/surface.h | 14 +++---- lib/web_ui/test/engine/scene_view_test.dart | 14 +------ lib/web_ui/test/ui/image_golden_test.dart | 25 +++++++++++ 19 files changed, 215 insertions(+), 70 deletions(-) diff --git a/lib/web_ui/lib/src/engine/canvaskit/canvaskit_api.dart b/lib/web_ui/lib/src/engine/canvaskit/canvaskit_api.dart index 51470249d2a06..9ae0d6c2463d6 100644 --- a/lib/web_ui/lib/src/engine/canvaskit/canvaskit_api.dart +++ b/lib/web_ui/lib/src/engine/canvaskit/canvaskit_api.dart @@ -219,14 +219,31 @@ extension CanvasKitExtension on CanvasKit { ) => _MakeImage(info, pixels.toJS, bytesPerRow.toJS); @JS('MakeLazyImageFromTextureSource') - external SkImage? _MakeLazyImageFromTextureSource( + external SkImage? _MakeLazyImageFromTextureSource1( JSAny src, SkPartialImageInfo info, ); - SkImage? MakeLazyImageFromTextureSource( + + @JS('MakeLazyImageFromTextureSource') + external SkImage? _MakeLazyImageFromTextureSource2( + JSAny src, + JSNumber zeroSecondArgument, + JSBoolean srcIsPremultiplied, + ); + + SkImage? MakeLazyImageFromTextureSourceWithInfo( Object src, SkPartialImageInfo info, - ) => _MakeLazyImageFromTextureSource(src.toJSAnyShallow, info); + ) => _MakeLazyImageFromTextureSource1(src.toJSAnyShallow, info); + + SkImage? MakeLazyImageFromImageBitmap( + DomImageBitmap imageBitmap, + bool hasPremultipliedAlpha, + ) => _MakeLazyImageFromTextureSource2( + imageBitmap as JSAny, + 0.toJS, + hasPremultipliedAlpha.toJS, + ); } @JS('window.CanvasKitInit') diff --git a/lib/web_ui/lib/src/engine/canvaskit/image_web_codecs.dart b/lib/web_ui/lib/src/engine/canvaskit/image_web_codecs.dart index 05ef367e1b1f9..13563d0b52902 100644 --- a/lib/web_ui/lib/src/engine/canvaskit/image_web_codecs.dart +++ b/lib/web_ui/lib/src/engine/canvaskit/image_web_codecs.dart @@ -60,7 +60,7 @@ class CkBrowserImageDecoder extends BrowserImageDecoder { @override ui.Image generateImageFromVideoFrame(VideoFrame frame) { - final SkImage? skImage = canvasKit.MakeLazyImageFromTextureSource( + final SkImage? skImage = canvasKit.MakeLazyImageFromTextureSourceWithInfo( frame, SkPartialImageInfo( alphaType: canvasKit.AlphaType.Premul, diff --git a/lib/web_ui/lib/src/engine/canvaskit/renderer.dart b/lib/web_ui/lib/src/engine/canvaskit/renderer.dart index b94f75047e607..cc189ab40d795 100644 --- a/lib/web_ui/lib/src/engine/canvaskit/renderer.dart +++ b/lib/web_ui/lib/src/engine/canvaskit/renderer.dart @@ -212,6 +212,18 @@ class CanvasKitRenderer implements Renderer { ui_web.ImageCodecChunkCallback? chunkCallback }) => skiaInstantiateWebImageCodec(uri.toString(), chunkCallback); + @override + ui.Image createImageFromImageBitmap(DomImageBitmap imageBitmap) { + final SkImage? skImage = canvasKit.MakeLazyImageFromImageBitmap( + imageBitmap, + true + ); + if (skImage == null) { + throw Exception('Failed to convert image bitmap to an SkImage.'); + } + return CkImage(skImage); + } + @override void decodeImageFromPixels( Uint8List pixels, diff --git a/lib/web_ui/lib/src/engine/dom.dart b/lib/web_ui/lib/src/engine/dom.dart index 4a5ac6e540dea..d9ee5c4fbd85e 100644 --- a/lib/web_ui/lib/src/engine/dom.dart +++ b/lib/web_ui/lib/src/engine/dom.dart @@ -1433,6 +1433,33 @@ extension DomImageBitmapExtension on DomImageBitmap { external void close(); } + +@JS('createImageBitmap') +external JSPromise _createImageBitmap1( + JSAny source, +); +@JS('createImageBitmap') +external JSPromise _createImageBitmap2( + JSAny source, + JSNumber x, + JSNumber y, + JSNumber width, + JSNumber height, +); +JSPromise createImageBitmap(JSAny source, [({int x, int y, int width, int height})? bounds]) { + if (bounds != null) { + return _createImageBitmap2( + source, + bounds.x.toJS, + bounds.y.toJS, + bounds.width.toJS, + bounds.height.toJS + ); + } else { + return _createImageBitmap1(source); + } +} + @JS() @staticInterop class DomCanvasPattern {} @@ -2264,14 +2291,24 @@ extension DomURLExtension on DomURL { @staticInterop class DomBlob { external factory DomBlob(JSArray parts); + + external factory DomBlob.withOptions(JSArray parts, JSAny options); } extension DomBlobExtension on DomBlob { external JSPromise arrayBuffer(); } -DomBlob createDomBlob(List parts) => - DomBlob(parts.toJSAnyShallow as JSArray); +DomBlob createDomBlob(List parts, [Map? options]) { + if (options == null) { + return DomBlob(parts.toJSAnyShallow as JSArray); + } else { + return DomBlob.withOptions( + parts.toJSAnyShallow as JSArray, + options.toJSAnyDeep + ); + } +} typedef DomMutationCallback = void Function( JSArray mutation, DomMutationObserver observer); diff --git a/lib/web_ui/lib/src/engine/html/renderer.dart b/lib/web_ui/lib/src/engine/html/renderer.dart index 5084a202d4de0..9aa45c5e08a1a 100644 --- a/lib/web_ui/lib/src/engine/html/renderer.dart +++ b/lib/web_ui/lib/src/engine/html/renderer.dart @@ -3,6 +3,7 @@ // found in the LICENSE file. import 'dart:async'; +import 'dart:js_interop'; import 'dart:math' as math; import 'dart:typed_data'; @@ -361,4 +362,31 @@ class HtmlRenderer implements Renderer { baseline: baseline, lineNumber: lineNumber ); + + @override + Future createImageFromImageBitmap(DomImageBitmap imageSource) async { + final int width = imageSource.width.toDartInt; + final int height = imageSource.height.toDartInt; + final OffScreenCanvas canvas = OffScreenCanvas(width, height); + final DomCanvasRenderingContextBitmapRenderer context = canvas.getBitmapRendererContext()!; + context.transferFromImageBitmap(imageSource); + final DomHTMLImageElement imageElement = createDomHTMLImageElement(); + late final DomEventListener loadListener; + late final DomEventListener errorListener; + final Completer completer = Completer(); + loadListener = createDomEventListener((DomEvent event) { + completer.complete(HtmlImage(imageElement, width, height)); + imageElement.removeEventListener('load', loadListener); + imageElement.removeEventListener('error', errorListener); + }); + errorListener = createDomEventListener((DomEvent event) { + completer.completeError(Exception('Failed to create image from image bitmap.')); + imageElement.removeEventListener('load', loadListener); + imageElement.removeEventListener('error', errorListener); + }); + imageElement.addEventListener('load', loadListener); + imageElement.addEventListener('error', errorListener); + imageElement.src = await canvas.toDataUrl(); + return completer.future; + } } diff --git a/lib/web_ui/lib/src/engine/renderer.dart b/lib/web_ui/lib/src/engine/renderer.dart index c6a84ab959600..0f4982de62eba 100644 --- a/lib/web_ui/lib/src/engine/renderer.dart +++ b/lib/web_ui/lib/src/engine/renderer.dart @@ -6,17 +6,11 @@ import 'dart:async'; import 'dart:math' as math; import 'dart:typed_data'; +import 'package:ui/src/engine.dart'; import 'package:ui/src/engine/skwasm/skwasm_stub.dart' if (dart.library.ffi) 'package:ui/src/engine/skwasm/skwasm_impl.dart'; import 'package:ui/ui.dart' as ui; import 'package:ui/ui_web/src/ui_web.dart' as ui_web; -import 'browser_detection.dart'; -import 'canvaskit/renderer.dart'; -import 'configuration.dart'; -import 'embedder.dart'; -import 'fonts.dart'; -import 'html/renderer.dart'; - final Renderer _renderer = Renderer._internal(); Renderer get renderer => _renderer; @@ -134,6 +128,8 @@ abstract class Renderer { ui_web.ImageCodecChunkCallback? chunkCallback, }); + FutureOr createImageFromImageBitmap(DomImageBitmap imageSource); + void decodeImageFromPixels( Uint8List pixels, int width, diff --git a/lib/web_ui/lib/src/engine/safe_browser_api.dart b/lib/web_ui/lib/src/engine/safe_browser_api.dart index 1335ad866ef48..da4ac66a09019 100644 --- a/lib/web_ui/lib/src/engine/safe_browser_api.dart +++ b/lib/web_ui/lib/src/engine/safe_browser_api.dart @@ -1003,6 +1003,12 @@ class OffScreenCanvas { : canvasElement!.getContext('2d'); } + DomCanvasRenderingContextBitmapRenderer? getBitmapRendererContext() { + return (offScreenCanvas != null + ? offScreenCanvas!.getContext('bitmaprenderer') + : canvasElement!.getContext('bitmaprenderer')) as DomCanvasRenderingContextBitmapRenderer?; + } + /// Feature detection for transferToImageBitmap on OffscreenCanvas. bool get transferToImageBitmapSupported => js_util.hasProperty(offScreenCanvas!, 'transferToImageBitmap'); diff --git a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/codecs.dart b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/codecs.dart index 7c1a91cbd29c7..1f767354b8355 100644 --- a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/codecs.dart +++ b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/codecs.dart @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +import 'dart:js_interop'; + import 'package:ui/src/engine.dart'; import 'package:ui/src/engine/skwasm/skwasm_impl.dart'; import 'package:ui/ui.dart' as ui; @@ -18,8 +20,8 @@ class SkwasmImageDecoder extends BrowserImageDecoder { final int width = frame.codedWidth.toInt(); final int height = frame.codedHeight.toInt(); final SkwasmSurface surface = (renderer as SkwasmRenderer).surface; - return SkwasmImage(imageCreateFromVideoFrame( - frame, + return SkwasmImage(imageCreateFromTextureSource( + frame as JSAny, width, height, surface.handle, diff --git a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/raw_image.dart b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/raw_image.dart index fb16ad086f9f1..8b6c968e9375c 100644 --- a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/raw_image.dart +++ b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/raw_image.dart @@ -8,7 +8,6 @@ library skwasm_impl; import 'dart:ffi'; import 'dart:js_interop'; -import 'package:ui/src/engine.dart'; import 'package:ui/src/engine/skwasm/skwasm_impl.dart'; final class RawImage extends Opaque {} @@ -47,9 +46,9 @@ external ImageHandle imageCreateFromPixels( // Int, // Int, // SurfaceHandle, -// )>(symbol: 'image_createFromVideoFrame', isLeaf: true) -// external ImageHandle imageCreateFromVideoFrame( -// JSAny videoFrame, +// )>(symbol: 'image_createFromTextureSource', isLeaf: true) +// external ImageHandle imageCreateFromTextureSource( +// JSAny textureSource, // int width, // int height, // SurfaceHandle handle, @@ -59,21 +58,21 @@ external ImageHandle imageCreateFromPixels( // annotations currently. For now, we can use JS interop to expose this function // instead. extension SkwasmImageExtension on SkwasmInstance { - @JS('wasmExports.image_createFromVideoFrame') - external JSNumber imageCreateFromVideoFrame( - VideoFrame frame, + @JS('wasmExports.image_createFromTextureSource') + external JSNumber imageCreateFromTextureSource( + JSAny textureSource, JSNumber width, JSNumber height, JSNumber surfaceHandle, ); } -ImageHandle imageCreateFromVideoFrame( - VideoFrame frame, +ImageHandle imageCreateFromTextureSource( + JSAny frame, int width, int height, SurfaceHandle handle ) => ImageHandle.fromAddress( - skwasmInstance.imageCreateFromVideoFrame( + skwasmInstance.imageCreateFromTextureSource( frame, width.toJS, height.toJS, diff --git a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/renderer.dart b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/renderer.dart index ed0a2481d2f6c..5b2636d8a8b7f 100644 --- a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/renderer.dart +++ b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/renderer.dart @@ -448,6 +448,16 @@ class SkwasmRenderer implements Renderer { baseline: baseline, lineNumber: lineNumber ); + + @override + ui.Image createImageFromImageBitmap(DomImageBitmap imageSource) { + return SkwasmImage(imageCreateFromTextureSource( + imageSource as JSAny, + imageSource.width.toDartInt, + imageSource.height.toDartInt, + surface.handle, + )); + } } class SkwasmPictureRenderer implements PictureRenderer { diff --git a/lib/web_ui/lib/src/engine/skwasm/skwasm_stub/renderer.dart b/lib/web_ui/lib/src/engine/skwasm/skwasm_stub/renderer.dart index 314cf0b616f95..3964590b3e70a 100644 --- a/lib/web_ui/lib/src/engine/skwasm/skwasm_stub/renderer.dart +++ b/lib/web_ui/lib/src/engine/skwasm/skwasm_stub/renderer.dart @@ -189,4 +189,9 @@ class SkwasmRenderer implements Renderer { required double baseline, required int lineNumber }) => throw UnimplementedError('Skwasm not implemented on this platform.'); + + @override + ui.Image createImageFromImageBitmap(DomImageBitmap imageSource) { + throw UnimplementedError('Skwasm not implemented on this platform.'); + } } diff --git a/lib/web_ui/lib/ui_web/src/ui_web/images.dart b/lib/web_ui/lib/ui_web/src/ui_web/images.dart index d8e4de718111d..342057bf31c95 100644 --- a/lib/web_ui/lib/ui_web/src/ui_web/images.dart +++ b/lib/web_ui/lib/ui_web/src/ui_web/images.dart @@ -2,6 +2,9 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +import 'dart:async'; +import 'dart:js_interop'; + import 'package:ui/src/engine.dart'; import 'package:ui/ui.dart' as ui; @@ -25,3 +28,18 @@ Future createImageCodecFromUrl( chunkCallback: chunkCallback, ); } + +/// Creates a [ui.Image] from an ImageBitmap object. +/// The contents of the ImageBitmap must have a premultiplied alpha. +/// The engine will take ownership of the ImageBitmap object and consume its +/// contents. +/// +/// See https://developer.mozilla.org/en-US/docs/Web/API/ImageBitmap +FutureOr createImageFromImageBitmap(JSAny imageSource) { + if (!domInstanceOfString(imageSource, 'ImageBitmap')) { + throw Exception('Image source $imageSource is not an ImageBitmap!'); + } + return renderer.createImageFromImageBitmap( + imageSource as DomImageBitmap, + ); +} diff --git a/lib/web_ui/skwasm/image.cpp b/lib/web_ui/skwasm/image.cpp index 70553cfe98a9a..74df647625600 100644 --- a/lib/web_ui/skwasm/image.cpp +++ b/lib/web_ui/skwasm/image.cpp @@ -79,20 +79,22 @@ class ExternalWebGLTexture : public GrExternalTexture { }; } // namespace -class VideoFrameImageGenerator : public GrExternalTextureGenerator { +class TextureSourceImageGenerator : public GrExternalTextureGenerator { public: - VideoFrameImageGenerator(SkImageInfo ii, - SkwasmObject videoFrame, - Skwasm::Surface* surface) + TextureSourceImageGenerator(SkImageInfo ii, + SkwasmObject textureSource, + Skwasm::Surface* surface) : GrExternalTextureGenerator(ii), - _videoFrameWrapper(surface->createVideoFrameWrapper(videoFrame)) {} + _textureSourceWrapper( + surface->createTextureSourceWrapper(textureSource)) {} std::unique_ptr generateExternalTexture( GrRecordingContext* context, GrMipMapped mipmapped) override { GrGLTextureInfo glInfo; - glInfo.fID = skwasm_createGlTextureFromVideoFrame( - _videoFrameWrapper->getVideoFrame(), fInfo.width(), fInfo.height()); + glInfo.fID = skwasm_createGlTextureFromTextureSource( + _textureSourceWrapper->getTextureSource(), fInfo.width(), + fInfo.height()); glInfo.fFormat = GL_RGBA8_OES; glInfo.fTarget = GL_TEXTURE_2D; @@ -103,7 +105,7 @@ class VideoFrameImageGenerator : public GrExternalTextureGenerator { } private: - std::unique_ptr _videoFrameWrapper; + std::unique_ptr _textureSourceWrapper; }; SKWASM_EXPORT SkImage* image_createFromPicture(SkPicture* picture, @@ -129,17 +131,17 @@ SKWASM_EXPORT SkImage* image_createFromPixels(SkData* data, .release(); } -SKWASM_EXPORT SkImage* image_createFromVideoFrame(SkwasmObject videoFrame, - int width, - int height, - Skwasm::Surface* surface) { +SKWASM_EXPORT SkImage* image_createFromTextureSource(SkwasmObject textureSource, + int width, + int height, + Skwasm::Surface* surface) { return SkImages::DeferredFromTextureGenerator( - std::unique_ptr( - new VideoFrameImageGenerator( + std::unique_ptr( + new TextureSourceImageGenerator( SkImageInfo::Make(width, height, SkColorType::kRGBA_8888_SkColorType, SkAlphaType::kPremul_SkAlphaType), - videoFrame, surface))) + textureSource, surface))) .release(); } diff --git a/lib/web_ui/skwasm/library_skwasm_support.js b/lib/web_ui/skwasm/library_skwasm_support.js index 87dfff43fdb94..d1c3459d10809 100644 --- a/lib/web_ui/skwasm/library_skwasm_support.js +++ b/lib/web_ui/skwasm/library_skwasm_support.js @@ -82,13 +82,13 @@ mergeInto(LibraryManager.library, { imageBitmap, }); }; - _skwasm_createGlTextureFromVideoFrame = function(videoFrame, width, height) { + _skwasm_createGlTextureFromTextureSource = function(textureSource, width, height) { const glCtx = GL.currentContext.GLctx; const newTexture = glCtx.createTexture(); glCtx.bindTexture(glCtx.TEXTURE_2D, newTexture); glCtx.pixelStorei(glCtx.UNPACK_PREMULTIPLY_ALPHA_WEBGL, true); - glCtx.texImage2D(glCtx.TEXTURE_2D, 0, glCtx.RGBA, width, height, 0, glCtx.RGBA, glCtx.UNSIGNED_BYTE, videoFrame); + glCtx.texImage2D(glCtx.TEXTURE_2D, 0, glCtx.RGBA, width, height, 0, glCtx.RGBA, glCtx.UNSIGNED_BYTE, textureSource); glCtx.pixelStorei(glCtx.UNPACK_PREMULTIPLY_ALPHA_WEBGL, false); glCtx.bindTexture(glCtx.TEXTURE_2D, null); @@ -118,7 +118,7 @@ mergeInto(LibraryManager.library, { skwasm_resizeCanvas__deps: ['$skwasm_support_setup'], skwasm_captureImageBitmap: function () {}, skwasm_captureImageBitmap__deps: ['$skwasm_support_setup'], - skwasm_createGlTextureFromVideoFrame: function () {}, - skwasm_createGlTextureFromVideoFrame__deps: ['$skwasm_support_setup'], + skwasm_createGlTextureFromTextureSource: function () {}, + skwasm_createGlTextureFromTextureSource__deps: ['$skwasm_support_setup'], }); \ No newline at end of file diff --git a/lib/web_ui/skwasm/skwasm_support.h b/lib/web_ui/skwasm/skwasm_support.h index 9547bc29e7a89..21c790d6507eb 100644 --- a/lib/web_ui/skwasm/skwasm_support.h +++ b/lib/web_ui/skwasm/skwasm_support.h @@ -26,8 +26,8 @@ extern void skwasm_captureImageBitmap(Skwasm::Surface* surfaceHandle, uint32_t bitmapId, int width, int height); -extern unsigned int skwasm_createGlTextureFromVideoFrame( - SkwasmObject videoFrame, +extern unsigned int skwasm_createGlTextureFromTextureSource( + SkwasmObject textureSource, int width, int height); } diff --git a/lib/web_ui/skwasm/surface.cpp b/lib/web_ui/skwasm/surface.cpp index 160037e4313b3..8497b18ef55a3 100644 --- a/lib/web_ui/skwasm/surface.cpp +++ b/lib/web_ui/skwasm/surface.cpp @@ -60,10 +60,10 @@ uint32_t Surface::rasterizeImage(SkImage* image, ImageByteFormat format) { return callbackId; } -std::unique_ptr Surface::createVideoFrameWrapper( - SkwasmObject videoFrame) { - return std::unique_ptr( - new VideoFrameWrapper(_thread, videoFrame)); +std::unique_ptr Surface::createTextureSourceWrapper( + SkwasmObject textureSource) { + return std::unique_ptr( + new TextureSourceWrapper(_thread, textureSource)); } // Main thread only diff --git a/lib/web_ui/skwasm/surface.h b/lib/web_ui/skwasm/surface.h index 22cc57be9876c..c7cd137a00fb5 100644 --- a/lib/web_ui/skwasm/surface.h +++ b/lib/web_ui/skwasm/surface.h @@ -33,18 +33,18 @@ enum class ImageByteFormat { png, }; -class VideoFrameWrapper { +class TextureSourceWrapper { public: - VideoFrameWrapper(unsigned long threadId, SkwasmObject videoFrame) + TextureSourceWrapper(unsigned long threadId, SkwasmObject textureSource) : _rasterThreadId(threadId) { - skwasm_setAssociatedObjectOnThread(_rasterThreadId, this, videoFrame); + skwasm_setAssociatedObjectOnThread(_rasterThreadId, this, textureSource); } - ~VideoFrameWrapper() { + ~TextureSourceWrapper() { skwasm_disposeAssociatedObjectOnThread(_rasterThreadId, this); } - SkwasmObject getVideoFrame() { return skwasm_getAssociatedObject(this); } + SkwasmObject getTextureSource() { return skwasm_getAssociatedObject(this); } private: unsigned long _rasterThreadId; @@ -67,8 +67,8 @@ class Surface { void onRenderComplete(uint32_t callbackId, SkwasmObject imageBitmap); // Any thread - std::unique_ptr createVideoFrameWrapper( - SkwasmObject videoFrame); + std::unique_ptr createTextureSourceWrapper( + SkwasmObject textureSource); private: void _runWorker(); diff --git a/lib/web_ui/test/engine/scene_view_test.dart b/lib/web_ui/test/engine/scene_view_test.dart index 5551ee6a3b47e..0b475d8842824 100644 --- a/lib/web_ui/test/engine/scene_view_test.dart +++ b/lib/web_ui/test/engine/scene_view_test.dart @@ -14,15 +14,6 @@ import 'package:ui/ui_web/src/ui_web.dart'; import 'scene_builder_utils.dart'; -@JS('createImageBitmap') -external JSPromise createImageBitmap( - JSAny source, - JSNumber x, - JSNumber y, - JSNumber width, - JSNumber height -); - void main() { internalBootstrapBrowserTest(() => testMain); } @@ -37,10 +28,7 @@ class StubPictureRenderer implements PictureRenderer { final ui.Rect cullRect = picture.cullRect; final DomImageBitmap bitmap = (await createImageBitmap( scratchCanvasElement as JSAny, - 0.toJS, - 0.toJS, - cullRect.width.toJS, - cullRect.height.toJS + (x: 0, y: 0, width: cullRect.width.toInt(), height: cullRect.height.toInt()) ).toDart)! as DomImageBitmap; return bitmap; } diff --git a/lib/web_ui/test/ui/image_golden_test.dart b/lib/web_ui/test/ui/image_golden_test.dart index aa877b97b654e..09040e0c0dd99 100644 --- a/lib/web_ui/test/ui/image_golden_test.dart +++ b/lib/web_ui/test/ui/image_golden_test.dart @@ -4,6 +4,7 @@ import 'dart:async'; import 'dart:convert'; +import 'dart:js_interop'; import 'dart:math'; import 'dart:typed_data'; @@ -294,6 +295,30 @@ Future testMain() async { return info.image; }); + emitImageTests('svg_image_bitmap', () async { + final DomBlob svgBlob = createDomBlob([ +''' + + + +''' + ], {'type': 'image/svg+xml'}); + final String url = domWindow.URL.createObjectURL(svgBlob); + final DomHTMLImageElement image = createDomHTMLImageElement(); + final Completer completer = Completer(); + late final DomEventListener loadListener; + loadListener = createDomEventListener((DomEvent event) { + completer.complete(); + image.removeEventListener('load', loadListener); + }); + image.addEventListener('load', loadListener); + image.src = url; + await completer.future; + + final DomImageBitmap bitmap = (await createImageBitmap(image as JSAny).toDart)! as DomImageBitmap; + return renderer.createImageFromImageBitmap(bitmap); + }); + emitImageTests('codec_list_resized', () async { final ByteBuffer data = await httpFetchByteBuffer('/test_images/mandrill_128.png'); final ui.Codec codec = await renderer.instantiateImageCodec( From a611f13914f9b7430b40e57ea1a7b97b16744cb6 Mon Sep 17 00:00:00 2001 From: Jackson Gardner Date: Tue, 29 Aug 2023 20:14:03 -0700 Subject: [PATCH 2/4] Skip this test on Firefox due to no WebGL on headless mode. --- lib/web_ui/test/ui/image_golden_test.dart | 32 +++++++++++++---------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/lib/web_ui/test/ui/image_golden_test.dart b/lib/web_ui/test/ui/image_golden_test.dart index 09040e0c0dd99..0dd448f831947 100644 --- a/lib/web_ui/test/ui/image_golden_test.dart +++ b/lib/web_ui/test/ui/image_golden_test.dart @@ -242,21 +242,25 @@ Future testMain() async { return data; } - emitImageTests('decodeImageFromPixels_unscaled', () { - final Uint8List pixels = generatePixelData(150, 150, (double x, double y) { - final double r = sqrt(x * x + y * y); - final double theta = atan2(x, y); - return ui.Color.fromRGBO( - (255 * (sin(r * 10.0) + 1.0) / 2.0).round(), - (255 * (sin(theta * 10.0) + 1.0) / 2.0).round(), - 0, - 1, - ); + // This API doesn't work in headless Firefox due to requiring WebGL + // See https://github.com/flutter/flutter/issues/109265 + if (!isFirefox) { + emitImageTests('decodeImageFromPixels_unscaled', () { + final Uint8List pixels = generatePixelData(150, 150, (double x, double y) { + final double r = sqrt(x * x + y * y); + final double theta = atan2(x, y); + return ui.Color.fromRGBO( + (255 * (sin(r * 10.0) + 1.0) / 2.0).round(), + (255 * (sin(theta * 10.0) + 1.0) / 2.0).round(), + 0, + 1, + ); + }); + final Completer completer = Completer(); + ui.decodeImageFromPixels(pixels, 150, 150, ui.PixelFormat.rgba8888, completer.complete); + return completer.future; }); - final Completer completer = Completer(); - ui.decodeImageFromPixels(pixels, 150, 150, ui.PixelFormat.rgba8888, completer.complete); - return completer.future; - }); + } // https://github.com/flutter/flutter/issues/126603 if (!isHtml) { From 4e600231eba916c805ba6705bcbe62cee4fe88e2 Mon Sep 17 00:00:00 2001 From: Jackson Gardner Date: Tue, 29 Aug 2023 20:48:29 -0700 Subject: [PATCH 3/4] Whoops, accidentally skipped the wrong test. --- lib/web_ui/test/ui/image_golden_test.dart | 80 +++++++++++------------ 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/lib/web_ui/test/ui/image_golden_test.dart b/lib/web_ui/test/ui/image_golden_test.dart index 0dd448f831947..fccb86c8b2951 100644 --- a/lib/web_ui/test/ui/image_golden_test.dart +++ b/lib/web_ui/test/ui/image_golden_test.dart @@ -242,25 +242,21 @@ Future testMain() async { return data; } - // This API doesn't work in headless Firefox due to requiring WebGL - // See https://github.com/flutter/flutter/issues/109265 - if (!isFirefox) { - emitImageTests('decodeImageFromPixels_unscaled', () { - final Uint8List pixels = generatePixelData(150, 150, (double x, double y) { - final double r = sqrt(x * x + y * y); - final double theta = atan2(x, y); - return ui.Color.fromRGBO( - (255 * (sin(r * 10.0) + 1.0) / 2.0).round(), - (255 * (sin(theta * 10.0) + 1.0) / 2.0).round(), - 0, - 1, - ); - }); - final Completer completer = Completer(); - ui.decodeImageFromPixels(pixels, 150, 150, ui.PixelFormat.rgba8888, completer.complete); - return completer.future; + emitImageTests('decodeImageFromPixels_unscaled', () { + final Uint8List pixels = generatePixelData(150, 150, (double x, double y) { + final double r = sqrt(x * x + y * y); + final double theta = atan2(x, y); + return ui.Color.fromRGBO( + (255 * (sin(r * 10.0) + 1.0) / 2.0).round(), + (255 * (sin(theta * 10.0) + 1.0) / 2.0).round(), + 0, + 1, + ); }); - } + final Completer completer = Completer(); + ui.decodeImageFromPixels(pixels, 150, 150, ui.PixelFormat.rgba8888, completer.complete); + return completer.future; + }); // https://github.com/flutter/flutter/issues/126603 if (!isHtml) { @@ -299,29 +295,33 @@ Future testMain() async { return info.image; }); - emitImageTests('svg_image_bitmap', () async { - final DomBlob svgBlob = createDomBlob([ -''' - - - -''' - ], {'type': 'image/svg+xml'}); - final String url = domWindow.URL.createObjectURL(svgBlob); - final DomHTMLImageElement image = createDomHTMLImageElement(); - final Completer completer = Completer(); - late final DomEventListener loadListener; - loadListener = createDomEventListener((DomEvent event) { - completer.complete(); - image.removeEventListener('load', loadListener); - }); - image.addEventListener('load', loadListener); - image.src = url; - await completer.future; + // This API doesn't work in headless Firefox due to requiring WebGL + // See https://github.com/flutter/flutter/issues/109265 + if (!isFirefox) { + emitImageTests('svg_image_bitmap', () async { + final DomBlob svgBlob = createDomBlob([ + ''' + + + + ''' + ], {'type': 'image/svg+xml'}); + final String url = domWindow.URL.createObjectURL(svgBlob); + final DomHTMLImageElement image = createDomHTMLImageElement(); + final Completer completer = Completer(); + late final DomEventListener loadListener; + loadListener = createDomEventListener((DomEvent event) { + completer.complete(); + image.removeEventListener('load', loadListener); + }); + image.addEventListener('load', loadListener); + image.src = url; + await completer.future; - final DomImageBitmap bitmap = (await createImageBitmap(image as JSAny).toDart)! as DomImageBitmap; - return renderer.createImageFromImageBitmap(bitmap); - }); + final DomImageBitmap bitmap = (await createImageBitmap(image as JSAny).toDart)! as DomImageBitmap; + return renderer.createImageFromImageBitmap(bitmap); + }); + } emitImageTests('codec_list_resized', () async { final ByteBuffer data = await httpFetchByteBuffer('/test_images/mandrill_128.png'); From 9d4206dfbd5efc2a773608218e743b63ad0645e2 Mon Sep 17 00:00:00 2001 From: Jackson Gardner Date: Wed, 30 Aug 2023 12:03:19 -0700 Subject: [PATCH 4/4] Addressed Yegor's comments. --- lib/web_ui/lib/src/engine/canvaskit/canvaskit_api.dart | 8 ++++---- lib/web_ui/lib/ui_web/src/ui_web/images.dart | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/web_ui/lib/src/engine/canvaskit/canvaskit_api.dart b/lib/web_ui/lib/src/engine/canvaskit/canvaskit_api.dart index 9ae0d6c2463d6..22c94573fdf6f 100644 --- a/lib/web_ui/lib/src/engine/canvaskit/canvaskit_api.dart +++ b/lib/web_ui/lib/src/engine/canvaskit/canvaskit_api.dart @@ -219,13 +219,13 @@ extension CanvasKitExtension on CanvasKit { ) => _MakeImage(info, pixels.toJS, bytesPerRow.toJS); @JS('MakeLazyImageFromTextureSource') - external SkImage? _MakeLazyImageFromTextureSource1( + external SkImage? _MakeLazyImageFromTextureSource2( JSAny src, SkPartialImageInfo info, ); @JS('MakeLazyImageFromTextureSource') - external SkImage? _MakeLazyImageFromTextureSource2( + external SkImage? _MakeLazyImageFromTextureSource3( JSAny src, JSNumber zeroSecondArgument, JSBoolean srcIsPremultiplied, @@ -234,12 +234,12 @@ extension CanvasKitExtension on CanvasKit { SkImage? MakeLazyImageFromTextureSourceWithInfo( Object src, SkPartialImageInfo info, - ) => _MakeLazyImageFromTextureSource1(src.toJSAnyShallow, info); + ) => _MakeLazyImageFromTextureSource2(src.toJSAnyShallow, info); SkImage? MakeLazyImageFromImageBitmap( DomImageBitmap imageBitmap, bool hasPremultipliedAlpha, - ) => _MakeLazyImageFromTextureSource2( + ) => _MakeLazyImageFromTextureSource3( imageBitmap as JSAny, 0.toJS, hasPremultipliedAlpha.toJS, diff --git a/lib/web_ui/lib/ui_web/src/ui_web/images.dart b/lib/web_ui/lib/ui_web/src/ui_web/images.dart index 342057bf31c95..5e297337f2c49 100644 --- a/lib/web_ui/lib/ui_web/src/ui_web/images.dart +++ b/lib/web_ui/lib/ui_web/src/ui_web/images.dart @@ -30,6 +30,7 @@ Future createImageCodecFromUrl( } /// Creates a [ui.Image] from an ImageBitmap object. +/// /// The contents of the ImageBitmap must have a premultiplied alpha. /// The engine will take ownership of the ImageBitmap object and consume its /// contents. @@ -37,7 +38,7 @@ Future createImageCodecFromUrl( /// See https://developer.mozilla.org/en-US/docs/Web/API/ImageBitmap FutureOr createImageFromImageBitmap(JSAny imageSource) { if (!domInstanceOfString(imageSource, 'ImageBitmap')) { - throw Exception('Image source $imageSource is not an ImageBitmap!'); + throw ArgumentError('Image source $imageSource is not an ImageBitmap.', 'imageSource'); } return renderer.createImageFromImageBitmap( imageSource as DomImageBitmap,