From 42c6da338385b6e0366f3e12ac6bec5d4508fc8b Mon Sep 17 00:00:00 2001 From: Kaushik Iska Date: Wed, 10 Feb 2021 05:45:40 -0800 Subject: [PATCH 1/6] [embedder] Refactor GL texture resolution This moves the texture resolution out of embedder.cc and into embedder external_texture_gl.cc --- shell/platform/embedder/embedder.cc | 48 ++------------ .../embedder/embedder_external_texture_gl.cc | 63 +++++++++++++++++-- .../embedder/embedder_external_texture_gl.h | 13 ++-- 3 files changed, 71 insertions(+), 53 deletions(-) diff --git a/shell/platform/embedder/embedder.cc b/shell/platform/embedder/embedder.cc index 00fd7633a2611..b0cc085921c29 100644 --- a/shell/platform/embedder/embedder.cc +++ b/shell/platform/embedder/embedder.cc @@ -1146,51 +1146,13 @@ FlutterEngineResult FlutterEngineInitialize(size_t version, nullptr) != nullptr) { external_texture_callback = [ptr = open_gl_config->gl_external_texture_frame_callback, user_data]( - int64_t texture_identifier, GrDirectContext* context, - const SkISize& size) -> sk_sp { - FlutterOpenGLTexture texture = {}; - - if (!ptr(user_data, texture_identifier, size.width(), size.height(), - &texture)) { - return nullptr; - } - - GrGLTextureInfo gr_texture_info = {texture.target, texture.name, - texture.format}; - - size_t width = size.width(); - size_t height = size.height(); - - if (texture.width != 0 && texture.height != 0) { - width = texture.width; - height = texture.height; - } - - GrBackendTexture gr_backend_texture(width, height, GrMipMapped::kNo, - gr_texture_info); - SkImage::TextureReleaseProc release_proc = texture.destruction_callback; - auto image = SkImage::MakeFromTexture( - context, // context - gr_backend_texture, // texture handle - kTopLeft_GrSurfaceOrigin, // origin - kRGBA_8888_SkColorType, // color type - kPremul_SkAlphaType, // alpha type - nullptr, // colorspace - release_proc, // texture release proc - texture.user_data // texture release context - ); - - if (!image) { - // In case Skia rejects the image, call the release proc so that - // embedders can perform collection of intermediates. - if (release_proc) { - release_proc(texture.user_data); - } - FML_LOG(ERROR) << "Could not create external texture."; + int64_t texture_identifier, size_t width, + size_t height) -> std::unique_ptr { + FlutterOpenGLTexture* texture = new FlutterOpenGLTexture(); + if (!ptr(user_data, texture_identifier, width, height, texture)) { return nullptr; } - - return image; + return std::unique_ptr(texture); }; } } diff --git a/shell/platform/embedder/embedder_external_texture_gl.cc b/shell/platform/embedder/embedder_external_texture_gl.cc index e7839e1ddc703..0b5ed43c78028 100644 --- a/shell/platform/embedder/embedder_external_texture_gl.cc +++ b/shell/platform/embedder/embedder_external_texture_gl.cc @@ -5,6 +5,10 @@ #include "flutter/shell/platform/embedder/embedder_external_texture_gl.h" #include "flutter/fml/logging.h" +#include "third_party/skia/include/core/SkImage.h" +#include "third_party/skia/include/core/SkSize.h" +#include "third_party/skia/include/gpu/GrBackendSurface.h" +#include "third_party/skia/include/gpu/GrDirectContext.h" namespace flutter { @@ -23,11 +27,11 @@ void EmbedderExternalTextureGL::Paint(SkCanvas& canvas, bool freeze, GrDirectContext* context, const SkSamplingOptions& sampling) { - if (auto image = external_texture_callback_( - Id(), // - context, // - SkISize::Make(bounds.width(), bounds.height()) // - )) { + if (auto image = + ResolveTexture(Id(), // + context, // + SkISize::Make(bounds.width(), bounds.height()) // + )) { last_image_ = image; } @@ -40,6 +44,55 @@ void EmbedderExternalTextureGL::Paint(SkCanvas& canvas, } } +sk_sp EmbedderExternalTextureGL::ResolveTexture( + int64_t texture_id, + GrDirectContext* context, + const SkISize& size) { + std::unique_ptr texture = + external_texture_callback_(texture_id, size.width(), size.height()); + + if (!texture) { + return nullptr; + } + + GrGLTextureInfo gr_texture_info = {texture->target, texture->name, + texture->format}; + + size_t width = size.width(); + size_t height = size.height(); + + if (texture->width != 0 && texture->height != 0) { + width = texture->width; + height = texture->height; + } + + GrBackendTexture gr_backend_texture(width, height, GrMipMapped::kNo, + gr_texture_info); + SkImage::TextureReleaseProc release_proc = texture->destruction_callback; + auto image = + SkImage::MakeFromTexture(context, // context + gr_backend_texture, // texture handle + kTopLeft_GrSurfaceOrigin, // origin + kRGBA_8888_SkColorType, // color type + kPremul_SkAlphaType, // alpha type + nullptr, // colorspace + release_proc, // texture release proc + texture->user_data // texture release context + ); + + if (!image) { + // In case Skia rejects the image, call the release proc so that + // embedders can perform collection of intermediates. + if (release_proc) { + release_proc(texture->user_data); + } + FML_LOG(ERROR) << "Could not create external texture->"; + return nullptr; + } + + return image; +} + // |flutter::Texture| void EmbedderExternalTextureGL::OnGrContextCreated() {} diff --git a/shell/platform/embedder/embedder_external_texture_gl.h b/shell/platform/embedder/embedder_external_texture_gl.h index 68d96cf4f3bee..f1373405aec98 100644 --- a/shell/platform/embedder/embedder_external_texture_gl.h +++ b/shell/platform/embedder/embedder_external_texture_gl.h @@ -7,6 +7,7 @@ #include "flutter/common/graphics/texture.h" #include "flutter/fml/macros.h" +#include "flutter/shell/platform/embedder/embedder.h" #include "third_party/skia/include/core/SkImage.h" #include "third_party/skia/include/core/SkSize.h" @@ -14,10 +15,8 @@ namespace flutter { class EmbedderExternalTextureGL : public flutter::Texture { public: - using ExternalTextureCallback = - std::function(int64_t texture_identifier, - GrDirectContext*, - const SkISize&)>; + using ExternalTextureCallback = std::function< + std::unique_ptr(int64_t, size_t, size_t)>; EmbedderExternalTextureGL(int64_t texture_identifier, const ExternalTextureCallback& callback); @@ -25,9 +24,13 @@ class EmbedderExternalTextureGL : public flutter::Texture { ~EmbedderExternalTextureGL(); private: - ExternalTextureCallback external_texture_callback_; + const ExternalTextureCallback& external_texture_callback_; sk_sp last_image_; + sk_sp ResolveTexture(int64_t texture_id, + GrDirectContext* context, + const SkISize& size); + // |flutter::Texture| void Paint(SkCanvas& canvas, const SkRect& bounds, From 90eacee6a7b89dd252b1e6866379c663feca1126 Mon Sep 17 00:00:00 2001 From: Kaushik Iska Date: Wed, 10 Feb 2021 04:31:53 -0800 Subject: [PATCH 2/6] [embedder] [metal] Add embedder support for metal external textures --- ci/licenses_golden/licenses_flutter | 4 + .../FlutterDarwinExternalTextureMetal.h | 17 ++- .../FlutterDarwinExternalTextureMetal.mm | 109 ++++++++++-------- shell/platform/embedder/BUILD.gn | 4 + shell/platform/embedder/embedder.cc | 48 ++++++-- shell/platform/embedder/embedder.h | 39 +++++++ shell/platform/embedder/embedder_engine.cc | 35 ++---- shell/platform/embedder/embedder_engine.h | 21 +--- .../embedder_external_texture_metal.h | 58 ++++++++++ .../embedder_external_texture_metal.mm | 97 ++++++++++++++++ .../embedder_external_texture_resolver.cc | 58 ++++++++++ .../embedder_external_texture_resolver.h | 54 +++++++++ 12 files changed, 442 insertions(+), 102 deletions(-) create mode 100644 shell/platform/embedder/embedder_external_texture_metal.h create mode 100644 shell/platform/embedder/embedder_external_texture_metal.mm create mode 100644 shell/platform/embedder/embedder_external_texture_resolver.cc create mode 100644 shell/platform/embedder/embedder_external_texture_resolver.h diff --git a/ci/licenses_golden/licenses_flutter b/ci/licenses_golden/licenses_flutter index a646d94e34f24..0857e4e1a3ece 100644 --- a/ci/licenses_golden/licenses_flutter +++ b/ci/licenses_golden/licenses_flutter @@ -1152,6 +1152,10 @@ FILE: ../../../flutter/shell/platform/embedder/embedder_engine.cc FILE: ../../../flutter/shell/platform/embedder/embedder_engine.h FILE: ../../../flutter/shell/platform/embedder/embedder_external_texture_gl.cc FILE: ../../../flutter/shell/platform/embedder/embedder_external_texture_gl.h +FILE: ../../../flutter/shell/platform/embedder/embedder_external_texture_metal.h +FILE: ../../../flutter/shell/platform/embedder/embedder_external_texture_metal.mm +FILE: ../../../flutter/shell/platform/embedder/embedder_external_texture_resolver.cc +FILE: ../../../flutter/shell/platform/embedder/embedder_external_texture_resolver.h FILE: ../../../flutter/shell/platform/embedder/embedder_external_view.cc FILE: ../../../flutter/shell/platform/embedder/embedder_external_view.h FILE: ../../../flutter/shell/platform/embedder/embedder_external_view_embedder.cc diff --git a/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.h b/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.h index 3962512e87f58..beadd1dfea0d1 100644 --- a/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.h +++ b/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.h @@ -7,7 +7,22 @@ #import "flutter/shell/platform/darwin/common/framework/Headers/FlutterTexture.h" #include "third_party/skia/include/core/SkCanvas.h" -#include "third_party/skia/include/gpu/GrDirectContext.h" +#include "third_party/skia/include/core/SkImage.h" + +@interface FlutterDarwinExternalTextureSkImageWrapper : NSObject + ++ (sk_sp)wrapYUVATexture:(nonnull id)yTex + UVTex:(nonnull id)uvTex + grContext:(nonnull GrDirectContext*)grContext + width:(size_t)width + height:(size_t)height; + ++ (sk_sp)wrapRGBATexture:(nonnull id)rgbaTex + grContext:(nonnull GrDirectContext*)grContext + width:(size_t)width + height:(size_t)height; + +@end @interface FlutterDarwinExternalTextureMetal : NSObject diff --git a/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.mm b/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.mm index 070e895a58a6c..2dc6febb048d0 100644 --- a/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.mm +++ b/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.mm @@ -14,17 +14,6 @@ FLUTTER_ASSERT_ARC -namespace { - -static sk_cf_obj SkiaTextureFromCVMetalTexture(CVMetalTextureRef cvMetalTexture) { - id texture = CVMetalTextureGetTexture(cvMetalTexture); - // CVMetal texture can be released as soon as we can the MTLTexture from it. - CVPixelBufferRelease(cvMetalTexture); - return sk_cf_obj{(__bridge_retained const void*)texture}; -} - -} - @implementation FlutterDarwinExternalTextureMetal { CVMetalTextureCacheRef _textureCache; NSObject* _externalTexture; @@ -185,31 +174,17 @@ - (void)onTextureUnregistered { } } - GrMtlTextureInfo ySkiaTextureInfo; - ySkiaTextureInfo.fTexture = SkiaTextureFromCVMetalTexture(yMetalTexture); + id yTex = CVMetalTextureGetTexture(yMetalTexture); + CVBufferRelease(yMetalTexture); - GrBackendTexture skiaBackendTextures[2]; - skiaBackendTextures[0] = GrBackendTexture(/*width=*/textureSize.width(), - /*height=*/textureSize.height(), - /*mipMapped=*/GrMipMapped ::kNo, - /*textureInfo=*/ySkiaTextureInfo); + id uvTex = CVMetalTextureGetTexture(uvMetalTexture); + CVBufferRelease(uvMetalTexture); - GrMtlTextureInfo uvSkiaTextureInfo; - uvSkiaTextureInfo.fTexture = SkiaTextureFromCVMetalTexture(uvMetalTexture); - - skiaBackendTextures[1] = GrBackendTexture(/*width=*/textureSize.width(), - /*height=*/textureSize.height(), - /*mipMapped=*/GrMipMapped ::kNo, - /*textureInfo=*/uvSkiaTextureInfo); - SkYUVAInfo yuvaInfo(skiaBackendTextures[0].dimensions(), SkYUVAInfo::PlaneConfig::kY_UV, - SkYUVAInfo::Subsampling::k444, kRec601_SkYUVColorSpace); - GrYUVABackendTextures yuvaBackendTextures(yuvaInfo, skiaBackendTextures, - kTopLeft_GrSurfaceOrigin); - - sk_sp image = - SkImage::MakeFromYUVATextures(grContext, yuvaBackendTextures, /*imageColorSpace=*/nullptr, - /*releaseProc*/ nullptr, /*releaseContext*/ nullptr); - return image; + return [FlutterDarwinExternalTextureSkImageWrapper wrapYUVATexture:yTex + UVTex:uvTex + grContext:grContext + width:textureSize.width() + height:textureSize.height()]; } - (sk_sp)wrapRGBAExternalPixelBuffer:(CVPixelBufferRef)pixelBuffer @@ -233,22 +208,64 @@ GrYUVABackendTextures yuvaBackendTextures(yuvaInfo, skiaBackendTextures, return nullptr; } + id rgbaTex = CVMetalTextureGetTexture(metalTexture); + CVBufferRelease(metalTexture); + + return [FlutterDarwinExternalTextureSkImageWrapper wrapRGBATexture:rgbaTex + grContext:grContext + width:textureSize.width() + height:textureSize.height()]; +} + +@end + +@implementation FlutterDarwinExternalTextureSkImageWrapper + ++ (sk_sp)wrapYUVATexture:(id)yTex + UVTex:(id)uvTex + grContext:(nonnull GrDirectContext*)grContext + width:(size_t)width + height:(size_t)height { + GrMtlTextureInfo ySkiaTextureInfo; + ySkiaTextureInfo.fTexture = sk_cf_obj{(__bridge_retained const void*)yTex}; + + GrBackendTexture skiaBackendTextures[2]; + skiaBackendTextures[0] = GrBackendTexture(/*width=*/width, + /*height=*/height, + /*mipMapped=*/GrMipMapped::kNo, + /*textureInfo=*/ySkiaTextureInfo); + + GrMtlTextureInfo uvSkiaTextureInfo; + uvSkiaTextureInfo.fTexture = sk_cf_obj{(__bridge_retained const void*)uvTex}; + + skiaBackendTextures[1] = GrBackendTexture(/*width=*/width, + /*height=*/height, + /*mipMapped=*/GrMipMapped::kNo, + /*textureInfo=*/uvSkiaTextureInfo); + SkYUVAInfo yuvaInfo(skiaBackendTextures[0].dimensions(), SkYUVAInfo::PlaneConfig::kY_UV, + SkYUVAInfo::Subsampling::k444, kRec601_SkYUVColorSpace); + GrYUVABackendTextures yuvaBackendTextures(yuvaInfo, skiaBackendTextures, + kTopLeft_GrSurfaceOrigin); + + return SkImage::MakeFromYUVATextures(grContext, yuvaBackendTextures, /*imageColorSpace=*/nullptr, + /*releaseProc*/ nullptr, /*releaseContext*/ nullptr); +} + ++ (sk_sp)wrapRGBATexture:(id)rgbaTex + grContext:(nonnull GrDirectContext*)grContext + width:(size_t)width + height:(size_t)height { GrMtlTextureInfo skiaTextureInfo; - skiaTextureInfo.fTexture = SkiaTextureFromCVMetalTexture(metalTexture); + skiaTextureInfo.fTexture = sk_cf_obj{(__bridge_retained const void*)rgbaTex}; - GrBackendTexture skiaBackendTexture(/*width=*/textureSize.width(), - /*height=*/textureSize.height(), + GrBackendTexture skiaBackendTexture(/*width=*/width, + /*height=*/height, /*mipMapped=*/GrMipMapped ::kNo, /*textureInfo=*/skiaTextureInfo); - sk_sp image = - SkImage::MakeFromTexture(grContext, skiaBackendTexture, kTopLeft_GrSurfaceOrigin, - kBGRA_8888_SkColorType, kPremul_SkAlphaType, - /*imageColorSpace=*/nullptr, /*releaseProc*/ nullptr, - /*releaseContext*/ nullptr - - ); - return image; + return SkImage::MakeFromTexture(grContext, skiaBackendTexture, kTopLeft_GrSurfaceOrigin, + kBGRA_8888_SkColorType, kPremul_SkAlphaType, + /*imageColorSpace=*/nullptr, /*releaseProc*/ nullptr, + /*releaseContext*/ nullptr); } - @end diff --git a/shell/platform/embedder/BUILD.gn b/shell/platform/embedder/BUILD.gn index f545de9691ef8..d1a7d150df037 100644 --- a/shell/platform/embedder/BUILD.gn +++ b/shell/platform/embedder/BUILD.gn @@ -34,6 +34,8 @@ template("embedder_source_set") { "embedder.cc", "embedder_engine.cc", "embedder_engine.h", + "embedder_external_texture_resolver.cc", + "embedder_external_texture_resolver.h", "embedder_external_view.cc", "embedder_external_view.h", "embedder_external_view_embedder.cc", @@ -90,6 +92,8 @@ template("embedder_source_set") { if (embedder_enable_metal) { sources += [ + "embedder_external_texture_metal.h", + "embedder_external_texture_metal.mm", "embedder_surface_metal.h", "embedder_surface_metal.mm", ] diff --git a/shell/platform/embedder/embedder.cc b/shell/platform/embedder/embedder.cc index b0cc085921c29..0bf4440bfcf78 100644 --- a/shell/platform/embedder/embedder.cc +++ b/shell/platform/embedder/embedder.cc @@ -48,6 +48,7 @@ extern const intptr_t kPlatformStrongDillSize; #include "flutter/shell/common/switches.h" #include "flutter/shell/platform/embedder/embedder.h" #include "flutter/shell/platform/embedder/embedder_engine.h" +#include "flutter/shell/platform/embedder/embedder_external_texture_resolver.h" #include "flutter/shell/platform/embedder/embedder_platform_message_response.h" #include "flutter/shell/platform/embedder/embedder_render_target.h" #include "flutter/shell/platform/embedder/embedder_struct_macros.h" @@ -1135,9 +1136,11 @@ FlutterEngineResult FlutterEngineInitialize(size_t version, return std::make_unique(shell); }; + using ExternalTextureResolver = flutter::EmbedderExternalTextureResolver; + std::unique_ptr external_texture_resolver; + external_texture_resolver = std::make_unique(); + #ifdef SHELL_ENABLE_GL - // TODO(chinmaygarde): This is the wrong spot for this. It belongs in the - // platform view jump table. flutter::EmbedderExternalTextureGL::ExternalTextureCallback external_texture_callback; if (config->type == kOpenGL) { @@ -1156,6 +1159,30 @@ FlutterEngineResult FlutterEngineInitialize(size_t version, }; } } + external_texture_resolver = + std::make_unique(external_texture_callback); +#endif +#ifdef SHELL_ENABLE_METAL + flutter::EmbedderExternalTextureMetal::ExternalTextureCallback + external_texture_metal_callback; + if (config->type == kMetal) { + const FlutterMetalRendererConfig* metal_config = &config->metal; + if (SAFE_ACCESS(metal_config, external_texture_frame_callback, nullptr)) { + external_texture_metal_callback = + [ptr = metal_config->external_texture_frame_callback, user_data]( + int64_t texture_identifier, size_t width, + size_t height) -> std::unique_ptr { + FlutterMetalExternalTexture* texture = + new FlutterMetalExternalTexture(); + if (!ptr(user_data, texture_identifier, width, height, texture)) { + return nullptr; + } + return std::unique_ptr(texture); + }; + } + } + external_texture_resolver = + std::make_unique(external_texture_callback); #endif auto thread_host = @@ -1207,16 +1234,13 @@ FlutterEngineResult FlutterEngineInitialize(size_t version, // Create the engine but don't launch the shell or run the root isolate. auto embedder_engine = std::make_unique( - std::move(thread_host), // - std::move(task_runners), // - std::move(settings), // - std::move(run_configuration), // - on_create_platform_view, // - on_create_rasterizer // -#ifdef SHELL_ENABLE_GL - , - external_texture_callback // -#endif + std::move(thread_host), // + std::move(task_runners), // + std::move(settings), // + std::move(run_configuration), // + on_create_platform_view, // + on_create_rasterizer, // + std::move(external_texture_resolver) // ); // Release the ownership of the embedder engine to the caller. diff --git a/shell/platform/embedder/embedder.h b/shell/platform/embedder/embedder.h index 6489edcf938ef..04bff041da0d0 100644 --- a/shell/platform/embedder/embedder.h +++ b/shell/platform/embedder/embedder.h @@ -447,6 +447,40 @@ typedef const void* FlutterMetalCommandQueueHandle; /// Alias for id. typedef const void* FlutterMetalTextureHandle; +/// Pixel format for the external texture. +typedef enum { + kYUVA, + kRGBA, +} FlutterMetalExternalTexturePixelFormat; + +typedef struct { + /// The size of this struct. Must be sizeof(FlutterMetalExternalTexture). + size_t struct_size; + /// Height of the texture. + size_t width; + /// Height of the texture. + size_t height; + /// The pixel format type of the external. + FlutterMetalExternalTexturePixelFormat pixel_format; + /// Represents the size of the `textures` array. + size_t num_textures; + /// Supported textures are YUVA and RGBA, in case of YUVA we expect 2 texture + /// handles to be provided by the embedder, Y first and UV next. In case of + /// RGBA only one should be passed. + /// These are individually aliases for id which will be released + /// once they have been composited: `external_texture_frame_callback`. + FlutterMetalTextureHandle* textures; +} FlutterMetalExternalTexture; + +/// Callback to provide an external texture for a given texture_id. +/// See: external_texture_frame_callback. +typedef bool (*FlutterMetalTextureFrameCallback)( + void* /* user data */, + int64_t /* texture identifier */, + size_t /* width */, + size_t /* height */, + FlutterMetalExternalTexture* /* texture out */); + typedef struct { /// The size of this struct. Must be sizeof(FlutterMetalTexture). size_t struct_size; @@ -485,6 +519,11 @@ typedef struct { /// The callback presented to the embedder to present a fully populated metal /// texture to the user. FlutterMetalPresentCallback present_drawable_callback; + /// When the embedder specifies that a texture has a frame available, the + /// engine will call this method (on an internal engine managed thread) so + /// that external texture details can be supplied to the engine for subsequent + /// composition. + FlutterMetalTextureFrameCallback external_texture_frame_callback; } FlutterMetalRendererConfig; typedef struct { diff --git a/shell/platform/embedder/embedder_engine.cc b/shell/platform/embedder/embedder_engine.cc index d4a55fff95f6a..0b0adf8793f63 100644 --- a/shell/platform/embedder/embedder_engine.cc +++ b/shell/platform/embedder/embedder_engine.cc @@ -27,24 +27,15 @@ EmbedderEngine::EmbedderEngine( flutter::Settings settings, RunConfiguration run_configuration, Shell::CreateCallback on_create_platform_view, - Shell::CreateCallback on_create_rasterizer -#ifdef SHELL_ENABLE_GL - , - EmbedderExternalTextureGL::ExternalTextureCallback external_texture_callback -#endif - ) + Shell::CreateCallback on_create_rasterizer, + std::unique_ptr external_texture_resolver) : thread_host_(std::move(thread_host)), task_runners_(task_runners), run_configuration_(std::move(run_configuration)), shell_args_(std::make_unique(std::move(settings), on_create_platform_view, - on_create_rasterizer)) -#ifdef SHELL_ENABLE_GL - , - external_texture_callback_(external_texture_callback) -#endif -{ -} + on_create_rasterizer)), + external_texture_resolver_(std::move(external_texture_resolver)) {} EmbedderEngine::~EmbedderEngine() = default; @@ -168,37 +159,27 @@ bool EmbedderEngine::SendPlatformMessage( } bool EmbedderEngine::RegisterTexture(int64_t texture) { -#ifdef SHELL_ENABLE_GL - if (!IsValid() || !external_texture_callback_) { + if (!IsValid()) { return false; } shell_->GetPlatformView()->RegisterTexture( - std::make_unique(texture, - external_texture_callback_)); -#endif - + external_texture_resolver_->ResolveExternalTexture(texture)); return true; } bool EmbedderEngine::UnregisterTexture(int64_t texture) { -#ifdef SHELL_ENABLE_GL - if (!IsValid() || !external_texture_callback_) { + if (!IsValid()) { return false; } shell_->GetPlatformView()->UnregisterTexture(texture); -#endif - return true; } bool EmbedderEngine::MarkTextureFrameAvailable(int64_t texture) { -#ifdef SHELL_ENABLE_GL - if (!IsValid() || !external_texture_callback_) { + if (!IsValid()) { return false; } shell_->GetPlatformView()->MarkTextureFrameAvailable(texture); -#endif - return true; } diff --git a/shell/platform/embedder/embedder_engine.h b/shell/platform/embedder/embedder_engine.h index 55afebdba7367..ab0de7b1f1d1b 100644 --- a/shell/platform/embedder/embedder_engine.h +++ b/shell/platform/embedder/embedder_engine.h @@ -12,12 +12,8 @@ #include "flutter/shell/common/shell.h" #include "flutter/shell/common/thread_host.h" #include "flutter/shell/platform/embedder/embedder.h" +#include "flutter/shell/platform/embedder/embedder_external_texture_resolver.h" #include "flutter/shell/platform/embedder/embedder_thread_host.h" - -#ifdef SHELL_ENABLE_GL -#include "flutter/shell/platform/embedder/embedder_external_texture_gl.h" -#endif - namespace flutter { struct ShellArgs; @@ -31,13 +27,9 @@ class EmbedderEngine { Settings settings, RunConfiguration run_configuration, Shell::CreateCallback on_create_platform_view, - Shell::CreateCallback on_create_rasterizer -#ifdef SHELL_ENABLE_GL - , - EmbedderExternalTextureGL::ExternalTextureCallback - external_texture_callback -#endif - ); + Shell::CreateCallback on_create_rasterizer, + std::unique_ptr + external_texture_resolver); ~EmbedderEngine(); @@ -113,10 +105,7 @@ class EmbedderEngine { RunConfiguration run_configuration_; std::unique_ptr shell_args_; std::unique_ptr shell_; -#ifdef SHELL_ENABLE_GL - const EmbedderExternalTextureGL::ExternalTextureCallback - external_texture_callback_; -#endif + std::unique_ptr external_texture_resolver_; FML_DISALLOW_COPY_AND_ASSIGN(EmbedderEngine); }; diff --git a/shell/platform/embedder/embedder_external_texture_metal.h b/shell/platform/embedder/embedder_external_texture_metal.h new file mode 100644 index 0000000000000..fb34a13eade49 --- /dev/null +++ b/shell/platform/embedder/embedder_external_texture_metal.h @@ -0,0 +1,58 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef FLUTTER_SHELL_PLATFORM_EMBEDDER_EMBEDDER_EXTERNAL_TEXTURE_METAL_H_ +#define FLUTTER_SHELL_PLATFORM_EMBEDDER_EMBEDDER_EXTERNAL_TEXTURE_METAL_H_ + +#include "flutter/common/graphics/texture.h" +#include "flutter/fml/macros.h" +#include "flutter/shell/platform/embedder/embedder.h" +#include "third_party/skia/include/core/SkImage.h" +#include "third_party/skia/include/core/SkSize.h" + +namespace flutter { + +class EmbedderExternalTextureMetal : public flutter::Texture { + public: + using ExternalTextureCallback = std::function< + std::unique_ptr(int64_t, size_t, size_t)>; + + EmbedderExternalTextureMetal(int64_t texture_identifier, + const ExternalTextureCallback& callback); + + ~EmbedderExternalTextureMetal(); + + private: + const ExternalTextureCallback& external_texture_callback_; + sk_sp last_image_; + + sk_sp ResolveTexture(int64_t texture_id, + GrDirectContext* context, + const SkISize& size); + + // |flutter::Texture| + void Paint(SkCanvas& canvas, + const SkRect& bounds, + bool freeze, + GrDirectContext* context, + const SkSamplingOptions& sampling) override; + + // |flutter::Texture| + void OnGrContextCreated() override; + + // |flutter::Texture| + void OnGrContextDestroyed() override; + + // |flutter::Texture| + void MarkNewFrameAvailable() override; + + // |flutter::Texture| + void OnTextureUnregistered() override; + + FML_DISALLOW_COPY_AND_ASSIGN(EmbedderExternalTextureMetal); +}; + +} // namespace flutter + +#endif // FLUTTER_SHELL_PLATFORM_EMBEDDER_EMBEDDER_EXTERNAL_TEXTURE_METAL_H_ diff --git a/shell/platform/embedder/embedder_external_texture_metal.mm b/shell/platform/embedder/embedder_external_texture_metal.mm new file mode 100644 index 0000000000000..1edadf3cc1cfd --- /dev/null +++ b/shell/platform/embedder/embedder_external_texture_metal.mm @@ -0,0 +1,97 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "flutter/shell/platform/embedder/embedder_external_texture_metal.h" + +#include "flutter/fml/logging.h" +#import "flutter/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.h" +#include "third_party/skia/include/core/SkImage.h" +#include "third_party/skia/include/core/SkSize.h" +#include "third_party/skia/include/gpu/GrBackendSurface.h" +#include "third_party/skia/include/gpu/GrDirectContext.h" + +namespace flutter { + +EmbedderExternalTextureMetal::EmbedderExternalTextureMetal(int64_t texture_identifier, + const ExternalTextureCallback& callback) + : Texture(texture_identifier), external_texture_callback_(callback) { + FML_DCHECK(external_texture_callback_); +} + +EmbedderExternalTextureMetal::~EmbedderExternalTextureMetal() = default; + +// |flutter::Texture| +void EmbedderExternalTextureMetal::Paint(SkCanvas& canvas, + const SkRect& bounds, + bool freeze, + GrDirectContext* context, + const SkSamplingOptions& sampling) { + if (auto image = ResolveTexture(Id(), context, SkISize::Make(bounds.width(), bounds.height()))) { + last_image_ = image; + } + + if (last_image_) { + if (bounds != SkRect::Make(last_image_->bounds())) { + canvas.drawImageRect(last_image_, bounds, sampling); + } else { + canvas.drawImage(last_image_, bounds.x(), bounds.y(), sampling, nullptr); + } + } +} + +sk_sp EmbedderExternalTextureMetal::ResolveTexture(int64_t texture_id, + GrDirectContext* context, + const SkISize& size) { + std::unique_ptr texture = + external_texture_callback_(texture_id, size.width(), size.height()); + + if (!texture) { + return nullptr; + } + + sk_sp image; + + switch (texture->pixel_format) { + case FlutterMetalExternalTexturePixelFormat::kRGBA: { + FML_CHECK(texture->num_textures == 1); + id rgbaTex = reinterpret_cast>(texture->textures[0]); + image = [FlutterDarwinExternalTextureSkImageWrapper wrapRGBATexture:rgbaTex + grContext:context + width:size.width() + height:size.height()]; + break; + } + case FlutterMetalExternalTexturePixelFormat::kYUVA: { + FML_CHECK(texture->num_textures == 2); + id yTex = reinterpret_cast>(texture->textures[0]); + id uvTex = reinterpret_cast>(texture->textures[1]); + image = [FlutterDarwinExternalTextureSkImageWrapper wrapYUVATexture:yTex + UVTex:uvTex + grContext:context + width:size.width() + height:size.height()]; + break; + } + } + + if (!image) { + FML_LOG(ERROR) << "Could not create external texture: " << texture_id; + } + + return image; +} + +// |flutter::Texture| +void EmbedderExternalTextureMetal::OnGrContextCreated() {} + +// |flutter::Texture| +void EmbedderExternalTextureMetal::OnGrContextDestroyed() {} + +// |flutter::Texture| +void EmbedderExternalTextureMetal::MarkNewFrameAvailable() {} + +// |flutter::Texture| +void EmbedderExternalTextureMetal::OnTextureUnregistered() {} + +} // namespace flutter diff --git a/shell/platform/embedder/embedder_external_texture_resolver.cc b/shell/platform/embedder/embedder_external_texture_resolver.cc new file mode 100644 index 0000000000000..f2aa18d11b3f0 --- /dev/null +++ b/shell/platform/embedder/embedder_external_texture_resolver.cc @@ -0,0 +1,58 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "flutter/shell/platform/embedder/embedder_external_texture_resolver.h" + +#include + +namespace flutter { + +#ifdef SHELL_ENABLE_GL +EmbedderExternalTextureResolver::EmbedderExternalTextureResolver( + EmbedderExternalTextureGL::ExternalTextureCallback gl_callback) + : gl_callback_(gl_callback) {} +#endif + +#ifdef SHELL_ENABLE_METAL +EmbedderExternalTextureResolver::EmbedderExternalTextureResolver( + EmbedderExternalTextureMetal::ExternalTextureCallback metal_callback) + : metal_callback_(metal_callback) {} +#endif + +std::unique_ptr +EmbedderExternalTextureResolver::ResolveExternalTexture(int64_t texture_id) { +#ifdef SHELL_ENABLE_GL + if (gl_callback_) { + return std::make_unique(texture_id, + gl_callback_); + } +#endif + +#ifdef SHELL_ENABLE_METAL + if (metal_callback_) { + return std::make_unique(texture_id, + metal_callback_); + } +#endif + + return nullptr; +} + +bool EmbedderExternalTextureResolver::SupportsExternalTextures() { +#ifdef SHELL_ENABLE_GL + if (gl_callback_) { + return true; + } +#endif + +#ifdef SHELL_ENABLE_METAL + if (metal_callback_) { + return true; + } +#endif + + return false; +} + +} // namespace flutter diff --git a/shell/platform/embedder/embedder_external_texture_resolver.h b/shell/platform/embedder/embedder_external_texture_resolver.h new file mode 100644 index 0000000000000..412301736e566 --- /dev/null +++ b/shell/platform/embedder/embedder_external_texture_resolver.h @@ -0,0 +1,54 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef FLUTTER_SHELL_PLATFORM_EMBEDDER_EMBEDDER_EXTERNAL_TEXTURE_RESOLVER_H_ +#define FLUTTER_SHELL_PLATFORM_EMBEDDER_EMBEDDER_EXTERNAL_TEXTURE_RESOLVER_H_ + +#include + +#include "flutter/common/graphics/texture.h" + +#ifdef SHELL_ENABLE_GL +#include "flutter/shell/platform/embedder/embedder_external_texture_gl.h" +#endif + +#ifdef SHELL_ENABLE_METAL +#include "flutter/shell/platform/embedder/embedder_external_texture_metal.h" +#endif + +namespace flutter { +class EmbedderExternalTextureResolver { + public: + EmbedderExternalTextureResolver() = default; + + ~EmbedderExternalTextureResolver() = default; + +#ifdef SHELL_ENABLE_GL + explicit EmbedderExternalTextureResolver( + EmbedderExternalTextureGL::ExternalTextureCallback gl_callback); +#endif + +#ifdef SHELL_ENABLE_METAL + explicit EmbedderExternalTextureResolver( + EmbedderExternalTextureMetal::ExternalTextureCallback metal_callback); +#endif + + std::unique_ptr ResolveExternalTexture(int64_t texture_id); + + bool SupportsExternalTextures(); + + private: +#ifdef SHELL_ENABLE_GL + EmbedderExternalTextureGL::ExternalTextureCallback gl_callback_; +#endif + +#ifdef SHELL_ENABLE_METAL + EmbedderExternalTextureMetal::ExternalTextureCallback metal_callback_; +#endif + + FML_DISALLOW_COPY_AND_ASSIGN(EmbedderExternalTextureResolver); +}; +} // namespace flutter + +#endif // FLUTTER_SHELL_PLATFORM_EMBEDDER_EMBEDDER_EXTERNAL_TEXTURE_RESOLVER_H_ From 89d98c6bb460634eef90868c21dfa5cd7d000473 Mon Sep 17 00:00:00 2001 From: Kaushik Iska Date: Thu, 11 Feb 2021 12:17:01 -0800 Subject: [PATCH 3/6] fix cr comments --- shell/platform/embedder/embedder.h | 9 ++-- .../embedder_external_texture_metal.mm | 41 ++++++++++++------- 2 files changed, 33 insertions(+), 17 deletions(-) diff --git a/shell/platform/embedder/embedder.h b/shell/platform/embedder/embedder.h index 04bff041da0d0..982480ec7d6ae 100644 --- a/shell/platform/embedder/embedder.h +++ b/shell/platform/embedder/embedder.h @@ -467,9 +467,12 @@ typedef struct { /// Supported textures are YUVA and RGBA, in case of YUVA we expect 2 texture /// handles to be provided by the embedder, Y first and UV next. In case of /// RGBA only one should be passed. - /// These are individually aliases for id which will be released - /// once they have been composited: `external_texture_frame_callback`. - FlutterMetalTextureHandle* textures; + /// These are individually aliases for id. These textures are + /// retained by the engine for the period of the composition. Once these + /// textures have been unregistered via the + /// `FlutterEngineUnregisterExternalTexture`, the embedder has to release + /// these textures. + const FlutterMetalTextureHandle* textures; } FlutterMetalExternalTexture; /// Callback to provide an external texture for a given texture_id. diff --git a/shell/platform/embedder/embedder_external_texture_metal.mm b/shell/platform/embedder/embedder_external_texture_metal.mm index 1edadf3cc1cfd..ef7110458c789 100644 --- a/shell/platform/embedder/embedder_external_texture_metal.mm +++ b/shell/platform/embedder/embedder_external_texture_metal.mm @@ -13,6 +13,15 @@ namespace flutter { +static bool ValidNumTextures(int expected, int actual) { + if (expected == actual) { + return true; + } else { + FML_LOG(ERROR) << "Invalid number of textures, expected: " << expected << ", got: " << actual; + return false; + } +} + EmbedderExternalTextureMetal::EmbedderExternalTextureMetal(int64_t texture_identifier, const ExternalTextureCallback& callback) : Texture(texture_identifier), external_texture_callback_(callback) { @@ -47,6 +56,8 @@ external_texture_callback_(texture_id, size.width(), size.height()); if (!texture) { + FML_LOG(ERROR) << "External texture callback for ID " << texture_id + << " did not return a valid texture."; return nullptr; } @@ -54,23 +65,25 @@ switch (texture->pixel_format) { case FlutterMetalExternalTexturePixelFormat::kRGBA: { - FML_CHECK(texture->num_textures == 1); - id rgbaTex = reinterpret_cast>(texture->textures[0]); - image = [FlutterDarwinExternalTextureSkImageWrapper wrapRGBATexture:rgbaTex - grContext:context - width:size.width() - height:size.height()]; + if (ValidNumTextures(1, texture->num_textures)) { + id rgbaTex = reinterpret_cast>(texture->textures[0]); + image = [FlutterDarwinExternalTextureSkImageWrapper wrapRGBATexture:rgbaTex + grContext:context + width:size.width() + height:size.height()]; + } break; } case FlutterMetalExternalTexturePixelFormat::kYUVA: { - FML_CHECK(texture->num_textures == 2); - id yTex = reinterpret_cast>(texture->textures[0]); - id uvTex = reinterpret_cast>(texture->textures[1]); - image = [FlutterDarwinExternalTextureSkImageWrapper wrapYUVATexture:yTex - UVTex:uvTex - grContext:context - width:size.width() - height:size.height()]; + if (ValidNumTextures(2, texture->num_textures)) { + id yTex = reinterpret_cast>(texture->textures[0]); + id uvTex = reinterpret_cast>(texture->textures[1]); + image = [FlutterDarwinExternalTextureSkImageWrapper wrapYUVATexture:yTex + UVTex:uvTex + grContext:context + width:size.width() + height:size.height()]; + } break; } } From d85498dad8c01fb21006cc4c5fd83e2a2f1000a9 Mon Sep 17 00:00:00 2001 From: Kaushik Iska Date: Thu, 11 Feb 2021 13:41:43 -0800 Subject: [PATCH 4/6] Add test for external textures on macOS --- shell/platform/darwin/macos/BUILD.gn | 2 + ...FlutterEmbedderExternalTextureUnittests.mm | 79 +++++++++++++++++++ 2 files changed, 81 insertions(+) create mode 100644 shell/platform/darwin/macos/framework/Source/FlutterEmbedderExternalTextureUnittests.mm diff --git a/shell/platform/darwin/macos/BUILD.gn b/shell/platform/darwin/macos/BUILD.gn index b68a1431a8e73..db743a6aebfe9 100644 --- a/shell/platform/darwin/macos/BUILD.gn +++ b/shell/platform/darwin/macos/BUILD.gn @@ -154,6 +154,7 @@ executable("flutter_desktop_darwin_unittests") { if (shell_enable_metal) { sources += [ + "framework/Source/FlutterEmbedderExternalTextureUnittests.mm", "framework/Source/FlutterMetalRendererTest.mm", "framework/Source/FlutterMetalSurfaceManagerTest.mm", ] @@ -169,6 +170,7 @@ executable("flutter_desktop_darwin_unittests") { ":flutter_desktop_darwin_fixtures", ":flutter_framework_source", "//flutter/shell/platform/darwin/common:framework_shared", + "//flutter/shell/platform/darwin/graphics", "//flutter/shell/platform/embedder:embedder_as_internal_library", "//flutter/shell/platform/embedder:embedder_test_utils", "//flutter/testing", diff --git a/shell/platform/darwin/macos/framework/Source/FlutterEmbedderExternalTextureUnittests.mm b/shell/platform/darwin/macos/framework/Source/FlutterEmbedderExternalTextureUnittests.mm new file mode 100644 index 0000000000000..700bd6ed2657f --- /dev/null +++ b/shell/platform/darwin/macos/framework/Source/FlutterEmbedderExternalTextureUnittests.mm @@ -0,0 +1,79 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#import +#import + +#include +#include + +#import "flutter/shell/platform/darwin/graphics/FlutterDarwinContextMetal.h" +#import "flutter/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.h" +#include "flutter/shell/platform/embedder/embedder.h" +#include "flutter/shell/platform/embedder/embedder_external_texture_metal.h" +#import "flutter/testing/testing.h" +#include "third_party/googletest/googletest/include/gtest/gtest.h" +#include "third_party/skia/include/core/SkImage.h" +#include "third_party/skia/include/core/SkSamplingOptions.h" +#include "third_party/skia/include/core/SkSurface.h" + +namespace flutter::testing { + +TEST(FlutterEmbedderExternalTextureUnittests, TestTextureResolution) { + // constants. + const size_t width = 100; + const size_t height = 100; + const int64_t texture_id = 1; + + // setup the surface. + FlutterDarwinContextMetal* darwinContextMetal = + [[FlutterDarwinContextMetal alloc] initWithDefaultMTLDevice]; + SkImageInfo info = SkImageInfo::MakeN32Premul(width, height); + GrDirectContext* grContext = darwinContextMetal.mainContext.get(); + sk_sp gpuSurface(SkSurface::MakeRenderTarget(grContext, SkBudgeted::kNo, info)); + + // create a texture. + MTLTextureDescriptor* textureDescriptor = [[MTLTextureDescriptor alloc] init]; + textureDescriptor.pixelFormat = MTLPixelFormatBGRA8Unorm; + textureDescriptor.width = width; + textureDescriptor.height = height; + textureDescriptor.usage = MTLTextureUsageRenderTarget | MTLTextureUsageShaderRead; + id mtlTexture = + [darwinContextMetal.device newTextureWithDescriptor:textureDescriptor]; + + // callback to resolve the texture. + EmbedderExternalTextureMetal::ExternalTextureCallback callback = [&](int64_t texture_id, size_t w, + size_t h) { + EXPECT_TRUE(w == width); + EXPECT_TRUE(h == height); + + FlutterMetalExternalTexture* texture = new FlutterMetalExternalTexture(); + texture->struct_size = sizeof(FlutterMetalExternalTexture); + texture->num_textures = 1; + texture->height = h; + texture->width = w; + texture->pixel_format = FlutterMetalExternalTexturePixelFormat::kRGBA; + + std::vector textures = { + (__bridge FlutterMetalTextureHandle)mtlTexture, + }; + + texture->textures = textures.data(); + + return std::unique_ptr(texture); + }; + + // render the texture. + std::unique_ptr texture = + std::make_unique(texture_id, callback); + SkRect bounds = SkRect::MakeWH(info.width(), info.height()); + SkSamplingOptions sampling = SkSamplingOptions(SkFilterMode::kNearest); + texture->Paint(*gpuSurface->getCanvas(), bounds, /*freeze=*/false, grContext, sampling); + + ASSERT_TRUE(mtlTexture != nil); + + gpuSurface->makeImageSnapshot(); +} + +} // namespace flutter::testing From c01798bfb4579a5b4cb284b37a510b2ee720cdbd Mon Sep 17 00:00:00 2001 From: Kaushik Iska Date: Thu, 11 Feb 2021 14:15:15 -0800 Subject: [PATCH 5/6] fix licenses --- ci/licenses_golden/licenses_flutter | 1 + shell/platform/darwin/macos/BUILD.gn | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/ci/licenses_golden/licenses_flutter b/ci/licenses_golden/licenses_flutter index 0857e4e1a3ece..99389cd5d8901 100644 --- a/ci/licenses_golden/licenses_flutter +++ b/ci/licenses_golden/licenses_flutter @@ -1099,6 +1099,7 @@ FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterBacki FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterBackingStoreData.mm FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterDartProject.mm FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterDartProject_Internal.h +FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterEmbedderExternalTextureUnittests.mm FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterEngine.mm FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterEngineTest.mm FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterEngine_Internal.h diff --git a/shell/platform/darwin/macos/BUILD.gn b/shell/platform/darwin/macos/BUILD.gn index db743a6aebfe9..7dac2895445a8 100644 --- a/shell/platform/darwin/macos/BUILD.gn +++ b/shell/platform/darwin/macos/BUILD.gn @@ -170,7 +170,6 @@ executable("flutter_desktop_darwin_unittests") { ":flutter_desktop_darwin_fixtures", ":flutter_framework_source", "//flutter/shell/platform/darwin/common:framework_shared", - "//flutter/shell/platform/darwin/graphics", "//flutter/shell/platform/embedder:embedder_as_internal_library", "//flutter/shell/platform/embedder:embedder_test_utils", "//flutter/testing", @@ -179,6 +178,10 @@ executable("flutter_desktop_darwin_unittests") { "//flutter/testing:testing_lib", "//third_party/ocmock:ocmock", ] + + if (shell_enable_metal) { + deps += [ "//flutter/shell/platform/darwin/graphics" ] + } } copy("copy_dylib") { From 967e0a4324666fbaf4c5e4c69d1d177a1eb4ff32 Mon Sep 17 00:00:00 2001 From: Kaushik Iska Date: Thu, 18 Feb 2021 10:26:43 -0800 Subject: [PATCH 6/6] add a test that exercises embedder codepath --- ci/licenses_golden/licenses_flutter | 1 + shell/platform/embedder/BUILD.gn | 3 +- shell/platform/embedder/embedder.cc | 20 +-- shell/platform/embedder/embedder.h | 2 +- .../fixtures/external_texture_metal.png | Bin 0 -> 3252 bytes shell/platform/embedder/fixtures/main.dart | 18 +++ .../embedder/tests/embedder_config_builder.cc | 8 ++ .../tests/embedder_test_context_metal.cc | 18 +++ .../tests/embedder_test_context_metal.h | 15 +++ .../tests/embedder_unittests_metal.cc | 52 -------- .../tests/embedder_unittests_metal.mm | 120 ++++++++++++++++++ 11 files changed, 194 insertions(+), 63 deletions(-) create mode 100644 shell/platform/embedder/fixtures/external_texture_metal.png delete mode 100644 shell/platform/embedder/tests/embedder_unittests_metal.cc create mode 100644 shell/platform/embedder/tests/embedder_unittests_metal.mm diff --git a/ci/licenses_golden/licenses_flutter b/ci/licenses_golden/licenses_flutter index 99389cd5d8901..5d8852e578d61 100644 --- a/ci/licenses_golden/licenses_flutter +++ b/ci/licenses_golden/licenses_flutter @@ -1192,6 +1192,7 @@ FILE: ../../../flutter/shell/platform/embedder/fixtures/compositor_with_platform FILE: ../../../flutter/shell/platform/embedder/fixtures/compositor_with_root_layer_only.png FILE: ../../../flutter/shell/platform/embedder/fixtures/dpr_noxform.png FILE: ../../../flutter/shell/platform/embedder/fixtures/dpr_xform.png +FILE: ../../../flutter/shell/platform/embedder/fixtures/external_texture_metal.png FILE: ../../../flutter/shell/platform/embedder/fixtures/gradient.png FILE: ../../../flutter/shell/platform/embedder/fixtures/gradient_metal.png FILE: ../../../flutter/shell/platform/embedder/fixtures/gradient_xform.png diff --git a/shell/platform/embedder/BUILD.gn b/shell/platform/embedder/BUILD.gn index d1a7d150df037..edaa0a57228cd 100644 --- a/shell/platform/embedder/BUILD.gn +++ b/shell/platform/embedder/BUILD.gn @@ -160,6 +160,7 @@ test_fixtures("fixtures") { "fixtures/dpr_xform.png", "fixtures/gradient.png", "fixtures/gradient_metal.png", + "fixtures/external_texture_metal.png", "fixtures/gradient_xform.png", "fixtures/scene_without_custom_compositor.png", "fixtures/scene_without_custom_compositor_with_xform.png", @@ -230,7 +231,7 @@ if (enable_unittests) { sources += [ "tests/embedder_test_context_metal.cc", "tests/embedder_test_context_metal.h", - "tests/embedder_unittests_metal.cc", + "tests/embedder_unittests_metal.mm", ] deps += [ "//flutter/testing:metal" ] diff --git a/shell/platform/embedder/embedder.cc b/shell/platform/embedder/embedder.cc index 0bf4440bfcf78..20522a25710b8 100644 --- a/shell/platform/embedder/embedder.cc +++ b/shell/platform/embedder/embedder.cc @@ -1151,11 +1151,12 @@ FlutterEngineResult FlutterEngineInitialize(size_t version, [ptr = open_gl_config->gl_external_texture_frame_callback, user_data]( int64_t texture_identifier, size_t width, size_t height) -> std::unique_ptr { - FlutterOpenGLTexture* texture = new FlutterOpenGLTexture(); - if (!ptr(user_data, texture_identifier, width, height, texture)) { + std::unique_ptr texture = + std::make_unique(); + if (!ptr(user_data, texture_identifier, width, height, texture.get())) { return nullptr; } - return std::unique_ptr(texture); + return texture; }; } } @@ -1172,17 +1173,18 @@ FlutterEngineResult FlutterEngineInitialize(size_t version, [ptr = metal_config->external_texture_frame_callback, user_data]( int64_t texture_identifier, size_t width, size_t height) -> std::unique_ptr { - FlutterMetalExternalTexture* texture = - new FlutterMetalExternalTexture(); - if (!ptr(user_data, texture_identifier, width, height, texture)) { + std::unique_ptr texture = + std::make_unique(); + texture->struct_size = sizeof(FlutterMetalExternalTexture); + if (!ptr(user_data, texture_identifier, width, height, texture.get())) { return nullptr; } - return std::unique_ptr(texture); + return texture; }; } } - external_texture_resolver = - std::make_unique(external_texture_callback); + external_texture_resolver = std::make_unique( + external_texture_metal_callback); #endif auto thread_host = diff --git a/shell/platform/embedder/embedder.h b/shell/platform/embedder/embedder.h index 982480ec7d6ae..6d1d95ee97451 100644 --- a/shell/platform/embedder/embedder.h +++ b/shell/platform/embedder/embedder.h @@ -472,7 +472,7 @@ typedef struct { /// textures have been unregistered via the /// `FlutterEngineUnregisterExternalTexture`, the embedder has to release /// these textures. - const FlutterMetalTextureHandle* textures; + FlutterMetalTextureHandle* textures; } FlutterMetalExternalTexture; /// Callback to provide an external texture for a given texture_id. diff --git a/shell/platform/embedder/fixtures/external_texture_metal.png b/shell/platform/embedder/fixtures/external_texture_metal.png new file mode 100644 index 0000000000000000000000000000000000000000..89f889b95dcb75ae7a60517e086d71ae98dec8b5 GIT binary patch literal 3252 zcmeAS@N?(olHy`uVBq!ia0y~yU{+vYV2a>i1B%QlYbpRzEX7WqAsj$Z!;#X#z`&F3 z>EaktG3V`dLq-M#o&y^i>(4(}B%{Lht8DH}W1#xRzg9r``U1A`G83j@Om2>}KM zg=7Z?hK8O|!O;L2O$?(MVYFlzEfq&=gwfh?v^E^A4M%Ik(b^DuZ5Wd;&bDrrenu>? PL&4zb>gTe~DWM4fq#N9) literal 0 HcmV?d00001 diff --git a/shell/platform/embedder/fixtures/main.dart b/shell/platform/embedder/fixtures/main.dart index 9d0458af6fdcd..070179b9b9a15 100644 --- a/shell/platform/embedder/fixtures/main.dart +++ b/shell/platform/embedder/fixtures/main.dart @@ -554,6 +554,24 @@ void render_gradient() { PlatformDispatcher.instance.scheduleFrame(); } +@pragma('vm:entry-point') +void render_texture() { + PlatformDispatcher.instance.onBeginFrame = (Duration duration) { + Size size = Size(800.0, 600.0); + + SceneBuilder builder = SceneBuilder(); + + builder.pushOffset(0.0, 0.0); + + builder.addTexture(/*textureId*/1, width: size.width, height: size.height); + + builder.pop(); + + PlatformDispatcher.instance.views.first.render(builder.build()); + }; + PlatformDispatcher.instance.scheduleFrame(); +} + @pragma('vm:entry-point') void render_gradient_on_non_root_backing_store() { PlatformDispatcher.instance.onBeginFrame = (Duration duration) { diff --git a/shell/platform/embedder/tests/embedder_config_builder.cc b/shell/platform/embedder/tests/embedder_config_builder.cc index 05cda13023304..37b2425136ebb 100644 --- a/shell/platform/embedder/tests/embedder_config_builder.cc +++ b/shell/platform/embedder/tests/embedder_config_builder.cc @@ -409,6 +409,14 @@ void EmbedderConfigBuilder::InitializeMetalRendererConfig() { reinterpret_cast(user_data); return metal_context->Present(texture->texture_id); }; + metal_renderer_config_.external_texture_frame_callback = + [](void* user_data, int64_t texture_id, size_t width, size_t height, + FlutterMetalExternalTexture* texture_out) -> bool { + EmbedderTestContextMetal* metal_context = + reinterpret_cast(user_data); + return metal_context->PopulateExternalTexture(texture_id, width, height, + texture_out); + }; } #endif // SHELL_ENABLE_METAL diff --git a/shell/platform/embedder/tests/embedder_test_context_metal.cc b/shell/platform/embedder/tests/embedder_test_context_metal.cc index 3514262b8b022..3a4fc008095f6 100644 --- a/shell/platform/embedder/tests/embedder_test_context_metal.cc +++ b/shell/platform/embedder/tests/embedder_test_context_metal.cc @@ -6,6 +6,7 @@ #include +#include "embedder.h" #include "flutter/fml/logging.h" namespace flutter { @@ -49,5 +50,22 @@ bool EmbedderTestContextMetal::Present(int64_t texture_id) { return metal_context_->Present(texture_id); } +void EmbedderTestContextMetal::SetExternalTextureCallback( + TestExternalTextureCallback external_texture_frame_callback) { + external_texture_frame_callback_ = external_texture_frame_callback; +} + +bool EmbedderTestContextMetal::PopulateExternalTexture( + int64_t texture_id, + size_t w, + size_t h, + FlutterMetalExternalTexture* output) { + if (external_texture_frame_callback_ != nullptr) { + return external_texture_frame_callback_(texture_id, w, h, output); + } else { + return false; + } +} + } // namespace testing } // namespace flutter diff --git a/shell/platform/embedder/tests/embedder_test_context_metal.h b/shell/platform/embedder/tests/embedder_test_context_metal.h index 788e7bfacfbc0..0e4653260e100 100644 --- a/shell/platform/embedder/tests/embedder_test_context_metal.h +++ b/shell/platform/embedder/tests/embedder_test_context_metal.h @@ -14,6 +14,12 @@ namespace testing { class EmbedderTestContextMetal : public EmbedderTestContext { public: + using TestExternalTextureCallback = + std::function; + explicit EmbedderTestContextMetal(std::string assets_path = ""); ~EmbedderTestContextMetal() override; @@ -27,14 +33,23 @@ class EmbedderTestContextMetal : public EmbedderTestContext { // |EmbedderTestContext| void SetupCompositor() override; + void SetExternalTextureCallback( + TestExternalTextureCallback external_texture_frame_callback); + bool Present(int64_t texture_id); + bool PopulateExternalTexture(int64_t texture_id, + size_t w, + size_t h, + FlutterMetalExternalTexture* output); + TestMetalContext* GetTestMetalContext(); private: // This allows the builder to access the hooks. friend class EmbedderConfigBuilder; + TestExternalTextureCallback external_texture_frame_callback_ = nullptr; SkISize surface_size_ = SkISize::MakeEmpty(); std::unique_ptr metal_context_; size_t present_count_ = 0; diff --git a/shell/platform/embedder/tests/embedder_unittests_metal.cc b/shell/platform/embedder/tests/embedder_unittests_metal.cc deleted file mode 100644 index a52822c86f284..0000000000000 --- a/shell/platform/embedder/tests/embedder_unittests_metal.cc +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright 2013 The Flutter Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#define FML_USED_ON_EMBEDDER - -#include -#include - -#include "embedder.h" -#include "flutter/shell/platform/embedder/tests/embedder_assertions.h" -#include "flutter/shell/platform/embedder/tests/embedder_config_builder.h" -#include "flutter/shell/platform/embedder/tests/embedder_test.h" -#include "flutter/shell/platform/embedder/tests/embedder_test_context_gl.h" -#include "flutter/shell/platform/embedder/tests/embedder_unittests_util.h" -#include "flutter/testing/assertions_skia.h" -#include "flutter/testing/testing.h" - -namespace flutter { -namespace testing { - -using EmbedderTest = testing::EmbedderTest; - -TEST_F(EmbedderTest, CanRenderGradientWithMetal) { - auto& context = GetEmbedderContext(EmbedderTestContextType::kMetalContext); - - EmbedderConfigBuilder builder(context); - - builder.SetDartEntrypoint("render_gradient"); - builder.SetMetalRendererConfig(SkISize::Make(800, 600)); - - auto rendered_scene = context.GetNextSceneImage(); - - auto engine = builder.LaunchEngine(); - ASSERT_TRUE(engine.is_valid()); - - // Send a window metrics events so frames may be scheduled. - FlutterWindowMetricsEvent event = {}; - event.struct_size = sizeof(event); - event.width = 800; - event.height = 600; - event.pixel_ratio = 1.0; - ASSERT_EQ(FlutterEngineSendWindowMetricsEvent(engine.get(), &event), - kSuccess); - - // TODO (https://github.com/flutter/flutter/issues/73590): re-enable once - // we are able to figure out why this fails on the bots. - // ASSERT_TRUE(ImageMatchesFixture("gradient_metal.png", rendered_scene)); -} - -} // namespace testing -} // namespace flutter diff --git a/shell/platform/embedder/tests/embedder_unittests_metal.mm b/shell/platform/embedder/tests/embedder_unittests_metal.mm new file mode 100644 index 0000000000000..49b761f65db29 --- /dev/null +++ b/shell/platform/embedder/tests/embedder_unittests_metal.mm @@ -0,0 +1,120 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#define FML_USED_ON_EMBEDDER + +#include +#include + +#import + +#include "embedder.h" +#include "flutter/shell/platform/embedder/tests/embedder_assertions.h" +#include "flutter/shell/platform/embedder/tests/embedder_config_builder.h" +#include "flutter/shell/platform/embedder/tests/embedder_test.h" +#include "flutter/shell/platform/embedder/tests/embedder_test_context_metal.h" +#include "flutter/shell/platform/embedder/tests/embedder_unittests_util.h" +#include "flutter/testing/assertions_skia.h" +#include "flutter/testing/testing.h" + +namespace flutter { +namespace testing { + +using EmbedderTest = testing::EmbedderTest; + +TEST_F(EmbedderTest, CanRenderGradientWithMetal) { + auto& context = GetEmbedderContext(EmbedderTestContextType::kMetalContext); + + EmbedderConfigBuilder builder(context); + + builder.SetDartEntrypoint("render_gradient"); + builder.SetMetalRendererConfig(SkISize::Make(800, 600)); + + auto rendered_scene = context.GetNextSceneImage(); + + auto engine = builder.LaunchEngine(); + ASSERT_TRUE(engine.is_valid()); + + // Send a window metrics events so frames may be scheduled. + FlutterWindowMetricsEvent event = {}; + event.struct_size = sizeof(event); + event.width = 800; + event.height = 600; + event.pixel_ratio = 1.0; + ASSERT_EQ(FlutterEngineSendWindowMetricsEvent(engine.get(), &event), kSuccess); + + // TODO (https://github.com/flutter/flutter/issues/73590): re-enable once + // we are able to figure out why this fails on the bots. + // ASSERT_TRUE(ImageMatchesFixture("gradient_metal.png", rendered_scene)); +} + +static sk_sp GetSurfaceFromTexture(sk_sp skia_context, + SkISize texture_size, + void* texture) { + GrMtlTextureInfo info; + info.fTexture.reset([(id)texture retain]); + GrBackendTexture backend_texture(texture_size.width(), texture_size.height(), GrMipmapped::kNo, + info); + + return SkSurface::MakeFromBackendTexture(skia_context.get(), backend_texture, + kTopLeft_GrSurfaceOrigin, 1, kBGRA_8888_SkColorType, + nullptr, nullptr); +} + +TEST_F(EmbedderTest, ExternalTextureMetal) { + EmbedderTestContextMetal& context = reinterpret_cast( + GetEmbedderContext(EmbedderTestContextType::kMetalContext)); + + const auto texture_size = SkISize::Make(800, 600); + const int64_t texture_id = 1; + + TestMetalContext* metal_context = context.GetTestMetalContext(); + TestMetalContext::TextureInfo texture_info = metal_context->CreateMetalTexture(texture_size); + + sk_sp surface = + GetSurfaceFromTexture(metal_context->GetSkiaContext(), texture_size, texture_info.texture); + auto canvas = surface->getCanvas(); + canvas->clear(SK_ColorRED); + metal_context->GetSkiaContext()->flushAndSubmit(); + + std::vector textures{texture_info.texture}; + + context.SetExternalTextureCallback( + [&](int64_t id, size_t w, size_t h, FlutterMetalExternalTexture* output) { + EXPECT_TRUE(w == texture_size.width()); + EXPECT_TRUE(h == texture_size.height()); + EXPECT_TRUE(texture_id == id); + output->num_textures = 1; + output->height = h; + output->width = w; + output->pixel_format = FlutterMetalExternalTexturePixelFormat::kRGBA; + output->textures = textures.data(); + return true; + }); + + EmbedderConfigBuilder builder(context); + + builder.SetDartEntrypoint("render_texture"); + builder.SetMetalRendererConfig(texture_size); + + auto engine = builder.LaunchEngine(); + ASSERT_TRUE(engine.is_valid()); + + ASSERT_EQ(FlutterEngineRegisterExternalTexture(engine.get(), texture_id), kSuccess); + + auto rendered_scene = context.GetNextSceneImage(); + + // Send a window metrics events so frames may be scheduled. + FlutterWindowMetricsEvent event = {}; + event.struct_size = sizeof(event); + event.width = texture_size.width(); + event.height = texture_size.height(); + event.pixel_ratio = 1.0; + ASSERT_EQ(FlutterEngineSendWindowMetricsEvent(engine.get(), &event), kSuccess); + + ASSERT_TRUE(ImageMatchesFixture("external_texture_metal.png", rendered_scene)); +} + +} // namespace testing +} // namespace flutter