Skip to content
This repository was archived by the owner on Feb 25, 2025. It is now read-only.

Commit 4fe5381

Browse files
iskakaushikiskakaushik
committed
[embedder] [metal] Add embedder support for metal external textures
1 parent 484e2d7 commit 4fe5381

11 files changed

+438
-94
lines changed

‎shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.h‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,22 @@
77

88
#import "flutter/shell/platform/darwin/common/framework/Headers/FlutterTexture.h"
99
#include "third_party/skia/include/core/SkCanvas.h"
10-
#include "third_party/skia/include/gpu/GrDirectContext.h"
10+
#include "third_party/skia/include/core/SkImage.h"
11+
12+
@interface FlutterDarwinExternalTextureSkImageWrapper : NSObject
13+
14+
+ (sk_sp<SkImage>)wrapYUVATexture:(nonnull id<MTLTexture>)yTex
15+
UVTex:(nonnull id<MTLTexture>)uvTex
16+
grContext:(nonnull GrDirectContext*)grContext
17+
width:(size_t)width
18+
height:(size_t)height;
19+
20+
+ (sk_sp<SkImage>)wrapRGBATexture:(nonnull id<MTLTexture>)rgbaTex
21+
grContext:(nonnull GrDirectContext*)grContext
22+
width:(size_t)width
23+
height:(size_t)height;
24+
25+
@end
1126

1227
@interface FlutterDarwinExternalTextureMetal : NSObject
1328

‎shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.mm‎

Lines changed: 63 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -173,33 +173,17 @@ - (void)onTextureUnregistered {
173173
}
174174
}
175175

176-
GrMtlTextureInfo ySkiaTextureInfo;
177-
ySkiaTextureInfo.fTexture = sk_cf_obj<const void*>{
178-
(__bridge_retained const void*)CVMetalTextureGetTexture(yMetalTexture)};
176+
id<MTLTexture> yTex = CVMetalTextureGetTexture(yMetalTexture);
177+
CVBufferRelease(yMetalTexture);
179178

180-
GrBackendTexture skiaBackendTextures[2];
181-
skiaBackendTextures[0] = GrBackendTexture(/*width=*/textureSize.width(),
182-
/*height=*/textureSize.height(),
183-
/*mipMapped=*/GrMipMapped ::kNo,
184-
/*textureInfo=*/ySkiaTextureInfo);
185-
186-
GrMtlTextureInfo uvSkiaTextureInfo;
187-
uvSkiaTextureInfo.fTexture = sk_cf_obj<const void*>{
188-
(__bridge_retained const void*)CVMetalTextureGetTexture(uvMetalTexture)};
189-
190-
skiaBackendTextures[1] = GrBackendTexture(/*width=*/textureSize.width(),
191-
/*height=*/textureSize.height(),
192-
/*mipMapped=*/GrMipMapped ::kNo,
193-
/*textureInfo=*/uvSkiaTextureInfo);
194-
SkYUVAInfo yuvaInfo(skiaBackendTextures[0].dimensions(), SkYUVAInfo::PlaneConfig::kY_UV,
195-
SkYUVAInfo::Subsampling::k444, kRec601_SkYUVColorSpace);
196-
GrYUVABackendTextures yuvaBackendTextures(yuvaInfo, skiaBackendTextures,
197-
kTopLeft_GrSurfaceOrigin);
179+
id<MTLTexture> uvTex = CVMetalTextureGetTexture(uvMetalTexture);
180+
CVBufferRelease(uvMetalTexture);
198181

199-
sk_sp<SkImage> image =
200-
SkImage::MakeFromYUVATextures(grContext, yuvaBackendTextures, /*imageColorSpace=*/nullptr,
201-
/*releaseProc*/ nullptr, /*releaseContext*/ nullptr);
202-
return image;
182+
return [FlutterDarwinExternalTextureSkImageWrapper wrapYUVATexture:yTex
183+
UVTex:uvTex
184+
grContext:grContext
185+
width:textureSize.width()
186+
height:textureSize.height()];
203187
}
204188

205189
- (sk_sp<SkImage>)wrapRGBAExternalPixelBuffer:(CVPixelBufferRef)pixelBuffer
@@ -223,23 +207,64 @@ GrYUVABackendTextures yuvaBackendTextures(yuvaInfo, skiaBackendTextures,
223207
return nullptr;
224208
}
225209

210+
id<MTLTexture> rgbaTex = CVMetalTextureGetTexture(metalTexture);
211+
CVBufferRelease(metalTexture);
212+
213+
return [FlutterDarwinExternalTextureSkImageWrapper wrapRGBATexture:rgbaTex
214+
grContext:grContext
215+
width:textureSize.width()
216+
height:textureSize.height()];
217+
}
218+
219+
@end
220+
221+
@implementation FlutterDarwinExternalTextureSkImageWrapper
222+
223+
+ (sk_sp<SkImage>)wrapYUVATexture:(id<MTLTexture>)yTex
224+
UVTex:(id<MTLTexture>)uvTex
225+
grContext:(nonnull GrDirectContext*)grContext
226+
width:(size_t)width
227+
height:(size_t)height {
228+
GrMtlTextureInfo ySkiaTextureInfo;
229+
ySkiaTextureInfo.fTexture = sk_cf_obj<const void*>{(__bridge_retained const void*)yTex};
230+
231+
GrBackendTexture skiaBackendTextures[2];
232+
skiaBackendTextures[0] = GrBackendTexture(/*width=*/width,
233+
/*height=*/height,
234+
/*mipMapped=*/GrMipMapped::kNo,
235+
/*textureInfo=*/ySkiaTextureInfo);
236+
237+
GrMtlTextureInfo uvSkiaTextureInfo;
238+
uvSkiaTextureInfo.fTexture = sk_cf_obj<const void*>{(__bridge_retained const void*)uvTex};
239+
240+
skiaBackendTextures[1] = GrBackendTexture(/*width=*/width,
241+
/*height=*/height,
242+
/*mipMapped=*/GrMipMapped::kNo,
243+
/*textureInfo=*/uvSkiaTextureInfo);
244+
SkYUVAInfo yuvaInfo(skiaBackendTextures[0].dimensions(), SkYUVAInfo::PlaneConfig::kY_UV,
245+
SkYUVAInfo::Subsampling::k444, kRec601_SkYUVColorSpace);
246+
GrYUVABackendTextures yuvaBackendTextures(yuvaInfo, skiaBackendTextures,
247+
kTopLeft_GrSurfaceOrigin);
248+
249+
return SkImage::MakeFromYUVATextures(grContext, yuvaBackendTextures, /*imageColorSpace=*/nullptr,
250+
/*releaseProc*/ nullptr, /*releaseContext*/ nullptr);
251+
}
252+
253+
+ (sk_sp<SkImage>)wrapRGBATexture:(id<MTLTexture>)rgbaTex
254+
grContext:(nonnull GrDirectContext*)grContext
255+
width:(size_t)width
256+
height:(size_t)height {
226257
GrMtlTextureInfo skiaTextureInfo;
227-
skiaTextureInfo.fTexture =
228-
sk_cf_obj<const void*>{(__bridge_retained const void*)CVMetalTextureGetTexture(metalTexture)};
258+
skiaTextureInfo.fTexture = sk_cf_obj<const void*>{(__bridge_retained const void*)rgbaTex};
229259

230-
GrBackendTexture skiaBackendTexture(/*width=*/textureSize.width(),
231-
/*height=*/textureSize.height(),
260+
GrBackendTexture skiaBackendTexture(/*width=*/width,
261+
/*height=*/height,
232262
/*mipMapped=*/GrMipMapped ::kNo,
233263
/*textureInfo=*/skiaTextureInfo);
234264

235-
sk_sp<SkImage> image =
236-
SkImage::MakeFromTexture(grContext, skiaBackendTexture, kTopLeft_GrSurfaceOrigin,
237-
kBGRA_8888_SkColorType, kPremul_SkAlphaType,
238-
/*imageColorSpace=*/nullptr, /*releaseProc*/ nullptr,
239-
/*releaseContext*/ nullptr
240-
241-
);
242-
return image;
265+
return SkImage::MakeFromTexture(grContext, skiaBackendTexture, kTopLeft_GrSurfaceOrigin,
266+
kBGRA_8888_SkColorType, kPremul_SkAlphaType,
267+
/*imageColorSpace=*/nullptr, /*releaseProc*/ nullptr,
268+
/*releaseContext*/ nullptr);
243269
}
244-
245270
@end

‎shell/platform/embedder/BUILD.gn‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ template("embedder_source_set") {
3434
"embedder.cc",
3535
"embedder_engine.cc",
3636
"embedder_engine.h",
37+
"embedder_external_texture_resolver.cc",
38+
"embedder_external_texture_resolver.h",
3739
"embedder_external_view.cc",
3840
"embedder_external_view.h",
3941
"embedder_external_view_embedder.cc",
@@ -90,6 +92,8 @@ template("embedder_source_set") {
9092

9193
if (embedder_enable_metal) {
9294
sources += [
95+
"embedder_external_texture_metal.h",
96+
"embedder_external_texture_metal.mm",
9397
"embedder_surface_metal.h",
9498
"embedder_surface_metal.mm",
9599
]

‎shell/platform/embedder/embedder.cc‎

Lines changed: 36 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ extern const intptr_t kPlatformStrongDillSize;
4848
#include "flutter/shell/common/switches.h"
4949
#include "flutter/shell/platform/embedder/embedder.h"
5050
#include "flutter/shell/platform/embedder/embedder_engine.h"
51+
#include "flutter/shell/platform/embedder/embedder_external_texture_resolver.h"
5152
#include "flutter/shell/platform/embedder/embedder_platform_message_response.h"
5253
#include "flutter/shell/platform/embedder/embedder_render_target.h"
5354
#include "flutter/shell/platform/embedder/embedder_struct_macros.h"
@@ -1135,9 +1136,11 @@ FlutterEngineResult FlutterEngineInitialize(size_t version,
11351136
return std::make_unique<flutter::Rasterizer>(shell);
11361137
};
11371138

1139+
using ExternalTextureResolver = flutter::EmbedderExternalTextureResolver;
1140+
std::unique_ptr<ExternalTextureResolver> external_texture_resolver;
1141+
external_texture_resolver = std::make_unique<ExternalTextureResolver>();
1142+
11381143
#ifdef SHELL_ENABLE_GL
1139-
// TODO(chinmaygarde): This is the wrong spot for this. It belongs in the
1140-
// platform view jump table.
11411144
flutter::EmbedderExternalTextureGL::ExternalTextureCallback
11421145
external_texture_callback;
11431146
if (config->type == kOpenGL) {
@@ -1156,6 +1159,30 @@ FlutterEngineResult FlutterEngineInitialize(size_t version,
11561159
};
11571160
}
11581161
}
1162+
external_texture_resolver =
1163+
std::make_unique<ExternalTextureResolver>(external_texture_callback);
1164+
#endif
1165+
#ifdef SHELL_ENABLE_METAL
1166+
flutter::EmbedderExternalTextureMetal::ExternalTextureCallback
1167+
external_texture_metal_callback;
1168+
if (config->type == kMetal) {
1169+
const FlutterMetalRendererConfig* metal_config = &config->metal;
1170+
if (SAFE_ACCESS(metal_config, external_texture_frame_callback, nullptr)) {
1171+
external_texture_metal_callback =
1172+
[ptr = metal_config->external_texture_frame_callback, user_data](
1173+
int64_t texture_identifier, size_t width,
1174+
size_t height) -> std::unique_ptr<FlutterMetalExternalTexture> {
1175+
FlutterMetalExternalTexture* texture =
1176+
new FlutterMetalExternalTexture();
1177+
if (!ptr(user_data, texture_identifier, width, height, texture)) {
1178+
return nullptr;
1179+
}
1180+
return std::unique_ptr<FlutterMetalExternalTexture>(texture);
1181+
};
1182+
}
1183+
}
1184+
external_texture_resolver =
1185+
std::make_unique<ExternalTextureResolver>(external_texture_callback);
11591186
#endif
11601187

11611188
auto thread_host =
@@ -1207,16 +1234,13 @@ FlutterEngineResult FlutterEngineInitialize(size_t version,
12071234

12081235
// Create the engine but don't launch the shell or run the root isolate.
12091236
auto embedder_engine = std::make_unique<flutter::EmbedderEngine>(
1210-
std::move(thread_host), //
1211-
std::move(task_runners), //
1212-
std::move(settings), //
1213-
std::move(run_configuration), //
1214-
on_create_platform_view, //
1215-
on_create_rasterizer //
1216-
#ifdef SHELL_ENABLE_GL
1217-
,
1218-
external_texture_callback //
1219-
#endif
1237+
std::move(thread_host), //
1238+
std::move(task_runners), //
1239+
std::move(settings), //
1240+
std::move(run_configuration), //
1241+
on_create_platform_view, //
1242+
on_create_rasterizer, //
1243+
std::move(external_texture_resolver) //
12201244
);
12211245

12221246
// Release the ownership of the embedder engine to the caller.

‎shell/platform/embedder/embedder.h‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -447,6 +447,40 @@ typedef const void* FlutterMetalCommandQueueHandle;
447447
/// Alias for id<MTLTexture>.
448448
typedef const void* FlutterMetalTextureHandle;
449449

450+
/// Pixel format for the external texture.
451+
typedef enum {
452+
kYUVA,
453+
kRGBA,
454+
} FlutterMetalExternalTexturePixelFormat;
455+
456+
typedef struct {
457+
/// The size of this struct. Must be sizeof(FlutterMetalExternalTexture).
458+
size_t struct_size;
459+
/// Height of the texture.
460+
size_t width;
461+
/// Height of the texture.
462+
size_t height;
463+
/// The pixel format type of the external.
464+
FlutterMetalExternalTexturePixelFormat pixel_format;
465+
/// Represents the size of the `textures` array.
466+
size_t num_textures;
467+
/// Supported textures are YUVA and RGBA, in case of YUVA we expect 2 texture
468+
/// handles to be provided by the embedder, Y first and UV next. In case of
469+
/// RGBA only one should be passed.
470+
/// These are individually aliases for id<MTLTexture> which will be released
471+
/// once they have been composited: `external_texture_frame_callback`.
472+
FlutterMetalTextureHandle* textures;
473+
} FlutterMetalExternalTexture;
474+
475+
/// Callback to provide an external texture for a given texture_id.
476+
/// See: external_texture_frame_callback.
477+
typedef bool (*FlutterMetalTextureFrameCallback)(
478+
void* /* user data */,
479+
int64_t /* texture identifier */,
480+
size_t /* width */,
481+
size_t /* height */,
482+
FlutterMetalExternalTexture* /* texture out */);
483+
450484
typedef struct {
451485
/// The size of this struct. Must be sizeof(FlutterMetalTexture).
452486
size_t struct_size;
@@ -485,6 +519,11 @@ typedef struct {
485519
/// The callback presented to the embedder to present a fully populated metal
486520
/// texture to the user.
487521
FlutterMetalPresentCallback present_drawable_callback;
522+
/// When the embedder specifies that a texture has a frame available, the
523+
/// engine will call this method (on an internal engine managed thread) so
524+
/// that external texture details can be supplied to the engine for subsequent
525+
/// composition.
526+
FlutterMetalTextureFrameCallback external_texture_frame_callback;
488527
} FlutterMetalRendererConfig;
489528

490529
typedef struct {

‎shell/platform/embedder/embedder_engine.cc‎

Lines changed: 8 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -27,24 +27,15 @@ EmbedderEngine::EmbedderEngine(
2727
flutter::Settings settings,
2828
RunConfiguration run_configuration,
2929
Shell::CreateCallback<PlatformView> on_create_platform_view,
30-
Shell::CreateCallback<Rasterizer> on_create_rasterizer
31-
#ifdef SHELL_ENABLE_GL
32-
,
33-
EmbedderExternalTextureGL::ExternalTextureCallback external_texture_callback
34-
#endif
35-
)
30+
Shell::CreateCallback<Rasterizer> on_create_rasterizer,
31+
std::unique_ptr<EmbedderExternalTextureResolver> external_texture_resolver)
3632
: thread_host_(std::move(thread_host)),
3733
task_runners_(task_runners),
3834
run_configuration_(std::move(run_configuration)),
3935
shell_args_(std::make_unique<ShellArgs>(std::move(settings),
4036
on_create_platform_view,
41-
on_create_rasterizer))
42-
#ifdef SHELL_ENABLE_GL
43-
,
44-
external_texture_callback_(external_texture_callback)
45-
#endif
46-
{
47-
}
37+
on_create_rasterizer)),
38+
external_texture_resolver_(std::move(external_texture_resolver)) {}
4839

4940
EmbedderEngine::~EmbedderEngine() = default;
5041

@@ -168,37 +159,27 @@ bool EmbedderEngine::SendPlatformMessage(
168159
}
169160

170161
bool EmbedderEngine::RegisterTexture(int64_t texture) {
171-
#ifdef SHELL_ENABLE_GL
172-
if (!IsValid() || !external_texture_callback_) {
162+
if (!IsValid()) {
173163
return false;
174164
}
175165
shell_->GetPlatformView()->RegisterTexture(
176-
std::make_unique<EmbedderExternalTextureGL>(texture,
177-
external_texture_callback_));
178-
#endif
179-
166+
external_texture_resolver_->ResolveExternalTexture(texture));
180167
return true;
181168
}
182169

183170
bool EmbedderEngine::UnregisterTexture(int64_t texture) {
184-
#ifdef SHELL_ENABLE_GL
185-
if (!IsValid() || !external_texture_callback_) {
171+
if (!IsValid()) {
186172
return false;
187173
}
188174
shell_->GetPlatformView()->UnregisterTexture(texture);
189-
#endif
190-
191175
return true;
192176
}
193177

194178
bool EmbedderEngine::MarkTextureFrameAvailable(int64_t texture) {
195-
#ifdef SHELL_ENABLE_GL
196-
if (!IsValid() || !external_texture_callback_) {
179+
if (!IsValid()) {
197180
return false;
198181
}
199182
shell_->GetPlatformView()->MarkTextureFrameAvailable(texture);
200-
#endif
201-
202183
return true;
203184
}
204185

0 commit comments

Comments
 (0)