Skip to content
This repository was archived by the owner on Feb 25, 2025. It is now read-only.

Commit 6a09435

Browse files
iskakaushikiskakaushik
committed
[embedder] [metal] Add embedder support for metal external textures
1 parent 5e7d329 commit 6a09435

12 files changed

+442
-102
lines changed

‎ci/licenses_golden/licenses_flutter‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1152,6 +1152,10 @@ FILE: ../../../flutter/shell/platform/embedder/embedder_engine.cc
11521152
FILE: ../../../flutter/shell/platform/embedder/embedder_engine.h
11531153
FILE: ../../../flutter/shell/platform/embedder/embedder_external_texture_gl.cc
11541154
FILE: ../../../flutter/shell/platform/embedder/embedder_external_texture_gl.h
1155+
FILE: ../../../flutter/shell/platform/embedder/embedder_external_texture_metal.h
1156+
FILE: ../../../flutter/shell/platform/embedder/embedder_external_texture_metal.mm
1157+
FILE: ../../../flutter/shell/platform/embedder/embedder_external_texture_resolver.cc
1158+
FILE: ../../../flutter/shell/platform/embedder/embedder_external_texture_resolver.h
11551159
FILE: ../../../flutter/shell/platform/embedder/embedder_external_view.cc
11561160
FILE: ../../../flutter/shell/platform/embedder/embedder_external_view.h
11571161
FILE: ../../../flutter/shell/platform/embedder/embedder_external_view_embedder.cc

‎shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.h‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,22 @@
77

88
#import "flutter/shell/platform/darwin/common/framework/Headers/FlutterTexture.h"
99
#include "third_party/skia/include/core/SkCanvas.h"
10-
#include "third_party/skia/include/gpu/GrDirectContext.h"
10+
#include "third_party/skia/include/core/SkImage.h"
11+
12+
@interface FlutterDarwinExternalTextureSkImageWrapper : NSObject
13+
14+
+ (sk_sp<SkImage>)wrapYUVATexture:(nonnull id<MTLTexture>)yTex
15+
UVTex:(nonnull id<MTLTexture>)uvTex
16+
grContext:(nonnull GrDirectContext*)grContext
17+
width:(size_t)width
18+
height:(size_t)height;
19+
20+
+ (sk_sp<SkImage>)wrapRGBATexture:(nonnull id<MTLTexture>)rgbaTex
21+
grContext:(nonnull GrDirectContext*)grContext
22+
width:(size_t)width
23+
height:(size_t)height;
24+
25+
@end
1126

1227
@interface FlutterDarwinExternalTextureMetal : NSObject
1328

‎shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.mm‎

Lines changed: 63 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -14,17 +14,6 @@
1414

1515
FLUTTER_ASSERT_ARC
1616

17-
namespace {
18-
19-
static sk_cf_obj<const void*> SkiaTextureFromCVMetalTexture(CVMetalTextureRef cvMetalTexture) {
20-
id<MTLTexture> texture = CVMetalTextureGetTexture(cvMetalTexture);
21-
// CVMetal texture can be released as soon as we can the MTLTexture from it.
22-
CVPixelBufferRelease(cvMetalTexture);
23-
return sk_cf_obj<const void*>{(__bridge_retained const void*)texture};
24-
}
25-
26-
}
27-
2817
@implementation FlutterDarwinExternalTextureMetal {
2918
CVMetalTextureCacheRef _textureCache;
3019
NSObject<FlutterTexture>* _externalTexture;
@@ -185,31 +174,17 @@ - (void)onTextureUnregistered {
185174
}
186175
}
187176

188-
GrMtlTextureInfo ySkiaTextureInfo;
189-
ySkiaTextureInfo.fTexture = SkiaTextureFromCVMetalTexture(yMetalTexture);
177+
id<MTLTexture> yTex = CVMetalTextureGetTexture(yMetalTexture);
178+
CVBufferRelease(yMetalTexture);
190179

191-
GrBackendTexture skiaBackendTextures[2];
192-
skiaBackendTextures[0] = GrBackendTexture(/*width=*/textureSize.width(),
193-
/*height=*/textureSize.height(),
194-
/*mipMapped=*/GrMipMapped ::kNo,
195-
/*textureInfo=*/ySkiaTextureInfo);
180+
id<MTLTexture> uvTex = CVMetalTextureGetTexture(uvMetalTexture);
181+
CVBufferRelease(uvMetalTexture);
196182

197-
GrMtlTextureInfo uvSkiaTextureInfo;
198-
uvSkiaTextureInfo.fTexture = SkiaTextureFromCVMetalTexture(uvMetalTexture);
199-
200-
skiaBackendTextures[1] = GrBackendTexture(/*width=*/textureSize.width(),
201-
/*height=*/textureSize.height(),
202-
/*mipMapped=*/GrMipMapped ::kNo,
203-
/*textureInfo=*/uvSkiaTextureInfo);
204-
SkYUVAInfo yuvaInfo(skiaBackendTextures[0].dimensions(), SkYUVAInfo::PlaneConfig::kY_UV,
205-
SkYUVAInfo::Subsampling::k444, kRec601_SkYUVColorSpace);
206-
GrYUVABackendTextures yuvaBackendTextures(yuvaInfo, skiaBackendTextures,
207-
kTopLeft_GrSurfaceOrigin);
208-
209-
sk_sp<SkImage> image =
210-
SkImage::MakeFromYUVATextures(grContext, yuvaBackendTextures, /*imageColorSpace=*/nullptr,
211-
/*releaseProc*/ nullptr, /*releaseContext*/ nullptr);
212-
return image;
183+
return [FlutterDarwinExternalTextureSkImageWrapper wrapYUVATexture:yTex
184+
UVTex:uvTex
185+
grContext:grContext
186+
width:textureSize.width()
187+
height:textureSize.height()];
213188
}
214189

215190
- (sk_sp<SkImage>)wrapRGBAExternalPixelBuffer:(CVPixelBufferRef)pixelBuffer
@@ -233,22 +208,64 @@ GrYUVABackendTextures yuvaBackendTextures(yuvaInfo, skiaBackendTextures,
233208
return nullptr;
234209
}
235210

211+
id<MTLTexture> rgbaTex = CVMetalTextureGetTexture(metalTexture);
212+
CVBufferRelease(metalTexture);
213+
214+
return [FlutterDarwinExternalTextureSkImageWrapper wrapRGBATexture:rgbaTex
215+
grContext:grContext
216+
width:textureSize.width()
217+
height:textureSize.height()];
218+
}
219+
220+
@end
221+
222+
@implementation FlutterDarwinExternalTextureSkImageWrapper
223+
224+
+ (sk_sp<SkImage>)wrapYUVATexture:(id<MTLTexture>)yTex
225+
UVTex:(id<MTLTexture>)uvTex
226+
grContext:(nonnull GrDirectContext*)grContext
227+
width:(size_t)width
228+
height:(size_t)height {
229+
GrMtlTextureInfo ySkiaTextureInfo;
230+
ySkiaTextureInfo.fTexture = sk_cf_obj<const void*>{(__bridge_retained const void*)yTex};
231+
232+
GrBackendTexture skiaBackendTextures[2];
233+
skiaBackendTextures[0] = GrBackendTexture(/*width=*/width,
234+
/*height=*/height,
235+
/*mipMapped=*/GrMipMapped::kNo,
236+
/*textureInfo=*/ySkiaTextureInfo);
237+
238+
GrMtlTextureInfo uvSkiaTextureInfo;
239+
uvSkiaTextureInfo.fTexture = sk_cf_obj<const void*>{(__bridge_retained const void*)uvTex};
240+
241+
skiaBackendTextures[1] = GrBackendTexture(/*width=*/width,
242+
/*height=*/height,
243+
/*mipMapped=*/GrMipMapped::kNo,
244+
/*textureInfo=*/uvSkiaTextureInfo);
245+
SkYUVAInfo yuvaInfo(skiaBackendTextures[0].dimensions(), SkYUVAInfo::PlaneConfig::kY_UV,
246+
SkYUVAInfo::Subsampling::k444, kRec601_SkYUVColorSpace);
247+
GrYUVABackendTextures yuvaBackendTextures(yuvaInfo, skiaBackendTextures,
248+
kTopLeft_GrSurfaceOrigin);
249+
250+
return SkImage::MakeFromYUVATextures(grContext, yuvaBackendTextures, /*imageColorSpace=*/nullptr,
251+
/*releaseProc*/ nullptr, /*releaseContext*/ nullptr);
252+
}
253+
254+
+ (sk_sp<SkImage>)wrapRGBATexture:(id<MTLTexture>)rgbaTex
255+
grContext:(nonnull GrDirectContext*)grContext
256+
width:(size_t)width
257+
height:(size_t)height {
236258
GrMtlTextureInfo skiaTextureInfo;
237-
skiaTextureInfo.fTexture = SkiaTextureFromCVMetalTexture(metalTexture);
259+
skiaTextureInfo.fTexture = sk_cf_obj<const void*>{(__bridge_retained const void*)rgbaTex};
238260

239-
GrBackendTexture skiaBackendTexture(/*width=*/textureSize.width(),
240-
/*height=*/textureSize.height(),
261+
GrBackendTexture skiaBackendTexture(/*width=*/width,
262+
/*height=*/height,
241263
/*mipMapped=*/GrMipMapped ::kNo,
242264
/*textureInfo=*/skiaTextureInfo);
243265

244-
sk_sp<SkImage> image =
245-
SkImage::MakeFromTexture(grContext, skiaBackendTexture, kTopLeft_GrSurfaceOrigin,
246-
kBGRA_8888_SkColorType, kPremul_SkAlphaType,
247-
/*imageColorSpace=*/nullptr, /*releaseProc*/ nullptr,
248-
/*releaseContext*/ nullptr
249-
250-
);
251-
return image;
266+
return SkImage::MakeFromTexture(grContext, skiaBackendTexture, kTopLeft_GrSurfaceOrigin,
267+
kBGRA_8888_SkColorType, kPremul_SkAlphaType,
268+
/*imageColorSpace=*/nullptr, /*releaseProc*/ nullptr,
269+
/*releaseContext*/ nullptr);
252270
}
253-
254271
@end

‎shell/platform/embedder/BUILD.gn‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ template("embedder_source_set") {
3434
"embedder.cc",
3535
"embedder_engine.cc",
3636
"embedder_engine.h",
37+
"embedder_external_texture_resolver.cc",
38+
"embedder_external_texture_resolver.h",
3739
"embedder_external_view.cc",
3840
"embedder_external_view.h",
3941
"embedder_external_view_embedder.cc",
@@ -90,6 +92,8 @@ template("embedder_source_set") {
9092

9193
if (embedder_enable_metal) {
9294
sources += [
95+
"embedder_external_texture_metal.h",
96+
"embedder_external_texture_metal.mm",
9397
"embedder_surface_metal.h",
9498
"embedder_surface_metal.mm",
9599
]

‎shell/platform/embedder/embedder.cc‎

Lines changed: 36 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ extern const intptr_t kPlatformStrongDillSize;
4848
#include "flutter/shell/common/switches.h"
4949
#include "flutter/shell/platform/embedder/embedder.h"
5050
#include "flutter/shell/platform/embedder/embedder_engine.h"
51+
#include "flutter/shell/platform/embedder/embedder_external_texture_resolver.h"
5152
#include "flutter/shell/platform/embedder/embedder_platform_message_response.h"
5253
#include "flutter/shell/platform/embedder/embedder_render_target.h"
5354
#include "flutter/shell/platform/embedder/embedder_struct_macros.h"
@@ -1135,9 +1136,11 @@ FlutterEngineResult FlutterEngineInitialize(size_t version,
11351136
return std::make_unique<flutter::Rasterizer>(shell);
11361137
};
11371138

1139+
using ExternalTextureResolver = flutter::EmbedderExternalTextureResolver;
1140+
std::unique_ptr<ExternalTextureResolver> external_texture_resolver;
1141+
external_texture_resolver = std::make_unique<ExternalTextureResolver>();
1142+
11381143
#ifdef SHELL_ENABLE_GL
1139-
// TODO(chinmaygarde): This is the wrong spot for this. It belongs in the
1140-
// platform view jump table.
11411144
flutter::EmbedderExternalTextureGL::ExternalTextureCallback
11421145
external_texture_callback;
11431146
if (config->type == kOpenGL) {
@@ -1156,6 +1159,30 @@ FlutterEngineResult FlutterEngineInitialize(size_t version,
11561159
};
11571160
}
11581161
}
1162+
external_texture_resolver =
1163+
std::make_unique<ExternalTextureResolver>(external_texture_callback);
1164+
#endif
1165+
#ifdef SHELL_ENABLE_METAL
1166+
flutter::EmbedderExternalTextureMetal::ExternalTextureCallback
1167+
external_texture_metal_callback;
1168+
if (config->type == kMetal) {
1169+
const FlutterMetalRendererConfig* metal_config = &config->metal;
1170+
if (SAFE_ACCESS(metal_config, external_texture_frame_callback, nullptr)) {
1171+
external_texture_metal_callback =
1172+
[ptr = metal_config->external_texture_frame_callback, user_data](
1173+
int64_t texture_identifier, size_t width,
1174+
size_t height) -> std::unique_ptr<FlutterMetalExternalTexture> {
1175+
FlutterMetalExternalTexture* texture =
1176+
new FlutterMetalExternalTexture();
1177+
if (!ptr(user_data, texture_identifier, width, height, texture)) {
1178+
return nullptr;
1179+
}
1180+
return std::unique_ptr<FlutterMetalExternalTexture>(texture);
1181+
};
1182+
}
1183+
}
1184+
external_texture_resolver =
1185+
std::make_unique<ExternalTextureResolver>(external_texture_callback);
11591186
#endif
11601187

11611188
auto thread_host =
@@ -1207,16 +1234,13 @@ FlutterEngineResult FlutterEngineInitialize(size_t version,
12071234

12081235
// Create the engine but don't launch the shell or run the root isolate.
12091236
auto embedder_engine = std::make_unique<flutter::EmbedderEngine>(
1210-
std::move(thread_host), //
1211-
std::move(task_runners), //
1212-
std::move(settings), //
1213-
std::move(run_configuration), //
1214-
on_create_platform_view, //
1215-
on_create_rasterizer //
1216-
#ifdef SHELL_ENABLE_GL
1217-
,
1218-
external_texture_callback //
1219-
#endif
1237+
std::move(thread_host), //
1238+
std::move(task_runners), //
1239+
std::move(settings), //
1240+
std::move(run_configuration), //
1241+
on_create_platform_view, //
1242+
on_create_rasterizer, //
1243+
std::move(external_texture_resolver) //
12201244
);
12211245

12221246
// Release the ownership of the embedder engine to the caller.

‎shell/platform/embedder/embedder.h‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -447,6 +447,40 @@ typedef const void* FlutterMetalCommandQueueHandle;
447447
/// Alias for id<MTLTexture>.
448448
typedef const void* FlutterMetalTextureHandle;
449449

450+
/// Pixel format for the external texture.
451+
typedef enum {
452+
kYUVA,
453+
kRGBA,
454+
} FlutterMetalExternalTexturePixelFormat;
455+
456+
typedef struct {
457+
/// The size of this struct. Must be sizeof(FlutterMetalExternalTexture).
458+
size_t struct_size;
459+
/// Height of the texture.
460+
size_t width;
461+
/// Height of the texture.
462+
size_t height;
463+
/// The pixel format type of the external.
464+
FlutterMetalExternalTexturePixelFormat pixel_format;
465+
/// Represents the size of the `textures` array.
466+
size_t num_textures;
467+
/// Supported textures are YUVA and RGBA, in case of YUVA we expect 2 texture
468+
/// handles to be provided by the embedder, Y first and UV next. In case of
469+
/// RGBA only one should be passed.
470+
/// These are individually aliases for id<MTLTexture> which will be released
471+
/// once they have been composited: `external_texture_frame_callback`.
472+
FlutterMetalTextureHandle* textures;
473+
} FlutterMetalExternalTexture;
474+
475+
/// Callback to provide an external texture for a given texture_id.
476+
/// See: external_texture_frame_callback.
477+
typedef bool (*FlutterMetalTextureFrameCallback)(
478+
void* /* user data */,
479+
int64_t /* texture identifier */,
480+
size_t /* width */,
481+
size_t /* height */,
482+
FlutterMetalExternalTexture* /* texture out */);
483+
450484
typedef struct {
451485
/// The size of this struct. Must be sizeof(FlutterMetalTexture).
452486
size_t struct_size;
@@ -485,6 +519,11 @@ typedef struct {
485519
/// The callback presented to the embedder to present a fully populated metal
486520
/// texture to the user.
487521
FlutterMetalPresentCallback present_drawable_callback;
522+
/// When the embedder specifies that a texture has a frame available, the
523+
/// engine will call this method (on an internal engine managed thread) so
524+
/// that external texture details can be supplied to the engine for subsequent
525+
/// composition.
526+
FlutterMetalTextureFrameCallback external_texture_frame_callback;
488527
} FlutterMetalRendererConfig;
489528

490529
typedef struct {

‎shell/platform/embedder/embedder_engine.cc‎

Lines changed: 8 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -27,24 +27,15 @@ EmbedderEngine::EmbedderEngine(
2727
flutter::Settings settings,
2828
RunConfiguration run_configuration,
2929
Shell::CreateCallback<PlatformView> on_create_platform_view,
30-
Shell::CreateCallback<Rasterizer> on_create_rasterizer
31-
#ifdef SHELL_ENABLE_GL
32-
,
33-
EmbedderExternalTextureGL::ExternalTextureCallback external_texture_callback
34-
#endif
35-
)
30+
Shell::CreateCallback<Rasterizer> on_create_rasterizer,
31+
std::unique_ptr<EmbedderExternalTextureResolver> external_texture_resolver)
3632
: thread_host_(std::move(thread_host)),
3733
task_runners_(task_runners),
3834
run_configuration_(std::move(run_configuration)),
3935
shell_args_(std::make_unique<ShellArgs>(std::move(settings),
4036
on_create_platform_view,
41-
on_create_rasterizer))
42-
#ifdef SHELL_ENABLE_GL
43-
,
44-
external_texture_callback_(external_texture_callback)
45-
#endif
46-
{
47-
}
37+
on_create_rasterizer)),
38+
external_texture_resolver_(std::move(external_texture_resolver)) {}
4839

4940
EmbedderEngine::~EmbedderEngine() = default;
5041

@@ -168,37 +159,27 @@ bool EmbedderEngine::SendPlatformMessage(
168159
}
169160

170161
bool EmbedderEngine::RegisterTexture(int64_t texture) {
171-
#ifdef SHELL_ENABLE_GL
172-
if (!IsValid() || !external_texture_callback_) {
162+
if (!IsValid()) {
173163
return false;
174164
}
175165
shell_->GetPlatformView()->RegisterTexture(
176-
std::make_unique<EmbedderExternalTextureGL>(texture,
177-
external_texture_callback_));
178-
#endif
179-
166+
external_texture_resolver_->ResolveExternalTexture(texture));
180167
return true;
181168
}
182169

183170
bool EmbedderEngine::UnregisterTexture(int64_t texture) {
184-
#ifdef SHELL_ENABLE_GL
185-
if (!IsValid() || !external_texture_callback_) {
171+
if (!IsValid()) {
186172
return false;
187173
}
188174
shell_->GetPlatformView()->UnregisterTexture(texture);
189-
#endif
190-
191175
return true;
192176
}
193177

194178
bool EmbedderEngine::MarkTextureFrameAvailable(int64_t texture) {
195-
#ifdef SHELL_ENABLE_GL
196-
if (!IsValid() || !external_texture_callback_) {
179+
if (!IsValid()) {
197180
return false;
198181
}
199182
shell_->GetPlatformView()->MarkTextureFrameAvailable(texture);
200-
#endif
201-
202183
return true;
203184
}
204185

0 commit comments

Comments
 (0)