[embedder] [metal] Embedder API can support externally composited Metal textures

ci/licenses_golden/licenses_flutter

@@ -1099,6 +1099,7 @@ FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterBacki
 FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterBackingStoreData.mm
 FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterDartProject.mm
 FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterDartProject_Internal.h
+FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterEmbedderExternalTextureUnittests.mm
 FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterEngine.mm
 FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterEngineTest.mm
 FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterEngine_Internal.h
@@ -1152,6 +1153,10 @@ FILE: ../../../flutter/shell/platform/embedder/embedder_engine.cc
 FILE: ../../../flutter/shell/platform/embedder/embedder_engine.h
 FILE: ../../../flutter/shell/platform/embedder/embedder_external_texture_gl.cc
 FILE: ../../../flutter/shell/platform/embedder/embedder_external_texture_gl.h
+FILE: ../../../flutter/shell/platform/embedder/embedder_external_texture_metal.h
+FILE: ../../../flutter/shell/platform/embedder/embedder_external_texture_metal.mm
+FILE: ../../../flutter/shell/platform/embedder/embedder_external_texture_resolver.cc
+FILE: ../../../flutter/shell/platform/embedder/embedder_external_texture_resolver.h
 FILE: ../../../flutter/shell/platform/embedder/embedder_external_view.cc
 FILE: ../../../flutter/shell/platform/embedder/embedder_external_view.h
 FILE: ../../../flutter/shell/platform/embedder/embedder_external_view_embedder.cc
@@ -1187,6 +1192,7 @@ FILE: ../../../flutter/shell/platform/embedder/fixtures/compositor_with_platform
 FILE: ../../../flutter/shell/platform/embedder/fixtures/compositor_with_root_layer_only.png
 FILE: ../../../flutter/shell/platform/embedder/fixtures/dpr_noxform.png
 FILE: ../../../flutter/shell/platform/embedder/fixtures/dpr_xform.png
+FILE: ../../../flutter/shell/platform/embedder/fixtures/external_texture_metal.png
 FILE: ../../../flutter/shell/platform/embedder/fixtures/gradient.png
 FILE: ../../../flutter/shell/platform/embedder/fixtures/gradient_metal.png
 FILE: ../../../flutter/shell/platform/embedder/fixtures/gradient_xform.png

shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.h

@@ -7,7 +7,22 @@
 
 #import "flutter/shell/platform/darwin/common/framework/Headers/FlutterTexture.h"
 #include "third_party/skia/include/core/SkCanvas.h"
-#include "third_party/skia/include/gpu/GrDirectContext.h"
+#include "third_party/skia/include/core/SkImage.h"
+
+@interface FlutterDarwinExternalTextureSkImageWrapper : NSObject
+
++ (sk_sp<SkImage>)wrapYUVATexture:(nonnull id<MTLTexture>)yTex
+                            UVTex:(nonnull id<MTLTexture>)uvTex
+                        grContext:(nonnull GrDirectContext*)grContext
+                            width:(size_t)width
+                           height:(size_t)height;
+
++ (sk_sp<SkImage>)wrapRGBATexture:(nonnull id<MTLTexture>)rgbaTex
+                        grContext:(nonnull GrDirectContext*)grContext
+                            width:(size_t)width
+                           height:(size_t)height;
+
+@end
 
 @interface FlutterDarwinExternalTextureMetal : NSObject
 

shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.mm

@@ -14,17 +14,6 @@
 
 FLUTTER_ASSERT_ARC
 
-namespace {
-
-static sk_cf_obj<const void*> SkiaTextureFromCVMetalTexture(CVMetalTextureRef cvMetalTexture) {
-  id<MTLTexture> texture = CVMetalTextureGetTexture(cvMetalTexture);
-  // CVMetal texture can be released as soon as we can the MTLTexture from it.
-  CVPixelBufferRelease(cvMetalTexture);
-  return sk_cf_obj<const void*>{(__bridge_retained const void*)texture};
-}
-
-}
-
 @implementation FlutterDarwinExternalTextureMetal {
   CVMetalTextureCacheRef _textureCache;
   NSObject<FlutterTexture>* _externalTexture;
@@ -185,31 +174,17 @@ - (void)onTextureUnregistered {
     }
   }
 
-  GrMtlTextureInfo ySkiaTextureInfo;
-  ySkiaTextureInfo.fTexture = SkiaTextureFromCVMetalTexture(yMetalTexture);
+  id<MTLTexture> yTex = CVMetalTextureGetTexture(yMetalTexture);
+  CVBufferRelease(yMetalTexture);
 
-  GrBackendTexture skiaBackendTextures[2];
-  skiaBackendTextures[0] = GrBackendTexture(/*width=*/textureSize.width(),
-                                            /*height=*/textureSize.height(),
-                                            /*mipMapped=*/GrMipMapped ::kNo,
-                                            /*textureInfo=*/ySkiaTextureInfo);
+  id<MTLTexture> uvTex = CVMetalTextureGetTexture(uvMetalTexture);
+  CVBufferRelease(uvMetalTexture);
 
-  GrMtlTextureInfo uvSkiaTextureInfo;
-  uvSkiaTextureInfo.fTexture = SkiaTextureFromCVMetalTexture(uvMetalTexture);
-
-  skiaBackendTextures[1] = GrBackendTexture(/*width=*/textureSize.width(),
-                                            /*height=*/textureSize.height(),
-                                            /*mipMapped=*/GrMipMapped ::kNo,
-                                            /*textureInfo=*/uvSkiaTextureInfo);
-  SkYUVAInfo yuvaInfo(skiaBackendTextures[0].dimensions(), SkYUVAInfo::PlaneConfig::kY_UV,
-                      SkYUVAInfo::Subsampling::k444, kRec601_SkYUVColorSpace);
-  GrYUVABackendTextures yuvaBackendTextures(yuvaInfo, skiaBackendTextures,
-                                            kTopLeft_GrSurfaceOrigin);
-
-  sk_sp<SkImage> image =
-      SkImage::MakeFromYUVATextures(grContext, yuvaBackendTextures, /*imageColorSpace=*/nullptr,
-                                    /*releaseProc*/ nullptr, /*releaseContext*/ nullptr);
-  return image;
+  return [FlutterDarwinExternalTextureSkImageWrapper wrapYUVATexture:yTex
+                                                               UVTex:uvTex
+                                                           grContext:grContext
+                                                               width:textureSize.width()
+                                                              height:textureSize.height()];
 }
 
 - (sk_sp<SkImage>)wrapRGBAExternalPixelBuffer:(CVPixelBufferRef)pixelBuffer
@@ -233,22 +208,64 @@ GrYUVABackendTextures yuvaBackendTextures(yuvaInfo, skiaBackendTextures,
     return nullptr;
   }
 
+  id<MTLTexture> rgbaTex = CVMetalTextureGetTexture(metalTexture);
+  CVBufferRelease(metalTexture);
+
+  return [FlutterDarwinExternalTextureSkImageWrapper wrapRGBATexture:rgbaTex
+                                                           grContext:grContext
+                                                               width:textureSize.width()
+                                                              height:textureSize.height()];
+}
+
+@end
+
+@implementation FlutterDarwinExternalTextureSkImageWrapper
+
++ (sk_sp<SkImage>)wrapYUVATexture:(id<MTLTexture>)yTex
+                            UVTex:(id<MTLTexture>)uvTex
+                        grContext:(nonnull GrDirectContext*)grContext
+                            width:(size_t)width
+                           height:(size_t)height {
+  GrMtlTextureInfo ySkiaTextureInfo;
+  ySkiaTextureInfo.fTexture = sk_cf_obj<const void*>{(__bridge_retained const void*)yTex};
+
+  GrBackendTexture skiaBackendTextures[2];
+  skiaBackendTextures[0] = GrBackendTexture(/*width=*/width,
+                                            /*height=*/height,
+                                            /*mipMapped=*/GrMipMapped::kNo,
+                                            /*textureInfo=*/ySkiaTextureInfo);
+
+  GrMtlTextureInfo uvSkiaTextureInfo;
+  uvSkiaTextureInfo.fTexture = sk_cf_obj<const void*>{(__bridge_retained const void*)uvTex};
+
+  skiaBackendTextures[1] = GrBackendTexture(/*width=*/width,
+                                            /*height=*/height,
+                                            /*mipMapped=*/GrMipMapped::kNo,
+                                            /*textureInfo=*/uvSkiaTextureInfo);
+  SkYUVAInfo yuvaInfo(skiaBackendTextures[0].dimensions(), SkYUVAInfo::PlaneConfig::kY_UV,
+                      SkYUVAInfo::Subsampling::k444, kRec601_SkYUVColorSpace);
+  GrYUVABackendTextures yuvaBackendTextures(yuvaInfo, skiaBackendTextures,
+                                            kTopLeft_GrSurfaceOrigin);
+
+  return SkImage::MakeFromYUVATextures(grContext, yuvaBackendTextures, /*imageColorSpace=*/nullptr,
+                                       /*releaseProc*/ nullptr, /*releaseContext*/ nullptr);
+}
+
++ (sk_sp<SkImage>)wrapRGBATexture:(id<MTLTexture>)rgbaTex
+                        grContext:(nonnull GrDirectContext*)grContext
+                            width:(size_t)width
+                           height:(size_t)height {
   GrMtlTextureInfo skiaTextureInfo;
-  skiaTextureInfo.fTexture = SkiaTextureFromCVMetalTexture(metalTexture);
+  skiaTextureInfo.fTexture = sk_cf_obj<const void*>{(__bridge_retained const void*)rgbaTex};
 
-  GrBackendTexture skiaBackendTexture(/*width=*/textureSize.width(),
-                                      /*height=*/textureSize.height(),
+  GrBackendTexture skiaBackendTexture(/*width=*/width,
+                                      /*height=*/height,
                                       /*mipMapped=*/GrMipMapped ::kNo,
                                       /*textureInfo=*/skiaTextureInfo);
 
-  sk_sp<SkImage> image =
-      SkImage::MakeFromTexture(grContext, skiaBackendTexture, kTopLeft_GrSurfaceOrigin,
-                               kBGRA_8888_SkColorType, kPremul_SkAlphaType,
-                               /*imageColorSpace=*/nullptr, /*releaseProc*/ nullptr,
-                               /*releaseContext*/ nullptr
-
-      );
-  return image;
+  return SkImage::MakeFromTexture(grContext, skiaBackendTexture, kTopLeft_GrSurfaceOrigin,
+                                  kBGRA_8888_SkColorType, kPremul_SkAlphaType,
+                                  /*imageColorSpace=*/nullptr, /*releaseProc*/ nullptr,
+                                  /*releaseContext*/ nullptr);
 }
-
 @end

shell/platform/darwin/macos/BUILD.gn

@@ -154,6 +154,7 @@ executable("flutter_desktop_darwin_unittests") {
 
   if (shell_enable_metal) {
     sources += [
+      "framework/Source/FlutterEmbedderExternalTextureUnittests.mm",
       "framework/Source/FlutterMetalRendererTest.mm",
       "framework/Source/FlutterMetalSurfaceManagerTest.mm",
     ]
@@ -177,6 +178,10 @@ executable("flutter_desktop_darwin_unittests") {
     "//flutter/testing:testing_lib",
     "//third_party/ocmock:ocmock",
   ]
+
+  if (shell_enable_metal) {
+    deps += [ "//flutter/shell/platform/darwin/graphics" ]
+  }
 }
 
 copy("copy_dylib") {

shell/platform/darwin/macos/framework/Source/FlutterEmbedderExternalTextureUnittests.mm

@@ -0,0 +1,79 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#import <Foundation/Foundation.h>
+#import <Metal/Metal.h>
+
+#include <memory>
+#include <vector>
+
+#import "flutter/shell/platform/darwin/graphics/FlutterDarwinContextMetal.h"
+#import "flutter/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.h"
+#include "flutter/shell/platform/embedder/embedder.h"
+#include "flutter/shell/platform/embedder/embedder_external_texture_metal.h"
+#import "flutter/testing/testing.h"
+#include "third_party/googletest/googletest/include/gtest/gtest.h"
+#include "third_party/skia/include/core/SkImage.h"
+#include "third_party/skia/include/core/SkSamplingOptions.h"
+#include "third_party/skia/include/core/SkSurface.h"
+
+namespace flutter::testing {
+
+TEST(FlutterEmbedderExternalTextureUnittests, TestTextureResolution) {
+  // constants.
+  const size_t width = 100;
+  const size_t height = 100;
+  const int64_t texture_id = 1;
+
+  // setup the surface.
+  FlutterDarwinContextMetal* darwinContextMetal =
+      [[FlutterDarwinContextMetal alloc] initWithDefaultMTLDevice];
+  SkImageInfo info = SkImageInfo::MakeN32Premul(width, height);
+  GrDirectContext* grContext = darwinContextMetal.mainContext.get();
+  sk_sp<SkSurface> gpuSurface(SkSurface::MakeRenderTarget(grContext, SkBudgeted::kNo, info));
+
+  // create a texture.
+  MTLTextureDescriptor* textureDescriptor = [[MTLTextureDescriptor alloc] init];
+  textureDescriptor.pixelFormat = MTLPixelFormatBGRA8Unorm;
+  textureDescriptor.width = width;
+  textureDescriptor.height = height;
+  textureDescriptor.usage = MTLTextureUsageRenderTarget | MTLTextureUsageShaderRead;
+  id<MTLTexture> mtlTexture =
+      [darwinContextMetal.device newTextureWithDescriptor:textureDescriptor];
+
+  // callback to resolve the texture.
+  EmbedderExternalTextureMetal::ExternalTextureCallback callback = [&](int64_t texture_id, size_t w,
+                                                                       size_t h) {
+    EXPECT_TRUE(w == width);
+    EXPECT_TRUE(h == height);
+
+    FlutterMetalExternalTexture* texture = new FlutterMetalExternalTexture();
+    texture->struct_size = sizeof(FlutterMetalExternalTexture);
+    texture->num_textures = 1;
+    texture->height = h;
+    texture->width = w;
+    texture->pixel_format = FlutterMetalExternalTexturePixelFormat::kRGBA;
+
+    std::vector<FlutterMetalTextureHandle> textures = {
+        (__bridge FlutterMetalTextureHandle)mtlTexture,
+    };
+
+    texture->textures = textures.data();
+
+    return std::unique_ptr<FlutterMetalExternalTexture>(texture);
+  };
+
+  // render the texture.
+  std::unique_ptr<flutter::Texture> texture =
+      std::make_unique<EmbedderExternalTextureMetal>(texture_id, callback);
+  SkRect bounds = SkRect::MakeWH(info.width(), info.height());
+  SkSamplingOptions sampling = SkSamplingOptions(SkFilterMode::kNearest);
+  texture->Paint(*gpuSurface->getCanvas(), bounds, /*freeze=*/false, grContext, sampling);
+
+  ASSERT_TRUE(mtlTexture != nil);
+
+  gpuSurface->makeImageSnapshot();
+}
+
+}  // namespace flutter::testing

shell/platform/embedder/BUILD.gn

@@ -34,6 +34,8 @@ template("embedder_source_set") {
       "embedder.cc",
       "embedder_engine.cc",
       "embedder_engine.h",
+      "embedder_external_texture_resolver.cc",
+      "embedder_external_texture_resolver.h",
       "embedder_external_view.cc",
       "embedder_external_view.h",
       "embedder_external_view_embedder.cc",
@@ -90,6 +92,8 @@ template("embedder_source_set") {
 
     if (embedder_enable_metal) {
       sources += [
+        "embedder_external_texture_metal.h",
+        "embedder_external_texture_metal.mm",
         "embedder_surface_metal.h",
         "embedder_surface_metal.mm",
       ]
@@ -156,6 +160,7 @@ test_fixtures("fixtures") {
     "fixtures/dpr_xform.png",
     "fixtures/gradient.png",
     "fixtures/gradient_metal.png",
+    "fixtures/external_texture_metal.png",
     "fixtures/gradient_xform.png",
     "fixtures/scene_without_custom_compositor.png",
     "fixtures/scene_without_custom_compositor_with_xform.png",
@@ -226,7 +231,7 @@ if (enable_unittests) {
       sources += [
         "tests/embedder_test_context_metal.cc",
         "tests/embedder_test_context_metal.h",
-        "tests/embedder_unittests_metal.cc",
+        "tests/embedder_unittests_metal.mm",
       ]
 
       deps += [ "//flutter/testing:metal" ]