From 4b4cde09d936aad0fd319a29f243bd580eed5053 Mon Sep 17 00:00:00 2001 From: Matan Lurey Date: Wed, 28 Feb 2024 11:05:12 -0800 Subject: [PATCH 1/3] Plumb and use --force-surface-producer-surface-texture in scenario_app. --- .../linux_android_emulator_opengles.json | 26 + ci/builders/linux_android_emulator_skia.json | 25 + .../engine/renderer/FlutterRenderer.java | 2328 ++++++++--------- .../SurfaceTextureSurfaceProducer.java | 12 +- .../android/io/flutter/view/FlutterView.java | 1 + .../platform/PlatformViewsControllerTest.java | 3 + .../java/dev/flutter/TestRunner.java | 6 + .../ExternalTextureFlutterActivity.java | 18 +- .../scenario_app/bin/run_android_tests.dart | 5 + testing/scenario_app/bin/utils/options.dart | 36 + 10 files changed, 1268 insertions(+), 1192 deletions(-) diff --git a/ci/builders/linux_android_emulator_opengles.json b/ci/builders/linux_android_emulator_opengles.json index 7eaeaebbdd541..ddd3f784520a1 100644 --- a/ci/builders/linux_android_emulator_opengles.json +++ b/ci/builders/linux_android_emulator_opengles.json @@ -57,6 +57,32 @@ "--enable-impeller", "--impeller-backend=opengles" ] + }, + { + "language": "dart", + "name": "Android Scenario App Integration Tests (Impeller/OpenGLES, SurfaceTexture)", + "test_timeout_secs": 900, + "max_attempts": 2, + "test_dependencies": [ + { + "dependency": "android_virtual_device", + "version": "android_34_google_apis_x64.textpb" + }, + { + "dependency": "avd_cipd_version", + "version": "build_id:8759428741582061553" + } + ], + "contexts": [ + "android_virtual_device" + ], + "script": "flutter/testing/scenario_app/bin/run_android_tests.dart", + "parameters": [ + "--out-dir=../out/android_emulator_opengles_debug_x64", + "--enable-impeller", + "--impeller-backend=opengles", + "--force-surface-producer-surface-texture" + ] } ] } diff --git a/ci/builders/linux_android_emulator_skia.json b/ci/builders/linux_android_emulator_skia.json index 35294517e2983..32fcbc96ac174 100644 --- a/ci/builders/linux_android_emulator_skia.json +++ b/ci/builders/linux_android_emulator_skia.json @@ -57,6 +57,31 @@ "--out-dir=../out/android_emulator_skia_debug_x64", "--no-enable-impeller" ] + }, + { + "language": "dart", + "name": "Android Scenario App Integration Tests (Skia, SurfaceTexture)", + "test_timeout_secs": 900, + "max_attempts": 2, + "test_dependencies": [ + { + "dependency": "android_virtual_device", + "version": "android_34_google_apis_x64.textpb" + }, + { + "dependency": "avd_cipd_version", + "version": "build_id:8759428741582061553" + } + ], + "contexts": [ + "android_virtual_device" + ], + "script": "flutter/testing/scenario_app/bin/run_android_tests.dart", + "parameters": [ + "--out-dir=../out/android_emulator_skia_debug_x64", + "--no-enable-impeller", + "--force-surface-producer-surface-texture" + ] } ] } diff --git a/shell/platform/android/io/flutter/embedding/engine/renderer/FlutterRenderer.java b/shell/platform/android/io/flutter/embedding/engine/renderer/FlutterRenderer.java index 82455af96b75d..8902f9a6c8355 100644 --- a/shell/platform/android/io/flutter/embedding/engine/renderer/FlutterRenderer.java +++ b/shell/platform/android/io/flutter/embedding/engine/renderer/FlutterRenderer.java @@ -16,13 +16,12 @@ import android.os.Build; import android.os.Handler; import android.view.Surface; + import androidx.annotation.Keep; import androidx.annotation.NonNull; import androidx.annotation.Nullable; import androidx.annotation.VisibleForTesting; -import io.flutter.Log; -import io.flutter.embedding.engine.FlutterJNI; -import io.flutter.view.TextureRegistry; + import java.io.IOException; import java.lang.ref.WeakReference; import java.nio.ByteBuffer; @@ -35,6 +34,10 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicLong; +import io.flutter.Log; +import io.flutter.embedding.engine.FlutterJNI; +import io.flutter.view.TextureRegistry; + /** * Represents the rendering responsibilities of a {@code FlutterEngine}. * @@ -49,1297 +52,1266 @@ * io.flutter.embedding.android.FlutterTextureView} are implementations of {@link RenderSurface}. */ public class FlutterRenderer implements TextureRegistry { - /** - * Whether to always use GL textures for {@link FlutterRenderer#createSurfaceProducer()}. - * - *

This is a debug-only API intended for local development. For example, when using a newer - * Android device (that normally would use {@link ImageReaderSurfaceProducer}, but wanting to test - * the OpenGLES/{@link SurfaceTextureSurfaceProducer} code branch. This flag has undefined - * behavior if set to true while running in a Vulkan (Impeller) context. - */ - @VisibleForTesting static boolean debugForceSurfaceProducerGlTextures = false; - - private static final String TAG = "FlutterRenderer"; - - @NonNull private final FlutterJNI flutterJNI; - @NonNull private final AtomicLong nextTextureId = new AtomicLong(0L); - @Nullable private Surface surface; - private boolean isDisplayingFlutterUi = false; - private final Handler handler = new Handler(); - - @NonNull - private final Set> onTrimMemoryListeners = - new HashSet<>(); - - @NonNull - private final FlutterUiDisplayListener flutterUiDisplayListener = - new FlutterUiDisplayListener() { - @Override - public void onFlutterUiDisplayed() { - isDisplayingFlutterUi = true; - } + /** + * Whether to always use GL textures for {@link FlutterRenderer#createSurfaceProducer()}. + * + *

This is a debug-only API intended for local development. For example, when using a newer + * Android device (that normally would use {@link ImageReaderSurfaceProducer}, but wanting to + * test the OpenGLES/{@link SurfaceTextureSurfaceProducer} code branch. This flag has undefined + * behavior if set to true while running in a Vulkan (Impeller) context. + */ + @VisibleForTesting + public static boolean debugForceSurfaceProducerGlTextures = false; - @Override - public void onFlutterUiNoLongerDisplayed() { - isDisplayingFlutterUi = false; - } - }; - - public FlutterRenderer(@NonNull FlutterJNI flutterJNI) { - this.flutterJNI = flutterJNI; - this.flutterJNI.addIsDisplayingFlutterUiListener(flutterUiDisplayListener); - } - - /** - * Returns true if this {@code FlutterRenderer} is painting pixels to an Android {@code View} - * hierarchy, false otherwise. - */ - public boolean isDisplayingFlutterUi() { - return isDisplayingFlutterUi; - } - - /** - * Adds a listener that is invoked whenever this {@code FlutterRenderer} starts and stops painting - * pixels to an Android {@code View} hierarchy. - */ - public void addIsDisplayingFlutterUiListener(@NonNull FlutterUiDisplayListener listener) { - flutterJNI.addIsDisplayingFlutterUiListener(listener); - - if (isDisplayingFlutterUi) { - listener.onFlutterUiDisplayed(); + private static final String TAG = "FlutterRenderer"; + + @NonNull + private final FlutterJNI flutterJNI; + @NonNull + private final AtomicLong nextTextureId = new AtomicLong(0L); + @Nullable + private Surface surface; + private boolean isDisplayingFlutterUi = false; + private final Handler handler = new Handler(); + + @NonNull + private final Set> onTrimMemoryListeners = + new HashSet<>(); + + @NonNull + private final FlutterUiDisplayListener flutterUiDisplayListener = + new FlutterUiDisplayListener() { + @Override + public void onFlutterUiDisplayed() { + isDisplayingFlutterUi = true; + } + + @Override + public void onFlutterUiNoLongerDisplayed() { + isDisplayingFlutterUi = false; + } + }; + + public FlutterRenderer(@NonNull FlutterJNI flutterJNI) { + this.flutterJNI = flutterJNI; + this.flutterJNI.addIsDisplayingFlutterUiListener(flutterUiDisplayListener); } - } - - /** - * Removes a listener added via {@link - * #addIsDisplayingFlutterUiListener(FlutterUiDisplayListener)}. - */ - public void removeIsDisplayingFlutterUiListener(@NonNull FlutterUiDisplayListener listener) { - flutterJNI.removeIsDisplayingFlutterUiListener(listener); - } - - private void clearDeadListeners() { - final Iterator> iterator = onTrimMemoryListeners.iterator(); - while (iterator.hasNext()) { - WeakReference listenerRef = iterator.next(); - final OnTrimMemoryListener listener = listenerRef.get(); - if (listener == null) { - iterator.remove(); - } + + /** + * Returns true if this {@code FlutterRenderer} is painting pixels to an Android {@code View} + * hierarchy, false otherwise. + */ + public boolean isDisplayingFlutterUi() { + return isDisplayingFlutterUi; } - } - - /** Adds a listener that is invoked when a memory pressure warning was forward. */ - @VisibleForTesting - /* package */ void addOnTrimMemoryListener(@NonNull OnTrimMemoryListener listener) { - // Purge dead listener to avoid accumulating. - clearDeadListeners(); - onTrimMemoryListeners.add(new WeakReference<>(listener)); - } - - /** - * Removes a {@link OnTrimMemoryListener} that was added with {@link - * #addOnTrimMemoryListener(OnTrimMemoryListener)}. - */ - @VisibleForTesting - /* package */ void removeOnTrimMemoryListener(@NonNull OnTrimMemoryListener listener) { - for (WeakReference listenerRef : onTrimMemoryListeners) { - if (listenerRef.get() == listener) { - onTrimMemoryListeners.remove(listenerRef); - break; - } + + /** + * Adds a listener that is invoked whenever this {@code FlutterRenderer} starts and stops + * painting pixels to an Android {@code View} hierarchy. + */ + public void addIsDisplayingFlutterUiListener(@NonNull FlutterUiDisplayListener listener) { + flutterJNI.addIsDisplayingFlutterUiListener(listener); + + if (isDisplayingFlutterUi) { + listener.onFlutterUiDisplayed(); + } } - } - - // ------ START TextureRegistry IMPLEMENTATION ----- - - /** - * Creates and returns a new external texture {@link SurfaceProducer} managed by the Flutter - * engine that is also made available to Flutter code. - */ - @NonNull - @Override - public SurfaceProducer createSurfaceProducer() { - // Prior to Impeller, Flutter on Android *only* ran on OpenGLES (via Skia). That - // meant that - // plugins (i.e. end-users) either explicitly created a SurfaceTexture (via - // createX/registerX) or an ImageTexture (via createX/registerX). - // - // In an Impeller world, which for the first time uses (if available) a Vulkan - // rendering - // backend, it is no longer possible (at least not trivially) to render an - // OpenGLES-provided - // texture (SurfaceTexture) in a Vulkan context. - // - // This function picks the "best" rendering surface based on the Android - // runtime, and - // provides a consumer-agnostic SurfaceProducer (which in turn vends a Surface), - // and has - // plugins (i.e. end-users) use the Surface instead, letting us "hide" the - // consumer-side - // of the implementation. - // - // tl;dr: If ImageTexture is available, we use it, otherwise we use a - // SurfaceTexture. - // Coincidentally, if ImageTexture is available, we are also on an Android - // version that is - // running Vulkan, so we don't have to worry about it not being supported. - final long id = nextTextureId.getAndIncrement(); - final SurfaceProducer entry; - if (!debugForceSurfaceProducerGlTextures && Build.VERSION.SDK_INT >= 29) { - final ImageReaderSurfaceProducer producer = new ImageReaderSurfaceProducer(id); - registerImageTexture(id, producer); - addOnTrimMemoryListener(producer); - Log.v(TAG, "New ImageReaderSurfaceProducer ID: " + id); - entry = producer; - } else { - final SurfaceTextureSurfaceProducer producer = - new SurfaceTextureSurfaceProducer(id, handler, flutterJNI); - registerSurfaceTexture(id, producer.getSurfaceTexture()); - Log.v(TAG, "New SurfaceTextureSurfaceProducer ID: " + id); - entry = producer; + + /** + * Removes a listener added via {@link + * #addIsDisplayingFlutterUiListener(FlutterUiDisplayListener)}. + */ + public void removeIsDisplayingFlutterUiListener(@NonNull FlutterUiDisplayListener listener) { + flutterJNI.removeIsDisplayingFlutterUiListener(listener); + } + + private void clearDeadListeners() { + final Iterator> iterator = + onTrimMemoryListeners.iterator(); + while (iterator.hasNext()) { + WeakReference listenerRef = iterator.next(); + final OnTrimMemoryListener listener = listenerRef.get(); + if (listener == null) { + iterator.remove(); + } + } } - return entry; - } - - /** - * Creates and returns a new {@link SurfaceTexture} managed by the Flutter engine that is also - * made available to Flutter code. - */ - @NonNull - @Override - public SurfaceTextureEntry createSurfaceTexture() { - Log.v(TAG, "Creating a SurfaceTexture."); - final SurfaceTexture surfaceTexture = new SurfaceTexture(0); - return registerSurfaceTexture(surfaceTexture); - } - - /** - * Registers and returns a {@link SurfaceTexture} managed by the Flutter engine that is also made - * available to Flutter code. - */ - @NonNull - @Override - public SurfaceTextureEntry registerSurfaceTexture(@NonNull SurfaceTexture surfaceTexture) { - return registerSurfaceTexture(nextTextureId.getAndIncrement(), surfaceTexture); - } - - /** - * Similar to {@link FlutterRenderer#registerSurfaceTexture} but with an existing @{code - * textureId}. - * - * @param surfaceTexture Surface texture to wrap. - * @param textureId A texture ID already created that should be assigned to the surface texture. - */ - @NonNull - private SurfaceTextureEntry registerSurfaceTexture( - long textureId, @NonNull SurfaceTexture surfaceTexture) { - surfaceTexture.detachFromGLContext(); - final SurfaceTextureRegistryEntry entry = - new SurfaceTextureRegistryEntry(textureId, surfaceTexture); - Log.v(TAG, "New SurfaceTexture ID: " + entry.id()); - registerTexture(entry.id(), entry.textureWrapper()); - addOnTrimMemoryListener(entry); - return entry; - } - - @NonNull - @Override - public ImageTextureEntry createImageTexture() { - final ImageTextureRegistryEntry entry = - new ImageTextureRegistryEntry(nextTextureId.getAndIncrement()); - Log.v(TAG, "New ImageTextureEntry ID: " + entry.id()); - registerImageTexture(entry.id(), entry); - return entry; - } - - @Override - public void onTrimMemory(int level) { - final Iterator> iterator = onTrimMemoryListeners.iterator(); - while (iterator.hasNext()) { - WeakReference listenerRef = iterator.next(); - final OnTrimMemoryListener listener = listenerRef.get(); - if (listener != null) { - listener.onTrimMemory(level); - } else { - // Purge cleared refs to avoid accumulating a lot of dead listener - iterator.remove(); - } + + /** Adds a listener that is invoked when a memory pressure warning was forward. */ + @VisibleForTesting + /* package */ void addOnTrimMemoryListener(@NonNull OnTrimMemoryListener listener) { + // Purge dead listener to avoid accumulating. + clearDeadListeners(); + onTrimMemoryListeners.add(new WeakReference<>(listener)); } - } - - final class SurfaceTextureRegistryEntry - implements TextureRegistry.SurfaceTextureEntry, TextureRegistry.OnTrimMemoryListener { - private final long id; - @NonNull private final SurfaceTextureWrapper textureWrapper; - private boolean released; - @Nullable private OnTrimMemoryListener trimMemoryListener; - @Nullable private OnFrameConsumedListener frameConsumedListener; - - SurfaceTextureRegistryEntry(long id, @NonNull SurfaceTexture surfaceTexture) { - this.id = id; - Runnable onFrameConsumed = - () -> { - if (frameConsumedListener != null) { - frameConsumedListener.onFrameConsumed(); - } - }; - this.textureWrapper = new SurfaceTextureWrapper(surfaceTexture, onFrameConsumed); - - // Even though we make sure to unregister the callback before releasing, as of - // Android O, SurfaceTexture has a data race when accessing the callback, so the - // callback may still be called by a stale reference after released==true and - // mNativeView==null. - SurfaceTexture.OnFrameAvailableListener onFrameListener = - texture -> { - if (released || !flutterJNI.isAttached()) { - // Even though we make sure to unregister the callback before releasing, as of - // Android O, SurfaceTexture has a data race when accessing the callback, so the - // callback may still be called by a stale reference after released==true and - // mNativeView==null. - return; + + /** + * Removes a {@link OnTrimMemoryListener} that was added with {@link + * #addOnTrimMemoryListener(OnTrimMemoryListener)}. + */ + @VisibleForTesting + /* package */ void removeOnTrimMemoryListener(@NonNull OnTrimMemoryListener listener) { + for (WeakReference listenerRef : onTrimMemoryListeners) { + if (listenerRef.get() == listener) { + onTrimMemoryListeners.remove(listenerRef); + break; } - textureWrapper.markDirty(); - scheduleEngineFrame(); - }; - // The callback relies on being executed on the UI thread (unsynchronised read of - // mNativeView and also the engine code check for platform thread in - // Shell::OnPlatformViewMarkTextureFrameAvailable), so we explicitly pass a Handler for the - // current thread. - this.surfaceTexture().setOnFrameAvailableListener(onFrameListener, new Handler()); + } } + // ------ START TextureRegistry IMPLEMENTATION ----- + + /** + * Creates and returns a new external texture {@link SurfaceProducer} managed by the Flutter + * engine that is also made available to Flutter code. + */ + @NonNull @Override - public void onTrimMemory(int level) { - if (trimMemoryListener != null) { - trimMemoryListener.onTrimMemory(level); - } + public SurfaceProducer createSurfaceProducer() { + // Prior to Impeller, Flutter on Android *only* ran on OpenGLES (via Skia). That + // meant that + // plugins (i.e. end-users) either explicitly created a SurfaceTexture (via + // createX/registerX) or an ImageTexture (via createX/registerX). + // + // In an Impeller world, which for the first time uses (if available) a Vulkan + // rendering + // backend, it is no longer possible (at least not trivially) to render an + // OpenGLES-provided + // texture (SurfaceTexture) in a Vulkan context. + // + // This function picks the "best" rendering surface based on the Android + // runtime, and + // provides a consumer-agnostic SurfaceProducer (which in turn vends a Surface), + // and has + // plugins (i.e. end-users) use the Surface instead, letting us "hide" the + // consumer-side + // of the implementation. + // + // tl;dr: If ImageTexture is available, we use it, otherwise we use a + // SurfaceTexture. + // Coincidentally, if ImageTexture is available, we are also on an Android + // version that is + // running Vulkan, so we don't have to worry about it not being supported. + final SurfaceProducer entry; + if (!debugForceSurfaceProducerGlTextures && Build.VERSION.SDK_INT >= 29) { + final long id = nextTextureId.getAndIncrement(); + final ImageReaderSurfaceProducer producer = new ImageReaderSurfaceProducer(id); + registerImageTexture(id, producer); + addOnTrimMemoryListener(producer); + Log.v(TAG, "New ImageReaderSurfaceProducer ID: " + id); + entry = producer; + } else { + // TODO(matanlurey): Actually have the class named "*Producer" to well, produce + // something. This is a code smell, but does guarantee the paths for both + // createSurfaceTexture and createSurfaceProducer doesn't diverge. As we get more + // confident in this API and any possible bugs (and have tests to check we don't + // regress), reconsider this pattern. + final SurfaceTextureEntry texture = createSurfaceTexture(); + final SurfaceTextureSurfaceProducer producer = + new SurfaceTextureSurfaceProducer(texture.id(), handler, flutterJNI, texture); + Log.v(TAG, "New SurfaceTextureSurfaceProducer ID: " + texture.id()); + entry = producer; + } + return entry; } - private void removeListener() { - removeOnTrimMemoryListener(this); + /** + * Creates and returns a new {@link SurfaceTexture} managed by the Flutter engine that is also + * made available to Flutter code. + */ + @NonNull + @Override + public SurfaceTextureEntry createSurfaceTexture() { + Log.v(TAG, "Creating a SurfaceTexture."); + final SurfaceTexture surfaceTexture = new SurfaceTexture(0); + return registerSurfaceTexture(surfaceTexture); } + /** + * Registers and returns a {@link SurfaceTexture} managed by the Flutter engine that is also + * made available to Flutter code. + */ @NonNull - public SurfaceTextureWrapper textureWrapper() { - return textureWrapper; + @Override + public SurfaceTextureEntry registerSurfaceTexture(@NonNull SurfaceTexture surfaceTexture) { + return registerSurfaceTexture(nextTextureId.getAndIncrement(), surfaceTexture); } - @Override + /** + * Similar to {@link FlutterRenderer#registerSurfaceTexture} but with an existing @{code + * textureId}. + * + * @param surfaceTexture Surface texture to wrap. + * @param textureId A texture ID already created that should be assigned to the surface texture. + */ @NonNull - public SurfaceTexture surfaceTexture() { - return textureWrapper.surfaceTexture(); + private SurfaceTextureEntry registerSurfaceTexture( + long textureId, @NonNull SurfaceTexture surfaceTexture) { + surfaceTexture.detachFromGLContext(); + final SurfaceTextureRegistryEntry entry = + new SurfaceTextureRegistryEntry(textureId, surfaceTexture); + Log.v(TAG, "New SurfaceTexture ID: " + entry.id()); + registerTexture(entry.id(), entry.textureWrapper()); + addOnTrimMemoryListener(entry); + return entry; } + @NonNull @Override - public long id() { - return id; + public ImageTextureEntry createImageTexture() { + final ImageTextureRegistryEntry entry = + new ImageTextureRegistryEntry(nextTextureId.getAndIncrement()); + Log.v(TAG, "New ImageTextureEntry ID: " + entry.id()); + registerImageTexture(entry.id(), entry); + return entry; } @Override - public void release() { - if (released) { - return; - } - Log.v(TAG, "Releasing a SurfaceTexture (" + id + ")."); - textureWrapper.release(); - unregisterTexture(id); - removeListener(); - released = true; + public void onTrimMemory(int level) { + final Iterator> iterator = + onTrimMemoryListeners.iterator(); + while (iterator.hasNext()) { + WeakReference listenerRef = iterator.next(); + final OnTrimMemoryListener listener = listenerRef.get(); + if (listener != null) { + listener.onTrimMemory(level); + } else { + // Purge cleared refs to avoid accumulating a lot of dead listener + iterator.remove(); + } + } } - @Override - protected void finalize() throws Throwable { - try { - if (released) { - return; + final class SurfaceTextureRegistryEntry + implements TextureRegistry.SurfaceTextureEntry, TextureRegistry.OnTrimMemoryListener { + private final long id; + @NonNull + private final SurfaceTextureWrapper textureWrapper; + private boolean released; + @Nullable + private OnTrimMemoryListener trimMemoryListener; + @Nullable + private OnFrameConsumedListener frameConsumedListener; + + SurfaceTextureRegistryEntry(long id, @NonNull SurfaceTexture surfaceTexture) { + this.id = id; + Runnable onFrameConsumed = () -> { + if (frameConsumedListener != null) { + frameConsumedListener.onFrameConsumed(); + } + }; + this.textureWrapper = new SurfaceTextureWrapper(surfaceTexture, onFrameConsumed); + + // Even though we make sure to unregister the callback before releasing, as of + // Android O, SurfaceTexture has a data race when accessing the callback, so the + // callback may still be called by a stale reference after released==true and + // mNativeView==null. + SurfaceTexture.OnFrameAvailableListener onFrameListener = texture -> { + if (released || !flutterJNI.isAttached()) { + // Even though we make sure to unregister the callback before releasing, as of + // Android O, SurfaceTexture has a data race when accessing the callback, so the + // callback may still be called by a stale reference after released==true and + // mNativeView==null. + return; + } + textureWrapper.markDirty(); + scheduleEngineFrame(); + }; + // The callback relies on being executed on the UI thread (unsynchronised read of + // mNativeView and also the engine code check for platform thread in + // Shell::OnPlatformViewMarkTextureFrameAvailable), so we explicitly pass a Handler for + // the current thread. + this.surfaceTexture().setOnFrameAvailableListener(onFrameListener, new Handler()); } - handler.post(new TextureFinalizerRunnable(id, flutterJNI)); - } finally { - super.finalize(); - } - } + @Override + public void onTrimMemory(int level) { + if (trimMemoryListener != null) { + trimMemoryListener.onTrimMemory(level); + } + } - @Override - public void setOnFrameConsumedListener(@Nullable OnFrameConsumedListener listener) { - frameConsumedListener = listener; - } + private void removeListener() { + removeOnTrimMemoryListener(this); + } - @Override - public void setOnTrimMemoryListener(@Nullable OnTrimMemoryListener listener) { - trimMemoryListener = listener; - } - } + @NonNull + public SurfaceTextureWrapper textureWrapper() { + return textureWrapper; + } - static final class TextureFinalizerRunnable implements Runnable { - private final long id; - private final FlutterJNI flutterJNI; + @Override + @NonNull + public SurfaceTexture surfaceTexture() { + return textureWrapper.surfaceTexture(); + } - TextureFinalizerRunnable(long id, @NonNull FlutterJNI flutterJNI) { - this.id = id; - this.flutterJNI = flutterJNI; - } + @Override + public long id() { + return id; + } - @Override - public void run() { - if (!flutterJNI.isAttached()) { - return; - } - Log.v(TAG, "Releasing a Texture (" + id + ")."); - flutterJNI.unregisterTexture(id); + @Override + public void release() { + if (released) { + return; + } + Log.v(TAG, "Releasing a SurfaceTexture (" + id + ")."); + textureWrapper.release(); + unregisterTexture(id); + removeListener(); + released = true; + } + + @Override + protected void finalize() throws Throwable { + try { + if (released) { + return; + } + + handler.post(new TextureFinalizerRunnable(id, flutterJNI)); + } finally { + super.finalize(); + } + } + + @Override + public void setOnFrameConsumedListener(@Nullable OnFrameConsumedListener listener) { + frameConsumedListener = listener; + } + + @Override + public void setOnTrimMemoryListener(@Nullable OnTrimMemoryListener listener) { + trimMemoryListener = listener; + } } - } - - // Keep a queue of ImageReaders. - // Each ImageReader holds acquired Images. - // When we acquire the next image, close any ImageReaders that don't have any - // more pending images. - @Keep - @TargetApi(29) - final class ImageReaderSurfaceProducer - implements TextureRegistry.SurfaceProducer, - TextureRegistry.ImageConsumer, - TextureRegistry.OnTrimMemoryListener { - private static final String TAG = "ImageReaderSurfaceProducer"; - private static final int MAX_IMAGES = 5; - - // Flip when debugging to see verbose logs. - private static final boolean VERBOSE_LOGS = false; - - private final long id; - - private boolean released; - // Will be true in tests and on Android API < 33. - private boolean ignoringFence = false; - - // The requested width and height are updated by setSize. - private int requestedWidth = 1; - private int requestedHeight = 1; - // Whenever the requested width and height change we set this to be true so we - // create a new ImageReader (inside getSurface) with the correct width and height. - // We use this flag so that we lazily create the ImageReader only when a frame - // will be produced at that size. - private boolean createNewReader = true; - - // State held to track latency of various stages. - private long lastDequeueTime = 0; - private long lastQueueTime = 0; - private long lastScheduleTime = 0; - - private Object lock = new Object(); - // REQUIRED: The following fields must only be accessed when lock is held. - private final LinkedList imageReaderQueue = new LinkedList(); - private final HashMap perImageReaders = - new HashMap(); - private PerImage lastDequeuedImage = null; - private PerImageReader lastReaderDequeuedFrom = null; - - /** Internal class: state held per Image produced by ImageReaders. */ - private class PerImage { - public final Image image; - public final long queuedTime; - - public PerImage(Image image, long queuedTime) { - this.image = image; - this.queuedTime = queuedTime; - } + + static final class TextureFinalizerRunnable implements Runnable { + private final long id; + private final FlutterJNI flutterJNI; + + TextureFinalizerRunnable(long id, @NonNull FlutterJNI flutterJNI) { + this.id = id; + this.flutterJNI = flutterJNI; + } + + @Override + public void run() { + if (!flutterJNI.isAttached()) { + return; + } + Log.v(TAG, "Releasing a Texture (" + id + ")."); + flutterJNI.unregisterTexture(id); + } } - /** Internal class: state held per ImageReader. */ - private class PerImageReader { - public final ImageReader reader; - private final LinkedList imageQueue = new LinkedList(); - private boolean closed = false; + // Keep a queue of ImageReaders. + // Each ImageReader holds acquired Images. + // When we acquire the next image, close any ImageReaders that don't have any + // more pending images. + @Keep + @TargetApi(29) + final class ImageReaderSurfaceProducer + implements TextureRegistry.SurfaceProducer, TextureRegistry.ImageConsumer, + TextureRegistry.OnTrimMemoryListener { + private static final String TAG = "ImageReaderSurfaceProducer"; + private static final int MAX_IMAGES = 5; + + // Flip when debugging to see verbose logs. + private static final boolean VERBOSE_LOGS = false; + + private final long id; + + private boolean released; + // Will be true in tests and on Android API < 33. + private boolean ignoringFence = false; + + // The requested width and height are updated by setSize. + private int requestedWidth = 1; + private int requestedHeight = 1; + // Whenever the requested width and height change we set this to be true so we + // create a new ImageReader (inside getSurface) with the correct width and height. + // We use this flag so that we lazily create the ImageReader only when a frame + // will be produced at that size. + private boolean createNewReader = true; + + // State held to track latency of various stages. + private long lastDequeueTime = 0; + private long lastQueueTime = 0; + private long lastScheduleTime = 0; + + private Object lock = new Object(); + // REQUIRED: The following fields must only be accessed when lock is held. + private final LinkedList imageReaderQueue = + new LinkedList(); + private final HashMap perImageReaders = + new HashMap(); + private PerImage lastDequeuedImage = null; + private PerImageReader lastReaderDequeuedFrom = null; + + /** Internal class: state held per Image produced by ImageReaders. */ + private class PerImage { + public final Image image; + public final long queuedTime; + + public PerImage(Image image, long queuedTime) { + this.image = image; + this.queuedTime = queuedTime; + } + } + + /** Internal class: state held per ImageReader. */ + private class PerImageReader { + public final ImageReader reader; + private final LinkedList imageQueue = new LinkedList(); + private boolean closed = false; + + private final ImageReader.OnImageAvailableListener onImageAvailableListener = + reader -> { + Image image = null; + try { + image = reader.acquireLatestImage(); + } catch (IllegalStateException e) { + Log.e(TAG, "onImageAvailable acquireLatestImage failed: " + e); + } + if (image == null) { + return; + } + if (released || closed) { + image.close(); + return; + } + onImage(reader, image); + }; + + public PerImageReader(ImageReader reader) { + this.reader = reader; + reader.setOnImageAvailableListener(onImageAvailableListener, new Handler()); + } + + PerImage queueImage(Image image) { + if (closed) { + return null; + } + PerImage perImage = new PerImage(image, System.nanoTime()); + imageQueue.add(perImage); + // If we fall too far behind we will skip some frames. + while (imageQueue.size() > 2) { + PerImage r = imageQueue.removeFirst(); + if (VERBOSE_LOGS) { + Log.i(TAG, + "" + reader.hashCode() + + " force closed image=" + r.image.hashCode()); + } + r.image.close(); + } + return perImage; + } + + PerImage dequeueImage() { + if (imageQueue.size() == 0) { + return null; + } + PerImage r = imageQueue.removeFirst(); + return r; + } + + /** returns true if we can prune this reader */ + boolean canPrune() { + return imageQueue.size() == 0 && lastReaderDequeuedFrom != this; + } + + void close() { + closed = true; + if (VERBOSE_LOGS) { + Log.i(TAG, "Closing reader=" + reader.hashCode()); + } + reader.close(); + imageQueue.clear(); + } + } + + double deltaMillis(long deltaNanos) { + double ms = (double) deltaNanos / (double) 1000000.0; + return ms; + } + + PerImageReader getOrCreatePerImageReader(ImageReader reader) { + PerImageReader r = perImageReaders.get(reader); + if (r == null) { + r = new PerImageReader(reader); + perImageReaders.put(reader, r); + imageReaderQueue.add(r); + if (VERBOSE_LOGS) { + Log.i(TAG, "imageReaderQueue#=" + imageReaderQueue.size()); + } + } + return r; + } - private final ImageReader.OnImageAvailableListener onImageAvailableListener = - reader -> { - Image image = null; + void pruneImageReaderQueue() { + boolean change = false; + // Prune nodes from the head of the ImageReader queue. + while (imageReaderQueue.size() > 1) { + PerImageReader r = imageReaderQueue.peekFirst(); + if (!r.canPrune()) { + // No more ImageReaders can be pruned this round. + break; + } + imageReaderQueue.removeFirst(); + perImageReaders.remove(r.reader); + r.close(); + change = true; + } + if (change && VERBOSE_LOGS) { + Log.i(TAG, "Pruned image reader queue length=" + imageReaderQueue.size()); + } + } + + void onImage(ImageReader reader, Image image) { + PerImage queuedImage = null; + synchronized (lock) { + PerImageReader perReader = getOrCreatePerImageReader(reader); + queuedImage = perReader.queueImage(image); + } + if (queuedImage == null) { + // We got a late image. + return; + } + if (VERBOSE_LOGS) { + if (lastQueueTime != 0) { + long now = System.nanoTime(); + long queueDelta = now - lastQueueTime; + Log.i(TAG, + "" + reader.hashCode() + + " enqueued image=" + queuedImage.image.hashCode() + + " queueDelta=" + deltaMillis(queueDelta)); + lastQueueTime = now; + } else { + lastQueueTime = System.nanoTime(); + } + } + scheduleEngineFrame(); + } + + PerImage dequeueImage() { + PerImage r = null; + synchronized (lock) { + for (PerImageReader reader : imageReaderQueue) { + r = reader.dequeueImage(); + if (r == null) { + // This reader is probably about to get pruned. + continue; + } + if (VERBOSE_LOGS) { + if (lastDequeueTime != 0) { + long now = System.nanoTime(); + long dequeueDelta = now - lastDequeueTime; + long queuedFor = now - r.queuedTime; + long scheduleDelay = now - lastScheduleTime; + Log.i(TAG, + "" + reader.reader.hashCode() + + " dequeued image=" + r.image.hashCode() + + " queuedFor= " + deltaMillis(queuedFor) + + " dequeueDelta=" + deltaMillis(dequeueDelta) + + " scheduleDelay=" + deltaMillis(scheduleDelay)); + lastDequeueTime = now; + } else { + lastDequeueTime = System.nanoTime(); + } + } + if (lastDequeuedImage != null) { + if (VERBOSE_LOGS) { + Log.i(TAG, + "" + lastReaderDequeuedFrom.reader.hashCode() + + " closing image=" + + lastDequeuedImage.image.hashCode()); + } + // We must keep the last image dequeued open until we are done presenting + // it. We have just dequeued a new image (r). Close the previously dequeued + // image. + lastDequeuedImage.image.close(); + lastDequeuedImage = null; + } + // Remember the last image and reader dequeued from. We do this because we must + // keep both of these alive until we are done presenting the image. + lastDequeuedImage = r; + lastReaderDequeuedFrom = reader; + break; + } + pruneImageReaderQueue(); + } + return r; + } + + @Override + public void onTrimMemory(int level) { + cleanup(); + createNewReader = true; + } + + private void releaseInternal() { + cleanup(); + released = true; + } + + private void cleanup() { + synchronized (lock) { + for (PerImageReader pir : perImageReaders.values()) { + if (lastReaderDequeuedFrom == pir) { + lastReaderDequeuedFrom = null; + } + pir.close(); + } + perImageReaders.clear(); + if (lastDequeuedImage != null) { + lastDequeuedImage.image.close(); + lastDequeuedImage = null; + } + if (lastReaderDequeuedFrom != null) { + lastReaderDequeuedFrom.close(); + lastReaderDequeuedFrom = null; + } + imageReaderQueue.clear(); + } + } + + @TargetApi(33) + private void waitOnFence(Image image) { try { - image = reader.acquireLatestImage(); - } catch (IllegalStateException e) { - Log.e(TAG, "onImageAvailable acquireLatestImage failed: " + e); + SyncFence fence = image.getFence(); + fence.awaitForever(); + } catch (IOException e) { + // Drop. } + } + + private void maybeWaitOnFence(Image image) { if (image == null) { - return; + return; } - if (released || closed) { - image.close(); - return; + if (ignoringFence) { + return; } - onImage(reader, image); - }; - - public PerImageReader(ImageReader reader) { - this.reader = reader; - reader.setOnImageAvailableListener(onImageAvailableListener, new Handler()); - } - - PerImage queueImage(Image image) { - if (closed) { - return null; - } - PerImage perImage = new PerImage(image, System.nanoTime()); - imageQueue.add(perImage); - // If we fall too far behind we will skip some frames. - while (imageQueue.size() > 2) { - PerImage r = imageQueue.removeFirst(); - if (VERBOSE_LOGS) { - Log.i(TAG, "" + reader.hashCode() + " force closed image=" + r.image.hashCode()); - } - r.image.close(); - } - return perImage; - } - - PerImage dequeueImage() { - if (imageQueue.size() == 0) { - return null; - } - PerImage r = imageQueue.removeFirst(); - return r; - } - - /** returns true if we can prune this reader */ - boolean canPrune() { - return imageQueue.size() == 0 && lastReaderDequeuedFrom != this; - } - - void close() { - closed = true; - if (VERBOSE_LOGS) { - Log.i(TAG, "Closing reader=" + reader.hashCode()); - } - reader.close(); - imageQueue.clear(); - } - } + if (Build.VERSION.SDK_INT >= 33) { + // The fence API is only available on Android >= 33. + waitOnFence(image); + return; + } + // Log once per ImageTextureEntry. + ignoringFence = true; + Log.w(TAG, "ImageTextureEntry can't wait on the fence on Android < 33"); + } - double deltaMillis(long deltaNanos) { - double ms = (double) deltaNanos / (double) 1000000.0; - return ms; - } + ImageReaderSurfaceProducer(long id) { + this.id = id; + } - PerImageReader getOrCreatePerImageReader(ImageReader reader) { - PerImageReader r = perImageReaders.get(reader); - if (r == null) { - r = new PerImageReader(reader); - perImageReaders.put(reader, r); - imageReaderQueue.add(r); - if (VERBOSE_LOGS) { - Log.i(TAG, "imageReaderQueue#=" + imageReaderQueue.size()); - } - } - return r; - } + @Override + public long id() { + return id; + } - void pruneImageReaderQueue() { - boolean change = false; - // Prune nodes from the head of the ImageReader queue. - while (imageReaderQueue.size() > 1) { - PerImageReader r = imageReaderQueue.peekFirst(); - if (!r.canPrune()) { - // No more ImageReaders can be pruned this round. - break; - } - imageReaderQueue.removeFirst(); - perImageReaders.remove(r.reader); - r.close(); - change = true; - } - if (change && VERBOSE_LOGS) { - Log.i(TAG, "Pruned image reader queue length=" + imageReaderQueue.size()); - } - } + @Override + public void release() { + if (released) { + return; + } + releaseInternal(); + unregisterTexture(id); + } - void onImage(ImageReader reader, Image image) { - PerImage queuedImage = null; - synchronized (lock) { - PerImageReader perReader = getOrCreatePerImageReader(reader); - queuedImage = perReader.queueImage(image); - } - if (queuedImage == null) { - // We got a late image. - return; - } - if (VERBOSE_LOGS) { - if (lastQueueTime != 0) { - long now = System.nanoTime(); - long queueDelta = now - lastQueueTime; - Log.i( - TAG, - "" - + reader.hashCode() - + " enqueued image=" - + queuedImage.image.hashCode() - + " queueDelta=" - + deltaMillis(queueDelta)); - lastQueueTime = now; - } else { - lastQueueTime = System.nanoTime(); + @Override + public void setSize(int width, int height) { + // Clamp to a minimum of 1. A 0x0 texture is a runtime exception in ImageReader. + width = Math.max(1, width); + height = Math.max(1, height); + + if (requestedWidth == width && requestedHeight == height) { + // No size change. + return; + } + this.createNewReader = true; + this.requestedHeight = height; + this.requestedWidth = width; } - } - scheduleEngineFrame(); - } - PerImage dequeueImage() { - PerImage r = null; - synchronized (lock) { - for (PerImageReader reader : imageReaderQueue) { - r = reader.dequeueImage(); - if (r == null) { - // This reader is probably about to get pruned. - continue; - } - if (VERBOSE_LOGS) { - if (lastDequeueTime != 0) { - long now = System.nanoTime(); - long dequeueDelta = now - lastDequeueTime; - long queuedFor = now - r.queuedTime; - long scheduleDelay = now - lastScheduleTime; - Log.i( - TAG, - "" - + reader.reader.hashCode() - + " dequeued image=" - + r.image.hashCode() - + " queuedFor= " - + deltaMillis(queuedFor) - + " dequeueDelta=" - + deltaMillis(dequeueDelta) - + " scheduleDelay=" - + deltaMillis(scheduleDelay)); - lastDequeueTime = now; - } else { - lastDequeueTime = System.nanoTime(); + @Override + public int getWidth() { + return this.requestedWidth; + } + + @Override + public int getHeight() { + return this.requestedHeight; + } + + @Override + public Surface getSurface() { + PerImageReader pir = getActiveReader(); + if (VERBOSE_LOGS) { + Log.i(TAG, + "" + pir.reader.hashCode() + " returning surface to render a new frame."); } - } - if (lastDequeuedImage != null) { + return pir.reader.getSurface(); + } + + @Override + public void scheduleFrame() { if (VERBOSE_LOGS) { - Log.i( - TAG, - "" - + lastReaderDequeuedFrom.reader.hashCode() - + " closing image=" - + lastDequeuedImage.image.hashCode()); + long now = System.nanoTime(); + if (lastScheduleTime != 0) { + long delta = now - lastScheduleTime; + Log.v(TAG, "scheduleFrame delta=" + deltaMillis(delta)); + } + lastScheduleTime = now; } - // We must keep the last image dequeued open until we are done presenting it. - // We have just dequeued a new image (r). Close the previously dequeued image. - lastDequeuedImage.image.close(); - lastDequeuedImage = null; - } - // Remember the last image and reader dequeued from. We do this because we must - // keep both of these alive until we are done presenting the image. - lastDequeuedImage = r; - lastReaderDequeuedFrom = reader; - break; - } - pruneImageReaderQueue(); - } - return r; - } + scheduleEngineFrame(); + } - @Override - public void onTrimMemory(int level) { - cleanup(); - createNewReader = true; - } + @Override + @TargetApi(29) + public Image acquireLatestImage() { + PerImage r = dequeueImage(); + if (r == null) { + return null; + } + maybeWaitOnFence(r.image); + return r.image; + } - private void releaseInternal() { - cleanup(); - released = true; - } + private PerImageReader getActiveReader() { + synchronized (lock) { + if (createNewReader) { + createNewReader = false; + // Create a new ImageReader and add it to the queue. + ImageReader reader = createImageReader(); + if (VERBOSE_LOGS) { + Log.i(TAG, + "" + reader.hashCode() + " created w=" + requestedWidth + + " h=" + requestedHeight); + } + return getOrCreatePerImageReader(reader); + } + return imageReaderQueue.peekLast(); + } + } - private void cleanup() { - synchronized (lock) { - for (PerImageReader pir : perImageReaders.values()) { - if (lastReaderDequeuedFrom == pir) { - lastReaderDequeuedFrom = null; - } - pir.close(); - } - perImageReaders.clear(); - if (lastDequeuedImage != null) { - lastDequeuedImage.image.close(); - lastDequeuedImage = null; - } - if (lastReaderDequeuedFrom != null) { - lastReaderDequeuedFrom.close(); - lastReaderDequeuedFrom = null; - } - imageReaderQueue.clear(); - } - } + @Override + protected void finalize() throws Throwable { + try { + if (released) { + return; + } + releaseInternal(); + handler.post(new TextureFinalizerRunnable(id, flutterJNI)); + } finally { + super.finalize(); + } + } - @TargetApi(33) - private void waitOnFence(Image image) { - try { - SyncFence fence = image.getFence(); - fence.awaitForever(); - } catch (IOException e) { - // Drop. - } - } + @TargetApi(33) + private ImageReader createImageReader33() { + final ImageReader.Builder builder = + new ImageReader.Builder(requestedWidth, requestedHeight); + // Allow for double buffering. + builder.setMaxImages(MAX_IMAGES); + // Use PRIVATE image format so that we can support video decoding. + // TODO(johnmccutchan): Should we always use PRIVATE here? It may impact our ability to + // read back texture data. If we don't always want to use it, how do we decide when to + // use it or not? Perhaps PlatformViews can indicate if they may contain DRM'd content. + // I need to investigate how PRIVATE impacts our ability to take screenshots or capture + // the output of Flutter application. + builder.setImageFormat(ImageFormat.PRIVATE); + // Hint that consumed images will only be read by GPU. + builder.setUsage(HardwareBuffer.USAGE_GPU_SAMPLED_IMAGE); + final ImageReader reader = builder.build(); + return reader; + } - private void maybeWaitOnFence(Image image) { - if (image == null) { - return; - } - if (ignoringFence) { - return; - } - if (Build.VERSION.SDK_INT >= 33) { - // The fence API is only available on Android >= 33. - waitOnFence(image); - return; - } - // Log once per ImageTextureEntry. - ignoringFence = true; - Log.w(TAG, "ImageTextureEntry can't wait on the fence on Android < 33"); - } + @TargetApi(29) + private ImageReader createImageReader29() { + final ImageReader reader = ImageReader.newInstance(requestedWidth, requestedHeight, + ImageFormat.PRIVATE, MAX_IMAGES, HardwareBuffer.USAGE_GPU_SAMPLED_IMAGE); + return reader; + } - ImageReaderSurfaceProducer(long id) { - this.id = id; - } + private ImageReader createImageReader() { + if (Build.VERSION.SDK_INT >= 33) { + return createImageReader33(); + } else if (Build.VERSION.SDK_INT >= 29) { + return createImageReader29(); + } + throw new UnsupportedOperationException( + "ImageReaderPlatformViewRenderTarget requires API version 29+"); + } - @Override - public long id() { - return id; - } + @VisibleForTesting + public void disableFenceForTest() { + // Roboelectric's implementation of SyncFence is borked. + ignoringFence = true; + } - @Override - public void release() { - if (released) { - return; - } - releaseInternal(); - unregisterTexture(id); - } + @VisibleForTesting + public int numImageReaders() { + synchronized (lock) { + return imageReaderQueue.size(); + } + } - @Override - public void setSize(int width, int height) { - // Clamp to a minimum of 1. A 0x0 texture is a runtime exception in ImageReader. - width = Math.max(1, width); - height = Math.max(1, height); - - if (requestedWidth == width && requestedHeight == height) { - // No size change. - return; - } - this.createNewReader = true; - this.requestedHeight = height; - this.requestedWidth = width; + @VisibleForTesting + public int numImages() { + int r = 0; + synchronized (lock) { + for (PerImageReader reader : imageReaderQueue) { + r += reader.imageQueue.size(); + } + } + return r; + } } - @Override - public int getWidth() { - return this.requestedWidth; - } + @Keep + final class ImageTextureRegistryEntry + implements TextureRegistry.ImageTextureEntry, TextureRegistry.ImageConsumer { + private static final String TAG = "ImageTextureRegistryEntry"; + private final long id; + private boolean released; + private boolean ignoringFence = false; + private Image image; - @Override - public int getHeight() { - return this.requestedHeight; - } + ImageTextureRegistryEntry(long id) { + this.id = id; + } - @Override - public Surface getSurface() { - PerImageReader pir = getActiveReader(); - if (VERBOSE_LOGS) { - Log.i(TAG, "" + pir.reader.hashCode() + " returning surface to render a new frame."); - } - return pir.reader.getSurface(); - } + @Override + public long id() { + return id; + } - @Override - public void scheduleFrame() { - if (VERBOSE_LOGS) { - long now = System.nanoTime(); - if (lastScheduleTime != 0) { - long delta = now - lastScheduleTime; - Log.v(TAG, "scheduleFrame delta=" + deltaMillis(delta)); - } - lastScheduleTime = now; - } - scheduleEngineFrame(); - } + @Override + public void release() { + if (released) { + return; + } + released = true; + if (image != null) { + image.close(); + image = null; + } + unregisterTexture(id); + } - @Override - @TargetApi(29) - public Image acquireLatestImage() { - PerImage r = dequeueImage(); - if (r == null) { - return null; - } - maybeWaitOnFence(r.image); - return r.image; - } + @Override + public void pushImage(Image image) { + if (released) { + return; + } + Image toClose; + synchronized (this) { + toClose = this.image; + this.image = image; + } + // Close the previously pushed buffer. + if (toClose != null) { + Log.e(TAG, "Dropping PlatformView Frame"); + toClose.close(); + } + if (image != null) { + scheduleEngineFrame(); + } + } - private PerImageReader getActiveReader() { - synchronized (lock) { - if (createNewReader) { - createNewReader = false; - // Create a new ImageReader and add it to the queue. - ImageReader reader = createImageReader(); - if (VERBOSE_LOGS) { - Log.i( - TAG, - "" + reader.hashCode() + " created w=" + requestedWidth + " h=" + requestedHeight); - } - return getOrCreatePerImageReader(reader); - } - return imageReaderQueue.peekLast(); - } - } + @TargetApi(33) + private void waitOnFence(Image image) { + try { + SyncFence fence = image.getFence(); + fence.awaitForever(); + } catch (IOException e) { + // Drop. + } + } - @Override - protected void finalize() throws Throwable { - try { - if (released) { - return; - } - releaseInternal(); - handler.post(new TextureFinalizerRunnable(id, flutterJNI)); - } finally { - super.finalize(); - } - } + @TargetApi(29) + private void maybeWaitOnFence(Image image) { + if (image == null) { + return; + } + if (ignoringFence) { + return; + } + if (Build.VERSION.SDK_INT >= 33) { + // The fence API is only available on Android >= 33. + waitOnFence(image); + return; + } + // Log once per ImageTextureEntry. + ignoringFence = true; + Log.w(TAG, "ImageTextureEntry can't wait on the fence on Android < 33"); + } - @TargetApi(33) - private ImageReader createImageReader33() { - final ImageReader.Builder builder = new ImageReader.Builder(requestedWidth, requestedHeight); - // Allow for double buffering. - builder.setMaxImages(MAX_IMAGES); - // Use PRIVATE image format so that we can support video decoding. - // TODO(johnmccutchan): Should we always use PRIVATE here? It may impact our ability to read - // back texture data. If we don't always want to use it, how do we decide when to use it or - // not? Perhaps PlatformViews can indicate if they may contain DRM'd content. I need to - // investigate how PRIVATE impacts our ability to take screenshots or capture the output of - // Flutter application. - builder.setImageFormat(ImageFormat.PRIVATE); - // Hint that consumed images will only be read by GPU. - builder.setUsage(HardwareBuffer.USAGE_GPU_SAMPLED_IMAGE); - final ImageReader reader = builder.build(); - return reader; - } + @Override + @TargetApi(29) + public Image acquireLatestImage() { + Image r; + synchronized (this) { + r = this.image; + this.image = null; + } + maybeWaitOnFence(r); + return r; + } - @TargetApi(29) - private ImageReader createImageReader29() { - final ImageReader reader = - ImageReader.newInstance( - requestedWidth, - requestedHeight, - ImageFormat.PRIVATE, - MAX_IMAGES, - HardwareBuffer.USAGE_GPU_SAMPLED_IMAGE); - return reader; + @Override + protected void finalize() throws Throwable { + try { + if (released) { + return; + } + if (image != null) { + // Be sure to finalize any cached image. + image.close(); + image = null; + } + released = true; + handler.post(new TextureFinalizerRunnable(id, flutterJNI)); + } finally { + super.finalize(); + } + } } + // ------ END TextureRegistry IMPLEMENTATION ---- - private ImageReader createImageReader() { - if (Build.VERSION.SDK_INT >= 33) { - return createImageReader33(); - } else if (Build.VERSION.SDK_INT >= 29) { - return createImageReader29(); - } - throw new UnsupportedOperationException( - "ImageReaderPlatformViewRenderTarget requires API version 29+"); - } + /** + * Notifies Flutter that the given {@code surface} was created and is available for Flutter + * rendering. + * + *

If called more than once, the current native resources are released. This can be undesired + * if the Engine expects to reuse this surface later. For example, this is true when platform + * views are displayed in a frame, and then removed in the next frame. + * + *

To avoid releasing the current surface resources, set {@code keepCurrentSurface} to true. + * + *

See {@link android.view.SurfaceHolder.Callback} and {@link + * android.view.TextureView.SurfaceTextureListener} + * + * @param surface The render surface. + * @param onlySwap True if the current active surface should not be detached. + */ + public void startRenderingToSurface(@NonNull Surface surface, boolean onlySwap) { + if (!onlySwap) { + // Stop rendering to the surface releases the associated native resources, which causes + // a glitch when toggling between rendering to an image view (hybrid composition) and + // rendering directly to a Surface or Texture view. For more, + // https://github.com/flutter/flutter/issues/95343 + stopRenderingToSurface(); + } - @VisibleForTesting - public void disableFenceForTest() { - // Roboelectric's implementation of SyncFence is borked. - ignoringFence = true; - } + this.surface = surface; - @VisibleForTesting - public int numImageReaders() { - synchronized (lock) { - return imageReaderQueue.size(); - } + if (onlySwap) { + // In the swap case we are just swapping the surface that we render to. + flutterJNI.onSurfaceWindowChanged(surface); + } else { + // In the non-swap case we are creating a new surface to render to. + flutterJNI.onSurfaceCreated(surface); + } } - @VisibleForTesting - public int numImages() { - int r = 0; - synchronized (lock) { - for (PerImageReader reader : imageReaderQueue) { - r += reader.imageQueue.size(); - } - } - return r; - } - } - - @Keep - final class ImageTextureRegistryEntry - implements TextureRegistry.ImageTextureEntry, TextureRegistry.ImageConsumer { - private static final String TAG = "ImageTextureRegistryEntry"; - private final long id; - private boolean released; - private boolean ignoringFence = false; - private Image image; - - ImageTextureRegistryEntry(long id) { - this.id = id; + /** + * Swaps the {@link Surface} used to render the current frame. + * + *

In hybrid composition, the root surfaces changes from {@link + * android.view.SurfaceHolder#getSurface()} to {@link android.media.ImageReader#getSurface()} + * when a platform view is in the current frame. + */ + public void swapSurface(@NonNull Surface surface) { + this.surface = surface; + flutterJNI.onSurfaceWindowChanged(surface); } - @Override - public long id() { - return id; + /** + * Notifies Flutter that a {@code surface} previously registered with {@link + * #startRenderingToSurface(Surface, boolean)} has changed size to the given {@code width} and + * {@code height}. + * + *

See {@link android.view.SurfaceHolder.Callback} and {@link + * android.view.TextureView.SurfaceTextureListener} + */ + public void surfaceChanged(int width, int height) { + flutterJNI.onSurfaceChanged(width, height); } - @Override - public void release() { - if (released) { - return; - } - released = true; - if (image != null) { - image.close(); - image = null; - } - unregisterTexture(id); - } + /** + * Notifies Flutter that a {@code surface} previously registered with {@link + * #startRenderingToSurface(Surface, boolean)} has been destroyed and needs to be released and + * cleaned up on the Flutter side. + * + *

See {@link android.view.SurfaceHolder.Callback} and {@link + * android.view.TextureView.SurfaceTextureListener} + */ + public void stopRenderingToSurface() { + if (surface != null) { + flutterJNI.onSurfaceDestroyed(); + + // TODO(mattcarroll): the source of truth for this call should be FlutterJNI, which is + // where the call to onFlutterUiDisplayed() comes from. However, no such native callback + // exists yet, so until the engine and FlutterJNI are configured to call us back when + // rendering stops, we will manually monitor that change here. + if (isDisplayingFlutterUi) { + flutterUiDisplayListener.onFlutterUiNoLongerDisplayed(); + } - @Override - public void pushImage(Image image) { - if (released) { - return; - } - Image toClose; - synchronized (this) { - toClose = this.image; - this.image = image; - } - // Close the previously pushed buffer. - if (toClose != null) { - Log.e(TAG, "Dropping PlatformView Frame"); - toClose.close(); - } - if (image != null) { - scheduleEngineFrame(); - } + isDisplayingFlutterUi = false; + surface = null; + } } - @TargetApi(33) - private void waitOnFence(Image image) { - try { - SyncFence fence = image.getFence(); - fence.awaitForever(); - } catch (IOException e) { - // Drop. - } - } + /** + * Notifies Flutter that the viewport metrics, e.g. window height and width, have changed. + * + *

If the width, height, or devicePixelRatio are less than or equal to 0, this update is + * ignored. + * + * @param viewportMetrics The metrics to send to the Dart application. + */ + public void setViewportMetrics(@NonNull ViewportMetrics viewportMetrics) { + // We might get called with just the DPR if width/height aren't available yet. + // Just ignore, as it will get called again when width/height are set. + if (!viewportMetrics.validate()) { + return; + } + Log.v(TAG, + "Setting viewport metrics\n" + + "Size: " + viewportMetrics.width + " x " + viewportMetrics.height + "\n" + + "Padding - L: " + viewportMetrics.viewPaddingLeft + + ", T: " + viewportMetrics.viewPaddingTop + + ", R: " + viewportMetrics.viewPaddingRight + + ", B: " + viewportMetrics.viewPaddingBottom + "\n" + + "Insets - L: " + viewportMetrics.viewInsetLeft + ", T: " + + viewportMetrics.viewInsetTop + ", R: " + viewportMetrics.viewInsetRight + + ", B: " + viewportMetrics.viewInsetBottom + "\n" + + "System Gesture Insets - L: " + viewportMetrics.systemGestureInsetLeft + + ", T: " + viewportMetrics.systemGestureInsetTop + + ", R: " + viewportMetrics.systemGestureInsetRight + + ", B: " + viewportMetrics.systemGestureInsetRight + "\n" + + "Display Features: " + viewportMetrics.displayFeatures.size()); + + int[] displayFeaturesBounds = new int[viewportMetrics.displayFeatures.size() * 4]; + int[] displayFeaturesType = new int[viewportMetrics.displayFeatures.size()]; + int[] displayFeaturesState = new int[viewportMetrics.displayFeatures.size()]; + for (int i = 0; i < viewportMetrics.displayFeatures.size(); i++) { + DisplayFeature displayFeature = viewportMetrics.displayFeatures.get(i); + displayFeaturesBounds[4 * i] = displayFeature.bounds.left; + displayFeaturesBounds[4 * i + 1] = displayFeature.bounds.top; + displayFeaturesBounds[4 * i + 2] = displayFeature.bounds.right; + displayFeaturesBounds[4 * i + 3] = displayFeature.bounds.bottom; + displayFeaturesType[i] = displayFeature.type.encodedValue; + displayFeaturesState[i] = displayFeature.state.encodedValue; + } - @TargetApi(29) - private void maybeWaitOnFence(Image image) { - if (image == null) { - return; - } - if (ignoringFence) { - return; - } - if (Build.VERSION.SDK_INT >= 33) { - // The fence API is only available on Android >= 33. - waitOnFence(image); - return; - } - // Log once per ImageTextureEntry. - ignoringFence = true; - Log.w(TAG, "ImageTextureEntry can't wait on the fence on Android < 33"); + flutterJNI.setViewportMetrics(viewportMetrics.devicePixelRatio, viewportMetrics.width, + viewportMetrics.height, viewportMetrics.viewPaddingTop, + viewportMetrics.viewPaddingRight, viewportMetrics.viewPaddingBottom, + viewportMetrics.viewPaddingLeft, viewportMetrics.viewInsetTop, + viewportMetrics.viewInsetRight, viewportMetrics.viewInsetBottom, + viewportMetrics.viewInsetLeft, viewportMetrics.systemGestureInsetTop, + viewportMetrics.systemGestureInsetRight, viewportMetrics.systemGestureInsetBottom, + viewportMetrics.systemGestureInsetLeft, viewportMetrics.physicalTouchSlop, + displayFeaturesBounds, displayFeaturesType, displayFeaturesState); } - @Override - @TargetApi(29) - public Image acquireLatestImage() { - Image r; - synchronized (this) { - r = this.image; - this.image = null; - } - maybeWaitOnFence(r); - return r; + // TODO(mattcarroll): describe the native behavior that this invokes + // TODO(mattcarroll): determine if this is nullable or nonnull + public Bitmap getBitmap() { + return flutterJNI.getBitmap(); } - @Override - protected void finalize() throws Throwable { - try { - if (released) { - return; - } - if (image != null) { - // Be sure to finalize any cached image. - image.close(); - image = null; - } - released = true; - handler.post(new TextureFinalizerRunnable(id, flutterJNI)); - } finally { - super.finalize(); - } - } - } - // ------ END TextureRegistry IMPLEMENTATION ---- - - /** - * Notifies Flutter that the given {@code surface} was created and is available for Flutter - * rendering. - * - *

If called more than once, the current native resources are released. This can be undesired - * if the Engine expects to reuse this surface later. For example, this is true when platform - * views are displayed in a frame, and then removed in the next frame. - * - *

To avoid releasing the current surface resources, set {@code keepCurrentSurface} to true. - * - *

See {@link android.view.SurfaceHolder.Callback} and {@link - * android.view.TextureView.SurfaceTextureListener} - * - * @param surface The render surface. - * @param onlySwap True if the current active surface should not be detached. - */ - public void startRenderingToSurface(@NonNull Surface surface, boolean onlySwap) { - if (!onlySwap) { - // Stop rendering to the surface releases the associated native resources, which causes a - // glitch when toggling between rendering to an image view (hybrid composition) and rendering - // directly to a Surface or Texture view. - // For more, https://github.com/flutter/flutter/issues/95343 - stopRenderingToSurface(); + // TODO(mattcarroll): describe the native behavior that this invokes + public void dispatchPointerDataPacket(@NonNull ByteBuffer buffer, int position) { + flutterJNI.dispatchPointerDataPacket(buffer, position); } - this.surface = surface; + // TODO(mattcarroll): describe the native behavior that this invokes + private void registerTexture(long textureId, @NonNull SurfaceTextureWrapper textureWrapper) { + flutterJNI.registerTexture(textureId, textureWrapper); + } - if (onlySwap) { - // In the swap case we are just swapping the surface that we render to. - flutterJNI.onSurfaceWindowChanged(surface); - } else { - // In the non-swap case we are creating a new surface to render to. - flutterJNI.onSurfaceCreated(surface); + private void registerImageTexture( + long textureId, @NonNull TextureRegistry.ImageConsumer imageTexture) { + flutterJNI.registerImageTexture(textureId, imageTexture); } - } - - /** - * Swaps the {@link Surface} used to render the current frame. - * - *

In hybrid composition, the root surfaces changes from {@link - * android.view.SurfaceHolder#getSurface()} to {@link android.media.ImageReader#getSurface()} when - * a platform view is in the current frame. - */ - public void swapSurface(@NonNull Surface surface) { - this.surface = surface; - flutterJNI.onSurfaceWindowChanged(surface); - } - - /** - * Notifies Flutter that a {@code surface} previously registered with {@link - * #startRenderingToSurface(Surface, boolean)} has changed size to the given {@code width} and - * {@code height}. - * - *

See {@link android.view.SurfaceHolder.Callback} and {@link - * android.view.TextureView.SurfaceTextureListener} - */ - public void surfaceChanged(int width, int height) { - flutterJNI.onSurfaceChanged(width, height); - } - - /** - * Notifies Flutter that a {@code surface} previously registered with {@link - * #startRenderingToSurface(Surface, boolean)} has been destroyed and needs to be released and - * cleaned up on the Flutter side. - * - *

See {@link android.view.SurfaceHolder.Callback} and {@link - * android.view.TextureView.SurfaceTextureListener} - */ - public void stopRenderingToSurface() { - if (surface != null) { - flutterJNI.onSurfaceDestroyed(); - - // TODO(mattcarroll): the source of truth for this call should be FlutterJNI, which is where - // the call to onFlutterUiDisplayed() comes from. However, no such native callback exists yet, - // so until the engine and FlutterJNI are configured to call us back when rendering stops, we - // will manually monitor that change here. - if (isDisplayingFlutterUi) { - flutterUiDisplayListener.onFlutterUiNoLongerDisplayed(); - } - - isDisplayingFlutterUi = false; - surface = null; + + private void scheduleEngineFrame() { + flutterJNI.scheduleFrame(); } - } - - /** - * Notifies Flutter that the viewport metrics, e.g. window height and width, have changed. - * - *

If the width, height, or devicePixelRatio are less than or equal to 0, this update is - * ignored. - * - * @param viewportMetrics The metrics to send to the Dart application. - */ - public void setViewportMetrics(@NonNull ViewportMetrics viewportMetrics) { - // We might get called with just the DPR if width/height aren't available yet. - // Just ignore, as it will get called again when width/height are set. - if (!viewportMetrics.validate()) { - return; + + // TODO(mattcarroll): describe the native behavior that this invokes + private void markTextureFrameAvailable(long textureId) { + flutterJNI.markTextureFrameAvailable(textureId); } - Log.v( - TAG, - "Setting viewport metrics\n" - + "Size: " - + viewportMetrics.width - + " x " - + viewportMetrics.height - + "\n" - + "Padding - L: " - + viewportMetrics.viewPaddingLeft - + ", T: " - + viewportMetrics.viewPaddingTop - + ", R: " - + viewportMetrics.viewPaddingRight - + ", B: " - + viewportMetrics.viewPaddingBottom - + "\n" - + "Insets - L: " - + viewportMetrics.viewInsetLeft - + ", T: " - + viewportMetrics.viewInsetTop - + ", R: " - + viewportMetrics.viewInsetRight - + ", B: " - + viewportMetrics.viewInsetBottom - + "\n" - + "System Gesture Insets - L: " - + viewportMetrics.systemGestureInsetLeft - + ", T: " - + viewportMetrics.systemGestureInsetTop - + ", R: " - + viewportMetrics.systemGestureInsetRight - + ", B: " - + viewportMetrics.systemGestureInsetRight - + "\n" - + "Display Features: " - + viewportMetrics.displayFeatures.size()); - - int[] displayFeaturesBounds = new int[viewportMetrics.displayFeatures.size() * 4]; - int[] displayFeaturesType = new int[viewportMetrics.displayFeatures.size()]; - int[] displayFeaturesState = new int[viewportMetrics.displayFeatures.size()]; - for (int i = 0; i < viewportMetrics.displayFeatures.size(); i++) { - DisplayFeature displayFeature = viewportMetrics.displayFeatures.get(i); - displayFeaturesBounds[4 * i] = displayFeature.bounds.left; - displayFeaturesBounds[4 * i + 1] = displayFeature.bounds.top; - displayFeaturesBounds[4 * i + 2] = displayFeature.bounds.right; - displayFeaturesBounds[4 * i + 3] = displayFeature.bounds.bottom; - displayFeaturesType[i] = displayFeature.type.encodedValue; - displayFeaturesState[i] = displayFeature.state.encodedValue; + + // TODO(mattcarroll): describe the native behavior that this invokes + private void unregisterTexture(long textureId) { + flutterJNI.unregisterTexture(textureId); } - flutterJNI.setViewportMetrics( - viewportMetrics.devicePixelRatio, - viewportMetrics.width, - viewportMetrics.height, - viewportMetrics.viewPaddingTop, - viewportMetrics.viewPaddingRight, - viewportMetrics.viewPaddingBottom, - viewportMetrics.viewPaddingLeft, - viewportMetrics.viewInsetTop, - viewportMetrics.viewInsetRight, - viewportMetrics.viewInsetBottom, - viewportMetrics.viewInsetLeft, - viewportMetrics.systemGestureInsetTop, - viewportMetrics.systemGestureInsetRight, - viewportMetrics.systemGestureInsetBottom, - viewportMetrics.systemGestureInsetLeft, - viewportMetrics.physicalTouchSlop, - displayFeaturesBounds, - displayFeaturesType, - displayFeaturesState); - } - - // TODO(mattcarroll): describe the native behavior that this invokes - // TODO(mattcarroll): determine if this is nullable or nonnull - public Bitmap getBitmap() { - return flutterJNI.getBitmap(); - } - - // TODO(mattcarroll): describe the native behavior that this invokes - public void dispatchPointerDataPacket(@NonNull ByteBuffer buffer, int position) { - flutterJNI.dispatchPointerDataPacket(buffer, position); - } - - // TODO(mattcarroll): describe the native behavior that this invokes - private void registerTexture(long textureId, @NonNull SurfaceTextureWrapper textureWrapper) { - flutterJNI.registerTexture(textureId, textureWrapper); - } - - private void registerImageTexture( - long textureId, @NonNull TextureRegistry.ImageConsumer imageTexture) { - flutterJNI.registerImageTexture(textureId, imageTexture); - } - - private void scheduleEngineFrame() { - flutterJNI.scheduleFrame(); - } - - // TODO(mattcarroll): describe the native behavior that this invokes - private void markTextureFrameAvailable(long textureId) { - flutterJNI.markTextureFrameAvailable(textureId); - } - - // TODO(mattcarroll): describe the native behavior that this invokes - private void unregisterTexture(long textureId) { - flutterJNI.unregisterTexture(textureId); - } - - // TODO(mattcarroll): describe the native behavior that this invokes - public boolean isSoftwareRenderingEnabled() { - return flutterJNI.getIsSoftwareRenderingEnabled(); - } - - // TODO(mattcarroll): describe the native behavior that this invokes - public void setAccessibilityFeatures(int flags) { - flutterJNI.setAccessibilityFeatures(flags); - } - - // TODO(mattcarroll): describe the native behavior that this invokes - public void setSemanticsEnabled(boolean enabled) { - flutterJNI.setSemanticsEnabled(enabled); - } - - // TODO(mattcarroll): describe the native behavior that this invokes - public void dispatchSemanticsAction( - int nodeId, int action, @Nullable ByteBuffer args, int argsPosition) { - flutterJNI.dispatchSemanticsAction(nodeId, action, args, argsPosition); - } - - /** - * Mutable data structure that holds all viewport metrics properties that Flutter cares about. - * - *

All distance measurements, e.g., width, height, padding, viewInsets, are measured in device - * pixels, not logical pixels. - */ - public static final class ViewportMetrics { - /** A value that indicates the setting has not been set. */ - public static final int unsetValue = -1; - - public float devicePixelRatio = 1.0f; - public int width = 0; - public int height = 0; - public int viewPaddingTop = 0; - public int viewPaddingRight = 0; - public int viewPaddingBottom = 0; - public int viewPaddingLeft = 0; - public int viewInsetTop = 0; - public int viewInsetRight = 0; - public int viewInsetBottom = 0; - public int viewInsetLeft = 0; - public int systemGestureInsetTop = 0; - public int systemGestureInsetRight = 0; - public int systemGestureInsetBottom = 0; - public int systemGestureInsetLeft = 0; - public int physicalTouchSlop = unsetValue; + // TODO(mattcarroll): describe the native behavior that this invokes + public boolean isSoftwareRenderingEnabled() { + return flutterJNI.getIsSoftwareRenderingEnabled(); + } - /** - * Whether this instance contains valid metrics for the Flutter application. - * - * @return True if width, height, and devicePixelRatio are > 0; false otherwise. - */ - boolean validate() { - return width > 0 && height > 0 && devicePixelRatio > 0; + // TODO(mattcarroll): describe the native behavior that this invokes + public void setAccessibilityFeatures(int flags) { + flutterJNI.setAccessibilityFeatures(flags); } - public List displayFeatures = new ArrayList<>(); - } - - /** - * Description of a physical feature on the display. - * - *

A display feature is a distinctive physical attribute located within the display panel of - * the device. It can intrude into the application window space and create a visual distortion, - * visual or touch discontinuity, make some area invisible or create a logical divider or - * separation in the screen space. - * - *

Based on {@link androidx.window.layout.DisplayFeature}, with added support for cutouts. - */ - public static final class DisplayFeature { - public final Rect bounds; - public final DisplayFeatureType type; - public final DisplayFeatureState state; - - public DisplayFeature(Rect bounds, DisplayFeatureType type, DisplayFeatureState state) { - this.bounds = bounds; - this.type = type; - this.state = state; + // TODO(mattcarroll): describe the native behavior that this invokes + public void setSemanticsEnabled(boolean enabled) { + flutterJNI.setSemanticsEnabled(enabled); } - public DisplayFeature(Rect bounds, DisplayFeatureType type) { - this.bounds = bounds; - this.type = type; - this.state = DisplayFeatureState.UNKNOWN; + // TODO(mattcarroll): describe the native behavior that this invokes + public void dispatchSemanticsAction( + int nodeId, int action, @Nullable ByteBuffer args, int argsPosition) { + flutterJNI.dispatchSemanticsAction(nodeId, action, args, argsPosition); } - } - - /** - * Types of display features that can appear on the viewport. - * - *

Some, like {@link #FOLD}, can be reported without actually occluding the screen. They are - * useful for knowing where the display is bent or has a crease. The {@link DisplayFeature#bounds} - * can be 0-width in such cases. - */ - public enum DisplayFeatureType { - /** - * Type of display feature not yet known to Flutter. This can happen if WindowManager is updated - * with new types. The {@link DisplayFeature#bounds} is the only known property. - */ - UNKNOWN(0), /** - * A fold in the flexible display that does not occlude the screen. Corresponds to {@link - * androidx.window.layout.FoldingFeature.OcclusionType#NONE} + * Mutable data structure that holds all viewport metrics properties that Flutter cares about. + * + *

All distance measurements, e.g., width, height, padding, viewInsets, are measured in + * device pixels, not logical pixels. */ - FOLD(1), + public static final class ViewportMetrics { + /** A value that indicates the setting has not been set. */ + public static final int unsetValue = -1; + + public float devicePixelRatio = 1.0f; + public int width = 0; + public int height = 0; + public int viewPaddingTop = 0; + public int viewPaddingRight = 0; + public int viewPaddingBottom = 0; + public int viewPaddingLeft = 0; + public int viewInsetTop = 0; + public int viewInsetRight = 0; + public int viewInsetBottom = 0; + public int viewInsetLeft = 0; + public int systemGestureInsetTop = 0; + public int systemGestureInsetRight = 0; + public int systemGestureInsetBottom = 0; + public int systemGestureInsetLeft = 0; + public int physicalTouchSlop = unsetValue; + + /** + * Whether this instance contains valid metrics for the Flutter application. + * + * @return True if width, height, and devicePixelRatio are > 0; false otherwise. + */ + boolean validate() { + return width > 0 && height > 0 && devicePixelRatio > 0; + } - /** - * Splits the display in two separate panels that can fold. Occludes the screen. Corresponds to - * {@link androidx.window.layout.FoldingFeature.OcclusionType#FULL} - */ - HINGE(2), + public List displayFeatures = new ArrayList<>(); + } /** - * Area of the screen that usually houses cameras or sensors. Occludes the screen. Corresponds - * to {@link android.view.DisplayCutout} + * Description of a physical feature on the display. + * + *

A display feature is a distinctive physical attribute located within the display panel of + * the device. It can intrude into the application window space and create a visual distortion, + * visual or touch discontinuity, make some area invisible or create a logical divider or + * separation in the screen space. + * + *

Based on {@link androidx.window.layout.DisplayFeature}, with added support for cutouts. */ - CUTOUT(3); - - public final int encodedValue; + public static final class DisplayFeature { + public final Rect bounds; + public final DisplayFeatureType type; + public final DisplayFeatureState state; + + public DisplayFeature(Rect bounds, DisplayFeatureType type, DisplayFeatureState state) { + this.bounds = bounds; + this.type = type; + this.state = state; + } - DisplayFeatureType(int encodedValue) { - this.encodedValue = encodedValue; + public DisplayFeature(Rect bounds, DisplayFeatureType type) { + this.bounds = bounds; + this.type = type; + this.state = DisplayFeatureState.UNKNOWN; + } } - } - - /** - * State of the display feature. - * - *

For foldables, the state is the posture. For cutouts, this property is {@link #UNKNOWN} - */ - public enum DisplayFeatureState { - /** The display feature is a cutout or this state is new and not yet known to Flutter. */ - UNKNOWN(0), /** - * The foldable device is completely open. The screen space that is presented to the user is - * flat. Corresponds to {@link androidx.window.layout.FoldingFeature.State#FLAT} + * Types of display features that can appear on the viewport. + * + *

Some, like {@link #FOLD}, can be reported without actually occluding the screen. They are + * useful for knowing where the display is bent or has a crease. The {@link + * DisplayFeature#bounds} can be 0-width in such cases. */ - POSTURE_FLAT(1), + public enum DisplayFeatureType { + /** + * Type of display feature not yet known to Flutter. This can happen if WindowManager is + * updated with new types. The {@link DisplayFeature#bounds} is the only known property. + */ + UNKNOWN(0), + + /** + * A fold in the flexible display that does not occlude the screen. Corresponds to {@link + * androidx.window.layout.FoldingFeature.OcclusionType#NONE} + */ + FOLD(1), + + /** + * Splits the display in two separate panels that can fold. Occludes the screen. Corresponds + * to + * {@link androidx.window.layout.FoldingFeature.OcclusionType#FULL} + */ + HINGE(2), + + /** + * Area of the screen that usually houses cameras or sensors. Occludes the screen. + * Corresponds to {@link android.view.DisplayCutout} + */ + CUTOUT(3); + + public final int encodedValue; + + DisplayFeatureType(int encodedValue) { + this.encodedValue = encodedValue; + } + } /** - * The foldable device's hinge is in an intermediate position between opened and closed state. - * There is a non-flat angle between parts of the flexible screen or between physical display - * panels. Corresponds to {@link androidx.window.layout.FoldingFeature.State#HALF_OPENED} + * State of the display feature. + * + *

For foldables, the state is the posture. For cutouts, this property is {@link #UNKNOWN} */ - POSTURE_HALF_OPENED(2); - - public final int encodedValue; - - DisplayFeatureState(int encodedValue) { - this.encodedValue = encodedValue; + public enum DisplayFeatureState { + /** The display feature is a cutout or this state is new and not yet known to Flutter. */ + UNKNOWN(0), + + /** + * The foldable device is completely open. The screen space that is presented to the user is + * flat. Corresponds to {@link androidx.window.layout.FoldingFeature.State#FLAT} + */ + POSTURE_FLAT(1), + + /** + * The foldable device's hinge is in an intermediate position between opened and closed + * state. There is a non-flat angle between parts of the flexible screen or between physical + * display panels. Corresponds to {@link + * androidx.window.layout.FoldingFeature.State#HALF_OPENED} + */ + POSTURE_HALF_OPENED(2); + + public final int encodedValue; + + DisplayFeatureState(int encodedValue) { + this.encodedValue = encodedValue; + } } - } } diff --git a/shell/platform/android/io/flutter/embedding/engine/renderer/SurfaceTextureSurfaceProducer.java b/shell/platform/android/io/flutter/embedding/engine/renderer/SurfaceTextureSurfaceProducer.java index 8153fc55826d9..7ff1399cbe411 100644 --- a/shell/platform/android/io/flutter/embedding/engine/renderer/SurfaceTextureSurfaceProducer.java +++ b/shell/platform/android/io/flutter/embedding/engine/renderer/SurfaceTextureSurfaceProducer.java @@ -5,6 +5,8 @@ import android.view.Surface; import androidx.annotation.NonNull; import androidx.annotation.Nullable; + +import io.flutter.Log; import io.flutter.embedding.engine.FlutterJNI; import io.flutter.view.TextureRegistry; @@ -16,15 +18,15 @@ final class SurfaceTextureSurfaceProducer private int requestedBufferHeight; private boolean released; @Nullable private Surface surface; - @NonNull private final SurfaceTexture texture; + @NonNull private final TextureRegistry.SurfaceTextureEntry texture; @NonNull private final Handler handler; @NonNull private final FlutterJNI flutterJNI; - SurfaceTextureSurfaceProducer(long id, @NonNull Handler handler, @NonNull FlutterJNI flutterJNI) { + SurfaceTextureSurfaceProducer(long id, @NonNull Handler handler, @NonNull FlutterJNI flutterJNI, @NonNull TextureRegistry.SurfaceTextureEntry texture) { this.id = id; this.handler = handler; this.flutterJNI = flutterJNI; - this.texture = new SurfaceTexture(0); + this.texture = texture; } @Override @@ -54,7 +56,7 @@ public void release() { @Override @NonNull public SurfaceTexture getSurfaceTexture() { - return texture; + return texture.surfaceTexture(); } @Override @@ -77,7 +79,7 @@ public int getHeight() { @Override public Surface getSurface() { if (surface == null) { - surface = new Surface(texture); + surface = new Surface(texture.surfaceTexture()); } return surface; } diff --git a/shell/platform/android/io/flutter/view/FlutterView.java b/shell/platform/android/io/flutter/view/FlutterView.java index 4b63c5d54661e..dde0422c6a10a 100644 --- a/shell/platform/android/io/flutter/view/FlutterView.java +++ b/shell/platform/android/io/flutter/view/FlutterView.java @@ -928,6 +928,7 @@ public SurfaceTextureWrapper textureWrapper() { return textureWrapper; } + @NonNull @Override public SurfaceTexture surfaceTexture() { return textureWrapper.surfaceTexture(); diff --git a/shell/platform/android/test/io/flutter/plugin/platform/PlatformViewsControllerTest.java b/shell/platform/android/test/io/flutter/plugin/platform/PlatformViewsControllerTest.java index c5871db27bf1c..0ef5de4734962 100644 --- a/shell/platform/android/test/io/flutter/plugin/platform/PlatformViewsControllerTest.java +++ b/shell/platform/android/test/io/flutter/plugin/platform/PlatformViewsControllerTest.java @@ -26,6 +26,8 @@ import android.view.ViewParent; import android.widget.FrameLayout; import android.widget.FrameLayout.LayoutParams; + +import androidx.annotation.NonNull; import androidx.test.core.app.ApplicationProvider; import androidx.test.ext.junit.runners.AndroidJUnit4; import io.flutter.embedding.android.FlutterImageView; @@ -1555,6 +1557,7 @@ public SurfaceTextureEntry createSurfaceTexture() { @Override public SurfaceTextureEntry registerSurfaceTexture(SurfaceTexture surfaceTexture) { return new SurfaceTextureEntry() { + @NonNull @Override public SurfaceTexture surfaceTexture() { return mock(SurfaceTexture.class); diff --git a/testing/scenario_app/android/app/src/androidTest/java/dev/flutter/TestRunner.java b/testing/scenario_app/android/app/src/androidTest/java/dev/flutter/TestRunner.java index 89494f200cc1d..6c2989c1517b4 100644 --- a/testing/scenario_app/android/app/src/androidTest/java/dev/flutter/TestRunner.java +++ b/testing/scenario_app/android/app/src/androidTest/java/dev/flutter/TestRunner.java @@ -9,11 +9,13 @@ import androidx.test.runner.AndroidJUnitRunner; import dev.flutter.scenariosui.ScreenshotUtil; import io.flutter.FlutterInjector; +import io.flutter.embedding.engine.renderer.FlutterRenderer; public class TestRunner extends AndroidJUnitRunner { @Override public void onCreate(@Nullable Bundle arguments) { String[] engineArguments = null; + assert arguments != null; if ("true".equals(arguments.getString("enable-impeller"))) { // Set up the global settings object so that Impeller is enabled for all tests. engineArguments = @@ -22,6 +24,10 @@ public void onCreate(@Nullable Bundle arguments) { "--impeller-backend=" + arguments.getString("impeller-backend", "vulkan") }; } + if ("true".equals(arguments.getString("force-surface-producer-surface-texture"))) { + // Set a test flag to force the SurfaceProducer to use SurfaceTexture. + FlutterRenderer.debugForceSurfaceProducerGlTextures = true; + } // For consistency, just always initilaize FlutterJNI etc. FlutterInjector.instance().flutterLoader().startInitialization(getTargetContext()); FlutterInjector.instance() diff --git a/testing/scenario_app/android/app/src/main/java/dev/flutter/scenarios/ExternalTextureFlutterActivity.java b/testing/scenario_app/android/app/src/main/java/dev/flutter/scenarios/ExternalTextureFlutterActivity.java index cb088ba8f0867..883ae4d34f2f4 100644 --- a/testing/scenario_app/android/app/src/main/java/dev/flutter/scenarios/ExternalTextureFlutterActivity.java +++ b/testing/scenario_app/android/app/src/main/java/dev/flutter/scenarios/ExternalTextureFlutterActivity.java @@ -11,7 +11,6 @@ import android.graphics.Paint; import android.graphics.Rect; import android.graphics.Shader.TileMode; -import android.graphics.SurfaceTexture; import android.hardware.HardwareBuffer; import android.media.Image; import android.media.ImageReader; @@ -36,6 +35,8 @@ import androidx.annotation.Nullable; import androidx.annotation.RequiresApi; import androidx.core.util.Supplier; + +import io.flutter.view.TextureRegistry; import io.flutter.view.TextureRegistry.SurfaceTextureEntry; import java.io.IOException; import java.nio.ByteBuffer; @@ -54,7 +55,7 @@ public class ExternalTextureFlutterActivity extends TestActivity { private final CountDownLatch firstFrameLatch = new CountDownLatch(2); private long textureId = 0; - private SurfaceTextureEntry surfaceTextureEntry; + private TextureRegistry.SurfaceProducer surfaceProducer; @Override protected void onCreate(@Nullable Bundle savedInstanceState) { @@ -139,19 +140,18 @@ private MediaExtractor createMediaExtractor() { public void onPause() { surfaceViewRenderer.destroy(); flutterRenderer.destroy(); - surfaceTextureEntry.release(); + surfaceProducer.release(); super.onPause(); } @Override public void onFlutterUiDisplayed() { - surfaceTextureEntry = - Objects.requireNonNull(getFlutterEngine()).getRenderer().createSurfaceTexture(); - SurfaceTexture surfaceTexture = surfaceTextureEntry.surfaceTexture(); - surfaceTexture.setDefaultBufferSize(SURFACE_WIDTH, SURFACE_HEIGHT); - flutterRenderer.attach(new Surface(surfaceTexture), firstFrameLatch); + surfaceProducer = + Objects.requireNonNull(getFlutterEngine()).getRenderer().createSurfaceProducer(); + surfaceProducer.setSize(SURFACE_WIDTH, SURFACE_HEIGHT); + flutterRenderer.attach(surfaceProducer.getSurface(), firstFrameLatch); flutterRenderer.repaint(); - textureId = surfaceTextureEntry.id(); + textureId = surfaceProducer.id(); super.onFlutterUiDisplayed(); } diff --git a/testing/scenario_app/bin/run_android_tests.dart b/testing/scenario_app/bin/run_android_tests.dart index 37db27a5b51b1..b85ad6436d79a 100644 --- a/testing/scenario_app/bin/run_android_tests.dart +++ b/testing/scenario_app/bin/run_android_tests.dart @@ -78,6 +78,7 @@ void main(List args) async { logsDir: Directory(options.logsDir), contentsGolden: options.outputContentsGolden, ndkStack: options.ndkStack, + forceSurfaceProducerSurfaceTexture: options.forceSurfaceProducerSurfaceTexture, ); onSigint.cancel(); exit(0); @@ -120,6 +121,7 @@ Future _run({ required Directory logsDir, required String? contentsGolden, required String ndkStack, + required bool forceSurfaceProducerSurfaceTexture, }) async { const ProcessManager pm = LocalProcessManager(); final String scenarioAppPath = join(outDir.path, 'scenario_app'); @@ -290,6 +292,7 @@ Future _run({ final Map dimensions = { 'AndroidAPILevel': connectedDeviceAPILevel, 'GraphicsBackend': enableImpeller ? 'impeller-${impellerBackend!.name}' : 'skia', + 'ForceSurfaceProducerSurfaceTexture': '$forceSurfaceProducerSurfaceTexture' }; log('using dimensions: ${json.encode(dimensions)}'); skiaGoldClient = SkiaGoldClient( @@ -345,6 +348,8 @@ Future _run({ '-e enable-impeller true', if (impellerBackend != null) '-e impeller-backend ${impellerBackend.name}', + if (forceSurfaceProducerSurfaceTexture) + '-e force-surface-producer-surface-texture true', 'dev.flutter.scenarios.test/dev.flutter.TestRunner', ]); if (exitCode != 0) { diff --git a/testing/scenario_app/bin/utils/options.dart b/testing/scenario_app/bin/utils/options.dart index 057256afe1cac..cc3ad5930fb89 100644 --- a/testing/scenario_app/bin/utils/options.dart +++ b/testing/scenario_app/bin/utils/options.dart @@ -46,6 +46,17 @@ extension type const Options._(ArgResults _args) { ); } + // Cannot use forceSurfaceProducerSurfaceTexture with Impeller+Vulkan. + if (options.forceSurfaceProducerSurfaceTexture && + options.enableImpeller && + options.impellerBackend != 'opengles') { + throw const FormatException( + 'Cannot use --force-surface-producer-surface-texture with ' + '--enable-impeller unless --impeller-backend="opengles" is used. See ' + 'https://github.com/flutter/flutter/issues/143539 for details.', + ); + } + return options; } @@ -134,6 +145,20 @@ extension type const Options._(ArgResults _args) { 'default backend will be used. To explicitly run with the Skia ' 'backend, set this to false (--no-enable-impeller).', ) + ..addFlag( + 'force-surface-producer-surface-texture', + help: + 'Whether to force the use of SurfaceTexture as the SurfaceProducer ' + 'rendering strategy. This is used to emulate the behavior of older ' + 'devices that do not support ImageReader, or to explicitly test ' + 'SurfaceTexture path for rendering plugins still using the older ' + 'createSurfaceTexture() API.' + '\n' + 'Cannot be used with --enable-impeller unless --impeller-backend=' + '"opengles" is used. See ' + 'https://github.com/flutter/flutter/issues/143539 for details.', + negatable: false + ) ..addOption( 'impeller-backend', help: 'The graphics backend to use when --enable-impeller is true. ' @@ -278,4 +303,15 @@ extension type const Options._(ArgResults _args) { /// Path to a file that contains the expected filenames of golden files. String? get outputContentsGolden => _args['output-contents-golden'] as String; + + /// Whether to force the use of `SurfaceTexture` for `SurfaceProducer`. + /// + /// Always returns `false` if `--enable-impeller` is `true` and + /// `--impeller-backend` is not `opengles`. + bool get forceSurfaceProducerSurfaceTexture { + if (enableImpeller && impellerBackend != 'opengles') { + return false; + } + return _args['force-surface-producer-surface-texture'] as bool; + } } From d4f7e086305e585162515afc9db467edd1f286e1 Mon Sep 17 00:00:00 2001 From: Matan Lurey Date: Wed, 28 Feb 2024 11:05:59 -0800 Subject: [PATCH 2/3] Format. --- .../engine/renderer/FlutterRenderer.java | 2334 +++++++++-------- .../SurfaceTextureSurfaceProducer.java | 8 +- .../platform/PlatformViewsControllerTest.java | 1 - .../ExternalTextureFlutterActivity.java | 2 - testing/scenario_app/bin/utils/options.dart | 2 +- 5 files changed, 1190 insertions(+), 1157 deletions(-) diff --git a/shell/platform/android/io/flutter/embedding/engine/renderer/FlutterRenderer.java b/shell/platform/android/io/flutter/embedding/engine/renderer/FlutterRenderer.java index 8902f9a6c8355..9b9f8468639e1 100644 --- a/shell/platform/android/io/flutter/embedding/engine/renderer/FlutterRenderer.java +++ b/shell/platform/android/io/flutter/embedding/engine/renderer/FlutterRenderer.java @@ -16,12 +16,13 @@ import android.os.Build; import android.os.Handler; import android.view.Surface; - import androidx.annotation.Keep; import androidx.annotation.NonNull; import androidx.annotation.Nullable; import androidx.annotation.VisibleForTesting; - +import io.flutter.Log; +import io.flutter.embedding.engine.FlutterJNI; +import io.flutter.view.TextureRegistry; import java.io.IOException; import java.lang.ref.WeakReference; import java.nio.ByteBuffer; @@ -34,10 +35,6 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicLong; -import io.flutter.Log; -import io.flutter.embedding.engine.FlutterJNI; -import io.flutter.view.TextureRegistry; - /** * Represents the rendering responsibilities of a {@code FlutterEngine}. * @@ -52,1266 +49,1303 @@ * io.flutter.embedding.android.FlutterTextureView} are implementations of {@link RenderSurface}. */ public class FlutterRenderer implements TextureRegistry { - /** - * Whether to always use GL textures for {@link FlutterRenderer#createSurfaceProducer()}. - * - *

This is a debug-only API intended for local development. For example, when using a newer - * Android device (that normally would use {@link ImageReaderSurfaceProducer}, but wanting to - * test the OpenGLES/{@link SurfaceTextureSurfaceProducer} code branch. This flag has undefined - * behavior if set to true while running in a Vulkan (Impeller) context. - */ - @VisibleForTesting - public static boolean debugForceSurfaceProducerGlTextures = false; - - private static final String TAG = "FlutterRenderer"; - - @NonNull - private final FlutterJNI flutterJNI; - @NonNull - private final AtomicLong nextTextureId = new AtomicLong(0L); - @Nullable - private Surface surface; - private boolean isDisplayingFlutterUi = false; - private final Handler handler = new Handler(); - - @NonNull - private final Set> onTrimMemoryListeners = - new HashSet<>(); - - @NonNull - private final FlutterUiDisplayListener flutterUiDisplayListener = - new FlutterUiDisplayListener() { - @Override - public void onFlutterUiDisplayed() { - isDisplayingFlutterUi = true; - } - - @Override - public void onFlutterUiNoLongerDisplayed() { - isDisplayingFlutterUi = false; - } - }; + /** + * Whether to always use GL textures for {@link FlutterRenderer#createSurfaceProducer()}. + * + *

This is a debug-only API intended for local development. For example, when using a newer + * Android device (that normally would use {@link ImageReaderSurfaceProducer}, but wanting to test + * the OpenGLES/{@link SurfaceTextureSurfaceProducer} code branch. This flag has undefined + * behavior if set to true while running in a Vulkan (Impeller) context. + */ + @VisibleForTesting public static boolean debugForceSurfaceProducerGlTextures = false; + + private static final String TAG = "FlutterRenderer"; + + @NonNull private final FlutterJNI flutterJNI; + @NonNull private final AtomicLong nextTextureId = new AtomicLong(0L); + @Nullable private Surface surface; + private boolean isDisplayingFlutterUi = false; + private final Handler handler = new Handler(); + + @NonNull + private final Set> onTrimMemoryListeners = + new HashSet<>(); + + @NonNull + private final FlutterUiDisplayListener flutterUiDisplayListener = + new FlutterUiDisplayListener() { + @Override + public void onFlutterUiDisplayed() { + isDisplayingFlutterUi = true; + } - public FlutterRenderer(@NonNull FlutterJNI flutterJNI) { - this.flutterJNI = flutterJNI; - this.flutterJNI.addIsDisplayingFlutterUiListener(flutterUiDisplayListener); + @Override + public void onFlutterUiNoLongerDisplayed() { + isDisplayingFlutterUi = false; + } + }; + + public FlutterRenderer(@NonNull FlutterJNI flutterJNI) { + this.flutterJNI = flutterJNI; + this.flutterJNI.addIsDisplayingFlutterUiListener(flutterUiDisplayListener); + } + + /** + * Returns true if this {@code FlutterRenderer} is painting pixels to an Android {@code View} + * hierarchy, false otherwise. + */ + public boolean isDisplayingFlutterUi() { + return isDisplayingFlutterUi; + } + + /** + * Adds a listener that is invoked whenever this {@code FlutterRenderer} starts and stops painting + * pixels to an Android {@code View} hierarchy. + */ + public void addIsDisplayingFlutterUiListener(@NonNull FlutterUiDisplayListener listener) { + flutterJNI.addIsDisplayingFlutterUiListener(listener); + + if (isDisplayingFlutterUi) { + listener.onFlutterUiDisplayed(); } - - /** - * Returns true if this {@code FlutterRenderer} is painting pixels to an Android {@code View} - * hierarchy, false otherwise. - */ - public boolean isDisplayingFlutterUi() { - return isDisplayingFlutterUi; + } + + /** + * Removes a listener added via {@link + * #addIsDisplayingFlutterUiListener(FlutterUiDisplayListener)}. + */ + public void removeIsDisplayingFlutterUiListener(@NonNull FlutterUiDisplayListener listener) { + flutterJNI.removeIsDisplayingFlutterUiListener(listener); + } + + private void clearDeadListeners() { + final Iterator> iterator = onTrimMemoryListeners.iterator(); + while (iterator.hasNext()) { + WeakReference listenerRef = iterator.next(); + final OnTrimMemoryListener listener = listenerRef.get(); + if (listener == null) { + iterator.remove(); + } } - - /** - * Adds a listener that is invoked whenever this {@code FlutterRenderer} starts and stops - * painting pixels to an Android {@code View} hierarchy. - */ - public void addIsDisplayingFlutterUiListener(@NonNull FlutterUiDisplayListener listener) { - flutterJNI.addIsDisplayingFlutterUiListener(listener); - - if (isDisplayingFlutterUi) { - listener.onFlutterUiDisplayed(); - } + } + + /** Adds a listener that is invoked when a memory pressure warning was forward. */ + @VisibleForTesting + /* package */ void addOnTrimMemoryListener(@NonNull OnTrimMemoryListener listener) { + // Purge dead listener to avoid accumulating. + clearDeadListeners(); + onTrimMemoryListeners.add(new WeakReference<>(listener)); + } + + /** + * Removes a {@link OnTrimMemoryListener} that was added with {@link + * #addOnTrimMemoryListener(OnTrimMemoryListener)}. + */ + @VisibleForTesting + /* package */ void removeOnTrimMemoryListener(@NonNull OnTrimMemoryListener listener) { + for (WeakReference listenerRef : onTrimMemoryListeners) { + if (listenerRef.get() == listener) { + onTrimMemoryListeners.remove(listenerRef); + break; + } } - - /** - * Removes a listener added via {@link - * #addIsDisplayingFlutterUiListener(FlutterUiDisplayListener)}. - */ - public void removeIsDisplayingFlutterUiListener(@NonNull FlutterUiDisplayListener listener) { - flutterJNI.removeIsDisplayingFlutterUiListener(listener); - } - - private void clearDeadListeners() { - final Iterator> iterator = - onTrimMemoryListeners.iterator(); - while (iterator.hasNext()) { - WeakReference listenerRef = iterator.next(); - final OnTrimMemoryListener listener = listenerRef.get(); - if (listener == null) { - iterator.remove(); - } - } + } + + // ------ START TextureRegistry IMPLEMENTATION ----- + + /** + * Creates and returns a new external texture {@link SurfaceProducer} managed by the Flutter + * engine that is also made available to Flutter code. + */ + @NonNull + @Override + public SurfaceProducer createSurfaceProducer() { + // Prior to Impeller, Flutter on Android *only* ran on OpenGLES (via Skia). That + // meant that + // plugins (i.e. end-users) either explicitly created a SurfaceTexture (via + // createX/registerX) or an ImageTexture (via createX/registerX). + // + // In an Impeller world, which for the first time uses (if available) a Vulkan + // rendering + // backend, it is no longer possible (at least not trivially) to render an + // OpenGLES-provided + // texture (SurfaceTexture) in a Vulkan context. + // + // This function picks the "best" rendering surface based on the Android + // runtime, and + // provides a consumer-agnostic SurfaceProducer (which in turn vends a Surface), + // and has + // plugins (i.e. end-users) use the Surface instead, letting us "hide" the + // consumer-side + // of the implementation. + // + // tl;dr: If ImageTexture is available, we use it, otherwise we use a + // SurfaceTexture. + // Coincidentally, if ImageTexture is available, we are also on an Android + // version that is + // running Vulkan, so we don't have to worry about it not being supported. + final SurfaceProducer entry; + if (!debugForceSurfaceProducerGlTextures && Build.VERSION.SDK_INT >= 29) { + final long id = nextTextureId.getAndIncrement(); + final ImageReaderSurfaceProducer producer = new ImageReaderSurfaceProducer(id); + registerImageTexture(id, producer); + addOnTrimMemoryListener(producer); + Log.v(TAG, "New ImageReaderSurfaceProducer ID: " + id); + entry = producer; + } else { + // TODO(matanlurey): Actually have the class named "*Producer" to well, produce + // something. This is a code smell, but does guarantee the paths for both + // createSurfaceTexture and createSurfaceProducer doesn't diverge. As we get more + // confident in this API and any possible bugs (and have tests to check we don't + // regress), reconsider this pattern. + final SurfaceTextureEntry texture = createSurfaceTexture(); + final SurfaceTextureSurfaceProducer producer = + new SurfaceTextureSurfaceProducer(texture.id(), handler, flutterJNI, texture); + Log.v(TAG, "New SurfaceTextureSurfaceProducer ID: " + texture.id()); + entry = producer; } - - /** Adds a listener that is invoked when a memory pressure warning was forward. */ - @VisibleForTesting - /* package */ void addOnTrimMemoryListener(@NonNull OnTrimMemoryListener listener) { - // Purge dead listener to avoid accumulating. - clearDeadListeners(); - onTrimMemoryListeners.add(new WeakReference<>(listener)); + return entry; + } + + /** + * Creates and returns a new {@link SurfaceTexture} managed by the Flutter engine that is also + * made available to Flutter code. + */ + @NonNull + @Override + public SurfaceTextureEntry createSurfaceTexture() { + Log.v(TAG, "Creating a SurfaceTexture."); + final SurfaceTexture surfaceTexture = new SurfaceTexture(0); + return registerSurfaceTexture(surfaceTexture); + } + + /** + * Registers and returns a {@link SurfaceTexture} managed by the Flutter engine that is also made + * available to Flutter code. + */ + @NonNull + @Override + public SurfaceTextureEntry registerSurfaceTexture(@NonNull SurfaceTexture surfaceTexture) { + return registerSurfaceTexture(nextTextureId.getAndIncrement(), surfaceTexture); + } + + /** + * Similar to {@link FlutterRenderer#registerSurfaceTexture} but with an existing @{code + * textureId}. + * + * @param surfaceTexture Surface texture to wrap. + * @param textureId A texture ID already created that should be assigned to the surface texture. + */ + @NonNull + private SurfaceTextureEntry registerSurfaceTexture( + long textureId, @NonNull SurfaceTexture surfaceTexture) { + surfaceTexture.detachFromGLContext(); + final SurfaceTextureRegistryEntry entry = + new SurfaceTextureRegistryEntry(textureId, surfaceTexture); + Log.v(TAG, "New SurfaceTexture ID: " + entry.id()); + registerTexture(entry.id(), entry.textureWrapper()); + addOnTrimMemoryListener(entry); + return entry; + } + + @NonNull + @Override + public ImageTextureEntry createImageTexture() { + final ImageTextureRegistryEntry entry = + new ImageTextureRegistryEntry(nextTextureId.getAndIncrement()); + Log.v(TAG, "New ImageTextureEntry ID: " + entry.id()); + registerImageTexture(entry.id(), entry); + return entry; + } + + @Override + public void onTrimMemory(int level) { + final Iterator> iterator = onTrimMemoryListeners.iterator(); + while (iterator.hasNext()) { + WeakReference listenerRef = iterator.next(); + final OnTrimMemoryListener listener = listenerRef.get(); + if (listener != null) { + listener.onTrimMemory(level); + } else { + // Purge cleared refs to avoid accumulating a lot of dead listener + iterator.remove(); + } } - - /** - * Removes a {@link OnTrimMemoryListener} that was added with {@link - * #addOnTrimMemoryListener(OnTrimMemoryListener)}. - */ - @VisibleForTesting - /* package */ void removeOnTrimMemoryListener(@NonNull OnTrimMemoryListener listener) { - for (WeakReference listenerRef : onTrimMemoryListeners) { - if (listenerRef.get() == listener) { - onTrimMemoryListeners.remove(listenerRef); - break; + } + + final class SurfaceTextureRegistryEntry + implements TextureRegistry.SurfaceTextureEntry, TextureRegistry.OnTrimMemoryListener { + private final long id; + @NonNull private final SurfaceTextureWrapper textureWrapper; + private boolean released; + @Nullable private OnTrimMemoryListener trimMemoryListener; + @Nullable private OnFrameConsumedListener frameConsumedListener; + + SurfaceTextureRegistryEntry(long id, @NonNull SurfaceTexture surfaceTexture) { + this.id = id; + Runnable onFrameConsumed = + () -> { + if (frameConsumedListener != null) { + frameConsumedListener.onFrameConsumed(); } - } + }; + this.textureWrapper = new SurfaceTextureWrapper(surfaceTexture, onFrameConsumed); + + // Even though we make sure to unregister the callback before releasing, as of + // Android O, SurfaceTexture has a data race when accessing the callback, so the + // callback may still be called by a stale reference after released==true and + // mNativeView==null. + SurfaceTexture.OnFrameAvailableListener onFrameListener = + texture -> { + if (released || !flutterJNI.isAttached()) { + // Even though we make sure to unregister the callback before releasing, as of + // Android O, SurfaceTexture has a data race when accessing the callback, so the + // callback may still be called by a stale reference after released==true and + // mNativeView==null. + return; + } + textureWrapper.markDirty(); + scheduleEngineFrame(); + }; + // The callback relies on being executed on the UI thread (unsynchronised read of + // mNativeView and also the engine code check for platform thread in + // Shell::OnPlatformViewMarkTextureFrameAvailable), so we explicitly pass a Handler for + // the current thread. + this.surfaceTexture().setOnFrameAvailableListener(onFrameListener, new Handler()); } - // ------ START TextureRegistry IMPLEMENTATION ----- - - /** - * Creates and returns a new external texture {@link SurfaceProducer} managed by the Flutter - * engine that is also made available to Flutter code. - */ - @NonNull @Override - public SurfaceProducer createSurfaceProducer() { - // Prior to Impeller, Flutter on Android *only* ran on OpenGLES (via Skia). That - // meant that - // plugins (i.e. end-users) either explicitly created a SurfaceTexture (via - // createX/registerX) or an ImageTexture (via createX/registerX). - // - // In an Impeller world, which for the first time uses (if available) a Vulkan - // rendering - // backend, it is no longer possible (at least not trivially) to render an - // OpenGLES-provided - // texture (SurfaceTexture) in a Vulkan context. - // - // This function picks the "best" rendering surface based on the Android - // runtime, and - // provides a consumer-agnostic SurfaceProducer (which in turn vends a Surface), - // and has - // plugins (i.e. end-users) use the Surface instead, letting us "hide" the - // consumer-side - // of the implementation. - // - // tl;dr: If ImageTexture is available, we use it, otherwise we use a - // SurfaceTexture. - // Coincidentally, if ImageTexture is available, we are also on an Android - // version that is - // running Vulkan, so we don't have to worry about it not being supported. - final SurfaceProducer entry; - if (!debugForceSurfaceProducerGlTextures && Build.VERSION.SDK_INT >= 29) { - final long id = nextTextureId.getAndIncrement(); - final ImageReaderSurfaceProducer producer = new ImageReaderSurfaceProducer(id); - registerImageTexture(id, producer); - addOnTrimMemoryListener(producer); - Log.v(TAG, "New ImageReaderSurfaceProducer ID: " + id); - entry = producer; - } else { - // TODO(matanlurey): Actually have the class named "*Producer" to well, produce - // something. This is a code smell, but does guarantee the paths for both - // createSurfaceTexture and createSurfaceProducer doesn't diverge. As we get more - // confident in this API and any possible bugs (and have tests to check we don't - // regress), reconsider this pattern. - final SurfaceTextureEntry texture = createSurfaceTexture(); - final SurfaceTextureSurfaceProducer producer = - new SurfaceTextureSurfaceProducer(texture.id(), handler, flutterJNI, texture); - Log.v(TAG, "New SurfaceTextureSurfaceProducer ID: " + texture.id()); - entry = producer; - } - return entry; + public void onTrimMemory(int level) { + if (trimMemoryListener != null) { + trimMemoryListener.onTrimMemory(level); + } } - /** - * Creates and returns a new {@link SurfaceTexture} managed by the Flutter engine that is also - * made available to Flutter code. - */ - @NonNull - @Override - public SurfaceTextureEntry createSurfaceTexture() { - Log.v(TAG, "Creating a SurfaceTexture."); - final SurfaceTexture surfaceTexture = new SurfaceTexture(0); - return registerSurfaceTexture(surfaceTexture); + private void removeListener() { + removeOnTrimMemoryListener(this); } - /** - * Registers and returns a {@link SurfaceTexture} managed by the Flutter engine that is also - * made available to Flutter code. - */ @NonNull - @Override - public SurfaceTextureEntry registerSurfaceTexture(@NonNull SurfaceTexture surfaceTexture) { - return registerSurfaceTexture(nextTextureId.getAndIncrement(), surfaceTexture); + public SurfaceTextureWrapper textureWrapper() { + return textureWrapper; } - /** - * Similar to {@link FlutterRenderer#registerSurfaceTexture} but with an existing @{code - * textureId}. - * - * @param surfaceTexture Surface texture to wrap. - * @param textureId A texture ID already created that should be assigned to the surface texture. - */ + @Override @NonNull - private SurfaceTextureEntry registerSurfaceTexture( - long textureId, @NonNull SurfaceTexture surfaceTexture) { - surfaceTexture.detachFromGLContext(); - final SurfaceTextureRegistryEntry entry = - new SurfaceTextureRegistryEntry(textureId, surfaceTexture); - Log.v(TAG, "New SurfaceTexture ID: " + entry.id()); - registerTexture(entry.id(), entry.textureWrapper()); - addOnTrimMemoryListener(entry); - return entry; + public SurfaceTexture surfaceTexture() { + return textureWrapper.surfaceTexture(); } - @NonNull @Override - public ImageTextureEntry createImageTexture() { - final ImageTextureRegistryEntry entry = - new ImageTextureRegistryEntry(nextTextureId.getAndIncrement()); - Log.v(TAG, "New ImageTextureEntry ID: " + entry.id()); - registerImageTexture(entry.id(), entry); - return entry; + public long id() { + return id; } @Override - public void onTrimMemory(int level) { - final Iterator> iterator = - onTrimMemoryListeners.iterator(); - while (iterator.hasNext()) { - WeakReference listenerRef = iterator.next(); - final OnTrimMemoryListener listener = listenerRef.get(); - if (listener != null) { - listener.onTrimMemory(level); - } else { - // Purge cleared refs to avoid accumulating a lot of dead listener - iterator.remove(); - } - } + public void release() { + if (released) { + return; + } + Log.v(TAG, "Releasing a SurfaceTexture (" + id + ")."); + textureWrapper.release(); + unregisterTexture(id); + removeListener(); + released = true; } - final class SurfaceTextureRegistryEntry - implements TextureRegistry.SurfaceTextureEntry, TextureRegistry.OnTrimMemoryListener { - private final long id; - @NonNull - private final SurfaceTextureWrapper textureWrapper; - private boolean released; - @Nullable - private OnTrimMemoryListener trimMemoryListener; - @Nullable - private OnFrameConsumedListener frameConsumedListener; - - SurfaceTextureRegistryEntry(long id, @NonNull SurfaceTexture surfaceTexture) { - this.id = id; - Runnable onFrameConsumed = () -> { - if (frameConsumedListener != null) { - frameConsumedListener.onFrameConsumed(); - } - }; - this.textureWrapper = new SurfaceTextureWrapper(surfaceTexture, onFrameConsumed); - - // Even though we make sure to unregister the callback before releasing, as of - // Android O, SurfaceTexture has a data race when accessing the callback, so the - // callback may still be called by a stale reference after released==true and - // mNativeView==null. - SurfaceTexture.OnFrameAvailableListener onFrameListener = texture -> { - if (released || !flutterJNI.isAttached()) { - // Even though we make sure to unregister the callback before releasing, as of - // Android O, SurfaceTexture has a data race when accessing the callback, so the - // callback may still be called by a stale reference after released==true and - // mNativeView==null. - return; - } - textureWrapper.markDirty(); - scheduleEngineFrame(); - }; - // The callback relies on being executed on the UI thread (unsynchronised read of - // mNativeView and also the engine code check for platform thread in - // Shell::OnPlatformViewMarkTextureFrameAvailable), so we explicitly pass a Handler for - // the current thread. - this.surfaceTexture().setOnFrameAvailableListener(onFrameListener, new Handler()); - } - - @Override - public void onTrimMemory(int level) { - if (trimMemoryListener != null) { - trimMemoryListener.onTrimMemory(level); - } - } - - private void removeListener() { - removeOnTrimMemoryListener(this); - } - - @NonNull - public SurfaceTextureWrapper textureWrapper() { - return textureWrapper; - } - - @Override - @NonNull - public SurfaceTexture surfaceTexture() { - return textureWrapper.surfaceTexture(); - } - - @Override - public long id() { - return id; - } - - @Override - public void release() { - if (released) { - return; - } - Log.v(TAG, "Releasing a SurfaceTexture (" + id + ")."); - textureWrapper.release(); - unregisterTexture(id); - removeListener(); - released = true; - } - - @Override - protected void finalize() throws Throwable { - try { - if (released) { - return; - } - - handler.post(new TextureFinalizerRunnable(id, flutterJNI)); - } finally { - super.finalize(); - } - } - - @Override - public void setOnFrameConsumedListener(@Nullable OnFrameConsumedListener listener) { - frameConsumedListener = listener; + @Override + protected void finalize() throws Throwable { + try { + if (released) { + return; } - @Override - public void setOnTrimMemoryListener(@Nullable OnTrimMemoryListener listener) { - trimMemoryListener = listener; - } + handler.post(new TextureFinalizerRunnable(id, flutterJNI)); + } finally { + super.finalize(); + } } - static final class TextureFinalizerRunnable implements Runnable { - private final long id; - private final FlutterJNI flutterJNI; - - TextureFinalizerRunnable(long id, @NonNull FlutterJNI flutterJNI) { - this.id = id; - this.flutterJNI = flutterJNI; - } - - @Override - public void run() { - if (!flutterJNI.isAttached()) { - return; - } - Log.v(TAG, "Releasing a Texture (" + id + ")."); - flutterJNI.unregisterTexture(id); - } + @Override + public void setOnFrameConsumedListener(@Nullable OnFrameConsumedListener listener) { + frameConsumedListener = listener; } - // Keep a queue of ImageReaders. - // Each ImageReader holds acquired Images. - // When we acquire the next image, close any ImageReaders that don't have any - // more pending images. - @Keep - @TargetApi(29) - final class ImageReaderSurfaceProducer - implements TextureRegistry.SurfaceProducer, TextureRegistry.ImageConsumer, - TextureRegistry.OnTrimMemoryListener { - private static final String TAG = "ImageReaderSurfaceProducer"; - private static final int MAX_IMAGES = 5; - - // Flip when debugging to see verbose logs. - private static final boolean VERBOSE_LOGS = false; - - private final long id; - - private boolean released; - // Will be true in tests and on Android API < 33. - private boolean ignoringFence = false; - - // The requested width and height are updated by setSize. - private int requestedWidth = 1; - private int requestedHeight = 1; - // Whenever the requested width and height change we set this to be true so we - // create a new ImageReader (inside getSurface) with the correct width and height. - // We use this flag so that we lazily create the ImageReader only when a frame - // will be produced at that size. - private boolean createNewReader = true; - - // State held to track latency of various stages. - private long lastDequeueTime = 0; - private long lastQueueTime = 0; - private long lastScheduleTime = 0; - - private Object lock = new Object(); - // REQUIRED: The following fields must only be accessed when lock is held. - private final LinkedList imageReaderQueue = - new LinkedList(); - private final HashMap perImageReaders = - new HashMap(); - private PerImage lastDequeuedImage = null; - private PerImageReader lastReaderDequeuedFrom = null; - - /** Internal class: state held per Image produced by ImageReaders. */ - private class PerImage { - public final Image image; - public final long queuedTime; - - public PerImage(Image image, long queuedTime) { - this.image = image; - this.queuedTime = queuedTime; - } - } - - /** Internal class: state held per ImageReader. */ - private class PerImageReader { - public final ImageReader reader; - private final LinkedList imageQueue = new LinkedList(); - private boolean closed = false; - - private final ImageReader.OnImageAvailableListener onImageAvailableListener = - reader -> { - Image image = null; - try { - image = reader.acquireLatestImage(); - } catch (IllegalStateException e) { - Log.e(TAG, "onImageAvailable acquireLatestImage failed: " + e); - } - if (image == null) { - return; - } - if (released || closed) { - image.close(); - return; - } - onImage(reader, image); - }; - - public PerImageReader(ImageReader reader) { - this.reader = reader; - reader.setOnImageAvailableListener(onImageAvailableListener, new Handler()); - } - - PerImage queueImage(Image image) { - if (closed) { - return null; - } - PerImage perImage = new PerImage(image, System.nanoTime()); - imageQueue.add(perImage); - // If we fall too far behind we will skip some frames. - while (imageQueue.size() > 2) { - PerImage r = imageQueue.removeFirst(); - if (VERBOSE_LOGS) { - Log.i(TAG, - "" + reader.hashCode() - + " force closed image=" + r.image.hashCode()); - } - r.image.close(); - } - return perImage; - } - - PerImage dequeueImage() { - if (imageQueue.size() == 0) { - return null; - } - PerImage r = imageQueue.removeFirst(); - return r; - } - - /** returns true if we can prune this reader */ - boolean canPrune() { - return imageQueue.size() == 0 && lastReaderDequeuedFrom != this; - } - - void close() { - closed = true; - if (VERBOSE_LOGS) { - Log.i(TAG, "Closing reader=" + reader.hashCode()); - } - reader.close(); - imageQueue.clear(); - } - } - - double deltaMillis(long deltaNanos) { - double ms = (double) deltaNanos / (double) 1000000.0; - return ms; - } - - PerImageReader getOrCreatePerImageReader(ImageReader reader) { - PerImageReader r = perImageReaders.get(reader); - if (r == null) { - r = new PerImageReader(reader); - perImageReaders.put(reader, r); - imageReaderQueue.add(r); - if (VERBOSE_LOGS) { - Log.i(TAG, "imageReaderQueue#=" + imageReaderQueue.size()); - } - } - return r; - } - - void pruneImageReaderQueue() { - boolean change = false; - // Prune nodes from the head of the ImageReader queue. - while (imageReaderQueue.size() > 1) { - PerImageReader r = imageReaderQueue.peekFirst(); - if (!r.canPrune()) { - // No more ImageReaders can be pruned this round. - break; - } - imageReaderQueue.removeFirst(); - perImageReaders.remove(r.reader); - r.close(); - change = true; - } - if (change && VERBOSE_LOGS) { - Log.i(TAG, "Pruned image reader queue length=" + imageReaderQueue.size()); - } - } - - void onImage(ImageReader reader, Image image) { - PerImage queuedImage = null; - synchronized (lock) { - PerImageReader perReader = getOrCreatePerImageReader(reader); - queuedImage = perReader.queueImage(image); - } - if (queuedImage == null) { - // We got a late image. - return; - } - if (VERBOSE_LOGS) { - if (lastQueueTime != 0) { - long now = System.nanoTime(); - long queueDelta = now - lastQueueTime; - Log.i(TAG, - "" + reader.hashCode() - + " enqueued image=" + queuedImage.image.hashCode() - + " queueDelta=" + deltaMillis(queueDelta)); - lastQueueTime = now; - } else { - lastQueueTime = System.nanoTime(); - } - } - scheduleEngineFrame(); - } + @Override + public void setOnTrimMemoryListener(@Nullable OnTrimMemoryListener listener) { + trimMemoryListener = listener; + } + } - PerImage dequeueImage() { - PerImage r = null; - synchronized (lock) { - for (PerImageReader reader : imageReaderQueue) { - r = reader.dequeueImage(); - if (r == null) { - // This reader is probably about to get pruned. - continue; - } - if (VERBOSE_LOGS) { - if (lastDequeueTime != 0) { - long now = System.nanoTime(); - long dequeueDelta = now - lastDequeueTime; - long queuedFor = now - r.queuedTime; - long scheduleDelay = now - lastScheduleTime; - Log.i(TAG, - "" + reader.reader.hashCode() - + " dequeued image=" + r.image.hashCode() - + " queuedFor= " + deltaMillis(queuedFor) - + " dequeueDelta=" + deltaMillis(dequeueDelta) - + " scheduleDelay=" + deltaMillis(scheduleDelay)); - lastDequeueTime = now; - } else { - lastDequeueTime = System.nanoTime(); - } - } - if (lastDequeuedImage != null) { - if (VERBOSE_LOGS) { - Log.i(TAG, - "" + lastReaderDequeuedFrom.reader.hashCode() - + " closing image=" - + lastDequeuedImage.image.hashCode()); - } - // We must keep the last image dequeued open until we are done presenting - // it. We have just dequeued a new image (r). Close the previously dequeued - // image. - lastDequeuedImage.image.close(); - lastDequeuedImage = null; - } - // Remember the last image and reader dequeued from. We do this because we must - // keep both of these alive until we are done presenting the image. - lastDequeuedImage = r; - lastReaderDequeuedFrom = reader; - break; - } - pruneImageReaderQueue(); - } - return r; - } + static final class TextureFinalizerRunnable implements Runnable { + private final long id; + private final FlutterJNI flutterJNI; - @Override - public void onTrimMemory(int level) { - cleanup(); - createNewReader = true; - } + TextureFinalizerRunnable(long id, @NonNull FlutterJNI flutterJNI) { + this.id = id; + this.flutterJNI = flutterJNI; + } - private void releaseInternal() { - cleanup(); - released = true; - } + @Override + public void run() { + if (!flutterJNI.isAttached()) { + return; + } + Log.v(TAG, "Releasing a Texture (" + id + ")."); + flutterJNI.unregisterTexture(id); + } + } + + // Keep a queue of ImageReaders. + // Each ImageReader holds acquired Images. + // When we acquire the next image, close any ImageReaders that don't have any + // more pending images. + @Keep + @TargetApi(29) + final class ImageReaderSurfaceProducer + implements TextureRegistry.SurfaceProducer, + TextureRegistry.ImageConsumer, + TextureRegistry.OnTrimMemoryListener { + private static final String TAG = "ImageReaderSurfaceProducer"; + private static final int MAX_IMAGES = 5; + + // Flip when debugging to see verbose logs. + private static final boolean VERBOSE_LOGS = false; + + private final long id; + + private boolean released; + // Will be true in tests and on Android API < 33. + private boolean ignoringFence = false; + + // The requested width and height are updated by setSize. + private int requestedWidth = 1; + private int requestedHeight = 1; + // Whenever the requested width and height change we set this to be true so we + // create a new ImageReader (inside getSurface) with the correct width and height. + // We use this flag so that we lazily create the ImageReader only when a frame + // will be produced at that size. + private boolean createNewReader = true; + + // State held to track latency of various stages. + private long lastDequeueTime = 0; + private long lastQueueTime = 0; + private long lastScheduleTime = 0; + + private Object lock = new Object(); + // REQUIRED: The following fields must only be accessed when lock is held. + private final LinkedList imageReaderQueue = new LinkedList(); + private final HashMap perImageReaders = + new HashMap(); + private PerImage lastDequeuedImage = null; + private PerImageReader lastReaderDequeuedFrom = null; + + /** Internal class: state held per Image produced by ImageReaders. */ + private class PerImage { + public final Image image; + public final long queuedTime; + + public PerImage(Image image, long queuedTime) { + this.image = image; + this.queuedTime = queuedTime; + } + } - private void cleanup() { - synchronized (lock) { - for (PerImageReader pir : perImageReaders.values()) { - if (lastReaderDequeuedFrom == pir) { - lastReaderDequeuedFrom = null; - } - pir.close(); - } - perImageReaders.clear(); - if (lastDequeuedImage != null) { - lastDequeuedImage.image.close(); - lastDequeuedImage = null; - } - if (lastReaderDequeuedFrom != null) { - lastReaderDequeuedFrom.close(); - lastReaderDequeuedFrom = null; - } - imageReaderQueue.clear(); - } - } + /** Internal class: state held per ImageReader. */ + private class PerImageReader { + public final ImageReader reader; + private final LinkedList imageQueue = new LinkedList(); + private boolean closed = false; - @TargetApi(33) - private void waitOnFence(Image image) { + private final ImageReader.OnImageAvailableListener onImageAvailableListener = + reader -> { + Image image = null; try { - SyncFence fence = image.getFence(); - fence.awaitForever(); - } catch (IOException e) { - // Drop. + image = reader.acquireLatestImage(); + } catch (IllegalStateException e) { + Log.e(TAG, "onImageAvailable acquireLatestImage failed: " + e); } - } - - private void maybeWaitOnFence(Image image) { if (image == null) { - return; - } - if (ignoringFence) { - return; + return; } - if (Build.VERSION.SDK_INT >= 33) { - // The fence API is only available on Android >= 33. - waitOnFence(image); - return; + if (released || closed) { + image.close(); + return; } - // Log once per ImageTextureEntry. - ignoringFence = true; - Log.w(TAG, "ImageTextureEntry can't wait on the fence on Android < 33"); - } - - ImageReaderSurfaceProducer(long id) { - this.id = id; - } - - @Override - public long id() { - return id; - } + onImage(reader, image); + }; + + public PerImageReader(ImageReader reader) { + this.reader = reader; + reader.setOnImageAvailableListener(onImageAvailableListener, new Handler()); + } + + PerImage queueImage(Image image) { + if (closed) { + return null; + } + PerImage perImage = new PerImage(image, System.nanoTime()); + imageQueue.add(perImage); + // If we fall too far behind we will skip some frames. + while (imageQueue.size() > 2) { + PerImage r = imageQueue.removeFirst(); + if (VERBOSE_LOGS) { + Log.i(TAG, "" + reader.hashCode() + " force closed image=" + r.image.hashCode()); + } + r.image.close(); + } + return perImage; + } + + PerImage dequeueImage() { + if (imageQueue.size() == 0) { + return null; + } + PerImage r = imageQueue.removeFirst(); + return r; + } + + /** returns true if we can prune this reader */ + boolean canPrune() { + return imageQueue.size() == 0 && lastReaderDequeuedFrom != this; + } + + void close() { + closed = true; + if (VERBOSE_LOGS) { + Log.i(TAG, "Closing reader=" + reader.hashCode()); + } + reader.close(); + imageQueue.clear(); + } + } - @Override - public void release() { - if (released) { - return; - } - releaseInternal(); - unregisterTexture(id); - } + double deltaMillis(long deltaNanos) { + double ms = (double) deltaNanos / (double) 1000000.0; + return ms; + } - @Override - public void setSize(int width, int height) { - // Clamp to a minimum of 1. A 0x0 texture is a runtime exception in ImageReader. - width = Math.max(1, width); - height = Math.max(1, height); - - if (requestedWidth == width && requestedHeight == height) { - // No size change. - return; - } - this.createNewReader = true; - this.requestedHeight = height; - this.requestedWidth = width; - } + PerImageReader getOrCreatePerImageReader(ImageReader reader) { + PerImageReader r = perImageReaders.get(reader); + if (r == null) { + r = new PerImageReader(reader); + perImageReaders.put(reader, r); + imageReaderQueue.add(r); + if (VERBOSE_LOGS) { + Log.i(TAG, "imageReaderQueue#=" + imageReaderQueue.size()); + } + } + return r; + } - @Override - public int getWidth() { - return this.requestedWidth; - } + void pruneImageReaderQueue() { + boolean change = false; + // Prune nodes from the head of the ImageReader queue. + while (imageReaderQueue.size() > 1) { + PerImageReader r = imageReaderQueue.peekFirst(); + if (!r.canPrune()) { + // No more ImageReaders can be pruned this round. + break; + } + imageReaderQueue.removeFirst(); + perImageReaders.remove(r.reader); + r.close(); + change = true; + } + if (change && VERBOSE_LOGS) { + Log.i(TAG, "Pruned image reader queue length=" + imageReaderQueue.size()); + } + } - @Override - public int getHeight() { - return this.requestedHeight; + void onImage(ImageReader reader, Image image) { + PerImage queuedImage = null; + synchronized (lock) { + PerImageReader perReader = getOrCreatePerImageReader(reader); + queuedImage = perReader.queueImage(image); + } + if (queuedImage == null) { + // We got a late image. + return; + } + if (VERBOSE_LOGS) { + if (lastQueueTime != 0) { + long now = System.nanoTime(); + long queueDelta = now - lastQueueTime; + Log.i( + TAG, + "" + + reader.hashCode() + + " enqueued image=" + + queuedImage.image.hashCode() + + " queueDelta=" + + deltaMillis(queueDelta)); + lastQueueTime = now; + } else { + lastQueueTime = System.nanoTime(); } + } + scheduleEngineFrame(); + } - @Override - public Surface getSurface() { - PerImageReader pir = getActiveReader(); - if (VERBOSE_LOGS) { - Log.i(TAG, - "" + pir.reader.hashCode() + " returning surface to render a new frame."); + PerImage dequeueImage() { + PerImage r = null; + synchronized (lock) { + for (PerImageReader reader : imageReaderQueue) { + r = reader.dequeueImage(); + if (r == null) { + // This reader is probably about to get pruned. + continue; + } + if (VERBOSE_LOGS) { + if (lastDequeueTime != 0) { + long now = System.nanoTime(); + long dequeueDelta = now - lastDequeueTime; + long queuedFor = now - r.queuedTime; + long scheduleDelay = now - lastScheduleTime; + Log.i( + TAG, + "" + + reader.reader.hashCode() + + " dequeued image=" + + r.image.hashCode() + + " queuedFor= " + + deltaMillis(queuedFor) + + " dequeueDelta=" + + deltaMillis(dequeueDelta) + + " scheduleDelay=" + + deltaMillis(scheduleDelay)); + lastDequeueTime = now; + } else { + lastDequeueTime = System.nanoTime(); } - return pir.reader.getSurface(); - } - - @Override - public void scheduleFrame() { + } + if (lastDequeuedImage != null) { if (VERBOSE_LOGS) { - long now = System.nanoTime(); - if (lastScheduleTime != 0) { - long delta = now - lastScheduleTime; - Log.v(TAG, "scheduleFrame delta=" + deltaMillis(delta)); - } - lastScheduleTime = now; - } - scheduleEngineFrame(); - } - - @Override - @TargetApi(29) - public Image acquireLatestImage() { - PerImage r = dequeueImage(); - if (r == null) { - return null; - } - maybeWaitOnFence(r.image); - return r.image; - } - - private PerImageReader getActiveReader() { - synchronized (lock) { - if (createNewReader) { - createNewReader = false; - // Create a new ImageReader and add it to the queue. - ImageReader reader = createImageReader(); - if (VERBOSE_LOGS) { - Log.i(TAG, - "" + reader.hashCode() + " created w=" + requestedWidth - + " h=" + requestedHeight); - } - return getOrCreatePerImageReader(reader); - } - return imageReaderQueue.peekLast(); + Log.i( + TAG, + "" + + lastReaderDequeuedFrom.reader.hashCode() + + " closing image=" + + lastDequeuedImage.image.hashCode()); } - } - - @Override - protected void finalize() throws Throwable { - try { - if (released) { - return; - } - releaseInternal(); - handler.post(new TextureFinalizerRunnable(id, flutterJNI)); - } finally { - super.finalize(); - } - } - - @TargetApi(33) - private ImageReader createImageReader33() { - final ImageReader.Builder builder = - new ImageReader.Builder(requestedWidth, requestedHeight); - // Allow for double buffering. - builder.setMaxImages(MAX_IMAGES); - // Use PRIVATE image format so that we can support video decoding. - // TODO(johnmccutchan): Should we always use PRIVATE here? It may impact our ability to - // read back texture data. If we don't always want to use it, how do we decide when to - // use it or not? Perhaps PlatformViews can indicate if they may contain DRM'd content. - // I need to investigate how PRIVATE impacts our ability to take screenshots or capture - // the output of Flutter application. - builder.setImageFormat(ImageFormat.PRIVATE); - // Hint that consumed images will only be read by GPU. - builder.setUsage(HardwareBuffer.USAGE_GPU_SAMPLED_IMAGE); - final ImageReader reader = builder.build(); - return reader; - } - - @TargetApi(29) - private ImageReader createImageReader29() { - final ImageReader reader = ImageReader.newInstance(requestedWidth, requestedHeight, - ImageFormat.PRIVATE, MAX_IMAGES, HardwareBuffer.USAGE_GPU_SAMPLED_IMAGE); - return reader; - } + // We must keep the last image dequeued open until we are done presenting + // it. We have just dequeued a new image (r). Close the previously dequeued + // image. + lastDequeuedImage.image.close(); + lastDequeuedImage = null; + } + // Remember the last image and reader dequeued from. We do this because we must + // keep both of these alive until we are done presenting the image. + lastDequeuedImage = r; + lastReaderDequeuedFrom = reader; + break; + } + pruneImageReaderQueue(); + } + return r; + } - private ImageReader createImageReader() { - if (Build.VERSION.SDK_INT >= 33) { - return createImageReader33(); - } else if (Build.VERSION.SDK_INT >= 29) { - return createImageReader29(); - } - throw new UnsupportedOperationException( - "ImageReaderPlatformViewRenderTarget requires API version 29+"); - } + @Override + public void onTrimMemory(int level) { + cleanup(); + createNewReader = true; + } - @VisibleForTesting - public void disableFenceForTest() { - // Roboelectric's implementation of SyncFence is borked. - ignoringFence = true; - } + private void releaseInternal() { + cleanup(); + released = true; + } - @VisibleForTesting - public int numImageReaders() { - synchronized (lock) { - return imageReaderQueue.size(); - } - } + private void cleanup() { + synchronized (lock) { + for (PerImageReader pir : perImageReaders.values()) { + if (lastReaderDequeuedFrom == pir) { + lastReaderDequeuedFrom = null; + } + pir.close(); + } + perImageReaders.clear(); + if (lastDequeuedImage != null) { + lastDequeuedImage.image.close(); + lastDequeuedImage = null; + } + if (lastReaderDequeuedFrom != null) { + lastReaderDequeuedFrom.close(); + lastReaderDequeuedFrom = null; + } + imageReaderQueue.clear(); + } + } - @VisibleForTesting - public int numImages() { - int r = 0; - synchronized (lock) { - for (PerImageReader reader : imageReaderQueue) { - r += reader.imageQueue.size(); - } - } - return r; - } + @TargetApi(33) + private void waitOnFence(Image image) { + try { + SyncFence fence = image.getFence(); + fence.awaitForever(); + } catch (IOException e) { + // Drop. + } } - @Keep - final class ImageTextureRegistryEntry - implements TextureRegistry.ImageTextureEntry, TextureRegistry.ImageConsumer { - private static final String TAG = "ImageTextureRegistryEntry"; - private final long id; - private boolean released; - private boolean ignoringFence = false; - private Image image; + private void maybeWaitOnFence(Image image) { + if (image == null) { + return; + } + if (ignoringFence) { + return; + } + if (Build.VERSION.SDK_INT >= 33) { + // The fence API is only available on Android >= 33. + waitOnFence(image); + return; + } + // Log once per ImageTextureEntry. + ignoringFence = true; + Log.w(TAG, "ImageTextureEntry can't wait on the fence on Android < 33"); + } - ImageTextureRegistryEntry(long id) { - this.id = id; - } + ImageReaderSurfaceProducer(long id) { + this.id = id; + } - @Override - public long id() { - return id; - } + @Override + public long id() { + return id; + } - @Override - public void release() { - if (released) { - return; - } - released = true; - if (image != null) { - image.close(); - image = null; - } - unregisterTexture(id); - } + @Override + public void release() { + if (released) { + return; + } + releaseInternal(); + unregisterTexture(id); + } - @Override - public void pushImage(Image image) { - if (released) { - return; - } - Image toClose; - synchronized (this) { - toClose = this.image; - this.image = image; - } - // Close the previously pushed buffer. - if (toClose != null) { - Log.e(TAG, "Dropping PlatformView Frame"); - toClose.close(); - } - if (image != null) { - scheduleEngineFrame(); - } - } + @Override + public void setSize(int width, int height) { + // Clamp to a minimum of 1. A 0x0 texture is a runtime exception in ImageReader. + width = Math.max(1, width); + height = Math.max(1, height); + + if (requestedWidth == width && requestedHeight == height) { + // No size change. + return; + } + this.createNewReader = true; + this.requestedHeight = height; + this.requestedWidth = width; + } - @TargetApi(33) - private void waitOnFence(Image image) { - try { - SyncFence fence = image.getFence(); - fence.awaitForever(); - } catch (IOException e) { - // Drop. - } - } + @Override + public int getWidth() { + return this.requestedWidth; + } - @TargetApi(29) - private void maybeWaitOnFence(Image image) { - if (image == null) { - return; - } - if (ignoringFence) { - return; - } - if (Build.VERSION.SDK_INT >= 33) { - // The fence API is only available on Android >= 33. - waitOnFence(image); - return; - } - // Log once per ImageTextureEntry. - ignoringFence = true; - Log.w(TAG, "ImageTextureEntry can't wait on the fence on Android < 33"); - } + @Override + public int getHeight() { + return this.requestedHeight; + } - @Override - @TargetApi(29) - public Image acquireLatestImage() { - Image r; - synchronized (this) { - r = this.image; - this.image = null; - } - maybeWaitOnFence(r); - return r; - } + @Override + public Surface getSurface() { + PerImageReader pir = getActiveReader(); + if (VERBOSE_LOGS) { + Log.i(TAG, "" + pir.reader.hashCode() + " returning surface to render a new frame."); + } + return pir.reader.getSurface(); + } - @Override - protected void finalize() throws Throwable { - try { - if (released) { - return; - } - if (image != null) { - // Be sure to finalize any cached image. - image.close(); - image = null; - } - released = true; - handler.post(new TextureFinalizerRunnable(id, flutterJNI)); - } finally { - super.finalize(); - } - } + @Override + public void scheduleFrame() { + if (VERBOSE_LOGS) { + long now = System.nanoTime(); + if (lastScheduleTime != 0) { + long delta = now - lastScheduleTime; + Log.v(TAG, "scheduleFrame delta=" + deltaMillis(delta)); + } + lastScheduleTime = now; + } + scheduleEngineFrame(); } - // ------ END TextureRegistry IMPLEMENTATION ---- - /** - * Notifies Flutter that the given {@code surface} was created and is available for Flutter - * rendering. - * - *

If called more than once, the current native resources are released. This can be undesired - * if the Engine expects to reuse this surface later. For example, this is true when platform - * views are displayed in a frame, and then removed in the next frame. - * - *

To avoid releasing the current surface resources, set {@code keepCurrentSurface} to true. - * - *

See {@link android.view.SurfaceHolder.Callback} and {@link - * android.view.TextureView.SurfaceTextureListener} - * - * @param surface The render surface. - * @param onlySwap True if the current active surface should not be detached. - */ - public void startRenderingToSurface(@NonNull Surface surface, boolean onlySwap) { - if (!onlySwap) { - // Stop rendering to the surface releases the associated native resources, which causes - // a glitch when toggling between rendering to an image view (hybrid composition) and - // rendering directly to a Surface or Texture view. For more, - // https://github.com/flutter/flutter/issues/95343 - stopRenderingToSurface(); - } + @Override + @TargetApi(29) + public Image acquireLatestImage() { + PerImage r = dequeueImage(); + if (r == null) { + return null; + } + maybeWaitOnFence(r.image); + return r.image; + } - this.surface = surface; + private PerImageReader getActiveReader() { + synchronized (lock) { + if (createNewReader) { + createNewReader = false; + // Create a new ImageReader and add it to the queue. + ImageReader reader = createImageReader(); + if (VERBOSE_LOGS) { + Log.i( + TAG, + "" + reader.hashCode() + " created w=" + requestedWidth + " h=" + requestedHeight); + } + return getOrCreatePerImageReader(reader); + } + return imageReaderQueue.peekLast(); + } + } - if (onlySwap) { - // In the swap case we are just swapping the surface that we render to. - flutterJNI.onSurfaceWindowChanged(surface); - } else { - // In the non-swap case we are creating a new surface to render to. - flutterJNI.onSurfaceCreated(surface); - } + @Override + protected void finalize() throws Throwable { + try { + if (released) { + return; + } + releaseInternal(); + handler.post(new TextureFinalizerRunnable(id, flutterJNI)); + } finally { + super.finalize(); + } } - /** - * Swaps the {@link Surface} used to render the current frame. - * - *

In hybrid composition, the root surfaces changes from {@link - * android.view.SurfaceHolder#getSurface()} to {@link android.media.ImageReader#getSurface()} - * when a platform view is in the current frame. - */ - public void swapSurface(@NonNull Surface surface) { - this.surface = surface; - flutterJNI.onSurfaceWindowChanged(surface); + @TargetApi(33) + private ImageReader createImageReader33() { + final ImageReader.Builder builder = new ImageReader.Builder(requestedWidth, requestedHeight); + // Allow for double buffering. + builder.setMaxImages(MAX_IMAGES); + // Use PRIVATE image format so that we can support video decoding. + // TODO(johnmccutchan): Should we always use PRIVATE here? It may impact our ability to + // read back texture data. If we don't always want to use it, how do we decide when to + // use it or not? Perhaps PlatformViews can indicate if they may contain DRM'd content. + // I need to investigate how PRIVATE impacts our ability to take screenshots or capture + // the output of Flutter application. + builder.setImageFormat(ImageFormat.PRIVATE); + // Hint that consumed images will only be read by GPU. + builder.setUsage(HardwareBuffer.USAGE_GPU_SAMPLED_IMAGE); + final ImageReader reader = builder.build(); + return reader; } - /** - * Notifies Flutter that a {@code surface} previously registered with {@link - * #startRenderingToSurface(Surface, boolean)} has changed size to the given {@code width} and - * {@code height}. - * - *

See {@link android.view.SurfaceHolder.Callback} and {@link - * android.view.TextureView.SurfaceTextureListener} - */ - public void surfaceChanged(int width, int height) { - flutterJNI.onSurfaceChanged(width, height); + @TargetApi(29) + private ImageReader createImageReader29() { + final ImageReader reader = + ImageReader.newInstance( + requestedWidth, + requestedHeight, + ImageFormat.PRIVATE, + MAX_IMAGES, + HardwareBuffer.USAGE_GPU_SAMPLED_IMAGE); + return reader; } - /** - * Notifies Flutter that a {@code surface} previously registered with {@link - * #startRenderingToSurface(Surface, boolean)} has been destroyed and needs to be released and - * cleaned up on the Flutter side. - * - *

See {@link android.view.SurfaceHolder.Callback} and {@link - * android.view.TextureView.SurfaceTextureListener} - */ - public void stopRenderingToSurface() { - if (surface != null) { - flutterJNI.onSurfaceDestroyed(); - - // TODO(mattcarroll): the source of truth for this call should be FlutterJNI, which is - // where the call to onFlutterUiDisplayed() comes from. However, no such native callback - // exists yet, so until the engine and FlutterJNI are configured to call us back when - // rendering stops, we will manually monitor that change here. - if (isDisplayingFlutterUi) { - flutterUiDisplayListener.onFlutterUiNoLongerDisplayed(); - } + private ImageReader createImageReader() { + if (Build.VERSION.SDK_INT >= 33) { + return createImageReader33(); + } else if (Build.VERSION.SDK_INT >= 29) { + return createImageReader29(); + } + throw new UnsupportedOperationException( + "ImageReaderPlatformViewRenderTarget requires API version 29+"); + } - isDisplayingFlutterUi = false; - surface = null; - } + @VisibleForTesting + public void disableFenceForTest() { + // Roboelectric's implementation of SyncFence is borked. + ignoringFence = true; } - /** - * Notifies Flutter that the viewport metrics, e.g. window height and width, have changed. - * - *

If the width, height, or devicePixelRatio are less than or equal to 0, this update is - * ignored. - * - * @param viewportMetrics The metrics to send to the Dart application. - */ - public void setViewportMetrics(@NonNull ViewportMetrics viewportMetrics) { - // We might get called with just the DPR if width/height aren't available yet. - // Just ignore, as it will get called again when width/height are set. - if (!viewportMetrics.validate()) { - return; - } - Log.v(TAG, - "Setting viewport metrics\n" - + "Size: " + viewportMetrics.width + " x " + viewportMetrics.height + "\n" - + "Padding - L: " + viewportMetrics.viewPaddingLeft - + ", T: " + viewportMetrics.viewPaddingTop - + ", R: " + viewportMetrics.viewPaddingRight - + ", B: " + viewportMetrics.viewPaddingBottom + "\n" - + "Insets - L: " + viewportMetrics.viewInsetLeft + ", T: " - + viewportMetrics.viewInsetTop + ", R: " + viewportMetrics.viewInsetRight - + ", B: " + viewportMetrics.viewInsetBottom + "\n" - + "System Gesture Insets - L: " + viewportMetrics.systemGestureInsetLeft - + ", T: " + viewportMetrics.systemGestureInsetTop - + ", R: " + viewportMetrics.systemGestureInsetRight - + ", B: " + viewportMetrics.systemGestureInsetRight + "\n" - + "Display Features: " + viewportMetrics.displayFeatures.size()); - - int[] displayFeaturesBounds = new int[viewportMetrics.displayFeatures.size() * 4]; - int[] displayFeaturesType = new int[viewportMetrics.displayFeatures.size()]; - int[] displayFeaturesState = new int[viewportMetrics.displayFeatures.size()]; - for (int i = 0; i < viewportMetrics.displayFeatures.size(); i++) { - DisplayFeature displayFeature = viewportMetrics.displayFeatures.get(i); - displayFeaturesBounds[4 * i] = displayFeature.bounds.left; - displayFeaturesBounds[4 * i + 1] = displayFeature.bounds.top; - displayFeaturesBounds[4 * i + 2] = displayFeature.bounds.right; - displayFeaturesBounds[4 * i + 3] = displayFeature.bounds.bottom; - displayFeaturesType[i] = displayFeature.type.encodedValue; - displayFeaturesState[i] = displayFeature.state.encodedValue; - } + @VisibleForTesting + public int numImageReaders() { + synchronized (lock) { + return imageReaderQueue.size(); + } + } - flutterJNI.setViewportMetrics(viewportMetrics.devicePixelRatio, viewportMetrics.width, - viewportMetrics.height, viewportMetrics.viewPaddingTop, - viewportMetrics.viewPaddingRight, viewportMetrics.viewPaddingBottom, - viewportMetrics.viewPaddingLeft, viewportMetrics.viewInsetTop, - viewportMetrics.viewInsetRight, viewportMetrics.viewInsetBottom, - viewportMetrics.viewInsetLeft, viewportMetrics.systemGestureInsetTop, - viewportMetrics.systemGestureInsetRight, viewportMetrics.systemGestureInsetBottom, - viewportMetrics.systemGestureInsetLeft, viewportMetrics.physicalTouchSlop, - displayFeaturesBounds, displayFeaturesType, displayFeaturesState); + @VisibleForTesting + public int numImages() { + int r = 0; + synchronized (lock) { + for (PerImageReader reader : imageReaderQueue) { + r += reader.imageQueue.size(); + } + } + return r; + } + } + + @Keep + final class ImageTextureRegistryEntry + implements TextureRegistry.ImageTextureEntry, TextureRegistry.ImageConsumer { + private static final String TAG = "ImageTextureRegistryEntry"; + private final long id; + private boolean released; + private boolean ignoringFence = false; + private Image image; + + ImageTextureRegistryEntry(long id) { + this.id = id; } - // TODO(mattcarroll): describe the native behavior that this invokes - // TODO(mattcarroll): determine if this is nullable or nonnull - public Bitmap getBitmap() { - return flutterJNI.getBitmap(); + @Override + public long id() { + return id; } - // TODO(mattcarroll): describe the native behavior that this invokes - public void dispatchPointerDataPacket(@NonNull ByteBuffer buffer, int position) { - flutterJNI.dispatchPointerDataPacket(buffer, position); + @Override + public void release() { + if (released) { + return; + } + released = true; + if (image != null) { + image.close(); + image = null; + } + unregisterTexture(id); } - // TODO(mattcarroll): describe the native behavior that this invokes - private void registerTexture(long textureId, @NonNull SurfaceTextureWrapper textureWrapper) { - flutterJNI.registerTexture(textureId, textureWrapper); + @Override + public void pushImage(Image image) { + if (released) { + return; + } + Image toClose; + synchronized (this) { + toClose = this.image; + this.image = image; + } + // Close the previously pushed buffer. + if (toClose != null) { + Log.e(TAG, "Dropping PlatformView Frame"); + toClose.close(); + } + if (image != null) { + scheduleEngineFrame(); + } } - private void registerImageTexture( - long textureId, @NonNull TextureRegistry.ImageConsumer imageTexture) { - flutterJNI.registerImageTexture(textureId, imageTexture); + @TargetApi(33) + private void waitOnFence(Image image) { + try { + SyncFence fence = image.getFence(); + fence.awaitForever(); + } catch (IOException e) { + // Drop. + } } - private void scheduleEngineFrame() { - flutterJNI.scheduleFrame(); + @TargetApi(29) + private void maybeWaitOnFence(Image image) { + if (image == null) { + return; + } + if (ignoringFence) { + return; + } + if (Build.VERSION.SDK_INT >= 33) { + // The fence API is only available on Android >= 33. + waitOnFence(image); + return; + } + // Log once per ImageTextureEntry. + ignoringFence = true; + Log.w(TAG, "ImageTextureEntry can't wait on the fence on Android < 33"); } - // TODO(mattcarroll): describe the native behavior that this invokes - private void markTextureFrameAvailable(long textureId) { - flutterJNI.markTextureFrameAvailable(textureId); + @Override + @TargetApi(29) + public Image acquireLatestImage() { + Image r; + synchronized (this) { + r = this.image; + this.image = null; + } + maybeWaitOnFence(r); + return r; } - // TODO(mattcarroll): describe the native behavior that this invokes - private void unregisterTexture(long textureId) { - flutterJNI.unregisterTexture(textureId); + @Override + protected void finalize() throws Throwable { + try { + if (released) { + return; + } + if (image != null) { + // Be sure to finalize any cached image. + image.close(); + image = null; + } + released = true; + handler.post(new TextureFinalizerRunnable(id, flutterJNI)); + } finally { + super.finalize(); + } + } + } + // ------ END TextureRegistry IMPLEMENTATION ---- + + /** + * Notifies Flutter that the given {@code surface} was created and is available for Flutter + * rendering. + * + *

If called more than once, the current native resources are released. This can be undesired + * if the Engine expects to reuse this surface later. For example, this is true when platform + * views are displayed in a frame, and then removed in the next frame. + * + *

To avoid releasing the current surface resources, set {@code keepCurrentSurface} to true. + * + *

See {@link android.view.SurfaceHolder.Callback} and {@link + * android.view.TextureView.SurfaceTextureListener} + * + * @param surface The render surface. + * @param onlySwap True if the current active surface should not be detached. + */ + public void startRenderingToSurface(@NonNull Surface surface, boolean onlySwap) { + if (!onlySwap) { + // Stop rendering to the surface releases the associated native resources, which causes + // a glitch when toggling between rendering to an image view (hybrid composition) and + // rendering directly to a Surface or Texture view. For more, + // https://github.com/flutter/flutter/issues/95343 + stopRenderingToSurface(); } - // TODO(mattcarroll): describe the native behavior that this invokes - public boolean isSoftwareRenderingEnabled() { - return flutterJNI.getIsSoftwareRenderingEnabled(); + this.surface = surface; + + if (onlySwap) { + // In the swap case we are just swapping the surface that we render to. + flutterJNI.onSurfaceWindowChanged(surface); + } else { + // In the non-swap case we are creating a new surface to render to. + flutterJNI.onSurfaceCreated(surface); + } + } + + /** + * Swaps the {@link Surface} used to render the current frame. + * + *

In hybrid composition, the root surfaces changes from {@link + * android.view.SurfaceHolder#getSurface()} to {@link android.media.ImageReader#getSurface()} when + * a platform view is in the current frame. + */ + public void swapSurface(@NonNull Surface surface) { + this.surface = surface; + flutterJNI.onSurfaceWindowChanged(surface); + } + + /** + * Notifies Flutter that a {@code surface} previously registered with {@link + * #startRenderingToSurface(Surface, boolean)} has changed size to the given {@code width} and + * {@code height}. + * + *

See {@link android.view.SurfaceHolder.Callback} and {@link + * android.view.TextureView.SurfaceTextureListener} + */ + public void surfaceChanged(int width, int height) { + flutterJNI.onSurfaceChanged(width, height); + } + + /** + * Notifies Flutter that a {@code surface} previously registered with {@link + * #startRenderingToSurface(Surface, boolean)} has been destroyed and needs to be released and + * cleaned up on the Flutter side. + * + *

See {@link android.view.SurfaceHolder.Callback} and {@link + * android.view.TextureView.SurfaceTextureListener} + */ + public void stopRenderingToSurface() { + if (surface != null) { + flutterJNI.onSurfaceDestroyed(); + + // TODO(mattcarroll): the source of truth for this call should be FlutterJNI, which is + // where the call to onFlutterUiDisplayed() comes from. However, no such native callback + // exists yet, so until the engine and FlutterJNI are configured to call us back when + // rendering stops, we will manually monitor that change here. + if (isDisplayingFlutterUi) { + flutterUiDisplayListener.onFlutterUiNoLongerDisplayed(); + } + + isDisplayingFlutterUi = false; + surface = null; + } + } + + /** + * Notifies Flutter that the viewport metrics, e.g. window height and width, have changed. + * + *

If the width, height, or devicePixelRatio are less than or equal to 0, this update is + * ignored. + * + * @param viewportMetrics The metrics to send to the Dart application. + */ + public void setViewportMetrics(@NonNull ViewportMetrics viewportMetrics) { + // We might get called with just the DPR if width/height aren't available yet. + // Just ignore, as it will get called again when width/height are set. + if (!viewportMetrics.validate()) { + return; } + Log.v( + TAG, + "Setting viewport metrics\n" + + "Size: " + + viewportMetrics.width + + " x " + + viewportMetrics.height + + "\n" + + "Padding - L: " + + viewportMetrics.viewPaddingLeft + + ", T: " + + viewportMetrics.viewPaddingTop + + ", R: " + + viewportMetrics.viewPaddingRight + + ", B: " + + viewportMetrics.viewPaddingBottom + + "\n" + + "Insets - L: " + + viewportMetrics.viewInsetLeft + + ", T: " + + viewportMetrics.viewInsetTop + + ", R: " + + viewportMetrics.viewInsetRight + + ", B: " + + viewportMetrics.viewInsetBottom + + "\n" + + "System Gesture Insets - L: " + + viewportMetrics.systemGestureInsetLeft + + ", T: " + + viewportMetrics.systemGestureInsetTop + + ", R: " + + viewportMetrics.systemGestureInsetRight + + ", B: " + + viewportMetrics.systemGestureInsetRight + + "\n" + + "Display Features: " + + viewportMetrics.displayFeatures.size()); + + int[] displayFeaturesBounds = new int[viewportMetrics.displayFeatures.size() * 4]; + int[] displayFeaturesType = new int[viewportMetrics.displayFeatures.size()]; + int[] displayFeaturesState = new int[viewportMetrics.displayFeatures.size()]; + for (int i = 0; i < viewportMetrics.displayFeatures.size(); i++) { + DisplayFeature displayFeature = viewportMetrics.displayFeatures.get(i); + displayFeaturesBounds[4 * i] = displayFeature.bounds.left; + displayFeaturesBounds[4 * i + 1] = displayFeature.bounds.top; + displayFeaturesBounds[4 * i + 2] = displayFeature.bounds.right; + displayFeaturesBounds[4 * i + 3] = displayFeature.bounds.bottom; + displayFeaturesType[i] = displayFeature.type.encodedValue; + displayFeaturesState[i] = displayFeature.state.encodedValue; + } + + flutterJNI.setViewportMetrics( + viewportMetrics.devicePixelRatio, + viewportMetrics.width, + viewportMetrics.height, + viewportMetrics.viewPaddingTop, + viewportMetrics.viewPaddingRight, + viewportMetrics.viewPaddingBottom, + viewportMetrics.viewPaddingLeft, + viewportMetrics.viewInsetTop, + viewportMetrics.viewInsetRight, + viewportMetrics.viewInsetBottom, + viewportMetrics.viewInsetLeft, + viewportMetrics.systemGestureInsetTop, + viewportMetrics.systemGestureInsetRight, + viewportMetrics.systemGestureInsetBottom, + viewportMetrics.systemGestureInsetLeft, + viewportMetrics.physicalTouchSlop, + displayFeaturesBounds, + displayFeaturesType, + displayFeaturesState); + } + + // TODO(mattcarroll): describe the native behavior that this invokes + // TODO(mattcarroll): determine if this is nullable or nonnull + public Bitmap getBitmap() { + return flutterJNI.getBitmap(); + } + + // TODO(mattcarroll): describe the native behavior that this invokes + public void dispatchPointerDataPacket(@NonNull ByteBuffer buffer, int position) { + flutterJNI.dispatchPointerDataPacket(buffer, position); + } + + // TODO(mattcarroll): describe the native behavior that this invokes + private void registerTexture(long textureId, @NonNull SurfaceTextureWrapper textureWrapper) { + flutterJNI.registerTexture(textureId, textureWrapper); + } + + private void registerImageTexture( + long textureId, @NonNull TextureRegistry.ImageConsumer imageTexture) { + flutterJNI.registerImageTexture(textureId, imageTexture); + } + + private void scheduleEngineFrame() { + flutterJNI.scheduleFrame(); + } + + // TODO(mattcarroll): describe the native behavior that this invokes + private void markTextureFrameAvailable(long textureId) { + flutterJNI.markTextureFrameAvailable(textureId); + } + + // TODO(mattcarroll): describe the native behavior that this invokes + private void unregisterTexture(long textureId) { + flutterJNI.unregisterTexture(textureId); + } + + // TODO(mattcarroll): describe the native behavior that this invokes + public boolean isSoftwareRenderingEnabled() { + return flutterJNI.getIsSoftwareRenderingEnabled(); + } + + // TODO(mattcarroll): describe the native behavior that this invokes + public void setAccessibilityFeatures(int flags) { + flutterJNI.setAccessibilityFeatures(flags); + } + + // TODO(mattcarroll): describe the native behavior that this invokes + public void setSemanticsEnabled(boolean enabled) { + flutterJNI.setSemanticsEnabled(enabled); + } + + // TODO(mattcarroll): describe the native behavior that this invokes + public void dispatchSemanticsAction( + int nodeId, int action, @Nullable ByteBuffer args, int argsPosition) { + flutterJNI.dispatchSemanticsAction(nodeId, action, args, argsPosition); + } + + /** + * Mutable data structure that holds all viewport metrics properties that Flutter cares about. + * + *

All distance measurements, e.g., width, height, padding, viewInsets, are measured in device + * pixels, not logical pixels. + */ + public static final class ViewportMetrics { + /** A value that indicates the setting has not been set. */ + public static final int unsetValue = -1; + + public float devicePixelRatio = 1.0f; + public int width = 0; + public int height = 0; + public int viewPaddingTop = 0; + public int viewPaddingRight = 0; + public int viewPaddingBottom = 0; + public int viewPaddingLeft = 0; + public int viewInsetTop = 0; + public int viewInsetRight = 0; + public int viewInsetBottom = 0; + public int viewInsetLeft = 0; + public int systemGestureInsetTop = 0; + public int systemGestureInsetRight = 0; + public int systemGestureInsetBottom = 0; + public int systemGestureInsetLeft = 0; + public int physicalTouchSlop = unsetValue; - // TODO(mattcarroll): describe the native behavior that this invokes - public void setAccessibilityFeatures(int flags) { - flutterJNI.setAccessibilityFeatures(flags); + /** + * Whether this instance contains valid metrics for the Flutter application. + * + * @return True if width, height, and devicePixelRatio are > 0; false otherwise. + */ + boolean validate() { + return width > 0 && height > 0 && devicePixelRatio > 0; } - // TODO(mattcarroll): describe the native behavior that this invokes - public void setSemanticsEnabled(boolean enabled) { - flutterJNI.setSemanticsEnabled(enabled); + public List displayFeatures = new ArrayList<>(); + } + + /** + * Description of a physical feature on the display. + * + *

A display feature is a distinctive physical attribute located within the display panel of + * the device. It can intrude into the application window space and create a visual distortion, + * visual or touch discontinuity, make some area invisible or create a logical divider or + * separation in the screen space. + * + *

Based on {@link androidx.window.layout.DisplayFeature}, with added support for cutouts. + */ + public static final class DisplayFeature { + public final Rect bounds; + public final DisplayFeatureType type; + public final DisplayFeatureState state; + + public DisplayFeature(Rect bounds, DisplayFeatureType type, DisplayFeatureState state) { + this.bounds = bounds; + this.type = type; + this.state = state; } - // TODO(mattcarroll): describe the native behavior that this invokes - public void dispatchSemanticsAction( - int nodeId, int action, @Nullable ByteBuffer args, int argsPosition) { - flutterJNI.dispatchSemanticsAction(nodeId, action, args, argsPosition); + public DisplayFeature(Rect bounds, DisplayFeatureType type) { + this.bounds = bounds; + this.type = type; + this.state = DisplayFeatureState.UNKNOWN; } + } + + /** + * Types of display features that can appear on the viewport. + * + *

Some, like {@link #FOLD}, can be reported without actually occluding the screen. They are + * useful for knowing where the display is bent or has a crease. The {@link DisplayFeature#bounds} + * can be 0-width in such cases. + */ + public enum DisplayFeatureType { + /** + * Type of display feature not yet known to Flutter. This can happen if WindowManager is updated + * with new types. The {@link DisplayFeature#bounds} is the only known property. + */ + UNKNOWN(0), /** - * Mutable data structure that holds all viewport metrics properties that Flutter cares about. - * - *

All distance measurements, e.g., width, height, padding, viewInsets, are measured in - * device pixels, not logical pixels. + * A fold in the flexible display that does not occlude the screen. Corresponds to {@link + * androidx.window.layout.FoldingFeature.OcclusionType#NONE} */ - public static final class ViewportMetrics { - /** A value that indicates the setting has not been set. */ - public static final int unsetValue = -1; - - public float devicePixelRatio = 1.0f; - public int width = 0; - public int height = 0; - public int viewPaddingTop = 0; - public int viewPaddingRight = 0; - public int viewPaddingBottom = 0; - public int viewPaddingLeft = 0; - public int viewInsetTop = 0; - public int viewInsetRight = 0; - public int viewInsetBottom = 0; - public int viewInsetLeft = 0; - public int systemGestureInsetTop = 0; - public int systemGestureInsetRight = 0; - public int systemGestureInsetBottom = 0; - public int systemGestureInsetLeft = 0; - public int physicalTouchSlop = unsetValue; - - /** - * Whether this instance contains valid metrics for the Flutter application. - * - * @return True if width, height, and devicePixelRatio are > 0; false otherwise. - */ - boolean validate() { - return width > 0 && height > 0 && devicePixelRatio > 0; - } + FOLD(1), - public List displayFeatures = new ArrayList<>(); - } + /** + * Splits the display in two separate panels that can fold. Occludes the screen. Corresponds to + * {@link androidx.window.layout.FoldingFeature.OcclusionType#FULL} + */ + HINGE(2), /** - * Description of a physical feature on the display. - * - *

A display feature is a distinctive physical attribute located within the display panel of - * the device. It can intrude into the application window space and create a visual distortion, - * visual or touch discontinuity, make some area invisible or create a logical divider or - * separation in the screen space. - * - *

Based on {@link androidx.window.layout.DisplayFeature}, with added support for cutouts. + * Area of the screen that usually houses cameras or sensors. Occludes the screen. Corresponds + * to {@link android.view.DisplayCutout} */ - public static final class DisplayFeature { - public final Rect bounds; - public final DisplayFeatureType type; - public final DisplayFeatureState state; - - public DisplayFeature(Rect bounds, DisplayFeatureType type, DisplayFeatureState state) { - this.bounds = bounds; - this.type = type; - this.state = state; - } + CUTOUT(3); - public DisplayFeature(Rect bounds, DisplayFeatureType type) { - this.bounds = bounds; - this.type = type; - this.state = DisplayFeatureState.UNKNOWN; - } + public final int encodedValue; + + DisplayFeatureType(int encodedValue) { + this.encodedValue = encodedValue; } + } + + /** + * State of the display feature. + * + *

For foldables, the state is the posture. For cutouts, this property is {@link #UNKNOWN} + */ + public enum DisplayFeatureState { + /** The display feature is a cutout or this state is new and not yet known to Flutter. */ + UNKNOWN(0), /** - * Types of display features that can appear on the viewport. - * - *

Some, like {@link #FOLD}, can be reported without actually occluding the screen. They are - * useful for knowing where the display is bent or has a crease. The {@link - * DisplayFeature#bounds} can be 0-width in such cases. + * The foldable device is completely open. The screen space that is presented to the user is + * flat. Corresponds to {@link androidx.window.layout.FoldingFeature.State#FLAT} */ - public enum DisplayFeatureType { - /** - * Type of display feature not yet known to Flutter. This can happen if WindowManager is - * updated with new types. The {@link DisplayFeature#bounds} is the only known property. - */ - UNKNOWN(0), - - /** - * A fold in the flexible display that does not occlude the screen. Corresponds to {@link - * androidx.window.layout.FoldingFeature.OcclusionType#NONE} - */ - FOLD(1), - - /** - * Splits the display in two separate panels that can fold. Occludes the screen. Corresponds - * to - * {@link androidx.window.layout.FoldingFeature.OcclusionType#FULL} - */ - HINGE(2), - - /** - * Area of the screen that usually houses cameras or sensors. Occludes the screen. - * Corresponds to {@link android.view.DisplayCutout} - */ - CUTOUT(3); - - public final int encodedValue; - - DisplayFeatureType(int encodedValue) { - this.encodedValue = encodedValue; - } - } + POSTURE_FLAT(1), /** - * State of the display feature. - * - *

For foldables, the state is the posture. For cutouts, this property is {@link #UNKNOWN} + * The foldable device's hinge is in an intermediate position between opened and closed state. + * There is a non-flat angle between parts of the flexible screen or between physical display + * panels. Corresponds to {@link androidx.window.layout.FoldingFeature.State#HALF_OPENED} */ - public enum DisplayFeatureState { - /** The display feature is a cutout or this state is new and not yet known to Flutter. */ - UNKNOWN(0), - - /** - * The foldable device is completely open. The screen space that is presented to the user is - * flat. Corresponds to {@link androidx.window.layout.FoldingFeature.State#FLAT} - */ - POSTURE_FLAT(1), - - /** - * The foldable device's hinge is in an intermediate position between opened and closed - * state. There is a non-flat angle between parts of the flexible screen or between physical - * display panels. Corresponds to {@link - * androidx.window.layout.FoldingFeature.State#HALF_OPENED} - */ - POSTURE_HALF_OPENED(2); - - public final int encodedValue; - - DisplayFeatureState(int encodedValue) { - this.encodedValue = encodedValue; - } + POSTURE_HALF_OPENED(2); + + public final int encodedValue; + + DisplayFeatureState(int encodedValue) { + this.encodedValue = encodedValue; } + } } diff --git a/shell/platform/android/io/flutter/embedding/engine/renderer/SurfaceTextureSurfaceProducer.java b/shell/platform/android/io/flutter/embedding/engine/renderer/SurfaceTextureSurfaceProducer.java index 7ff1399cbe411..0592a0bab1299 100644 --- a/shell/platform/android/io/flutter/embedding/engine/renderer/SurfaceTextureSurfaceProducer.java +++ b/shell/platform/android/io/flutter/embedding/engine/renderer/SurfaceTextureSurfaceProducer.java @@ -5,8 +5,6 @@ import android.view.Surface; import androidx.annotation.NonNull; import androidx.annotation.Nullable; - -import io.flutter.Log; import io.flutter.embedding.engine.FlutterJNI; import io.flutter.view.TextureRegistry; @@ -22,7 +20,11 @@ final class SurfaceTextureSurfaceProducer @NonNull private final Handler handler; @NonNull private final FlutterJNI flutterJNI; - SurfaceTextureSurfaceProducer(long id, @NonNull Handler handler, @NonNull FlutterJNI flutterJNI, @NonNull TextureRegistry.SurfaceTextureEntry texture) { + SurfaceTextureSurfaceProducer( + long id, + @NonNull Handler handler, + @NonNull FlutterJNI flutterJNI, + @NonNull TextureRegistry.SurfaceTextureEntry texture) { this.id = id; this.handler = handler; this.flutterJNI = flutterJNI; diff --git a/shell/platform/android/test/io/flutter/plugin/platform/PlatformViewsControllerTest.java b/shell/platform/android/test/io/flutter/plugin/platform/PlatformViewsControllerTest.java index 0ef5de4734962..0b69e6a801880 100644 --- a/shell/platform/android/test/io/flutter/plugin/platform/PlatformViewsControllerTest.java +++ b/shell/platform/android/test/io/flutter/plugin/platform/PlatformViewsControllerTest.java @@ -26,7 +26,6 @@ import android.view.ViewParent; import android.widget.FrameLayout; import android.widget.FrameLayout.LayoutParams; - import androidx.annotation.NonNull; import androidx.test.core.app.ApplicationProvider; import androidx.test.ext.junit.runners.AndroidJUnit4; diff --git a/testing/scenario_app/android/app/src/main/java/dev/flutter/scenarios/ExternalTextureFlutterActivity.java b/testing/scenario_app/android/app/src/main/java/dev/flutter/scenarios/ExternalTextureFlutterActivity.java index 883ae4d34f2f4..fcced923eb563 100644 --- a/testing/scenario_app/android/app/src/main/java/dev/flutter/scenarios/ExternalTextureFlutterActivity.java +++ b/testing/scenario_app/android/app/src/main/java/dev/flutter/scenarios/ExternalTextureFlutterActivity.java @@ -35,9 +35,7 @@ import androidx.annotation.Nullable; import androidx.annotation.RequiresApi; import androidx.core.util.Supplier; - import io.flutter.view.TextureRegistry; -import io.flutter.view.TextureRegistry.SurfaceTextureEntry; import java.io.IOException; import java.nio.ByteBuffer; import java.util.Map; diff --git a/testing/scenario_app/bin/utils/options.dart b/testing/scenario_app/bin/utils/options.dart index cc3ad5930fb89..43cb0e6bed5d3 100644 --- a/testing/scenario_app/bin/utils/options.dart +++ b/testing/scenario_app/bin/utils/options.dart @@ -305,7 +305,7 @@ extension type const Options._(ArgResults _args) { String? get outputContentsGolden => _args['output-contents-golden'] as String; /// Whether to force the use of `SurfaceTexture` for `SurfaceProducer`. - /// + /// /// Always returns `false` if `--enable-impeller` is `true` and /// `--impeller-backend` is not `opengles`. bool get forceSurfaceProducerSurfaceTexture { From 3d8deaf67c468c957a11ba2de7d78953ee89adf1 Mon Sep 17 00:00:00 2001 From: Matan Lurey Date: Wed, 28 Feb 2024 12:05:23 -0800 Subject: [PATCH 3/3] Fix test, prepare to update README. --- .../SurfaceTextureSurfaceProducerTest.java | 6 +- testing/scenario_app/android/README.md | 63 +++++++++++++------ testing/scenario_app/bin/README.md | 9 +-- 3 files changed, 51 insertions(+), 27 deletions(-) diff --git a/shell/platform/android/test/io/flutter/embedding/engine/renderer/SurfaceTextureSurfaceProducerTest.java b/shell/platform/android/test/io/flutter/embedding/engine/renderer/SurfaceTextureSurfaceProducerTest.java index 81aae38880613..fb97be732e42b 100644 --- a/shell/platform/android/test/io/flutter/embedding/engine/renderer/SurfaceTextureSurfaceProducerTest.java +++ b/shell/platform/android/test/io/flutter/embedding/engine/renderer/SurfaceTextureSurfaceProducerTest.java @@ -6,6 +6,7 @@ import android.annotation.TargetApi; import android.graphics.Canvas; +import android.graphics.SurfaceTexture; import android.os.Handler; import android.os.Looper; import android.view.Surface; @@ -22,10 +23,13 @@ public final class SurfaceTextureSurfaceProducerTest { @Test public void createsSurfaceTextureOfGivenSizeAndResizesWhenRequested() { + final FlutterRenderer flutterRenderer = new FlutterRenderer(fakeJNI); + // Create a surface and set the initial size. final Handler handler = new Handler(Looper.getMainLooper()); final SurfaceTextureSurfaceProducer producer = - new SurfaceTextureSurfaceProducer(0, handler, fakeJNI); + new SurfaceTextureSurfaceProducer( + 0, handler, fakeJNI, flutterRenderer.registerSurfaceTexture(new SurfaceTexture(0))); final Surface surface = producer.getSurface(); AtomicInteger frames = new AtomicInteger(); producer diff --git a/testing/scenario_app/android/README.md b/testing/scenario_app/android/README.md index 4dd6d2fb04218..a023eb3461991 100644 --- a/testing/scenario_app/android/README.md +++ b/testing/scenario_app/android/README.md @@ -17,6 +17,13 @@ Or for a specific, build, such as `android_debug_unopt_arm64`: dart ./testing/scenario_app/bin/run_android_tests.dart --out-dir=../out/android_debug_unopt_arm64 ``` +See also: + +- [File an issue][file_issue] with the `e: scenario-app, platform-android` + labels. + +[file_issue]: https://github.com/flutter/flutter/issues/new?labels=e:%20scenario-app,engine,platform-android,fyi-android,team-engine + ## Debugging Debugging the tests on CI is not straightforward but is being improved: @@ -38,31 +45,51 @@ or locally in `out/.../scenario_app/logs`. You can then view the logs and screenshots on LUCI. [For example](https://ci.chromium.org/ui/p/flutter/builders/try/Linux%20Engine%20Drone/2003164/overview): -![Screenshot of the Logs on LUCI](https://github.com/flutter/engine/assets/168174/79dc864c-c18b-4df9-a733-fd55301cc69c). +![Screenshot of the Logs on LUCI](https://github.com/flutter/engine/assets/168174/79dc864c-c18b-4df9-a733-fd55301cc69c) + +For a full list of flags, see [the runner](../bin/README.md). ## CI Configuration -See [`ci/builders/linux_android_emulator.json`](../../../ci/builders/linux_android_emulator.json) -, and grep for `run_android_tests.dart`. +See [`ci/builders`](../../../ci/builders) and grep for `run_android_tests.dart`. + +### Skia + +> [!NOTE] +> As of 2024-02-28, Flutter on Android defaults to the Skia graphics backend. + +There are two code branches we test using `scenario_app`: + +- Older Android devices, that use `SurfaceTexture`. + - CI Configuration (TODO: Link) + - CI History (TODO: Link) + - Skia Gold (TODO: Link) +- Newer Android devices, (API 34) that use `ImageReader`. + - CI Configuration (TODO: Link) + - CI History (TODO: Link) + - Skia Gold (TODO: Link) + +### Impeller with OpenGLES + +There are two code branches we test using `scenario_app`: -The following matrix of configurations is tested on the CI: +- Older Android devices, that use `SurfaceTexture`. + - CI Configuration (TODO: Link) + - CI History (TODO: Link) + - Skia Gold (TODO: Link) +- Newer Android devices, (API 34) that use `ImageReader`. + - CI Configuration (TODO: Link) + - CI History (TODO: Link) + - Skia Gold (TODO: Link) - +### Impeller with Vulkan -| API Version | Graphics Backend | Skia Gold | Rationale | -| ----------- | ------------------- | ---------------------------------------------------------------- | ------------------------------------------------ | -| 34 | Skia | [Android 34 + Skia][skia-gold-skia-34] | Newer Android devices on Skia. | -| 34 | Impeller (OpenGLES) | [Android 34 + Impeller OpenGLES][skia-gold-impeller-opengles-34] | Newer Android devices on Impeller with OpenGLES. | -| 34 | Impeller (Vulkan) | [Android 34 + Impeller Vulkan][skia-gold-impeller-vulkan-34] | Newer Android devices on Impeller. | +There is only a single code branch we test using `scenario_app`: -[skia-gold-skia-34]: https://flutter-engine-gold.skia.org/search?left_filter=AndroidAPILevel%3D34%26GraphicsBackend%3Dskia&negative=true&positive=true&right_filter=AndroidAPILevel%3D34%26GraphicsBackend%3Dskia -[skia-gold-impeller-opengles-34]: https://flutter-engine-gold.skia.org/search?left_filter=AndroidAPILevel%3D34%26GraphicsBackend%3Dimpeller-opengles&negative=true&positive=true&right_filter=AndroidAPILevel%3D34%26GraphicsBackend%3Dimpeller-opengles -[skia-gold-impeller-vulkan-34]: https://flutter-engine-gold.skia.org/search?left_filter=AndroidAPILevel%3D34%26GraphicsBackend%3Dimpeller-vulkan&negative=true&positive=true&right_filter=AndroidAPILevel%3D34%26GraphicsBackend%3Dimpeller-vulkan +- Newer Android devices, (API 34) + - CI Configuration (TODO: Link) + - CI History (TODO: Link) + - Skia Gold (TODO: Link) ## Updating Gradle dependencies diff --git a/testing/scenario_app/bin/README.md b/testing/scenario_app/bin/README.md index 6125827a8b75d..40863acbb5a0b 100644 --- a/testing/scenario_app/bin/README.md +++ b/testing/scenario_app/bin/README.md @@ -1,17 +1,10 @@ -# `android_integration_tests` runner +# Scenario App Android Test Runner This directory contains code specific to running Android integration tests. The tests are uploaded and run on the device using `adb`, and screenshots are captured and compared using Skia Gold (if available, for example on CI). -See also: - -- [File an issue][file_issue] with the `e: scenario-app, platform-android` - labels. - -[file_issue]: https://github.com/flutter/flutter/issues/new?labels=e:%20scenario-app,engine,platform-android,fyi-android,team-engine - ## Usage ```sh