diff --git a/shell/platform/android/io/flutter/plugin/editing/TextInputPlugin.java b/shell/platform/android/io/flutter/plugin/editing/TextInputPlugin.java index a96b0deeabb22..c11d264c21988 100644 --- a/shell/platform/android/io/flutter/plugin/editing/TextInputPlugin.java +++ b/shell/platform/android/io/flutter/plugin/editing/TextInputPlugin.java @@ -5,7 +5,9 @@ package io.flutter.plugin.editing; import android.annotation.SuppressLint; +import android.annotation.TargetApi; import android.content.Context; +import android.graphics.Insets; import android.graphics.Rect; import android.os.Build; import android.os.Bundle; @@ -16,6 +18,8 @@ import android.util.SparseArray; import android.view.View; import android.view.ViewStructure; +import android.view.WindowInsets; +import android.view.WindowInsetsAnimation; import android.view.autofill.AutofillId; import android.view.autofill.AutofillManager; import android.view.autofill.AutofillValue; @@ -26,10 +30,12 @@ import android.view.inputmethod.InputMethodSubtype; import androidx.annotation.NonNull; import androidx.annotation.Nullable; +import androidx.annotation.RequiresApi; import androidx.annotation.VisibleForTesting; import io.flutter.embedding.engine.systemchannels.TextInputChannel; import io.flutter.plugin.platform.PlatformViewsController; import java.util.HashMap; +import java.util.List; /** Android implementation of the text input plugin. */ public class TextInputPlugin { @@ -46,6 +52,7 @@ public class TextInputPlugin { @NonNull private PlatformViewsController platformViewsController; @Nullable private Rect lastClientRect; private final boolean restartAlwaysRequired; + private ImeSyncDeferringInsetsCallback imeSyncCallback; // When true following calls to createInputConnection will return the cached lastInputConnection // if the input @@ -53,6 +60,7 @@ public class TextInputPlugin { // details. private boolean isInputConnectionLocked; + @SuppressLint("NewApi") public TextInputPlugin( View view, @NonNull TextInputChannel textInputChannel, @@ -65,6 +73,27 @@ public TextInputPlugin( afm = null; } + // Sets up syncing ime insets with the framework, allowing + // the Flutter view to grow and shrink to accomodate Android + // controlled keyboard animations. + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { + int mask = 0; + if ((View.SYSTEM_UI_FLAG_HIDE_NAVIGATION & mView.getWindowSystemUiVisibility()) == 0) { + mask = mask | WindowInsets.Type.navigationBars(); + } + if ((View.SYSTEM_UI_FLAG_FULLSCREEN & mView.getWindowSystemUiVisibility()) == 0) { + mask = mask | WindowInsets.Type.statusBars(); + } + imeSyncCallback = + new ImeSyncDeferringInsetsCallback( + view, + mask, // Overlay, insets that should be merged with the deferred insets + WindowInsets.Type.ime() // Deferred, insets that will animate + ); + mView.setWindowInsetsAnimationCallback(imeSyncCallback); + mView.setOnApplyWindowInsetsListener(imeSyncCallback); + } + this.textInputChannel = textInputChannel; textInputChannel.setTextInputMethodHandler( new TextInputChannel.TextInputMethodHandler() { @@ -134,6 +163,139 @@ public void sendAppPrivateCommand(String action, Bundle data) { restartAlwaysRequired = isRestartAlwaysRequired(); } + // Loosely based off of + // https://github.com/android/user-interface-samples/blob/master/WindowInsetsAnimation/app/src/main/java/com/google/android/samples/insetsanimation/RootViewDeferringInsetsCallback.kt + // + // When the IME is shown or hidden, it immediately sends an onApplyWindowInsets call + // with the final state of the IME. This initial call disrupts the animation, which + // causes a flicker in the beginning. + // + // To fix this, this class extends WindowInsetsAnimation.Callback and implements + // OnApplyWindowInsetsListener. We capture and defer the initial call to + // onApplyWindowInsets while the animation completes. When the animation + // finishes, we can then release the call by invoking it in the onEnd callback + // + // The WindowInsetsAnimation.Callback extension forwards the new state of the + // IME inset from onProgress() to the framework. We also make use of the + // onStart callback to detect which calls to onApplyWindowInsets would + // interrupt the animation and defer it. + // + // By implementing OnApplyWindowInsetsListener, we are able to capture Android's + // attempts to call the FlutterView's onApplyWindowInsets. When a call to onStart + // occurs, we can mark any non-animation calls to onApplyWindowInsets() that + // occurs between prepare and start as deferred by using this class' wrapper + // implementation to cache the WindowInsets passed in and turn the current call into + // a no-op. When onEnd indicates the end of the animation, the deferred call is + // dispatched again, this time avoiding any flicker since the animation is now + // complete. + @VisibleForTesting + @TargetApi(30) + @RequiresApi(30) + @SuppressLint({"NewApi", "Override"}) + class ImeSyncDeferringInsetsCallback extends WindowInsetsAnimation.Callback + implements View.OnApplyWindowInsetsListener { + private int overlayInsetTypes; + private int deferredInsetTypes; + + private View view; + private WindowInsets lastWindowInsets; + private boolean started = false; + + ImeSyncDeferringInsetsCallback( + @NonNull View view, int overlayInsetTypes, int deferredInsetTypes) { + super(WindowInsetsAnimation.Callback.DISPATCH_MODE_CONTINUE_ON_SUBTREE); + this.overlayInsetTypes = overlayInsetTypes; + this.deferredInsetTypes = deferredInsetTypes; + this.view = view; + } + + @Override + public WindowInsets onApplyWindowInsets(View view, WindowInsets windowInsets) { + this.view = view; + if (started) { + // While animation is running, we consume the insets to prevent disrupting + // the animation, which skips this implementation and calls the view's + // onApplyWindowInsets directly to avoid being consumed here. + return WindowInsets.CONSUMED; + } + + // Store the view and insets for us in onEnd() below + lastWindowInsets = windowInsets; + + // If no animation is happening, pass the insets on to the view's own + // inset handling. + return view.onApplyWindowInsets(windowInsets); + } + + @Override + public WindowInsetsAnimation.Bounds onStart( + WindowInsetsAnimation animation, WindowInsetsAnimation.Bounds bounds) { + if ((animation.getTypeMask() & deferredInsetTypes) != 0) { + started = true; + } + return bounds; + } + + @Override + public WindowInsets onProgress( + WindowInsets insets, List runningAnimations) { + if (!started) { + return insets; + } + boolean matching = false; + for (WindowInsetsAnimation animation : runningAnimations) { + if ((animation.getTypeMask() & deferredInsetTypes) != 0) { + matching = true; + continue; + } + } + if (!matching) { + return insets; + } + WindowInsets.Builder builder = new WindowInsets.Builder(lastWindowInsets); + // Overlay the ime-only insets with the full insets. + // + // The IME insets passed in by onProgress assumes that the entire animation + // occurs above any present navigation and status bars. This causes the + // IME inset to be too large for the animation. To remedy this, we merge the + // IME inset with other insets present via a subtract + reLu, which causes the + // IME inset to be overlaid with any bars present. + Insets newImeInsets = + Insets.of( + 0, + 0, + 0, + Math.max( + insets.getInsets(deferredInsetTypes).bottom + - insets.getInsets(overlayInsetTypes).bottom, + 0)); + builder.setInsets(deferredInsetTypes, newImeInsets); + // Directly call onApplyWindowInsets of the view as we do not want to pass through + // the onApplyWindowInsets defined in this class, which would consume the insets + // as if they were a non-animation inset change and cache it for re-dispatch in + // onEnd instead. + view.onApplyWindowInsets(builder.build()); + return insets; + } + + @Override + public void onEnd(WindowInsetsAnimation animation) { + if (started && (animation.getTypeMask() & deferredInsetTypes) != 0) { + // If we deferred the IME insets and an IME animation has finished, we need to reset + // the flags + started = false; + + // And finally dispatch the deferred insets to the view now. + // Ideally we would just call view.requestApplyInsets() and let the normal dispatch + // cycle happen, but this happens too late resulting in a visual flicker. + // Instead we manually dispatch the most recent WindowInsets to the view. + if (lastWindowInsets != null && view != null) { + view.dispatchApplyWindowInsets(lastWindowInsets); + } + } + } + } + @NonNull public InputMethodManager getInputMethodManager() { return mImm; @@ -144,6 +306,11 @@ Editable getEditable() { return mEditable; } + @VisibleForTesting + ImeSyncDeferringInsetsCallback getImeSyncCallback() { + return imeSyncCallback; + } + /** * Use the current platform view input connection until unlockPlatformViewInputConnection is * called. @@ -177,9 +344,14 @@ public void unlockPlatformViewInputConnection() { * *

The TextInputPlugin instance should not be used after calling this. */ + @SuppressLint("NewApi") public void destroy() { platformViewsController.detachTextInputPlugin(); textInputChannel.setTextInputMethodHandler(null); + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { + mView.setWindowInsetsAnimationCallback(null); + mView.setOnApplyWindowInsetsListener(null); + } } private static int inputTypeFromTextInputType( diff --git a/shell/platform/android/test/io/flutter/plugin/editing/TextInputPluginTest.java b/shell/platform/android/test/io/flutter/plugin/editing/TextInputPluginTest.java index 1a6b53c6b7080..562f1f51dcf13 100644 --- a/shell/platform/android/test/io/flutter/plugin/editing/TextInputPluginTest.java +++ b/shell/platform/android/test/io/flutter/plugin/editing/TextInputPluginTest.java @@ -15,8 +15,10 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import android.annotation.TargetApi; import android.content.Context; import android.content.res.AssetManager; +import android.graphics.Insets; import android.os.Build; import android.os.Bundle; import android.provider.Settings; @@ -24,14 +26,19 @@ import android.view.KeyEvent; import android.view.View; import android.view.ViewStructure; +import android.view.WindowInsets; +import android.view.WindowInsetsAnimation; import android.view.inputmethod.CursorAnchorInfo; import android.view.inputmethod.EditorInfo; import android.view.inputmethod.InputConnection; import android.view.inputmethod.InputMethodManager; import android.view.inputmethod.InputMethodSubtype; import io.flutter.embedding.android.FlutterView; +import io.flutter.embedding.engine.FlutterEngine; import io.flutter.embedding.engine.FlutterJNI; import io.flutter.embedding.engine.dart.DartExecutor; +import io.flutter.embedding.engine.loader.FlutterLoader; +import io.flutter.embedding.engine.renderer.FlutterRenderer; import io.flutter.embedding.engine.systemchannels.TextInputChannel; import io.flutter.plugin.common.BinaryMessenger; import io.flutter.plugin.common.JSONMethodCodec; @@ -39,12 +46,14 @@ import io.flutter.plugin.platform.PlatformViewsController; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.List; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.ArgumentCaptor; +import org.mockito.Mock; import org.robolectric.RobolectricTestRunner; import org.robolectric.RuntimeEnvironment; import org.robolectric.annotation.Config; @@ -57,6 +66,9 @@ @Config(manifest = Config.NONE, shadows = TextInputPluginTest.TestImm.class) @RunWith(RobolectricTestRunner.class) public class TextInputPluginTest { + @Mock FlutterJNI mockFlutterJni; + @Mock FlutterLoader mockFlutterLoader; + // Verifies the method and arguments for a captured method call. private void verifyMethodCall(ByteBuffer buffer, String methodName, String[] expectedArgs) throws JSONException { @@ -632,6 +644,92 @@ public void sendAppPrivateCommand_hasData() throws JSONException { assertEquals("actionData", bundleCaptor.getValue().getCharSequence("data")); } + @Test + @TargetApi(30) + @Config(sdk = 30) + public void ime_windowInsetsSync() { + FlutterView testView = new FlutterView(RuntimeEnvironment.application); + TextInputChannel textInputChannel = new TextInputChannel(mock(DartExecutor.class)); + TextInputPlugin textInputPlugin = + new TextInputPlugin(testView, textInputChannel, mock(PlatformViewsController.class)); + TextInputPlugin.ImeSyncDeferringInsetsCallback imeSyncCallback = + textInputPlugin.getImeSyncCallback(); + FlutterEngine flutterEngine = + spy(new FlutterEngine(RuntimeEnvironment.application, mockFlutterLoader, mockFlutterJni)); + FlutterRenderer flutterRenderer = spy(new FlutterRenderer(mockFlutterJni)); + when(flutterEngine.getRenderer()).thenReturn(flutterRenderer); + testView.attachToFlutterEngine(flutterEngine); + + WindowInsetsAnimation animation = mock(WindowInsetsAnimation.class); + when(animation.getTypeMask()).thenReturn(WindowInsets.Type.ime()); + + List animationList = new ArrayList(); + animationList.add(animation); + + WindowInsets.Builder builder = new WindowInsets.Builder(); + WindowInsets noneInsets = builder.build(); + + builder.setInsets(WindowInsets.Type.ime(), Insets.of(0, 0, 0, 100)); + builder.setInsets(WindowInsets.Type.navigationBars(), Insets.of(10, 10, 10, 40)); + WindowInsets imeInsets0 = builder.build(); + + builder.setInsets(WindowInsets.Type.ime(), Insets.of(0, 0, 0, 30)); + builder.setInsets(WindowInsets.Type.navigationBars(), Insets.of(10, 10, 10, 40)); + WindowInsets imeInsets1 = builder.build(); + + builder.setInsets(WindowInsets.Type.ime(), Insets.of(0, 0, 0, 200)); + builder.setInsets(WindowInsets.Type.navigationBars(), Insets.of(10, 10, 10, 0)); + WindowInsets deferredInsets = builder.build(); + + ArgumentCaptor viewportMetricsCaptor = + ArgumentCaptor.forClass(FlutterRenderer.ViewportMetrics.class); + + imeSyncCallback.onApplyWindowInsets(testView, deferredInsets); + imeSyncCallback.onApplyWindowInsets(testView, noneInsets); + + verify(flutterRenderer).setViewportMetrics(viewportMetricsCaptor.capture()); + assertEquals(0, viewportMetricsCaptor.getValue().paddingBottom); + assertEquals(0, viewportMetricsCaptor.getValue().paddingTop); + assertEquals(0, viewportMetricsCaptor.getValue().viewInsetBottom); + assertEquals(0, viewportMetricsCaptor.getValue().viewInsetTop); + + imeSyncCallback.onPrepare(animation); + imeSyncCallback.onApplyWindowInsets(testView, deferredInsets); + imeSyncCallback.onStart(animation, null); + + verify(flutterRenderer).setViewportMetrics(viewportMetricsCaptor.capture()); + // No change, as deferredInset is stored to be passed in onEnd() + assertEquals(0, viewportMetricsCaptor.getValue().paddingBottom); + assertEquals(0, viewportMetricsCaptor.getValue().paddingTop); + assertEquals(0, viewportMetricsCaptor.getValue().viewInsetBottom); + assertEquals(0, viewportMetricsCaptor.getValue().viewInsetTop); + + imeSyncCallback.onProgress(imeInsets0, animationList); + + verify(flutterRenderer).setViewportMetrics(viewportMetricsCaptor.capture()); + assertEquals(40, viewportMetricsCaptor.getValue().paddingBottom); + assertEquals(10, viewportMetricsCaptor.getValue().paddingTop); + assertEquals(60, viewportMetricsCaptor.getValue().viewInsetBottom); + assertEquals(0, viewportMetricsCaptor.getValue().viewInsetTop); + + imeSyncCallback.onProgress(imeInsets1, animationList); + + verify(flutterRenderer).setViewportMetrics(viewportMetricsCaptor.capture()); + assertEquals(40, viewportMetricsCaptor.getValue().paddingBottom); + assertEquals(10, viewportMetricsCaptor.getValue().paddingTop); + assertEquals(0, viewportMetricsCaptor.getValue().viewInsetBottom); // Cannot be negative + assertEquals(0, viewportMetricsCaptor.getValue().viewInsetTop); + + imeSyncCallback.onEnd(animation); + + verify(flutterRenderer).setViewportMetrics(viewportMetricsCaptor.capture()); + // Values should be of deferredInsets + assertEquals(0, viewportMetricsCaptor.getValue().paddingBottom); + assertEquals(10, viewportMetricsCaptor.getValue().paddingTop); + assertEquals(200, viewportMetricsCaptor.getValue().viewInsetBottom); + assertEquals(0, viewportMetricsCaptor.getValue().viewInsetTop); + } + interface EventHandler { void sendAppPrivateCommand(View view, String action, Bundle data); }