[Android R] Sync keyboard animation with view insets vs Android 11/R/API 30 WindowInsetsAnimation

shell/platform/android/io/flutter/plugin/editing/TextInputPlugin.java

@@ -5,7 +5,9 @@
 package io.flutter.plugin.editing;
 
 import android.annotation.SuppressLint;
+import android.annotation.TargetApi;
 import android.content.Context;
+import android.graphics.Insets;
 import android.graphics.Rect;
 import android.os.Build;
 import android.os.Bundle;
@@ -16,6 +18,8 @@
 import android.util.SparseArray;
 import android.view.View;
 import android.view.ViewStructure;
+import android.view.WindowInsets;
+import android.view.WindowInsetsAnimation;
 import android.view.autofill.AutofillId;
 import android.view.autofill.AutofillManager;
 import android.view.autofill.AutofillValue;
@@ -26,10 +30,12 @@
 import android.view.inputmethod.InputMethodSubtype;
 import androidx.annotation.NonNull;
 import androidx.annotation.Nullable;
+import androidx.annotation.RequiresApi;
 import androidx.annotation.VisibleForTesting;
 import io.flutter.embedding.engine.systemchannels.TextInputChannel;
 import io.flutter.plugin.platform.PlatformViewsController;
 import java.util.HashMap;
+import java.util.List;
 
 /** Android implementation of the text input plugin. */
 public class TextInputPlugin {
@@ -46,13 +52,15 @@ public class TextInputPlugin {
   @NonNull private PlatformViewsController platformViewsController;
   @Nullable private Rect lastClientRect;
   private final boolean restartAlwaysRequired;
+  private ImeSyncDeferringInsetsCallback imeSyncCallback;
 
   // When true following calls to createInputConnection will return the cached lastInputConnection
   // if the input
   // target is a platform view. See the comments on lockPlatformViewInputConnection for more
   // details.
   private boolean isInputConnectionLocked;
 
+  @SuppressLint("NewApi")
   public TextInputPlugin(
       View view,
       @NonNull TextInputChannel textInputChannel,
@@ -65,6 +73,27 @@ public TextInputPlugin(
       afm = null;
     }
 
+    // Sets up syncing ime insets with the framework, allowing
+    // the Flutter view to grow and shrink to accomodate Android
+    // controlled keyboard animations.
+    if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
+      int mask = 0;
+      if ((View.SYSTEM_UI_FLAG_HIDE_NAVIGATION & mView.getWindowSystemUiVisibility()) == 0) {
+        mask = mask | WindowInsets.Type.navigationBars();
+      }
+      if ((View.SYSTEM_UI_FLAG_FULLSCREEN & mView.getWindowSystemUiVisibility()) == 0) {
+        mask = mask | WindowInsets.Type.statusBars();
+      }
+      imeSyncCallback =
+          new ImeSyncDeferringInsetsCallback(
+              view,
+              mask, // Overlay, insets that should be merged with the deferred insets
+              WindowInsets.Type.ime() // Deferred, insets that will animate
+              );
+      mView.setWindowInsetsAnimationCallback(imeSyncCallback);
+      mView.setOnApplyWindowInsetsListener(imeSyncCallback);
+    }
+
     this.textInputChannel = textInputChannel;
     textInputChannel.setTextInputMethodHandler(
         new TextInputChannel.TextInputMethodHandler() {
@@ -134,6 +163,139 @@ public void sendAppPrivateCommand(String action, Bundle data) {
     restartAlwaysRequired = isRestartAlwaysRequired();
   }
 
+  // Loosely based off of
+  // https://github.com/android/user-interface-samples/blob/master/WindowInsetsAnimation/app/src/main/java/com/google/android/samples/insetsanimation/RootViewDeferringInsetsCallback.kt
+  //
+  // When the IME is shown or hidden, it immediately sends an onApplyWindowInsets call
+  // with the final state of the IME. This initial call disrupts the animation, which
+  // causes a flicker in the beginning.
+  //
+  // To fix this, this class extends WindowInsetsAnimation.Callback and implements
+  // OnApplyWindowInsetsListener. We capture and defer the initial call to
+  // onApplyWindowInsets while the animation completes. When the animation
+  // finishes, we can then release the call by invoking it in the onEnd callback
+  //
+  // The WindowInsetsAnimation.Callback extension forwards the new state of the
+  // IME inset from onProgress() to the framework. We also make use of the
+  // onStart callback to detect which calls to onApplyWindowInsets would
+  // interrupt the animation and defer it.
+  //
+  // By implementing OnApplyWindowInsetsListener, we are able to capture Android's
+  // attempts to call the FlutterView's onApplyWindowInsets. When a call to onStart
+  // occurs, we can mark any non-animation calls to onApplyWindowInsets() that
+  // occurs between prepare and start as deferred by using this class' wrapper
+  // implementation to cache the WindowInsets passed in and turn the current call into
+  // a no-op. When onEnd indicates the end of the animation, the deferred call is
+  // dispatched again, this time avoiding any flicker since the animation is now
+  // complete.
+  @VisibleForTesting
+  @TargetApi(30)
+  @RequiresApi(30)
+  @SuppressLint({"NewApi", "Override"})
+  class ImeSyncDeferringInsetsCallback extends WindowInsetsAnimation.Callback
+      implements View.OnApplyWindowInsetsListener {
+    private int overlayInsetTypes;
+    private int deferredInsetTypes;
+
+    private View view;
+    private WindowInsets lastWindowInsets;
+    private boolean started = false;
+
+    ImeSyncDeferringInsetsCallback(
+        @NonNull View view, int overlayInsetTypes, int deferredInsetTypes) {
+      super(WindowInsetsAnimation.Callback.DISPATCH_MODE_CONTINUE_ON_SUBTREE);
+      this.overlayInsetTypes = overlayInsetTypes;
+      this.deferredInsetTypes = deferredInsetTypes;
+      this.view = view;
+    }
+
+    @Override
+    public WindowInsets onApplyWindowInsets(View view, WindowInsets windowInsets) {
+      this.view = view;
+      if (started) {
+        // While animation is running, we consume the insets to prevent disrupting
+        // the animation, which skips this implementation and calls the view's
+        // onApplyWindowInsets directly to avoid being consumed here.
+        return WindowInsets.CONSUMED;
+      }
+
+      // Store the view and insets for us in onEnd() below
+      lastWindowInsets = windowInsets;
+
+      // If no animation is happening, pass the insets on to the view's own
+      // inset handling.
+      return view.onApplyWindowInsets(windowInsets);
+    }
+
+    @Override
+    public WindowInsetsAnimation.Bounds onStart(
+        WindowInsetsAnimation animation, WindowInsetsAnimation.Bounds bounds) {
+      if ((animation.getTypeMask() & deferredInsetTypes) != 0) {
+        started = true;
+      }
+      return bounds;
+    }
+
+    @Override
+    public WindowInsets onProgress(
+        WindowInsets insets, List<WindowInsetsAnimation> runningAnimations) {
+      if (!started) {
+        return insets;
+      }
+      boolean matching = false;
+      for (WindowInsetsAnimation animation : runningAnimations) {
+        if ((animation.getTypeMask() & deferredInsetTypes) != 0) {
+          matching = true;
+          continue;
+        }
+      }
+      if (!matching) {
+        return insets;
+      }
+      WindowInsets.Builder builder = new WindowInsets.Builder(lastWindowInsets);
+      // Overlay the ime-only insets with the full insets.
+      //
+      // The IME insets passed in by onProgress assumes that the entire animation
+      // occurs above any present navigation and status bars. This causes the
+      // IME inset to be too large for the animation. To remedy this, we merge the
+      // IME inset with other insets present via a subtract + reLu, which causes the
+      // IME inset to be overlaid with any bars present.
+      Insets newImeInsets =
+          Insets.of(
+              0,
+              0,
+              0,
+              Math.max(
+                  insets.getInsets(deferredInsetTypes).bottom
+                      - insets.getInsets(overlayInsetTypes).bottom,
+                  0));
+      builder.setInsets(deferredInsetTypes, newImeInsets);
+      // Directly call onApplyWindowInsets of the view as we do not want to pass through
+      // the onApplyWindowInsets defined in this class, which would consume the insets
+      // as if they were a non-animation inset change and cache it for re-dispatch in
+      // onEnd instead.
+      view.onApplyWindowInsets(builder.build());
+      return insets;
+    }
+
+    @Override
+    public void onEnd(WindowInsetsAnimation animation) {
+      if (started && (animation.getTypeMask() & deferredInsetTypes) != 0) {
+        // If we deferred the IME insets and an IME animation has finished, we need to reset
+        // the flags
+        started = false;
+
+        // And finally dispatch the deferred insets to the view now.
+        // Ideally we would just call view.requestApplyInsets() and let the normal dispatch
+        // cycle happen, but this happens too late resulting in a visual flicker.
+        // Instead we manually dispatch the most recent WindowInsets to the view.
+        if (lastWindowInsets != null && view != null) {
+          view.dispatchApplyWindowInsets(lastWindowInsets);
+        }
+      }
+    }
+  }
+
   @NonNull
   public InputMethodManager getInputMethodManager() {
     return mImm;
@@ -144,6 +306,11 @@ Editable getEditable() {
     return mEditable;
   }
 
+  @VisibleForTesting
+  ImeSyncDeferringInsetsCallback getImeSyncCallback() {
+    return imeSyncCallback;
+  }
+
   /**
    * Use the current platform view input connection until unlockPlatformViewInputConnection is
    * called.
@@ -177,9 +344,14 @@ public void unlockPlatformViewInputConnection() {
    *
    * <p>The TextInputPlugin instance should not be used after calling this.
    */
+  @SuppressLint("NewApi")
   public void destroy() {
     platformViewsController.detachTextInputPlugin();
     textInputChannel.setTextInputMethodHandler(null);
+    if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
+      mView.setWindowInsetsAnimationCallback(null);
+      mView.setOnApplyWindowInsetsListener(null);
+    }
   }
 
   private static int inputTypeFromTextInputType(

shell/platform/android/test/io/flutter/plugin/editing/TextInputPluginTest.java

@@ -15,36 +15,45 @@
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
+import android.annotation.TargetApi;
 import android.content.Context;
 import android.content.res.AssetManager;
+import android.graphics.Insets;
 import android.os.Build;
 import android.os.Bundle;
 import android.provider.Settings;
 import android.util.SparseIntArray;
 import android.view.KeyEvent;
 import android.view.View;
 import android.view.ViewStructure;
+import android.view.WindowInsets;
+import android.view.WindowInsetsAnimation;
 import android.view.inputmethod.CursorAnchorInfo;
 import android.view.inputmethod.EditorInfo;
 import android.view.inputmethod.InputConnection;
 import android.view.inputmethod.InputMethodManager;
 import android.view.inputmethod.InputMethodSubtype;
 import io.flutter.embedding.android.FlutterView;
+import io.flutter.embedding.engine.FlutterEngine;
 import io.flutter.embedding.engine.FlutterJNI;
 import io.flutter.embedding.engine.dart.DartExecutor;
+import io.flutter.embedding.engine.loader.FlutterLoader;
+import io.flutter.embedding.engine.renderer.FlutterRenderer;
 import io.flutter.embedding.engine.systemchannels.TextInputChannel;
 import io.flutter.plugin.common.BinaryMessenger;
 import io.flutter.plugin.common.JSONMethodCodec;
 import io.flutter.plugin.common.MethodCall;
 import io.flutter.plugin.platform.PlatformViewsController;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
+import java.util.List;
 import org.json.JSONArray;
 import org.json.JSONException;
 import org.json.JSONObject;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.mockito.ArgumentCaptor;
+import org.mockito.Mock;
 import org.robolectric.RobolectricTestRunner;
 import org.robolectric.RuntimeEnvironment;
 import org.robolectric.annotation.Config;
@@ -57,6 +66,9 @@
 @Config(manifest = Config.NONE, shadows = TextInputPluginTest.TestImm.class)
 @RunWith(RobolectricTestRunner.class)
 public class TextInputPluginTest {
+  @Mock FlutterJNI mockFlutterJni;
+  @Mock FlutterLoader mockFlutterLoader;
+
   // Verifies the method and arguments for a captured method call.
   private void verifyMethodCall(ByteBuffer buffer, String methodName, String[] expectedArgs)
       throws JSONException {
@@ -632,6 +644,92 @@ public void sendAppPrivateCommand_hasData() throws JSONException {
     assertEquals("actionData", bundleCaptor.getValue().getCharSequence("data"));
   }
 
+  @Test
+  @TargetApi(30)
+  @Config(sdk = 30)
+  public void ime_windowInsetsSync() {
+    FlutterView testView = new FlutterView(RuntimeEnvironment.application);
+    TextInputChannel textInputChannel = new TextInputChannel(mock(DartExecutor.class));
+    TextInputPlugin textInputPlugin =
+        new TextInputPlugin(testView, textInputChannel, mock(PlatformViewsController.class));
+    TextInputPlugin.ImeSyncDeferringInsetsCallback imeSyncCallback =
+        textInputPlugin.getImeSyncCallback();
+    FlutterEngine flutterEngine =
+        spy(new FlutterEngine(RuntimeEnvironment.application, mockFlutterLoader, mockFlutterJni));
+    FlutterRenderer flutterRenderer = spy(new FlutterRenderer(mockFlutterJni));
+    when(flutterEngine.getRenderer()).thenReturn(flutterRenderer);
+    testView.attachToFlutterEngine(flutterEngine);
+
+    WindowInsetsAnimation animation = mock(WindowInsetsAnimation.class);
+    when(animation.getTypeMask()).thenReturn(WindowInsets.Type.ime());
+
+    List<WindowInsetsAnimation> animationList = new ArrayList();
+    animationList.add(animation);
+
+    WindowInsets.Builder builder = new WindowInsets.Builder();
+    WindowInsets noneInsets = builder.build();
+
+    builder.setInsets(WindowInsets.Type.ime(), Insets.of(0, 0, 0, 100));
+    builder.setInsets(WindowInsets.Type.navigationBars(), Insets.of(10, 10, 10, 40));
+    WindowInsets imeInsets0 = builder.build();
+
+    builder.setInsets(WindowInsets.Type.ime(), Insets.of(0, 0, 0, 30));
+    builder.setInsets(WindowInsets.Type.navigationBars(), Insets.of(10, 10, 10, 40));
+    WindowInsets imeInsets1 = builder.build();
+
+    builder.setInsets(WindowInsets.Type.ime(), Insets.of(0, 0, 0, 200));
+    builder.setInsets(WindowInsets.Type.navigationBars(), Insets.of(10, 10, 10, 0));
+    WindowInsets deferredInsets = builder.build();
+
+    ArgumentCaptor<FlutterRenderer.ViewportMetrics> viewportMetricsCaptor =
+        ArgumentCaptor.forClass(FlutterRenderer.ViewportMetrics.class);
+
+    imeSyncCallback.onApplyWindowInsets(testView, deferredInsets);
+    imeSyncCallback.onApplyWindowInsets(testView, noneInsets);
+
+    verify(flutterRenderer).setViewportMetrics(viewportMetricsCaptor.capture());
+    assertEquals(0, viewportMetricsCaptor.getValue().paddingBottom);
+    assertEquals(0, viewportMetricsCaptor.getValue().paddingTop);
+    assertEquals(0, viewportMetricsCaptor.getValue().viewInsetBottom);
+    assertEquals(0, viewportMetricsCaptor.getValue().viewInsetTop);
+
+    imeSyncCallback.onPrepare(animation);
+    imeSyncCallback.onApplyWindowInsets(testView, deferredInsets);
+    imeSyncCallback.onStart(animation, null);
+
+    verify(flutterRenderer).setViewportMetrics(viewportMetricsCaptor.capture());
+    // No change, as deferredInset is stored to be passed in onEnd()
+    assertEquals(0, viewportMetricsCaptor.getValue().paddingBottom);
+    assertEquals(0, viewportMetricsCaptor.getValue().paddingTop);
+    assertEquals(0, viewportMetricsCaptor.getValue().viewInsetBottom);
+    assertEquals(0, viewportMetricsCaptor.getValue().viewInsetTop);
+
+    imeSyncCallback.onProgress(imeInsets0, animationList);
+
+    verify(flutterRenderer).setViewportMetrics(viewportMetricsCaptor.capture());
+    assertEquals(40, viewportMetricsCaptor.getValue().paddingBottom);
+    assertEquals(10, viewportMetricsCaptor.getValue().paddingTop);
+    assertEquals(60, viewportMetricsCaptor.getValue().viewInsetBottom);
+    assertEquals(0, viewportMetricsCaptor.getValue().viewInsetTop);
+
+    imeSyncCallback.onProgress(imeInsets1, animationList);
+
+    verify(flutterRenderer).setViewportMetrics(viewportMetricsCaptor.capture());
+    assertEquals(40, viewportMetricsCaptor.getValue().paddingBottom);
+    assertEquals(10, viewportMetricsCaptor.getValue().paddingTop);
+    assertEquals(0, viewportMetricsCaptor.getValue().viewInsetBottom); // Cannot be negative
+    assertEquals(0, viewportMetricsCaptor.getValue().viewInsetTop);
+
+    imeSyncCallback.onEnd(animation);
+
+    verify(flutterRenderer).setViewportMetrics(viewportMetricsCaptor.capture());
+    // Values should be of deferredInsets
+    assertEquals(0, viewportMetricsCaptor.getValue().paddingBottom);
+    assertEquals(10, viewportMetricsCaptor.getValue().paddingTop);
+    assertEquals(200, viewportMetricsCaptor.getValue().viewInsetBottom);
+    assertEquals(0, viewportMetricsCaptor.getValue().viewInsetTop);
+  }
+
   interface EventHandler {
     void sendAppPrivateCommand(View view, String action, Bundle data);
   }