[Windows] Synthesize modifier keys events on pointer events

Author: bleroux

shell/platform/windows/flutter_window.cc

@@ -156,8 +156,10 @@ void FlutterWindow::OnPaint() {
 void FlutterWindow::OnPointerMove(double x,
                                   double y,
                                   FlutterPointerDeviceKind device_kind,
-                                  int32_t device_id) {
-  binding_handler_delegate_->OnPointerMove(x, y, device_kind, device_id);
+                                  int32_t device_id,
+                                  int modifiers_state) {
+  binding_handler_delegate_->OnPointerMove(x, y, device_kind, device_id,
+                                           modifiers_state);
 }
 
 void FlutterWindow::OnPointerDown(double x,

shell/platform/windows/flutter_window.h

@@ -41,7 +41,8 @@ class FlutterWindow : public Window, public WindowBindingHandler {
   void OnPointerMove(double x,
                      double y,
                      FlutterPointerDeviceKind device_kind,
-                     int32_t device_id) override;
+                     int32_t device_id,
+                     int modifiers_state) override;
 
   // |Window|
   void OnPointerDown(double x,

shell/platform/windows/flutter_window_unittests.cc

@@ -42,6 +42,9 @@ class SpyKeyboardKeyHandler : public KeyboardHandlerBase {
     ON_CALL(*this, KeyboardHook(_, _, _, _, _, _, _))
         .WillByDefault(Invoke(real_implementation_.get(),
                               &KeyboardKeyHandler::KeyboardHook));
+    ON_CALL(*this, SyncModifiersIfNeeded(_))
+        .WillByDefault(Invoke(real_implementation_.get(),
+                              &KeyboardKeyHandler::SyncModifiersIfNeeded));
   }
 
   MOCK_METHOD7(KeyboardHook,
@@ -53,6 +56,8 @@ class SpyKeyboardKeyHandler : public KeyboardHandlerBase {
                     bool was_down,
                     KeyEventCallback callback));
 
+  MOCK_METHOD1(SyncModifiersIfNeeded, void(int modifiers_state));
+
  private:
   std::unique_ptr<KeyboardKeyHandler> real_implementation_;
 };
@@ -266,15 +271,15 @@ TEST(FlutterWindowTest, OnPointerStarSendsDeviceType) {
   // Move
   EXPECT_CALL(delegate,
               OnPointerMove(10.0, 10.0, kFlutterPointerDeviceKindMouse,
-                            kDefaultPointerDeviceId))
+                            kDefaultPointerDeviceId, 0))
       .Times(1);
   EXPECT_CALL(delegate,
               OnPointerMove(10.0, 10.0, kFlutterPointerDeviceKindTouch,
-                            kDefaultPointerDeviceId))
+                            kDefaultPointerDeviceId, 0))
       .Times(1);
   EXPECT_CALL(delegate,
               OnPointerMove(10.0, 10.0, kFlutterPointerDeviceKindStylus,
-                            kDefaultPointerDeviceId))
+                            kDefaultPointerDeviceId, 0))
       .Times(1);
 
   // Down
@@ -323,7 +328,7 @@ TEST(FlutterWindowTest, OnPointerStarSendsDeviceType) {
       .Times(1);
 
   win32window.OnPointerMove(10.0, 10.0, kFlutterPointerDeviceKindMouse,
-                            kDefaultPointerDeviceId);
+                            kDefaultPointerDeviceId, 0);
   win32window.OnPointerDown(10.0, 10.0, kFlutterPointerDeviceKindMouse,
                             kDefaultPointerDeviceId, WM_LBUTTONDOWN);
   win32window.OnPointerUp(10.0, 10.0, kFlutterPointerDeviceKindMouse,
@@ -333,7 +338,7 @@ TEST(FlutterWindowTest, OnPointerStarSendsDeviceType) {
 
   // Touch
   win32window.OnPointerMove(10.0, 10.0, kFlutterPointerDeviceKindTouch,
-                            kDefaultPointerDeviceId);
+                            kDefaultPointerDeviceId, 0);
   win32window.OnPointerDown(10.0, 10.0, kFlutterPointerDeviceKindTouch,
                             kDefaultPointerDeviceId, WM_LBUTTONDOWN);
   win32window.OnPointerUp(10.0, 10.0, kFlutterPointerDeviceKindTouch,
@@ -343,7 +348,7 @@ TEST(FlutterWindowTest, OnPointerStarSendsDeviceType) {
 
   // Pen
   win32window.OnPointerMove(10.0, 10.0, kFlutterPointerDeviceKindStylus,
-                            kDefaultPointerDeviceId);
+                            kDefaultPointerDeviceId, 0);
   win32window.OnPointerDown(10.0, 10.0, kFlutterPointerDeviceKindStylus,
                             kDefaultPointerDeviceId, WM_LBUTTONDOWN);
   win32window.OnPointerUp(10.0, 10.0, kFlutterPointerDeviceKindStylus,

shell/platform/windows/flutter_windows_view.cc

@@ -166,7 +166,9 @@ void FlutterWindowsView::OnWindowRepaint() {
 void FlutterWindowsView::OnPointerMove(double x,
                                        double y,
                                        FlutterPointerDeviceKind device_kind,
-                                       int32_t device_id) {
+                                       int32_t device_id,
+                                       int modifiers_state) {
+  keyboard_key_handler_->SyncModifiersIfNeeded(modifiers_state);
   SendPointerMove(x, y, GetOrCreatePointerState(device_kind, device_id));
 }
 
@@ -285,8 +287,6 @@ void FlutterWindowsView::OnResetImeComposing() {
 
 void FlutterWindowsView::InitializeKeyboard() {
   auto internal_plugin_messenger = internal_plugin_registrar_->messenger();
-  // TODO(cbracken): This can be inlined into KeyboardKeyEmedderHandler once
-  // UWP code is removed. https://github.com/flutter/flutter/issues/102172.
   KeyboardKeyEmbedderHandler::GetKeyStateHandler get_key_state = GetKeyState;
   KeyboardKeyEmbedderHandler::MapVirtualKeyToScanCode map_vk_to_scan =
       [](UINT virtual_key, bool extended) {

shell/platform/windows/flutter_windows_view.h

@@ -112,7 +112,8 @@ class FlutterWindowsView : public WindowBindingHandlerDelegate,
   void OnPointerMove(double x,
                      double y,
                      FlutterPointerDeviceKind device_kind,
-                     int32_t device_id) override;
+                     int32_t device_id,
+                     int modifiers_state) override;
 
   // |WindowBindingHandlerDelegate|
   void OnPointerDown(double x,

shell/platform/windows/keyboard_handler_base.h

@@ -29,6 +29,10 @@ class KeyboardHandlerBase {
                             bool extended,
                             bool was_down,
                             KeyEventCallback callback) = 0;
+
+  // If needed, synthesize modifier keys events by comparing the
+  // given modifiers state to the known pressing state..
+  virtual void SyncModifiersIfNeeded(int modifiers_state) = 0;
 };
 
 }  // namespace flutter

shell/platform/windows/keyboard_key_channel_handler.cc

@@ -109,6 +109,10 @@ KeyboardKeyChannelHandler::KeyboardKeyChannelHandler(
 
 KeyboardKeyChannelHandler::~KeyboardKeyChannelHandler() = default;
 
+void KeyboardKeyChannelHandler::SyncModifiersIfNeeded(int modifiers_state) {
+  // Do nothing
+}
+
 void KeyboardKeyChannelHandler::KeyboardHook(
     int key,
     int scancode,

shell/platform/windows/keyboard_key_channel_handler.h

@@ -38,6 +38,8 @@ class KeyboardKeyChannelHandler
                     bool was_down,
                     std::function<void(bool)> callback);
 
+  void SyncModifiersIfNeeded(int modifiers_state);
+
  private:
   // The Flutter system channel for key event messages.
   std::unique_ptr<flutter::BasicMessageChannel<rapidjson::Document>> channel_;

shell/platform/windows/keyboard_key_embedder_handler.cc

@@ -187,7 +187,7 @@ void KeyboardKeyEmbedderHandler::KeyboardHookImpl(
   const bool is_event_down = action == WM_KEYDOWN || action == WM_SYSKEYDOWN;
 
   bool event_key_can_be_repeat = true;
-  UpdateLastSeenCritialKey(key, physical_key, sequence_logical_key);
+  UpdateLastSeenCriticalKey(key, physical_key, sequence_logical_key);
   // Synchronize the toggled states of critical keys (such as whether CapsLocks
   // is enabled). Toggled states can only be changed upon a down event, so if
   // the recorded toggled state does not match the true state, this function
@@ -197,16 +197,17 @@ void KeyboardKeyEmbedderHandler::KeyboardHookImpl(
   // After this function, all critical keys will have their toggled state
   // updated to the true state, while the critical keys whose toggled state have
   // been changed will be pressed regardless of their true pressed state.
-  // Updating the pressed state will be done by SynchronizeCritialPressedStates.
-  SynchronizeCritialToggledStates(key, is_event_down, &event_key_can_be_repeat);
+  // Updating the pressed state will be done by SynchronizeCriticalPressedStates.
+  SynchronizeCriticalToggledStates(key, is_event_down,
+                                   &event_key_can_be_repeat);
   // Synchronize the pressed states of critical keys (such as whether CapsLocks
   // is pressed).
   //
   // After this function, all critical keys except for the target key will have
   // their toggled state and pressed state matched with their true states. The
   // target key's pressed state will be updated immediately after this.
-  SynchronizeCritialPressedStates(key, physical_key, is_event_down,
-                                  event_key_can_be_repeat);
+  SynchronizeCriticalPressedStates(key, physical_key, is_event_down,
+                                   event_key_can_be_repeat);
 
   // Reassess the last logical record in case pressingRecords_ was modified
   // by the above synchronization methods.
@@ -317,8 +318,8 @@ void KeyboardKeyEmbedderHandler::KeyboardHookImpl(
   // received despite that GetKeyState says that CapsLock is not pressed. In
   // such case, post-event synchronization will synthesize a CapsLock up event
   // after the main event.
-  SynchronizeCritialPressedStates(key, physical_key, is_event_down,
-                                  event_key_can_be_repeat);
+  SynchronizeCriticalPressedStates(key, physical_key, is_event_down,
+                                   event_key_can_be_repeat);
 }
 
 void KeyboardKeyEmbedderHandler::KeyboardHook(
@@ -349,7 +350,7 @@ void KeyboardKeyEmbedderHandler::KeyboardHook(
   }
 }
 
-void KeyboardKeyEmbedderHandler::UpdateLastSeenCritialKey(
+void KeyboardKeyEmbedderHandler::UpdateLastSeenCriticalKey(
     int virtual_key,
     uint64_t physical_key,
     uint64_t logical_key) {
@@ -360,7 +361,7 @@ void KeyboardKeyEmbedderHandler::UpdateLastSeenCritialKey(
   }
 }
 
-void KeyboardKeyEmbedderHandler::SynchronizeCritialToggledStates(
+void KeyboardKeyEmbedderHandler::SynchronizeCriticalToggledStates(
     int event_virtual_key,
     bool is_event_down,
     bool* event_key_can_be_repeat) {
@@ -415,7 +416,7 @@ void KeyboardKeyEmbedderHandler::SynchronizeCritialToggledStates(
   }
 }
 
-void KeyboardKeyEmbedderHandler::SynchronizeCritialPressedStates(
+void KeyboardKeyEmbedderHandler::SynchronizeCriticalPressedStates(
     int event_virtual_key,
     int event_physical_key,
     bool is_event_down,
@@ -492,6 +493,72 @@ void KeyboardKeyEmbedderHandler::SynchronizeCritialPressedStates(
   }
 }
 
+void KeyboardKeyEmbedderHandler::SyncModifiersIfNeeded(int modifiers_state) {
+  // TODO(bleroux): consider exposing these constants in flutter_key_map.g.cc?
+  const uint64_t physical_shift_left =
+      windowsToPhysicalMap_.at(kScanCodeShiftLeft);
+  const uint64_t physical_shift_right =
+      windowsToPhysicalMap_.at(kScanCodeShiftRight);
+  const uint64_t logical_shift_left =
+      windowsToLogicalMap_.at(kKeyCodeShiftLeft);
+  const uint64_t physical_control_left =
+      windowsToPhysicalMap_.at(kScanCodeControlLeft);
+  const uint64_t physical_control_right =
+      windowsToPhysicalMap_.at(kScanCodeControlRight);
+  const uint64_t logical_control_left =
+      windowsToLogicalMap_.at(kKeyCodeControlLeft);
+
+  bool shift_pressed = (modifiers_state & kShift) != 0;
+  SynthesizeIfNeeded(physical_shift_left, physical_shift_right,
+                     logical_shift_left, shift_pressed);
+  bool control_pressed = (modifiers_state & kControl) != 0;
+  SynthesizeIfNeeded(physical_control_left, physical_control_right,
+                     logical_control_left, control_pressed);
+}
+
+void KeyboardKeyEmbedderHandler::SynthesizeIfNeeded(uint64_t physical_left,
+                                                    uint64_t physical_right,
+                                                    uint64_t logical_left,
+                                                    bool is_pressed) {
+  auto pressing_record_iter_left = pressingRecords_.find(physical_left);
+  bool left_pressed = pressing_record_iter_left != pressingRecords_.end();
+  auto pressing_record_iter_right = pressingRecords_.find(physical_right);
+  bool right_pressed = pressing_record_iter_right != pressingRecords_.end();
+  bool already_pressed = left_pressed || right_pressed;
+  bool synthesize_down = is_pressed && !already_pressed;
+  bool synthesize_up = !is_pressed && already_pressed;
+
+  if (synthesize_down) {
+    SendSynthesizeDownEvent(physical_left, logical_left);
+  }
+
+  if (synthesize_up && left_pressed) {
+    uint64_t known_logical = pressing_record_iter_left->second;
+    SendSynthesizeUpEvent(physical_left, known_logical);
+  }
+
+  if (synthesize_up && right_pressed) {
+    uint64_t known_logical = pressing_record_iter_right->second;
+    SendSynthesizeUpEvent(physical_right, known_logical);
+  }
+}
+
+void KeyboardKeyEmbedderHandler::SendSynthesizeDownEvent(uint64_t physical,
+                                                         uint64_t logical) {
+  SendEvent(
+      SynthesizeSimpleEvent(kFlutterKeyEventTypeDown, physical, logical, ""),
+      nullptr, nullptr);
+  pressingRecords_[physical] = logical;
+};
+
+void KeyboardKeyEmbedderHandler::SendSynthesizeUpEvent(uint64_t physical,
+                                                       uint64_t logical) {
+  SendEvent(
+      SynthesizeSimpleEvent(kFlutterKeyEventTypeUp, physical, logical, ""),
+      nullptr, nullptr);
+  pressingRecords_.erase(physical);
+};
+
 void KeyboardKeyEmbedderHandler::HandleResponse(bool handled, void* user_data) {
   PendingResponse* pending = reinterpret_cast<PendingResponse*>(user_data);
   auto callback = std::move(pending->callback);
@@ -516,8 +583,6 @@ void KeyboardKeyEmbedderHandler::InitCriticalKeys(
     };
   };
 
-  // TODO(dkwingsmt): Consider adding more critical keys here.
-  // https://github.com/flutter/flutter/issues/76736
   critical_keys_.emplace(VK_LSHIFT,
                          createCheckedKey(VK_LSHIFT, false, true, false));
   critical_keys_.emplace(VK_RSHIFT,
@@ -532,7 +597,6 @@ void KeyboardKeyEmbedderHandler::InitCriticalKeys(
                          createCheckedKey(VK_RMENU, true, true, false));
   critical_keys_.emplace(VK_LWIN, createCheckedKey(VK_LWIN, true, true, false));
   critical_keys_.emplace(VK_RWIN, createCheckedKey(VK_RWIN, true, true, false));
-
   critical_keys_.emplace(VK_CAPITAL,
                          createCheckedKey(VK_CAPITAL, false, true, true));
   critical_keys_.emplace(VK_SCROLL,

shell/platform/windows/keyboard_key_embedder_handler.h

@@ -70,6 +70,8 @@ class KeyboardKeyEmbedderHandler
                     bool was_down,
                     std::function<void(bool)> callback) override;
 
+  void SyncModifiersIfNeeded(int modifiers_state) override;
+
  private:
   struct PendingResponse {
     std::function<void(bool, uint64_t)> callback;
@@ -109,27 +111,40 @@ class KeyboardKeyEmbedderHandler
   // Assign |critical_keys_| with basic information.
   void InitCriticalKeys(MapVirtualKeyToScanCode map_virtual_key_to_scan_code);
   // Update |critical_keys_| with last seen logical and physical key.
-  void UpdateLastSeenCritialKey(int virtual_key,
-                                uint64_t physical_key,
-                                uint64_t logical_key);
+  void UpdateLastSeenCriticalKey(int virtual_key,
+                                 uint64_t physical_key,
+                                 uint64_t logical_key);
   // Check each key's state from |get_key_state_| and synthesize events
   // if their toggling states have been desynchronized.
-  void SynchronizeCritialToggledStates(int event_virtual_key,
-                                       bool is_event_down,
-                                       bool* event_key_can_be_repeat);
+  void SynchronizeCriticalToggledStates(int event_virtual_key,
+                                        bool is_event_down,
+                                        bool* event_key_can_be_repeat);
   // Check each key's state from |get_key_state_| and synthesize events
   // if their pressing states have been desynchronized.
-  void SynchronizeCritialPressedStates(int event_virtual_key,
-                                       int event_physical_key,
-                                       bool is_event_down,
-                                       bool event_key_can_be_repeat);
+  void SynchronizeCriticalPressedStates(int event_virtual_key,
+                                        int event_physical_key,
+                                        bool is_event_down,
+                                        bool event_key_can_be_repeat);
 
   // Wraps perform_send_event_ with state tracking. Use this instead of
   // |perform_send_event_| to send events to the framework.
   void SendEvent(const FlutterKeyEvent& event,
                  FlutterKeyEventCallback callback,
                  void* user_data);
 
+  // Send a synthesized down event and update pressing records.
+  void SendSynthesizeDownEvent(uint64_t physical, uint64_t logical);
+
+  // Send a synthesized up event and update pressing records.
+  void SendSynthesizeUpEvent(uint64_t physical, uint64_t logical);
+
+  // Send a synthesized up or down event depending on the current pressing
+  // state.
+  void SynthesizeIfNeeded(uint64_t physical_left,
+                          uint64_t physical_right,
+                          uint64_t logical_left,
+                          bool is_pressed);
+
   std::function<void(const FlutterKeyEvent&, FlutterKeyEventCallback, void*)>
       perform_send_event_;
   GetKeyStateHandler get_key_state_;