diff --git a/ci/licenses_golden/licenses_flutter b/ci/licenses_golden/licenses_flutter index 5f784be07af1e..674c3c741454c 100644 --- a/ci/licenses_golden/licenses_flutter +++ b/ci/licenses_golden/licenses_flutter @@ -868,6 +868,9 @@ FILE: ../../../flutter/shell/platform/android/surface/android_surface_mock.cc FILE: ../../../flutter/shell/platform/android/surface/android_surface_mock.h FILE: ../../../flutter/shell/platform/android/vsync_waiter_android.cc FILE: ../../../flutter/shell/platform/android/vsync_waiter_android.h +FILE: ../../../flutter/shell/platform/common/cpp/accessibility_bridge.cc +FILE: ../../../flutter/shell/platform/common/cpp/accessibility_bridge.h +FILE: ../../../flutter/shell/platform/common/cpp/accessibility_bridge_unittests.cc FILE: ../../../flutter/shell/platform/common/cpp/client_wrapper/basic_message_channel_unittests.cc FILE: ../../../flutter/shell/platform/common/cpp/client_wrapper/binary_messenger_impl.h FILE: ../../../flutter/shell/platform/common/cpp/client_wrapper/byte_buffer_streams.h @@ -909,6 +912,9 @@ FILE: ../../../flutter/shell/platform/common/cpp/client_wrapper/texture_registra FILE: ../../../flutter/shell/platform/common/cpp/engine_switches.cc FILE: ../../../flutter/shell/platform/common/cpp/engine_switches.h FILE: ../../../flutter/shell/platform/common/cpp/engine_switches_unittests.cc +FILE: ../../../flutter/shell/platform/common/cpp/flutter_platform_node_delegate.cc +FILE: ../../../flutter/shell/platform/common/cpp/flutter_platform_node_delegate.h +FILE: ../../../flutter/shell/platform/common/cpp/flutter_platform_node_delegate_unittests.cc FILE: ../../../flutter/shell/platform/common/cpp/incoming_message_dispatcher.cc FILE: ../../../flutter/shell/platform/common/cpp/incoming_message_dispatcher.h FILE: ../../../flutter/shell/platform/common/cpp/json_message_codec.cc @@ -924,6 +930,8 @@ FILE: ../../../flutter/shell/platform/common/cpp/public/flutter_export.h FILE: ../../../flutter/shell/platform/common/cpp/public/flutter_messenger.h FILE: ../../../flutter/shell/platform/common/cpp/public/flutter_plugin_registrar.h FILE: ../../../flutter/shell/platform/common/cpp/public/flutter_texture_registrar.h +FILE: ../../../flutter/shell/platform/common/cpp/test_accessibility_bridge.cc +FILE: ../../../flutter/shell/platform/common/cpp/test_accessibility_bridge.h FILE: ../../../flutter/shell/platform/common/cpp/text_input_model.cc FILE: ../../../flutter/shell/platform/common/cpp/text_input_model.h FILE: ../../../flutter/shell/platform/common/cpp/text_input_model_unittests.cc diff --git a/shell/platform/common/cpp/BUILD.gn b/shell/platform/common/cpp/BUILD.gn index 36c9760c419c2..148c39ad91709 100644 --- a/shell/platform/common/cpp/BUILD.gn +++ b/shell/platform/common/cpp/BUILD.gn @@ -68,6 +68,26 @@ source_set("common_cpp_switches") { ] } +source_set("common_cpp_accessibility") { + public = [ + "accessibility_bridge.h", + "flutter_platform_node_delegate.h", + ] + + sources = [ + "accessibility_bridge.cc", + "flutter_platform_node_delegate.cc", + ] + + public_configs = + [ "//flutter/third_party/accessibility:accessibility_config" ] + + public_deps = [ + "//flutter/shell/platform/embedder:embedder_as_internal_library", + "//flutter/third_party/accessibility", + ] +} + source_set("common_cpp") { public = [ "incoming_message_dispatcher.h", @@ -156,6 +176,18 @@ if (enable_unittests) { "//flutter/testing", ] + # The accessibility bridge only supports MacOS for now. + if (is_mac) { + sources += [ + "accessibility_bridge_unittests.cc", + "flutter_platform_node_delegate_unittests.cc", + "test_accessibility_bridge.cc", + "test_accessibility_bridge.h", + ] + + deps += [ ":common_cpp_accessibility" ] + } + public_configs = [ "//flutter:config" ] } } diff --git a/shell/platform/common/cpp/accessibility_bridge.cc b/shell/platform/common/cpp/accessibility_bridge.cc new file mode 100644 index 0000000000000..20e42914bbfaf --- /dev/null +++ b/shell/platform/common/cpp/accessibility_bridge.cc @@ -0,0 +1,539 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "accessibility_bridge.h" + +#include +#include + +#include "flutter/third_party/accessibility/ax/ax_tree_update.h" +#include "flutter/third_party/accessibility/base/logging.h" + +namespace flutter { // namespace + +constexpr int kHasScrollingAction = + FlutterSemanticsAction::kFlutterSemanticsActionScrollLeft | + FlutterSemanticsAction::kFlutterSemanticsActionScrollRight | + FlutterSemanticsAction::kFlutterSemanticsActionScrollUp | + FlutterSemanticsAction::kFlutterSemanticsActionScrollDown; + +// AccessibilityBridge +AccessibilityBridge::AccessibilityBridge( + std::unique_ptr delegate) + : delegate_(std::move(delegate)) { + event_generator_.SetTree(&tree_); + tree_.AddObserver(static_cast(this)); +} + +AccessibilityBridge::~AccessibilityBridge() { + event_generator_.ReleaseTree(); + tree_.RemoveObserver(static_cast(this)); +} + +void AccessibilityBridge::AddFlutterSemanticsNodeUpdate( + const FlutterSemanticsNode* node) { + pending_semantics_node_updates_[node->id] = FromFlutterSemanticsNode(node); +} + +void AccessibilityBridge::AddFlutterSemanticsCustomActionUpdate( + const FlutterSemanticsCustomAction* action) { + pending_semantics_custom_action_updates_[action->id] = + FromFlutterSemanticsCustomAction(action); +} + +void AccessibilityBridge::CommitUpdates() { + ui::AXTreeUpdate update{.tree_data = tree_.data()}; + // Figure out update order, ui::AXTree only accepts update in tree order, + // where parent node must come before the child node in + // ui::AXTreeUpdate.nodes. We start with picking a random node and turn the + // entire subtree into a list. We pick another node from the remaining update, + // and keep doing so until the update map is empty. We then concatenate the + // lists in the reversed order, this guarantees parent updates always come + // before child updates. + std::vector> results; + while (!pending_semantics_node_updates_.empty()) { + auto begin = pending_semantics_node_updates_.begin(); + SemanticsNode target = begin->second; + std::vector sub_tree_list; + GetSubTreeList(target, sub_tree_list); + results.push_back(sub_tree_list); + pending_semantics_node_updates_.erase(begin); + } + + for (size_t i = results.size(); i > 0; i--) { + for (SemanticsNode node : results[i - 1]) { + ConvertFluterUpdate(node, update); + } + } + + tree_.Unserialize(update); + pending_semantics_node_updates_.clear(); + pending_semantics_custom_action_updates_.clear(); + + std::string error = tree_.error(); + if (!error.empty()) { + BASE_LOG() << "Failed to update ui::AXTree, error: " << error; + return; + } + // Handles accessibility events as the result of the semantics update. + for (const auto& targeted_event : event_generator_) { + auto event_target = + GetFlutterPlatformNodeDelegateFromID(targeted_event.node->id()); + if (event_target.expired()) { + continue; + } + + delegate_->OnAccessibilityEvent(targeted_event); + } + event_generator_.ClearEvents(); +} + +std::weak_ptr +AccessibilityBridge::GetFlutterPlatformNodeDelegateFromID( + AccessibilityNodeId id) const { + const auto iter = id_wrapper_map_.find(id); + if (iter != id_wrapper_map_.end()) { + return iter->second; + } + + return std::weak_ptr(); +} + +const ui::AXTreeData& AccessibilityBridge::GetAXTreeData() const { + return tree_.data(); +} + +void AccessibilityBridge::OnNodeWillBeDeleted(ui::AXTree* tree, + ui::AXNode* node) {} + +void AccessibilityBridge::OnSubtreeWillBeDeleted(ui::AXTree* tree, + ui::AXNode* node) {} + +void AccessibilityBridge::OnNodeReparented(ui::AXTree* tree, ui::AXNode* node) { +} + +void AccessibilityBridge::OnRoleChanged(ui::AXTree* tree, + ui::AXNode* node, + ax::mojom::Role old_role, + ax::mojom::Role new_role) {} + +void AccessibilityBridge::OnNodeCreated(ui::AXTree* tree, ui::AXNode* node) { + BASE_DCHECK(node); + id_wrapper_map_[node->id()] = delegate_->CreateFlutterPlatformNodeDelegate(); + id_wrapper_map_[node->id()]->Init( + std::static_pointer_cast( + shared_from_this()), + node); +} + +void AccessibilityBridge::OnNodeDeleted(ui::AXTree* tree, + AccessibilityNodeId node_id) { + BASE_DCHECK(node_id != ui::AXNode::kInvalidAXID); + if (id_wrapper_map_.find(node_id) != id_wrapper_map_.end()) { + id_wrapper_map_.erase(node_id); + } +} + +void AccessibilityBridge::OnAtomicUpdateFinished( + ui::AXTree* tree, + bool root_changed, + const std::vector& changes) { + // The Flutter semantics update does not include child->parent relationship + // We have to update the relative bound offset container id here in order + // to calculate the screen bound correctly. + for (const auto& change : changes) { + ui::AXNode* node = change.node; + const ui::AXNodeData& data = node->data(); + AccessibilityNodeId offset_container_id = -1; + if (node->parent()) { + offset_container_id = node->parent()->id(); + } + node->SetLocation(offset_container_id, data.relative_bounds.bounds, + data.relative_bounds.transform.get()); + } +} + +// Private method. +void AccessibilityBridge::GetSubTreeList(SemanticsNode target, + std::vector& result) { + result.push_back(target); + for (int32_t child : target.children_in_traversal_order) { + auto iter = pending_semantics_node_updates_.find(child); + if (iter != pending_semantics_node_updates_.end()) { + SemanticsNode node = iter->second; + GetSubTreeList(node, result); + pending_semantics_node_updates_.erase(iter); + } + } +} + +void AccessibilityBridge::ConvertFluterUpdate(const SemanticsNode& node, + ui::AXTreeUpdate& tree_update) { + ui::AXNodeData node_data; + node_data.id = node.id; + SetRoleFromFlutterUpdate(node_data, node); + SetStateFromFlutterUpdate(node_data, node); + SetActionsFromFlutterUpdate(node_data, node); + SetBooleanAttributesFromFlutterUpdate(node_data, node); + SetIntAttributesFromFlutterUpdate(node_data, node); + SetIntListAttributesFromFlutterUpdate(node_data, node); + SetStringListAttributesFromFlutterUpdate(node_data, node); + SetNameFromFlutterUpdate(node_data, node); + SetValueFromFlutterUpdate(node_data, node); + node_data.relative_bounds.bounds.SetRect(node.rect.left, node.rect.top, + node.rect.right - node.rect.left, + node.rect.bottom - node.rect.top); + node_data.relative_bounds.transform = std::make_unique( + node.transform.scaleX, node.transform.skewX, node.transform.transX, 0, + node.transform.skewY, node.transform.scaleY, node.transform.transY, 0, + node.transform.pers0, node.transform.pers1, node.transform.pers2, 0, 0, 0, + 0, 0); + for (auto child : node.children_in_traversal_order) { + node_data.child_ids.push_back(child); + } + SetTreeData(node, tree_update); + tree_update.nodes.push_back(node_data); +} + +void AccessibilityBridge::SetRoleFromFlutterUpdate(ui::AXNodeData& node_data, + const SemanticsNode& node) { + FlutterSemanticsFlag flags = node.flags; + if (flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsButton) { + node_data.role = ax::mojom::Role::kButton; + return; + } + if (flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsTextField && + !(flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsReadOnly)) { + node_data.role = ax::mojom::Role::kTextField; + return; + } + if (flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsHeader) { + node_data.role = ax::mojom::Role::kHeader; + return; + } + if (flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsImage) { + node_data.role = ax::mojom::Role::kImage; + return; + } + if (flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsLink) { + node_data.role = ax::mojom::Role::kLink; + return; + } + + if (flags & kFlutterSemanticsFlagIsInMutuallyExclusiveGroup && + flags & kFlutterSemanticsFlagHasCheckedState) { + node_data.role = ax::mojom::Role::kRadioButton; + return; + } + if (flags & kFlutterSemanticsFlagHasCheckedState) { + node_data.role = ax::mojom::Role::kCheckBox; + return; + } + // If the state cannot be derived from the flutter flags, we fallback to group + // or static text. + if (node.children_in_traversal_order.size() == 0) { + node_data.role = ax::mojom::Role::kStaticText; + } else { + node_data.role = ax::mojom::Role::kGroup; + } +} + +void AccessibilityBridge::SetStateFromFlutterUpdate(ui::AXNodeData& node_data, + const SemanticsNode& node) { + FlutterSemanticsFlag flags = node.flags; + FlutterSemanticsAction actions = node.actions; + if (flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsTextField && + (flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsReadOnly) == 0) { + node_data.AddState(ax::mojom::State::kEditable); + } + if (node_data.role == ax::mojom::Role::kStaticText && + (actions & kHasScrollingAction) == 0 && node.value.empty() && + node.label.empty() && node.hint.empty()) { + node_data.AddState(ax::mojom::State::kIgnored); + } else { + // kFlutterSemanticsFlagIsFocusable means a keyboard focusable, it is + // different from semantics focusable. + // TODO(chunhtai): figure out whether something is not semantics focusable. + node_data.AddState(ax::mojom::State::kFocusable); + } +} + +void AccessibilityBridge::SetActionsFromFlutterUpdate( + ui::AXNodeData& node_data, + const SemanticsNode& node) { + FlutterSemanticsAction actions = node.actions; + if (actions & FlutterSemanticsAction::kFlutterSemanticsActionTap) { + node_data.AddAction(ax::mojom::Action::kDoDefault); + } + if (actions & FlutterSemanticsAction::kFlutterSemanticsActionScrollLeft) { + node_data.AddAction(ax::mojom::Action::kScrollLeft); + } + if (actions & FlutterSemanticsAction::kFlutterSemanticsActionScrollRight) { + node_data.AddAction(ax::mojom::Action::kScrollRight); + } + if (actions & FlutterSemanticsAction::kFlutterSemanticsActionScrollUp) { + node_data.AddAction(ax::mojom::Action::kScrollUp); + } + if (actions & FlutterSemanticsAction::kFlutterSemanticsActionScrollDown) { + node_data.AddAction(ax::mojom::Action::kScrollDown); + } + if (actions & FlutterSemanticsAction::kFlutterSemanticsActionIncrease) { + node_data.AddAction(ax::mojom::Action::kIncrement); + } + if (actions & FlutterSemanticsAction::kFlutterSemanticsActionDecrease) { + node_data.AddAction(ax::mojom::Action::kDecrement); + } + // Every node has show on screen action. + node_data.AddAction(ax::mojom::Action::kScrollToMakeVisible); + + if (actions & FlutterSemanticsAction::kFlutterSemanticsActionSetSelection) { + node_data.AddAction(ax::mojom::Action::kSetSelection); + } + if (actions & FlutterSemanticsAction:: + kFlutterSemanticsActionDidGainAccessibilityFocus) { + node_data.AddAction(ax::mojom::Action::kSetAccessibilityFocus); + } + if (actions & FlutterSemanticsAction:: + kFlutterSemanticsActionDidLoseAccessibilityFocus) { + node_data.AddAction(ax::mojom::Action::kClearAccessibilityFocus); + } + if (actions & FlutterSemanticsAction::kFlutterSemanticsActionCustomAction) { + node_data.AddAction(ax::mojom::Action::kCustomAction); + } +} + +void AccessibilityBridge::SetBooleanAttributesFromFlutterUpdate( + ui::AXNodeData& node_data, + const SemanticsNode& node) { + FlutterSemanticsAction actions = node.actions; + FlutterSemanticsFlag flags = node.flags; + node_data.AddBoolAttribute(ax::mojom::BoolAttribute::kScrollable, + actions & kHasScrollingAction); + node_data.AddBoolAttribute( + ax::mojom::BoolAttribute::kClickable, + actions & FlutterSemanticsAction::kFlutterSemanticsActionTap); + // TODO(chunhtai): figure out if there is a node that does not clip overflow. + node_data.AddBoolAttribute(ax::mojom::BoolAttribute::kClipsChildren, + node.children_in_traversal_order.size() != 0); + node_data.AddBoolAttribute( + ax::mojom::BoolAttribute::kSelected, + flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsSelected); + node_data.AddBoolAttribute( + ax::mojom::BoolAttribute::kEditableRoot, + flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsTextField && + (flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsReadOnly) > 0); +} + +void AccessibilityBridge::SetIntAttributesFromFlutterUpdate( + ui::AXNodeData& node_data, + const SemanticsNode& node) { + FlutterSemanticsFlag flags = node.flags; + node_data.AddIntAttribute(ax::mojom::IntAttribute::kTextDirection, + node.text_direction); + + int sel_start = node.text_selection_base; + int sel_end = node.text_selection_extent; + if (flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsTextField && + (flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsReadOnly) == 0 && + !node.value.empty()) { + // By default the text field selection should be at the end. + sel_start = sel_start == -1 ? node.value.length() : sel_start; + sel_end = sel_end == -1 ? node.value.length() : sel_end; + } + node_data.AddIntAttribute(ax::mojom::IntAttribute::kTextSelStart, sel_start); + node_data.AddIntAttribute(ax::mojom::IntAttribute::kTextSelEnd, sel_end); + + if (node_data.role == ax::mojom::Role::kRadioButton) { + node_data.AddIntAttribute( + ax::mojom::IntAttribute::kCheckedState, + static_cast( + flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsChecked + ? ax::mojom::CheckedState::kTrue + : ax::mojom::CheckedState::kFalse)); + } +} + +void AccessibilityBridge::SetIntListAttributesFromFlutterUpdate( + ui::AXNodeData& node_data, + const SemanticsNode& node) { + FlutterSemanticsAction actions = node.actions; + if (actions & FlutterSemanticsAction::kFlutterSemanticsActionCustomAction) { + std::vector custom_action_ids; + for (size_t i = 0; i < node.custom_accessibility_actions.size(); i++) { + custom_action_ids.push_back(node.custom_accessibility_actions[i]); + } + node_data.AddIntListAttribute(ax::mojom::IntListAttribute::kCustomActionIds, + custom_action_ids); + } +} + +void AccessibilityBridge::SetStringListAttributesFromFlutterUpdate( + ui::AXNodeData& node_data, + const SemanticsNode& node) { + FlutterSemanticsAction actions = node.actions; + if (actions & FlutterSemanticsAction::kFlutterSemanticsActionCustomAction) { + std::vector custom_action_description; + for (size_t i = 0; i < node.custom_accessibility_actions.size(); i++) { + auto iter = pending_semantics_custom_action_updates_.find( + node.custom_accessibility_actions[i]); + BASE_DCHECK(iter != pending_semantics_custom_action_updates_.end()); + custom_action_description.push_back(iter->second.label); + } + node_data.AddStringListAttribute( + ax::mojom::StringListAttribute::kCustomActionDescriptions, + custom_action_description); + } +} + +void AccessibilityBridge::SetNameFromFlutterUpdate(ui::AXNodeData& node_data, + const SemanticsNode& node) { + node_data.SetName(node.label); +} + +void AccessibilityBridge::SetValueFromFlutterUpdate(ui::AXNodeData& node_data, + const SemanticsNode& node) { + node_data.SetValue(node.value); +} + +void AccessibilityBridge::SetTreeData(const SemanticsNode& node, + ui::AXTreeUpdate& tree_update) { + FlutterSemanticsFlag flags = node.flags; + // Set selection if: + // 1. this text field has a valid selection + // 2. this text field doesn't have a valid selection but had selection stored + // in the tree. + if (flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsTextField) { + if (node.text_selection_base != -1) { + tree_update.tree_data.sel_anchor_object_id = node.id; + tree_update.tree_data.sel_anchor_offset = node.text_selection_base; + tree_update.tree_data.sel_focus_object_id = node.id; + tree_update.tree_data.sel_focus_offset = node.text_selection_extent; + tree_update.has_tree_data = true; + } else if (tree_update.tree_data.sel_anchor_object_id == node.id) { + tree_update.tree_data.sel_anchor_object_id = ui::AXNode::kInvalidAXID; + tree_update.tree_data.sel_anchor_offset = -1; + tree_update.tree_data.sel_focus_object_id = ui::AXNode::kInvalidAXID; + tree_update.tree_data.sel_focus_offset = -1; + tree_update.has_tree_data = true; + } + } + + if (flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsFocused && + tree_update.tree_data.focus_id != node.id) { + tree_update.tree_data.focus_id = node.id; + tree_update.has_tree_data = true; + } else if ((flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsFocused) == + 0 && + tree_update.tree_data.focus_id == node.id) { + tree_update.tree_data.focus_id = ui::AXNode::kInvalidAXID; + tree_update.has_tree_data = true; + } +} + +AccessibilityBridge::SemanticsNode +AccessibilityBridge::FromFlutterSemanticsNode( + const FlutterSemanticsNode* flutter_node) { + SemanticsNode result; + result.id = flutter_node->id; + result.flags = flutter_node->flags; + result.actions = flutter_node->actions; + result.text_selection_base = flutter_node->text_selection_base; + result.text_selection_extent = flutter_node->text_selection_extent; + result.scroll_child_count = flutter_node->scroll_child_count; + result.scroll_index = flutter_node->scroll_index; + result.scroll_position = flutter_node->scroll_position; + result.scroll_extent_max = flutter_node->scroll_extent_max; + result.scroll_extent_min = flutter_node->scroll_extent_min; + result.elevation = flutter_node->elevation; + result.thickness = flutter_node->thickness; + if (flutter_node->label) { + result.label = std::string(flutter_node->label); + } + if (flutter_node->hint) { + result.hint = std::string(flutter_node->hint); + } + if (flutter_node->value) { + result.value = std::string(flutter_node->value); + } + if (flutter_node->increased_value) { + result.increased_value = std::string(flutter_node->increased_value); + } + if (flutter_node->decreased_value) { + result.decreased_value = std::string(flutter_node->decreased_value); + } + result.text_direction = flutter_node->text_direction; + result.rect = flutter_node->rect; + result.transform = flutter_node->transform; + if (flutter_node->child_count > 0) { + result.children_in_traversal_order = std::vector( + flutter_node->children_in_traversal_order, + flutter_node->children_in_traversal_order + flutter_node->child_count); + } + if (flutter_node->custom_accessibility_actions_count > 0) { + result.custom_accessibility_actions = std::vector( + flutter_node->custom_accessibility_actions, + flutter_node->custom_accessibility_actions + + flutter_node->custom_accessibility_actions_count); + } + return result; +} + +AccessibilityBridge::SemanticsCustomAction +AccessibilityBridge::FromFlutterSemanticsCustomAction( + const FlutterSemanticsCustomAction* flutter_custom_action) { + SemanticsCustomAction result; + result.id = flutter_custom_action->id; + result.override_action = flutter_custom_action->override_action; + if (flutter_custom_action->label) { + result.label = std::string(flutter_custom_action->label); + } + if (flutter_custom_action->hint) { + result.hint = std::string(flutter_custom_action->hint); + } + return result; +} + +void AccessibilityBridge::SetLastFocusedId(AccessibilityNodeId node_id) { + if (last_focused_id_ != node_id) { + auto last_focused_child = + GetFlutterPlatformNodeDelegateFromID(last_focused_id_); + if (!last_focused_child.expired()) { + delegate_->DispatchAccessibilityAction( + last_focused_id_, + FlutterSemanticsAction:: + kFlutterSemanticsActionDidLoseAccessibilityFocus, + {}); + } + last_focused_id_ = node_id; + } +} + +AccessibilityNodeId AccessibilityBridge::GetLastFocusedId() { + return last_focused_id_; +} + +gfx::NativeViewAccessible AccessibilityBridge::GetNativeAccessibleFromId( + AccessibilityNodeId id) { + auto platform_node_delegate = GetFlutterPlatformNodeDelegateFromID(id).lock(); + if (!platform_node_delegate) { + return nullptr; + } + return platform_node_delegate->GetNativeViewAccessible(); +} + +gfx::RectF AccessibilityBridge::RelativeToGlobalBounds(const ui::AXNode* node, + bool& offscreen, + bool clip_bounds) { + return tree_.RelativeToTreeBounds(node, gfx::RectF(), &offscreen, + clip_bounds); +} + +void AccessibilityBridge::DispatchAccessibilityAction( + AccessibilityNodeId target, + FlutterSemanticsAction action, + std::vector data) { + delegate_->DispatchAccessibilityAction(target, action, data); +} + +} // namespace flutter diff --git a/shell/platform/common/cpp/accessibility_bridge.h b/shell/platform/common/cpp/accessibility_bridge.h new file mode 100644 index 0000000000000..f2f2bfd381bc6 --- /dev/null +++ b/shell/platform/common/cpp/accessibility_bridge.h @@ -0,0 +1,283 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef FLUTTER_SHELL_PLATFORM_COMMON_CPP_ACCESSIBILITY_BRIDGE_H_ +#define FLUTTER_SHELL_PLATFORM_COMMON_CPP_ACCESSIBILITY_BRIDGE_H_ + +#include + +#include "flutter/shell/platform/embedder/embedder.h" + +#include "flutter/third_party/accessibility/ax/ax_event_generator.h" +#include "flutter/third_party/accessibility/ax/ax_tree.h" +#include "flutter/third_party/accessibility/ax/ax_tree_observer.h" +#include "flutter/third_party/accessibility/ax/platform/ax_platform_node_delegate.h" + +#include "flutter_platform_node_delegate.h" + +namespace flutter { + +//------------------------------------------------------------------------------ +/// Use this class to maintain an accessibility tree. This class consumes +/// semantics updates from the embedder API and produces an accessibility tree +/// in the native format. +/// +/// The bridge creates an AXTree to hold the semantics data that comes from +/// Flutter semantics updates. The tree holds AXNode[s] which contain the +/// semantics information for semantics node. The AXTree ressemble the Flutter +/// semantics tree in the Flutter framework. The bridge also uses +/// FlutterPlatformNodeDelegate to wrap each AXNode in order to provide +/// an accessibility tree in the native format. +/// +/// This class takes in a AccessibilityBridgeDelegate instance and is in charge +/// of its lifecycle. The delegate are used to handle the accessibility events +/// and actions. +/// +/// To use this class, you must provide your own implementation of +/// FlutterPlatformNodeDelegate and AccessibilityBridgeDelegate. +class AccessibilityBridge + : public std::enable_shared_from_this, + public FlutterPlatformNodeDelegate::OwnerBridge, + private ui::AXTreeObserver { + public: + //----------------------------------------------------------------------------- + /// Delegate to handle requests from the accessibility bridge. The requests + /// include sending accessibility event to native accessibility system, + /// routing accessibility action to the Flutter framework, and creating + /// platform specific FlutterPlatformNodeDelegate. + /// + /// The accessibility events are generated when accessibility tree changes. + /// These events must be sent to the native accessibility system through + /// the native API for the system to pick up the changes + /// (e.g. NSAccessibilityPostNotification in MacOS). + /// + /// The accessibility actions are generated by the native accessibility system + /// when users interacted with the assistive technologies. Those actions + /// needed to be sent to the Flutter framework. + /// + /// Each platform needs to implement the FlutterPlatformNodeDelegate and + /// returns its platform specific instance of FlutterPlatformNodeDelegate + /// in this delegate. + class AccessibilityBridgeDelegate { + public: + virtual ~AccessibilityBridgeDelegate() = default; + //--------------------------------------------------------------------------- + /// @brief Handle accessibility events generated due to accessibility + /// tree changes. These events are generated in accessibility + /// bridge and needed to be sent to native accessibility system. + /// See ui::AXEventGenerator::Event for possible events. + /// + /// @param[in] targeted_event The object that contains both the + /// generated event and the event target. + virtual void OnAccessibilityEvent( + ui::AXEventGenerator::TargetedEvent targeted_event) = 0; + + //--------------------------------------------------------------------------- + /// @brief Dispatch accessibility action back to the Flutter framework. + /// These actions are generated in the native accessibility + /// system when users interact with the assistive technologies. + /// For example, a + /// FlutterSemanticsAction::kFlutterSemanticsActionTap is + /// fired when user click or touch the screen. + /// + /// @param[in] target The semantics node id of the action + /// target. + /// @param[in] action The generated flutter semantics action. + /// @param[in] data Additional data associated with the + /// action. + virtual void DispatchAccessibilityAction(AccessibilityNodeId target, + FlutterSemanticsAction action, + std::vector data) = 0; + + //--------------------------------------------------------------------------- + /// @brief Creates a platform specific FlutterPlatformNodeDelegate. + /// Ownership passes to the caller. This method will be called + /// by accessibility bridge whenever a new AXNode is created in + /// AXTree. Each platform needs to implement this method in + /// order to inject its subclass into the accessibility bridge. + virtual std::unique_ptr + CreateFlutterPlatformNodeDelegate() = 0; + }; + //----------------------------------------------------------------------------- + /// @brief Creates a new instance of a accessibility bridge. + /// + /// @param[in] user_data A custom pointer to the data of your + /// choice. This pointer can be retrieve later + /// through GetUserData(). + AccessibilityBridge(std::unique_ptr delegate); + ~AccessibilityBridge(); + + //------------------------------------------------------------------------------ + /// @brief Adds a semantics node update to the pending semantics update. + /// Calling this method alone will NOT update the semantics tree. + /// To flush the pending updates, call the CommitUpdates(). + /// + /// @param[in] node A pointer to the semantics node update. + void AddFlutterSemanticsNodeUpdate(const FlutterSemanticsNode* node); + + //------------------------------------------------------------------------------ + /// @brief Adds a custom semantics action update to the pending semantics + /// update. Calling this method alone will NOT update the + /// semantics tree. To flush the pending updates, call the + /// CommitUpdates(). + /// + /// @param[in] action A pointer to the custom semantics action + /// update. + void AddFlutterSemanticsCustomActionUpdate( + const FlutterSemanticsCustomAction* action); + + //------------------------------------------------------------------------------ + /// @brief Flushes the pending updates and applies them to this + /// accessibility bridge. Calling this with no pending updates + /// does nothing, and callers should call this method at the end + /// of an atomic batch to avoid leaving the tree in a unstable + /// state. For example if a node reparents from A to B, callers + /// should only call this method when both removal from A and + /// addition to B are in the pending updates. + void CommitUpdates(); + + //------------------------------------------------------------------------------ + /// @brief Get the flutter platform node delegate with the given id from + /// this accessibility bridge. Returns expired weak_ptr if the + /// delegate associated with the id does not exist or has been + /// removed from the accessibility tree. + /// + /// @param[in] id The id of the flutter accessibility node you want + /// to retrieve. + std::weak_ptr + GetFlutterPlatformNodeDelegateFromID(AccessibilityNodeId id) const; + + //------------------------------------------------------------------------------ + /// @brief Get the ax tree data from this accessibility bridge. The tree + /// data contains information such as the id of the node that + /// has the keyboard focus or the text selection range. + const ui::AXTreeData& GetAXTreeData() const; + + private: + // See FlutterSemanticsNode in embedder.h + typedef struct { + int32_t id; + FlutterSemanticsFlag flags; + FlutterSemanticsAction actions; + int32_t text_selection_base; + int32_t text_selection_extent; + int32_t scroll_child_count; + int32_t scroll_index; + double scroll_position; + double scroll_extent_max; + double scroll_extent_min; + double elevation; + double thickness; + std::string label; + std::string hint; + std::string value; + std::string increased_value; + std::string decreased_value; + FlutterTextDirection text_direction; + FlutterRect rect; + FlutterTransformation transform; + std::vector children_in_traversal_order; + std::vector custom_accessibility_actions; + } SemanticsNode; + + // See FlutterSemanticsCustomAction in embedder.h + typedef struct { + int32_t id; + FlutterSemanticsAction override_action; + std::string label; + std::string hint; + } SemanticsCustomAction; + + std::unordered_map> + id_wrapper_map_; + ui::AXTree tree_; + ui::AXEventGenerator event_generator_; + std::unordered_map pending_semantics_node_updates_; + std::unordered_map + pending_semantics_custom_action_updates_; + AccessibilityNodeId last_focused_id_ = ui::AXNode::kInvalidAXID; + std::unique_ptr delegate_; + + void InitAXTree(const ui::AXTreeUpdate& initial_state); + void GetSubTreeList(SemanticsNode target, std::vector& result); + void ConvertFluterUpdate(const SemanticsNode& node, + ui::AXTreeUpdate& tree_update); + void SetRoleFromFlutterUpdate(ui::AXNodeData& node_data, + const SemanticsNode& node); + void SetStateFromFlutterUpdate(ui::AXNodeData& node_data, + const SemanticsNode& node); + void SetActionsFromFlutterUpdate(ui::AXNodeData& node_data, + const SemanticsNode& node); + void SetBooleanAttributesFromFlutterUpdate(ui::AXNodeData& node_data, + const SemanticsNode& node); + void SetIntAttributesFromFlutterUpdate(ui::AXNodeData& node_data, + const SemanticsNode& node); + void SetIntListAttributesFromFlutterUpdate(ui::AXNodeData& node_data, + const SemanticsNode& node); + void SetStringListAttributesFromFlutterUpdate(ui::AXNodeData& node_data, + const SemanticsNode& node); + void SetNameFromFlutterUpdate(ui::AXNodeData& node_data, + const SemanticsNode& node); + void SetValueFromFlutterUpdate(ui::AXNodeData& node_data, + const SemanticsNode& node); + void SetTreeData(const SemanticsNode& node, ui::AXTreeUpdate& tree_update); + SemanticsNode FromFlutterSemanticsNode( + const FlutterSemanticsNode* flutter_node); + SemanticsCustomAction FromFlutterSemanticsCustomAction( + const FlutterSemanticsCustomAction* flutter_custom_action); + + // |AXTreeObserver| + void OnNodeWillBeDeleted(ui::AXTree* tree, ui::AXNode* node) override; + + // |AXTreeObserver| + void OnSubtreeWillBeDeleted(ui::AXTree* tree, ui::AXNode* node) override; + + // |AXTreeObserver| + void OnNodeCreated(ui::AXTree* tree, ui::AXNode* node) override; + + // |AXTreeObserver| + void OnNodeDeleted(ui::AXTree* tree, AccessibilityNodeId node_id) override; + + // |AXTreeObserver| + void OnNodeReparented(ui::AXTree* tree, ui::AXNode* node) override; + + // |AXTreeObserver| + void OnRoleChanged(ui::AXTree* tree, + ui::AXNode* node, + ax::mojom::Role old_role, + ax::mojom::Role new_role) override; + + // |AXTreeObserver| + void OnAtomicUpdateFinished( + ui::AXTree* tree, + bool root_changed, + const std::vector& changes) override; + + // |FlutterPlatformNodeDelegate::OwnerBridge| + void SetLastFocusedId(AccessibilityNodeId node_id) override; + + // |FlutterPlatformNodeDelegate::OwnerBridge| + AccessibilityNodeId GetLastFocusedId() override; + + // |FlutterPlatformNodeDelegate::OwnerBridge| + gfx::NativeViewAccessible GetNativeAccessibleFromId( + AccessibilityNodeId id) override; + + // |FlutterPlatformNodeDelegate::OwnerBridge| + void DispatchAccessibilityAction(AccessibilityNodeId target, + FlutterSemanticsAction action, + std::vector data) override; + + // |FlutterPlatformNodeDelegate::OwnerBridge| + gfx::RectF RelativeToGlobalBounds(const ui::AXNode* node, + bool& offscreen, + bool clip_bounds) override; + + BASE_DISALLOW_COPY_AND_ASSIGN(AccessibilityBridge); +}; + +} // namespace flutter + +#endif // FLUTTER_SHELL_PLATFORM_COMMON_CPP_ACCESSIBILITY_BRIDGE_H_ diff --git a/shell/platform/common/cpp/accessibility_bridge_unittests.cc b/shell/platform/common/cpp/accessibility_bridge_unittests.cc new file mode 100644 index 0000000000000..90d50a571fcd5 --- /dev/null +++ b/shell/platform/common/cpp/accessibility_bridge_unittests.cc @@ -0,0 +1,204 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "accessibility_bridge.h" + +#include "gtest/gtest.h" + +#include "test_accessibility_bridge.h" + +namespace flutter { +namespace testing { + +TEST(AccessibilityBridgeTest, basicTest) { + std::shared_ptr bridge = + std::make_shared( + std::make_unique()); + FlutterSemanticsNode root; + root.id = 0; + root.label = "root"; + root.hint = ""; + root.value = ""; + root.increased_value = ""; + root.decreased_value = ""; + root.child_count = 2; + int32_t children[] = {1, 2}; + root.children_in_traversal_order = children; + root.custom_accessibility_actions_count = 0; + bridge->AddFlutterSemanticsNodeUpdate(&root); + + FlutterSemanticsNode child1; + child1.id = 1; + child1.label = "child 1"; + child1.hint = ""; + child1.value = ""; + child1.increased_value = ""; + child1.decreased_value = ""; + child1.child_count = 0; + child1.custom_accessibility_actions_count = 0; + bridge->AddFlutterSemanticsNodeUpdate(&child1); + + FlutterSemanticsNode child2; + child2.id = 2; + child2.label = "child 2"; + child2.hint = ""; + child2.value = ""; + child2.increased_value = ""; + child2.decreased_value = ""; + child2.child_count = 0; + child2.custom_accessibility_actions_count = 0; + bridge->AddFlutterSemanticsNodeUpdate(&child2); + + bridge->CommitUpdates(); + + auto root_node = bridge->GetFlutterPlatformNodeDelegateFromID(0).lock(); + auto child1_node = bridge->GetFlutterPlatformNodeDelegateFromID(1).lock(); + auto child2_node = bridge->GetFlutterPlatformNodeDelegateFromID(2).lock(); + EXPECT_EQ(root_node->GetChildCount(), 2); + EXPECT_EQ(root_node->GetData().child_ids[0], 1); + EXPECT_EQ(root_node->GetData().child_ids[1], 2); + EXPECT_EQ(root_node->GetName(), "root"); + + EXPECT_EQ(child1_node->GetChildCount(), 0); + EXPECT_EQ(child1_node->GetName(), "child 1"); + + EXPECT_EQ(child2_node->GetChildCount(), 0); + EXPECT_EQ(child2_node->GetName(), "child 2"); +} + +TEST(AccessibilityBridgeTest, canFireChildrenChangedCorrectly) { + TestAccessibilityBridgeDelegate* delegate = + new TestAccessibilityBridgeDelegate(); + std::unique_ptr ptr(delegate); + std::shared_ptr bridge = + std::make_shared(std::move(ptr)); + FlutterSemanticsNode root; + root.id = 0; + root.flags = static_cast(0); + root.actions = static_cast(0); + root.text_selection_base = -1; + root.text_selection_extent = -1; + root.label = "root"; + root.hint = ""; + root.value = ""; + root.increased_value = ""; + root.decreased_value = ""; + root.child_count = 1; + int32_t children[] = {1}; + root.children_in_traversal_order = children; + root.custom_accessibility_actions_count = 0; + bridge->AddFlutterSemanticsNodeUpdate(&root); + + FlutterSemanticsNode child1; + child1.id = 1; + child1.flags = static_cast(0); + child1.actions = static_cast(0); + child1.text_selection_base = -1; + child1.text_selection_extent = -1; + child1.label = "child 1"; + child1.hint = ""; + child1.value = ""; + child1.increased_value = ""; + child1.decreased_value = ""; + child1.child_count = 0; + child1.custom_accessibility_actions_count = 0; + bridge->AddFlutterSemanticsNodeUpdate(&child1); + + bridge->CommitUpdates(); + + auto root_node = bridge->GetFlutterPlatformNodeDelegateFromID(0).lock(); + auto child1_node = bridge->GetFlutterPlatformNodeDelegateFromID(1).lock(); + EXPECT_EQ(root_node->GetChildCount(), 1); + EXPECT_EQ(root_node->GetData().child_ids[0], 1); + EXPECT_EQ(root_node->GetName(), "root"); + + EXPECT_EQ(child1_node->GetChildCount(), 0); + EXPECT_EQ(child1_node->GetName(), "child 1"); + delegate->accessibilitiy_events.clear(); + + // Add a child to root. + root.child_count = 2; + int32_t new_children[] = {1, 2}; + root.children_in_traversal_order = new_children; + bridge->AddFlutterSemanticsNodeUpdate(&root); + + FlutterSemanticsNode child2; + child2.id = 2; + child2.flags = static_cast(0); + child2.actions = static_cast(0); + child2.text_selection_base = -1; + child2.text_selection_extent = -1; + child2.label = "child 2"; + child2.hint = ""; + child2.value = ""; + child2.increased_value = ""; + child2.decreased_value = ""; + child2.child_count = 0; + child2.custom_accessibility_actions_count = 0; + bridge->AddFlutterSemanticsNodeUpdate(&child2); + + bridge->CommitUpdates(); + + root_node = bridge->GetFlutterPlatformNodeDelegateFromID(0).lock(); + + EXPECT_EQ(root_node->GetChildCount(), 2); + EXPECT_EQ(root_node->GetData().child_ids[0], 1); + EXPECT_EQ(root_node->GetData().child_ids[1], 2); + EXPECT_EQ(delegate->accessibilitiy_events.size(), size_t{2}); + std::set actual_event; + actual_event.insert(delegate->accessibilitiy_events[0].event_params.event); + actual_event.insert(delegate->accessibilitiy_events[1].event_params.event); + EXPECT_NE(actual_event.find(ui::AXEventGenerator::Event::CHILDREN_CHANGED), + actual_event.end()); + EXPECT_NE(actual_event.find(ui::AXEventGenerator::Event::SUBTREE_CREATED), + actual_event.end()); +} + +TEST(AccessibilityBridgeTest, canHandleSelectionChangeCorrectly) { + TestAccessibilityBridgeDelegate* delegate = + new TestAccessibilityBridgeDelegate(); + std::unique_ptr ptr(delegate); + std::shared_ptr bridge = + std::make_shared(std::move(ptr)); + FlutterSemanticsNode root; + root.id = 0; + root.flags = FlutterSemanticsFlag::kFlutterSemanticsFlagIsTextField; + root.actions = static_cast(0); + root.text_selection_base = -1; + root.text_selection_extent = -1; + root.label = "root"; + root.hint = ""; + root.value = ""; + root.increased_value = ""; + root.decreased_value = ""; + root.child_count = 0; + root.custom_accessibility_actions_count = 0; + bridge->AddFlutterSemanticsNodeUpdate(&root); + + bridge->CommitUpdates(); + + const ui::AXTreeData& tree = bridge->GetAXTreeData(); + EXPECT_EQ(tree.sel_anchor_object_id, ui::AXNode::kInvalidAXID); + delegate->accessibilitiy_events.clear(); + + // Update the selection. + root.text_selection_base = 0; + root.text_selection_extent = 5; + bridge->AddFlutterSemanticsNodeUpdate(&root); + + bridge->CommitUpdates(); + + EXPECT_EQ(tree.sel_anchor_object_id, 0); + EXPECT_EQ(tree.sel_anchor_offset, 0); + EXPECT_EQ(tree.sel_focus_object_id, 0); + EXPECT_EQ(tree.sel_focus_offset, 5); + EXPECT_EQ(delegate->accessibilitiy_events.size(), size_t{2}); + EXPECT_EQ(delegate->accessibilitiy_events[0].event_params.event, + ui::AXEventGenerator::Event::DOCUMENT_SELECTION_CHANGED); + EXPECT_EQ(delegate->accessibilitiy_events[1].event_params.event, + ui::AXEventGenerator::Event::OTHER_ATTRIBUTE_CHANGED); +} + +} // namespace testing +} // namespace flutter diff --git a/shell/platform/common/cpp/flutter_platform_node_delegate.cc b/shell/platform/common/cpp/flutter_platform_node_delegate.cc new file mode 100644 index 0000000000000..9eff09cc19350 --- /dev/null +++ b/shell/platform/common/cpp/flutter_platform_node_delegate.cc @@ -0,0 +1,110 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "flutter_platform_node_delegate.h" + +#include "flutter/third_party/accessibility/ax/ax_action_data.h" +#include "flutter/third_party/accessibility/gfx/geometry/rect_conversions.h" + +namespace flutter { + +FlutterPlatformNodeDelegate::FlutterPlatformNodeDelegate() = default; + +FlutterPlatformNodeDelegate::~FlutterPlatformNodeDelegate() = default; + +void FlutterPlatformNodeDelegate::Init(std::weak_ptr bridge, + ui::AXNode* node) { + bridge_ = bridge; + ax_node_ = node; +} + +ui::AXNode* FlutterPlatformNodeDelegate::GetAXNode() const { + return ax_node_; +} + +bool FlutterPlatformNodeDelegate::AccessibilityPerformAction( + const ui::AXActionData& data) { + AccessibilityNodeId target = ax_node_->id(); + auto bridge_ptr = bridge_.lock(); + BASE_DCHECK(bridge_ptr); + switch (data.action) { + case ax::mojom::Action::kDoDefault: + bridge_ptr->DispatchAccessibilityAction( + target, FlutterSemanticsAction::kFlutterSemanticsActionTap, {}); + return true; + case ax::mojom::Action::kFocus: + bridge_ptr->SetLastFocusedId(target); + bridge_ptr->DispatchAccessibilityAction( + target, + FlutterSemanticsAction:: + kFlutterSemanticsActionDidGainAccessibilityFocus, + {}); + return true; + case ax::mojom::Action::kScrollToMakeVisible: + bridge_ptr->DispatchAccessibilityAction( + target, FlutterSemanticsAction::kFlutterSemanticsActionShowOnScreen, + {}); + return true; + // TODO(chunhtai): support more actions. + default: + return false; + } + return false; +} + +const ui::AXNodeData& FlutterPlatformNodeDelegate::GetData() const { + return ax_node_->data(); +} + +gfx::NativeViewAccessible FlutterPlatformNodeDelegate::GetParent() { + if (!ax_node_->parent()) { + return nullptr; + } + auto bridge_ptr = bridge_.lock(); + BASE_DCHECK(bridge_ptr); + return bridge_ptr->GetNativeAccessibleFromId(ax_node_->parent()->id()); +} + +gfx::NativeViewAccessible FlutterPlatformNodeDelegate::GetFocus() { + auto bridge_ptr = bridge_.lock(); + BASE_DCHECK(bridge_ptr); + AccessibilityNodeId last_focused = bridge_ptr->GetLastFocusedId(); + if (last_focused == ui::AXNode::kInvalidAXID) { + return nullptr; + } + return bridge_ptr->GetNativeAccessibleFromId(last_focused); +} + +int FlutterPlatformNodeDelegate::GetChildCount() const { + return static_cast(ax_node_->GetUnignoredChildCount()); +} + +gfx::NativeViewAccessible FlutterPlatformNodeDelegate::ChildAtIndex(int index) { + auto bridge_ptr = bridge_.lock(); + BASE_DCHECK(bridge_ptr); + AccessibilityNodeId child = ax_node_->GetUnignoredChildAtIndex(index)->id(); + return bridge_ptr->GetNativeAccessibleFromId(child); +} + +gfx::Rect FlutterPlatformNodeDelegate::GetBoundsRect( + const ui::AXCoordinateSystem coordinate_system, + const ui::AXClippingBehavior clipping_behavior, + ui::AXOffscreenResult* offscreen_result) const { + auto bridge_ptr = bridge_.lock(); + BASE_DCHECK(bridge_ptr); + // TODO(chunhtai): We need to apply screen dpr in here. + // https://github.com/flutter/flutter/issues/74283 + const bool clip_bounds = + clipping_behavior == ui::AXClippingBehavior::kClipped; + bool offscreen = false; + gfx::RectF bounds = + bridge_ptr->RelativeToGlobalBounds(ax_node_, offscreen, clip_bounds); + if (offscreen_result != nullptr) { + *offscreen_result = offscreen ? ui::AXOffscreenResult::kOffscreen + : ui::AXOffscreenResult::kOnscreen; + } + return gfx::ToEnclosingRect(bounds); +} + +} // namespace flutter diff --git a/shell/platform/common/cpp/flutter_platform_node_delegate.h b/shell/platform/common/cpp/flutter_platform_node_delegate.h new file mode 100644 index 0000000000000..26d757c8119c1 --- /dev/null +++ b/shell/platform/common/cpp/flutter_platform_node_delegate.h @@ -0,0 +1,144 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef FLUTTER_SHELL_PLATFORM_COMMON_CPP_FLUTTER_PLATFORM_NODE_DELEGATE_H_ +#define FLUTTER_SHELL_PLATFORM_COMMON_CPP_FLUTTER_PLATFORM_NODE_DELEGATE_H_ + +#include "flutter/shell/platform/embedder/embedder.h" + +#include "flutter/third_party/accessibility/ax/ax_event_generator.h" +#include "flutter/third_party/accessibility/ax/platform/ax_platform_node_delegate_base.h" + +namespace flutter { + +typedef ui::AXNode::AXID AccessibilityNodeId; + +//------------------------------------------------------------------------------ +/// The platform node delegate to be used in accessibility bridge. This +/// class is responsible for providing native accessibility object with +/// appropriate information, such as accessibility label/value/bounds. +/// +/// While most methods have default implementations and are ready to be used +/// as-is, the subclasses must override the GetNativeViewAccessible to return +/// native accessibility objects. To do that, subclasses should create and +/// maintain AXPlatformNode[s] which delegate their accessibility attributes to +/// this class. +/// +/// For desktop platforms, subclasses also need to override the GetBoundsRect +/// to apply window-to-screen transform. +/// +/// This class transforms bounds assuming the device pixel ratio is 1.0. See +/// the https://github.com/flutter/flutter/issues/74283 for more information. +class FlutterPlatformNodeDelegate : public ui::AXPlatformNodeDelegateBase { + public: + //---------------------------------------------------------------------------- + /// The required interface to be able to own the flutter platform node + /// delegate. + class OwnerBridge { + public: + ~OwnerBridge() = default; + + protected: + friend class FlutterPlatformNodeDelegate; + + //--------------------------------------------------------------------------- + /// @brief Dispatch accessibility action back to the Flutter framework. + /// These actions are generated in the native accessibility + /// system when users interact with the assistive technologies. + /// For example, a + /// FlutterSemanticsAction::kFlutterSemanticsActionTap is + /// fired when user click or touch the screen. + /// + /// @param[in] target The semantics node id of the action + /// target. + /// @param[in] action The generated flutter semantics action. + /// @param[in] data Additional data associated with the + /// action. + virtual void DispatchAccessibilityAction(AccessibilityNodeId target, + FlutterSemanticsAction action, + std::vector data) = 0; + + //--------------------------------------------------------------------------- + /// @brief Get the native accessibility node with the given id. + /// + /// @param[in] id The id of the native accessibility node you + /// want to retrieve. + virtual gfx::NativeViewAccessible GetNativeAccessibleFromId( + AccessibilityNodeId id) = 0; + + //--------------------------------------------------------------------------- + /// @brief Get the last id of the node that received accessibility + /// focus. + virtual AccessibilityNodeId GetLastFocusedId() = 0; + + //--------------------------------------------------------------------------- + /// @brief Update the id of the node that is currently foucsed by the + /// native accessibility system. + /// + /// @param[in] node_id The id of the focused node. + virtual void SetLastFocusedId(AccessibilityNodeId node_id) = 0; + + //--------------------------------------------------------------------------- + /// @brief Gets the rectangular bounds of the ax node relative to + /// global coordinate + /// + /// @param[in] node The ax node to look up. + /// @param[in] offscreen the bool reference to hold the result whether + /// the ax node is outside of its ancestors' bounds. + /// @param[in] clip_bounds whether to clip the result if the ax node cannot + /// be fully contained in its ancestors' bounds. + virtual gfx::RectF RelativeToGlobalBounds(const ui::AXNode* node, + bool& offscreen, + bool clip_bounds) = 0; + }; + + FlutterPlatformNodeDelegate(); + + // |ui::AXPlatformNodeDelegateBase| + ~FlutterPlatformNodeDelegate() override; + + // |ui::AXPlatformNodeDelegateBase| + const ui::AXNodeData& GetData() const override; + + // |ui::AXPlatformNodeDelegateBase| + bool AccessibilityPerformAction(const ui::AXActionData& data) override; + + // |ui::AXPlatformNodeDelegateBase| + gfx::NativeViewAccessible GetParent() override; + + // |ui::AXPlatformNodeDelegateBase| + gfx::NativeViewAccessible GetFocus() override; + + // |ui::AXPlatformNodeDelegateBase| + int GetChildCount() const override; + + // |ui::AXPlatformNodeDelegateBase| + gfx::NativeViewAccessible ChildAtIndex(int index) override; + + // |ui::AXPlatformNodeDelegateBase| + gfx::Rect GetBoundsRect( + const ui::AXCoordinateSystem coordinate_system, + const ui::AXClippingBehavior clipping_behavior, + ui::AXOffscreenResult* offscreen_result) const override; + + //------------------------------------------------------------------------------ + /// @brief Called only once, immediately after construction. The + /// constructor doesn't take any arguments because in the Windows + /// subclass we use a special function to construct a COM object. + /// Subclasses must call super. + virtual void Init(std::weak_ptr bridge, ui::AXNode* node); + + protected: + //------------------------------------------------------------------------------ + /// @brief Gets the underlying ax node for this accessibility node. + ui::AXNode* GetAXNode() const; + + private: + ui::AXNode* ax_node_; + std::weak_ptr bridge_; +}; + +} // namespace flutter + +#endif // FLUTTER_SHELL_PLATFORM_COMMON_CPP_FLUTTER_PLATFORM_NODE_DELEGATE_H_ diff --git a/shell/platform/common/cpp/flutter_platform_node_delegate_unittests.cc b/shell/platform/common/cpp/flutter_platform_node_delegate_unittests.cc new file mode 100644 index 0000000000000..d4753265f4b07 --- /dev/null +++ b/shell/platform/common/cpp/flutter_platform_node_delegate_unittests.cc @@ -0,0 +1,178 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "flutter_platform_node_delegate.h" + +#include "flutter/third_party/accessibility/ax/ax_action_data.h" +#include "gtest/gtest.h" + +#include "test_accessibility_bridge.h" + +namespace flutter { +namespace testing { + +TEST(FlutterPlatformNodeDelegateTest, canPerfomActions) { + TestAccessibilityBridgeDelegate* delegate = + new TestAccessibilityBridgeDelegate(); + std::unique_ptr ptr(delegate); + std::shared_ptr bridge = + std::make_shared(std::move(ptr)); + FlutterSemanticsNode root; + root.id = 0; + root.flags = FlutterSemanticsFlag::kFlutterSemanticsFlagIsTextField; + root.actions = static_cast(0); + root.text_selection_base = -1; + root.text_selection_extent = -1; + root.label = "root"; + root.hint = ""; + root.value = ""; + root.increased_value = ""; + root.decreased_value = ""; + root.child_count = 0; + root.custom_accessibility_actions_count = 0; + bridge->AddFlutterSemanticsNodeUpdate(&root); + + bridge->CommitUpdates(); + + auto accessibility = bridge->GetFlutterPlatformNodeDelegateFromID(0).lock(); + // Performs an AXAction. + ui::AXActionData action_data; + action_data.action = ax::mojom::Action::kDoDefault; + accessibility->AccessibilityPerformAction(action_data); + EXPECT_EQ(delegate->performed_actions.size(), size_t{1}); + EXPECT_EQ(delegate->performed_actions[0], + FlutterSemanticsAction::kFlutterSemanticsActionTap); + + action_data.action = ax::mojom::Action::kFocus; + accessibility->AccessibilityPerformAction(action_data); + EXPECT_EQ(delegate->performed_actions.size(), size_t{2}); + EXPECT_EQ( + delegate->performed_actions[1], + FlutterSemanticsAction::kFlutterSemanticsActionDidGainAccessibilityFocus); + + action_data.action = ax::mojom::Action::kScrollToMakeVisible; + accessibility->AccessibilityPerformAction(action_data); + EXPECT_EQ(delegate->performed_actions.size(), size_t{3}); + EXPECT_EQ(delegate->performed_actions[2], + FlutterSemanticsAction::kFlutterSemanticsActionShowOnScreen); +} + +TEST(FlutterPlatformNodeDelegateTest, canGetAXNode) { + // Set up a flutter accessibility node. + std::shared_ptr bridge = + std::make_shared( + std::make_unique()); + FlutterSemanticsNode root; + root.id = 0; + root.flags = FlutterSemanticsFlag::kFlutterSemanticsFlagIsTextField; + root.actions = static_cast(0); + root.text_selection_base = -1; + root.text_selection_extent = -1; + root.label = "root"; + root.hint = ""; + root.value = ""; + root.increased_value = ""; + root.decreased_value = ""; + root.child_count = 0; + root.custom_accessibility_actions_count = 0; + bridge->AddFlutterSemanticsNodeUpdate(&root); + + bridge->CommitUpdates(); + + auto accessibility = bridge->GetFlutterPlatformNodeDelegateFromID(0).lock(); + EXPECT_EQ(accessibility->GetData().id, 0); +} + +TEST(FlutterPlatformNodeDelegateTest, canCalculateBoundsCorrectly) { + std::shared_ptr bridge = + std::make_shared( + std::make_unique()); + FlutterSemanticsNode root; + root.id = 0; + root.label = "root"; + root.hint = ""; + root.value = ""; + root.increased_value = ""; + root.decreased_value = ""; + root.child_count = 1; + int32_t children[] = {1}; + root.children_in_traversal_order = children; + root.custom_accessibility_actions_count = 0; + root.rect = {0, 0, 100, 100}; // LTRB + root.transform = {1, 0, 0, 0, 1, 0, 0, 0, 1}; + bridge->AddFlutterSemanticsNodeUpdate(&root); + + FlutterSemanticsNode child1; + child1.id = 1; + child1.label = "child 1"; + child1.hint = ""; + child1.value = ""; + child1.increased_value = ""; + child1.decreased_value = ""; + child1.child_count = 0; + child1.custom_accessibility_actions_count = 0; + child1.rect = {0, 0, 50, 50}; // LTRB + child1.transform = {0.5, 0, 0, 0, 0.5, 0, 0, 0, 1}; + bridge->AddFlutterSemanticsNodeUpdate(&child1); + + bridge->CommitUpdates(); + auto child1_node = bridge->GetFlutterPlatformNodeDelegateFromID(1).lock(); + ui::AXOffscreenResult result; + gfx::Rect bounds = + child1_node->GetBoundsRect(ui::AXCoordinateSystem::kScreenDIPs, + ui::AXClippingBehavior::kClipped, &result); + EXPECT_EQ(bounds.x(), 0); + EXPECT_EQ(bounds.y(), 0); + EXPECT_EQ(bounds.width(), 25); + EXPECT_EQ(bounds.height(), 25); + EXPECT_EQ(result, ui::AXOffscreenResult::kOnscreen); +} + +TEST(FlutterPlatformNodeDelegateTest, canCalculateOffScreenBoundsCorrectly) { + std::shared_ptr bridge = + std::make_shared( + std::make_unique()); + FlutterSemanticsNode root; + root.id = 0; + root.label = "root"; + root.hint = ""; + root.value = ""; + root.increased_value = ""; + root.decreased_value = ""; + root.child_count = 1; + int32_t children[] = {1}; + root.children_in_traversal_order = children; + root.custom_accessibility_actions_count = 0; + root.rect = {0, 0, 100, 100}; // LTRB + root.transform = {1, 0, 0, 0, 1, 0, 0, 0, 1}; + bridge->AddFlutterSemanticsNodeUpdate(&root); + + FlutterSemanticsNode child1; + child1.id = 1; + child1.label = "child 1"; + child1.hint = ""; + child1.value = ""; + child1.increased_value = ""; + child1.decreased_value = ""; + child1.child_count = 0; + child1.custom_accessibility_actions_count = 0; + child1.rect = {90, 90, 100, 100}; // LTRB + child1.transform = {2, 0, 0, 0, 2, 0, 0, 0, 1}; + bridge->AddFlutterSemanticsNodeUpdate(&child1); + + bridge->CommitUpdates(); + auto child1_node = bridge->GetFlutterPlatformNodeDelegateFromID(1).lock(); + ui::AXOffscreenResult result; + gfx::Rect bounds = + child1_node->GetBoundsRect(ui::AXCoordinateSystem::kScreenDIPs, + ui::AXClippingBehavior::kUnclipped, &result); + EXPECT_EQ(bounds.x(), 180); + EXPECT_EQ(bounds.y(), 180); + EXPECT_EQ(bounds.width(), 20); + EXPECT_EQ(bounds.height(), 20); + EXPECT_EQ(result, ui::AXOffscreenResult::kOffscreen); +} + +} // namespace testing +} // namespace flutter diff --git a/shell/platform/common/cpp/test_accessibility_bridge.cc b/shell/platform/common/cpp/test_accessibility_bridge.cc new file mode 100644 index 0000000000000..22f88ea1f3289 --- /dev/null +++ b/shell/platform/common/cpp/test_accessibility_bridge.cc @@ -0,0 +1,26 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "test_accessibility_bridge.h" + +namespace flutter { + +std::unique_ptr +TestAccessibilityBridgeDelegate::CreateFlutterPlatformNodeDelegate() { + return std::make_unique(); +}; + +void TestAccessibilityBridgeDelegate::OnAccessibilityEvent( + ui::AXEventGenerator::TargetedEvent targeted_event) { + accessibilitiy_events.push_back(targeted_event); +} + +void TestAccessibilityBridgeDelegate::DispatchAccessibilityAction( + AccessibilityNodeId target, + FlutterSemanticsAction action, + std::vector data) { + performed_actions.push_back(action); +} + +} // namespace flutter diff --git a/shell/platform/common/cpp/test_accessibility_bridge.h b/shell/platform/common/cpp/test_accessibility_bridge.h new file mode 100644 index 0000000000000..e0ec2b434df35 --- /dev/null +++ b/shell/platform/common/cpp/test_accessibility_bridge.h @@ -0,0 +1,31 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef FLUTTER_SHELL_PLATFORM_COMMON_CPP_TEST_ACCESSIBILITY_BRIDGE_H_ +#define FLUTTER_SHELL_PLATFORM_COMMON_CPP_TEST_ACCESSIBILITY_BRIDGE_H_ + +#include "accessibility_bridge.h" + +namespace flutter { + +class TestAccessibilityBridgeDelegate + : public AccessibilityBridge::AccessibilityBridgeDelegate { + public: + TestAccessibilityBridgeDelegate() = default; + + void OnAccessibilityEvent( + ui::AXEventGenerator::TargetedEvent targeted_event) override; + void DispatchAccessibilityAction(AccessibilityNodeId target, + FlutterSemanticsAction action, + std::vector data) override; + std::unique_ptr + CreateFlutterPlatformNodeDelegate(); + + std::vector accessibilitiy_events; + std::vector performed_actions; +}; + +} // namespace flutter + +#endif // FLUTTER_SHELL_PLATFORM_COMMON_CPP_TEST_ACCESSIBILITY_BRIDGE_H_ diff --git a/third_party/accessibility/build.gn b/third_party/accessibility/BUILD.gn similarity index 100% rename from third_party/accessibility/build.gn rename to third_party/accessibility/BUILD.gn diff --git a/third_party/accessibility/gfx/native_widget_types.h b/third_party/accessibility/gfx/native_widget_types.h index 8c5479755dd42..287689be82af9 100644 --- a/third_party/accessibility/gfx/native_widget_types.h +++ b/third_party/accessibility/gfx/native_widget_types.h @@ -191,6 +191,10 @@ typedef ui::WindowAndroid* NativeWindow; typedef base::android::ScopedJavaGlobalRef NativeEvent; constexpr NativeView kNullNativeView = nullptr; constexpr NativeWindow kNullNativeWindow = nullptr; +#elif defined(OS_LINUX) +// TODO(chunhtai): Figures out what is the correct type for Linux +// https://github.com/flutter/flutter/issues/74270 +typedef void* NativeCursor; #else #error Unknown build environment. #endif @@ -245,6 +249,11 @@ constexpr AcceleratedWidget kNullAcceleratedWidget = 0; #elif defined(USE_OZONE) || defined(USE_X11) typedef uint32_t AcceleratedWidget; constexpr AcceleratedWidget kNullAcceleratedWidget = 0; +#elif defined(OS_LINUX) +// TODO(chunhtai): Figure out what the correct type is for the Linux. +// https://github.com/flutter/flutter/issues/74270 +typedef void* AcceleratedWidget; +constexpr AcceleratedWidget kNullAcceleratedWidget = nullptr; #else #error unknown platform #endif