Skip to content
This repository was archived by the owner on Feb 25, 2025. It is now read-only.

Commit 0b56cb8

Browse files
LongCatIsLooongLongCatIsLooong
authored
Bare-bones iOS FKA implementation (#55964)
A bare-bones implementation for iOS focus engine support, to enable basic full keyboard access (FKA) (except for scrolling which will be implemented in a different patch). Partially f1xes flutter/flutter#76497 https://github.com/user-attachments/assets/427db87e-cc15-483a-85a1-56bf1c02c285 On iOS 15 and above, FKA, if enabled, always consumes relevant key events, so the Flutter framework can't see those key events as they won't be delivered via the `UIResponder` chain (https://developer.apple.com/documentation/uikit/uikeycommand/3780513-wantspriorityoversystembehavior). This patch provides the basic focus-related information to the iOS focus engine, derived from the information already available in the accessibility tree, so the iOS focus engine can navigate the UI hierarchy and invoke `accessibilityActivate` on the current focus when the user presses the space key. This at the moment seems to be the best option: - There doesn't seem to be a way to reliably prevent FKA from consuming the key events and that seems to be by design. - The user can remap the FKA keys in iOS system settings, but that key mapping isn't available to apps, so even if the framework can get the key events it won't be able to honor custom key maps. - When FKA is on, `-[FlutterView isAccessibilityElement]` is called without user interaction (presumably it's called when the view appears), so when the user interacts with the app using FKA, it's likely that the accessibility is already enabled, we don't have to worry detecting whether FKA is on (at least for now). Scrolling using FKA currently does not work despite `FlutterScrollableSemanticsObject` conforms to `UIFocusItemScrollableContainer`. `setContentOffset:` must be implemented using a new API that informs the framework of the new contentOffset in the scroll view. `accessibilityScroll` does not work because it scrolls too much in most cases, and it tells the framework "how much to scroll" instead of "where to scroll to" so it tends to be jumpy. ## What happens on iOS versions earlier than 15 I couldn't find iOS 14 runtime for simulators in xcode 16 so I couldn't test it. But since the key events will be delivered to the framework first regardless of whether FKA is enabled, the framework is supposed to handle keyboard focus navigation even when FKA is on. [C++, Objective-C, Java style guides]: https://github.com/flutter/engine/blob/main/CONTRIBUTING.md#style
1 parent 5565cbc commit 0b56cb8

File tree

5 files changed

+186
-0
lines changed

5 files changed

+186
-0
lines changed

‎ci/licenses_golden/licenses_flutter‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44558,6 +44558,7 @@ ORIGIN: ../../../flutter/shell/platform/darwin/ios/framework/Source/FlutterViewT
4455844558
ORIGIN: ../../../flutter/shell/platform/darwin/ios/framework/Source/IOKit.h + ../../../flutter/LICENSE
4455944559
ORIGIN: ../../../flutter/shell/platform/darwin/ios/framework/Source/KeyCodeMap.g.mm + ../../../flutter/LICENSE
4456044560
ORIGIN: ../../../flutter/shell/platform/darwin/ios/framework/Source/KeyCodeMap_Internal.h + ../../../flutter/LICENSE
44561+
ORIGIN: ../../../flutter/shell/platform/darwin/ios/framework/Source/SemanticsObject+UIFocusSystem.mm + ../../../flutter/LICENSE
4456144562
ORIGIN: ../../../flutter/shell/platform/darwin/ios/framework/Source/SemanticsObject.h + ../../../flutter/LICENSE
4456244563
ORIGIN: ../../../flutter/shell/platform/darwin/ios/framework/Source/SemanticsObject.mm + ../../../flutter/LICENSE
4456344564
ORIGIN: ../../../flutter/shell/platform/darwin/ios/framework/Source/SemanticsObjectTest.mm + ../../../flutter/LICENSE
@@ -47436,6 +47437,7 @@ FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/FlutterViewTes
4743647437
FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/IOKit.h
4743747438
FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/KeyCodeMap.g.mm
4743847439
FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/KeyCodeMap_Internal.h
47440+
FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/SemanticsObject+UIFocusSystem.mm
4743947441
FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/SemanticsObject.h
4744047442
FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/SemanticsObject.mm
4744147443
FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/SemanticsObjectTest.mm

‎shell/platform/darwin/ios/BUILD.gn‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,7 @@ source_set("flutter_framework_source_arc") {
108108
"framework/Source/FlutterViewResponder.h",
109109
"framework/Source/KeyCodeMap.g.mm",
110110
"framework/Source/KeyCodeMap_Internal.h",
111+
"framework/Source/SemanticsObject+UIFocusSystem.mm",
111112
"framework/Source/SemanticsObject.h",
112113
"framework/Source/SemanticsObject.mm",
113114
"framework/Source/TextInputSemanticsObject.h",

‎shell/platform/darwin/ios/framework/Source/FlutterView.mm‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
// found in the LICENSE file.
44

55
#import "flutter/shell/platform/darwin/ios/framework/Source/FlutterView.h"
6+
#import "flutter/shell/platform/darwin/ios/framework/Source/SemanticsObject.h"
67

78
#include "flutter/fml/platform/darwin/cf_utils.h"
89

@@ -226,4 +227,30 @@ - (BOOL)isAccessibilityElement {
226227
return NO;
227228
}
228229

230+
// Enables keyboard-based navigation when the user turns on
231+
// full keyboard access (FKA), using existing accessibility information.
232+
//
233+
// iOS does not provide any API for monitoring or querying whether FKA is on,
234+
// but it does call isAccessibilityElement if FKA is on,
235+
// so the isAccessibilityElement implementation above will be called
236+
// when the view appears and the accessibility information will most likely
237+
// be available by the time the user starts to interact with the app using FKA.
238+
//
239+
// See SemanticsObject+UIFocusSystem.mm for more details.
240+
- (NSArray<id<UIFocusItem>>*)focusItemsInRect:(CGRect)rect {
241+
NSObject* rootAccessibilityElement =
242+
[self.accessibilityElements count] > 0 ? self.accessibilityElements[0] : nil;
243+
return [rootAccessibilityElement isKindOfClass:[SemanticsObjectContainer class]]
244+
? @[ [rootAccessibilityElement accessibilityElementAtIndex:0] ]
245+
: nil;
246+
}
247+
248+
- (NSArray<id<UIFocusEnvironment>>*)preferredFocusEnvironments {
249+
// Occasionally we add subviews to FlutterView (text fields for example).
250+
// These views shouldn't be directly visible to the iOS focus engine, instead
251+
// the focus engine should only interact with the designated focus items
252+
// (SemanticsObjects).
253+
return nil;
254+
}
255+
229256
@end

‎shell/platform/darwin/ios/framework/Source/SemanticsObject+UIFocusSystem.mm‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
// Copyright 2013 The Flutter Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style license that can be
3+
// found in the LICENSE file.
4+
5+
#import "SemanticsObject.h"
6+
#import "flutter/shell/platform/darwin/common/framework/Headers/FlutterMacros.h"
7+
8+
FLUTTER_ASSERT_ARC
9+
10+
// The SemanticsObject class conforms to UIFocusItem and UIFocusItemContainer
11+
// protocols, so the SemanticsObject tree can also be used to represent
12+
// interactive UI components on screen that can receive UIFocusSystem focus.
13+
//
14+
// Typically, physical key events received by the FlutterViewController is
15+
// first delivered to the framework, but that stopped working for navigation keys
16+
// since iOS 15 when full keyboard access (FKA) is on, because those events are
17+
// consumed by the UIFocusSystem and never dispatched to the UIResponders in the
18+
// application (see
19+
// https://developer.apple.com/documentation/uikit/uikeycommand/3780513-wantspriorityoversystembehavior
20+
// ). FKA relies on the iOS focus engine, to enable FKA on iOS 15+, we use
21+
// SemanticsObject to provide the iOS focus engine with the required hierarchical
22+
// information and geometric context.
23+
//
24+
// The focus engine focus is different from accessibility focus, or even the
25+
// currentFocus of the Flutter FocusManager in the framework. On iOS 15+, FKA
26+
// key events are dispatched to the current iOS focus engine focus (and
27+
// translated to calls such as -[NSObject accessibilityActivate]), while most
28+
// other key events are dispatched to the framework.
29+
@interface SemanticsObject (UIFocusSystem) <UIFocusItem, UIFocusItemContainer>
30+
@end
31+
32+
@implementation SemanticsObject (UIFocusSystem)
33+
34+
#pragma mark - UIFocusEnvironment Conformance
35+
36+
- (void)setNeedsFocusUpdate {
37+
}
38+
39+
- (void)updateFocusIfNeeded {
40+
}
41+
42+
- (BOOL)shouldUpdateFocusInContext:(UIFocusUpdateContext*)context {
43+
return YES;
44+
}
45+
46+
- (void)didUpdateFocusInContext:(UIFocusUpdateContext*)context
47+
withAnimationCoordinator:(UIFocusAnimationCoordinator*)coordinator {
48+
}
49+
50+
- (id<UIFocusEnvironment>)parentFocusEnvironment {
51+
// The root SemanticsObject node's parent is the FlutterView.
52+
return self.parent ?: self.bridge->view();
53+
}
54+
55+
- (NSArray<id<UIFocusEnvironment>>*)preferredFocusEnvironments {
56+
return nil;
57+
}
58+
59+
- (id<UIFocusItemContainer>)focusItemContainer {
60+
return self;
61+
}
62+
63+
#pragma mark - UIFocusItem Conformance
64+
65+
- (BOOL)canBecomeFocused {
66+
if ((self.node.flags & static_cast<int32_t>(flutter::SemanticsFlags::kIsHidden)) != 0) {
67+
return NO;
68+
}
69+
// Currently only supports SemanticsObjects that handle
70+
// -[NSObject accessibilityActivate].
71+
return self.node.HasAction(flutter::SemanticsAction::kTap);
72+
}
73+
74+
- (CGRect)frame {
75+
return self.accessibilityFrame;
76+
}
77+
78+
#pragma mark - UIFocusItemContainer Conformance
79+
80+
- (NSArray<id<UIFocusItem>>*)focusItemsInRect:(CGRect)rect {
81+
// It seems the iOS focus system relies heavily on focusItemsInRect
82+
// (instead of preferredFocusEnvironments) for directional navigation.
83+
//
84+
// The order of the items seems to be important, menus and dialogs become
85+
// unreachable via FKA if the returned children are organized
86+
// in hit-test order.
87+
//
88+
// This method is only supposed to return items within the given
89+
// rect but returning everything in the subtree seems to work fine.
90+
NSMutableArray<SemanticsObject*>* reversedItems =
91+
[[NSMutableArray alloc] initWithCapacity:self.childrenInHitTestOrder.count];
92+
for (NSUInteger i = 0; i < self.childrenInHitTestOrder.count; ++i) {
93+
[reversedItems
94+
addObject:self.childrenInHitTestOrder[self.childrenInHitTestOrder.count - 1 - i]];
95+
}
96+
return reversedItems;
97+
}
98+
99+
- (id<UICoordinateSpace>)coordinateSpace {
100+
return self.bridge->view();
101+
}
102+
@end

‎shell/platform/darwin/ios/framework/Source/SemanticsObjectTest.mm‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,9 @@
1616

1717
const float kFloatCompareEpsilon = 0.001;
1818

19+
@interface SemanticsObject (UIFocusSystem) <UIFocusItem, UIFocusItemContainer>
20+
@end
21+
1922
@interface TextInputSemanticsObject (Test)
2023
- (UIView<UITextInput>*)textInputSurrogate;
2124
@end
@@ -1152,4 +1155,55 @@ - (void)testTextInputSemanticsObject_editActions {
11521155
[self waitForExpectationsWithTimeout:1 handler:nil];
11531156
}
11541157

1158+
- (void)testUIFocusItemConformance {
1159+
fml::WeakPtrFactory<flutter::AccessibilityBridgeIos> factory(
1160+
new flutter::testing::MockAccessibilityBridge());
1161+
fml::WeakPtr<flutter::AccessibilityBridgeIos> bridge = factory.GetWeakPtr();
1162+
SemanticsObject* parent = [[SemanticsObject alloc] initWithBridge:bridge uid:0];
1163+
SemanticsObject* child = [[SemanticsObject alloc] initWithBridge:bridge uid:1];
1164+
parent.children = @[ child ];
1165+
1166+
// parentFocusEnvironment
1167+
XCTAssertTrue([parent.parentFocusEnvironment isKindOfClass:[UIView class]]);
1168+
XCTAssertEqual(child.parentFocusEnvironment, child.parent);
1169+
1170+
// canBecomeFocused
1171+
flutter::SemanticsNode childNode;
1172+
childNode.flags = static_cast<int32_t>(flutter::SemanticsFlags::kIsHidden);
1173+
childNode.actions = static_cast<int32_t>(flutter::SemanticsAction::kTap);
1174+
[child setSemanticsNode:&childNode];
1175+
XCTAssertFalse(child.canBecomeFocused);
1176+
childNode.flags = 0;
1177+
[child setSemanticsNode:&childNode];
1178+
XCTAssertTrue(child.canBecomeFocused);
1179+
childNode.actions = 0;
1180+
[child setSemanticsNode:&childNode];
1181+
XCTAssertFalse(child.canBecomeFocused);
1182+
1183+
CGFloat scale = ((bridge->view().window.screen ?: UIScreen.mainScreen)).scale;
1184+
1185+
childNode.rect = SkRect::MakeXYWH(0, 0, 100 * scale, 100 * scale);
1186+
[child setSemanticsNode:&childNode];
1187+
flutter::SemanticsNode parentNode;
1188+
parentNode.rect = SkRect::MakeXYWH(0, 0, 200, 200);
1189+
[parent setSemanticsNode:&parentNode];
1190+
1191+
XCTAssertTrue(CGRectEqualToRect(child.frame, CGRectMake(0, 0, 100, 100)));
1192+
}
1193+
1194+
- (void)testUIFocusItemContainerConformance {
1195+
fml::WeakPtrFactory<flutter::AccessibilityBridgeIos> factory(
1196+
new flutter::testing::MockAccessibilityBridge());
1197+
fml::WeakPtr<flutter::AccessibilityBridgeIos> bridge = factory.GetWeakPtr();
1198+
SemanticsObject* parent = [[SemanticsObject alloc] initWithBridge:bridge uid:0];
1199+
SemanticsObject* child1 = [[SemanticsObject alloc] initWithBridge:bridge uid:1];
1200+
SemanticsObject* child2 = [[SemanticsObject alloc] initWithBridge:bridge uid:2];
1201+
parent.childrenInHitTestOrder = @[ child1, child2 ];
1202+
1203+
// focusItemsInRect
1204+
NSArray<id<UIFocusItem>>* itemsInRect = [parent focusItemsInRect:CGRectMake(0, 0, 100, 100)];
1205+
XCTAssertEqual(itemsInRect.count, (unsigned long)2);
1206+
XCTAssertTrue([itemsInRect containsObject:child1]);
1207+
XCTAssertTrue([itemsInRect containsObject:child2]);
1208+
}
11551209
@end

0 commit comments

Comments
 (0)