Reland ios accessibility scrolling support (#26860)

Author: chunhtai

lib/ui/semantics/semantics_node.h

@@ -44,12 +44,17 @@ enum class SemanticsAction : int32_t {
   kSetText = 1 << 21,
 };
 
-const int kScrollableSemanticsActions =
-    static_cast<int32_t>(SemanticsAction::kScrollLeft) |
-    static_cast<int32_t>(SemanticsAction::kScrollRight) |
+const int kVerticalScrollSemanticsActions =
     static_cast<int32_t>(SemanticsAction::kScrollUp) |
     static_cast<int32_t>(SemanticsAction::kScrollDown);
 
+const int kHorizontalScrollSemanticsActions =
+    static_cast<int32_t>(SemanticsAction::kScrollLeft) |
+    static_cast<int32_t>(SemanticsAction::kScrollRight);
+
+const int kScrollableSemanticsActions =
+    kVerticalScrollSemanticsActions | kHorizontalScrollSemanticsActions;
+
 /// C/C++ representation of `SemanticsFlags` defined in
 /// `lib/ui/semantics.dart`.
 ///\warning This must match the `SemanticsFlags` enum in

shell/platform/darwin/ios/framework/Source/SemanticsObject.h

@@ -13,6 +13,8 @@
 #import "flutter/shell/platform/darwin/ios/framework/Source/accessibility_bridge_ios.h"
 
 constexpr int32_t kRootNodeId = 0;
+// This can be arbitrary number as long as it is bigger than 0.
+constexpr float kScrollExtentMaxForInf = 1000;
 
 @class FlutterCustomAccessibilityAction;
 @class FlutterPlatformViewSemanticsContainer;
@@ -31,7 +33,7 @@ constexpr int32_t kRootNodeId = 0;
  * The parent of this node in the node tree. Will be nil for the root node and
  * during transient state changes.
  */
-@property(nonatomic, readonly) SemanticsObject* parent;
+@property(nonatomic, assign) SemanticsObject* parent;
 
 /**
  * The accessibility bridge that this semantics object is attached to. This
@@ -94,6 +96,14 @@ constexpr int32_t kRootNodeId = 0;
 
 - (BOOL)onCustomAccessibilityAction:(FlutterCustomAccessibilityAction*)action;
 
+/**
+ * Called after accessibility bridge finishes a semantics update.
+ *
+ * Subclasses can override this method if they contain states that can only be
+ * updated once every node in the accessibility tree has finished updating.
+ */
+- (void)accessibilityBridgeDidFinishUpdate;
+
 #pragma mark - Designated initializers
 
 - (instancetype)init __attribute__((unavailable("Use initWithBridge instead")));
@@ -159,6 +169,18 @@ constexpr int32_t kRootNodeId = 0;
 
 @end
 
+/// The semantics object for scrollable. This class creates an UIScrollView to interact with the
+/// iOS.
+@interface FlutterScrollableSemanticsObject : UIScrollView
+
+- (instancetype)init NS_UNAVAILABLE;
+- (instancetype)initWithFrame:(CGRect)frame NS_UNAVAILABLE;
+- (instancetype)initWithCoder:(NSCoder*)coder NS_UNAVAILABLE;
+- (instancetype)initWithSemanticsObject:(SemanticsObject*)semanticsObject NS_DESIGNATED_INITIALIZER;
+- (void)accessibilityBridgeDidFinishUpdate;
+
+@end
+
 /**
  * Represents a semantics object that has children and hence has to be presented to the OS as a
  * UIAccessibilityContainer.

shell/platform/darwin/ios/framework/Source/SemanticsObject.mm

@@ -34,6 +34,58 @@
   return flutter::SemanticsAction::kScrollUp;
 }
 
+SkM44 GetGlobalTransform(SemanticsObject* reference) {
+  SkM44 globalTransform = [reference node].transform;
+  for (SemanticsObject* parent = [reference parent]; parent; parent = parent.parent) {
+    globalTransform = parent.node.transform * globalTransform;
+  }
+  return globalTransform;
+}
+
+SkPoint ApplyTransform(SkPoint& point, const SkM44& transform) {
+  SkV4 vector = transform.map(point.x(), point.y(), 0, 1);
+  return SkPoint::Make(vector.x / vector.w, vector.y / vector.w);
+}
+
+CGPoint ConvertPointToGlobal(SemanticsObject* reference, CGPoint local_point) {
+  SkM44 globalTransform = GetGlobalTransform(reference);
+  SkPoint point = SkPoint::Make(local_point.x, local_point.y);
+  point = ApplyTransform(point, globalTransform);
+  // `rect` is in the physical pixel coordinate system. iOS expects the accessibility frame in
+  // the logical pixel coordinate system. Therefore, we divide by the `scale` (pixel ratio) to
+  // convert.
+  CGFloat scale = [[[reference bridge]->view() window] screen].scale;
+  auto result = CGPointMake(point.x() / scale, point.y() / scale);
+  return [[reference bridge]->view() convertPoint:result toView:nil];
+}
+
+CGRect ConvertRectToGlobal(SemanticsObject* reference, CGRect local_rect) {
+  SkM44 globalTransform = GetGlobalTransform(reference);
+
+  SkPoint quad[4] = {
+      SkPoint::Make(local_rect.origin.x, local_rect.origin.y),                          // top left
+      SkPoint::Make(local_rect.origin.x + local_rect.size.width, local_rect.origin.y),  // top right
+      SkPoint::Make(local_rect.origin.x + local_rect.size.width,
+                    local_rect.origin.y + local_rect.size.height),  // bottom right
+      SkPoint::Make(local_rect.origin.x,
+                    local_rect.origin.y + local_rect.size.height)  // bottom left
+  };
+  for (auto& point : quad) {
+    point = ApplyTransform(point, globalTransform);
+  }
+  SkRect rect;
+  NSCAssert(rect.setBoundsCheck(quad, 4), @"Transformed points can't form a rect");
+  rect.setBounds(quad, 4);
+
+  // `rect` is in the physical pixel coordinate system. iOS expects the accessibility frame in
+  // the logical pixel coordinate system. Therefore, we divide by the `scale` (pixel ratio) to
+  // convert.
+  CGFloat scale = [[[reference bridge]->view() window] screen].scale;
+  auto result =
+      CGRectMake(rect.x() / scale, rect.y() / scale, rect.width() / scale, rect.height() / scale);
+  return UIAccessibilityConvertFrameToScreenCoordinates(result, [reference bridge]->view());
+}
+
 }  // namespace
 
 @implementation FlutterSwitchSemanticsObject {
@@ -88,6 +140,175 @@ - (UIAccessibilityTraits)accessibilityTraits {
 
 @end  // FlutterSwitchSemanticsObject
 
+@interface FlutterScrollableSemanticsObject ()
+@property(nonatomic, strong) SemanticsObject* semanticsObject;
+@end
+
+@implementation FlutterScrollableSemanticsObject {
+  fml::scoped_nsobject<SemanticsObjectContainer> _container;
+}
+
+- (instancetype)initWithSemanticsObject:(SemanticsObject*)semanticsObject {
+  self = [super initWithFrame:CGRectZero];
+  if (self) {
+    _semanticsObject = [semanticsObject retain];
+    [semanticsObject.bridge->view() addSubview:self];
+  }
+  return self;
+}
+
+- (void)dealloc {
+  _container.get().semanticsObject = nil;
+  [_semanticsObject release];
+  [self removeFromSuperview];
+  [super dealloc];
+}
+
+- (UIView*)hitTest:(CGPoint)point withEvent:(UIEvent*)event {
+  return nil;
+}
+
+- (NSMethodSignature*)methodSignatureForSelector:(SEL)sel {
+  NSMethodSignature* result = [super methodSignatureForSelector:sel];
+  if (!result) {
+    result = [_semanticsObject methodSignatureForSelector:sel];
+  }
+  return result;
+}
+
+- (void)forwardInvocation:(NSInvocation*)anInvocation {
+  [anInvocation setTarget:_semanticsObject];
+  [anInvocation invoke];
+}
+
+- (void)accessibilityBridgeDidFinishUpdate {
+  // In order to make iOS think this UIScrollView is scrollable, the following
+  // requirements must be true.
+  // 1. contentSize must be bigger than the frame size.
+  // 2. The scrollable isAccessibilityElement must return YES
+  //
+  // Once the requirements are met, the iOS uses contentOffset to determine
+  // what scroll actions are available. e.g. If the view scrolls vertically and
+  // contentOffset is 0.0, only the scroll down action is available.
+  [self setFrame:[_semanticsObject accessibilityFrame]];
+  [self setContentSize:[self contentSizeInternal]];
+  [self setContentOffset:[self contentOffsetInternal] animated:NO];
+  if (self.contentSize.width > self.frame.size.width ||
+      self.contentSize.height > self.frame.size.height) {
+    self.isAccessibilityElement = YES;
+  } else {
+    self.isAccessibilityElement = NO;
+  }
+}
+
+- (void)setChildren:(NSArray<SemanticsObject*>*)children {
+  [_semanticsObject setChildren:children];
+  // The children's parent is pointing to _semanticsObject, need to manually
+  // set it this object.
+  for (SemanticsObject* child in _semanticsObject.children) {
+    child.parent = (SemanticsObject*)self;
+  }
+}
+
+- (id)accessibilityContainer {
+  if (_container == nil) {
+    _container.reset([[SemanticsObjectContainer alloc]
+        initWithSemanticsObject:(SemanticsObject*)self
+                         bridge:[_semanticsObject bridge]]);
+  }
+  return _container.get();
+}
+
+// private methods
+
+- (float)scrollExtentMax {
+  if (![_semanticsObject isAccessibilityBridgeAlive]) {
+    return 0.0f;
+  }
+  float scrollExtentMax = _semanticsObject.node.scrollExtentMax;
+  if (isnan(scrollExtentMax)) {
+    scrollExtentMax = 0.0f;
+  } else if (!isfinite(scrollExtentMax)) {
+    scrollExtentMax = kScrollExtentMaxForInf + [self scrollPosition];
+  }
+  return scrollExtentMax;
+}
+
+- (float)scrollPosition {
+  if (![_semanticsObject isAccessibilityBridgeAlive]) {
+    return 0.0f;
+  }
+  float scrollPosition = _semanticsObject.node.scrollPosition;
+  if (isnan(scrollPosition)) {
+    scrollPosition = 0.0f;
+  }
+  NSCAssert(isfinite(scrollPosition), @"The scrollPosition must not be infinity");
+  return scrollPosition;
+}
+
+- (CGSize)contentSizeInternal {
+  CGRect result;
+  const SkRect& rect = _semanticsObject.node.rect;
+
+  if (_semanticsObject.node.actions & flutter::kVerticalScrollSemanticsActions) {
+    result = CGRectMake(rect.x(), rect.y(), rect.width(), rect.height() + [self scrollExtentMax]);
+  } else if (_semanticsObject.node.actions & flutter::kHorizontalScrollSemanticsActions) {
+    result = CGRectMake(rect.x(), rect.y(), rect.width() + [self scrollExtentMax], rect.height());
+  } else {
+    result = CGRectMake(rect.x(), rect.y(), rect.width(), rect.height());
+  }
+  return ConvertRectToGlobal(_semanticsObject, result).size;
+}
+
+- (CGPoint)contentOffsetInternal {
+  CGPoint result;
+  CGPoint origin = self.frame.origin;
+  const SkRect& rect = _semanticsObject.node.rect;
+  if (_semanticsObject.node.actions & flutter::kVerticalScrollSemanticsActions) {
+    result = ConvertPointToGlobal(_semanticsObject,
+                                  CGPointMake(rect.x(), rect.y() + [self scrollPosition]));
+  } else if (_semanticsObject.node.actions & flutter::kHorizontalScrollSemanticsActions) {
+    result = ConvertPointToGlobal(_semanticsObject,
+                                  CGPointMake(rect.x() + [self scrollPosition], rect.y()));
+  } else {
+    result = origin;
+  }
+  return CGPointMake(result.x - origin.x, result.y - origin.y);
+}
+
+// The following methods are explicitly forwarded to the wrapped SemanticsObject because the
+// forwarding logic above doesn't apply to them since they are also implemented in the
+// UIScrollView class, the base class.
+
+- (BOOL)accessibilityActivate {
+  return [_semanticsObject accessibilityActivate];
+}
+
+- (void)accessibilityIncrement {
+  [_semanticsObject accessibilityIncrement];
+}
+
+- (void)accessibilityDecrement {
+  [_semanticsObject accessibilityDecrement];
+}
+
+- (BOOL)accessibilityScroll:(UIAccessibilityScrollDirection)direction {
+  return [_semanticsObject accessibilityScroll:direction];
+}
+
+- (BOOL)accessibilityPerformEscape {
+  return [_semanticsObject accessibilityPerformEscape];
+}
+
+- (void)accessibilityElementDidBecomeFocused {
+  [_semanticsObject accessibilityElementDidBecomeFocused];
+}
+
+- (void)accessibilityElementDidLoseFocus {
+  [_semanticsObject accessibilityElementDidLoseFocus];
+}
+@end  // FlutterScrollableSemanticsObject
+
 @implementation FlutterCustomAccessibilityAction {
 }
 @end
@@ -174,6 +395,9 @@ - (void)setSemanticsNode:(const flutter::SemanticsNode*)node {
   _node = *node;
 }
 
+- (void)accessibilityBridgeDidFinishUpdate { /* Do nothing by default */
+}
+
 /**
  * Whether calling `setSemanticsNode:` with `node` would cause a layout change.
  */
@@ -398,27 +622,9 @@ - (CGRect)accessibilityFrame {
 }
 
 - (CGRect)globalRect {
-  SkM44 globalTransform = [self node].transform;
-  for (SemanticsObject* parent = [self parent]; parent; parent = parent.parent) {
-    globalTransform = parent.node.transform * globalTransform;
-  }
-
-  SkPoint quad[4];
-  [self node].rect.toQuad(quad);
-  for (auto& point : quad) {
-    SkV4 vector = globalTransform.map(point.x(), point.y(), 0, 1);
-    point.set(vector.x / vector.w, vector.y / vector.w);
-  }
-  SkRect rect;
-  rect.setBounds(quad, 4);
-
-  // `rect` is in the physical pixel coordinate system. iOS expects the accessibility frame in
-  // the logical pixel coordinate system. Therefore, we divide by the `scale` (pixel ratio) to
-  // convert.
-  CGFloat scale = [[[self bridge]->view() window] screen].scale;
-  auto result =
-      CGRectMake(rect.x() / scale, rect.y() / scale, rect.width() / scale, rect.height() / scale);
-  return UIAccessibilityConvertFrameToScreenCoordinates(result, [self bridge]->view());
+  const SkRect& rect = [self node].rect;
+  CGRect localRect = CGRectMake(rect.x(), rect.y(), rect.width(), rect.height());
+  return ConvertRectToGlobal(self, localRect);
 }
 
 #pragma mark - UIAccessibilityElement protocol