From 7cfa1ca9d4528a62b0683cb3922c3bad0732b43c Mon Sep 17 00:00:00 2001 From: Chun-Heng Tai Date: Thu, 25 Feb 2021 13:59:17 -0800 Subject: [PATCH 1/7] Enables semantics when voice control is turned on --- shell/platform/darwin/ios/BUILD.gn | 1 + .../darwin/ios/framework/Source/FlutterView.h | 1 + .../ios/framework/Source/FlutterView.mm | 15 ++++++ .../ios/framework/Source/FlutterViewTest.mm | 50 +++++++++++++++++++ 4 files changed, 67 insertions(+) create mode 100644 shell/platform/darwin/ios/framework/Source/FlutterViewTest.mm diff --git a/shell/platform/darwin/ios/BUILD.gn b/shell/platform/darwin/ios/BUILD.gn index dafb1dce0ed92..4402d1d13641e 100644 --- a/shell/platform/darwin/ios/BUILD.gn +++ b/shell/platform/darwin/ios/BUILD.gn @@ -186,6 +186,7 @@ source_set("ios_test_flutter_mrc") { "framework/Source/FlutterEngineTest_mrc.mm", "framework/Source/FlutterPlatformPluginTest.mm", "framework/Source/FlutterPlatformViewsTest.mm", + "framework/Source/FlutterViewTest.mm", "framework/Source/accessibility_bridge_test.mm", ] deps = [ diff --git a/shell/platform/darwin/ios/framework/Source/FlutterView.h b/shell/platform/darwin/ios/framework/Source/FlutterView.h index 0cdd56d2cfbdc..4922b09606f73 100644 --- a/shell/platform/darwin/ios/framework/Source/FlutterView.h +++ b/shell/platform/darwin/ios/framework/Source/FlutterView.h @@ -21,6 +21,7 @@ asBase64Encoded:(BOOL)base64Encode; - (std::shared_ptr&)platformViewsController; +- (void)ensureSemanticsEnabled; @end @interface FlutterView : UIView diff --git a/shell/platform/darwin/ios/framework/Source/FlutterView.mm b/shell/platform/darwin/ios/framework/Source/FlutterView.mm index 73a9b8b529088..05bb8132297cb 100644 --- a/shell/platform/darwin/ios/framework/Source/FlutterView.mm +++ b/shell/platform/darwin/ios/framework/Source/FlutterView.mm @@ -130,4 +130,19 @@ - (void)drawLayer:(CALayer*)layer inContext:(CGContextRef)context { CGContextRestoreGState(context); } +- (BOOL)isAccessibilityElement { + // iOS does not provide an API to query whether the voice control + // is turned on or off. It is likely at least one of the assitive + // technologies is turned on if this method is called. If we do + // not catch it in notification center, we will catch it here. + // + // TODO(chunhtai): Remove this workaround once iOS provides an + // API to query whether voice control is enabled. + // https://github.com/flutter/flutter/issues/76808. + if (self.accessibilityElements == nil) { + [_delegate ensureSemanticsEnabled]; + } + return NO; +} + @end diff --git a/shell/platform/darwin/ios/framework/Source/FlutterViewTest.mm b/shell/platform/darwin/ios/framework/Source/FlutterViewTest.mm new file mode 100644 index 0000000000000..342ed52694776 --- /dev/null +++ b/shell/platform/darwin/ios/framework/Source/FlutterViewTest.mm @@ -0,0 +1,50 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#import + +#import "flutter/shell/platform/darwin/ios/framework/Source/FlutterView.h" + +@interface FakeDelegate : NSObject +@property(nonatomic) BOOL ensureSemanticsEnabledCalled; +@end + +@implementation FakeDelegate { + std::shared_ptr _platformViewsController; +} + +- (instancetype)init{ + _ensureSemanticsEnabledCalled = NO; + _platformViewsController = std::shared_ptr(nullptr); + return self; +} + +- (flutter::Rasterizer::Screenshot)takeScreenshot:(flutter::Rasterizer::ScreenshotType)type + asBase64Encoded:(BOOL)base64Encode { + return {}; +} + +- (std::shared_ptr&)platformViewsController { + return _platformViewsController; +} +- (void)ensureSemanticsEnabled { + _ensureSemanticsEnabledCalled = YES; +} +@end + +@interface FlutterViewTest : XCTestCase +@end + +@implementation FlutterViewTest + +- (void)testFlutterViewEnableSemanticsWhenIsAccessibilityElementIsCalled { + FakeDelegate* delegate = [[FakeDelegate alloc] init]; + FlutterView* view = [[FlutterView alloc] initWithDelegate:delegate + opaque:NO]; + delegate.ensureSemanticsEnabledCalled = NO; + XCTAssertFalse(view.isAccessibilityElement); + XCTAssertTrue(delegate.ensureSemanticsEnabledCalled); +} + +@end From 269576cb04f3e1b4eac4286ea04e6752e1d3524f Mon Sep 17 00:00:00 2001 From: Chun-Heng Tai Date: Mon, 1 Mar 2021 13:21:31 -0800 Subject: [PATCH 2/7] fix license --- ci/licenses_golden/licenses_flutter | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/licenses_golden/licenses_flutter b/ci/licenses_golden/licenses_flutter index 75865d537786f..e69eaee8688b4 100755 --- a/ci/licenses_golden/licenses_flutter +++ b/ci/licenses_golden/licenses_flutter @@ -1036,6 +1036,7 @@ FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/FlutterView.mm FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/FlutterViewController.mm FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/FlutterViewControllerTest.mm FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/FlutterViewController_Internal.h +FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/FlutterViewTest.mm FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/IOKit.h FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/SemanticsObject.h FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/SemanticsObject.mm From c5b17af0189d82e2c3d8be23656ada20faa08d7e Mon Sep 17 00:00:00 2001 From: Chun-Heng Tai Date: Tue, 2 Mar 2021 13:45:24 -0800 Subject: [PATCH 3/7] update --- .../ios/framework/Source/FlutterEngine.mm | 8 +++++- .../darwin/ios/framework/Source/FlutterView.h | 11 +++++++- .../ios/framework/Source/FlutterView.mm | 7 ++--- .../ios/framework/Source/FlutterViewTest.mm | 27 +++++-------------- 4 files changed, 27 insertions(+), 26 deletions(-) diff --git a/shell/platform/darwin/ios/framework/Source/FlutterEngine.mm b/shell/platform/darwin/ios/framework/Source/FlutterEngine.mm index 4585c392b3eac..915445cc1c0da 100644 --- a/shell/platform/darwin/ios/framework/Source/FlutterEngine.mm +++ b/shell/platform/darwin/ios/framework/Source/FlutterEngine.mm @@ -761,7 +761,7 @@ - (void)showAutocorrectionPromptRectForStart:(NSUInteger)start arguments:@[ @(client), @(start), @(end) ]]; } -#pragma mark - Screenshot Delegate +#pragma mark - FlutterViewEngineDelegate - (flutter::Rasterizer::Screenshot)takeScreenshot:(flutter::Rasterizer::ScreenshotType)type asBase64Encoded:(BOOL)base64Encode { @@ -769,6 +769,12 @@ - (void)showAutocorrectionPromptRectForStart:(NSUInteger)start return _shell->Screenshot(type, base64Encode); } +- (void)futterViewAccessibilityDidCall { + if (self.viewController.view.accessibilityElements == nil) { + [self ensureSemanticsEnabled]; + } +} + - (NSObject*)binaryMessenger { return _binaryMessenger; } diff --git a/shell/platform/darwin/ios/framework/Source/FlutterView.h b/shell/platform/darwin/ios/framework/Source/FlutterView.h index 4922b09606f73..106efd47be69e 100644 --- a/shell/platform/darwin/ios/framework/Source/FlutterView.h +++ b/shell/platform/darwin/ios/framework/Source/FlutterView.h @@ -21,7 +21,16 @@ asBase64Encoded:(BOOL)base64Encode; - (std::shared_ptr&)platformViewsController; -- (void)ensureSemanticsEnabled; + +/** + * A callback that is called when iOS queries accessibility information of the Flutter view. + * + * This is useful to predict the current iOS accessibility status. For example, there is + * no API to listen whether voice control is turned on or off. The Flutter engine uses + * this callback to enable semantics in order to catch the case that voice control is + * on. + */ +- (void)futterViewAccessibilityDidCall; @end @interface FlutterView : UIView diff --git a/shell/platform/darwin/ios/framework/Source/FlutterView.mm b/shell/platform/darwin/ios/framework/Source/FlutterView.mm index 05bb8132297cb..6d0f019606541 100644 --- a/shell/platform/darwin/ios/framework/Source/FlutterView.mm +++ b/shell/platform/darwin/ios/framework/Source/FlutterView.mm @@ -139,9 +139,10 @@ - (BOOL)isAccessibilityElement { // TODO(chunhtai): Remove this workaround once iOS provides an // API to query whether voice control is enabled. // https://github.com/flutter/flutter/issues/76808. - if (self.accessibilityElements == nil) { - [_delegate ensureSemanticsEnabled]; - } + [_delegate futterViewAccessibilityDidCall]; + // if (self.accessibilityElements == nil) { + // [_delegate futterViewAccessibilityDidCall]; + // } return NO; } diff --git a/shell/platform/darwin/ios/framework/Source/FlutterViewTest.mm b/shell/platform/darwin/ios/framework/Source/FlutterViewTest.mm index 342ed52694776..4b5d52c3bb086 100644 --- a/shell/platform/darwin/ios/framework/Source/FlutterViewTest.mm +++ b/shell/platform/darwin/ios/framework/Source/FlutterViewTest.mm @@ -4,33 +4,19 @@ #import +#import "flutter/shell/platform/darwin/ios/framework/Source/FlutterEngine_Internal.h" #import "flutter/shell/platform/darwin/ios/framework/Source/FlutterView.h" -@interface FakeDelegate : NSObject +@interface FakeDelegate : FlutterEngine @property(nonatomic) BOOL ensureSemanticsEnabledCalled; @end -@implementation FakeDelegate { - std::shared_ptr _platformViewsController; -} - -- (instancetype)init{ - _ensureSemanticsEnabledCalled = NO; - _platformViewsController = std::shared_ptr(nullptr); - return self; -} +@implementation FakeDelegate -- (flutter::Rasterizer::Screenshot)takeScreenshot:(flutter::Rasterizer::ScreenshotType)type - asBase64Encoded:(BOOL)base64Encode { - return {}; -} - -- (std::shared_ptr&)platformViewsController { - return _platformViewsController; -} - (void)ensureSemanticsEnabled { _ensureSemanticsEnabledCalled = YES; } + @end @interface FlutterViewTest : XCTestCase @@ -39,9 +25,8 @@ @interface FlutterViewTest : XCTestCase @implementation FlutterViewTest - (void)testFlutterViewEnableSemanticsWhenIsAccessibilityElementIsCalled { - FakeDelegate* delegate = [[FakeDelegate alloc] init]; - FlutterView* view = [[FlutterView alloc] initWithDelegate:delegate - opaque:NO]; + FakeDelegate* delegate = [[FakeDelegate alloc] initWithName:@"foobar"]; + FlutterView* view = [[FlutterView alloc] initWithDelegate:delegate opaque:NO]; delegate.ensureSemanticsEnabledCalled = NO; XCTAssertFalse(view.isAccessibilityElement); XCTAssertTrue(delegate.ensureSemanticsEnabledCalled); From 6c411373640ac88aedafd92a4510c8f9ed6f3d29 Mon Sep 17 00:00:00 2001 From: Chun-Heng Tai Date: Tue, 2 Mar 2021 13:47:22 -0800 Subject: [PATCH 4/7] typo --- shell/platform/darwin/ios/framework/Source/FlutterEngine.mm | 2 +- shell/platform/darwin/ios/framework/Source/FlutterView.mm | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/shell/platform/darwin/ios/framework/Source/FlutterEngine.mm b/shell/platform/darwin/ios/framework/Source/FlutterEngine.mm index 915445cc1c0da..23e56fdc41dc7 100644 --- a/shell/platform/darwin/ios/framework/Source/FlutterEngine.mm +++ b/shell/platform/darwin/ios/framework/Source/FlutterEngine.mm @@ -769,7 +769,7 @@ - (void)showAutocorrectionPromptRectForStart:(NSUInteger)start return _shell->Screenshot(type, base64Encode); } -- (void)futterViewAccessibilityDidCall { +- (void)flutterViewAccessibilityDidCall { if (self.viewController.view.accessibilityElements == nil) { [self ensureSemanticsEnabled]; } diff --git a/shell/platform/darwin/ios/framework/Source/FlutterView.mm b/shell/platform/darwin/ios/framework/Source/FlutterView.mm index 6d0f019606541..00ac8f042a3fc 100644 --- a/shell/platform/darwin/ios/framework/Source/FlutterView.mm +++ b/shell/platform/darwin/ios/framework/Source/FlutterView.mm @@ -139,9 +139,9 @@ - (BOOL)isAccessibilityElement { // TODO(chunhtai): Remove this workaround once iOS provides an // API to query whether voice control is enabled. // https://github.com/flutter/flutter/issues/76808. - [_delegate futterViewAccessibilityDidCall]; + [_delegate flutterViewAccessibilityDidCall]; // if (self.accessibilityElements == nil) { - // [_delegate futterViewAccessibilityDidCall]; + // [_delegate flutterViewAccessibilityDidCall]; // } return NO; } From 34f34363eafe3a8b8e6bce239040b1cc682d6e47 Mon Sep 17 00:00:00 2001 From: Chun-Heng Tai Date: Tue, 2 Mar 2021 13:48:20 -0800 Subject: [PATCH 5/7] update --- shell/platform/darwin/ios/framework/Source/FlutterView.mm | 3 --- 1 file changed, 3 deletions(-) diff --git a/shell/platform/darwin/ios/framework/Source/FlutterView.mm b/shell/platform/darwin/ios/framework/Source/FlutterView.mm index 00ac8f042a3fc..36b1bbb346e3e 100644 --- a/shell/platform/darwin/ios/framework/Source/FlutterView.mm +++ b/shell/platform/darwin/ios/framework/Source/FlutterView.mm @@ -140,9 +140,6 @@ - (BOOL)isAccessibilityElement { // API to query whether voice control is enabled. // https://github.com/flutter/flutter/issues/76808. [_delegate flutterViewAccessibilityDidCall]; - // if (self.accessibilityElements == nil) { - // [_delegate flutterViewAccessibilityDidCall]; - // } return NO; } From 9afca0a2d8e456715b7909b646482799710b0787 Mon Sep 17 00:00:00 2001 From: Chun-Heng Tai Date: Wed, 3 Mar 2021 09:46:15 -0800 Subject: [PATCH 6/7] fix typo --- shell/platform/darwin/ios/framework/Source/FlutterView.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shell/platform/darwin/ios/framework/Source/FlutterView.h b/shell/platform/darwin/ios/framework/Source/FlutterView.h index 106efd47be69e..a795953c823f4 100644 --- a/shell/platform/darwin/ios/framework/Source/FlutterView.h +++ b/shell/platform/darwin/ios/framework/Source/FlutterView.h @@ -30,7 +30,7 @@ * this callback to enable semantics in order to catch the case that voice control is * on. */ -- (void)futterViewAccessibilityDidCall; +- (void)flutterViewAccessibilityDidCall; @end @interface FlutterView : UIView From 81c07b00fc2ebdefec209cf0537519ab9554722e Mon Sep 17 00:00:00 2001 From: Chun-Heng Tai Date: Wed, 3 Mar 2021 16:20:57 -0800 Subject: [PATCH 7/7] fix tests --- .../framework/Source/FlutterEngineTest_mrc.mm | 19 +++++++++++ .../ios/framework/Source/FlutterViewTest.mm | 33 ++++++++++++++----- 2 files changed, 44 insertions(+), 8 deletions(-) diff --git a/shell/platform/darwin/ios/framework/Source/FlutterEngineTest_mrc.mm b/shell/platform/darwin/ios/framework/Source/FlutterEngineTest_mrc.mm index 46a7e4abac965..a13b669b13c2f 100644 --- a/shell/platform/darwin/ios/framework/Source/FlutterEngineTest_mrc.mm +++ b/shell/platform/darwin/ios/framework/Source/FlutterEngineTest_mrc.mm @@ -12,6 +12,18 @@ FLUTTER_ASSERT_NOT_ARC +@interface FlutterEngineSpy : FlutterEngine +@property(nonatomic) BOOL ensureSemanticsEnabledCalled; +@end + +@implementation FlutterEngineSpy + +- (void)ensureSemanticsEnabled { + _ensureSemanticsEnabledCalled = YES; +} + +@end + @interface FlutterEngineTest_mrc : XCTestCase @end @@ -41,4 +53,11 @@ - (void)testSpawnsShareGpuContext { [spawn release]; } +- (void)testEnableSemanticsWhenFlutterViewAccessibilityDidCall { + FlutterEngineSpy* engine = [[FlutterEngineSpy alloc] initWithName:@"foobar"]; + engine.ensureSemanticsEnabledCalled = NO; + [engine flutterViewAccessibilityDidCall]; + XCTAssertTrue(engine.ensureSemanticsEnabledCalled); +} + @end diff --git a/shell/platform/darwin/ios/framework/Source/FlutterViewTest.mm b/shell/platform/darwin/ios/framework/Source/FlutterViewTest.mm index 4b5d52c3bb086..c14bee16a6104 100644 --- a/shell/platform/darwin/ios/framework/Source/FlutterViewTest.mm +++ b/shell/platform/darwin/ios/framework/Source/FlutterViewTest.mm @@ -7,14 +7,31 @@ #import "flutter/shell/platform/darwin/ios/framework/Source/FlutterEngine_Internal.h" #import "flutter/shell/platform/darwin/ios/framework/Source/FlutterView.h" -@interface FakeDelegate : FlutterEngine -@property(nonatomic) BOOL ensureSemanticsEnabledCalled; +@interface FakeDelegate : NSObject +@property(nonatomic) BOOL callbackCalled; @end -@implementation FakeDelegate +@implementation FakeDelegate { + std::shared_ptr _platformViewsController; +} + +- (instancetype)init { + _callbackCalled = NO; + _platformViewsController = std::shared_ptr(nullptr); + return self; +} + +- (flutter::Rasterizer::Screenshot)takeScreenshot:(flutter::Rasterizer::ScreenshotType)type + asBase64Encoded:(BOOL)base64Encode { + return {}; +} + +- (std::shared_ptr&)platformViewsController { + return _platformViewsController; +} -- (void)ensureSemanticsEnabled { - _ensureSemanticsEnabledCalled = YES; +- (void)flutterViewAccessibilityDidCall { + _callbackCalled = YES; } @end @@ -25,11 +42,11 @@ @interface FlutterViewTest : XCTestCase @implementation FlutterViewTest - (void)testFlutterViewEnableSemanticsWhenIsAccessibilityElementIsCalled { - FakeDelegate* delegate = [[FakeDelegate alloc] initWithName:@"foobar"]; + FakeDelegate* delegate = [[FakeDelegate alloc] init]; FlutterView* view = [[FlutterView alloc] initWithDelegate:delegate opaque:NO]; - delegate.ensureSemanticsEnabledCalled = NO; + delegate.callbackCalled = NO; XCTAssertFalse(view.isAccessibilityElement); - XCTAssertTrue(delegate.ensureSemanticsEnabledCalled); + XCTAssertTrue(delegate.callbackCalled); } @end