Skip to content
This repository was archived by the owner on Feb 25, 2025. It is now read-only.

Fixes android voice access delete text, redo, and undo actions. #25050

Merged
merged 2 commits into from
Mar 25, 2021
Merged

Fixes android voice access delete text, redo, and undo actions. #25050

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f83212a
Fixes android voice access delete text, redo, and undo actions
chunhtai Mar 12, 2021
0147ebc
account for unicode
chunhtai Mar 23, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
100 changes: 96 additions & 4 deletions shell/platform/android/io/flutter/view/AccessibilityBridge.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
* Bridge between Android's OS accessibility system and Flutter's accessibility system.
Expand Down Expand Up @@ -633,8 +635,7 @@ public AccessibilityNodeInfo createAccessibilityNodeInfo(int virtualViewId) {
}

// These are non-ops on older devices. Attempting to interact with the text will cause Talkback
// to read the
// contents of the text box instead.
// to read the contents of the text box instead.
if (Build.VERSION.SDK_INT > Build.VERSION_CODES.JELLY_BEAN_MR2) {
if (semanticsNode.hasAction(Action.SET_SELECTION)) {
result.addAction(AccessibilityNodeInfo.ACTION_SET_SELECTION);
Expand All @@ -650,6 +651,13 @@ public AccessibilityNodeInfo createAccessibilityNodeInfo(int virtualViewId) {
}
}

// Set text API isn't available until API 21.
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
if (semanticsNode.hasAction(Action.SET_TEXT)) {
result.addAction(AccessibilityNodeInfo.ACTION_SET_TEXT);
}
}

if (semanticsNode.hasFlag(Flag.IS_BUTTON) || semanticsNode.hasFlag(Flag.IS_LINK)) {
result.setClassName("android.widget.Button");
}
Expand Down Expand Up @@ -1034,6 +1042,12 @@ public boolean performAction(
}
accessibilityChannel.dispatchSemanticsAction(
virtualViewId, Action.SET_SELECTION, selection);
// The voice access expects the semantics node to update immediately. We update the
// semantics node based on prediction. If the result is incorrect, it will be updated in
// the next frame.
SemanticsNode node = flutterSemanticsTree.get(virtualViewId);
node.textSelectionBase = selection.get("base");
node.textSelectionExtent = selection.get("extent");
return true;
}
case AccessibilityNodeInfo.ACTION_COPY:
Expand Down Expand Up @@ -1064,7 +1078,7 @@ public boolean performAction(
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.LOLLIPOP) {
return false;
}
return performSetText(virtualViewId, arguments);
return performSetText(semanticsNode, virtualViewId, arguments);
}
default:
// might be a custom accessibility accessibilityAction.
Expand Down Expand Up @@ -1094,6 +1108,9 @@ private boolean performCursorMoveAction(
arguments.getInt(AccessibilityNodeInfo.ACTION_ARGUMENT_MOVEMENT_GRANULARITY_INT);
final boolean extendSelection =
arguments.getBoolean(AccessibilityNodeInfo.ACTION_ARGUMENT_EXTEND_SELECTION_BOOLEAN);
// The voice access expects the semantics node to update immediately. We update the semantics
// node based on prediction. If the result is incorrect, it will be updated in the next frame.
predictCursorMovement(semanticsNode, granularity, forward, extendSelection);
switch (granularity) {
case AccessibilityNodeInfo.MOVEMENT_GRANULARITY_CHARACTER:
{
Expand Down Expand Up @@ -1121,23 +1138,98 @@ private boolean performCursorMoveAction(
return true;
}
break;
case AccessibilityNodeInfo.MOVEMENT_GRANULARITY_LINE:
case AccessibilityNodeInfo.MOVEMENT_GRANULARITY_PARAGRAPH:
case AccessibilityNodeInfo.MOVEMENT_GRANULARITY_PAGE:
return true;
Copy link
Contributor Author

@chunhtai chunhtai Mar 18, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We don't have semantics action for these actions. I feel weird to add them, too, because they aren't really going to be used. voice access only care about the action return true and the semantics node gets updated immediately, which is done by the predictCursorMovement.

If we do want to add them, I feel we should just merge all the cursor movement action into one semantics action that takes in granularity and direction.

}
return false;
}

private void predictCursorMovement(
@NonNull SemanticsNode node, int granularity, boolean forward, boolean extendSelection) {
if (node.textSelectionExtent < 0 || node.textSelectionBase < 0) {
return;
}

switch (granularity) {
case AccessibilityNodeInfo.MOVEMENT_GRANULARITY_CHARACTER:
if (forward && node.textSelectionExtent < node.value.length()) {
node.textSelectionExtent += 1;
} else if (!forward && node.textSelectionExtent > 0) {
node.textSelectionExtent -= 1;
}
break;
case AccessibilityNodeInfo.MOVEMENT_GRANULARITY_WORD:
if (forward && node.textSelectionExtent < node.value.length()) {
Pattern pattern = Pattern.compile("\\p{L}(\\b)");
Matcher result = pattern.matcher(node.value.substring(node.textSelectionExtent));
// we discard the first result because we want to find the "next" word
if (result.find() && result.find()) {
Copy link
Member

@xster xster Mar 29, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the test failure is from this condition and'ing the same expression twice

Copy link
Contributor Author

@chunhtai chunhtai Mar 29, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The lint does not consider the side affect of calling the condition. Not sure what is the correct way of writing this..

node.textSelectionExtent += result.start(1);
} else {
node.textSelectionExtent = node.value.length();
}
} else if (!forward && node.textSelectionExtent > 0) {
// Finds last beginning of the word boundary.
Pattern pattern = Pattern.compile("(?s:.*)(\\b)\\p{L}");
Matcher result = pattern.matcher(node.value.substring(0, node.textSelectionExtent));
if (result.find()) {
node.textSelectionExtent = result.start(1);
}
}
break;
case AccessibilityNodeInfo.MOVEMENT_GRANULARITY_LINE:
if (forward && node.textSelectionExtent < node.value.length()) {
// Finds the next new line.
Pattern pattern = Pattern.compile("(?!^)(\\n)");
Matcher result = pattern.matcher(node.value.substring(node.textSelectionExtent));
if (result.find()) {
node.textSelectionExtent += result.start(1);
} else {
node.textSelectionExtent = node.value.length();
}
} else if (!forward && node.textSelectionExtent > 0) {
// Finds the last new line.
Pattern pattern = Pattern.compile("(?s:.*)(\\n)");
Matcher result = pattern.matcher(node.value.substring(0, node.textSelectionExtent));
if (result.find()) {
node.textSelectionExtent = result.start(1);
} else {
node.textSelectionExtent = 0;
}
}
break;
case AccessibilityNodeInfo.MOVEMENT_GRANULARITY_PARAGRAPH:
case AccessibilityNodeInfo.MOVEMENT_GRANULARITY_PAGE:
if (forward) {
node.textSelectionExtent = node.value.length();
} else {
node.textSelectionExtent = 0;
}
break;
}
if (!extendSelection) {
node.textSelectionBase = node.textSelectionExtent;
}
}

/**
* Handles the responsibilities of {@link #performAction(int, int, Bundle)} for the specific
* scenario of cursor movement.
*/
@TargetApi(21)
@RequiresApi(21)
private boolean performSetText(int virtualViewId, @NonNull Bundle arguments) {
private boolean performSetText(SemanticsNode node, int virtualViewId, @NonNull Bundle arguments) {
String newText = "";
if (arguments != null
&& arguments.containsKey(AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE)) {
newText = arguments.getString(AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE);
}
accessibilityChannel.dispatchSemanticsAction(virtualViewId, Action.SET_TEXT, newText);
// The voice access expects the semantics node to update immediately. We update the semantics
// node based on prediction. If the result is incorrect, it will be updated in the next frame.
node.value = newText;
return true;
}

Expand Down
Loading