Skip to content
This repository was archived by the owner on Feb 25, 2025. It is now read-only.

[web:semantics] fix node positioning; expose debug tree #24903

Merged
merged 1 commit into from
Mar 11, 2021
Merged

[web:semantics] fix node positioning; expose debug tree #24903

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 40 additions & 3 deletions lib/web_ui/lib/src/engine/dom_renderer.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,12 +54,27 @@ class DomRenderer {
/// This element is created and inserted in the HTML DOM once. It is never
/// removed or moved. However the [sceneElement] may be replaced inside it.
///
/// This element precedes the [glassPaneElement] so that it never receives
/// input events. All input events are processed by [glassPaneElement] and the
/// semantics tree.
/// This element is inserted after the [semanticsHostElement] so that
/// platform views take precedence in DOM event handling.
html.Element? get sceneHostElement => _sceneHostElement;
html.Element? _sceneHostElement;

/// The element that contains the semantics tree.
///
/// This element is created and inserted in the HTML DOM once. It is never
/// removed or moved.
///
/// We render semantics inside the glasspane for proper focus and event
/// handling. If semantics is behind the glasspane, the phone will disable
/// focusing by touch, only by tabbing around the UI. If semantics is in
/// front of glasspane, then DOM event won't bubble up to the glasspane so
/// it can forward events to the framework.
///
/// This element is inserted before the [semanticsHostElement] so that
/// platform views take precedence in DOM event handling.
html.Element? get semanticsHostElement => _semanticsHostElement;
html.Element? _semanticsHostElement;

/// The last scene element rendered by the [render] method.
html.Element? get sceneElement => _sceneElement;
html.Element? _sceneElement;
Expand Down Expand Up @@ -427,6 +442,14 @@ flt-glass-pane * {

_sceneHostElement = createElement('flt-scene-host');

final html.Element semanticsHostElement = createElement('flt-semantics-host');
semanticsHostElement.style
..position = 'absolute'
..transformOrigin = '0 0 0';
_semanticsHostElement = semanticsHostElement;
updateSemanticsScreenProperties();
glassPaneElement.append(semanticsHostElement);

// Don't allow the scene to receive pointer events.
_sceneHostElement!.style.pointerEvents = 'none';

Expand All @@ -443,6 +466,12 @@ flt-glass-pane * {
// by the platform view.
glassPaneElement.insertBefore(_accesibilityPlaceholder, _sceneHostElement);

// When debugging semantics, make the scene semi-transparent so that the
// semantics tree is visible.
if (_debugShowSemanticsNodes) {
_sceneHostElement!.style.opacity = '0.3';
}

PointerBinding.initInstance(glassPaneElement);
KeyboardBinding.initInstance(glassPaneElement);

Expand Down Expand Up @@ -559,6 +588,13 @@ flt-glass-pane * {
EnginePlatformDispatcher.instance._updateLocales();
}

/// The framework specifies semantics in physical pixels, but CSS uses
/// logical pixels. To compensate, we inject an inverse scale at the root
/// level.
void updateSemanticsScreenProperties() {
_semanticsHostElement!.style.transform = 'scale(${1 / html.window.devicePixelRatio})';
}

/// Called immediately after browser window metrics change.
///
/// When there is a text editing going on in mobile devices, do not change
Expand All @@ -569,6 +605,7 @@ flt-glass-pane * {
/// Note: always check for rotations for a mobile device. Update the physical
/// size if the change is caused by a rotation.
void _metricsDidChange(html.Event? event) {
updateSemanticsScreenProperties();
if (isMobile && !window.isRotation() && textEditing.isEditing) {
window.computeOnScreenKeyboardInsets();
EnginePlatformDispatcher.instance.invokeOnMetricsChanged();
Expand Down
6 changes: 5 additions & 1 deletion lib/web_ui/lib/src/engine/semantics/label_and_value.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,11 @@ class LabelAndValue extends RoleManager {
..width = '${semanticsObject.rect!.width}px'
..height = '${semanticsObject.rect!.height}px';
}
_auxiliaryValueElement!.style.fontSize = '6px';

// Normally use a small font size so that text doesn't leave the scope
// of the semantics node. When debugging semantics, use a font size
// that's reasonably visible.
_auxiliaryValueElement!.style.fontSize = _debugShowSemanticsNodes ? '12px' : '6px';
semanticsObject.element.append(_auxiliaryValueElement!);
}
_auxiliaryValueElement!.text = combinedValue.toString();
Expand Down
140 changes: 55 additions & 85 deletions lib/web_ui/lib/src/engine/semantics/semantics.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,20 @@
part of engine;

/// Set this flag to `true` to cause the engine to visualize the semantics tree
/// on the screen.
/// on the screen for debugging.
///
/// This is useful for debugging.
const bool _debugShowSemanticsNodes = false;
/// This only works in profile and release modes. Debug mode does not support
/// passing compile-time constants.
///
/// Example:
///
/// ```
/// flutter run -d chrome --profile --dart-define=FLUTTER_WEB_DEBUG_SHOW_SEMANTICS=true
/// ```
const bool _debugShowSemanticsNodes = bool.fromEnvironment(
'FLUTTER_WEB_DEBUG_SHOW_SEMANTICS',
defaultValue: false,
);

/// Contains updates for the semantics tree.
///
Expand Down Expand Up @@ -233,27 +243,29 @@ class SemanticsObject {
/// Creates a semantics tree node with the given [id] and [owner].
SemanticsObject(this.id, this.owner) {
// DOM nodes created for semantics objects are positioned absolutely using
// transforms. We use a transparent color instead of "visibility:hidden" or
// "display:none" so that a screen reader does not ignore these elements.
// transforms.
element.style.position = 'absolute';

// The root node has some properties that other nodes do not.
if (id == 0) {
if (id == 0 && !_debugShowSemanticsNodes) {
// Make all semantics transparent. We use `filter` instead of `opacity`
// attribute because `filter` is stronger. `opacity` does not apply to
// some elements, particularly on iOS, such as the slider thumb and track.
//
// We use transparency instead of "visibility:hidden" or "display:none"
// so that a screen reader does not ignore these elements.
element.style.filter = 'opacity(0%)';

// Make text explicitly transparent to signal to the browser that no
// rasterization needs to be done.
element.style.color = 'rgba(0,0,0,0)';
}

// Make semantic elements visible for debugging by outlining them using a
// green border. We do not use `border` attribute because it affects layout
// (`outline` does not).
if (_debugShowSemanticsNodes) {
element.style
..filter = 'opacity(90%)'
..outline = '1px solid green'
..color = 'purple';
element.style.outline = '1px solid green';
}
}

Expand Down Expand Up @@ -853,9 +865,9 @@ class SemanticsObject {
hasIdentityTransform &&
verticalContainerAdjustment == 0.0 &&
horizontalContainerAdjustment == 0.0) {
_resetElementOffsets(element);
_clearSemanticElementTransform(element);
if (containerElement != null) {
_resetElementOffsets(containerElement);
_clearSemanticElementTransform(containerElement);
}
return;
}
Expand All @@ -879,81 +891,48 @@ class SemanticsObject {
effectiveTransformIsIdentity = false;
}

if (!effectiveTransformIsIdentity || isMacOrIOS) {
if (effectiveTransformIsIdentity) {
effectiveTransform = Matrix4.identity();
}
if (isDesktop) {
element.style
..transformOrigin = '0 0 0'
..transform = (effectiveTransformIsIdentity ? 'translate(0px 0px 0px)'
: matrix4ToCssTransform(effectiveTransform));
} else {
// Mobile screen readers observed to have errors while calculating the
// semantics focus borders if css `transform` properties are used.
// See: https://github.com/flutter/flutter/issues/68225
// Therefore we are calculating a bounding rectangle for the
// effective transform and use that rectangle to set TLWH css style
// properties.
// Note: Identity matrix is not using this code path.
final ui.Rect rect =
computeBoundingRectangleFromMatrix(effectiveTransform, _rect!);
element.style
..top = '${rect.top}px'
..left = '${rect.left}px'
..width = '${rect.width}px'
..height = '${rect.height}px';
}
if (!effectiveTransformIsIdentity) {
Copy link
Contributor

@ferhatb ferhatb Mar 10, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

have you verified flutter/flutter#68225 is fine using this version?

Copy link
Contributor Author

@yjbanov yjbanov Mar 10, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, this PR replaces the previous fix with updateSemanticsScreenProperties (see dom_renderer.dart). I tested both on Android and iOS (desktop didn't change as it already used transform due to if (isDesktop)).

element.style
..transformOrigin = '0 0 0'
..transform = matrix4ToCssTransform(effectiveTransform);
} else {
_resetElementOffsets(element);
// TODO: https://github.com/flutter/flutter/issues/73347
_clearSemanticElementTransform(element);
}

if (containerElement != null) {
if (!hasZeroRectOffset ||
isMacOrIOS ||
verticalContainerAdjustment != 0.0 ||
horizontalContainerAdjustment != 0.0) {
final double translateX = -_rect!.left + horizontalContainerAdjustment;
final double translateY = -_rect!.top + verticalContainerAdjustment;
if (isDesktop) {
containerElement.style
..transformOrigin = '0 0 0'
..transform = 'translate(${translateX}px, ${translateY}px)';
} else {
containerElement.style
..top = '${translateY}px'
..left = '${translateX}px';
}
containerElement.style
..top = '${translateY}px'
..left = '${translateX}px';
} else {
_resetElementOffsets(containerElement);
_clearSemanticElementTransform(containerElement);
}
}
}

// On Mac OS and iOS, VoiceOver requires left=0 top=0 value to correctly
// handle order. See https://github.com/flutter/flutter/issues/73347.
static void _resetElementOffsets(html.Element element) {
/// Clears the transform on a semantic element as if an identity transform is
/// applied.
///
/// On macOS and iOS, VoiceOver requires `left=0; top=0` value to correctly
/// handle traversal order.
///
/// See https://github.com/flutter/flutter/issues/73347.
static void _clearSemanticElementTransform(html.Element element) {
element.style
..removeProperty('transform-origin')
..removeProperty('transform');
if (isMacOrIOS) {
if (isDesktop) {
element.style
..transformOrigin = '0 0 0'
..transform = 'translate(0px, 0px)';
} else {
element.style
..top = '0px'
..left = '0px';
}
element.style
..top = '0px'
..left = '0px';
} else {
if (isDesktop) {
element.style
..removeProperty('transform-origin')
..removeProperty('transform');
} else {
element.style
..removeProperty('top')
..removeProperty('left');
}
element.style
..removeProperty('top')
..removeProperty('left');
}
}

Expand Down Expand Up @@ -1493,7 +1472,10 @@ class EngineSemanticsOwner {
/// Updates the semantics tree from data in the [uiUpdate].
void updateSemantics(ui.SemanticsUpdate uiUpdate) {
if (!_semanticsEnabled) {
return;
// If we're receiving a semantics update from the framework, it means the
// developer enabled it programmatically, so we enable it in the engine
// too.
semanticsEnabled = true;
}

final SemanticsUpdate update = uiUpdate as SemanticsUpdate;
Expand All @@ -1505,19 +1487,7 @@ class EngineSemanticsOwner {
if (_rootSemanticsElement == null) {
final SemanticsObject root = _semanticsTree[0]!;
_rootSemanticsElement = root.element;
// We render semantics inside the glasspane for proper focus and event
// handling. If semantics is behind the glasspane, the phone will disable
// focusing by touch, only by tabbing around the UI. If semantics is in
// front of glasspane, then DOM event won't bubble up to the glasspane so
// it can forward events to the framework.
//
// We insert the semantics root before the scene host. For all widgets
// in the scene, except for platform widgets, the scene host will pass the
// pointer events through to the semantics tree. However, for platform
// views, the pointer events will not pass through, and will be handled
// by the platform view.
domRenderer.glassPaneElement!
.insertBefore(_rootSemanticsElement!, domRenderer.sceneHostElement);
domRenderer.semanticsHostElement!.append(root.element);
}

_finalizeTree();
Expand Down
44 changes: 0 additions & 44 deletions lib/web_ui/lib/src/engine/util.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -595,47 +595,3 @@ int clampInt(int value, int min, int max) {
return value;
}
}

ui.Rect computeBoundingRectangleFromMatrix(Matrix4 transform, ui.Rect rect) {
final Float32List m = transform.storage;
// Apply perspective transform to all 4 corners. Can't use left,top, bottom,
// right since for example rotating 45 degrees would yield inaccurate size.
double x = rect.left;
double y = rect.top;
double wp = 1.0 / ((m[3] * x) + (m[7] * y) + m[15]);
double xp = ((m[0] * x) + (m[4] * y) + m[12]) * wp;
double yp = ((m[1] * x) + (m[5] * y) + m[13]) * wp;
double minX = xp, maxX = xp;
double minY = yp, maxY = yp;
x = rect.right;
y = rect.bottom;
wp = 1.0 / ((m[3] * x) + (m[7] * y) + m[15]);
xp = ((m[0] * x) + (m[4] * y) + m[12]) * wp;
yp = ((m[1] * x) + (m[5] * y) + m[13]) * wp;

minX = math.min(minX, xp);
maxX = math.max(maxX, xp);
minY = math.min(minY, yp);
maxY = math.max(maxY, yp);

x = rect.left;
y = rect.bottom;
wp = 1.0 / ((m[3] * x) + (m[7] * y) + m[15]);
xp = ((m[0] * x) + (m[4] * y) + m[12]) * wp;
yp = ((m[1] * x) + (m[5] * y) + m[13]) * wp;
minX = math.min(minX, xp);
maxX = math.max(maxX, xp);
minY = math.min(minY, yp);
maxY = math.max(maxY, yp);

x = rect.right;
y = rect.top;
wp = 1.0 / ((m[3] * x) + (m[7] * y) + m[15]);
xp = ((m[0] * x) + (m[4] * y) + m[12]) * wp;
yp = ((m[1] * x) + (m[5] * y) + m[13]) * wp;
minX = math.min(minX, xp);
maxX = math.max(maxX, xp);
minY = math.min(minY, yp);
maxY = math.max(maxY, yp);
return ui.Rect.fromLTWH(minX, minY, maxX - minX, maxY - minY);
}
Loading