From 98ae0067cd9403f24b0c8dc35c935536867580d9 Mon Sep 17 00:00:00 2001 From: Jackson Gardner Date: Wed, 7 Feb 2024 12:02:48 -0800 Subject: [PATCH 1/6] Implement frame timing callbacks in Skwasm. https://github.com/flutter/flutter/issues/140429 --- ci/licenses_golden/licenses_flutter | 4 + lib/web_ui/lib/frame_timings.dart | 105 ++++++++++++++++ lib/web_ui/lib/platform_dispatcher.dart | 100 --------------- lib/web_ui/lib/src/engine.dart | 1 + .../lib/src/engine/canvaskit/rasterizer.dart | 1 + .../lib/src/engine/canvaskit/renderer.dart | 16 +-- .../lib/src/engine/frame_timing_recorder.dart | 101 +++++++++++++++ lib/web_ui/lib/src/engine/html/renderer.dart | 7 +- lib/web_ui/lib/src/engine/html/scene.dart | 12 +- .../lib/src/engine/html/scene_builder.dart | 9 +- lib/web_ui/lib/src/engine/initialization.dart | 21 ++-- lib/web_ui/lib/src/engine/profiler.dart | 115 ------------------ lib/web_ui/lib/src/engine/scene_view.dart | 59 ++++++--- .../src/engine/skwasm/skwasm_impl/image.dart | 6 +- .../skwasm/skwasm_impl/raw/raw_surface.dart | 6 +- .../engine/skwasm/skwasm_impl/renderer.dart | 9 +- .../engine/skwasm/skwasm_impl/surface.dart | 32 ++++- lib/web_ui/lib/ui.dart | 1 + lib/web_ui/skwasm/library_skwasm_support.js | 42 +++++-- lib/web_ui/skwasm/skwasm_support.h | 22 ++-- lib/web_ui/skwasm/surface.cpp | 75 ++++++++---- lib/web_ui/skwasm/surface.h | 7 +- .../test/canvaskit/frame_timings_test.dart | 23 ---- .../test/common/frame_timings_common.dart | 53 -------- lib/web_ui/test/engine/scene_view_test.dart | 34 +++--- .../engine/surface/frame_timings_test.dart | 23 ---- lib/web_ui/test/ui/frame_timings_test.dart | 60 +++++++++ 27 files changed, 509 insertions(+), 435 deletions(-) create mode 100644 lib/web_ui/lib/frame_timings.dart create mode 100644 lib/web_ui/lib/src/engine/frame_timing_recorder.dart delete mode 100644 lib/web_ui/test/canvaskit/frame_timings_test.dart delete mode 100644 lib/web_ui/test/common/frame_timings_common.dart delete mode 100644 lib/web_ui/test/engine/surface/frame_timings_test.dart create mode 100644 lib/web_ui/test/ui/frame_timings_test.dart diff --git a/ci/licenses_golden/licenses_flutter b/ci/licenses_golden/licenses_flutter index 8e2c1a0f81175..7e9691da09e59 100644 --- a/ci/licenses_golden/licenses_flutter +++ b/ci/licenses_golden/licenses_flutter @@ -9682,6 +9682,7 @@ ORIGIN: ../../../flutter/lib/web_ui/lib/annotations.dart + ../../../flutter/LICE ORIGIN: ../../../flutter/lib/web_ui/lib/canvas.dart + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/web_ui/lib/channel_buffers.dart + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/web_ui/lib/compositing.dart + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/lib/web_ui/lib/frame_timings.dart + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/web_ui/lib/geometry.dart + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/web_ui/lib/hash_codes.dart + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/web_ui/lib/initialization.dart + ../../../flutter/LICENSE @@ -9747,6 +9748,7 @@ ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/font_fallback_data.dart + ../ ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/font_fallbacks.dart + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/fonts.dart + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/frame_reference.dart + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/frame_timing_recorder.dart + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/html/backdrop_filter.dart + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/html/bitmap_canvas.dart + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/html/canvas.dart + ../../../flutter/LICENSE @@ -12519,6 +12521,7 @@ FILE: ../../../flutter/lib/web_ui/lib/annotations.dart FILE: ../../../flutter/lib/web_ui/lib/canvas.dart FILE: ../../../flutter/lib/web_ui/lib/channel_buffers.dart FILE: ../../../flutter/lib/web_ui/lib/compositing.dart +FILE: ../../../flutter/lib/web_ui/lib/frame_timings.dart FILE: ../../../flutter/lib/web_ui/lib/geometry.dart FILE: ../../../flutter/lib/web_ui/lib/hash_codes.dart FILE: ../../../flutter/lib/web_ui/lib/initialization.dart @@ -12584,6 +12587,7 @@ FILE: ../../../flutter/lib/web_ui/lib/src/engine/font_fallback_data.dart FILE: ../../../flutter/lib/web_ui/lib/src/engine/font_fallbacks.dart FILE: ../../../flutter/lib/web_ui/lib/src/engine/fonts.dart FILE: ../../../flutter/lib/web_ui/lib/src/engine/frame_reference.dart +FILE: ../../../flutter/lib/web_ui/lib/src/engine/frame_timing_recorder.dart FILE: ../../../flutter/lib/web_ui/lib/src/engine/html/backdrop_filter.dart FILE: ../../../flutter/lib/web_ui/lib/src/engine/html/bitmap_canvas.dart FILE: ../../../flutter/lib/web_ui/lib/src/engine/html/canvas.dart diff --git a/lib/web_ui/lib/frame_timings.dart b/lib/web_ui/lib/frame_timings.dart new file mode 100644 index 0000000000000..e010819b5ca73 --- /dev/null +++ b/lib/web_ui/lib/frame_timings.dart @@ -0,0 +1,105 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +part of ui; + +enum FramePhase { + vsyncStart, + buildStart, + buildFinish, + rasterStart, + rasterFinish, + rasterFinishWallTime, +} + +enum _FrameTimingInfo { + layerCacheCount, + layerCacheBytes, + pictureCacheCount, + pictureCacheBytes, + frameNumber, +} + +class FrameTiming { + factory FrameTiming({ + required int vsyncStart, + required int buildStart, + required int buildFinish, + required int rasterStart, + required int rasterFinish, + required int rasterFinishWallTime, + int layerCacheCount = 0, + int layerCacheBytes = 0, + int pictureCacheCount = 0, + int pictureCacheBytes = 0, + int frameNumber = 1, + }) { + return FrameTiming._([ + vsyncStart, + buildStart, + buildFinish, + rasterStart, + rasterFinish, + rasterFinishWallTime, + layerCacheCount, + layerCacheBytes, + pictureCacheCount, + pictureCacheBytes, + frameNumber, + ]); + } + + FrameTiming._(this._data) + : assert(_data.length == _dataLength); + + static final int _dataLength = FramePhase.values.length + _FrameTimingInfo.values.length; + + int timestampInMicroseconds(FramePhase phase) => _data[phase.index]; + + Duration _rawDuration(FramePhase phase) => Duration(microseconds: _data[phase.index]); + + int _rawInfo(_FrameTimingInfo info) => _data[FramePhase.values.length + info.index]; + + Duration get buildDuration => + _rawDuration(FramePhase.buildFinish) - _rawDuration(FramePhase.buildStart); + + Duration get rasterDuration => + _rawDuration(FramePhase.rasterFinish) - _rawDuration(FramePhase.rasterStart); + + Duration get vsyncOverhead => _rawDuration(FramePhase.buildStart) - _rawDuration(FramePhase.vsyncStart); + + Duration get totalSpan => + _rawDuration(FramePhase.rasterFinish) - _rawDuration(FramePhase.vsyncStart); + + int get layerCacheCount => _rawInfo(_FrameTimingInfo.layerCacheCount); + + int get layerCacheBytes => _rawInfo(_FrameTimingInfo.layerCacheBytes); + + double get layerCacheMegabytes => layerCacheBytes / 1024.0 / 1024.0; + + int get pictureCacheCount => _rawInfo(_FrameTimingInfo.pictureCacheCount); + + int get pictureCacheBytes => _rawInfo(_FrameTimingInfo.pictureCacheBytes); + + double get pictureCacheMegabytes => pictureCacheBytes / 1024.0 / 1024.0; + + int get frameNumber => _data.last; + + final List _data; // some elements in microseconds, some in bytes, some are counts + + String _formatMS(Duration duration) => '${duration.inMicroseconds * 0.001}ms'; + + @override + String toString() { + return '$runtimeType(buildDuration: ${_formatMS(buildDuration)}, ' + 'rasterDuration: ${_formatMS(rasterDuration)}, ' + 'vsyncOverhead: ${_formatMS(vsyncOverhead)}, ' + 'totalSpan: ${_formatMS(totalSpan)}, ' + 'layerCacheCount: $layerCacheCount, ' + 'layerCacheBytes: $layerCacheBytes, ' + 'pictureCacheCount: $pictureCacheCount, ' + 'pictureCacheBytes: $pictureCacheBytes, ' + 'frameNumber: ${_data.last})'; + } +} diff --git a/lib/web_ui/lib/platform_dispatcher.dart b/lib/web_ui/lib/platform_dispatcher.dart index fb351b27c298e..a4326af23c651 100644 --- a/lib/web_ui/lib/platform_dispatcher.dart +++ b/lib/web_ui/lib/platform_dispatcher.dart @@ -158,106 +158,6 @@ abstract class PlatformDispatcher { double scaleFontSize(double unscaledFontSize); } -enum FramePhase { - vsyncStart, - buildStart, - buildFinish, - rasterStart, - rasterFinish, - rasterFinishWallTime, -} - -enum _FrameTimingInfo { - layerCacheCount, - layerCacheBytes, - pictureCacheCount, - pictureCacheBytes, - frameNumber, -} - -class FrameTiming { - factory FrameTiming({ - required int vsyncStart, - required int buildStart, - required int buildFinish, - required int rasterStart, - required int rasterFinish, - required int rasterFinishWallTime, - int layerCacheCount = 0, - int layerCacheBytes = 0, - int pictureCacheCount = 0, - int pictureCacheBytes = 0, - int frameNumber = 1, - }) { - return FrameTiming._([ - vsyncStart, - buildStart, - buildFinish, - rasterStart, - rasterFinish, - rasterFinishWallTime, - layerCacheCount, - layerCacheBytes, - pictureCacheCount, - pictureCacheBytes, - frameNumber, - ]); - } - - FrameTiming._(this._data) - : assert(_data.length == _dataLength); - - static final int _dataLength = FramePhase.values.length + _FrameTimingInfo.values.length; - - int timestampInMicroseconds(FramePhase phase) => _data[phase.index]; - - Duration _rawDuration(FramePhase phase) => Duration(microseconds: _data[phase.index]); - - int _rawInfo(_FrameTimingInfo info) => _data[FramePhase.values.length + info.index]; - - Duration get buildDuration => - _rawDuration(FramePhase.buildFinish) - _rawDuration(FramePhase.buildStart); - - Duration get rasterDuration => - _rawDuration(FramePhase.rasterFinish) - _rawDuration(FramePhase.rasterStart); - - Duration get vsyncOverhead => _rawDuration(FramePhase.buildStart) - _rawDuration(FramePhase.vsyncStart); - - Duration get totalSpan => - _rawDuration(FramePhase.rasterFinish) - _rawDuration(FramePhase.vsyncStart); - - int get layerCacheCount => _rawInfo(_FrameTimingInfo.layerCacheCount); - - int get layerCacheBytes => _rawInfo(_FrameTimingInfo.layerCacheBytes); - - double get layerCacheMegabytes => layerCacheBytes / 1024.0 / 1024.0; - - int get pictureCacheCount => _rawInfo(_FrameTimingInfo.pictureCacheCount); - - int get pictureCacheBytes => _rawInfo(_FrameTimingInfo.pictureCacheBytes); - - double get pictureCacheMegabytes => pictureCacheBytes / 1024.0 / 1024.0; - - int get frameNumber => _data.last; - - final List _data; // some elements in microseconds, some in bytes, some are counts - - String _formatMS(Duration duration) => '${duration.inMicroseconds * 0.001}ms'; - - @override - String toString() { - return '$runtimeType(buildDuration: ${_formatMS(buildDuration)}, ' - 'rasterDuration: ${_formatMS(rasterDuration)}, ' - 'vsyncOverhead: ${_formatMS(vsyncOverhead)}, ' - 'totalSpan: ${_formatMS(totalSpan)}, ' - 'layerCacheCount: $layerCacheCount, ' - 'layerCacheBytes: $layerCacheBytes, ' - 'pictureCacheCount: $pictureCacheCount, ' - 'pictureCacheBytes: $pictureCacheBytes, ' - 'frameNumber: ${_data.last})'; - } -} - enum AppLifecycleState { detached, resumed, diff --git a/lib/web_ui/lib/src/engine.dart b/lib/web_ui/lib/src/engine.dart index 9697966fe3bf0..0fbda333aac87 100644 --- a/lib/web_ui/lib/src/engine.dart +++ b/lib/web_ui/lib/src/engine.dart @@ -65,6 +65,7 @@ export 'engine/font_fallback_data.dart'; export 'engine/font_fallbacks.dart'; export 'engine/fonts.dart'; export 'engine/frame_reference.dart'; +export 'engine/frame_timing_recorder.dart'; export 'engine/html/backdrop_filter.dart'; export 'engine/html/bitmap_canvas.dart'; export 'engine/html/canvas.dart'; diff --git a/lib/web_ui/lib/src/engine/canvaskit/rasterizer.dart b/lib/web_ui/lib/src/engine/canvaskit/rasterizer.dart index 58a23e871cc92..bdf6d744f7152 100644 --- a/lib/web_ui/lib/src/engine/canvaskit/rasterizer.dart +++ b/lib/web_ui/lib/src/engine/canvaskit/rasterizer.dart @@ -131,6 +131,7 @@ abstract class DisplayCanvas { typedef RenderRequest = ({ ui.Scene scene, Completer completer, + FrameTimingRecorder? recorder, }); /// A per-view queue of render requests. Only contains the current render diff --git a/lib/web_ui/lib/src/engine/canvaskit/renderer.dart b/lib/web_ui/lib/src/engine/canvaskit/renderer.dart index f4fdaef67c0f9..1b9c3b7cc73b4 100644 --- a/lib/web_ui/lib/src/engine/canvaskit/renderer.dart +++ b/lib/web_ui/lib/src/engine/canvaskit/renderer.dart @@ -417,16 +417,17 @@ class CanvasKitRenderer implements Renderer { "Unable to render to a view which hasn't been registered"); final ViewRasterizer rasterizer = _rasterizers[view.viewId]!; final RenderQueue renderQueue = rasterizer.queue; + final FrameTimingRecorder? recorder = FrameTimingRecorder.currentRecorder; if (renderQueue.current != null) { // If a scene is already queued up, drop it and queue this one up instead // so that the scene view always displays the most recently requested scene. renderQueue.next?.completer.complete(); final Completer completer = Completer(); - renderQueue.next = (scene: scene, completer: completer); + renderQueue.next = (scene: scene, completer: completer, recorder: recorder); return completer.future; } final Completer completer = Completer(); - renderQueue.current = (scene: scene, completer: completer); + renderQueue.current = (scene: scene, completer: completer, recorder: recorder); unawaited(_kickRenderLoop(rasterizer)); return completer.future; } @@ -435,7 +436,7 @@ class CanvasKitRenderer implements Renderer { final RenderQueue renderQueue = rasterizer.queue; final RenderRequest current = renderQueue.current!; try { - await _renderScene(current.scene, rasterizer); + await _renderScene(current.scene, rasterizer, current.recorder); current.completer.complete(); } catch (error, stackTrace) { current.completer.completeError(error, stackTrace); @@ -449,7 +450,7 @@ class CanvasKitRenderer implements Renderer { } } - Future _renderScene(ui.Scene scene, ViewRasterizer rasterizer) async { + Future _renderScene(ui.Scene scene, ViewRasterizer rasterizer, FrameTimingRecorder? recorder) async { // "Build finish" and "raster start" happen back-to-back because we // render on the same thread, so there's no overhead from hopping to // another thread. @@ -457,11 +458,12 @@ class CanvasKitRenderer implements Renderer { // CanvasKit works differently from the HTML renderer in that in HTML // we update the DOM in SceneBuilder.build, which is these function calls // here are CanvasKit-only. - frameTimingsOnBuildFinish(); - frameTimingsOnRasterStart(); + recorder?.recordBuildFinish(); + recorder?.recordRasterStart(); await rasterizer.draw((scene as LayerScene).layerTree); - frameTimingsOnRasterFinish(); + recorder?.recordRasterFinish(); + recorder?.submitTimings(); } // Map from view id to the associated Rasterizer for that view. diff --git a/lib/web_ui/lib/src/engine/frame_timing_recorder.dart b/lib/web_ui/lib/src/engine/frame_timing_recorder.dart new file mode 100644 index 0000000000000..b0fe2f506e15b --- /dev/null +++ b/lib/web_ui/lib/src/engine/frame_timing_recorder.dart @@ -0,0 +1,101 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:ui/src/engine.dart'; +import 'package:ui/ui.dart' as ui; + +class FrameTimingRecorder { + int? _vsyncStartMicros; + int? _buildStartMicros; + int? _buildFinishMicros; + int? _rasterStartMicros; + int? _rasterFinishMicros; + + /// Collects frame timings from frames. + /// + /// This list is periodically reported to the framework (see [_kFrameTimingsSubmitInterval]). + static List _frameTimings = []; + + static void startFrame() { + _currentFrameTimingRecorder = frameTimingsEnabled + ? FrameTimingRecorder() + : null; + } + + static FrameTimingRecorder? _currentFrameTimingRecorder; + static FrameTimingRecorder? get currentRecorder => _currentFrameTimingRecorder; + + /// The last time (in microseconds) we submitted frame timings. + static int _frameTimingsLastSubmitTime = _nowMicros(); + /// The amount of time in microseconds we wait between submitting + /// frame timings. + static const int _kFrameTimingsSubmitInterval = 100000; // 100 milliseconds + + /// Whether we are collecting [ui.FrameTiming]s. + static bool get frameTimingsEnabled { + return EnginePlatformDispatcher.instance.onReportTimings != null; + } + + /// Current timestamp in microseconds taken from the high-precision + /// monotonically increasing timer. + /// + /// See also: + /// + /// * https://developer.mozilla.org/en-US/docs/Web/API/Performance/now, + /// particularly notes about Firefox rounding to 1ms for security reasons, + /// which can be bypassed in tests by setting certain browser options. + static int _nowMicros() { + return (domWindow.performance.now() * 1000).toInt(); + } + + void recordVsyncStart([int? vsyncStart]) { + assert(_vsyncStartMicros == null, "can't record vsync start more than once"); + _vsyncStartMicros = vsyncStart ?? _nowMicros(); + } + + void recordBuildStart([int? buildStart]) { + assert(_buildStartMicros == null, "can't record build start more than once"); + _buildStartMicros = buildStart ?? _nowMicros(); + } + + void recordBuildFinish([int? buildFinish]) { + assert(_buildFinishMicros == null, "can't record build finish more than once"); + _buildFinishMicros = buildFinish ?? _nowMicros(); + } + + void recordRasterStart([int? rasterStart]) { + assert(_rasterStartMicros == null, "can't record raster start more than once"); + _rasterStartMicros = rasterStart ?? _nowMicros(); + } + + void recordRasterFinish([int? rasterFinish]) { + assert(_rasterFinishMicros == null, "can't record raster finish more than once"); + _rasterFinishMicros = rasterFinish ?? _nowMicros(); + } + + void submitTimings() { + assert( + _buildStartMicros != null && + _buildFinishMicros != null && + _rasterStartMicros != null && + _rasterFinishMicros != null, + 'Attempted to submit an incomplete timings.' + ); + final ui.FrameTiming timing = ui.FrameTiming( + vsyncStart: _vsyncStartMicros!, + buildStart: _buildStartMicros!, + buildFinish: _buildFinishMicros!, + rasterStart: _rasterStartMicros!, + rasterFinish: _rasterFinishMicros!, + rasterFinishWallTime: _rasterFinishMicros!, + ); + _frameTimings.add(timing); + final int now = _nowMicros(); + if (now - _frameTimingsLastSubmitTime > _kFrameTimingsSubmitInterval) { + _frameTimingsLastSubmitTime = now; + EnginePlatformDispatcher.instance.invokeOnReportTimings(_frameTimings); + _frameTimings = []; + } + } +} diff --git a/lib/web_ui/lib/src/engine/html/renderer.dart b/lib/web_ui/lib/src/engine/html/renderer.dart index c9febef391e37..b41fac3739234 100644 --- a/lib/web_ui/lib/src/engine/html/renderer.dart +++ b/lib/web_ui/lib/src/engine/html/renderer.dart @@ -323,8 +323,11 @@ class HtmlRenderer implements Renderer { @override Future renderScene(ui.Scene scene, ui.FlutterView view) async { final EngineFlutterView implicitView = EnginePlatformDispatcher.instance.implicitView!; - implicitView.dom.setScene((scene as SurfaceScene).webOnlyRootElement!); - frameTimingsOnRasterFinish(); + scene as SurfaceScene; + implicitView.dom.setScene(scene.webOnlyRootElement!); + final FrameTimingRecorder? recorder = scene.timingRecorder; + recorder?.recordRasterFinish(); + recorder?.submitTimings(); } @override diff --git a/lib/web_ui/lib/src/engine/html/scene.dart b/lib/web_ui/lib/src/engine/html/scene.dart index f15d043447e4b..b4deb9a6dac69 100644 --- a/lib/web_ui/lib/src/engine/html/scene.dart +++ b/lib/web_ui/lib/src/engine/html/scene.dart @@ -2,22 +2,20 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -import 'package:ui/src/engine/display.dart'; +import 'package:ui/src/engine.dart'; import 'package:ui/ui.dart' as ui; -import '../dom.dart'; -import '../vector_math.dart'; -import '../window.dart'; -import 'surface.dart'; - class SurfaceScene implements ui.Scene { /// This class is created by the engine, and should not be instantiated /// or extended directly. /// /// To create a Scene object, use a [SceneBuilder]. - SurfaceScene(this.webOnlyRootElement); + SurfaceScene(this.webOnlyRootElement, { + required this.timingRecorder, + }); final DomElement? webOnlyRootElement; + final FrameTimingRecorder? timingRecorder; /// Creates a raster image representation of the current state of the scene. /// This is a slow operation that is performed on a background thread. diff --git a/lib/web_ui/lib/src/engine/html/scene_builder.dart b/lib/web_ui/lib/src/engine/html/scene_builder.dart index e721aa5a5576b..28e692c139e4a 100644 --- a/lib/web_ui/lib/src/engine/html/scene_builder.dart +++ b/lib/web_ui/lib/src/engine/html/scene_builder.dart @@ -7,7 +7,7 @@ import 'dart:typed_data'; import 'package:ui/ui.dart' as ui; import 'package:ui/ui_web/src/ui_web.dart' as ui_web; -import '../../engine.dart' show kProfileApplyFrame, kProfilePrerollFrame; +import '../../engine.dart' show FrameTimingRecorder, kProfileApplyFrame, kProfilePrerollFrame; import '../display.dart'; import '../dom.dart'; import '../picture.dart'; @@ -511,8 +511,9 @@ class SurfaceSceneBuilder implements ui.SceneBuilder { // In the HTML renderer we time the beginning of the rasterization phase // (counter-intuitively) in SceneBuilder.build because DOM updates happen // here. This is different from CanvasKit. - frameTimingsOnBuildFinish(); - frameTimingsOnRasterStart(); + final FrameTimingRecorder? recorder = FrameTimingRecorder.currentRecorder; + recorder?.recordBuildFinish(); + recorder?.recordRasterStart(); timeAction(kProfilePrerollFrame, () { while (_surfaceStack.length > 1) { // Auto-pop layers that were pushed without a corresponding pop. @@ -528,7 +529,7 @@ class SurfaceSceneBuilder implements ui.SceneBuilder { } commitScene(_persistedScene); _lastFrameScene = _persistedScene; - return SurfaceScene(_persistedScene.rootElement); + return SurfaceScene(_persistedScene.rootElement, timingRecorder: recorder); }); } diff --git a/lib/web_ui/lib/src/engine/initialization.dart b/lib/web_ui/lib/src/engine/initialization.dart index 78352cf091af3..af6c3479a787a 100644 --- a/lib/web_ui/lib/src/engine/initialization.dart +++ b/lib/web_ui/lib/src/engine/initialization.dart @@ -158,7 +158,19 @@ Future initializeEngineServices({ if (!waitingForAnimation) { waitingForAnimation = true; domWindow.requestAnimationFrame((JSNumber highResTime) { - frameTimingsOnVsync(); + FrameTimingRecorder.startFrame(); + final FrameTimingRecorder? recorder = FrameTimingRecorder.currentRecorder; + if (recorder != null) { + recorder.recordVsyncStart(); + + // In Flutter terminology "building a frame" consists of "beginning + // frame" and "drawing frame". + // + // We do not call `recordBuildFinish` from here because + // part of the rasterization process, particularly in the HTML + // renderer, takes place in the `SceneBuilder.build()`. + recorder.recordBuildStart(); + } // Reset immediately, because `frameHandler` can schedule more frames. waitingForAnimation = false; @@ -171,13 +183,6 @@ Future initializeEngineServices({ final int highResTimeMicroseconds = (1000 * highResTime.toDartDouble).toInt(); - // In Flutter terminology "building a frame" consists of "beginning - // frame" and "drawing frame". - // - // We do not call `frameTimingsOnBuildFinish` from here because - // part of the rasterization process, particularly in the HTML - // renderer, takes place in the `SceneBuilder.build()`. - frameTimingsOnBuildStart(); if (EnginePlatformDispatcher.instance.onBeginFrame != null) { EnginePlatformDispatcher.instance.invokeOnBeginFrame( Duration(microseconds: highResTimeMicroseconds)); diff --git a/lib/web_ui/lib/src/engine/profiler.dart b/lib/web_ui/lib/src/engine/profiler.dart index ffabd12d2156b..d5ef8b3fa831b 100644 --- a/lib/web_ui/lib/src/engine/profiler.dart +++ b/lib/web_ui/lib/src/engine/profiler.dart @@ -5,11 +5,8 @@ import 'dart:async'; import 'dart:js_interop'; -import 'package:ui/ui.dart' as ui; import 'package:ui/ui_web/src/ui_web.dart' as ui_web; -import 'dom.dart'; -import 'platform_dispatcher.dart'; import 'util.dart'; // TODO(mdebbar): Deprecate this and remove it. @@ -127,118 +124,6 @@ class Profiler { } } -/// Whether we are collecting [ui.FrameTiming]s. -bool get _frameTimingsEnabled { - return EnginePlatformDispatcher.instance.onReportTimings != null; -} - -/// Collects frame timings from frames. -/// -/// This list is periodically reported to the framework (see -/// [_kFrameTimingsSubmitInterval]). -List _frameTimings = []; - -/// The amount of time in microseconds we wait between submitting -/// frame timings. -const int _kFrameTimingsSubmitInterval = 100000; // 100 milliseconds - -/// The last time (in microseconds) we submitted frame timings. -int _frameTimingsLastSubmitTime = _nowMicros(); - -// These variables store individual [ui.FrameTiming] properties. -int _vsyncStartMicros = -1; -int _buildStartMicros = -1; -int _buildFinishMicros = -1; -int _rasterStartMicros = -1; -int _rasterFinishMicros = -1; - -/// Records the vsync timestamp for this frame. -void frameTimingsOnVsync() { - if (!_frameTimingsEnabled) { - return; - } - _vsyncStartMicros = _nowMicros(); -} - -/// Records the time when the framework started building the frame. -void frameTimingsOnBuildStart() { - if (!_frameTimingsEnabled) { - return; - } - _buildStartMicros = _nowMicros(); -} - -/// Records the time when the framework finished building the frame. -void frameTimingsOnBuildFinish() { - if (!_frameTimingsEnabled) { - return; - } - _buildFinishMicros = _nowMicros(); -} - -/// Records the time when the framework started rasterizing the frame. -/// -/// On the web, this value is almost always the same as [_buildFinishMicros] -/// because it's single-threaded so there's no delay between building -/// and rasterization. -/// -/// This also means different things between HTML and CanvasKit renderers. -/// -/// In HTML "rasterization" only captures DOM updates, but not the work that -/// the browser performs after the DOM updates are committed. The browser -/// does not report that information. -/// -/// CanvasKit captures everything because we control the rasterization -/// process, so we know exactly when rasterization starts and ends. -void frameTimingsOnRasterStart() { - if (!_frameTimingsEnabled) { - return; - } - _rasterStartMicros = _nowMicros(); -} - -/// Records the time when the framework started rasterizing the frame. -/// -/// See [_frameTimingsOnRasterStart] for more details on what rasterization -/// timings mean on the web. -void frameTimingsOnRasterFinish() { - if (!_frameTimingsEnabled) { - return; - } - final int now = _nowMicros(); - _rasterFinishMicros = now; - _frameTimings.add(ui.FrameTiming( - vsyncStart: _vsyncStartMicros, - buildStart: _buildStartMicros, - buildFinish: _buildFinishMicros, - rasterStart: _rasterStartMicros, - rasterFinish: _rasterFinishMicros, - rasterFinishWallTime: _rasterFinishMicros, - )); - _vsyncStartMicros = -1; - _buildStartMicros = -1; - _buildFinishMicros = -1; - _rasterStartMicros = -1; - _rasterFinishMicros = -1; - if (now - _frameTimingsLastSubmitTime > _kFrameTimingsSubmitInterval) { - _frameTimingsLastSubmitTime = now; - EnginePlatformDispatcher.instance.invokeOnReportTimings(_frameTimings); - _frameTimings = []; - } -} - -/// Current timestamp in microseconds taken from the high-precision -/// monotonically increasing timer. -/// -/// See also: -/// -/// * https://developer.mozilla.org/en-US/docs/Web/API/Performance/now, -/// particularly notes about Firefox rounding to 1ms for security reasons, -/// which can be bypassed in tests by setting certain browser options. -int _nowMicros() { - return (domWindow.performance.now() * 1000).toInt(); -} - /// Counts various events that take place while the app is running. /// /// This class will slow down the app, and therefore should be disabled while diff --git a/lib/web_ui/lib/src/engine/scene_view.dart b/lib/web_ui/lib/src/engine/scene_view.dart index 23010c6db0bff..b137b70f4f909 100644 --- a/lib/web_ui/lib/src/engine/scene_view.dart +++ b/lib/web_ui/lib/src/engine/scene_view.dart @@ -9,20 +9,31 @@ import 'package:ui/ui.dart' as ui; const String kCanvasContainerTag = 'flt-canvas-container'; +typedef RenderResult = ({ + List imageBitmaps, + int rasterStartMicros, + int rasterEndMicros, +}); + // This is an interface that renders a `ScenePicture` as a `DomImageBitmap`. // It is optionally asynchronous. It is required for the `EngineSceneView` to // composite pictures into the canvases in the DOM tree it builds. abstract class PictureRenderer { - FutureOr renderPicture(ScenePicture picture); + FutureOr renderPictures(List picture); } class _SceneRender { - _SceneRender(this.scene, this._completer) { + _SceneRender( + this.scene, + this._completer, { + this.recorder, + }) { scene.beginRender(); } final EngineScene scene; final Completer _completer; + final FrameTimingRecorder? recorder; void done() { scene.endRender(); @@ -47,24 +58,24 @@ class EngineSceneView { _SceneRender? _currentRender; _SceneRender? _nextRender; - Future renderScene(EngineScene scene) { + Future renderScene(EngineScene scene, FrameTimingRecorder? recorder) { if (_currentRender != null) { // If a scene is already queued up, drop it and queue this one up instead // so that the scene view always displays the most recently requested scene. _nextRender?.done(); final Completer completer = Completer(); - _nextRender = _SceneRender(scene, completer); + _nextRender = _SceneRender(scene, completer, recorder: recorder); return completer.future; } final Completer completer = Completer(); - _currentRender = _SceneRender(scene, completer); + _currentRender = _SceneRender(scene, completer, recorder: recorder); _kickRenderLoop(); return completer.future; } Future _kickRenderLoop() async { final _SceneRender current = _currentRender!; - await _renderScene(current.scene); + await _renderScene(current.scene, current.recorder); current.done(); _currentRender = _nextRender; _nextRender = null; @@ -75,19 +86,33 @@ class EngineSceneView { } } - Future _renderScene(EngineScene scene) async { + Future _renderScene(EngineScene scene, FrameTimingRecorder? recorder) async { final List slices = scene.rootLayer.slices; - final Iterable> renderFutures = slices.map( - (LayerSlice slice) async => switch (slice) { - PlatformViewSlice() => null, - PictureSlice() => pictureRenderer.renderPicture(slice.picture), - } - ); - final List renderedBitmaps = await Future.wait(renderFutures); + final List picturesToRender = []; + for (final LayerSlice slice in slices) { + if (slice is PictureSlice) { + picturesToRender.add(slice.picture); + } + } + final Map renderMap; + if (picturesToRender.isNotEmpty) { + final RenderResult renderResult = await pictureRenderer.renderPictures(picturesToRender); + renderMap = { + for (int i = 0; i < picturesToRender.length; i++) + picturesToRender[i]: renderResult.imageBitmaps[i], + }; + recorder?.recordRasterStart(renderResult.rasterStartMicros); + recorder?.recordRasterFinish(renderResult.rasterEndMicros); + } else { + renderMap = {}; + recorder?.recordRasterStart(); + recorder?.recordRasterFinish(); + } + recorder?.submitTimings(); + final List reusableContainers = List.from(containers); final List newContainers = []; - for (int i = 0; i < slices.length; i++) { - final LayerSlice slice = slices[i]; + for (final LayerSlice slice in slices) { switch (slice) { case PictureSlice(): PictureSliceContainer? container; @@ -106,7 +131,7 @@ class EngineSceneView { container = PictureSliceContainer(slice.picture.cullRect); } container.updateContents(); - container.renderBitmap(renderedBitmaps[i]!); + container.renderBitmap(renderMap[slice.picture]!); newContainers.add(container); case PlatformViewSlice(): diff --git a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/image.dart b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/image.dart index ee32ffd987350..80a5e733da3b5 100644 --- a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/image.dart +++ b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/image.dart @@ -62,9 +62,9 @@ class SkwasmImage extends SkwasmObjectWrapper implements ui.Image { final ui.Canvas canvas = ui.Canvas(recorder); canvas.drawImage(this, ui.Offset.zero, ui.Paint()); final DomImageBitmap bitmap = - await (renderer as SkwasmRenderer).surface.renderPicture( - recorder.endRecording() as SkwasmPicture, - ); + (await (renderer as SkwasmRenderer).surface.renderPictures( + [recorder.endRecording() as SkwasmPicture], + )).imageBitmaps.first; final DomOffscreenCanvas offscreenCanvas = createDomOffscreenCanvas(bitmap.width.toDartInt, bitmap.height.toDartInt); final DomCanvasRenderingContextBitmapRenderer context = diff --git a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/raw_surface.dart b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/raw_surface.dart index 3831188c4df63..22b7462eec9bc 100644 --- a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/raw_surface.dart +++ b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/raw_surface.dart @@ -35,10 +35,10 @@ external void surfaceSetCallbackHandler( isLeaf: true) external void surfaceDestroy(SurfaceHandle surface); -@Native( - symbol: 'surface_renderPicture', +@Native, Int)>( + symbol: 'surface_renderPictures', isLeaf: true) -external CallbackId surfaceRenderPicture(SurfaceHandle surface, PictureHandle picture); +external CallbackId surfaceRenderPictures(SurfaceHandle surface, Pointer picture, int count); @Native renderScene(ui.Scene scene, ui.FlutterView view) { + final FrameTimingRecorder? recorder = FrameTimingRecorder.currentRecorder; + recorder?.recordBuildFinish(); + view as EngineFlutterView; assert(view is EngineFlutterWindow, 'Skwasm does not support multi-view mode yet'); final EngineSceneView sceneView = _getSceneViewForView(view); - return sceneView.renderScene(scene as EngineScene); + return sceneView.renderScene(scene as EngineScene, recorder); } EngineSceneView _getSceneViewForView(EngineFlutterView view) { @@ -477,6 +480,6 @@ class SkwasmPictureRenderer implements PictureRenderer { SkwasmSurface surface; @override - FutureOr renderPicture(ScenePicture picture) => - surface.renderPicture(picture as SkwasmPicture); + FutureOr renderPictures(List pictures) => + surface.renderPictures(pictures.cast()); } diff --git a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/surface.dart b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/surface.dart index 34af06d0d1fd7..261ba8381be9c 100644 --- a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/surface.dart +++ b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/surface.dart @@ -12,6 +12,17 @@ import 'package:ui/src/engine.dart'; import 'package:ui/src/engine/skwasm/skwasm_impl.dart'; import 'package:ui/ui.dart' as ui; +@JS() +@staticInterop +@anonymous +class RasterResult {} + +extension RasterResultExtension on RasterResult { + external JSNumber get rasterStart; + external JSNumber get rasterEnd; + external JSArray get imageBitmaps; +} + @pragma('wasm:export') WasmVoid callbackHandler(WasmI32 callbackId, WasmI32 context, WasmExternRef? jsContext) { // Actually hide this call behind whether skwasm is enabled. Otherwise, the SkwasmCallbackHandler @@ -78,11 +89,22 @@ class SkwasmSurface { surfaceSetCallbackHandler(handle, SkwasmCallbackHandler.instance.callbackPointer); } - Future renderPicture(SkwasmPicture picture) async { - final int callbackId = surfaceRenderPicture(handle, picture.handle); - final DomImageBitmap bitmap = (await SkwasmCallbackHandler.instance.registerCallback(callbackId)) as DomImageBitmap; - return bitmap; - } + Future renderPictures(List pictures) => + withStackScope((StackScope scope) async { + final Pointer pictureHandles = + scope.allocPointerArray(pictures.length).cast(); + for (int i = 0; i < pictures.length; i++) { + pictureHandles[i] = pictures[i].handle; + } + final int callbackId = surfaceRenderPictures(handle, pictureHandles, pictures.length); + final RasterResult rasterResult = (await SkwasmCallbackHandler.instance.registerCallback(callbackId)) as RasterResult; + final RenderResult result = ( + imageBitmaps: rasterResult.imageBitmaps.toDart.cast(), + rasterStartMicros: (rasterResult.rasterStart.toDartDouble * 1000).toInt(), + rasterEndMicros: (rasterResult.rasterEnd.toDartDouble * 1000).toInt(), + ); + return result; + }); Future rasterizeImage(SkwasmImage image, ui.ImageByteFormat format) async { final int callbackId = surfaceRasterizeImage( diff --git a/lib/web_ui/lib/ui.dart b/lib/web_ui/lib/ui.dart index 8b5130e44d2d0..b0c8a346e001c 100644 --- a/lib/web_ui/lib/ui.dart +++ b/lib/web_ui/lib/ui.dart @@ -20,6 +20,7 @@ part 'annotations.dart'; part 'canvas.dart'; part 'channel_buffers.dart'; part 'compositing.dart'; +part 'frame_timings.dart'; part 'geometry.dart'; part 'hash_codes.dart'; part 'initialization.dart'; diff --git a/lib/web_ui/skwasm/library_skwasm_support.js b/lib/web_ui/skwasm/library_skwasm_support.js index 5e62614e009c4..63ef28eaa2bbe 100644 --- a/lib/web_ui/skwasm/library_skwasm_support.js +++ b/lib/web_ui/skwasm/library_skwasm_support.js @@ -27,11 +27,18 @@ mergeInto(LibraryManager.library, { return; } switch (skwasmMessage) { - case 'renderPicture': - _surface_renderPictureOnWorker(data.surface, data.picture, data.callbackId); + case 'renderPictures': + _surface_renderPicturesOnWorker(data.surface, data.pictures, data.pictureCount, data.callbackId, performance.now()); return; case 'onRenderComplete': - _surface_onRenderComplete(data.surface, data.callbackId, data.imageBitmap); + _surface_onRenderComplete( + data.surface, + data.callbackId, { + "imageBitmaps": data.imageBitmaps, + "rasterStart": data.rasterStart, + "rasterEnd": data.rasterEnd, + }, + ); return; case 'setAssociatedObject': associatedObjectsMap.set(data.pointer, data.object); @@ -54,11 +61,12 @@ mergeInto(LibraryManager.library, { PThread.pthreads[threadId].addEventListener("message", eventListener); } }; - _skwasm_dispatchRenderPicture = function(threadId, surfaceHandle, pictureHandle, callbackId) { + _skwasm_dispatchRenderPictures = function(threadId, surfaceHandle, pictures, pictureCount, callbackId) { PThread.pthreads[threadId].postMessage({ - skwasmMessage: 'renderPicture', + skwasmMessage: 'renderPictures', surface: surfaceHandle, - picture: pictureHandle, + pictures, + pictureCount, callbackId, }); }; @@ -85,15 +93,23 @@ mergeInto(LibraryManager.library, { canvas.width = width; canvas.height = height; }; - _skwasm_captureImageBitmap = async function(surfaceHandle, contextHandle, callbackId, width, height) { + _skwasm_captureImageBitmap = function(contextHandle, width, height, imagePromises) { + if (!imagePromises) imagePromises = Array(); const canvas = handleToCanvasMap.get(contextHandle); - const imageBitmap = await createImageBitmap(canvas, 0, 0, width, height); + imagePromises.push(createImageBitmap(canvas, 0, 0, width, height)); + return imagePromises; + }; + _skwasm_resolveAndPostImages = async function(surfaceHandle, imagePromises, rasterStart, callbackId) { + const imageBitmaps = imagePromises ? await Promise.all(imagePromises) : []; + const rasterEnd = performance.now(); postMessage({ skwasmMessage: 'onRenderComplete', surface: surfaceHandle, callbackId, - imageBitmap, - }, [imageBitmap]); + imageBitmaps, + rasterStart, + rasterEnd, + }, [...imageBitmaps]); }; _skwasm_createGlTextureFromTextureSource = function(textureSource, width, height) { const glCtx = GL.currentContext.GLctx; @@ -125,14 +141,16 @@ mergeInto(LibraryManager.library, { skwasm_disposeAssociatedObjectOnThread__deps: ['$skwasm_support_setup'], skwasm_registerMessageListener: function() {}, skwasm_registerMessageListener__deps: ['$skwasm_support_setup'], - skwasm_dispatchRenderPicture: function() {}, - skwasm_dispatchRenderPicture__deps: ['$skwasm_support_setup'], + skwasm_dispatchRenderPictures: function() {}, + skwasm_dispatchRenderPictures__deps: ['$skwasm_support_setup'], skwasm_createOffscreenCanvas: function () {}, skwasm_createOffscreenCanvas__deps: ['$skwasm_support_setup'], skwasm_resizeCanvas: function () {}, skwasm_resizeCanvas__deps: ['$skwasm_support_setup'], skwasm_captureImageBitmap: function () {}, skwasm_captureImageBitmap__deps: ['$skwasm_support_setup'], + skwasm_resolveAndPostImages: function () {}, + skwasm_resolveAndPostImages__deps: ['$skwasm_support_setup'], skwasm_createGlTextureFromTextureSource: function () {}, skwasm_createGlTextureFromTextureSource__deps: ['$skwasm_support_setup'], }); diff --git a/lib/web_ui/skwasm/skwasm_support.h b/lib/web_ui/skwasm/skwasm_support.h index ce36a192a69b6..c9132b89dd166 100644 --- a/lib/web_ui/skwasm/skwasm_support.h +++ b/lib/web_ui/skwasm/skwasm_support.h @@ -23,17 +23,21 @@ extern SkwasmObject skwasm_getAssociatedObject(void* pointer); extern void skwasm_disposeAssociatedObjectOnThread(unsigned long threadId, void* pointer); extern void skwasm_registerMessageListener(pthread_t threadId); -extern void skwasm_dispatchRenderPicture(unsigned long threadId, - Skwasm::Surface* surface, - SkPicture* picture, - uint32_t callbackId); +extern void skwasm_dispatchRenderPictures(unsigned long threadId, + Skwasm::Surface* surface, + sk_sp* pictures, + int count, + uint32_t callbackId); extern uint32_t skwasm_createOffscreenCanvas(int width, int height); extern void skwasm_resizeCanvas(uint32_t contextHandle, int width, int height); -extern void skwasm_captureImageBitmap(Skwasm::Surface* surfaceHandle, - uint32_t contextHandle, - uint32_t bitmapId, - int width, - int height); +extern SkwasmObject skwasm_captureImageBitmap(uint32_t contextHandle, + int width, + int height, + SkwasmObject imagePromises); +extern void skwasm_resolveAndPostImages(Skwasm::Surface* surface, + SkwasmObject imagePromises, + double rasterStart, + uint32_t callbackId); extern unsigned int skwasm_createGlTextureFromTextureSource( SkwasmObject textureSource, int width, diff --git a/lib/web_ui/skwasm/surface.cpp b/lib/web_ui/skwasm/surface.cpp index 28b64bd25a328..f661de69cd657 100644 --- a/lib/web_ui/skwasm/surface.cpp +++ b/lib/web_ui/skwasm/surface.cpp @@ -39,11 +39,19 @@ void Surface::dispose() { } // Main thread only -uint32_t Surface::renderPicture(SkPicture* picture) { +uint32_t Surface::renderPictures(SkPicture** pictures, int count) { assert(emscripten_is_main_browser_thread()); uint32_t callbackId = ++_currentCallbackId; - picture->ref(); - skwasm_dispatchRenderPicture(_thread, this, picture, callbackId); + std::unique_ptr[]> picturePointers = + std::make_unique[]>(count); + for (int i = 0; i < count; i++) { + picturePointers[i] = sk_ref_sp(pictures[i]); + } + + // Releasing picturePointers here and will recreate the unique_ptr on the + // other thread See skwasm_renderPicturesOnWorker + skwasm_dispatchRenderPictures(_thread, this, picturePointers.release(), count, + callbackId); return callbackId; } @@ -136,20 +144,31 @@ void Surface::_recreateSurface() { } // Worker thread only -void Surface::renderPictureOnWorker(SkPicture* picture, uint32_t callbackId) { - SkRect pictureRect = picture->cullRect(); - SkIRect roundedOutRect; - pictureRect.roundOut(&roundedOutRect); - _resizeCanvasToFit(roundedOutRect.width(), roundedOutRect.height()); - SkMatrix matrix = - SkMatrix::Translate(-roundedOutRect.fLeft, -roundedOutRect.fTop); - makeCurrent(_glContext); - auto canvas = _surface->getCanvas(); - canvas->drawColor(SK_ColorTRANSPARENT, SkBlendMode::kSrc); - canvas->drawPicture(sk_ref_sp(picture), &matrix, nullptr); - _grContext->flush(_surface.get()); - skwasm_captureImageBitmap(this, _glContext, callbackId, - roundedOutRect.width(), roundedOutRect.height()); +void Surface::renderPicturesOnWorker(sk_sp* pictures, + int pictureCount, + uint32_t callbackId, + double rasterStart) { + // This is populated by the `captureImageBitmap` call the first time it is + // passed in. + SkwasmObject imagePromiseArray = __builtin_wasm_ref_null_extern(); + for (int i = 0; i < pictureCount; i++) { + sk_sp picture = pictures[i]; + SkRect pictureRect = picture->cullRect(); + SkIRect roundedOutRect; + pictureRect.roundOut(&roundedOutRect); + _resizeCanvasToFit(roundedOutRect.width(), roundedOutRect.height()); + SkMatrix matrix = + SkMatrix::Translate(-roundedOutRect.fLeft, -roundedOutRect.fTop); + makeCurrent(_glContext); + auto canvas = _surface->getCanvas(); + canvas->drawColor(SK_ColorTRANSPARENT, SkBlendMode::kSrc); + canvas->drawPicture(picture, &matrix, nullptr); + _grContext->flush(_surface.get()); + imagePromiseArray = + skwasm_captureImageBitmap(_glContext, roundedOutRect.width(), + roundedOutRect.height(), imagePromiseArray); + } + skwasm_resolveAndPostImages(this, imagePromiseArray, rasterStart, callbackId); } void Surface::_rasterizeImage(SkImage* image, @@ -225,16 +244,22 @@ SKWASM_EXPORT void surface_destroy(Surface* surface) { surface->dispose(); } -SKWASM_EXPORT uint32_t surface_renderPicture(Surface* surface, - SkPicture* picture) { - return surface->renderPicture(picture); +SKWASM_EXPORT uint32_t surface_renderPictures(Surface* surface, + SkPicture** pictures, + int count) { + return surface->renderPictures(pictures, count); } -SKWASM_EXPORT void surface_renderPictureOnWorker(Surface* surface, - SkPicture* picture, - uint32_t callbackId) { - surface->renderPictureOnWorker(picture, callbackId); - picture->unref(); +SKWASM_EXPORT void surface_renderPicturesOnWorker(Surface* surface, + sk_sp* pictures, + int pictureCount, + uint32_t callbackId, + double rasterStart) { + // This will release the pictures when they leave scope. + std::unique_ptr> picturesPointer = + std::unique_ptr>(pictures); + surface->renderPicturesOnWorker(pictures, pictureCount, callbackId, + rasterStart); } SKWASM_EXPORT uint32_t surface_rasterizeImage(Surface* surface, diff --git a/lib/web_ui/skwasm/surface.h b/lib/web_ui/skwasm/surface.h index 3577a947c782a..7e1d5fbb526e1 100644 --- a/lib/web_ui/skwasm/surface.h +++ b/lib/web_ui/skwasm/surface.h @@ -62,7 +62,7 @@ class Surface { // Main thread only void dispose(); - uint32_t renderPicture(SkPicture* picture); + uint32_t renderPictures(SkPicture** picture, int count); uint32_t rasterizeImage(SkImage* image, ImageByteFormat format); void setCallbackHandler(CallbackHandler* callbackHandler); void onRenderComplete(uint32_t callbackId, SkwasmObject imageBitmap); @@ -72,7 +72,10 @@ class Surface { SkwasmObject textureSource); // Worker thread - void renderPictureOnWorker(SkPicture* picture, uint32_t callbackId); + void renderPicturesOnWorker(sk_sp* picture, + int pictureCount, + uint32_t callbackId, + double rasterStart); private: void _runWorker(); diff --git a/lib/web_ui/test/canvaskit/frame_timings_test.dart b/lib/web_ui/test/canvaskit/frame_timings_test.dart deleted file mode 100644 index 0cacd4246616b..0000000000000 --- a/lib/web_ui/test/canvaskit/frame_timings_test.dart +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2013 The Flutter Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -import 'package:test/bootstrap/browser.dart'; -import 'package:test/test.dart'; - -import '../common/frame_timings_common.dart'; -import 'common.dart'; - -void main() { - internalBootstrapBrowserTest(() => testMain); -} - -void testMain() { - group('frame timings', () { - setUpCanvasKitTest(withImplicitView: true); - - test('collects frame timings', () async { - await runFrameTimingsTest(); - }); - }); -} diff --git a/lib/web_ui/test/common/frame_timings_common.dart b/lib/web_ui/test/common/frame_timings_common.dart deleted file mode 100644 index 314e1a808861e..0000000000000 --- a/lib/web_ui/test/common/frame_timings_common.dart +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2013 The Flutter Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -import 'dart:async'; - -import 'package:test/test.dart'; -import 'package:ui/src/engine.dart' show EnginePlatformDispatcher; -import 'package:ui/ui.dart' as ui; - -/// Tests frame timings in a renderer-agnostic way. -/// -/// See CanvasKit-specific and HTML-specific test files `frame_timings_test.dart`. -Future runFrameTimingsTest() async { - final EnginePlatformDispatcher dispatcher = ui.PlatformDispatcher.instance as EnginePlatformDispatcher; - - List? timings; - dispatcher.onReportTimings = (List data) { - timings = data; - }; - Completer frameDone = Completer(); - dispatcher.onDrawFrame = () { - final ui.SceneBuilder sceneBuilder = ui.SceneBuilder(); - sceneBuilder - ..pushOffset(0, 0) - ..pop(); - dispatcher.render(sceneBuilder.build()).then((_) { - frameDone.complete(); - }); - }; - - // Frame 1. - dispatcher.scheduleFrame(); - await frameDone.future; - expect(timings, isNull, reason: "100 ms hasn't passed yet"); - await Future.delayed(const Duration(milliseconds: 150)); - - // Frame 2. - frameDone = Completer(); - dispatcher.scheduleFrame(); - await frameDone.future; - expect(timings, hasLength(2), reason: '100 ms passed. 2 frames pumped.'); - for (final ui.FrameTiming timing in timings!) { - expect(timing.vsyncOverhead, greaterThanOrEqualTo(Duration.zero)); - expect(timing.buildDuration, greaterThanOrEqualTo(Duration.zero)); - expect(timing.rasterDuration, greaterThanOrEqualTo(Duration.zero)); - expect(timing.totalSpan, greaterThanOrEqualTo(Duration.zero)); - expect(timing.layerCacheCount, equals(0)); - expect(timing.layerCacheBytes, equals(0)); - expect(timing.pictureCacheCount, equals(0)); - expect(timing.pictureCacheBytes, equals(0)); - } -} diff --git a/lib/web_ui/test/engine/scene_view_test.dart b/lib/web_ui/test/engine/scene_view_test.dart index 48e84b717f5eb..93d54b09b226f 100644 --- a/lib/web_ui/test/engine/scene_view_test.dart +++ b/lib/web_ui/test/engine/scene_view_test.dart @@ -24,17 +24,23 @@ class StubPictureRenderer implements PictureRenderer { createDomCanvasElement(width: 500, height: 500); @override - Future renderPicture(ScenePicture picture) async { - renderedPictures.add(picture); - final ui.Rect cullRect = picture.cullRect; - final DomImageBitmap bitmap = - await createImageBitmap(scratchCanvasElement as JSObject, ( - x: 0, - y: 0, - width: cullRect.width.toInt(), - height: cullRect.height.toInt(), - )); - return bitmap; + Future renderPictures(List pictures) async { + renderedPictures.addAll(pictures); + final List bitmaps = await Future.wait(pictures.map((ScenePicture picture) { + final ui.Rect cullRect = picture.cullRect; + final Future bitmap = createImageBitmap(scratchCanvasElement as JSObject, ( + x: 0, + y: 0, + width: cullRect.width.toInt(), + height: cullRect.height.toInt(), + )); + return bitmap; + })); + return ( + imageBitmaps: bitmaps, + rasterStartMicros: 0, + rasterEndMicros: 0, + ); } List renderedPictures = []; @@ -65,7 +71,7 @@ void testMain() { final EngineRootLayer rootLayer = EngineRootLayer(); rootLayer.slices.add(PictureSlice(picture)); final EngineScene scene = EngineScene(rootLayer); - await sceneView.renderScene(scene); + await sceneView.renderScene(scene, null); final DomElement sceneElement = sceneView.sceneElement; final List children = sceneElement.children.toList(); @@ -100,7 +106,7 @@ void testMain() { final EngineRootLayer rootLayer = EngineRootLayer(); rootLayer.slices.add(PlatformViewSlice([platformView], null)); final EngineScene scene = EngineScene(rootLayer); - await sceneView.renderScene(scene); + await sceneView.renderScene(scene, null); final DomElement sceneElement = sceneView.sceneElement; final List children = sceneElement.children.toList(); @@ -134,7 +140,7 @@ void testMain() { final EngineRootLayer rootLayer = EngineRootLayer(); rootLayer.slices.add(PictureSlice(picture)); final EngineScene scene = EngineScene(rootLayer); - renderFutures.add(sceneView.renderScene(scene)); + renderFutures.add(sceneView.renderScene(scene, null)); } await Future.wait(renderFutures); diff --git a/lib/web_ui/test/engine/surface/frame_timings_test.dart b/lib/web_ui/test/engine/surface/frame_timings_test.dart deleted file mode 100644 index 14ec8f2e353da..0000000000000 --- a/lib/web_ui/test/engine/surface/frame_timings_test.dart +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2013 The Flutter Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -import 'package:test/bootstrap/browser.dart'; -import 'package:test/test.dart'; - -import '../../common/frame_timings_common.dart'; -import '../../common/test_initialization.dart'; - -void main() { - internalBootstrapBrowserTest(() => testMain); -} - -void testMain() { - setUp(() async { - await bootstrapAndRunApp(withImplicitView: true); - }); - - test('collects frame timings', () async { - await runFrameTimingsTest(); - }); -} diff --git a/lib/web_ui/test/ui/frame_timings_test.dart b/lib/web_ui/test/ui/frame_timings_test.dart new file mode 100644 index 0000000000000..a49baab650635 --- /dev/null +++ b/lib/web_ui/test/ui/frame_timings_test.dart @@ -0,0 +1,60 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +import 'package:test/bootstrap/browser.dart'; +import 'package:test/test.dart'; +import 'package:ui/ui.dart' as ui; + +import '../common/test_initialization.dart'; + +void main() { + internalBootstrapBrowserTest(() => testMain); +} + +void testMain() { + setUp(() async { + await bootstrapAndRunApp(withImplicitView: true); + }); + + test('collects frame timings', () async { + List? timings; + ui.PlatformDispatcher.instance.onReportTimings = (List data) { + timings = data; + }; + Completer frameDone = Completer(); + ui.PlatformDispatcher.instance.onDrawFrame = () { + final ui.SceneBuilder sceneBuilder = ui.SceneBuilder(); + sceneBuilder + ..pushOffset(0, 0) + ..pop(); + ui.PlatformDispatcher.instance.render(sceneBuilder.build()).then((_) { + frameDone.complete(); + }); + }; + + // Frame 1. + ui.PlatformDispatcher.instance.scheduleFrame(); + await frameDone.future; + expect(timings, isNull, reason: "100 ms hasn't passed yet"); + await Future.delayed(const Duration(milliseconds: 150)); + + // Frame 2. + frameDone = Completer(); + ui.PlatformDispatcher.instance.scheduleFrame(); + await frameDone.future; + expect(timings, hasLength(2), reason: '100 ms passed. 2 frames pumped.'); + for (final ui.FrameTiming timing in timings!) { + expect(timing.vsyncOverhead, greaterThanOrEqualTo(Duration.zero)); + expect(timing.buildDuration, greaterThanOrEqualTo(Duration.zero)); + expect(timing.rasterDuration, greaterThanOrEqualTo(Duration.zero)); + expect(timing.totalSpan, greaterThanOrEqualTo(Duration.zero)); + expect(timing.layerCacheCount, equals(0)); + expect(timing.layerCacheBytes, equals(0)); + expect(timing.pictureCacheCount, equals(0)); + expect(timing.pictureCacheBytes, equals(0)); + } + }); +} From 13038117050b839c14efa3858b13467d2e6b08d1 Mon Sep 17 00:00:00 2001 From: Jackson Gardner Date: Fri, 16 Feb 2024 16:04:17 -0800 Subject: [PATCH 2/6] Fix issue from merge conflict. --- lib/web_ui/test/ui/frame_timings_test.dart | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/web_ui/test/ui/frame_timings_test.dart b/lib/web_ui/test/ui/frame_timings_test.dart index a49baab650635..62f83b71d4d09 100644 --- a/lib/web_ui/test/ui/frame_timings_test.dart +++ b/lib/web_ui/test/ui/frame_timings_test.dart @@ -6,6 +6,7 @@ import 'dart:async'; import 'package:test/bootstrap/browser.dart'; import 'package:test/test.dart'; +import 'package:ui/src/engine.dart'; import 'package:ui/ui.dart' as ui; import '../common/test_initialization.dart'; @@ -20,30 +21,31 @@ void testMain() { }); test('collects frame timings', () async { + final EnginePlatformDispatcher dispatcher = ui.PlatformDispatcher.instance as EnginePlatformDispatcher; List? timings; - ui.PlatformDispatcher.instance.onReportTimings = (List data) { + dispatcher.onReportTimings = (List data) { timings = data; }; Completer frameDone = Completer(); - ui.PlatformDispatcher.instance.onDrawFrame = () { + dispatcher.onDrawFrame = () { final ui.SceneBuilder sceneBuilder = ui.SceneBuilder(); sceneBuilder ..pushOffset(0, 0) ..pop(); - ui.PlatformDispatcher.instance.render(sceneBuilder.build()).then((_) { + dispatcher.render(sceneBuilder.build()).then((_) { frameDone.complete(); }); }; // Frame 1. - ui.PlatformDispatcher.instance.scheduleFrame(); + dispatcher.scheduleFrame(); await frameDone.future; expect(timings, isNull, reason: "100 ms hasn't passed yet"); await Future.delayed(const Duration(milliseconds: 150)); // Frame 2. frameDone = Completer(); - ui.PlatformDispatcher.instance.scheduleFrame(); + dispatcher.scheduleFrame(); await frameDone.future; expect(timings, hasLength(2), reason: '100 ms passed. 2 frames pumped.'); for (final ui.FrameTiming timing in timings!) { From 890cff3e68744651e0e2316a7be7df518db5125d Mon Sep 17 00:00:00 2001 From: Jackson Gardner Date: Wed, 21 Feb 2024 13:04:41 -0800 Subject: [PATCH 3/6] Don't fail if more than one scene is rendered in a frame. --- .../lib/src/engine/canvaskit/renderer.dart | 2 +- .../lib/src/engine/frame_timing_recorder.dart | 41 +++++++++---------- .../lib/src/engine/html/scene_builder.dart | 2 +- lib/web_ui/lib/src/engine/initialization.dart | 22 ++++------ .../engine/skwasm/skwasm_impl/renderer.dart | 2 +- 5 files changed, 32 insertions(+), 37 deletions(-) diff --git a/lib/web_ui/lib/src/engine/canvaskit/renderer.dart b/lib/web_ui/lib/src/engine/canvaskit/renderer.dart index 1b9c3b7cc73b4..7d673a63c6b3e 100644 --- a/lib/web_ui/lib/src/engine/canvaskit/renderer.dart +++ b/lib/web_ui/lib/src/engine/canvaskit/renderer.dart @@ -417,7 +417,7 @@ class CanvasKitRenderer implements Renderer { "Unable to render to a view which hasn't been registered"); final ViewRasterizer rasterizer = _rasterizers[view.viewId]!; final RenderQueue renderQueue = rasterizer.queue; - final FrameTimingRecorder? recorder = FrameTimingRecorder.currentRecorder; + final FrameTimingRecorder? recorder = FrameTimingRecorder.frameTimingsEnabled ? FrameTimingRecorder() : null; if (renderQueue.current != null) { // If a scene is already queued up, drop it and queue this one up instead // so that the scene view always displays the most recently requested scene. diff --git a/lib/web_ui/lib/src/engine/frame_timing_recorder.dart b/lib/web_ui/lib/src/engine/frame_timing_recorder.dart index b0fe2f506e15b..8fa85108cbab6 100644 --- a/lib/web_ui/lib/src/engine/frame_timing_recorder.dart +++ b/lib/web_ui/lib/src/engine/frame_timing_recorder.dart @@ -6,8 +6,9 @@ import 'package:ui/src/engine.dart'; import 'package:ui/ui.dart' as ui; class FrameTimingRecorder { - int? _vsyncStartMicros; - int? _buildStartMicros; + final int _vsyncStartMicros = _currentFrameVsyncStart; + final int _buildStartMicros = _currentFrameBuildStart; + int? _buildFinishMicros; int? _rasterStartMicros; int? _rasterFinishMicros; @@ -17,14 +18,23 @@ class FrameTimingRecorder { /// This list is periodically reported to the framework (see [_kFrameTimingsSubmitInterval]). static List _frameTimings = []; - static void startFrame() { - _currentFrameTimingRecorder = frameTimingsEnabled - ? FrameTimingRecorder() - : null; + /// These two metrics are collected early in the process, before the respective + /// scene builders are created. These are instead treated as global state, which + /// are used to initialize any recorders that are created by the scene builders. + static late int _currentFrameVsyncStart; + static late int _currentFrameBuildStart; + + static void recordCurrentFrameVsync() { + if (frameTimingsEnabled) { + _currentFrameVsyncStart = _nowMicros(); + } } - static FrameTimingRecorder? _currentFrameTimingRecorder; - static FrameTimingRecorder? get currentRecorder => _currentFrameTimingRecorder; + static void recordCurrentFrameBuildStart() { + if (frameTimingsEnabled) { + _currentFrameBuildStart = _nowMicros(); + } + } /// The last time (in microseconds) we submitted frame timings. static int _frameTimingsLastSubmitTime = _nowMicros(); @@ -49,16 +59,6 @@ class FrameTimingRecorder { return (domWindow.performance.now() * 1000).toInt(); } - void recordVsyncStart([int? vsyncStart]) { - assert(_vsyncStartMicros == null, "can't record vsync start more than once"); - _vsyncStartMicros = vsyncStart ?? _nowMicros(); - } - - void recordBuildStart([int? buildStart]) { - assert(_buildStartMicros == null, "can't record build start more than once"); - _buildStartMicros = buildStart ?? _nowMicros(); - } - void recordBuildFinish([int? buildFinish]) { assert(_buildFinishMicros == null, "can't record build finish more than once"); _buildFinishMicros = buildFinish ?? _nowMicros(); @@ -76,15 +76,14 @@ class FrameTimingRecorder { void submitTimings() { assert( - _buildStartMicros != null && _buildFinishMicros != null && _rasterStartMicros != null && _rasterFinishMicros != null, 'Attempted to submit an incomplete timings.' ); final ui.FrameTiming timing = ui.FrameTiming( - vsyncStart: _vsyncStartMicros!, - buildStart: _buildStartMicros!, + vsyncStart: _vsyncStartMicros, + buildStart: _buildStartMicros, buildFinish: _buildFinishMicros!, rasterStart: _rasterStartMicros!, rasterFinish: _rasterFinishMicros!, diff --git a/lib/web_ui/lib/src/engine/html/scene_builder.dart b/lib/web_ui/lib/src/engine/html/scene_builder.dart index 28e692c139e4a..701bb11ef92ad 100644 --- a/lib/web_ui/lib/src/engine/html/scene_builder.dart +++ b/lib/web_ui/lib/src/engine/html/scene_builder.dart @@ -511,7 +511,7 @@ class SurfaceSceneBuilder implements ui.SceneBuilder { // In the HTML renderer we time the beginning of the rasterization phase // (counter-intuitively) in SceneBuilder.build because DOM updates happen // here. This is different from CanvasKit. - final FrameTimingRecorder? recorder = FrameTimingRecorder.currentRecorder; + final FrameTimingRecorder? recorder = FrameTimingRecorder.frameTimingsEnabled ? FrameTimingRecorder() : null; recorder?.recordBuildFinish(); recorder?.recordRasterStart(); timeAction(kProfilePrerollFrame, () { diff --git a/lib/web_ui/lib/src/engine/initialization.dart b/lib/web_ui/lib/src/engine/initialization.dart index af6c3479a787a..c2e27c2cf08f7 100644 --- a/lib/web_ui/lib/src/engine/initialization.dart +++ b/lib/web_ui/lib/src/engine/initialization.dart @@ -158,19 +158,15 @@ Future initializeEngineServices({ if (!waitingForAnimation) { waitingForAnimation = true; domWindow.requestAnimationFrame((JSNumber highResTime) { - FrameTimingRecorder.startFrame(); - final FrameTimingRecorder? recorder = FrameTimingRecorder.currentRecorder; - if (recorder != null) { - recorder.recordVsyncStart(); - - // In Flutter terminology "building a frame" consists of "beginning - // frame" and "drawing frame". - // - // We do not call `recordBuildFinish` from here because - // part of the rasterization process, particularly in the HTML - // renderer, takes place in the `SceneBuilder.build()`. - recorder.recordBuildStart(); - } + FrameTimingRecorder.recordCurrentFrameVsync(); + + // In Flutter terminology "building a frame" consists of "beginning + // frame" and "drawing frame". + // + // We do not call `recordBuildFinish` from here because + // part of the rasterization process, particularly in the HTML + // renderer, takes place in the `SceneBuilder.build()`. + FrameTimingRecorder.recordCurrentFrameBuildStart(); // Reset immediately, because `frameHandler` can schedule more frames. waitingForAnimation = false; diff --git a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/renderer.dart b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/renderer.dart index b87a5c0a3aef2..a0bfd4780da08 100644 --- a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/renderer.dart +++ b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/renderer.dart @@ -401,7 +401,7 @@ class SkwasmRenderer implements Renderer { // https://github.com/flutter/flutter/issues/137073. @override Future renderScene(ui.Scene scene, ui.FlutterView view) { - final FrameTimingRecorder? recorder = FrameTimingRecorder.currentRecorder; + final FrameTimingRecorder? recorder = FrameTimingRecorder.frameTimingsEnabled ? FrameTimingRecorder() : null; recorder?.recordBuildFinish(); view as EngineFlutterView; From b9e1b1f9e1d16eb261bdc15c40f4d8065f15bc92 Mon Sep 17 00:00:00 2001 From: Jackson Gardner Date: Wed, 21 Feb 2024 13:45:17 -0800 Subject: [PATCH 4/6] Address comments. --- lib/web_ui/lib/frame_timings.dart | 105 ------------------ lib/web_ui/lib/platform_dispatcher.dart | 100 +++++++++++++++++ lib/web_ui/lib/src/engine/dom.dart | 2 +- .../src/engine/skwasm/skwasm_impl/image.dart | 5 +- .../engine/skwasm/skwasm_impl/surface.dart | 8 +- lib/web_ui/lib/ui.dart | 1 - lib/web_ui/skwasm/library_skwasm_support.js | 4 +- lib/web_ui/skwasm/surface.cpp | 2 +- 8 files changed, 110 insertions(+), 117 deletions(-) delete mode 100644 lib/web_ui/lib/frame_timings.dart diff --git a/lib/web_ui/lib/frame_timings.dart b/lib/web_ui/lib/frame_timings.dart deleted file mode 100644 index e010819b5ca73..0000000000000 --- a/lib/web_ui/lib/frame_timings.dart +++ /dev/null @@ -1,105 +0,0 @@ -// Copyright 2013 The Flutter Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -part of ui; - -enum FramePhase { - vsyncStart, - buildStart, - buildFinish, - rasterStart, - rasterFinish, - rasterFinishWallTime, -} - -enum _FrameTimingInfo { - layerCacheCount, - layerCacheBytes, - pictureCacheCount, - pictureCacheBytes, - frameNumber, -} - -class FrameTiming { - factory FrameTiming({ - required int vsyncStart, - required int buildStart, - required int buildFinish, - required int rasterStart, - required int rasterFinish, - required int rasterFinishWallTime, - int layerCacheCount = 0, - int layerCacheBytes = 0, - int pictureCacheCount = 0, - int pictureCacheBytes = 0, - int frameNumber = 1, - }) { - return FrameTiming._([ - vsyncStart, - buildStart, - buildFinish, - rasterStart, - rasterFinish, - rasterFinishWallTime, - layerCacheCount, - layerCacheBytes, - pictureCacheCount, - pictureCacheBytes, - frameNumber, - ]); - } - - FrameTiming._(this._data) - : assert(_data.length == _dataLength); - - static final int _dataLength = FramePhase.values.length + _FrameTimingInfo.values.length; - - int timestampInMicroseconds(FramePhase phase) => _data[phase.index]; - - Duration _rawDuration(FramePhase phase) => Duration(microseconds: _data[phase.index]); - - int _rawInfo(_FrameTimingInfo info) => _data[FramePhase.values.length + info.index]; - - Duration get buildDuration => - _rawDuration(FramePhase.buildFinish) - _rawDuration(FramePhase.buildStart); - - Duration get rasterDuration => - _rawDuration(FramePhase.rasterFinish) - _rawDuration(FramePhase.rasterStart); - - Duration get vsyncOverhead => _rawDuration(FramePhase.buildStart) - _rawDuration(FramePhase.vsyncStart); - - Duration get totalSpan => - _rawDuration(FramePhase.rasterFinish) - _rawDuration(FramePhase.vsyncStart); - - int get layerCacheCount => _rawInfo(_FrameTimingInfo.layerCacheCount); - - int get layerCacheBytes => _rawInfo(_FrameTimingInfo.layerCacheBytes); - - double get layerCacheMegabytes => layerCacheBytes / 1024.0 / 1024.0; - - int get pictureCacheCount => _rawInfo(_FrameTimingInfo.pictureCacheCount); - - int get pictureCacheBytes => _rawInfo(_FrameTimingInfo.pictureCacheBytes); - - double get pictureCacheMegabytes => pictureCacheBytes / 1024.0 / 1024.0; - - int get frameNumber => _data.last; - - final List _data; // some elements in microseconds, some in bytes, some are counts - - String _formatMS(Duration duration) => '${duration.inMicroseconds * 0.001}ms'; - - @override - String toString() { - return '$runtimeType(buildDuration: ${_formatMS(buildDuration)}, ' - 'rasterDuration: ${_formatMS(rasterDuration)}, ' - 'vsyncOverhead: ${_formatMS(vsyncOverhead)}, ' - 'totalSpan: ${_formatMS(totalSpan)}, ' - 'layerCacheCount: $layerCacheCount, ' - 'layerCacheBytes: $layerCacheBytes, ' - 'pictureCacheCount: $pictureCacheCount, ' - 'pictureCacheBytes: $pictureCacheBytes, ' - 'frameNumber: ${_data.last})'; - } -} diff --git a/lib/web_ui/lib/platform_dispatcher.dart b/lib/web_ui/lib/platform_dispatcher.dart index a4326af23c651..fb351b27c298e 100644 --- a/lib/web_ui/lib/platform_dispatcher.dart +++ b/lib/web_ui/lib/platform_dispatcher.dart @@ -158,6 +158,106 @@ abstract class PlatformDispatcher { double scaleFontSize(double unscaledFontSize); } +enum FramePhase { + vsyncStart, + buildStart, + buildFinish, + rasterStart, + rasterFinish, + rasterFinishWallTime, +} + +enum _FrameTimingInfo { + layerCacheCount, + layerCacheBytes, + pictureCacheCount, + pictureCacheBytes, + frameNumber, +} + +class FrameTiming { + factory FrameTiming({ + required int vsyncStart, + required int buildStart, + required int buildFinish, + required int rasterStart, + required int rasterFinish, + required int rasterFinishWallTime, + int layerCacheCount = 0, + int layerCacheBytes = 0, + int pictureCacheCount = 0, + int pictureCacheBytes = 0, + int frameNumber = 1, + }) { + return FrameTiming._([ + vsyncStart, + buildStart, + buildFinish, + rasterStart, + rasterFinish, + rasterFinishWallTime, + layerCacheCount, + layerCacheBytes, + pictureCacheCount, + pictureCacheBytes, + frameNumber, + ]); + } + + FrameTiming._(this._data) + : assert(_data.length == _dataLength); + + static final int _dataLength = FramePhase.values.length + _FrameTimingInfo.values.length; + + int timestampInMicroseconds(FramePhase phase) => _data[phase.index]; + + Duration _rawDuration(FramePhase phase) => Duration(microseconds: _data[phase.index]); + + int _rawInfo(_FrameTimingInfo info) => _data[FramePhase.values.length + info.index]; + + Duration get buildDuration => + _rawDuration(FramePhase.buildFinish) - _rawDuration(FramePhase.buildStart); + + Duration get rasterDuration => + _rawDuration(FramePhase.rasterFinish) - _rawDuration(FramePhase.rasterStart); + + Duration get vsyncOverhead => _rawDuration(FramePhase.buildStart) - _rawDuration(FramePhase.vsyncStart); + + Duration get totalSpan => + _rawDuration(FramePhase.rasterFinish) - _rawDuration(FramePhase.vsyncStart); + + int get layerCacheCount => _rawInfo(_FrameTimingInfo.layerCacheCount); + + int get layerCacheBytes => _rawInfo(_FrameTimingInfo.layerCacheBytes); + + double get layerCacheMegabytes => layerCacheBytes / 1024.0 / 1024.0; + + int get pictureCacheCount => _rawInfo(_FrameTimingInfo.pictureCacheCount); + + int get pictureCacheBytes => _rawInfo(_FrameTimingInfo.pictureCacheBytes); + + double get pictureCacheMegabytes => pictureCacheBytes / 1024.0 / 1024.0; + + int get frameNumber => _data.last; + + final List _data; // some elements in microseconds, some in bytes, some are counts + + String _formatMS(Duration duration) => '${duration.inMicroseconds * 0.001}ms'; + + @override + String toString() { + return '$runtimeType(buildDuration: ${_formatMS(buildDuration)}, ' + 'rasterDuration: ${_formatMS(rasterDuration)}, ' + 'vsyncOverhead: ${_formatMS(vsyncOverhead)}, ' + 'totalSpan: ${_formatMS(totalSpan)}, ' + 'layerCacheCount: $layerCacheCount, ' + 'layerCacheBytes: $layerCacheBytes, ' + 'pictureCacheCount: $pictureCacheCount, ' + 'pictureCacheBytes: $pictureCacheBytes, ' + 'frameNumber: ${_data.last})'; + } +} + enum AppLifecycleState { detached, resumed, diff --git a/lib/web_ui/lib/src/engine/dom.dart b/lib/web_ui/lib/src/engine/dom.dart index cd2c9a9b9af91..271acfca43567 100644 --- a/lib/web_ui/lib/src/engine/dom.dart +++ b/lib/web_ui/lib/src/engine/dom.dart @@ -1487,7 +1487,7 @@ class DomCanvasRenderingContextBitmapRenderer {} extension DomCanvasRenderingContextBitmapRendererExtension on DomCanvasRenderingContextBitmapRenderer { - external void transferFromImageBitmap(DomImageBitmap bitmap); + external void transferFromImageBitmap(DomImageBitmap? bitmap); } @JS('ImageData') diff --git a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/image.dart b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/image.dart index 80a5e733da3b5..2b800ba276964 100644 --- a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/image.dart +++ b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/image.dart @@ -64,7 +64,7 @@ class SkwasmImage extends SkwasmObjectWrapper implements ui.Image { final DomImageBitmap bitmap = (await (renderer as SkwasmRenderer).surface.renderPictures( [recorder.endRecording() as SkwasmPicture], - )).imageBitmaps.first; + )).imageBitmaps.single; final DomOffscreenCanvas offscreenCanvas = createDomOffscreenCanvas(bitmap.width.toDartInt, bitmap.height.toDartInt); final DomCanvasRenderingContextBitmapRenderer context = @@ -75,8 +75,7 @@ class SkwasmImage extends SkwasmObjectWrapper implements ui.Image { // Zero out the contents of the canvas so that resources can be reclaimed // by the browser. - offscreenCanvas.width = 0; - offscreenCanvas.height = 0; + context.transferFromImageBitmap(null); return ByteData.view(arrayBuffer.toDart); } else { return (renderer as SkwasmRenderer).surface.rasterizeImage(this, format); diff --git a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/surface.dart b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/surface.dart index 261ba8381be9c..eddcc19ff402b 100644 --- a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/surface.dart +++ b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/surface.dart @@ -18,8 +18,8 @@ import 'package:ui/ui.dart' as ui; class RasterResult {} extension RasterResultExtension on RasterResult { - external JSNumber get rasterStart; - external JSNumber get rasterEnd; + external JSNumber get rasterStartMilliseconds; + external JSNumber get rasterEndMilliseconds; external JSArray get imageBitmaps; } @@ -100,8 +100,8 @@ class SkwasmSurface { final RasterResult rasterResult = (await SkwasmCallbackHandler.instance.registerCallback(callbackId)) as RasterResult; final RenderResult result = ( imageBitmaps: rasterResult.imageBitmaps.toDart.cast(), - rasterStartMicros: (rasterResult.rasterStart.toDartDouble * 1000).toInt(), - rasterEndMicros: (rasterResult.rasterEnd.toDartDouble * 1000).toInt(), + rasterStartMicros: (rasterResult.rasterStartMilliseconds.toDartDouble * 1000).toInt(), + rasterEndMicros: (rasterResult.rasterEndMilliseconds.toDartDouble * 1000).toInt(), ); return result; }); diff --git a/lib/web_ui/lib/ui.dart b/lib/web_ui/lib/ui.dart index b0c8a346e001c..8b5130e44d2d0 100644 --- a/lib/web_ui/lib/ui.dart +++ b/lib/web_ui/lib/ui.dart @@ -20,7 +20,6 @@ part 'annotations.dart'; part 'canvas.dart'; part 'channel_buffers.dart'; part 'compositing.dart'; -part 'frame_timings.dart'; part 'geometry.dart'; part 'hash_codes.dart'; part 'initialization.dart'; diff --git a/lib/web_ui/skwasm/library_skwasm_support.js b/lib/web_ui/skwasm/library_skwasm_support.js index 63ef28eaa2bbe..76cbac2db2a87 100644 --- a/lib/web_ui/skwasm/library_skwasm_support.js +++ b/lib/web_ui/skwasm/library_skwasm_support.js @@ -35,8 +35,8 @@ mergeInto(LibraryManager.library, { data.surface, data.callbackId, { "imageBitmaps": data.imageBitmaps, - "rasterStart": data.rasterStart, - "rasterEnd": data.rasterEnd, + "rasterStartMilliseconds": data.rasterStart, + "rasterEndMilliseconds": data.rasterEnd, }, ); return; diff --git a/lib/web_ui/skwasm/surface.cpp b/lib/web_ui/skwasm/surface.cpp index f661de69cd657..a1c24b4bad83e 100644 --- a/lib/web_ui/skwasm/surface.cpp +++ b/lib/web_ui/skwasm/surface.cpp @@ -49,7 +49,7 @@ uint32_t Surface::renderPictures(SkPicture** pictures, int count) { } // Releasing picturePointers here and will recreate the unique_ptr on the - // other thread See skwasm_renderPicturesOnWorker + // other thread See surface_renderPicturesOnWorker skwasm_dispatchRenderPictures(_thread, this, picturePointers.release(), count, callbackId); return callbackId; From 769775e944ae8286ab5b416a8819aa7a1d88c5d7 Mon Sep 17 00:00:00 2001 From: Jackson Gardner Date: Wed, 21 Feb 2024 13:47:37 -0800 Subject: [PATCH 5/6] Don't use late variables to be more lenient in test scenarios. --- lib/web_ui/lib/src/engine/frame_timing_recorder.dart | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/web_ui/lib/src/engine/frame_timing_recorder.dart b/lib/web_ui/lib/src/engine/frame_timing_recorder.dart index 8fa85108cbab6..ec2944bb41327 100644 --- a/lib/web_ui/lib/src/engine/frame_timing_recorder.dart +++ b/lib/web_ui/lib/src/engine/frame_timing_recorder.dart @@ -21,8 +21,8 @@ class FrameTimingRecorder { /// These two metrics are collected early in the process, before the respective /// scene builders are created. These are instead treated as global state, which /// are used to initialize any recorders that are created by the scene builders. - static late int _currentFrameVsyncStart; - static late int _currentFrameBuildStart; + static int _currentFrameVsyncStart = 0; + static int _currentFrameBuildStart = 0; static void recordCurrentFrameVsync() { if (frameTimingsEnabled) { From 698c43dbeb486774dd9bdecc3edd6beb3c54c4dc Mon Sep 17 00:00:00 2001 From: Jackson Gardner Date: Wed, 21 Feb 2024 14:03:48 -0800 Subject: [PATCH 6/6] Update license golden. --- ci/licenses_golden/licenses_flutter | 2 -- 1 file changed, 2 deletions(-) diff --git a/ci/licenses_golden/licenses_flutter b/ci/licenses_golden/licenses_flutter index 7e9691da09e59..f72553daaee65 100644 --- a/ci/licenses_golden/licenses_flutter +++ b/ci/licenses_golden/licenses_flutter @@ -9682,7 +9682,6 @@ ORIGIN: ../../../flutter/lib/web_ui/lib/annotations.dart + ../../../flutter/LICE ORIGIN: ../../../flutter/lib/web_ui/lib/canvas.dart + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/web_ui/lib/channel_buffers.dart + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/web_ui/lib/compositing.dart + ../../../flutter/LICENSE -ORIGIN: ../../../flutter/lib/web_ui/lib/frame_timings.dart + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/web_ui/lib/geometry.dart + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/web_ui/lib/hash_codes.dart + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/web_ui/lib/initialization.dart + ../../../flutter/LICENSE @@ -12521,7 +12520,6 @@ FILE: ../../../flutter/lib/web_ui/lib/annotations.dart FILE: ../../../flutter/lib/web_ui/lib/canvas.dart FILE: ../../../flutter/lib/web_ui/lib/channel_buffers.dart FILE: ../../../flutter/lib/web_ui/lib/compositing.dart -FILE: ../../../flutter/lib/web_ui/lib/frame_timings.dart FILE: ../../../flutter/lib/web_ui/lib/geometry.dart FILE: ../../../flutter/lib/web_ui/lib/hash_codes.dart FILE: ../../../flutter/lib/web_ui/lib/initialization.dart