From 72bb4176c25deba8df7292abfc8384c84271511a Mon Sep 17 00:00:00 2001 From: Jake Macdonald Date: Tue, 22 Jul 2025 21:18:48 +0000 Subject: [PATCH 1/5] update all flutter driver schema arguments to be strings --- pkgs/dart_mcp_server/lib/src/mixins/dtd.dart | 71 +++++++++++--------- 1 file changed, 38 insertions(+), 33 deletions(-) diff --git a/pkgs/dart_mcp_server/lib/src/mixins/dtd.dart b/pkgs/dart_mcp_server/lib/src/mixins/dtd.dart index 7f84b305..44121e11 100644 --- a/pkgs/dart_mcp_server/lib/src/mixins/dtd.dart +++ b/pkgs/dart_mcp_server/lib/src/mixins/dtd.dart @@ -181,6 +181,7 @@ base mixin DartToolingDaemonSupport return _flutterDriverNotRegistered; } final vm = await vmService.getVM(); + final timeout = request.arguments?['timeout'] as String?; final result = await vmService .callServiceExtension( _flutterDriverService, @@ -189,9 +190,9 @@ base mixin DartToolingDaemonSupport ) .timeout( Duration( - milliseconds: - (request.arguments?['timeout'] as int?) ?? - _defaultTimeoutMs, + milliseconds: timeout != null + ? int.parse(timeout) + : _defaultTimeoutMs, ), onTimeout: () => Response.parse({ 'isError': true, @@ -469,7 +470,7 @@ base mixin DartToolingDaemonSupport 'groupName': inspectorObjectGroup, // TODO: consider making these configurable or using defaults that // are better for the LLM. - 'isSummaryTree': 'true', + 'isSummaryTree': 'false', 'withPreviews': 'true', 'fullDetails': 'false', }, @@ -684,35 +685,36 @@ base mixin DartToolingDaemonSupport ], description: 'The name of the driver command', ), - 'alignment': Schema.num( + 'alignment': Schema.string( description: - 'How the widget should be aligned. ' - 'Required for the scrollIntoView command', + 'Required for the scrollIntoView command, how the widget should ' + 'be aligned', ), - 'duration': Schema.int( + 'duration': Schema.string( description: - 'The duration of the scrolling action in microseconds. ' - 'Required for the scroll command', + 'Required for the scroll command, the duration of the ' + 'scrolling action in microseconds as a stringified integer.', ), - 'dx': Schema.int( + 'dx': Schema.string( description: - 'Delta X offset for move event. Required for the scroll command', + 'Required for the scroll command, the delta X offset for move ' + 'event as a stringified double', ), - 'dy': Schema.int( + 'dy': Schema.string( description: - 'Delta Y offset for move event. Required for the scroll command', + 'Required for the scroll command, the delta Y offset for move ' + 'event as a stringified double', ), - 'frequency': Schema.int( + 'frequency': Schema.string( description: - 'The frequency in Hz of the generated move events. ' - 'Required for the scroll command', + 'Required for the scroll command, the frequency in Hz of the ' + 'generated move events as a stringified integer', ), 'finderType': Schema.string( description: - 'The kind of finder to use, if required for the command. ' 'Required for get_text, scroll, scroll_into_view, tap, waitFor, ' 'waitForAbsent, waitForTappable, get_offset, and ' - 'get_diagnostics_tree', + 'get_diagnostics_tree. The kind of finder to use.', enumValues: [ 'ByType', 'ByValueKey', @@ -733,10 +735,11 @@ base mixin DartToolingDaemonSupport description: 'Required for the ByValueKey finder, the type of the key', ), - 'isRegExp': Schema.bool( + 'isRegExp': Schema.string( description: 'Used by the BySemanticsLabel finder, indicates whether ' 'the value should be treated as a regex', + enumValues: ['true', 'false'], ), 'label': Schema.string( description: @@ -745,8 +748,8 @@ base mixin DartToolingDaemonSupport ), 'text': Schema.string( description: - 'The relevant text for the command. Required for the ByText and ' - 'ByTooltipMessage finders, as well as the enter_text command.', + 'Required for the ByText and ByTooltipMessage finders, as well ' + 'as the enter_text command. The relevant text for the command', ), 'type': Schema.string( description: @@ -807,9 +810,7 @@ base mixin DartToolingDaemonSupport 'complete. Defaults to $_defaultTimeoutMs.', ), 'offsetType': Schema.string( - description: - 'Offset types that can be requested by get_offset. ' - 'Required for get_offset.', + description: 'Required for get_offset, the offset type to get', enumValues: [ 'topLeft', 'topRight', @@ -820,22 +821,26 @@ base mixin DartToolingDaemonSupport ), 'diagnosticsType': Schema.string( description: - 'The type of diagnostics tree to request. ' - 'Required for get_diagnostics_tree', + 'Required for get_diagnostics_tree, the type of diagnostics tree ' + 'to request', enumValues: ['renderObject', 'widget'], ), - 'subtreeDepth': Schema.int( + 'subtreeDepth': Schema.string( description: - 'How many levels of children to include in the result. ' - 'Required for get_diagnostics_tree', + 'Required for get_diagnostics_tree, how many levels of children ' + 'to include in the result, as a stringified integer', ), - 'includeProperties': Schema.bool( + 'includeProperties': Schema.string( description: 'Whether the properties of a diagnostics node should be included ' 'in get_diagnostics_tree results', + enumValues: const ['true', 'false'], ), - 'enabled': Schema.bool( - description: 'Used by set_text_entry_emulation, defaults to false', + 'enabled': Schema.string( + description: + 'Used by set_text_entry_emulation, defaults to ' + 'false', + enumValues: const ['true', 'false'], ), }, required: ['command'], From acd439c405597cb4c4be52b3fa72f45da69e1065 Mon Sep 17 00:00:00 2001 From: Jake Macdonald Date: Tue, 22 Jul 2025 21:41:23 +0000 Subject: [PATCH 2/5] enable screenshots through flutter driver --- pkgs/dart_mcp_server/lib/src/mixins/dtd.dart | 28 +++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/pkgs/dart_mcp_server/lib/src/mixins/dtd.dart b/pkgs/dart_mcp_server/lib/src/mixins/dtd.dart index 44121e11..b917a867 100644 --- a/pkgs/dart_mcp_server/lib/src/mixins/dtd.dart +++ b/pkgs/dart_mcp_server/lib/src/mixins/dtd.dart @@ -182,6 +182,10 @@ base mixin DartToolingDaemonSupport } final vm = await vmService.getVM(); final timeout = request.arguments?['timeout'] as String?; + final isScreenshot = request.arguments?['command'] == 'screenshot'; + if (isScreenshot) { + request.arguments?.putIfAbsent('format', () => '4' /*png*/); + } final result = await vmService .callServiceExtension( _flutterDriverService, @@ -200,7 +204,17 @@ base mixin DartToolingDaemonSupport })!, ); return CallToolResult( - content: [Content.text(text: jsonEncode(result.json))], + content: [ + isScreenshot && result.json?['isError'] == false + ? Content.image( + data: + (result.json!['response'] + as Map)['data'] + as String, + mimeType: 'image/png', + ) + : Content.text(text: jsonEncode(result.json)), + ], isError: result.json?['isError'] as bool?, ); }, @@ -648,11 +662,11 @@ base mixin DartToolingDaemonSupport inputSchema: Schema.object( additionalProperties: true, description: - 'The flutter driver command to run. Command arguments should be ' - 'passed as additional properties to this map.\n\nWhen searching for ' - 'widgets, you should first inspect the widget tree in order to ' - 'figure out how to find the widget instead of just guessing tooltip ' - 'text or other things.', + 'Command arguments are passed as additional properties to this map.' + 'To specify a widgets, you should first use the ' + '"${getWidgetTreeTool.name}" tool to inspect the widget tree for the ' + 'value id of the widget and then use the "ByValueKey" finder type ' + 'with that id.', properties: { 'command': Schema.string( // Commented out values are flutter_driver commands that are not @@ -681,7 +695,7 @@ base mixin DartToolingDaemonSupport // 'get_semantics_id', 'get_offset', 'get_diagnostics_tree', - // 'screenshot', + 'screenshot', ], description: 'The name of the driver command', ), From d611adec295f760c762ef0cd89e34c5c9b9fea33 Mon Sep 17 00:00:00 2001 From: Jake Macdonald Date: Wed, 23 Jul 2025 14:55:48 +0000 Subject: [PATCH 3/5] update changelog --- pkgs/dart_mcp_server/CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkgs/dart_mcp_server/CHANGELOG.md b/pkgs/dart_mcp_server/CHANGELOG.md index df084034..20e4576c 100644 --- a/pkgs/dart_mcp_server/CHANGELOG.md +++ b/pkgs/dart_mcp_server/CHANGELOG.md @@ -7,6 +7,10 @@ * Add a flutter_driver command for executing flutter driver commands on a device. * Allow for multiple package arguments to `pub add` and `pub remove`. * Require dart_mcp version 0.3.1. +* Add support for the flutter_driver screenshot command. +* Change the widget tree to the full version instead of the summary. The summary + tends to hide nested text widgets which makes it difficult to find widgets + based on their text values. # 0.1.0 (Dart SDK 3.9.0) From 2b296079aa9672b67b8b4a863366857c3a1fe990 Mon Sep 17 00:00:00 2001 From: Jake Macdonald Date: Wed, 23 Jul 2025 15:00:45 +0000 Subject: [PATCH 4/5] add screenshot command for flutter_driver version --- pkgs/dart_mcp_server/test/tools/dtd_test.dart | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/pkgs/dart_mcp_server/test/tools/dtd_test.dart b/pkgs/dart_mcp_server/test/tools/dtd_test.dart index 8191534d..c2a005c9 100644 --- a/pkgs/dart_mcp_server/test/tools/dtd_test.dart +++ b/pkgs/dart_mcp_server/test/tools/dtd_test.dart @@ -232,6 +232,25 @@ void main() { }); }); + test('can take a screenshot using flutter_driver', () async { + await testHarness.startDebugSession( + counterAppPath, + 'lib/driver_main.dart', + isFlutter: true, + ); + final screenshotResult = await testHarness.callToolWithRetry( + CallToolRequest( + name: DartToolingDaemonSupport.flutterDriverTool.name, + arguments: {'command': 'screenshot'}, + ), + ); + expect(screenshotResult.content.single, { + 'data': anything, + 'mimeType': 'image/png', + 'type': ImageContent.expectedType, + }); + }); + group('get selected widget', () { test('when a selected widget exists', () async { final server = testHarness.serverConnectionPair.server!; From 8f7281454d38ba34291aeb7039518024eae87320 Mon Sep 17 00:00:00 2001 From: Jake Macdonald Date: Wed, 23 Jul 2025 16:31:04 +0000 Subject: [PATCH 5/5] fix the widget selection test, add callTool helper, update tests to use that --- pkgs/dart_mcp_server/lib/src/mixins/dtd.dart | 19 +++++++---- pkgs/dart_mcp_server/test/test_harness.dart | 33 ++++++++++++------- .../test/tools/analyzer_test.dart | 6 ++-- .../test/tools/dart_cli_test.dart | 25 +++----------- pkgs/dart_mcp_server/test/tools/dtd_test.dart | 32 +++++++----------- .../test/tools/pub_dev_search_test.dart | 12 ++----- pkgs/dart_mcp_server/test/tools/pub_test.dart | 6 ++-- 7 files changed, 60 insertions(+), 73 deletions(-) diff --git a/pkgs/dart_mcp_server/lib/src/mixins/dtd.dart b/pkgs/dart_mcp_server/lib/src/mixins/dtd.dart index b917a867..39263841 100644 --- a/pkgs/dart_mcp_server/lib/src/mixins/dtd.dart +++ b/pkgs/dart_mcp_server/lib/src/mixins/dtd.dart @@ -476,15 +476,14 @@ base mixin DartToolingDaemonSupport callback: (vmService) async { final vm = await vmService.getVM(); final isolateId = vm.isolates!.first.id; + final summaryOnly = request.arguments?['summaryOnly'] as bool? ?? false; try { final result = await vmService.callServiceExtension( '$_inspectorServiceExtensionPrefix.getRootWidgetTree', isolateId: isolateId, args: { 'groupName': inspectorObjectGroup, - // TODO: consider making these configurable or using defaults that - // are better for the LLM. - 'isSummaryTree': 'false', + 'isSummaryTree': summaryOnly ? 'true' : 'false', 'withPreviews': 'true', 'fullDetails': 'false', }, @@ -673,8 +672,8 @@ base mixin DartToolingDaemonSupport // supported, but may be in the future. enumValues: [ 'get_health', - 'get_layer_tree', - 'get_render_tree', + // 'get_layer_tree', + // 'get_render_tree', 'enter_text', 'send_text_input_action', 'get_text', @@ -939,7 +938,15 @@ base mixin DartToolingDaemonSupport 'Retrieves the widget tree from the active Flutter application. ' 'Requires "${connectTool.name}" to be successfully called first.', annotations: ToolAnnotations(title: 'Get widget tree', readOnlyHint: true), - inputSchema: Schema.object(), + inputSchema: Schema.object( + properties: { + 'summaryOnly': Schema.bool( + description: + 'Defaults to false. If true, only widgets created by user code ' + 'are returned.', + ), + }, + ), ); @visibleForTesting diff --git a/pkgs/dart_mcp_server/test/test_harness.dart b/pkgs/dart_mcp_server/test/test_harness.dart index 9333d7e4..a8da3f02 100644 --- a/pkgs/dart_mcp_server/test/test_harness.dart +++ b/pkgs/dart_mcp_server/test/test_harness.dart @@ -163,27 +163,38 @@ class TestHarness { expect(result.isError, isNot(true), reason: result.content.join('\n')); } + /// Helper to send [request] to [mcpServerConnection]. + /// + /// Some methods will fail if the DTD connection is not yet ready. + Future callTool( + CallToolRequest request, { + bool expectError = false, + }) async { + final result = await mcpServerConnection.callTool(request); + expect( + result.isError, + expectError ? true : isNot(true), + reason: result.content.join('\n'), + ); + return result; + } + /// Sends [request] to [mcpServerConnection], retrying [maxTries] times. /// /// Some methods will fail if the DTD connection is not yet ready. Future callToolWithRetry( CallToolRequest request, { int maxTries = 5, - bool expectError = false, }) async { var tryCount = 0; - late CallToolResult lastResult; - while (tryCount++ < maxTries) { - lastResult = await mcpServerConnection.callTool(request); - if (lastResult.isError != true) return lastResult; + while (true) { + try { + return await callTool(request); + } catch (_) { + if (tryCount++ >= maxTries) rethrow; + } await Future.delayed(Duration(milliseconds: 100 * tryCount)); } - expect( - lastResult.isError, - expectError ? true : isNot(true), - reason: lastResult.content.join('\n'), - ); - return lastResult; } } diff --git a/pkgs/dart_mcp_server/test/tools/analyzer_test.dart b/pkgs/dart_mcp_server/test/tools/analyzer_test.dart index 2cfb10d2..f6d7e656 100644 --- a/pkgs/dart_mcp_server/test/tools/analyzer_test.dart +++ b/pkgs/dart_mcp_server/test/tools/analyzer_test.dart @@ -188,7 +188,7 @@ void printIt({required int x}) { }); test('cannot analyze without roots set', () async { - final result = await testHarness.callToolWithRetry( + final result = await testHarness.callTool( CallToolRequest(name: DartAnalyzerSupport.analyzeFilesTool.name), expectError: true, ); @@ -203,7 +203,7 @@ void printIt({required int x}) { }); test('cannot look up symbols without roots set', () async { - final result = await testHarness.callToolWithRetry( + final result = await testHarness.callTool( CallToolRequest( name: DartAnalyzerSupport.resolveWorkspaceSymbolTool.name, arguments: {ParameterNames.query: 'DartAnalyzerSupport'}, @@ -221,7 +221,7 @@ void printIt({required int x}) { }); test('cannot get hover information without roots set', () async { - final result = await testHarness.callToolWithRetry( + final result = await testHarness.callTool( CallToolRequest( name: DartAnalyzerSupport.hoverTool.name, arguments: { diff --git a/pkgs/dart_mcp_server/test/tools/dart_cli_test.dart b/pkgs/dart_mcp_server/test/tools/dart_cli_test.dart index ea2df139..f7e30d1d 100644 --- a/pkgs/dart_mcp_server/test/tools/dart_cli_test.dart +++ b/pkgs/dart_mcp_server/test/tools/dart_cli_test.dart @@ -344,10 +344,7 @@ dependencies: ParameterNames.platform: ['atari_jaguar', 'web'], // One invalid }, ); - final result = await testHarness.callToolWithRetry( - request, - expectError: true, - ); + final result = await testHarness.callTool(request, expectError: true); expect(result.isError, isTrue); expect( @@ -372,10 +369,7 @@ dependencies: ParameterNames.directory: 'my_app_no_type', }, ); - final result = await testHarness.callToolWithRetry( - request, - expectError: true, - ); + final result = await testHarness.callTool(request, expectError: true); expect(result.isError, isTrue); expect( @@ -395,10 +389,7 @@ dependencies: ParameterNames.projectType: 'java', // Invalid type }, ); - final result = await testHarness.callToolWithRetry( - request, - expectError: true, - ); + final result = await testHarness.callTool(request, expectError: true); expect(result.isError, isTrue); expect( @@ -418,10 +409,7 @@ dependencies: ParameterNames.projectType: 'dart', }, ); - final result = await testHarness.callToolWithRetry( - request, - expectError: true, - ); + final result = await testHarness.callTool(request, expectError: true); expect(result.isError, isTrue); expect( @@ -441,10 +429,7 @@ dependencies: ParameterNames.template: 'cli', }, ); - final result = await testHarness.callToolWithRetry( - request, - expectError: true, - ); + final result = await testHarness.callTool(request, expectError: true); expect(result.isError, true); expect( diff --git a/pkgs/dart_mcp_server/test/tools/dtd_test.dart b/pkgs/dart_mcp_server/test/tools/dtd_test.dart index c2a005c9..79ef04cf 100644 --- a/pkgs/dart_mcp_server/test/tools/dtd_test.dart +++ b/pkgs/dart_mcp_server/test/tools/dtd_test.dart @@ -254,8 +254,6 @@ void main() { group('get selected widget', () { test('when a selected widget exists', () async { final server = testHarness.serverConnectionPair.server!; - final tools = - (await testHarness.mcpServerConnection.listTools()).tools; await testHarness.startDebugSession( counterAppPath, @@ -264,11 +262,11 @@ void main() { ); await server.updateActiveVmServices(); - final getWidgetTreeTool = tools.singleWhere( - (t) => t.name == DartToolingDaemonSupport.getWidgetTreeTool.name, - ); final getWidgetTreeResult = await testHarness.callToolWithRetry( - CallToolRequest(name: getWidgetTreeTool.name), + CallToolRequest( + name: DartToolingDaemonSupport.getWidgetTreeTool.name, + arguments: {'summaryOnly': true}, + ), ); // Select the first child of the [root] widget. @@ -292,12 +290,10 @@ void main() { ); // Confirm we can get the selected widget from the MCP tool. - final getSelectedWidgetTool = tools.singleWhere( - (t) => - t.name == DartToolingDaemonSupport.getSelectedWidgetTool.name, - ); - final getSelectedWidgetResult = await testHarness.callToolWithRetry( - CallToolRequest(name: getSelectedWidgetTool.name), + final getSelectedWidgetResult = await testHarness.callTool( + CallToolRequest( + name: DartToolingDaemonSupport.getSelectedWidgetTool.name, + ), ); expect(getSelectedWidgetResult.isError, isNot(true)); expect( @@ -312,14 +308,10 @@ void main() { 'lib/main.dart', isFlutter: true, ); - final tools = - (await testHarness.mcpServerConnection.listTools()).tools; - final getSelectedWidgetTool = tools.singleWhere( - (t) => - t.name == DartToolingDaemonSupport.getSelectedWidgetTool.name, - ); final getSelectedWidgetResult = await testHarness.callToolWithRetry( - CallToolRequest(name: getSelectedWidgetTool.name), + CallToolRequest( + name: DartToolingDaemonSupport.getSelectedWidgetTool.name, + ), ); expect(getSelectedWidgetResult.isError, isNot(true)); @@ -620,7 +612,7 @@ void main() { ]); // Test missing 'enabled' argument - final missingArgResult = await testHarness.callToolWithRetry( + final missingArgResult = await testHarness.callTool( CallToolRequest(name: setSelectionModeTool.name), expectError: true, ); diff --git a/pkgs/dart_mcp_server/test/tools/pub_dev_search_test.dart b/pkgs/dart_mcp_server/test/tools/pub_dev_search_test.dart index ad4e00f7..27e6c826 100644 --- a/pkgs/dart_mcp_server/test/tools/pub_dev_search_test.dart +++ b/pkgs/dart_mcp_server/test/tools/pub_dev_search_test.dart @@ -129,11 +129,7 @@ void main() { arguments: {'query': 'retry'}, ); - final result = await testHarness.callToolWithRetry( - request, - maxTries: 1, - expectError: true, - ); + final result = await testHarness.callTool(request, expectError: true); expect(result.isError, isTrue); expect( (result.content[0] as TextContent).text, @@ -152,11 +148,7 @@ void main() { arguments: {'query': 'retry'}, ); - final result = await testHarness.callToolWithRetry( - request, - maxTries: 1, - expectError: true, - ); + final result = await testHarness.callTool(request); expect(result.content.length, 1); expect(json.decode((result.content[0] as TextContent).text), { 'packageName': 'retry', diff --git a/pkgs/dart_mcp_server/test/tools/pub_test.dart b/pkgs/dart_mcp_server/test/tools/pub_test.dart index a9658f7e..50d0cc61 100644 --- a/pkgs/dart_mcp_server/test/tools/pub_test.dart +++ b/pkgs/dart_mcp_server/test/tools/pub_test.dart @@ -190,7 +190,7 @@ void main() { group('returns error', () { test('for missing command', () async { final request = CallToolRequest(name: dartPubTool.name); - final result = await testHarness.callToolWithRetry( + final result = await testHarness.callTool( request, expectError: true, ); @@ -207,7 +207,7 @@ void main() { name: dartPubTool.name, arguments: {ParameterNames.command: 'publish'}, ); - final result = await testHarness.callToolWithRetry( + final result = await testHarness.callTool( request, expectError: true, ); @@ -227,7 +227,7 @@ void main() { name: dartPubTool.name, arguments: {ParameterNames.command: command.name}, ); - final result = await testHarness.callToolWithRetry( + final result = await testHarness.callTool( request, expectError: true, );