Skip to content

Commit e7a6911

Browse files
author
GitLab CI
committed
feat: Add native platform interaction tools and fix VM Service reconnection
Add native_screenshot, native_tap, native_input_text, native_swipe MCP tools that bypass Flutter's VM Service to interact with native OS views (photo pickers, permission dialogs, share sheets). iOS uses macOS Accessibility API, Android uses adb. Fix LateInitializationError crash in VM Service client by adding automatic reconnection with proper error handling and cleanup.
1 parent bac6f0d commit e7a6911

3 files changed

Lines changed: 1149 additions & 36 deletions

File tree

lib/src/cli/server.dart

Lines changed: 254 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import 'dart:io';
44

55
import 'package:http/http.dart' as http;
66
import '../flutter_skill_client.dart';
7+
import '../native_driver.dart';
78
import '../diagnostics/error_reporter.dart';
89
import 'setup.dart';
910

@@ -136,6 +137,28 @@ class FlutterMcpServer {
136137

137138
Process? _flutterProcess;
138139

140+
// Native platform drivers (for interacting with native OS views)
141+
final Map<String, NativeDriver> _nativeDrivers = {};
142+
143+
/// Get or create native driver for the active session
144+
Future<NativeDriver?> _getNativeDriver(Map<String, dynamic> args) async {
145+
final sessionId = args['session_id'] as String? ?? _activeSessionId;
146+
final key = sessionId ?? '_default';
147+
148+
if (_nativeDrivers.containsKey(key)) return _nativeDrivers[key];
149+
150+
String? deviceId;
151+
if (sessionId != null && _sessions.containsKey(sessionId)) {
152+
deviceId = _sessions[sessionId]!.deviceId;
153+
}
154+
155+
final driver = await NativeDriver.create(deviceId);
156+
if (driver != null) {
157+
_nativeDrivers[key] = driver;
158+
}
159+
return driver;
160+
}
161+
139162
Future<void> run() async {
140163
stdin
141164
.transform(utf8.decoder)
@@ -926,6 +949,140 @@ By default, saves screenshot to a temporary file and returns file path. Optional
926949
"description": "Trigger hot restart (slower, resets app state)",
927950
"inputSchema": {"type": "object", "properties": {}},
928951
},
952+
{
953+
// Native platform interaction tools
954+
"name": "native_screenshot",
955+
"description": """Take a screenshot at the OS level (bypasses Flutter).
956+
957+
[USE WHEN]
958+
• A native dialog is shown (photo picker, permission dialog, share sheet)
959+
• Flutter's screenshot returns a blank/stale image
960+
• You need to see system-level UI (status bar, keyboard, etc.)
961+
• The app is presenting a platform view not rendered by Flutter
962+
963+
[HOW IT WORKS]
964+
• iOS Simulator: Uses xcrun simctl screenshot
965+
• Android Emulator: Uses adb shell screencap
966+
967+
[RETURNS]
968+
Screenshot saved to a temporary file (default) or base64-encoded PNG.
969+
This captures the ENTIRE device screen, not just the Flutter app content.""",
970+
"inputSchema": {
971+
"type": "object",
972+
"properties": {
973+
"save_to_file": {
974+
"type": "boolean",
975+
"description":
976+
"Save to file and return path (default: true)"
977+
},
978+
},
979+
},
980+
},
981+
{
982+
"name": "native_tap",
983+
"description": """Tap at device coordinates using OS-level input (bypasses Flutter).
984+
985+
[USE WHEN]
986+
• Interacting with native dialogs (photo picker, permission "Allow", share sheet)
987+
• Flutter's tap() doesn't work because the target is a native view
988+
• Tapping system UI elements (status bar, notification)
989+
990+
[HOW IT WORKS]
991+
• iOS Simulator: Uses macOS Accessibility API to find and press UI elements at device coordinates
992+
• Android Emulator: Uses adb shell input tap
993+
994+
[IMPORTANT]
995+
• Coordinates are in device pixels (same as native_screenshot dimensions)
996+
• Take a native_screenshot first to identify tap targets
997+
• iOS: No external tools needed (uses built-in osascript + Accessibility API)
998+
• The Simulator window must be visible and not minimized""",
999+
"inputSchema": {
1000+
"type": "object",
1001+
"properties": {
1002+
"x": {
1003+
"type": "number",
1004+
"description": "X coordinate in device pixels"
1005+
},
1006+
"y": {
1007+
"type": "number",
1008+
"description": "Y coordinate in device pixels"
1009+
},
1010+
},
1011+
"required": ["x", "y"],
1012+
},
1013+
},
1014+
{
1015+
"name": "native_input_text",
1016+
"description": """Enter text using OS-level input (bypasses Flutter).
1017+
1018+
[USE WHEN]
1019+
• Typing into native text fields (search bars in native pickers, etc.)
1020+
• Flutter's enter_text() doesn't work because the field is in a native view
1021+
• Entering text in system dialogs
1022+
1023+
[HOW IT WORKS]
1024+
• iOS Simulator: Copies text to pasteboard via simctl, then pastes with Cmd+V
1025+
• Android Emulator: Uses adb shell input text
1026+
1027+
[IMPORTANT]
1028+
• The target text field must already be focused (tap it first with native_tap)
1029+
• iOS method uses paste, so it replaces clipboard content
1030+
• iOS paste confirmation dialog ("Allow Paste") is automatically dismissed""",
1031+
"inputSchema": {
1032+
"type": "object",
1033+
"properties": {
1034+
"text": {
1035+
"type": "string",
1036+
"description": "Text to enter"
1037+
},
1038+
},
1039+
"required": ["text"],
1040+
},
1041+
},
1042+
{
1043+
"name": "native_swipe",
1044+
"description": """Swipe using OS-level input (bypasses Flutter).
1045+
1046+
[USE WHEN]
1047+
• Swiping in native views (photo gallery scroll, native list)
1048+
• Dismissing native dialogs with swipe
1049+
• Flutter's swipe doesn't work because the scrollable is a native view
1050+
1051+
[HOW IT WORKS]
1052+
• iOS Simulator: Uses macOS Accessibility API scroll actions on elements at device coordinates
1053+
• Android Emulator: Uses adb shell input swipe
1054+
1055+
[IMPORTANT]
1056+
• Coordinates are in device pixels
1057+
• Take a native_screenshot first to plan your swipe path
1058+
• iOS: Scrolls by page using accessibility actions (AXScrollUpByPage/AXScrollDownByPage)""",
1059+
"inputSchema": {
1060+
"type": "object",
1061+
"properties": {
1062+
"start_x": {
1063+
"type": "number",
1064+
"description": "Start X in device pixels"
1065+
},
1066+
"start_y": {
1067+
"type": "number",
1068+
"description": "Start Y in device pixels"
1069+
},
1070+
"end_x": {
1071+
"type": "number",
1072+
"description": "End X in device pixels"
1073+
},
1074+
"end_y": {
1075+
"type": "number",
1076+
"description": "End Y in device pixels"
1077+
},
1078+
"duration": {
1079+
"type": "integer",
1080+
"description": "Swipe duration in ms (default: 300)"
1081+
},
1082+
},
1083+
"required": ["start_x", "start_y", "end_x", "end_y"],
1084+
},
1085+
},
9291086
{
9301087
"name": "diagnose_project",
9311088
"description": """⚡ DIAGNOSTIC & AUTO-FIX TOOL ⚡
@@ -2096,6 +2253,103 @@ Detailed diagnostic report with:
20962253
return await client!.getIndicatorStatus();
20972254
}
20982255

2256+
// Native platform interaction tools (no VM Service connection required)
2257+
if (name == 'native_screenshot') {
2258+
final driver = await _getNativeDriver(args);
2259+
if (driver == null) {
2260+
return {
2261+
"success": false,
2262+
"error": {
2263+
"code": "E501",
2264+
"message": "No supported platform detected",
2265+
},
2266+
"suggestions": [
2267+
"Ensure an iOS Simulator or Android emulator is running",
2268+
"If using a physical device, native tools are not yet supported",
2269+
],
2270+
};
2271+
}
2272+
final saveToFile = args['save_to_file'] ?? true;
2273+
final result = await driver.screenshot(saveToFile: saveToFile);
2274+
return result.toJson();
2275+
}
2276+
2277+
if (name == 'native_tap') {
2278+
final driver = await _getNativeDriver(args);
2279+
if (driver == null) {
2280+
return {
2281+
"success": false,
2282+
"error": {
2283+
"code": "E501",
2284+
"message": "No supported platform detected",
2285+
},
2286+
};
2287+
}
2288+
2289+
final toolCheck = await driver.checkToolAvailability();
2290+
final missingTools = toolCheck.entries
2291+
.where((e) => !e.value)
2292+
.map((e) => e.key)
2293+
.toList();
2294+
if (missingTools.isNotEmpty) {
2295+
return {
2296+
"success": false,
2297+
"error": {
2298+
"code": "E502",
2299+
"message": "Missing required tools: ${missingTools.join(', ')}",
2300+
},
2301+
"suggestions":
2302+
driver.platform == NativePlatform.iosSimulator
2303+
? ["Ensure Xcode command line tools are installed: xcode-select --install"]
2304+
: [
2305+
"Install Android platform tools: brew install android-platform-tools"
2306+
],
2307+
};
2308+
}
2309+
2310+
final x = (args['x'] as num).toDouble();
2311+
final y = (args['y'] as num).toDouble();
2312+
final result = await driver.tap(x, y);
2313+
return result.toJson();
2314+
}
2315+
2316+
if (name == 'native_input_text') {
2317+
final driver = await _getNativeDriver(args);
2318+
if (driver == null) {
2319+
return {
2320+
"success": false,
2321+
"error": {
2322+
"code": "E501",
2323+
"message": "No supported platform detected",
2324+
},
2325+
};
2326+
}
2327+
final text = args['text'] as String;
2328+
final result = await driver.inputText(text);
2329+
return result.toJson();
2330+
}
2331+
2332+
if (name == 'native_swipe') {
2333+
final driver = await _getNativeDriver(args);
2334+
if (driver == null) {
2335+
return {
2336+
"success": false,
2337+
"error": {
2338+
"code": "E501",
2339+
"message": "No supported platform detected",
2340+
},
2341+
};
2342+
}
2343+
final startX = (args['start_x'] as num).toDouble();
2344+
final startY = (args['start_y'] as num).toDouble();
2345+
final endX = (args['end_x'] as num).toDouble();
2346+
final endY = (args['end_y'] as num).toDouble();
2347+
final duration = args['duration'] as int? ?? 300;
2348+
final result = await driver.swipe(startX, startY, endX, endY,
2349+
durationMs: duration);
2350+
return result.toJson();
2351+
}
2352+
20992353
// Require connection for all other tools
21002354
final client = _getClient(args);
21012355
_requireConnection(client);

0 commit comments

Comments
 (0)