@@ -4,6 +4,7 @@ import 'dart:io';
44
55import 'package:http/http.dart' as http;
66import '../flutter_skill_client.dart' ;
7+ import '../native_driver.dart' ;
78import '../diagnostics/error_reporter.dart' ;
89import 'setup.dart' ;
910
@@ -136,6 +137,28 @@ class FlutterMcpServer {
136137
137138 Process ? _flutterProcess;
138139
140+ // Native platform drivers (for interacting with native OS views)
141+ final Map <String , NativeDriver > _nativeDrivers = {};
142+
143+ /// Get or create native driver for the active session
144+ Future <NativeDriver ?> _getNativeDriver (Map <String , dynamic > args) async {
145+ final sessionId = args['session_id' ] as String ? ?? _activeSessionId;
146+ final key = sessionId ?? '_default' ;
147+
148+ if (_nativeDrivers.containsKey (key)) return _nativeDrivers[key];
149+
150+ String ? deviceId;
151+ if (sessionId != null && _sessions.containsKey (sessionId)) {
152+ deviceId = _sessions[sessionId]! .deviceId;
153+ }
154+
155+ final driver = await NativeDriver .create (deviceId);
156+ if (driver != null ) {
157+ _nativeDrivers[key] = driver;
158+ }
159+ return driver;
160+ }
161+
139162 Future <void > run () async {
140163 stdin
141164 .transform (utf8.decoder)
@@ -926,6 +949,140 @@ By default, saves screenshot to a temporary file and returns file path. Optional
926949 "description" : "Trigger hot restart (slower, resets app state)" ,
927950 "inputSchema" : {"type" : "object" , "properties" : {}},
928951 },
952+ {
953+ // Native platform interaction tools
954+ "name" : "native_screenshot" ,
955+ "description" : """Take a screenshot at the OS level (bypasses Flutter).
956+
957+ [USE WHEN]
958+ • A native dialog is shown (photo picker, permission dialog, share sheet)
959+ • Flutter's screenshot returns a blank/stale image
960+ • You need to see system-level UI (status bar, keyboard, etc.)
961+ • The app is presenting a platform view not rendered by Flutter
962+
963+ [HOW IT WORKS]
964+ • iOS Simulator: Uses xcrun simctl screenshot
965+ • Android Emulator: Uses adb shell screencap
966+
967+ [RETURNS]
968+ Screenshot saved to a temporary file (default) or base64-encoded PNG.
969+ This captures the ENTIRE device screen, not just the Flutter app content.""" ,
970+ "inputSchema" : {
971+ "type" : "object" ,
972+ "properties" : {
973+ "save_to_file" : {
974+ "type" : "boolean" ,
975+ "description" :
976+ "Save to file and return path (default: true)"
977+ },
978+ },
979+ },
980+ },
981+ {
982+ "name" : "native_tap" ,
983+ "description" : """Tap at device coordinates using OS-level input (bypasses Flutter).
984+
985+ [USE WHEN]
986+ • Interacting with native dialogs (photo picker, permission "Allow", share sheet)
987+ • Flutter's tap() doesn't work because the target is a native view
988+ • Tapping system UI elements (status bar, notification)
989+
990+ [HOW IT WORKS]
991+ • iOS Simulator: Uses macOS Accessibility API to find and press UI elements at device coordinates
992+ • Android Emulator: Uses adb shell input tap
993+
994+ [IMPORTANT]
995+ • Coordinates are in device pixels (same as native_screenshot dimensions)
996+ • Take a native_screenshot first to identify tap targets
997+ • iOS: No external tools needed (uses built-in osascript + Accessibility API)
998+ • The Simulator window must be visible and not minimized""" ,
999+ "inputSchema" : {
1000+ "type" : "object" ,
1001+ "properties" : {
1002+ "x" : {
1003+ "type" : "number" ,
1004+ "description" : "X coordinate in device pixels"
1005+ },
1006+ "y" : {
1007+ "type" : "number" ,
1008+ "description" : "Y coordinate in device pixels"
1009+ },
1010+ },
1011+ "required" : ["x" , "y" ],
1012+ },
1013+ },
1014+ {
1015+ "name" : "native_input_text" ,
1016+ "description" : """Enter text using OS-level input (bypasses Flutter).
1017+
1018+ [USE WHEN]
1019+ • Typing into native text fields (search bars in native pickers, etc.)
1020+ • Flutter's enter_text() doesn't work because the field is in a native view
1021+ • Entering text in system dialogs
1022+
1023+ [HOW IT WORKS]
1024+ • iOS Simulator: Copies text to pasteboard via simctl, then pastes with Cmd+V
1025+ • Android Emulator: Uses adb shell input text
1026+
1027+ [IMPORTANT]
1028+ • The target text field must already be focused (tap it first with native_tap)
1029+ • iOS method uses paste, so it replaces clipboard content
1030+ • iOS paste confirmation dialog ("Allow Paste") is automatically dismissed""" ,
1031+ "inputSchema" : {
1032+ "type" : "object" ,
1033+ "properties" : {
1034+ "text" : {
1035+ "type" : "string" ,
1036+ "description" : "Text to enter"
1037+ },
1038+ },
1039+ "required" : ["text" ],
1040+ },
1041+ },
1042+ {
1043+ "name" : "native_swipe" ,
1044+ "description" : """Swipe using OS-level input (bypasses Flutter).
1045+
1046+ [USE WHEN]
1047+ • Swiping in native views (photo gallery scroll, native list)
1048+ • Dismissing native dialogs with swipe
1049+ • Flutter's swipe doesn't work because the scrollable is a native view
1050+
1051+ [HOW IT WORKS]
1052+ • iOS Simulator: Uses macOS Accessibility API scroll actions on elements at device coordinates
1053+ • Android Emulator: Uses adb shell input swipe
1054+
1055+ [IMPORTANT]
1056+ • Coordinates are in device pixels
1057+ • Take a native_screenshot first to plan your swipe path
1058+ • iOS: Scrolls by page using accessibility actions (AXScrollUpByPage/AXScrollDownByPage)""" ,
1059+ "inputSchema" : {
1060+ "type" : "object" ,
1061+ "properties" : {
1062+ "start_x" : {
1063+ "type" : "number" ,
1064+ "description" : "Start X in device pixels"
1065+ },
1066+ "start_y" : {
1067+ "type" : "number" ,
1068+ "description" : "Start Y in device pixels"
1069+ },
1070+ "end_x" : {
1071+ "type" : "number" ,
1072+ "description" : "End X in device pixels"
1073+ },
1074+ "end_y" : {
1075+ "type" : "number" ,
1076+ "description" : "End Y in device pixels"
1077+ },
1078+ "duration" : {
1079+ "type" : "integer" ,
1080+ "description" : "Swipe duration in ms (default: 300)"
1081+ },
1082+ },
1083+ "required" : ["start_x" , "start_y" , "end_x" , "end_y" ],
1084+ },
1085+ },
9291086 {
9301087 "name" : "diagnose_project" ,
9311088 "description" : """⚡ DIAGNOSTIC & AUTO-FIX TOOL ⚡
@@ -2096,6 +2253,103 @@ Detailed diagnostic report with:
20962253 return await client! .getIndicatorStatus ();
20972254 }
20982255
2256+ // Native platform interaction tools (no VM Service connection required)
2257+ if (name == 'native_screenshot' ) {
2258+ final driver = await _getNativeDriver (args);
2259+ if (driver == null ) {
2260+ return {
2261+ "success" : false ,
2262+ "error" : {
2263+ "code" : "E501" ,
2264+ "message" : "No supported platform detected" ,
2265+ },
2266+ "suggestions" : [
2267+ "Ensure an iOS Simulator or Android emulator is running" ,
2268+ "If using a physical device, native tools are not yet supported" ,
2269+ ],
2270+ };
2271+ }
2272+ final saveToFile = args['save_to_file' ] ?? true ;
2273+ final result = await driver.screenshot (saveToFile: saveToFile);
2274+ return result.toJson ();
2275+ }
2276+
2277+ if (name == 'native_tap' ) {
2278+ final driver = await _getNativeDriver (args);
2279+ if (driver == null ) {
2280+ return {
2281+ "success" : false ,
2282+ "error" : {
2283+ "code" : "E501" ,
2284+ "message" : "No supported platform detected" ,
2285+ },
2286+ };
2287+ }
2288+
2289+ final toolCheck = await driver.checkToolAvailability ();
2290+ final missingTools = toolCheck.entries
2291+ .where ((e) => ! e.value)
2292+ .map ((e) => e.key)
2293+ .toList ();
2294+ if (missingTools.isNotEmpty) {
2295+ return {
2296+ "success" : false ,
2297+ "error" : {
2298+ "code" : "E502" ,
2299+ "message" : "Missing required tools: ${missingTools .join (', ' )}" ,
2300+ },
2301+ "suggestions" :
2302+ driver.platform == NativePlatform .iosSimulator
2303+ ? ["Ensure Xcode command line tools are installed: xcode-select --install" ]
2304+ : [
2305+ "Install Android platform tools: brew install android-platform-tools"
2306+ ],
2307+ };
2308+ }
2309+
2310+ final x = (args['x' ] as num ).toDouble ();
2311+ final y = (args['y' ] as num ).toDouble ();
2312+ final result = await driver.tap (x, y);
2313+ return result.toJson ();
2314+ }
2315+
2316+ if (name == 'native_input_text' ) {
2317+ final driver = await _getNativeDriver (args);
2318+ if (driver == null ) {
2319+ return {
2320+ "success" : false ,
2321+ "error" : {
2322+ "code" : "E501" ,
2323+ "message" : "No supported platform detected" ,
2324+ },
2325+ };
2326+ }
2327+ final text = args['text' ] as String ;
2328+ final result = await driver.inputText (text);
2329+ return result.toJson ();
2330+ }
2331+
2332+ if (name == 'native_swipe' ) {
2333+ final driver = await _getNativeDriver (args);
2334+ if (driver == null ) {
2335+ return {
2336+ "success" : false ,
2337+ "error" : {
2338+ "code" : "E501" ,
2339+ "message" : "No supported platform detected" ,
2340+ },
2341+ };
2342+ }
2343+ final startX = (args['start_x' ] as num ).toDouble ();
2344+ final startY = (args['start_y' ] as num ).toDouble ();
2345+ final endX = (args['end_x' ] as num ).toDouble ();
2346+ final endY = (args['end_y' ] as num ).toDouble ();
2347+ final duration = args['duration' ] as int ? ?? 300 ;
2348+ final result = await driver.swipe (startX, startY, endX, endY,
2349+ durationMs: duration);
2350+ return result.toJson ();
2351+ }
2352+
20992353 // Require connection for all other tools
21002354 final client = _getClient (args);
21012355 _requireConnection (client);
0 commit comments