Skip to content

Commit e784831

Browse files
committed
fix: visual fallback truncation + cache results + sidebar fallback
- mobile_blueprint_runner.py: increase max_tokens 500->1024 for visual fallback - mobile_blueprint_runner.py: handle truncated AI responses (detect not_found/need_scroll from partial text) - mobile_blueprint_runner.py: cache visual fallback results into page library for reuse across scenarios - sidebarProvider.ts: show minimal result card from WS test_done even without full report (safety net) - copilot-instructions.md: add mandatory English language rule for AI interactions
1 parent c1970b5 commit e784831

3 files changed

Lines changed: 114 additions & 9 deletions

File tree

.github/copilot-instructions.md

Lines changed: 59 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,16 @@
1-
# TestPilot AI - 项目编码规范
1+
# TestPilot AI - Project Coding Standards
2+
3+
## Language Rule (MANDATORY)
4+
5+
All AI thinking, reasoning, and responses MUST be in **English**. This applies to:
6+
- Code comments in new/modified code
7+
- Commit messages
8+
- AI-generated explanations and dialog
9+
- Variable/function naming (already English)
10+
11+
Exception: User-facing strings in the product (UI text, log messages, prompts sent to AI models) may remain in Chinese as they are part of the product's Chinese localization.
12+
13+
---
214

315
## 蓝本文件(testpilot.json)管理规则
416

@@ -367,3 +379,49 @@ Flutter 的 `Semantics` 组件在 Android 端映射为无障碍属性。**选择
367379
| 等待元素 | `{"action": "wait", "target": "accessibility_id:xxx", "timeout_ms": 15000}` | 轮询等元素出现 |
368380

369381
> Flutter 每次 `navigate` 重启后的标准流程:先 `wait 3000`,再 `wait target` 等关键元素就绪。
382+
383+
核心工作流程:四步法
384+
385+
任务分析:分析需求、技术难点,默认使用北京时间。
386+
387+
实施规划:制定3-5个步骤,Python项目第一步必须包含虚拟环境设置。
388+
389+
分步执行:单次代码生成严格限制在150行以内,复杂任务分批次进行。
390+
391+
总结回顾:提供代码使用说明和注意事项。
392+
393+
重点优化规范
394+
395+
代码生成控制:避免卡顿,严禁一次性生成超长代码。
396+
397+
Python专项:强制使用虚拟环境,代码中显式指定 Asia/Shanghai时区。
398+
399+
项目文档管理:每个项目必须创建并实时维护 开发备忘录.md文件。
400+
401+
项目文档强制要求
402+
403+
文件位置:项目根目录下的 开发备忘录.md。
404+
405+
实时同步:每次代码修改、Bug修复、功能更新后,必须同步更新该备忘录。
406+
407+
核心内容:
408+
409+
项目简介与核心功能。
410+
411+
环境配置(重点强调 python -m venv venv)。
412+
413+
开发、测试与部署步骤。
414+
415+
版本更新日志(明确标注已淘汰的旧代码和新增功能)。
416+
417+
已知问题与解决方案。
418+
419+
更新原则:确保代码与文档完全一致,淘汰内容需特别注明。
420+
421+
交互与输出
422+
423+
全程使用简洁专业的简体中文。
424+
425+
代码注释重点解释核心逻辑,避免冗长。
426+
427+
分批次输出代码时,需明确标注进度(例如:[第1/3部分])。

extension/src/sidebarProvider.ts

Lines changed: 23 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2227,11 +2227,29 @@ ${commonRules}`;
22272227
screenshotImg.src = "data:image/png;base64," + wsMsg.data.image;
22282228
return;
22292229
}
2230-
// test_done 含完整报告时,直接渲染结果(HTTP后备)
2231-
// 即使 HTTP fetch failed,WebView 也能通过 WS 拿到完整结果
2232-
if (wsMsg.type === "test_done" && wsMsg.data?.report) {
2233-
onTestResult(wsMsg.data.report);
2234-
return;
2230+
// test_done: render full report if available, otherwise show summary
2231+
if (wsMsg.type === "test_done") {
2232+
if (wsMsg.data?.report) {
2233+
onTestResult(wsMsg.data.report);
2234+
return;
2235+
}
2236+
// Fallback: build a minimal report from WS summary data
2237+
// This ensures result card always shows even if full_report is missing
2238+
if (wsMsg.data?.pass_rate !== undefined) {
2239+
const minimal = {
2240+
test_name: "蓝本测试",
2241+
pass_rate: wsMsg.data.pass_rate,
2242+
passed_steps: 0,
2243+
total_steps: 0,
2244+
bug_count: wsMsg.data.bug_count || 0,
2245+
duration_seconds: 0,
2246+
bugs: [],
2247+
steps: [],
2248+
};
2249+
onTestResult(minimal);
2250+
addLog("⚠️ 结果摘要已显示,完整Bug详情等待HTTP返回...", "warn");
2251+
return;
2252+
}
22352253
}
22362254
const level = typeMap[wsMsg.type] || "info";
22372255
const text = wsMsg.data?.message || wsMsg.type;

src/testing/mobile_blueprint_runner.py

Lines changed: 32 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1194,6 +1194,15 @@ async def _smart_tap(
11941194
logger.info(" [视觉定位] 截图+AI识别 {}...", target)
11951195
coords = await self._visual_find_element(target, desc)
11961196
if coords:
1197+
# Cache the visual result into page library for future reuse
1198+
try:
1199+
xml_now = await self._ctrl.dump_ui_tree()
1200+
if xml_now:
1201+
fp = self._extract_page_fingerprint(xml_now)
1202+
if fp:
1203+
self._register_page(f"visual_{target[:30]}", fp, {target: coords})
1204+
except Exception:
1205+
pass
11971206
await self._ctrl.tap_xy(coords[0], coords[1])
11981207
else:
11991208
raise RuntimeError(f"元素找不到(Appium+视觉均失败): {target} | {desc}")
@@ -1329,6 +1338,15 @@ async def _smart_fill(
13291338
logger.info(" [视觉定位] 截图+AI识别 {}...", target)
13301339
coords = await self._visual_find_element(target, desc)
13311340
if coords:
1341+
# Cache the visual result into page library for future reuse
1342+
try:
1343+
xml_now = await self._ctrl.dump_ui_tree()
1344+
if xml_now:
1345+
fp = self._extract_page_fingerprint(xml_now)
1346+
if fp:
1347+
self._register_page(f"visual_{target[:30]}", fp, {target: coords})
1348+
except Exception:
1349+
pass
13321350
await self._ctrl.input_text_xy(coords[0], coords[1], value)
13331351
else:
13341352
raise RuntimeError(f"元素找不到(Appium+视觉均失败): {target} | {desc}")
@@ -1390,13 +1408,24 @@ async def _visual_find_element(
13901408
loop = asyncio.get_event_loop()
13911409
response = await loop.run_in_executor(
13921410
None, lambda p=prompt, sp=str(screenshot_path): self._ai.analyze_screenshot(
1393-
sp, p, reasoning_effort="low", timeout=45, max_tokens=500,
1411+
sp, p, reasoning_effort="low", timeout=45, max_tokens=1024,
13941412
)
13951413
)
13961414

1397-
# 解析AI返回
1398-
json_match = re.search(r'\{[^}]+\}', response)
1415+
# Parse AI response — try to extract JSON even from truncated output
1416+
json_match = re.search(r'\{[^{}]*\}', response)
13991417
if not json_match:
1418+
# Truncated JSON: try to salvage key fields
1419+
if '"not_found"' in response or '"reason"' in response:
1420+
logger.info(" 视觉降级: AI确认元素不存在(截断响应)")
1421+
return None
1422+
if '"need_scroll"' in response:
1423+
logger.info(" 视觉降级: AI建议滚动(截断响应)")
1424+
if scroll_attempt < max_scrolls:
1425+
await self._ctrl.swipe_screen("down")
1426+
await asyncio.sleep(1.0)
1427+
continue
1428+
return None
14001429
logger.warning(" 视觉降级: AI返回格式异常: {}", response[:100])
14011430
return None
14021431

0 commit comments

Comments
 (0)