merge: sync main and preserve platform contract

Author: ErrorAP717

main_logic/core.py

@@ -4958,6 +4958,22 @@ async def _stream_data_now(self, message: dict):
         # 在锁外检查是否需要创建新session（不要在锁内创建session，避免死锁）
         if not self.session_ready and self._starting_session_count == 0:
             if not self.session or not self.is_active:
+                # 音频 / 屏幕 / 摄像头都是持续流，只负责给已经存在（或由前端
+                # 显式 start_session 正在创建）的会话供数，不能把单个媒体帧
+                # 当成“用户要求启动会话”。
+                #
+                # API session 异常断开时，前端会停止录音并延迟发送一次
+                # start_session。屏幕采集可能在这段窗口内继续产生帧；旧逻辑把
+                # screen/camera/audio 都映射为 audio 并立即自动建连，抢在前端
+                # 恢复流程之前启动，且 cleanup 已清空 self.websocket，最终得到
+                # 一个无法向 UI 回传的孤儿会话。媒体流在这里直接丢弃，恢复入口
+                # 统一收口到显式 start_session；文本保留旧的兼容性自动启动。
+                if input_type != 'text':
+                    logger.debug(
+                        "Session未就绪，丢弃 %s 流数据并等待显式 start_session",
+                        input_type,
+                    )
+                    return
                 # Memory Server 专属冷却检查
                 if self._emit_cooldown_turn_end_if_needed():
                     return
@@ -4966,9 +4982,7 @@ async def _stream_data_now(self, message: dict):
                 if self._session_start_circuit_open:
                     return
                 logger.info(f"Session未就绪且不存在，根据输入类型 {input_type} 自动创建 session")
-                # 根据输入类型确定模式
-                mode = 'text' if input_type == 'text' else 'audio'
-                await self.start_session(self.websocket, new=False, input_mode=mode)
+                await self.start_session(self.websocket, new=False, input_mode='text')
 
                 # 检查启动是否成功
                 if not self.session or not self.is_active:
@@ -4996,6 +5010,14 @@ async def _process_stream_data_internal(self, message: dict):
 
         # 如果 session 不存在或不活跃，检查是否可以自动重建
         if not self.session or not self.is_active:
+            # 与 _stream_data_now 的入口规则保持对偶：持续媒体帧不得自行
+            # 创建会话。这个二次守卫覆盖直接调用本方法、flush 及竞态窗口。
+            if input_type != 'text':
+                logger.debug(
+                    "Session不存在或未激活，丢弃 %s 流数据并等待显式 start_session",
+                    input_type,
+                )
+                return
             # Memory Server 专属冷却检查
             if self._emit_cooldown_turn_end_if_needed():
                 return
@@ -5024,9 +5046,7 @@ async def _process_stream_data_internal(self, message: dict):
                 self.sync_message_queue.put({'type': 'system', 'data': 'websocket disconnected'})
                 return
 
-            # 根据输入类型确定模式
-            mode = 'text' if input_type == 'text' else 'audio'
-            await self.start_session(self.websocket, new=False, input_mode=mode)
+            await self.start_session(self.websocket, new=False, input_mode='text')
 
             # 检查启动是否成功
             if not self.session or not self.is_active:

main_logic/omni_realtime_client.py

@@ -50,6 +50,10 @@
 # ── Proactive audio prompt cache ──────────────────────────────────────
 _PROACTIVE_AUDIO_DIR = Path(__file__).resolve().parent.parent / "static" / "proactive_audio"
 _PROACTIVE_AUDIO_CACHE: Dict[str, bytes] = {}
+_IMAGE_ANALYSIS_PENDING_CONTEXT = (
+    "[实时屏幕截图或相机画面正在分析中。先不要瞎编内容，可以稍等片刻。"
+    "在此期间不要用搜索功能应付。等收到画面分析结果后再描述画面。]"
+)
 
 
 def _load_proactive_audio(filename: str) -> bytes:
@@ -261,7 +265,7 @@ def __init__(
         self._image_recognized_this_turn = False
         self._image_sent_this_turn = False
         self._image_being_analyzed = False
-        self._image_description = "[实时屏幕截图或相机画面正在分析中。先不要瞎编内容，可以稍等片刻。在此期间不要用搜索功能应付。等收到画面分析结果后再描述画面。]"
+        self._image_description = _IMAGE_ANALYSIS_PENDING_CONTEXT
         self._latest_image_b64 = None  # Cached latest screenshot for proactive injection
         self._proactive_image_consumed = True  # Whether the cached image has been used by a proactive nudge
         self._proactive_injecting = False  # True while prompt_ephemeral is injecting audio — suppresses mic input
@@ -1268,15 +1272,63 @@ async def _analyze_image_with_vision_model(self, image_b64: str) -> str:
 
         except Exception as e:
             logger.error(f"Error analyzing image with vision model: {e}")
-            self.image_recognized_this_turn = True
-            self._image_being_analyzed = False
+            self._image_recognized_this_turn = True
             self._image_description = f"[实时屏幕截图或相机画面]: 分析出错: {str(e)}"
             # 检测内容审查错误并发送中文提示到前端（不关闭session）
             error_str = str(e)
             if 'censorship' in error_str:
                 if self.on_status_message:
                     await self.on_status_message(json.dumps({"code": "IMAGE_BLOCKED"}))
             return "图片识别发生严重错误！"
+
+    async def _stream_image_with_vision_model(self, image_b64: str) -> None:
+        """Analyze one frame and inject both pending and ready context.
+
+        Realtime backends without native image input need a text description
+        from VISION_MODEL.  The previous first-frame fast path only updated
+        ``_image_description`` locally, so the realtime model could answer
+        before the description ever reached its conversation.  Serialize the
+        expensive analysis and inject the completed description immediately;
+        this does not depend on another frame arriving while the user speaks.
+        """
+        async with self._image_lock:
+            if self._image_recognized_this_turn:
+                return
+
+            self._image_being_analyzed = True
+            try:
+                pending_event = {
+                    "type": "conversation.item.create",
+                    "item": {
+                        "type": "message",
+                        "role": "user",
+                        "content": [{
+                            "type": "input_text",
+                            "text": _IMAGE_ANALYSIS_PENDING_CONTEXT,
+                        }],
+                    },
+                }
+                logger.info("Sending image analysis pending context.")
+                await self.send_event(pending_event)
+
+                await self._analyze_image_with_vision_model(image_b64)
+
+                ready_event = {
+                    "type": "conversation.item.create",
+                    "item": {
+                        "type": "message",
+                        "role": "user",
+                        "content": [{
+                            "type": "input_text",
+                            "text": self._image_description,
+                        }],
+                    },
+                }
+                logger.info("Sending image description after recognition.")
+                await self.send_event(ready_event)
+                self._image_sent_this_turn = True
+            finally:
+                self._image_being_analyzed = False
 
     async def stream_image(self, image_b64: str) -> None:
         """Stream raw image data to the API."""
@@ -1285,9 +1337,12 @@ async def stream_image(self, image_b64: str) -> None:
         self._proactive_image_consumed = False
 
         try:
-            # Models without native vision (step, free on lanlan.tech) — first frame triggers VISION_MODEL analysis
-            if '实时屏幕截图或相机画面正在分析中' in self._image_description and not self._supports_native_image:
-                await self._analyze_image_with_vision_model(image_b64)
+            # Models without native vision (step, free on lanlan.tech) use
+            # VISION_MODEL text context.  This path is independent of
+            # _audio_in_buffer so a completed analysis is available before
+            # the user's next utterance even while the microphone is idle.
+            if not self._supports_native_image:
+                await self._stream_image_with_vision_model(image_b64)
                 return
 
             # Rate limiting for native image input (with VAD-based throttling)
@@ -1349,47 +1404,6 @@ async def stream_image(self, image_b64: str) -> None:
                             ]
                         }
                     }
-                else:
-                    # Model does not support video streaming, use VISION_MODEL to analyze
-                    # Only recognize one image per conversation turn
-                    async with self._image_lock:
-                        if not self._image_recognized_this_turn:
-                            if not self._image_being_analyzed:
-                                self._image_being_analyzed = True
-                                text_event = {
-                                    "type": "conversation.item.create",
-                                    "item": {
-                                        "type": "message",
-                                        "role": "user",
-                                        "content": [
-                                            {
-                                                "type": "input_text",
-                                                "text": self._image_description
-                                            }
-                                        ]
-                                    }
-                                }
-                                logger.info("Sending image description before recognition.")
-                                await self.send_event(text_event)
-                                await self._analyze_image_with_vision_model(image_b64)
-                        elif not self._image_sent_this_turn:
-                            self._image_sent_this_turn = True
-                            text_event = {
-                                    "type": "conversation.item.create",
-                                    "item": {
-                                        "type": "message",
-                                        "role": "user",
-                                        "content": [
-                                            {
-                                                "type": "input_text",
-                                                "text": self._image_description
-                                            }
-                                        ]
-                                    }
-                                }
-                            logger.info("Sending image description after recognition.")
-                            await self.send_event(text_event)
-                    return
 
                 await self.send_event(append_event)
         except Exception as e: