feat(chat): wire chat role into ProviderResolver.resolve_with_fallback (#55) (#75)

The failover resolver (PR-D) had no production caller: chat.py still picked
the first enabled provider directly, so model_defaults.candidates order and
cooldown were dead UI surface.

- chat endpoint: when the chat role has model_defaults.candidates, run
  through resolve_with_fallback() — connection-level failures (connect
  error/timeout/5xx, decision #7) fail over to the next candidate, business
  failures (4xx) re-raise immediately. Explicit provider_id and roles with
  no candidates keep the legacy single-provider path unchanged.
- tool loops (JSON + XML) now raise LlmAdapterError with classify_retryable
  instead of a bare 502, so the resolver can distinguish failover-worthy
  failures; 502 conversion stays at the endpoint boundary.
- resolver: skip disabled providers in resolve()/resolve_with_fallback()
  (governance choice, not liveness failure — no cooldown), add public
  has_candidates() for the legacy fallback branch.
- tests: unit coverage for disabled-skip + has_candidates; new
  tests/integration/test_chat_failover_api.py exercises the production
  chain through POST /api/v1/chat (legacy path, retryable failover,
  business-error no-failover, all-unavailable 502, disabled-skip,
  explicit-provider bypass).
- fix latent patch leak in test_resolver concurrency test (per-task
  unittest.mock.patch around an await races under asyncio.gather and leaked
  a get_adapter mock into later tests).

Author: lunnynight

backend/api/v1/chat.py

@@ -23,6 +23,8 @@
 
 from backend.control.agent_control import ACTION_REGISTRY, agent_control_service
 from backend.database import get_db
+from backend.llm import ResolverError, resolver
+from backend.llm.base import LlmAdapterError, classify_retryable
 from backend.models.provider import ModelProvider
 from backend.schemas.common import ApiResponse
 from backend.security.identity import RequestIdentity, get_request_identity
@@ -341,16 +343,25 @@ async def _build_proposal(
     )
 
 
-@router.post("", response_model=ApiResponse[ChatReply])
-async def chat(
+async def _chat_with_client(
+    client: Any,
+    model: str,
     body: ChatRequest,
-    identity: RequestIdentity | None = Depends(_optional_request_identity),
-    db: AsyncSession = Depends(get_db),
+    db: AsyncSession,
+    identity: RequestIdentity | None,
 ) -> ApiResponse:
-    provider = await _pick_provider(db, body.provider_id)
-    client = await _build_client(provider)
-    model = provider.default_model or "gpt-4o-mini"
-
+    """Run the agent-dock tool loop against ``client`` with ``model``.
+
+    Extracted from the ``chat`` endpoint so the same loop serves both the
+    legacy single-provider path and each candidate tried inside
+    ``ProviderResolver.resolve_with_fallback`` (model-provider runtime
+    PR-E, issue #55). LLM-call failures are re-raised as
+    :class:`~backend.llm.base.LlmAdapterError` carrying
+    :func:`~backend.llm.base.classify_retryable`'s connection-vs-business
+    split (decision #7) — the resolver only fails over connection-level
+    failures, and the HTTP 502 conversion happens once at the endpoint
+    boundary instead of being baked into the loop.
+    """
     system = SYSTEM_PROMPT
     if body.context:
         system += f"\n\n当前用户操作上下文 (JSON): {json.dumps(body.context, ensure_ascii=False)}"
@@ -366,9 +377,13 @@ async def chat(
             response = await client.chat.completions.create(
                 model=model, messages=messages, tools=TOOLS, tool_choice="auto"
             )
+        except LlmAdapterError:
+            raise
         except Exception as exc:
             logger.error("chat llm error | %s", exc)
-            raise HTTPException(status_code=502, detail=f"模型调用失败: {exc}") from exc
+            raise LlmAdapterError(
+                f"chat llm error: {exc}", retryable=classify_retryable(exc)
+            ) from exc
 
         msg = response.choices[0].message
         tool_calls = msg.tool_calls or []
@@ -413,6 +428,61 @@ async def chat(
     return ApiResponse.ok(ChatReply(type="message", content="(达到工具调用步数上限, 请换个说法再试)"))
 
 
+async def _chat_single_provider(
+    db: AsyncSession,
+    body: ChatRequest,
+    identity: RequestIdentity | None,
+    provider_id: Optional[str],
+) -> ApiResponse:
+    """Legacy pre-failover chat path (issue #55): one provider — the
+    explicit ``provider_id`` or the first enabled one — no candidate
+    failover. Kept as the fallback when the ``chat`` role has no
+    ``model_defaults.candidates`` configured, so existing installs behave
+    exactly as before the failover wiring.
+    """
+    provider = await _pick_provider(db, provider_id)
+    client = await _build_client(provider)
+    model = provider.default_model or "gpt-4o-mini"
+    return await _chat_with_client(client, model, body, db, identity)
+
+
+@router.post("", response_model=ApiResponse[ChatReply])
+async def chat(
+    body: ChatRequest,
+    identity: RequestIdentity | None = Depends(_optional_request_identity),
+    db: AsyncSession = Depends(get_db),
+) -> ApiResponse:
+    """Agent dock chat (model-provider runtime PR-E, issue #55).
+
+    Provider selection now runs through
+    :func:`backend.llm.resolver.ProviderResolver.resolve_with_fallback` for
+    the ``chat`` role whenever ``model_defaults.candidates`` are configured
+    for it: candidates are tried in order and a connection-level failure
+    (connect error / timeout / 5xx — decision #7) fails over to the next
+    candidate, while a business failure (4xx: bad key, malformed request)
+    is re-raised immediately. An explicit ``provider_id`` still bypasses
+    failover (the user picked that provider on purpose), and a role with no
+    candidates falls back to the legacy first-enabled-provider lookup.
+    """
+    if body.provider_id or not await resolver.has_candidates(db, "chat"):
+        return await _chat_single_provider(db, body, identity, body.provider_id)
+
+    async def operation(adapter: Any, model_id: str) -> ApiResponse:
+        provider = adapter.provider
+        client = await _build_client(provider)
+        model = model_id or provider.default_model or "gpt-4o-mini"
+        return await _chat_with_client(client, model, body, db, identity)
+
+    try:
+        return await resolver.resolve_with_fallback(db, "chat", operation)
+    except (LlmAdapterError, ResolverError) as exc:
+        # Business-level failure (re-raised immediately, no candidate left
+        # untried) or every candidate skipped/failed → single 502 explaining
+        # why; never leaks a provider api_key (adapters sanitize).
+        logger.error("chat failover | %s", exc)
+        raise HTTPException(status_code=502, detail=f"模型调用失败: {exc}") from exc
+
+
 @router.post("/confirm", response_model=ApiResponse[dict])
 async def confirm(
     body: ConfirmRequest,
@@ -502,9 +572,13 @@ async def _chat_xml(
     for _step in range(MAX_TOOL_STEPS):
         try:
             response = await client.chat.completions.create(model=model, messages=messages, max_tokens=1024)
+        except LlmAdapterError:
+            raise
         except Exception as exc:
             logger.error("chat(xml) llm error | %s", exc)
-            raise HTTPException(status_code=502, detail=f"模型调用失败: {exc}") from exc
+            raise LlmAdapterError(
+                f"chat(xml) llm error: {exc}", retryable=classify_retryable(exc)
+            ) from exc
 
         content = response.choices[0].message.content or ""
         calls = _parse_tool_use(content)

backend/llm/resolver.py

@@ -102,24 +102,37 @@ async def _candidates_for(self, db: AsyncSession, role: str) -> list[dict[str, A
             return []
         return list(row.candidates or [])
 
+    async def has_candidates(self, db: AsyncSession, role: str) -> bool:
+        """``True`` iff a non-empty ``model_defaults.candidates`` list exists
+        for ``role``.
+
+        A cheap pre-check for callers that want to keep a legacy fallback
+        path when the role has no failover order configured (e.g.
+        ``chat.py``: candidates configured → failover runtime; no candidates
+        → the pre-failover single-provider lookup, so existing installs
+        keep working unchanged).
+        """
+        return bool(await self._candidates_for(db, role))
+
     async def resolve(self, db: AsyncSession, role: str) -> ResolvedModel | None:
         """Return the primary (first) candidate configured for ``role``.
 
         Returns ``None`` — rather than raising — when no ``model_defaults``
-        row exists for ``role``, its ``candidates`` list is empty, or the
+        row exists for ``role``, its ``candidates`` list is empty, the
         first candidate's ``provider_id`` no longer resolves to a real
-        provider row: this is a direct "what's configured" lookup for
-        callers that want the single default, not a "try until one works"
-        search (that's :meth:`resolve_with_fallback`) — there is nothing to
-        fail over to here, so a clean ``None`` is more useful to a caller
-        than an exception for what is often just "nothing configured yet".
+        provider row, or that provider is currently disabled: this is a
+        direct "what's configured AND usable" lookup for callers that want
+        the single default, not a "try until one works" search (that's
+        :meth:`resolve_with_fallback`) — there is nothing to fail over to
+        here, so a clean ``None`` is more useful to a caller than an
+        exception for what is often just "nothing configured yet".
         """
         candidates = await self._candidates_for(db, role)
         if not candidates:
             return None
         candidate = candidates[0]
         provider = await db.get(ModelProvider, candidate["provider_id"])
-        if provider is None:
+        if provider is None or not provider.enabled:
             return None
         return ResolvedModel(
             provider=provider,
@@ -140,7 +153,10 @@ async def resolve_with_fallback(
         - A candidate whose ``provider_id`` no longer resolves to a real
           provider row (deleted since the default was configured) is
           skipped the same way, without being put in cooldown (there is no
-          "it" to cool down).
+          "it" to cool down). A candidate whose provider row is currently
+          ``enabled=False`` is skipped identically — disabling a provider
+          in governance is an operator choice, not a liveness failure, so
+          it must not be used and must not go into cooldown either.
         - ``operation`` succeeding returns that result immediately — no
           further candidates are tried.
         - ``operation`` raising :class:`~backend.llm.base.LlmAdapterError`
@@ -170,6 +186,11 @@ async def resolve_with_fallback(
                 # Candidate references a since-deleted provider — dead, but
                 # not "this provider just failed", so no cooldown to set.
                 continue
+            if not provider.enabled:
+                # Disabled in governance — an operator choice, not a
+                # liveness failure: skip without cooldown, same as a
+                # deleted provider.
+                continue
             adapter = get_adapter(provider)
             tried += 1
             try: