1717from __future__ import annotations
1818
1919import asyncio
20+ import functools
2021import os
2122from contextlib import nullcontext
2223from typing import Any
@@ -527,9 +528,67 @@ def auditor_kwargs(*, target: dict, auditor: dict, judge: dict, generation: dict
527528 extra_body = {** (extra_body or {}), ** _p .pop ("extra_body" )}
528529 if extra_body is not None :
529530 kwargs ["extra_body" ] = extra_body
531+ # A decision model answers once and cannot take part in a follow-up turn
532+ # (``DecisionTarget.max_turns`` is 1), so the generation config's max_turns
533+ # must not reach it. Forced here rather than at either construction site so
534+ # the single-repetition path and the repetition runner agree — the latter
535+ # also derives max_retries_per_rep and its model entry from these kwargs.
536+ from model_registry .decision import is_decision_snapshot
537+
538+ if is_decision_snapshot (target ):
539+ kwargs ["max_turns" ] = 1
530540 return kwargs , gen .get ("language" ) or "English"
531541
532542
543+ def decision_target_for (target : dict , * , resolve_key = snapshot_api_key ):
544+ """The ``DecisionTarget`` a target snapshot calls for, or ``None``.
545+
546+ The capability was settled when the model was registered (the /connections/
547+ probe writes ``capabilities["decision"]``, read by
548+ ``RegisteredModel.is_decision``) and frozen into the run's snapshot, so a
549+ run never re-probes. An OpenRouter decision model needs its key, resolved
550+ here from the snapshot's ``secret_reference`` like every other role.
551+ """
552+ from model_registry .decision import decision_target_for_snapshot
553+
554+ return decision_target_for_snapshot (target , api_key = resolve_key (target ) or "" )
555+
556+
557+ @functools .cache
558+ def without_target_client (auditor_cls ):
559+ """``auditor_cls`` built without its (unused) AnyLLM chat client.
560+
561+ SimpleAudit's own ``Auditor`` facade does exactly this when an explicit
562+ Target is supplied: the chat client is never called, and building it would
563+ demand the provider's any_llm extra and an API key for nothing. A decision
564+ run is that case — a local Ollama decision model would otherwise fail on
565+ ``any-llm-sdk[ollama]`` not being installed.
566+
567+ A subclass rather than a flag set on the shared class: scenarios run in
568+ worker threads, and a chat auditor built meanwhile must keep its client.
569+ """
570+ return type (auditor_cls .__name__ , (auditor_cls ,), {"_skip_target_client" : True })
571+
572+
573+ def install_target (instance , decision_target ) -> bool :
574+ """Install the Target a run sends to; ``True`` if it is a decision target.
575+
576+ A decision model answers on ``/v1/systemone`` and is not a chat client, so
577+ it replaces the target outright rather than being wrapped by the
578+ trace-context adapter.
579+ """
580+ if decision_target is not None :
581+ instance .set_target (decision_target )
582+ return True
583+ # SimpleAudit 0.3.1's stock ModelTarget accepts TargetContext but drops it
584+ # before calling the OpenAI-compatible client. Install the narrow adapter
585+ # so the engine's per-turn W3C traceparent reaches the target process.
586+ from infra .trace_target import install_trace_context_target
587+
588+ install_trace_context_target (instance )
589+ return False
590+
591+
533592def build_model_auditor (* , target : dict , auditor : dict , judge : dict , generation : dict | None = None ):
534593 """Construct a ModelAuditor from three frozen endpoint snapshots.
535594
@@ -545,20 +604,25 @@ def build_model_auditor(*, target: dict, auditor: dict, judge: dict, generation:
545604 _kw , language = auditor_kwargs (target = target , auditor = auditor , judge = judge , generation = generation )
546605 extra_body = _kw .pop ("extra_body" , None )
547606 try :
548- instance = ModelAuditor (** _kw )
607+ # Inside the try: a decision snapshot that cannot be turned into a
608+ # target (no base URL, no key) is an auditor that cannot be built.
609+ decision = decision_target_for (target )
610+ auditor_cls = ModelAuditor if decision is None else without_target_client (ModelAuditor )
611+ instance = auditor_cls (** _kw )
549612 except Exception as exc :
550613 raise EngineError (f"Failed to construct ModelAuditor: { type (exc ).__name__ } : { exc } " ) from exc
551- # SimpleAudit 0.3.1's stock ModelTarget accepts TargetContext but drops it
552- # before calling the OpenAI-compatible client. Install the narrow adapter
553- # so the engine's per-turn W3C traceparent reaches the target process.
554- from infra . trace_target import install_trace_context_target
555-
556- install_trace_context_target (instance )
614+ # Exactly one target is installed: both paths call ``set_target``, so the
615+ # trace-context adapter must not run after a decision target (it would put
616+ # a ModelTarget over the skipped chat client back in its place). A decision
617+ # target still carries the traceparent: ``DecisionTarget.send`` merges
618+ # ``TargetContext.trace_headers`` into its own request headers.
619+ install_target (instance , decision )
557620 # ``extra_body`` (session_id / tool_ids for Open WebUI's saved-chat path)
558621 # is a per-request parameter. The engine merges self.target_params over
559622 # its own base params on every turn, so injecting it here is what makes
560623 # TraceContextModelTarget route agent turns through OWUI's server-side
561- # tool-execution chat.
624+ # tool-execution chat. A DecisionTarget ignores per-request params, so
625+ # there it is inert.
562626 if extra_body :
563627 tp = dict (instance .target_params or {})
564628 tp ["extra_body" ] = {** (tp .get ("extra_body" ) or {}), ** extra_body }
@@ -1069,9 +1133,24 @@ def run_scenario_repeated(
10691133 # subclass used by the single-repetition path.
10701134 import simpleaudit .experiment as experiment_module
10711135
1072- from infra .trace_target import TraceContextModelAuditor
1136+ from infra .trace_target import TraceContextModelAuditor , decision_auditor_class
10731137
1074- experiment_module .ModelAuditor = TraceContextModelAuditor
1138+ # A decision run replaces the auditor's target entirely; a chat run gets
1139+ # the trace-context adapter. The repetition runner builds its own
1140+ # auditors, so the choice has to be made on the class.
1141+ # Built once here so a malformed decision snapshot fails before the
1142+ # experiment is set up; each auditor then gets its own (a DecisionTarget
1143+ # owns an HTTP client).
1144+ if decision_target_for (target ) is None :
1145+ experiment_module .ModelAuditor = TraceContextModelAuditor
1146+ else :
1147+ # The key is resolved here, before the loop, as auditor_kwargs does for
1148+ # every role: AuditExperiment builds each repetition's auditor inside
1149+ # the running event loop, where Django refuses the connection lookup.
1150+ target_key = snapshot_api_key (target ) or ""
1151+ experiment_module .ModelAuditor = decision_auditor_class (
1152+ lambda : decision_target_for (target , resolve_key = lambda _s : target_key )
1153+ )
10751154
10761155 kwargs , language = auditor_kwargs (target = target , auditor = auditor , judge = judge , generation = generation )
10771156 max_turns = kwargs ["max_turns" ]
0 commit comments