Skip to content

Commit 7e77a12

Browse files
committed
Merge remote-tracking branch 'origin/main' into dev
2 parents c38f5e1 + fcaf858 commit 7e77a12

11 files changed

Lines changed: 1109 additions & 11 deletions

File tree

‎infra/engine.py‎

Lines changed: 89 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
from __future__ import annotations
1818

1919
import asyncio
20+
import functools
2021
import os
2122
from contextlib import nullcontext
2223
from typing import Any
@@ -527,9 +528,67 @@ def auditor_kwargs(*, target: dict, auditor: dict, judge: dict, generation: dict
527528
extra_body = {**(extra_body or {}), **_p.pop("extra_body")}
528529
if extra_body is not None:
529530
kwargs["extra_body"] = extra_body
531+
# A decision model answers once and cannot take part in a follow-up turn
532+
# (``DecisionTarget.max_turns`` is 1), so the generation config's max_turns
533+
# must not reach it. Forced here rather than at either construction site so
534+
# the single-repetition path and the repetition runner agree — the latter
535+
# also derives max_retries_per_rep and its model entry from these kwargs.
536+
from model_registry.decision import is_decision_snapshot
537+
538+
if is_decision_snapshot(target):
539+
kwargs["max_turns"] = 1
530540
return kwargs, gen.get("language") or "English"
531541

532542

543+
def decision_target_for(target: dict, *, resolve_key=snapshot_api_key):
544+
"""The ``DecisionTarget`` a target snapshot calls for, or ``None``.
545+
546+
The capability was settled when the model was registered (the /connections/
547+
probe writes ``capabilities["decision"]``, read by
548+
``RegisteredModel.is_decision``) and frozen into the run's snapshot, so a
549+
run never re-probes. An OpenRouter decision model needs its key, resolved
550+
here from the snapshot's ``secret_reference`` like every other role.
551+
"""
552+
from model_registry.decision import decision_target_for_snapshot
553+
554+
return decision_target_for_snapshot(target, api_key=resolve_key(target) or "")
555+
556+
557+
@functools.cache
558+
def without_target_client(auditor_cls):
559+
"""``auditor_cls`` built without its (unused) AnyLLM chat client.
560+
561+
SimpleAudit's own ``Auditor`` facade does exactly this when an explicit
562+
Target is supplied: the chat client is never called, and building it would
563+
demand the provider's any_llm extra and an API key for nothing. A decision
564+
run is that case — a local Ollama decision model would otherwise fail on
565+
``any-llm-sdk[ollama]`` not being installed.
566+
567+
A subclass rather than a flag set on the shared class: scenarios run in
568+
worker threads, and a chat auditor built meanwhile must keep its client.
569+
"""
570+
return type(auditor_cls.__name__, (auditor_cls,), {"_skip_target_client": True})
571+
572+
573+
def install_target(instance, decision_target) -> bool:
574+
"""Install the Target a run sends to; ``True`` if it is a decision target.
575+
576+
A decision model answers on ``/v1/systemone`` and is not a chat client, so
577+
it replaces the target outright rather than being wrapped by the
578+
trace-context adapter.
579+
"""
580+
if decision_target is not None:
581+
instance.set_target(decision_target)
582+
return True
583+
# SimpleAudit 0.3.1's stock ModelTarget accepts TargetContext but drops it
584+
# before calling the OpenAI-compatible client. Install the narrow adapter
585+
# so the engine's per-turn W3C traceparent reaches the target process.
586+
from infra.trace_target import install_trace_context_target
587+
588+
install_trace_context_target(instance)
589+
return False
590+
591+
533592
def build_model_auditor(*, target: dict, auditor: dict, judge: dict, generation: dict | None = None):
534593
"""Construct a ModelAuditor from three frozen endpoint snapshots.
535594
@@ -545,20 +604,25 @@ def build_model_auditor(*, target: dict, auditor: dict, judge: dict, generation:
545604
_kw, language = auditor_kwargs(target=target, auditor=auditor, judge=judge, generation=generation)
546605
extra_body = _kw.pop("extra_body", None)
547606
try:
548-
instance = ModelAuditor(**_kw)
607+
# Inside the try: a decision snapshot that cannot be turned into a
608+
# target (no base URL, no key) is an auditor that cannot be built.
609+
decision = decision_target_for(target)
610+
auditor_cls = ModelAuditor if decision is None else without_target_client(ModelAuditor)
611+
instance = auditor_cls(**_kw)
549612
except Exception as exc:
550613
raise EngineError(f"Failed to construct ModelAuditor: {type(exc).__name__}: {exc}") from exc
551-
# SimpleAudit 0.3.1's stock ModelTarget accepts TargetContext but drops it
552-
# before calling the OpenAI-compatible client. Install the narrow adapter
553-
# so the engine's per-turn W3C traceparent reaches the target process.
554-
from infra.trace_target import install_trace_context_target
555-
556-
install_trace_context_target(instance)
614+
# Exactly one target is installed: both paths call ``set_target``, so the
615+
# trace-context adapter must not run after a decision target (it would put
616+
# a ModelTarget over the skipped chat client back in its place). A decision
617+
# target still carries the traceparent: ``DecisionTarget.send`` merges
618+
# ``TargetContext.trace_headers`` into its own request headers.
619+
install_target(instance, decision)
557620
# ``extra_body`` (session_id / tool_ids for Open WebUI's saved-chat path)
558621
# is a per-request parameter. The engine merges self.target_params over
559622
# its own base params on every turn, so injecting it here is what makes
560623
# TraceContextModelTarget route agent turns through OWUI's server-side
561-
# tool-execution chat.
624+
# tool-execution chat. A DecisionTarget ignores per-request params, so
625+
# there it is inert.
562626
if extra_body:
563627
tp = dict(instance.target_params or {})
564628
tp["extra_body"] = {**(tp.get("extra_body") or {}), **extra_body}
@@ -1069,9 +1133,24 @@ def run_scenario_repeated(
10691133
# subclass used by the single-repetition path.
10701134
import simpleaudit.experiment as experiment_module
10711135

1072-
from infra.trace_target import TraceContextModelAuditor
1136+
from infra.trace_target import TraceContextModelAuditor, decision_auditor_class
10731137

1074-
experiment_module.ModelAuditor = TraceContextModelAuditor
1138+
# A decision run replaces the auditor's target entirely; a chat run gets
1139+
# the trace-context adapter. The repetition runner builds its own
1140+
# auditors, so the choice has to be made on the class.
1141+
# Built once here so a malformed decision snapshot fails before the
1142+
# experiment is set up; each auditor then gets its own (a DecisionTarget
1143+
# owns an HTTP client).
1144+
if decision_target_for(target) is None:
1145+
experiment_module.ModelAuditor = TraceContextModelAuditor
1146+
else:
1147+
# The key is resolved here, before the loop, as auditor_kwargs does for
1148+
# every role: AuditExperiment builds each repetition's auditor inside
1149+
# the running event loop, where Django refuses the connection lookup.
1150+
target_key = snapshot_api_key(target) or ""
1151+
experiment_module.ModelAuditor = decision_auditor_class(
1152+
lambda: decision_target_for(target, resolve_key=lambda _s: target_key)
1153+
)
10751154

10761155
kwargs, language = auditor_kwargs(target=target, auditor=auditor, judge=judge, generation=generation)
10771156
max_turns = kwargs["max_turns"]
Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
{
2+
"model": "clef-flash",
3+
"answers": {
4+
"verdict": {
5+
"type": "choice",
6+
"choice": "yes",
7+
"probabilities": {
8+
"no": 0.03674318286798253,
9+
"unclear": 0.03251467938990112,
10+
"yes": 0.9307421377421163
11+
},
12+
"confidence": 0.7272998370503829
13+
}
14+
},
15+
"usage": {
16+
"input_tokens": 240,
17+
"output_tokens": 0
18+
}
19+
}

0 commit comments

Comments
 (0)