Skip to content

Commit 1befa23

Browse files
committed
Merge dev: decision-target UI enforcement (choice_match auto-judge, max_turns=1)
2 parents 449665c + f4b44f2 commit 1befa23

4 files changed

Lines changed: 169 additions & 26 deletions

File tree

‎audits/experiments.py‎

Lines changed: 64 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -158,31 +158,37 @@ def parse_design(post, project) -> dict:
158158
)
159159
models[axis] = [found[i] for i in dict.fromkeys(ids)]
160160

161-
# Judges: version ids, and/or "latest:<judge id>" (always latest: unpinned,
162-
# so repeating runs follow new versions).
163-
raw = [v for v in dict.fromkeys(post.getlist("judge")) if v]
164-
if not raw:
165-
raise DesignError("Pick at least one judge.")
166-
follow_ids = [v.split(":", 1)[1] for v in raw if v.startswith("latest:")]
167-
pinned_ids = [v for v in raw if not v.startswith("latest:")]
168-
judges = list(
169-
JudgeVersion.objects.filter(pk__in=pinned_ids, judge__project=project).select_related("judge")
170-
)
171-
follow = list(Judge.objects.filter(pk__in=follow_ids, project=project))
172-
if len(judges) != len(pinned_ids) or len(follow) != len(follow_ids):
173-
raise DesignError("A selected judge is not in this workspace.")
174-
for judge in follow:
175-
latest = judge.versions.select_related("judge").order_by("-version").first()
176-
if latest is None:
177-
raise DesignError(f"Judge “{judge.name}” has no version.")
178-
latest.follow_latest = True
179-
judges.append(latest)
180-
judges.sort(key=lambda v: (v.judge.name, is_follow(v), v.version))
181-
182161
# When an agent is the target, its base_model IS the target model.
183162
if agent is not None and not models.get("target"):
184163
models["target"] = [agent.base_model]
185164

165+
all_decision = _all_decision_targets(models["target"])
166+
167+
# Judges: version ids, and/or "latest:<judge id>" (always latest: unpinned,
168+
# so repeating runs follow new versions). All-decision designs skip this:
169+
# choice_match grades in code and no judge model is called.
170+
if all_decision:
171+
judges = [_decision_judge_version(project)]
172+
else:
173+
raw = [v for v in dict.fromkeys(post.getlist("judge")) if v]
174+
if not raw:
175+
raise DesignError("Pick at least one judge.")
176+
follow_ids = [v.split(":", 1)[1] for v in raw if v.startswith("latest:")]
177+
pinned_ids = [v for v in raw if not v.startswith("latest:")]
178+
judges = list(
179+
JudgeVersion.objects.filter(pk__in=pinned_ids, judge__project=project).select_related("judge")
180+
)
181+
follow = list(Judge.objects.filter(pk__in=follow_ids, project=project))
182+
if len(judges) != len(pinned_ids) or len(follow) != len(follow_ids):
183+
raise DesignError("A selected judge is not in this workspace.")
184+
for judge in follow:
185+
latest = judge.versions.select_related("judge").order_by("-version").first()
186+
if latest is None:
187+
raise DesignError(f"Judge “{judge.name}” has no version.")
188+
latest.follow_latest = True
189+
judges.append(latest)
190+
judges.sort(key=lambda v: (v.judge.name, is_follow(v), v.version))
191+
186192
n_reps_raw = (post.get("n_repetitions") or "").strip()
187193
try:
188194
n_reps = int(n_reps_raw) if n_reps_raw else None
@@ -293,16 +299,46 @@ def run_count(design: dict) -> int:
293299
return n
294300

295301

302+
def _all_decision_targets(targets: list) -> bool:
303+
"""True when every selected target is a decision model."""
304+
return bool(targets) and all(m.is_decision for m in targets)
305+
306+
307+
def _decision_judge_version(project):
308+
"""A Choice Match judge version for this workspace, created on first use."""
309+
from judges.models import JudgeVersion
310+
from judges.services import create_judge, unique_name
311+
312+
existing = JudgeVersion.objects.filter(
313+
judge__project=project, base="choice_match"
314+
).order_by("judge__name", "version").first()
315+
if existing:
316+
return existing
317+
name = unique_name(project, "Choice Match")
318+
judge = create_judge(project=project, name=name, base="choice_match")
319+
return judge.latest
320+
321+
296322
def expand(design: dict) -> list[dict]:
297323
"""Every combination of the design's values, as run specs."""
298324
if run_count(design) > MAX_RUNS_PER_EXPERIMENT:
299325
raise DesignError(
300326
f"This design makes {run_count(design)} runs; the limit is {MAX_RUNS_PER_EXPERIMENT}. "
301327
"Fix more inputs to one value, or split it into several experiments."
302328
)
329+
all_decision = _all_decision_targets(design["target"])
330+
# All-decision designs have no auditor or judge model; give them a
331+
# placeholder so the cartesian product isn't empty (overwritten below).
332+
axes = {k: design[k] for k in DESIGN_AXES}
333+
if all_decision:
334+
axes["auditor"] = [None]
335+
axes["judge_model"] = [None]
303336
specs = []
304-
for combo in itertools.product(*(design[key] for key in DESIGN_AXES)):
337+
for combo in itertools.product(*(axes[key] for key in DESIGN_AXES)):
305338
spec = dict(zip(DESIGN_AXES, combo, strict=True))
339+
if all_decision:
340+
spec["auditor"] = spec["judge_model"] = spec["target"]
341+
spec["max_turns"] = 1
306342
spec["n_repetitions"] = design["n_repetitions"]
307343
spec["gen_config"] = design["gen_config"]
308344
spec["system_prompt_version"] = design.get("system_prompt_version")
@@ -360,7 +396,12 @@ def spec_label(spec: dict, factors: list[str]) -> str:
360396

361397
def spec_warnings(spec: dict) -> list[str]:
362398
warnings = []
363-
if spec["judge_model"].pk == spec["target"].pk:
399+
if spec["target"].is_decision:
400+
warnings.append(
401+
"Decision targets are graded by the Choice Match judge (choice_match); "
402+
"no judge model is called."
403+
)
404+
elif spec["judge_model"].pk == spec["target"].pk:
364405
warnings.append("Judge model is the target model: it grades itself.")
365406
for role, model in (("Target", spec["target"]), ("Auditor", spec["auditor"]), ("Judge", spec["judge_model"])):
366407
if not model.has_key:

‎infra/tests/test_experiments.py‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,9 @@ def setUp(self):
4646
self.t1 = RegisteredModelFactory(connection=conn, display_name="GPT")
4747
self.t2 = RegisteredModelFactory(connection=conn, display_name="Claude")
4848
self.judge = RegisteredModelFactory(connection=conn, display_name="Qwen")
49+
self.decision = RegisteredModelFactory(
50+
connection=conn, display_name="Jev", capabilities={"decision": True}
51+
)
4952
for target in (
5053
mock.patch("audits.services.resolve_engine_provenance", return_value=_PROVENANCE),
5154
mock.patch("audits.services.submit_audit_run"),
@@ -69,6 +72,19 @@ def _design(self, **extra):
6972
data.update(extra)
7073
return data
7174

75+
def _decision(self, **extra):
76+
"""A design POST for a decision-only target: no judge or auditor fields."""
77+
data = {
78+
"scenario_set": [self.set_a.id],
79+
"target_model": [self.decision.id],
80+
"max_turns": "3",
81+
"language": "",
82+
"n_repetitions": "1",
83+
"gen_config_json": "",
84+
}
85+
data.update(extra)
86+
return data
87+
7288
def _review_to_launch_payload(self, review, include=None, **overrides):
7389
data = {
7490
"action": "launch",
@@ -116,6 +132,12 @@ def test_design_page_renders_multi_pickers(self):
116132
self.assertNotContains(page, f'name="judge_model" value="{agent_model.id}"')
117133
self.assertContains(page, 'name="scenario_version"')
118134

135+
def test_decision_target_design_page_still_shows_all_pickers(self):
136+
# GET design page: JS hides columns client-side; server renders all.
137+
page = self.client.get("/experiments/new/")
138+
self.assertContains(page, 'name="auditor_model"')
139+
self.assertContains(page, 'name="judge_model"')
140+
119141
def test_single_combination_becomes_a_singleton_experiment(self):
120142
# Anything launched from New Experiment is an Experiment, even one run.
121143
resp = self.client.post("/experiments/new/", self._design(target_model=[self.t1.id], max_turns="3"))
@@ -127,6 +149,20 @@ def test_single_combination_becomes_a_singleton_experiment(self):
127149
self.assertEqual(run.experiment, exp)
128150
self.assertEqual(run.generation_parameters_snapshot["max_turns"], 3)
129151

152+
def test_decision_target_launches_without_judge_and_forces_max_turns_one(self):
153+
resp = self.client.post("/experiments/new/", self._decision())
154+
exp = Experiment.objects.get()
155+
run = AuditRun.objects.get()
156+
self.assertRedirects(resp, f"/experiments/{exp.id}/", fetch_redirect_response=False)
157+
# max_turns forced to 1 regardless of the submitted value.
158+
self.assertEqual(run.generation_parameters_snapshot["max_turns"], 1)
159+
# The target model stands in for both auditor and judge_model FKs.
160+
self.assertEqual(run.auditor_model_id, self.decision.id)
161+
self.assertEqual(run.judge_model_id, self.decision.id)
162+
# The judge version is choice_match, not the workspace default.
163+
snap = run.judge_config_snapshot
164+
self.assertEqual(snap["judge"]["base"], "choice_match")
165+
130166
def test_singleton_experiment_page_renders(self):
131167
self.client.post("/experiments/new/", self._design(target_model=[self.t1.id], max_turns="3"))
132168
exp = Experiment.objects.get()
@@ -149,6 +185,23 @@ def test_multi_design_shows_review_without_launching(self):
149185
self.assertContains(resp, "Launch")
150186
self.assertFalse(AuditRun.objects.exists())
151187

188+
def test_decision_target_review_forces_max_turns_and_hides_judge_model_column(self):
189+
# Two scenario sets × one decision target → multi-run review screen.
190+
review = self.client.post(
191+
"/experiments/new/",
192+
self._decision(scenario_set=[self.set_a.id, self.set_b.id]),
193+
)
194+
self.assertTemplateUsed(review, "experiment_review.html")
195+
rows = review.context["rows"]
196+
self.assertEqual(len(rows), 2)
197+
# max_turns forced to 1 on every row.
198+
self.assertTrue(all(r["max_turns"] == 1 for r in rows))
199+
# No judge_model column; choice_match note shown once at top.
200+
self.assertNotIn("judge_model", [k for k, _ in review.context["columns"]])
201+
self.assertIn(
202+
"All runs: Decision targets are graded by the Choice Match judge (choice_match); no judge model is called.",
203+
review.context["design_warnings"],
204+
)
152205

153206
def test_launch_creates_experiment_with_selected_runs(self):
154207
review = self.client.post("/experiments/new/", self._design())
@@ -185,12 +238,44 @@ def test_back_to_design_keeps_choices(self):
185238
self.assertEqual(set(page.context["sel"]["target"]), {str(self.t1.id), str(self.t2.id)})
186239
self.assertEqual(page.context["clone"]["max_turns"], "3, 5")
187240

241+
def test_decision_target_back_to_design_keeps_all_pickers(self):
242+
# Going back from review to design re-renders the full form; JS
243+
# hides columns client-side based on the selected targets.
244+
review = self.client.post(
245+
"/experiments/new/",
246+
self._decision(scenario_set=[self.set_a.id, self.set_b.id]),
247+
)
248+
back = {"action": "edit"}
249+
for key, value in review.context["design_post"]:
250+
back.setdefault(f"design__{key}", []).append(value)
251+
page = self.client.post("/experiments/new/", back)
252+
self.assertTemplateUsed(page, "experiment_new.html")
253+
# All pickers are rendered; JS handles visibility.
254+
self.assertContains(page, 'name="auditor_model"')
255+
self.assertContains(page, 'name="judge_model"')
256+
188257
def test_design_errors_are_explained(self):
189258
resp = self.client.post("/experiments/new/", self._design(judge=[]))
190259
self.assertContains(resp, "Pick at least one judge.")
191260
resp = self.client.post("/experiments/new/", self._design(max_turns="3, lots"))
192261
self.assertContains(resp, "Max turns")
193262

263+
def test_mixed_decision_and_chat_targets_show_choice_match_warning(self):
264+
# Mixed targets: both rows get the same auditor/judge; only the
265+
# decision row gets the choice_match warning.
266+
review = self.client.post(
267+
"/experiments/new/",
268+
self._design(target_model=[self.t1.id, self.decision.id], max_turns=""),
269+
)
270+
self.assertTemplateUsed(review, "experiment_review.html")
271+
rows = review.context["rows"]
272+
decision_row = next(r for r in rows if json.loads(r["spec"])["t"] == self.decision.id)
273+
chat_row = next(r for r in rows if json.loads(r["spec"])["t"] == self.t1.id)
274+
# Decision row: choice_match warning present.
275+
self.assertTrue(any("Choice Match judge" in w for w in decision_row["warnings"]))
276+
# Chat row: no choice_match warning.
277+
self.assertFalse(any("Choice Match judge" in w for w in chat_row["warnings"]))
278+
194279
def test_run_cap(self):
195280
from audits.experiments import MAX_RUNS_PER_EXPERIMENT
196281

@@ -335,6 +420,18 @@ def test_row_specific_warning_stays_on_its_row(self):
335420
rows = review.context["rows"]
336421
self.assertEqual(sum("grades itself" in " ".join(r["warnings"]) for r in rows), 1)
337422

423+
def test_decision_target_warning_shown_once_for_all_rows(self):
424+
# Two scenario sets × one decision target → 2 runs; note is shared.
425+
review = self.client.post(
426+
"/experiments/new/",
427+
self._decision(scenario_set=[self.set_a.id, self.set_b.id]),
428+
)
429+
self.assertIn(
430+
"All runs: Decision targets are graded by the Choice Match judge (choice_match); no judge model is called.",
431+
review.context["design_warnings"],
432+
)
433+
self.assertTrue(all(not r["warnings"] for r in review.context["rows"]))
434+
338435

339436
class ScenarioVersionPickTests(_ExperimentBase):
340437
def setUp(self):

‎infra/ui.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1719,15 +1719,17 @@ def lookup(found, pk, model):
17191719
if spec.get("jf"):
17201720
judge = copy.copy(judge) # the flag is per row
17211721
judge.follow_latest = True
1722+
target_model = lookup(models, spec["t"], RegisteredModel)
1723+
is_dec = target_model.is_decision
17221724
objs = {
17231725
"version": version,
1724-
"target": lookup(models, spec["t"], RegisteredModel),
1726+
"target": target_model,
17251727
"auditor": lookup(models, spec["a"], RegisteredModel),
17261728
"judge_model": lookup(models, spec["jm"], RegisteredModel),
17271729
"judge": judge,
17281730
}
17291731
name = (post.get(f"run-{i}-name") or "").strip() or f"Run {i + 1}"
1730-
mt = (post.get(f"run-{i}-max_turns") or "").strip()
1732+
mt = "1" if is_dec else (post.get(f"run-{i}-max_turns") or "").strip()
17311733
lang = (post.get(f"run-{i}-language") or "").strip()
17321734
reps = (post.get(f"run-{i}-n_repetitions") or "").strip()
17331735
system_prompt = (post.get(f"run-{i}-system_prompt") or "").strip()

‎templates/experiment_new.html‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -174,7 +174,10 @@ <h3 class="text-sm font-semibold text-gray-300 mb-1">Schedule</h3>
174174
const summary = document.getElementById('design-summary');
175175
const canLaunch = {{ can_launch|yesno:"true,false" }};
176176
const MAX_RUNS = {{ max_runs }};
177-
const AXES = [['scenario_version', 'scenario set'], ['target_model', 'target'], ['auditor_model', 'auditor'], ['judge_model', 'judge model'], ['judge', 'judge']];
177+
const ALL_DECISION = {{ all_decision|yesno:"true,false" }};
178+
const AXES = ALL_DECISION
179+
? [['scenario_version', 'scenario set'], ['target_model', 'target'], ['judge', 'judge']]
180+
: [['scenario_version', 'scenario set'], ['target_model', 'target'], ['auditor_model', 'auditor'], ['judge_model', 'judge model'], ['judge', 'judge']];
178181

179182
function listCount(name) {
180183
const el = form.querySelector(`[name="${name}"]`);

0 commit comments

Comments
 (0)