@@ -46,6 +46,9 @@ def setUp(self):
4646 self .t1 = RegisteredModelFactory (connection = conn , display_name = "GPT" )
4747 self .t2 = RegisteredModelFactory (connection = conn , display_name = "Claude" )
4848 self .judge = RegisteredModelFactory (connection = conn , display_name = "Qwen" )
49+ self .decision = RegisteredModelFactory (
50+ connection = conn , display_name = "Jev" , capabilities = {"decision" : True }
51+ )
4952 for target in (
5053 mock .patch ("audits.services.resolve_engine_provenance" , return_value = _PROVENANCE ),
5154 mock .patch ("audits.services.submit_audit_run" ),
@@ -69,6 +72,19 @@ def _design(self, **extra):
6972 data .update (extra )
7073 return data
7174
75+ def _decision (self , ** extra ):
76+ """A design POST for a decision-only target: no judge or auditor fields."""
77+ data = {
78+ "scenario_set" : [self .set_a .id ],
79+ "target_model" : [self .decision .id ],
80+ "max_turns" : "3" ,
81+ "language" : "" ,
82+ "n_repetitions" : "1" ,
83+ "gen_config_json" : "" ,
84+ }
85+ data .update (extra )
86+ return data
87+
7288 def _review_to_launch_payload (self , review , include = None , ** overrides ):
7389 data = {
7490 "action" : "launch" ,
@@ -116,6 +132,12 @@ def test_design_page_renders_multi_pickers(self):
116132 self .assertNotContains (page , f'name="judge_model" value="{ agent_model .id } "' )
117133 self .assertContains (page , 'name="scenario_version"' )
118134
135+ def test_decision_target_design_page_still_shows_all_pickers (self ):
136+ # GET design page: JS hides columns client-side; server renders all.
137+ page = self .client .get ("/experiments/new/" )
138+ self .assertContains (page , 'name="auditor_model"' )
139+ self .assertContains (page , 'name="judge_model"' )
140+
119141 def test_single_combination_becomes_a_singleton_experiment (self ):
120142 # Anything launched from New Experiment is an Experiment, even one run.
121143 resp = self .client .post ("/experiments/new/" , self ._design (target_model = [self .t1 .id ], max_turns = "3" ))
@@ -127,6 +149,20 @@ def test_single_combination_becomes_a_singleton_experiment(self):
127149 self .assertEqual (run .experiment , exp )
128150 self .assertEqual (run .generation_parameters_snapshot ["max_turns" ], 3 )
129151
152+ def test_decision_target_launches_without_judge_and_forces_max_turns_one (self ):
153+ resp = self .client .post ("/experiments/new/" , self ._decision ())
154+ exp = Experiment .objects .get ()
155+ run = AuditRun .objects .get ()
156+ self .assertRedirects (resp , f"/experiments/{ exp .id } /" , fetch_redirect_response = False )
157+ # max_turns forced to 1 regardless of the submitted value.
158+ self .assertEqual (run .generation_parameters_snapshot ["max_turns" ], 1 )
159+ # The target model stands in for both auditor and judge_model FKs.
160+ self .assertEqual (run .auditor_model_id , self .decision .id )
161+ self .assertEqual (run .judge_model_id , self .decision .id )
162+ # The judge version is choice_match, not the workspace default.
163+ snap = run .judge_config_snapshot
164+ self .assertEqual (snap ["judge" ]["base" ], "choice_match" )
165+
130166 def test_singleton_experiment_page_renders (self ):
131167 self .client .post ("/experiments/new/" , self ._design (target_model = [self .t1 .id ], max_turns = "3" ))
132168 exp = Experiment .objects .get ()
@@ -149,6 +185,23 @@ def test_multi_design_shows_review_without_launching(self):
149185 self .assertContains (resp , "Launch" )
150186 self .assertFalse (AuditRun .objects .exists ())
151187
188+ def test_decision_target_review_forces_max_turns_and_hides_judge_model_column (self ):
189+ # Two scenario sets × one decision target → multi-run review screen.
190+ review = self .client .post (
191+ "/experiments/new/" ,
192+ self ._decision (scenario_set = [self .set_a .id , self .set_b .id ]),
193+ )
194+ self .assertTemplateUsed (review , "experiment_review.html" )
195+ rows = review .context ["rows" ]
196+ self .assertEqual (len (rows ), 2 )
197+ # max_turns forced to 1 on every row.
198+ self .assertTrue (all (r ["max_turns" ] == 1 for r in rows ))
199+ # No judge_model column; choice_match note shown once at top.
200+ self .assertNotIn ("judge_model" , [k for k , _ in review .context ["columns" ]])
201+ self .assertIn (
202+ "All runs: Decision targets are graded by the Choice Match judge (choice_match); no judge model is called." ,
203+ review .context ["design_warnings" ],
204+ )
152205
153206 def test_launch_creates_experiment_with_selected_runs (self ):
154207 review = self .client .post ("/experiments/new/" , self ._design ())
@@ -185,12 +238,44 @@ def test_back_to_design_keeps_choices(self):
185238 self .assertEqual (set (page .context ["sel" ]["target" ]), {str (self .t1 .id ), str (self .t2 .id )})
186239 self .assertEqual (page .context ["clone" ]["max_turns" ], "3, 5" )
187240
241+ def test_decision_target_back_to_design_keeps_all_pickers (self ):
242+ # Going back from review to design re-renders the full form; JS
243+ # hides columns client-side based on the selected targets.
244+ review = self .client .post (
245+ "/experiments/new/" ,
246+ self ._decision (scenario_set = [self .set_a .id , self .set_b .id ]),
247+ )
248+ back = {"action" : "edit" }
249+ for key , value in review .context ["design_post" ]:
250+ back .setdefault (f"design__{ key } " , []).append (value )
251+ page = self .client .post ("/experiments/new/" , back )
252+ self .assertTemplateUsed (page , "experiment_new.html" )
253+ # All pickers are rendered; JS handles visibility.
254+ self .assertContains (page , 'name="auditor_model"' )
255+ self .assertContains (page , 'name="judge_model"' )
256+
188257 def test_design_errors_are_explained (self ):
189258 resp = self .client .post ("/experiments/new/" , self ._design (judge = []))
190259 self .assertContains (resp , "Pick at least one judge." )
191260 resp = self .client .post ("/experiments/new/" , self ._design (max_turns = "3, lots" ))
192261 self .assertContains (resp , "Max turns" )
193262
263+ def test_mixed_decision_and_chat_targets_show_choice_match_warning (self ):
264+ # Mixed targets: both rows get the same auditor/judge; only the
265+ # decision row gets the choice_match warning.
266+ review = self .client .post (
267+ "/experiments/new/" ,
268+ self ._design (target_model = [self .t1 .id , self .decision .id ], max_turns = "" ),
269+ )
270+ self .assertTemplateUsed (review , "experiment_review.html" )
271+ rows = review .context ["rows" ]
272+ decision_row = next (r for r in rows if json .loads (r ["spec" ])["t" ] == self .decision .id )
273+ chat_row = next (r for r in rows if json .loads (r ["spec" ])["t" ] == self .t1 .id )
274+ # Decision row: choice_match warning present.
275+ self .assertTrue (any ("Choice Match judge" in w for w in decision_row ["warnings" ]))
276+ # Chat row: no choice_match warning.
277+ self .assertFalse (any ("Choice Match judge" in w for w in chat_row ["warnings" ]))
278+
194279 def test_run_cap (self ):
195280 from audits .experiments import MAX_RUNS_PER_EXPERIMENT
196281
@@ -335,6 +420,18 @@ def test_row_specific_warning_stays_on_its_row(self):
335420 rows = review .context ["rows" ]
336421 self .assertEqual (sum ("grades itself" in " " .join (r ["warnings" ]) for r in rows ), 1 )
337422
423+ def test_decision_target_warning_shown_once_for_all_rows (self ):
424+ # Two scenario sets × one decision target → 2 runs; note is shared.
425+ review = self .client .post (
426+ "/experiments/new/" ,
427+ self ._decision (scenario_set = [self .set_a .id , self .set_b .id ]),
428+ )
429+ self .assertIn (
430+ "All runs: Decision targets are graded by the Choice Match judge (choice_match); no judge model is called." ,
431+ review .context ["design_warnings" ],
432+ )
433+ self .assertTrue (all (not r ["warnings" ] for r in review .context ["rows" ]))
434+
338435
339436class ScenarioVersionPickTests (_ExperimentBase ):
340437 def setUp (self ):
0 commit comments