Skip to content

Commit 80f73b6

Browse files
wfzyxwfzyx
committed
gate: von-1.4-judge-mix (1.2 + 75k judge/diversity + 30k replay) vs von-1.2, chains off — null (pooled +0.2pp p=0.95, heldout -0.1pp); ordinal 3/12→9/12 p=0.031 offset by intent/routing losses
1 parent 2477078 commit 80f73b6

1 file changed

Lines changed: 235 additions & 0 deletions

File tree

‎results/v1.4/gate_judge_mix.json‎

Lines changed: 235 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,235 @@
1+
{
2+
"base": "checkpoints/von-1.2",
3+
"cand": "/home/wfzyx/scratch/ckpt/von-1.4-judge-mix",
4+
"cand_chains": "",
5+
"gates": {
6+
"jev_standard": {
7+
"label": "jev_standard",
8+
"n": 72,
9+
"acc_baseline": 0.5555555555555556,
10+
"acc_candidate": 0.5694444444444444,
11+
"delta_pp": 1.388888888888884,
12+
"discordant_base_only": 10,
13+
"discordant_cand_only": 11,
14+
"mcnemar_p": 1.0,
15+
"mde_pp_80pct": 16.40627714548672,
16+
"verdict": "UNRESOLVABLE"
17+
},
18+
"jabr_v2": {
19+
"label": "jabr_v2",
20+
"n": 869,
21+
"acc_baseline": 0.713463751438435,
22+
"acc_candidate": 0.7077100115074798,
23+
"delta_pp": -0.5753739930955182,
24+
"discordant_base_only": 84,
25+
"discordant_cand_only": 79,
26+
"mcnemar_p": 0.7541603898054375,
27+
"mde_pp_80pct": 4.297042579187413,
28+
"verdict": "UNRESOLVABLE"
29+
},
30+
"jev_hard": {
31+
"label": "jev_hard",
32+
"n": 111,
33+
"acc_baseline": 0.3783783783783784,
34+
"acc_candidate": 0.3963963963963964,
35+
"delta_pp": 1.8018018018018,
36+
"discordant_base_only": 11,
37+
"discordant_cand_only": 13,
38+
"mcnemar_p": 0.8388197422027588,
39+
"mde_pp_80pct": 12.896400918980339,
40+
"verdict": "UNRESOLVABLE"
41+
},
42+
"jev_easy": {
43+
"label": "jev_easy",
44+
"n": 48,
45+
"acc_baseline": 1.0,
46+
"acc_candidate": 1.0,
47+
"delta_pp": 0.0,
48+
"discordant_base_only": 0,
49+
"discordant_cand_only": 0,
50+
"mcnemar_p": 1.0,
51+
"mde_pp_80pct": 20.21869817384522,
52+
"verdict": "UNRESOLVABLE"
53+
},
54+
"probes": {
55+
"label": "probes",
56+
"n": 52,
57+
"acc_baseline": 0.6538461538461539,
58+
"acc_candidate": 0.7307692307692307,
59+
"delta_pp": 7.692307692307687,
60+
"discordant_base_only": 4,
61+
"discordant_cand_only": 8,
62+
"mcnemar_p": 0.3876953125,
63+
"mde_pp_80pct": 18.4830866323031,
64+
"verdict": "UNRESOLVABLE"
65+
},
66+
"heldout_jabr+probes": {
67+
"label": "heldout_jabr+probes",
68+
"n": 921,
69+
"acc_baseline": 0.7100977198697068,
70+
"acc_candidate": 0.7090119435396308,
71+
"delta_pp": -0.10857763300760048,
72+
"discordant_base_only": 88,
73+
"discordant_cand_only": 87,
74+
"mcnemar_p": 1.0,
75+
"mde_pp_80pct": 4.188503310482239,
76+
"verdict": "UNRESOLVABLE"
77+
},
78+
"pooled_standard+jabr": {
79+
"label": "pooled_standard+jabr",
80+
"n": 941,
81+
"acc_baseline": 0.7013815090329437,
82+
"acc_candidate": 0.6971307120085016,
83+
"delta_pp": -0.4250797024442088,
84+
"discordant_base_only": 94,
85+
"discordant_cand_only": 90,
86+
"mcnemar_p": 0.8250403507533893,
87+
"mde_pp_80pct": 4.1796940799303774,
88+
"verdict": "UNRESOLVABLE"
89+
},
90+
"pooled_all": {
91+
"label": "pooled_all",
92+
"n": 1152,
93+
"acc_baseline": 0.6805555555555556,
94+
"acc_candidate": 0.6822916666666666,
95+
"delta_pp": 0.17361111111110494,
96+
"discordant_base_only": 109,
97+
"discordant_cand_only": 111,
98+
"mcnemar_p": 0.9462677277802586,
99+
"mde_pp_80pct": 3.8486376626278362,
100+
"verdict": "UNRESOLVABLE"
101+
},
102+
"jev_standard/adequacy": {
103+
"label": "jev_standard/adequacy",
104+
"n": 12,
105+
"acc_baseline": 0.16666666666666666,
106+
"acc_candidate": 0.5,
107+
"delta_pp": 33.333333333333336,
108+
"discordant_base_only": 1,
109+
"discordant_cand_only": 5,
110+
"mcnemar_p": 0.21875,
111+
"mde_pp_80pct": 30.14025568884585,
112+
"verdict": "UNRESOLVABLE"
113+
},
114+
"jev_standard/extraction": {
115+
"label": "jev_standard/extraction",
116+
"n": 12,
117+
"acc_baseline": 0.5833333333333334,
118+
"acc_candidate": 0.5,
119+
"delta_pp": -8.333333333333337,
120+
"discordant_base_only": 1,
121+
"discordant_cand_only": 0,
122+
"mcnemar_p": 1.0,
123+
"mde_pp_80pct": 39.87181050229295,
124+
"verdict": "UNRESOLVABLE"
125+
},
126+
"jev_standard/intent": {
127+
"label": "jev_standard/intent",
128+
"n": 12,
129+
"acc_baseline": 0.9166666666666666,
130+
"acc_candidate": 0.5833333333333334,
131+
"delta_pp": -33.33333333333333,
132+
"discordant_base_only": 4,
133+
"discordant_cand_only": 0,
134+
"mcnemar_p": 0.125,
135+
"mde_pp_80pct": 22.35261186402951,
136+
"verdict": "UNRESOLVABLE"
137+
},
138+
"jev_standard/ordinal": {
139+
"label": "jev_standard/ordinal",
140+
"n": 12,
141+
"acc_baseline": 0.25,
142+
"acc_candidate": 0.75,
143+
"delta_pp": 50.0,
144+
"discordant_base_only": 0,
145+
"discordant_cand_only": 6,
146+
"mcnemar_p": 0.03125,
147+
"mde_pp_80pct": 35.0198125,
148+
"verdict": "PASS"
149+
},
150+
"jev_standard/policy": {
151+
"label": "jev_standard/policy",
152+
"n": 12,
153+
"acc_baseline": 0.75,
154+
"acc_candidate": 0.6666666666666666,
155+
"delta_pp": -8.333333333333337,
156+
"discordant_base_only": 1,
157+
"discordant_cand_only": 0,
158+
"mcnemar_p": 1.0,
159+
"mde_pp_80pct": 35.0198125,
160+
"verdict": "UNRESOLVABLE"
161+
},
162+
"jev_standard/routing": {
163+
"label": "jev_standard/routing",
164+
"n": 12,
165+
"acc_baseline": 0.6666666666666666,
166+
"acc_candidate": 0.4166666666666667,
167+
"delta_pp": -24.999999999999993,
168+
"discordant_base_only": 3,
169+
"discordant_cand_only": 0,
170+
"mcnemar_p": 0.25,
171+
"mde_pp_80pct": 38.12474289464006,
172+
"verdict": "UNRESOLVABLE"
173+
},
174+
"probes/evidence_as_choice": {
175+
"label": "probes/evidence_as_choice",
176+
"n": 10,
177+
"acc_baseline": 0.8,
178+
"acc_candidate": 0.8,
179+
"delta_pp": 0.0,
180+
"discordant_base_only": 2,
181+
"discordant_cand_only": 2,
182+
"mcnemar_p": 1.0,
183+
"mde_pp_80pct": 35.43755863425132,
184+
"verdict": "UNRESOLVABLE"
185+
},
186+
"probes/evidence_noul": {
187+
"label": "probes/evidence_noul",
188+
"n": 14,
189+
"acc_baseline": 0.5714285714285714,
190+
"acc_candidate": 0.7142857142857143,
191+
"delta_pp": 14.28571428571429,
192+
"discordant_base_only": 1,
193+
"discordant_cand_only": 3,
194+
"mcnemar_p": 0.625,
195+
"mde_pp_80pct": 37.0537672031612,
196+
"verdict": "UNRESOLVABLE"
197+
},
198+
"probes/failure_triage": {
199+
"label": "probes/failure_triage",
200+
"n": 9,
201+
"acc_baseline": 0.5555555555555556,
202+
"acc_candidate": 0.7777777777777778,
203+
"delta_pp": 22.22222222222222,
204+
"discordant_base_only": 0,
205+
"discordant_cand_only": 2,
206+
"mcnemar_p": 0.5,
207+
"mde_pp_80pct": 46.403959294398135,
208+
"verdict": "UNRESOLVABLE"
209+
},
210+
"probes/note_relation_typing": {
211+
"label": "probes/note_relation_typing",
212+
"n": 7,
213+
"acc_baseline": 0.5714285714285714,
214+
"acc_candidate": 0.5714285714285714,
215+
"delta_pp": 0.0,
216+
"discordant_base_only": 0,
217+
"discordant_cand_only": 0,
218+
"mcnemar_p": 1.0,
219+
"mde_pp_80pct": 52.40194011572594,
220+
"verdict": "UNRESOLVABLE"
221+
},
222+
"probes/turn_tagging": {
223+
"label": "probes/turn_tagging",
224+
"n": 12,
225+
"acc_baseline": 0.75,
226+
"acc_candidate": 0.75,
227+
"delta_pp": 0.0,
228+
"discordant_base_only": 1,
229+
"discordant_cand_only": 1,
230+
"mcnemar_p": 1.0,
231+
"mde_pp_80pct": 35.0198125,
232+
"verdict": "UNRESOLVABLE"
233+
}
234+
}
235+
}

0 commit comments

Comments
 (0)