Skip to content
This repository was archived by the owner on Oct 17, 2026. It is now read-only.

Commit 0acaa06

Browse files
wfzyxwfzyx
committed
gate: judge_heldout — von-1.2 at chance (51.7%); judge-mix 57.7% (+6.0pp p=0.105), gliclass-init 57.7% (+6.0pp p=0.076), both under MDE 8.1pp
1 parent 1fa1655 commit 0acaa06

2 files changed

Lines changed: 182 additions & 0 deletions

File tree

‎results/v1.4/gate_judge_heldout_gliclass.json‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
{
2+
"base": "checkpoints/von-1.2",
3+
"cand": "/home/wfzyx/scratch/ckpt/von-1.4-gliclass-init",
4+
"cand_chains": "",
5+
"gates": {
6+
"judge_heldout": {
7+
"label": "judge_heldout",
8+
"n": 300,
9+
"acc_baseline": 0.5166666666666667,
10+
"acc_candidate": 0.5766666666666667,
11+
"delta_pp": 5.999999999999995,
12+
"discordant_base_only": 37,
13+
"discordant_cand_only": 55,
14+
"mcnemar_p": 0.07576023576406506,
15+
"mde_pp_80pct": 8.082984976737816,
16+
"verdict": "UNRESOLVABLE"
17+
},
18+
"pooled_all": {
19+
"label": "pooled_all",
20+
"n": 300,
21+
"acc_baseline": 0.5166666666666667,
22+
"acc_candidate": 0.5766666666666667,
23+
"delta_pp": 5.999999999999995,
24+
"discordant_base_only": 37,
25+
"discordant_cand_only": 55,
26+
"mcnemar_p": 0.07576023576406506,
27+
"mde_pp_80pct": 8.082984976737816,
28+
"verdict": "UNRESOLVABLE"
29+
},
30+
"judge_heldout/anthropic_hhh": {
31+
"label": "judge_heldout/anthropic_hhh",
32+
"n": 30,
33+
"acc_baseline": 0.7,
34+
"acc_candidate": 0.7333333333333333,
35+
"delta_pp": 3.3333333333333326,
36+
"discordant_base_only": 4,
37+
"discordant_cand_only": 5,
38+
"mcnemar_p": 1.0,
39+
"mde_pp_80pct": 23.439741804374684,
40+
"verdict": "UNRESOLVABLE"
41+
},
42+
"judge_heldout/helpsteer3_val": {
43+
"label": "judge_heldout/helpsteer3_val",
44+
"n": 120,
45+
"acc_baseline": 0.4666666666666667,
46+
"acc_candidate": 0.5583333333333333,
47+
"delta_pp": 9.166666666666668,
48+
"discordant_base_only": 15,
49+
"discordant_cand_only": 26,
50+
"mcnemar_p": 0.11727520595741225,
51+
"mde_pp_80pct": 12.758979360733743,
52+
"verdict": "UNRESOLVABLE"
53+
},
54+
"judge_heldout/pku_better": {
55+
"label": "judge_heldout/pku_better",
56+
"n": 30,
57+
"acc_baseline": 0.5,
58+
"acc_candidate": 0.36666666666666664,
59+
"delta_pp": -13.333333333333336,
60+
"discordant_base_only": 7,
61+
"discordant_cand_only": 3,
62+
"mcnemar_p": 0.34375,
63+
"mde_pp_80pct": 25.57485502113518,
64+
"verdict": "UNRESOLVABLE"
65+
},
66+
"judge_heldout/shp": {
67+
"label": "judge_heldout/shp",
68+
"n": 60,
69+
"acc_baseline": 0.4666666666666667,
70+
"acc_candidate": 0.55,
71+
"delta_pp": 8.333333333333337,
72+
"discordant_base_only": 4,
73+
"discordant_cand_only": 9,
74+
"mcnemar_p": 0.266845703125,
75+
"mde_pp_80pct": 18.04392165398806,
76+
"verdict": "UNRESOLVABLE"
77+
},
78+
"judge_heldout/summarize": {
79+
"label": "judge_heldout/summarize",
80+
"n": 60,
81+
"acc_baseline": 0.5833333333333334,
82+
"acc_candidate": 0.6666666666666666,
83+
"delta_pp": 8.333333333333325,
84+
"discordant_base_only": 7,
85+
"discordant_cand_only": 12,
86+
"mcnemar_p": 0.359283447265625,
87+
"mde_pp_80pct": 17.831215733823413,
88+
"verdict": "UNRESOLVABLE"
89+
}
90+
}
91+
}

‎results/v1.4/gate_judge_heldout_judge_mix.json‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
{
2+
"base": "checkpoints/von-1.2",
3+
"cand": "/home/wfzyx/scratch/ckpt/von-1.4-judge-mix",
4+
"cand_chains": "",
5+
"gates": {
6+
"judge_heldout": {
7+
"label": "judge_heldout",
8+
"n": 300,
9+
"acc_baseline": 0.5166666666666667,
10+
"acc_candidate": 0.5766666666666667,
11+
"delta_pp": 5.999999999999995,
12+
"discordant_base_only": 46,
13+
"discordant_cand_only": 64,
14+
"mcnemar_p": 0.10461305811126677,
15+
"mde_pp_80pct": 8.082984976737816,
16+
"verdict": "UNRESOLVABLE"
17+
},
18+
"pooled_all": {
19+
"label": "pooled_all",
20+
"n": 300,
21+
"acc_baseline": 0.5166666666666667,
22+
"acc_candidate": 0.5766666666666667,
23+
"delta_pp": 5.999999999999995,
24+
"discordant_base_only": 46,
25+
"discordant_cand_only": 64,
26+
"mcnemar_p": 0.10461305811126677,
27+
"mde_pp_80pct": 8.082984976737816,
28+
"verdict": "UNRESOLVABLE"
29+
},
30+
"judge_heldout/anthropic_hhh": {
31+
"label": "judge_heldout/anthropic_hhh",
32+
"n": 30,
33+
"acc_baseline": 0.7,
34+
"acc_candidate": 0.6,
35+
"delta_pp": -9.999999999999998,
36+
"discordant_base_only": 6,
37+
"discordant_cand_only": 3,
38+
"mcnemar_p": 0.5078125,
39+
"mde_pp_80pct": 23.439741804374684,
40+
"verdict": "UNRESOLVABLE"
41+
},
42+
"judge_heldout/helpsteer3_val": {
43+
"label": "judge_heldout/helpsteer3_val",
44+
"n": 120,
45+
"acc_baseline": 0.4666666666666667,
46+
"acc_candidate": 0.5666666666666667,
47+
"delta_pp": 9.999999999999998,
48+
"discordant_base_only": 19,
49+
"discordant_cand_only": 31,
50+
"mcnemar_p": 0.11892045255943628,
51+
"mde_pp_80pct": 12.758979360733743,
52+
"verdict": "UNRESOLVABLE"
53+
},
54+
"judge_heldout/pku_better": {
55+
"label": "judge_heldout/pku_better",
56+
"n": 30,
57+
"acc_baseline": 0.5,
58+
"acc_candidate": 0.5,
59+
"delta_pp": 0.0,
60+
"discordant_base_only": 4,
61+
"discordant_cand_only": 4,
62+
"mcnemar_p": 1.0,
63+
"mde_pp_80pct": 25.57485502113518,
64+
"verdict": "UNRESOLVABLE"
65+
},
66+
"judge_heldout/shp": {
67+
"label": "judge_heldout/shp",
68+
"n": 60,
69+
"acc_baseline": 0.4666666666666667,
70+
"acc_candidate": 0.6166666666666667,
71+
"delta_pp": 15.000000000000002,
72+
"discordant_base_only": 8,
73+
"discordant_cand_only": 17,
74+
"mcnemar_p": 0.10775214433670044,
75+
"mde_pp_80pct": 18.04392165398806,
76+
"verdict": "UNRESOLVABLE"
77+
},
78+
"judge_heldout/summarize": {
79+
"label": "judge_heldout/summarize",
80+
"n": 60,
81+
"acc_baseline": 0.5833333333333334,
82+
"acc_candidate": 0.5833333333333334,
83+
"delta_pp": 0.0,
84+
"discordant_base_only": 9,
85+
"discordant_cand_only": 9,
86+
"mcnemar_p": 1.0,
87+
"mde_pp_80pct": 17.831215733823413,
88+
"verdict": "UNRESOLVABLE"
89+
}
90+
}
91+
}

0 commit comments

Comments
 (0)