Skip to content

Commit e1d1a31

Browse files
committed
Stop scoring hints as a numeric deduction
Bundle 10 retires the instruction that every hint should meaningfully reduce the communication score. A hint is evidence about independence, but how much to subtract was left to the model, so two interviews with the same hint count were marked differently for it. Hints stay as context the reviewer weighs against everything else. The register in docs is now read back by the bundle test, because it is what a maintainer consults before a bump and it had nothing holding it to the constants.
1 parent e2c45b1 commit e1d1a31

8 files changed

Lines changed: 42 additions & 20 deletions

File tree

‎docs/interview-contract-versions.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,11 @@ can select it.
88

99
## The active bundle
1010

11-
Bundle 9: live prompt 3, report prompt 8, rubric 1, report schema 2.
11+
Bundle 10: live prompt 3, report prompt 9, rubric 1, report schema 2.
1212

1313
| Bundle | Introduced |
1414
|---|---|
15+
| 10 | Hints remain qualitative context for candidate independence rather than automatic numeric communication-score deductions. |
1516
| 9 | Reports keep the fixed mid-level hiring bar and state the optional level the candidate practiced for beside it. |
1617
| 8 | Candidate-authored test cases reach the live interviewer and report brief, while judge pass totals remain separate. |
1718
| 7 | The post-interview server stamp adds optional debrief, topics, and practice level fields. `tests/golden/report-schema.json` remains the model output shape only; server-stamped fields are versioned at the browser sanitizer. |

‎src/agent.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -115,9 +115,9 @@ const ROUND_TRANSITION_SKEW: std::time::Duration = std::time::Duration::from_sec
115115
/// `the_time_warning_threshold_is_the_same_number_on_both_sides`.
116116
pub const TIME_WARNING_S: u64 = 300;
117117

118-
pub const INTERVIEW_CONTRACT_BUNDLE_VERSION: u32 = 9;
118+
pub const INTERVIEW_CONTRACT_BUNDLE_VERSION: u32 = 10;
119119
pub const LIVE_PROMPT_VERSION: u32 = 3;
120-
pub const REPORT_PROMPT_VERSION: u32 = 8;
120+
pub const REPORT_PROMPT_VERSION: u32 = 9;
121121
pub const RUBRIC_VERSION: u32 = 1;
122122
pub const REPORT_SCHEMA_VERSION: u32 = 2;
123123

‎src/agent/prompts.rs‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -840,8 +840,7 @@ Score two independent dimensions from 0 to 100:
840840
optimization, and accurately answered follow-ups. Also consider completeness
841841
of Situation, Task, personal Action, and Result only if the interviewer actually
842842
asked a behavioral question. If none was asked, say behavioral communication
843-
was not assessed and do not deduct for it. Consider independence too: each hint
844-
should meaningfully reduce this score; {} hint(s) were given."#,
843+
was not assessed and do not deduct for it."#,
845844
input.duration_min,
846845
input.elapsed_min,
847846
input.problem.title,
@@ -857,8 +856,7 @@ Score two independent dimensions from 0 to 100:
857856
input.hints_used,
858857
input.hint_rung,
859858
volunteered_hints,
860-
test_summary,
861-
input.hints_used
859+
test_summary
862860
)
863861
}
864862

‎tests/agent.rs‎

Lines changed: 27 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -740,6 +740,10 @@ fn prompt_golden_digest_matches_versions() {
740740
(3, 8),
741741
"1be28b91ea6148c278f262c01d61670563ff4752b37ad517675284e9304a5825",
742742
),
743+
(
744+
(3, 9),
745+
"4f9e1aac81d6479abbe454d3eeb8dad27f414c2b311bc6cf89165dad9566ce1f",
746+
),
743747
]
744748
.into_iter()
745749
.find_map(|(candidate, digest)| (candidate == versions).then_some(digest));
@@ -5323,17 +5327,36 @@ fn generated_problem_metadata_exposes_no_private_rubric() {
53235327

53245328
#[test]
53255329
fn interview_contract_versions_are_one_closed_bundle() {
5326-
assert_eq!(INTERVIEW_CONTRACT_BUNDLE_VERSION, 9);
5330+
// The doc is the register a maintainer reads before bumping anything, so a
5331+
// bump that leaves it behind is worse than no register at all. Read the
5332+
// active-bundle line back rather than trusting the two to be edited
5333+
// together, and require the table to carry a row for that bundle.
5334+
let doc = std::fs::read_to_string("docs/interview-contract-versions.md")
5335+
.expect("the contract register is readable");
5336+
assert!(
5337+
doc.contains(&format!(
5338+
"Bundle {INTERVIEW_CONTRACT_BUNDLE_VERSION}: live prompt {LIVE_PROMPT_VERSION}, \
5339+
report prompt {REPORT_PROMPT_VERSION}, rubric {RUBRIC_VERSION}, report schema \
5340+
{REPORT_SCHEMA_VERSION}."
5341+
)),
5342+
"docs/interview-contract-versions.md does not name the active bundle"
5343+
);
5344+
assert!(
5345+
doc.contains(&format!("\n| {INTERVIEW_CONTRACT_BUNDLE_VERSION} | ")),
5346+
"the bundle table has no row for {INTERVIEW_CONTRACT_BUNDLE_VERSION}"
5347+
);
5348+
5349+
assert_eq!(INTERVIEW_CONTRACT_BUNDLE_VERSION, 10);
53275350
assert_eq!(LIVE_PROMPT_VERSION, 3);
5328-
assert_eq!(REPORT_PROMPT_VERSION, 8);
5351+
assert_eq!(REPORT_PROMPT_VERSION, 9);
53295352
assert_eq!(RUBRIC_VERSION, 1);
53305353
assert_eq!(REPORT_SCHEMA_VERSION, 2);
53315354
assert_eq!(
53325355
interview_contract_json(),
53335356
json!({
5334-
"bundleVersion": 9,
5357+
"bundleVersion": 10,
53355358
"livePromptVersion": 3,
5336-
"reportPromptVersion": 8,
5359+
"reportPromptVersion": 9,
53375360
"rubricVersion": 1,
53385361
"reportSchemaVersion": 2,
53395362
})

‎tests/browser/replay-render.test.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -523,7 +523,7 @@ test("the report card this page renders names no finding either", () => {
523523
said,
524524
new Set([
525525
"(.md)", "(Sxlang)", "(editor", "(none", "-", "/", "0", "01:05", "1", "10",
526-
"100", "2", "2.", "2;", "3", "37", "7", "70", "8", "9", "95%",
526+
"100", "2", "2.", "2;", "3", "37", "7", "70", "8", "95%",
527527
"Approach", "Chain", "CodeTrial.", "Coding", "Committee", "Common",
528528
"Communication", "Contract", "Done", "Download", "Evidence",
529529
"FACE_MISSING", "Follow-ups", "Framework", "Given:", "HIRE", "Held",

‎tests/fixtures/framework-evaluation-cases.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@
5151
{
5252
"id": "explicit-hint-use",
5353
"coverage": ["explicit-hint-use"],
54-
"reaction": {"kind":"report", "code":"def two_sum(nums, target): return []", "required":["each hint", "2 hint(s) were given"], "forbidden":["0 hint(s)"]},
54+
"reaction": {"kind":"report", "code":"def two_sum(nums, target): return []", "required":["2 total", "never as a numeric deduction"], "forbidden":["meaningfully reduce this score", "0 total"]},
5555
"transcript":"Interviewer suggested tracking prior complements; candidate then described a map.",
5656
"evidence":[{"phase":"algorithm","source":"candidate_speech","kind":"inferred","confidence":70,"summary":"Described the map after an explicit interviewer hint."}],
5757
"hintsUsed":2, "round":"none", "assessed":["Algorithm"]

‎tests/golden/prompts.json‎

Lines changed: 5 additions & 5 deletions
Large diffs are not rendered by default.

‎web/lib.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -421,9 +421,9 @@ const textEncoder = new TextEncoder();
421421
/// function-local, moving it left the whole suite green with the supported-card
422422
/// branch no longer rendering, which is the defect a local constant invites.
423423
export const ACTIVE_CONTRACT = {
424-
bundleVersion: 9,
424+
bundleVersion: 10,
425425
livePromptVersion: 3,
426-
reportPromptVersion: 8,
426+
reportPromptVersion: 9,
427427
reportSchemaVersion: 2,
428428
rubricVersion: 1,
429429
};

0 commit comments

Comments
 (0)