Skip to content

Commit 8775cd7

Browse files
committed
Judge typed comments as typed, and only comments
Three gaps in letting a comment be the judge's quote. Any line beginning with a comment marker of any language counted, so a C line beginning with a dereference could ground a step; comments are now read as the editor's language writes them, a C-family line beginning with an asterisk only inside a block. A row grounded in a comment was labelled candidate speech; it is now an editor snapshot. And after Coding was recorded an edit was judged only for Coding, so a comment typed for a step still open waited for the candidate to speak again; a changed comment now makes an editor-only judgment due.
1 parent 848d41c commit 8775cd7

2 files changed

Lines changed: 112 additions & 38 deletions

File tree

‎src/agent.rs‎

Lines changed: 51 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -1495,7 +1495,13 @@ pub fn phase_judge_need(state: &RuntimeState) -> PhaseJudgeDue {
14951495
let open = phase_judge_open(state);
14961496
if spoke && !open.is_empty() {
14971497
PhaseJudgeDue::Spoken
1498-
} else if typed && open.contains(&"coding") {
1498+
} else if typed
1499+
&& (open.contains(&"coding")
1500+
|| (!state.behavioral_round_started
1501+
&& !open.is_empty()
1502+
&& typed_comments(&state.language, &state.code)
1503+
!= typed_comments(&state.judged_language, &state.judged_code)))
1504+
{
14991505
PhaseJudgeDue::EditorOnly
15001506
} else {
15011507
PhaseJudgeDue::No
@@ -1570,19 +1576,34 @@ fn quote_words(text: &str) -> String {
15701576
.join(" ")
15711577
}
15721578

1573-
/// The comments in `code`: what follows a line's leading comment marker. Read
1574-
/// apart from the code, so a step is quoted from an explanation, not from the
1575-
/// code that is Coding's.
1576-
fn typed_comments(code: &str) -> String {
1577-
code.lines()
1578-
.filter_map(|line| {
1579-
let line = line.trim_start();
1580-
["//", "#", "/*", "*", "--"]
1581-
.into_iter()
1582-
.find_map(|marker| line.strip_prefix(marker))
1583-
})
1584-
.collect::<Vec<_>>()
1585-
.join("\n")
1579+
/// The comments in `code`, as `language` writes them: whole-line `#` comments
1580+
/// in Python, and `//` lines and `/* */` blocks in the others. Read apart from
1581+
/// the code, so a step is quoted from an explanation, not from the code that
1582+
/// is Coding's, and only from lines that cannot be code: a C line beginning
1583+
/// with `*` is a dereference outside a block. A block's leading `*` needs no
1584+
/// stripping, since a quote is matched on its words.
1585+
fn typed_comments(language: &str, code: &str) -> String {
1586+
let mut comments = Vec::new();
1587+
let mut in_block = false;
1588+
for line in code.lines().map(str::trim_start) {
1589+
let text = if language == "python" {
1590+
line.strip_prefix('#')
1591+
} else if in_block {
1592+
Some(line)
1593+
} else if let Some(text) = line.strip_prefix("/*") {
1594+
in_block = true;
1595+
Some(text)
1596+
} else {
1597+
line.strip_prefix("//")
1598+
};
1599+
let Some(mut text) = text else { continue };
1600+
if in_block && let Some((head, _)) = text.split_once("*/") {
1601+
in_block = false;
1602+
text = head;
1603+
}
1604+
comments.push(text);
1605+
}
1606+
comments.join("\n")
15861607
}
15871608

15881609
/// One speaker's turn, as `speaker_turns` reads the transcript.
@@ -1634,8 +1655,9 @@ fn speaker_turns(state: &RuntimeState) -> Vec<Turn> {
16341655
turns
16351656
}
16361657

1637-
/// Whether `quote` is the candidate's own: words from one recognized candidate
1638-
/// turn, or for Coding, from what they typed into the editor.
1658+
/// Where `quote` is the candidate's own, if it is: words from one recognized
1659+
/// candidate turn, or what they typed into the editor, its code for Coding and
1660+
/// a comment for another coding-round step.
16391661
///
16401662
/// Proof of who said it, not that the step was done; that judgment stays with
16411663
/// the model and its prompt. What can be checked mechanically is checked: the
@@ -1648,7 +1670,7 @@ fn quote_is_grounded(
16481670
turns: &[Turn],
16491671
phase: FrameworkPhase,
16501672
quote: &str,
1651-
) -> bool {
1673+
) -> Option<EvidenceSource> {
16521674
let words = quote_words(quote);
16531675

16541676
// A spoken trace is a sentence like any other answer, though Test is about
@@ -1662,32 +1684,30 @@ fn quote_is_grounded(
16621684
MIN_SPOKEN_QUOTE_WORDS
16631685
};
16641686
if words.split(' ').count() < least {
1665-
return false;
1687+
return None;
16661688
}
16671689
let needle = format!(" {words} ");
1690+
let typed = |text: &str| format!(" {} ", quote_words(text)).contains(&needle);
1691+
let editor = Some(EvidenceSource::EditorSnapshot);
16681692
if phase == FrameworkPhase::Coding {
1669-
let typed = |code: &str| format!(" {} ", quote_words(code)).contains(&needle);
1670-
return typed(&state.code) && !typed(starter(state, &state.language));
1693+
return editor.filter(|_| typed(&state.code) && !typed(starter(state, &state.language)));
16711694
}
16721695

16731696
// A candidate whose speech was not recognized is asked to type their
16741697
// explanation as a comment, and at an editor without the evidence tool
16751698
// nobody else can record it from there.
1676-
let commented =
1677-
|code: &str| format!(" {} ", quote_words(&typed_comments(code))).contains(&needle);
1699+
let commented = |code: &str| typed(&typed_comments(&state.language, code));
16781700
if !phase.is_star()
16791701
&& !state.interview_mode.is_whiteboard()
16801702
&& commented(&state.code)
16811703
&& !commented(starter(state, &state.language))
16821704
{
1683-
return true;
1705+
return editor;
16841706
}
16851707
let mut said = turns.iter().filter(|turn| turn.words.contains(&needle));
1686-
let Some(first) = said.clone().next() else {
1687-
return false;
1688-
};
1708+
let first = said.clone().next()?;
16891709
if !said.any(|turn| turn.speaker == CANDIDATE_SPEAKER) {
1690-
return false;
1710+
return None;
16911711
}
16921712

16931713
// A trace the interviewer walked first, or a story they told back before
@@ -1698,7 +1718,7 @@ fn quote_is_grounded(
16981718
FrameworkPhase::Algorithm | FrameworkPhase::Optimizations | FrameworkPhase::Test
16991719
))
17001720
&& first.speaker == INTERVIEWER_SPEAKER;
1701-
!echo
1721+
(!echo).then_some(EvidenceSource::CandidateSpeech)
17021722
}
17031723

17041724
/// Records what the judge found, through the same gate the interviewer's calls
@@ -1778,9 +1798,9 @@ pub fn apply_phase_judgment(state: &mut RuntimeState, text: &str) -> Vec<Framewo
17781798
continue;
17791799
}
17801800
let turns = turns.get_or_insert_with(|| speaker_turns(state));
1781-
if !quote_is_grounded(state, turns, phase, quote) {
1801+
let Some(source) = quote_is_grounded(state, turns, phase, quote) else {
17821802
continue;
1783-
}
1803+
};
17841804
let summary = step
17851805
.get("summary")
17861806
.and_then(serde_json::Value::as_str)
@@ -1797,16 +1817,11 @@ pub fn apply_phase_judgment(state: &mut RuntimeState, text: &str) -> Vec<Framewo
17971817
),
17981818
None => format!("Quote: {quoted}"),
17991819
};
1800-
let source = if phase == FrameworkPhase::Coding {
1801-
"editor_snapshot"
1802-
} else {
1803-
"candidate_speech"
1804-
};
18051820
if let Ok((evidence, RecordOutcome::Added)) = record_evidence(
18061821
state,
18071822
&serde_json::json!({
18081823
"phase": id,
1809-
"source": source,
1824+
"source": evidence_source_id(source),
18101825
"kind": "observed",
18111826
"confidence": 70,
18121827
"summary": summary,

‎tests/agent/framework.rs‎

Lines changed: 61 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4086,7 +4086,13 @@ fn a_step_explained_in_a_typed_comment_is_the_candidates_own() {
40864086
let mut typed = with_written_code(RuntimeState::default());
40874087
typed.code = format!("# {quote}\n{}", typed.code);
40884088
let algorithm = |quote: &str| json!([{"step": "algorithm", "quote": quote, "summary": "s"}]);
4089-
assert_eq!(judge(&mut typed.clone(), algorithm(quote)).len(), 1);
4089+
let recorded = judge(&mut typed.clone(), algorithm(quote));
4090+
assert_eq!(recorded.len(), 1);
4091+
assert_eq!(
4092+
recorded[0].source,
4093+
EvidenceSource::EditorSnapshot,
4094+
"typed, not said"
4095+
);
40904096
assert!(
40914097
judge(
40924098
&mut typed.clone(),
@@ -4106,12 +4112,65 @@ fn a_step_explained_in_a_typed_comment_is_the_candidates_own() {
41064112
"the starter's"
41074113
);
41084114

4109-
let mut board = typed;
4115+
let mut board = typed.clone();
41104116
board.interview_mode = InterviewMode::Whiteboard;
41114117
assert!(
41124118
judge(&mut board, algorithm(quote)).is_empty(),
41134119
"a whiteboard"
41144120
);
4121+
4122+
// In C a line beginning with `*` is a comment only inside a block.
4123+
let mut c = with_written_code(RuntimeState {
4124+
language: "c".into(),
4125+
..RuntimeState::default()
4126+
});
4127+
let body = "int f(int *rest) {\n *rest = keep_every_other_interval_as_it_was;\n}\n";
4128+
c.code = body.to_string();
4129+
let deref = "rest keep every other interval as it was";
4130+
assert!(
4131+
judge(&mut c.clone(), algorithm(deref)).is_empty(),
4132+
"a dereference"
4133+
);
4134+
c.code = format!("/*\n * {quote}\n */\n{body}");
4135+
assert!(
4136+
judge(&mut c.clone(), algorithm(deref)).is_empty(),
4137+
"code after the block"
4138+
);
4139+
assert_eq!(judge(&mut c, algorithm(quote)).len(), 1, "a block comment");
4140+
4141+
// A comment typed after Coding is recorded is judged without waiting for
4142+
// the candidate to speak again; other edits are not.
4143+
record_framework_evidence(
4144+
&mut typed,
4145+
&json!({"phase": "coding", "source": "editor_snapshot", "kind": "observed",
4146+
"confidence": 90, "summary": "Wrote it."}),
4147+
)
4148+
.unwrap();
4149+
take_phase_judge_window(&mut typed, get_problem(Some("insert-interval")));
4150+
typed.code.push_str("x = 1\n");
4151+
assert_eq!(phase_judge_need(&typed), PhaseJudgeDue::No);
4152+
typed
4153+
.code
4154+
.push_str("# then I insert the new interval in order\n");
4155+
assert_eq!(phase_judge_need(&typed), PhaseJudgeDue::EditorOnly);
4156+
let mut behavioral = typed.clone();
4157+
behavioral.behavioral_round_started = true;
4158+
assert_eq!(
4159+
phase_judge_need(&behavioral),
4160+
PhaseJudgeDue::No,
4161+
"STAR is not typed"
4162+
);
4163+
record_framework_evidence(
4164+
&mut typed,
4165+
&json!({"phase": "optimizations", "source": "candidate_speech", "kind": "observed",
4166+
"confidence": 90, "summary": "O(n)."}),
4167+
)
4168+
.unwrap();
4169+
assert_eq!(
4170+
phase_judge_need(&typed),
4171+
PhaseJudgeDue::No,
4172+
"nothing left open"
4173+
);
41154174
}
41164175

41174176
/// A whiteboard interviewer holds the evidence tool and records a hand trace

0 commit comments

Comments
 (0)