Skip to content

Commit 9af474a

Browse files
fix(skill-eval): record every Codex agent message in a conversation turn (EveryInc#1822)
Co-authored-by: Claude Code <noreply@anthropic.com>
1 parent 752b0bc commit 9af474a

4 files changed

Lines changed: 55 additions & 19 deletions

File tree

‎tests/skill-eval-cell/README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ bun run test:skill-eval-cell -- \
1919

2020
`--read-only` enforces the fake boundary, it does not merely suggest it: Codex drops `--dangerously-bypass-approvals-and-sandbox` and runs `--sandbox read-only` (the two contradict each other), and Claude pairs `--allowedTools Read,Glob,Grep` with a `--disallowedTools` list that also names `Task,Skill,WebFetch,WebSearch,NotebookEdit` — under `--dangerously-skip-permissions` those stay callable, so allow-listing alone leaves the boundary open.
2121

22-
Prints a `summary.json` path. Each host gets its own workspace copy plus stdout/stderr, git status/log, and a file list. PATH shims live beside that workspace, never inside it, so the skill under test never sees harness files as its own dirty tree. Grade those; Grok narrates before the answer (grep `FILES_READ:`). Codex transcript is stderr, final message is stdout. `claude -p` is one-tick only.
22+
Prints a `summary.json` path. Each host gets its own workspace copy plus stdout/stderr, git status/log, and a file list. PATH shims live beside that workspace, never inside it, so the skill under test never sees harness files as its own dirty tree. Grade those; Grok narrates before the answer (grep `FILES_READ:`). Codex transcript is stderr, final message is stdout. A conversation cell (`--persona`) differs: Codex runs with `--json`, `stdout.txt` is the conversation with every agent message of each turn, and the event stream, tool calls included, lands in `stderr.txt`. `claude -p` is one-tick only.
2323

2424
Each invocation requires a new or empty `--out` directory and records input and
2525
evidence fingerprints. Packs freeze their scenario criteria and grader hashes.

‎tests/skill-eval-cell/converse.test.ts‎

Lines changed: 24 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import { describe, expect, test } from "bun:test"
2-
import { USER_DONE, hostTurnArgv, runUserSim, simReplyOrDone, toollessClaudeArgv, userSimPrompt } from "./converse"
2+
import { USER_DONE, codexAgentText, hostTurnArgv, runUserSim, simReplyOrDone, toollessClaudeArgv, userSimPrompt } from "./converse"
33

4-
const base = { sessionId: "11111111-1111-4111-8111-111111111111", message: "hi", cwd: "/w", lastMessageFile: "/h/last.txt" }
4+
const base = { sessionId: "11111111-1111-4111-8111-111111111111", message: "hi", cwd: "/w" }
55

66
describe("hostTurnArgv", () => {
77
test("claude pins the session on the first turn and resumes it after", () => {
@@ -19,7 +19,11 @@ describe("hostTurnArgv", () => {
1919
expect(first.at(-1)).toBe("hi")
2020
const next = hostTurnArgv("codex", { ...base, first: false })
2121
expect(next.slice(0, 4)).toEqual(["codex", "exec", "resume", "--last"])
22-
expect(next).toContain(base.lastMessageFile)
22+
// Every turn must emit events: the last-message outputs drop what Codex said before its closing line.
23+
for (const argv of [first, next]) {
24+
expect(argv).toContain("--json")
25+
expect(argv).not.toContain("-o")
26+
}
2327
})
2428

2529
test("hosts without a scriptable resume are rejected", () => {
@@ -28,6 +32,23 @@ describe("hostTurnArgv", () => {
2832
})
2933

3034
describe("conversation control", () => {
35+
test("a codex turn records every agent message in order and nothing else", () => {
36+
// Event shapes as emitted by codex-cli 0.160.0 `exec --json`.
37+
const events = [
38+
`{"type":"thread.started","thread_id":"t"}`,
39+
`{"type":"turn.started"}`,
40+
`{"type":"item.completed","item":{"id":"item_0","type":"reasoning","text":"thinking"}}`,
41+
`{"type":"item.completed","item":{"id":"item_1","type":"agent_message","text":"| role | model |\\n| plan | opus |"}}`,
42+
`{"type":"item.started","item":{"id":"item_2","type":"command_execution","command":"cat config.yaml","aggregated_output":"","exit_code":null,"status":"in_progress"}}`,
43+
`{"type":"item.completed","item":{"id":"item_2","type":"command_execution","command":"cat config.yaml","aggregated_output":"secret: 1\\n","exit_code":0,"status":"completed"}}`,
44+
`{"type":"item.completed","item":{"id":"item_3","type":"agent_message","text":"I'm waiting for your list of roles."}}`,
45+
`{"type":"turn.completed","usage":{"input_tokens":1,"output_tokens":1}}`,
46+
"",
47+
].join("\n")
48+
expect(codexAgentText(events)).toBe("| role | model |\n| plan | opus |\n\nI'm waiting for your list of roles.")
49+
expect(codexAgentText("warning: not an event\n")).toBe("")
50+
})
51+
3152
test("the simulated user can decline to reply", () => {
3253
expect(simReplyOrDone(` ${USER_DONE}\n`)).toBeNull()
3354
expect(simReplyOrDone("")).toBeNull()

‎tests/skill-eval-cell/converse.ts‎

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ export function formatTranscript(turns: Turn[], userLabel: string): string {
2424

2525
export function hostTurnArgv(
2626
host: Host,
27-
opts: { first: boolean; sessionId: string; message: string; cwd: string; lastMessageFile: string },
27+
opts: { first: boolean; sessionId: string; message: string; cwd: string },
2828
): string[] {
2929
if (host === "claude") {
3030
// AskUserQuestion cannot reach a person in print mode; removing it makes the
@@ -36,7 +36,9 @@ export function hostTurnArgv(
3636
]
3737
}
3838
if (host === "codex") {
39-
const common = ["--dangerously-bypass-approvals-and-sandbox", "--skip-git-repo-check", "-o", opts.lastMessageFile]
39+
// --json puts the turn's events on stdout so codexAgentText can keep every agent
40+
// message; -o and plain stdout carry only the last one.
41+
const common = ["--dangerously-bypass-approvals-and-sandbox", "--skip-git-repo-check", "--json"]
4042
// Resume without an id picks the newest session recorded for this cwd; every
4143
// host runs in its own workspace, so that is this conversation.
4244
return opts.first
@@ -46,6 +48,26 @@ export function hostTurnArgv(
4648
throw new Error(`--persona supports ${CONVERSE_HOSTS.join(", ")} only, not ${host}`)
4749
}
4850

51+
/**
52+
* What Codex said to the user in one `codex exec --json` turn: every agent message, in
53+
* order. Codex spreads a reply over several messages and tends to end on a one-line
54+
* prompt, so the last message alone drops most of it. Tool calls, command output, and
55+
* reasoning are other item types and stay out.
56+
*/
57+
export function codexAgentText(events: string): string {
58+
const messages: string[] = []
59+
for (const line of events.split("\n")) {
60+
let event
61+
try {
62+
event = JSON.parse(line)
63+
} catch {
64+
continue
65+
}
66+
if (event?.type === "item.completed" && event.item?.type === "agent_message") messages.push(event.item.text)
67+
}
68+
return messages.join("\n\n")
69+
}
70+
4971
export function userSimPrompt(persona: string, turns: Turn[]): string {
5072
const transcript = formatTranscript(turns, "YOU")
5173
return [

‎tests/skill-eval-cell/run.ts‎

Lines changed: 6 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ import fs from "node:fs"
1111
import path from "node:path"
1212
import { arg, flag } from "./cli"
1313
import { REPO_ROOT, WORKTREE_REF, extractSkill, mintCellDir } from "./extract"
14-
import { CONVERSE_HOSTS, formatTranscript, hostTurnArgv, runUserSim, type ConversationEnd, type Turn } from "./converse"
14+
import { CONVERSE_HOSTS, codexAgentText, formatTranscript, hostTurnArgv, runUserSim, type ConversationEnd, type Turn } from "./converse"
1515
import { HOSTS, planHost, resolveRunHosts, wrapPrompt, type Host, type HostPlan } from "./hosts"
1616
import { installPathShims, type PathShim } from "./path-shim"
1717
import { fingerprint, prepareOutput, sealEvidence, sha256, writeJSON } from "./provenance"
@@ -157,7 +157,6 @@ async function converse(
157157
opts: { prompt: string; persona: string; timeoutMs: number; maxTurns: number },
158158
): Promise<RunResult & { turns: number; ended: ConversationEnd; argvs: string[][] }> {
159159
const sessionId = crypto.randomUUID()
160-
const lastMessageFile = path.join(hostDir, "last-message.txt")
161160
const deadline = Date.now() + opts.timeoutMs
162161
const turns: Turn[] = []
163162
const log = path.join(hostDir, "transcript.jsonl")
@@ -171,20 +170,14 @@ async function converse(
171170
for (let i = 0; i < opts.maxTurns; i++) {
172171
const remaining = deadline - Date.now()
173172
if (remaining <= 0) { ended = "timeout"; last.timedOut = true; break }
174-
fs.rmSync(lastMessageFile, { force: true })
175-
const argv = hostTurnArgv(host, { first: i === 0, sessionId, message, cwd, lastMessageFile })
173+
const argv = hostTurnArgv(host, { first: i === 0, sessionId, message, cwd })
176174
argvs.push(argv)
177175
last = await runPlan({ ...plan, argv }, cwd, remaining)
178176
stderr += last.stderr
179-
let agentText = last.stdout
180-
if (host === "codex") {
181-
try {
182-
agentText = fs.readFileSync(lastMessageFile, "utf8")
183-
} catch {
184-
// codex wrote no final message; keep its stdout
185-
}
186-
}
187-
agentText = agentText.trim()
177+
// Codex's stdout is its event stream here; the tool calls it holds stay in the
178+
// cell's stderr.txt, out of the graded transcript.
179+
if (host === "codex") stderr += last.stdout
180+
const agentText = (host === "codex" ? codexAgentText(last.stdout) : last.stdout).trim()
188181
turns.push({ role: "agent", text: agentText })
189182
fs.appendFileSync(log, `${JSON.stringify({ role: "agent", text: agentText, exitCode: last.exitCode })}\n`)
190183
if (last.timedOut) { ended = "timeout"; break }

0 commit comments

Comments
 (0)