Skip to content

Commit cd330f4

Browse files
mchwangclaude
andauthored
D: settle deferred output on the container's exit, not the acknowledgement exec (#77)
The wrapper exits as soon as it sees the acknowledgement file, and that exit tears the container down and can SIGKILL the acknowledging `docker exec` before it reports success. The supervisor treated that exec status as a capture failure, so "delimits READY after finite newline-free stderr" failed intermittently in the real-docker job (run 36470233043). A failed acknowledgement is now retried until one succeeds, the container exits, or a stop ends the invocation, and deferred output is accepted only if the container exited with the READY status. The acknowledgement script is idempotent, so a retry repairs a file an earlier attempt left partial. Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
1 parent 9854fe3 commit cd330f4

3 files changed

Lines changed: 235 additions & 24 deletions

File tree

‎agents/adapters/supervisor.ts‎

Lines changed: 49 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ export const OUTPUT_LIMITS = Object.freeze({
1414
});
1515
const DEFAULT_TIMEOUT_MS = 10 * 60_000;
1616
const CAPTURE_ABORT_GRACE_MS = 1_000;
17+
const ACKNOWLEDGEMENT_RETRY_MS = 250;
1718
const DIAGNOSTIC_BYTES = 1024;
1819
const active = new Map<string, InvocationHandle>();
1920
const activeProfiles = new WeakSet<ContainerProfile>();
@@ -363,6 +364,27 @@ export function readBoundedContainerFile(container: string, source: string, maxi
363364
});
364365
}
365366

367+
/**
368+
* Run as root inside the container: write the deferred output acknowledgement `collected-<token>` that the wrapper
369+
* waits for. It is idempotent, so a retry succeeds after an attempt that died once it had created the file: a
370+
* complete file is kept, and a partial one is unlinked and written again. Only root can create entries in the
371+
* control directory, and without capabilities root cannot reopen its own read-only file for writing.
372+
*/
373+
export const ACKNOWLEDGEMENT_SCRIPT = [
374+
"const fs=require('node:fs'),c=fs.constants,token=process.argv[1],directory='/run/codeboost-control';",
375+
"let dirfd,fd,existing;try{dirfd=fs.openSync(directory,c.O_RDONLY|c.O_DIRECTORY|c.O_NOFOLLOW);",
376+
"const name='/proc/self/fd/'+dirfd+'/collected-'+token,create=()=>fs.openSync(name,",
377+
'c.O_WRONLY|c.O_CREAT|c.O_EXCL|c.O_NOFOLLOW,0o444);',
378+
"try{fd=create()}catch(error){if(error.code!=='EEXIST')throw error;",
379+
'existing=fs.openSync(name,c.O_RDONLY|c.O_NOFOLLOW|c.O_NONBLOCK);const stat=fs.fstatSync(existing,{bigint:true});',
380+
"if(!stat.isFile()||stat.nlink!==1n)throw new Error('UNSAFE_ACK');",
381+
"const complete=fs.readFileSync(existing,'utf8')===token;fs.closeSync(existing);existing=undefined;",
382+
'if(!complete){fs.unlinkSync(name);fd=create()}}',
383+
"if(fd!==undefined){fs.writeFileSync(fd,token);fs.fsyncSync(fd);const stat=fs.fstatSync(fd,{bigint:true});",
384+
"if(!stat.isFile()||stat.nlink!==1n)throw new Error('UNSAFE_ACK')}}finally{if(existing!==undefined)fs.closeSync(existing);",
385+
'if(fd!==undefined)fs.closeSync(fd);if(dirfd!==undefined)fs.closeSync(dirfd)}',
386+
].join('');
387+
366388
export function isInvocationActive(attemptId: string): boolean {
367389
return ownsAttempt(attemptId);
368390
}
@@ -417,6 +439,7 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super
417439
let decodedOutput: DecodedOutput | undefined, decodePromise: Promise<void> | undefined;
418440
let decodeAbort: AbortController | undefined;
419441
let protocolToken: string | undefined, protocolStarted = false, protocolReady = false;
442+
let readyStatus: number | undefined, acknowledgementFailures = 0;
420443
let protocolBuffer = Buffer.alloc(0);
421444
// Setup (container create and validation) runs inside the handle; a stop kills its in-flight Docker call.
422445
const setupAbort = new AbortController();
@@ -443,22 +466,28 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super
443466
void operation.finally(() => controls.delete(operation));
444467
return operation;
445468
};
446-
const acknowledgeDeferredOutput = (token: string) => {
447-
const script = [
448-
"const fs=require('node:fs'),token=process.argv[1],directory='/run/codeboost-control';",
449-
'let dirfd,fd;try{dirfd=fs.openSync(directory,fs.constants.O_RDONLY|fs.constants.O_DIRECTORY|fs.constants.O_NOFOLLOW);',
450-
"fd=fs.openSync('/proc/self/fd/'+dirfd+'/collected-'+token,",
451-
'fs.constants.O_WRONLY|fs.constants.O_CREAT|fs.constants.O_EXCL|fs.constants.O_NOFOLLOW,0o444);',
452-
"fs.writeFileSync(fd,token);fs.fsyncSync(fd);const stat=fs.fstatSync(fd,{bigint:true});",
453-
"if(!stat.isFile()||stat.nlink!==1n)throw new Error('UNSAFE_ACK')}finally{if(fd!==undefined)fs.closeSync(fd);",
454-
'if(dirfd!==undefined)fs.closeSync(dirfd)}',
455-
].join('');
456-
return runControl(['exec', '--user', '0', container, 'node', '-e', script, token]);
457-
};
469+
const acknowledgeDeferredOutput = (token: string) =>
470+
runControl(['exec', '--user', '0', container, 'node', '-e', ACKNOWLEDGEMENT_SCRIPT, token]);
458471
const later = (callback: () => void, delay: number) => {
459472
const timer = setTimeout(() => { timers.delete(timer); callback(); }, delay);
460473
timer.unref(); timers.add(timer); return timer;
461474
};
475+
/**
476+
* The wrapper exits as soon as it sees the acknowledgement, and that exit can kill the acknowledging `docker exec`
477+
* before it reports success. The exec's status therefore decides nothing: a failed attempt is retried until one
478+
* succeeds, the container exits, or a stop ends the invocation. The close handler accepts the output only if the
479+
* container exited with the READY status, as the wrapper does after it has seen the acknowledgement; any other
480+
* exit status, or a signal to the attached client, fails closed. A container killed from outside cannot be told
481+
* apart when its kill status equals the READY status, but its output was already collected in full by then.
482+
*/
483+
const acknowledge = (token: string) => {
484+
if (closed || stopReason) return;
485+
void acknowledgeDeferredOutput(token).then(success => {
486+
if (success || closed || stopReason) return;
487+
acknowledgementFailures += 1;
488+
later(() => acknowledge(token), ACKNOWLEDGEMENT_RETRY_MS);
489+
});
490+
};
462491
const terminate = () => {
463492
if (terminating || closed || !child) return;
464493
terminating = true;
@@ -600,18 +629,9 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super
600629
return true;
601630
}
602631
protocolReady = true;
632+
readyStatus = Number(ready[2]);
603633
const readyToken = ready[1]!;
604-
void decodeOutput().then(() => {
605-
if (decodedOutput && !stopReason) {
606-
void acknowledgeDeferredOutput(readyToken)
607-
.then(success => {
608-
if (!success) {
609-
failureDetail ??= 'Deferred output acknowledgement failed.';
610-
stop('capture-failure');
611-
}
612-
});
613-
}
614-
});
634+
void decodeOutput().then(() => { if (decodedOutput) acknowledge(readyToken); });
615635
return true;
616636
};
617637
const captureStderr = (value: Buffer | string) => {
@@ -733,6 +753,12 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super
733753
if (!stopReason && profile.deferredOutput && !decodedOutput) {
734754
stopReason = 'capture-failure'; failureDetail ??= 'Deferred output protocol did not complete.';
735755
}
756+
if (stopReason === 'timeout' && acknowledgementFailures)
757+
failureDetail ??= `Deferred output acknowledgement failed ${acknowledgementFailures} times.`;
758+
if (!stopReason && profile.deferredOutput && (signal !== null || exitCode !== readyStatus)) {
759+
stopReason = 'capture-failure';
760+
failureDetail ??= 'Container exited without taking the deferred output acknowledgement.';
761+
}
736762
if (decodedOutput && !stopReason) {
737763
finalStdout = Buffer.from(decodedOutput.text);
738764
if (decodedOutput.providerFailed) exitCode = exitCode === 0 ? 1 : exitCode;
Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
import { EventEmitter } from 'node:events';
2+
import { PassThrough } from 'node:stream';
3+
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
4+
import type { InvocationResult } from '../agents/contract.ts';
5+
import type { ContainerProfile } from '../agents/container/profile.ts';
6+
7+
// Replays, without Docker, the interleavings of the deferred-output acknowledgement with the container's exit. The
8+
// wrapper (probe.sh) exits as soon as it sees the acknowledgement, and that exit can kill the acknowledging
9+
// `docker exec` before it reports success; CI run 36470233043 failed that way.
10+
type FakeChild = EventEmitter & { stdout: PassThrough; stderr: PassThrough; kill: ReturnType<typeof vi.fn> };
11+
const state = vi.hoisted(() => ({
12+
attach: undefined as FakeChild | undefined,
13+
acknowledgements: [] as { child: FakeChild; args: readonly string[] }[],
14+
}));
15+
vi.mock('node:child_process', () => ({
16+
execFile: vi.fn(),
17+
spawn: vi.fn((_command: string, args: readonly string[]) => {
18+
const child = Object.assign(new EventEmitter(), { stdout: new PassThrough(), stderr: new PassThrough(), kill: vi.fn() });
19+
if (args[0] === 'start') state.attach = child;
20+
else if (args[0] === 'exec') state.acknowledgements.push({ child, args });
21+
else process.nextTick(() => child.emit('close', 0, null));
22+
return child;
23+
}),
24+
}));
25+
vi.mock('../agents/policy.ts', () => ({ assertPhasePolicy: (policy: { invocation: unknown }) => policy.invocation }));
26+
vi.mock('../agents/container/profile.ts', () => ({
27+
assertContainerProfileAuthenticity: () => {},
28+
isContainerProfileAuthentic: () => true,
29+
disposeContainerProfile: async () => {},
30+
containerProfileResources: () => [],
31+
}));
32+
vi.mock('../agents/container/run.ts', () => ({
33+
createValidatedContainer: async () => {},
34+
validateContainer: async () => {},
35+
disposeValidatedContainer: async () => {},
36+
retireContainerProfile: () => {},
37+
isContainerProfileRetired: () => false,
38+
agentContainerId: () => 'c'.repeat(64),
39+
agentContainerResources: () => [],
40+
}));
41+
const { startProfileInvocation } = await import('../agents/adapters/supervisor.ts');
42+
43+
const token = '11111111-2222-3333-4444-555555555555';
44+
const fakeProfile = (attemptId: string) => ({
45+
name: `codeboost-agent-${attemptId}`, deferredOutput: true, resources: [],
46+
policy: { invocation: { attemptId, deadline: Date.now() + 10 * 60_000,
47+
context: { snapshotId: 's', planId: 'p', planRevision: 1, assignmentId: 'a', referencedCodeHash: 'c', stateVersion: 1 } } },
48+
}) as unknown as ContainerProfile;
49+
50+
describe('deferred output acknowledgement', () => {
51+
beforeEach(() => {
52+
vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'performance'] });
53+
state.attach = undefined;
54+
state.acknowledgements = [];
55+
});
56+
afterEach(() => { vi.useRealTimers(); });
57+
58+
/** Start an invocation whose agent writes newline-free stderr, then READY with `status`. */
59+
const readyInvocation = async (attemptId: string, status = 0, timeoutMs?: number) => {
60+
const box: { result?: InvocationResult } = {};
61+
void startProfileInvocation(fakeProfile(attemptId), { timeoutMs, decode: () => ({ text: 'captured' }) })
62+
.settled.then(result => { box.result = result; });
63+
await vi.advanceTimersByTimeAsync(0);
64+
const stderr = state.attach!.stderr;
65+
stderr.emit('data', Buffer.from(`\x1eCODEBOOST_START:${token}\x1e\n`));
66+
stderr.emit('data', Buffer.from('trailing-diagnostic'));
67+
stderr.emit('data', Buffer.from(`\n\x1eCODEBOOST_READY:${token}:${status}\x1e\n`));
68+
await vi.advanceTimersByTimeAsync(0);
69+
// The decoded output was collected, so the supervisor acknowledges it with the READY token.
70+
expect(state.acknowledgements).toHaveLength(1);
71+
expect(state.acknowledgements[0]!.args.at(-1)).toBe(token);
72+
return box;
73+
};
74+
const acknowledgementExits = (index: number, code: number | null, signal: NodeJS.Signals | null = null) =>
75+
state.acknowledgements[index]!.child.emit('close', code, signal);
76+
const containerExits = (code: number | null, signal: NodeJS.Signals | null = null) =>
77+
state.attach!.emit('close', code, signal);
78+
79+
it.each([
80+
['before the container exit is observed', 'acknowledgement-first'],
81+
['after the container exit is observed', 'exit-first'],
82+
] as const)('keeps the result when the wrapper takes the acknowledgement and its exec then fails %s',
83+
async (_label, order) => {
84+
const box = await readyInvocation(`ack-killed-${order}`);
85+
// The wrapper saw the acknowledgement and exited with the READY status; the teardown killed the exec.
86+
if (order === 'acknowledgement-first') { acknowledgementExits(0, 137); containerExits(0); }
87+
else { containerExits(0); acknowledgementExits(0, 137); }
88+
await vi.advanceTimersByTimeAsync(0);
89+
expect(box.result?.stopReason, box.result?.stderr).toBeUndefined();
90+
expect(box.result?.exitCode).toBe(0);
91+
expect(box.result?.stdout).toBe('captured');
92+
expect(box.result?.stderr).toBe('trailing-diagnostic\n');
93+
// Nothing retries once the container has exited.
94+
await vi.advanceTimersByTimeAsync(5_000);
95+
expect(state.acknowledgements).toHaveLength(1);
96+
});
97+
98+
it('retries an acknowledgement that failed while the container keeps waiting for it', async () => {
99+
const box = await readyInvocation('ack-retried');
100+
acknowledgementExits(0, 1);
101+
await vi.advanceTimersByTimeAsync(0);
102+
expect(box.result).toBeUndefined();
103+
await vi.advanceTimersByTimeAsync(250);
104+
expect(state.acknowledgements).toHaveLength(2);
105+
expect(state.acknowledgements[1]!.args.at(-1)).toBe(token);
106+
acknowledgementExits(1, 0);
107+
containerExits(0);
108+
await vi.advanceTimersByTimeAsync(0);
109+
expect(box.result?.stopReason, box.result?.stderr).toBeUndefined();
110+
expect(box.result?.stderr).toBe('trailing-diagnostic\n');
111+
});
112+
113+
it('reports the READY status of an agent that failed', async () => {
114+
const box = await readyInvocation('ack-nonzero', 3);
115+
acknowledgementExits(0, 0);
116+
containerExits(3);
117+
await vi.advanceTimersByTimeAsync(0);
118+
expect(box.result?.stopReason, box.result?.stderr).toBeUndefined();
119+
expect(box.result?.exitCode).toBe(3);
120+
});
121+
122+
it.each([
123+
['an exit status other than READY', 137, null],
124+
['a signal', null, 'SIGKILL'],
125+
] as const)('fails closed when the container exits with %s instead of taking the acknowledgement',
126+
async (_label, code, signal) => {
127+
const box = await readyInvocation(`ack-bypassed-${signal ?? code}`);
128+
acknowledgementExits(0, 137);
129+
containerExits(code, signal);
130+
await vi.advanceTimersByTimeAsync(0);
131+
expect(box.result?.stopReason).toBe('capture-failure');
132+
expect(box.result?.stderr).toContain('trailing-diagnostic\n');
133+
expect(box.result?.stderr).toContain('Container exited without taking the deferred output acknowledgement.');
134+
});
135+
136+
it('times out instead of succeeding when no acknowledgement ever takes effect', async () => {
137+
const box = await readyInvocation('ack-never', 0, 5_000);
138+
for (let index = 0; index < 4; index += 1) {
139+
acknowledgementExits(index, 1);
140+
await vi.advanceTimersByTimeAsync(250);
141+
}
142+
expect(state.acknowledgements).toHaveLength(5);
143+
await vi.advanceTimersByTimeAsync(5_000);
144+
expect(box.result).toBeUndefined();
145+
// The deadline stopped the container, which exits by signal and ends the last attempt without a retry.
146+
acknowledgementExits(4, 137);
147+
containerExits(null, 'SIGKILL');
148+
await vi.advanceTimersByTimeAsync(0);
149+
expect(box.result?.stopReason).toBe('timeout');
150+
expect(box.result?.stderr).toContain('trailing-diagnostic\n');
151+
expect(box.result?.stderr).toContain('Deferred output acknowledgement failed 4 times.');
152+
});
153+
});

‎test/agent-supervisor.test.ts‎

Lines changed: 33 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import { dirname, join } from 'node:path';
66
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
77
import { startClaudeInvocation } from '../agents/adapters/claude.ts';
88
import { readCodexOutput, startCodexInvocation } from '../agents/adapters/codex.ts';
9-
import { isInvocationActive, readBoundedContainerFile, retainSetupCleanup,
9+
import { ACKNOWLEDGEMENT_SCRIPT, isInvocationActive, readBoundedContainerFile, retainSetupCleanup,
1010
startProfileInvocation } from '../agents/adapters/supervisor.ts';
1111
import { captureInvocation, type InvocationInput, type InvocationResult } from '../agents/contract.ts';
1212
import { buildAgentImage } from '../agents/container/image.ts';
@@ -543,6 +543,38 @@ describe('container invocation supervisor', () => {
543543
expect(result.stderr).toContain('trailing-diagnostic');
544544
}, 60_000);
545545

546+
it('writes the deferred output acknowledgement idempotently, repairing a partial one', () => {
547+
const token = randomUUID(), file = `/run/codeboost-control/collected-${token}`;
548+
const name = `codeboost-ack-script-${randomUUID()}`;
549+
execFileSync('docker', ['run', '-d', '--rm', '--name', name, '--network', 'none', '--cap-drop=ALL',
550+
'--security-opt=no-new-privileges', '--user', '10001:10001', '--entrypoint', 'sleep',
551+
'--tmpfs', '/run/codeboost-control:rw,nosuid,nodev,noexec,size=65536,nr_inodes=16,uid=0,gid=0,mode=0711',
552+
imageId, '120'], { stdio: 'ignore' });
553+
try {
554+
const acknowledge = () => spawnSync('docker', ['exec', '--user', '0', name, 'node', '-e', ACKNOWLEDGEMENT_SCRIPT,
555+
token], { encoding: 'utf8' });
556+
const content = () => execFileSync('docker', ['exec', name, 'cat', file], { encoding: 'utf8' });
557+
const root = (script: string) => execFileSync('docker', ['exec', '--user', '0', name, 'sh', '-c', script]);
558+
expect(acknowledge().status).toBe(0);
559+
expect(content()).toBe(token);
560+
// A retry after an attempt that completed.
561+
expect(acknowledge().status).toBe(0);
562+
expect(content()).toBe(token);
563+
// A retry after an attempt that died once it had created the file.
564+
for (const partial of ['', token.slice(0, 8)]) {
565+
root(`rm ${file} && printf '${partial}' > ${file} && chmod 444 ${file}`);
566+
expect(acknowledge().status).toBe(0);
567+
expect(content()).toBe(token);
568+
}
569+
expect(execFileSync('docker', ['exec', name, 'stat', '-c', '%a %u %h', file], { encoding: 'utf8' }).trim())
570+
.toBe('444 0 1');
571+
root(`rm ${file} && ln -s /etc/passwd ${file}`);
572+
expect(acknowledge().status).not.toBe(0);
573+
} finally {
574+
spawnSync('docker', ['rm', '--force', name], { stdio: 'ignore' });
575+
}
576+
}, 60_000);
577+
546578
if (process.env.CODEBOOST_RUN_AUTH_PROBES === '1') {
547579
const schemaPrompt = 'Read /run/codeboost-input/schema.json and reply only with the exact value of its probe field, '
548580
+ 'without quotes or Markdown formatting.';

0 commit comments

Comments
 (0)