Skip to content

Commit 56c6a9a

Browse files
committed
feat(computer-use): align native app automation with Codex
Add a persistent isolated JavaScript worker for native app actions and batch known operations without a model round trip between each input. Preserve per-cell context, native errors, screenshot coordinates, and image types. Align macOS gesture, key, inventory, capture, scroll, and clipboard behavior; include a signed native receiver fixture and compiled sidecar regression tests. Keep the Windows pixel route and fix cancellation with session-owned mouse cleanup, lock revalidation, and portable signing-fixture tests.
1 parent a2b3f59 commit 56c6a9a

84 files changed

Lines changed: 8712 additions & 719 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎desktop/sidecars/claude-sidecar.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,15 @@
2020

2121
import { parseLauncherArgs, resolveSidecarInvocation } from './launcherRouting'
2222

23+
// The compiled Computer Use runtime relaunches this executable directly. Its
24+
// isolated worker has no app-root and must not load preload, CLI configuration,
25+
// providers, or any desktop mode before serving the JSON-only kernel protocol.
26+
if (process.argv[2] === '--computer-use-repl-worker') {
27+
const { runComputerUseReplWorker } = await import('../../src/utils/computerUse/replWorker')
28+
await runComputerUseReplWorker()
29+
process.exit(0)
30+
}
31+
2332
type AdapterConfigShape = Awaited<
2433
ReturnType<typeof import('../../adapters/common/config.ts')['loadConfig']>
2534
>
Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
// @vitest-environment node
2+
3+
import { spawn, type ChildProcessWithoutNullStreams } from 'node:child_process'
4+
import { mkdtemp, readdir, realpath, rm } from 'node:fs/promises'
5+
import { tmpdir } from 'node:os'
6+
import path from 'node:path'
7+
import { createInterface } from 'node:readline'
8+
import { describe, expect, it } from 'vitest'
9+
import { createComputerUseReplSandboxCommand } from '../../src/utils/computerUse/replRuntime'
10+
import { REPL_BOOTSTRAP_SOURCE } from '../../src/vendor/computer-use-mcp/replApi'
11+
import type { ReplInput, ReplOutput } from '../../src/vendor/computer-use-mcp/replProtocol'
12+
13+
const repoRoot = path.resolve(import.meta.dirname, '../..')
14+
15+
async function run(command: string, args: string[], cwd: string, env: NodeJS.ProcessEnv) {
16+
const child = spawn(command, args, { cwd, env, stdio: ['ignore', 'pipe', 'pipe'], timeout: 90_000 })
17+
let stdout = ''
18+
let stderr = ''
19+
child.stdout.on('data', chunk => { stdout += String(chunk) })
20+
child.stderr.on('data', chunk => { stderr += String(chunk) })
21+
const code = await new Promise<number | null>((resolve, reject) => {
22+
child.once('error', reject)
23+
child.once('exit', resolve)
24+
})
25+
if (code !== 0) throw new Error(`${path.basename(command)} exited ${code}: ${stderr}\n${stdout}`)
26+
return stdout
27+
}
28+
29+
describe.skipIf(process.platform !== 'darwin')('compiled desktop Computer Use worker routing', () => {
30+
it('boots the real merged entrypoint inside the production sandbox without preload and retains native App bindings', async () => {
31+
const directory = await realpath(await mkdtemp(path.join(tmpdir(), 'cc-haha-sidecar-cu-worker-')))
32+
const executable = path.join(directory, 'claude-sidecar-aarch64-apple-darwin')
33+
const env: NodeJS.ProcessEnv = {
34+
PATH: process.env.PATH,
35+
HOME: directory,
36+
TMPDIR: directory,
37+
TMP: directory,
38+
TEMP: directory,
39+
CLAUDE_CONFIG_DIR: path.join(directory, '.claude'),
40+
BUN_OPTIONS: '--no-env-file',
41+
// preload.ts would chdir here. The internal worker must never load it.
42+
CALLER_DIR: path.join(directory, 'must-not-enter-preload'),
43+
}
44+
let child: ChildProcessWithoutNullStreams | undefined
45+
let closed: Promise<{ code: number | null; signal: NodeJS.Signals | null }> | undefined
46+
try {
47+
// Compile the real production entrypoint, not a handwritten worker shim.
48+
// These are the optional externals used by build-sidecars.ts; Acorn and
49+
// sandbox-runtime remain bundled. No signing identity/keychain discovery.
50+
const build = {
51+
entrypoints: [path.join(repoRoot, 'desktop/sidecars/claude-sidecar.ts')],
52+
features: ['TRANSCRIPT_CLASSIFIER'],
53+
minify: { whitespace: true, identifiers: true, syntax: true },
54+
sourcemap: 'none',
55+
target: 'bun',
56+
external: [
57+
'@opentelemetry/exporter-trace-otlp-grpc', '@opentelemetry/exporter-trace-otlp-http',
58+
'@opentelemetry/exporter-trace-otlp-proto', '@opentelemetry/exporter-logs-otlp-grpc',
59+
'@opentelemetry/exporter-logs-otlp-http', '@opentelemetry/exporter-logs-otlp-proto',
60+
'@opentelemetry/exporter-metrics-otlp-grpc', '@opentelemetry/exporter-metrics-otlp-http',
61+
'@opentelemetry/exporter-metrics-otlp-proto', '@opentelemetry/exporter-prometheus',
62+
'@aws-sdk/client-bedrock', '@aws-sdk/client-sts', '@anthropic-ai/bedrock-sdk',
63+
'@anthropic-ai/foundry-sdk', '@anthropic-ai/vertex-sdk', '@azure/identity',
64+
'@anthropic-ai/mcpb', 'fflate', 'sharp', 'react-devtools-core',
65+
],
66+
compile: { outfile: executable, autoloadTsconfig: true, autoloadPackageJson: true },
67+
}
68+
await run('bun', ['--no-env-file', '-e', `const r=await Bun.build(${JSON.stringify(build)});if(!r.success){console.error(r.logs);process.exit(1)}`], repoRoot, { ...env, CALLER_DIR: undefined })
69+
await run('/usr/bin/codesign', ['--remove-signature', executable], directory, env)
70+
await run('/usr/bin/codesign', ['--sign', '-', '--force', '--timestamp=none', executable], directory, env)
71+
72+
const command = createComputerUseReplSandboxCommand({
73+
args: [executable, '--computer-use-repl-worker'],
74+
readable: [executable, directory],
75+
directory,
76+
})
77+
child = spawn('/bin/sh', ['-c', `exec ${command}`], { cwd: directory, env, stdio: 'pipe' })
78+
closed = new Promise(resolve => child!.once('close', (code, signal) => resolve({ code, signal })))
79+
const messages: ReplOutput[] = []
80+
const invocations: Array<{ name: string; args: unknown }> = []
81+
let stderr = ''
82+
let failure: Error | undefined
83+
child.stderr.on('data', chunk => { stderr += String(chunk) })
84+
child.on('error', error => { failure = error })
85+
child.stdin.on('error', error => { failure = error })
86+
const lines = createInterface({ input: child.stdout })
87+
const send = (message: ReplInput) => child!.stdin.write(`${JSON.stringify(message)}\n`)
88+
lines.on('line', line => {
89+
try {
90+
const message = JSON.parse(line) as ReplOutput
91+
messages.push(message)
92+
if (message.type === 'invoke') {
93+
invocations.push({ name: message.name, args: message.args })
94+
send({
95+
type: 'response', cellId: message.cellId, requestId: message.requestId,
96+
result: {
97+
app: (message.args as { app?: string }).app === 'Replacement'
98+
? '/Applications/Replacement.app' : '/Applications/Fixture.app',
99+
content: [{ type: 'text', text: '<app_state>\ng1:0 window\n\tg1:1 button Test\n</app_state>' }],
100+
},
101+
})
102+
}
103+
} catch (error) {
104+
failure = error instanceof Error ? error : new Error(String(error))
105+
}
106+
})
107+
async function until(predicate: () => boolean) {
108+
const deadline = Date.now() + 10_000
109+
while (!predicate()) {
110+
if (failure) throw failure
111+
if (child!.exitCode !== null || child!.signalCode !== null) {
112+
throw new Error(`Worker exited before readiness/completion: ${stderr}`)
113+
}
114+
if (Date.now() > deadline) throw new Error(`Worker timed out: ${stderr}`)
115+
await new Promise(resolve => setTimeout(resolve, 10))
116+
}
117+
}
118+
send({ type: 'init', bootstrap: REPL_BOOTSTRAP_SOURCE })
119+
await until(() => messages.some(message => message.type === 'ready'))
120+
send({ type: 'run', cellId: 1, code: 'let app = await cua.getApp("Fixture"); let count = 0; async function press(){await app.click([10,10]);count++}; function current(){return count}' })
121+
await until(() => messages.some(message => message.type === 'done' && message.cellId === 1))
122+
send({ type: 'run', cellId: 2, code: 'for (let i=0;i<3;i++) await press(); nodeRepl.write(count)' })
123+
await until(() => messages.some(message => message.type === 'done' && message.cellId === 2))
124+
send({ type: 'run', cellId: 3, code: 'app = await cua.getApp("Replacement"); count=10; await press(); nodeRepl.write({count,current:current()})' })
125+
await until(() => messages.some(message => message.type === 'done' && message.cellId === 3))
126+
expect(messages.filter(message => message.type === 'done' && message.error)).toEqual([])
127+
expect(invocations.map(call => call.name)).toEqual(['get_app_state', 'click', 'click', 'click', 'get_app_state', 'click'])
128+
expect(invocations[1]?.args).toEqual({ app: '/Applications/Fixture.app', x: 10, y: 10 })
129+
expect(invocations[5]?.args).toEqual({ app: '/Applications/Replacement.app', x: 10, y: 10 })
130+
expect(messages).toContainEqual({ type: 'emit', cellId: 2, content: { type: 'text', text: '3' } })
131+
const closureResult = messages.find(message => message.type === 'emit' && message.cellId === 3 && message.content.type === 'text' && message.content.text.includes('current'))
132+
expect(closureResult?.type === 'emit' && closureResult.content.type === 'text' && JSON.parse(closureResult.content.text)).toEqual({ count: 11, current: 11 })
133+
child.stdin.end()
134+
expect(await closed).toEqual({ code: 0, signal: null })
135+
expect(stderr).toBe('')
136+
expect((await readdir(directory)).filter(name => name !== path.basename(executable))).toEqual([])
137+
} finally {
138+
child?.kill('SIGKILL')
139+
if (closed) await closed
140+
await rm(directory, { recursive: true, force: true })
141+
}
142+
}, 120_000)
143+
})

0 commit comments

Comments
 (0)