Skip to content

Commit 2d8fe7f

Browse files
committed
fix(persistence): keep ui_messages readable when a write is interrupted
safeWriteJson replaced a file with two renames (target to backup, then replacement to target). A process exit between the renames left the canonical path absent, so readers saw a permanent ENOENT that no retry could repair. The replacement is now committed with a single atomic rename. Only a Windows-style EPERM (target held open by a reader) falls back to the two-step sequence, and a recovery pass under the advisory lock restores the newest orphaned backup and removes leftover temp files before any write. Task-message saves emit trace points with the task ID and process ID, the marker check and shutdown are traced, and a failed restart-persistence scenario now preserves phase results, VS Code logs, and task files for CI artifact upload.
1 parent 9cb83c3 commit 2d8fe7f

14 files changed

Lines changed: 780 additions & 81 deletions

File tree

‎.github/workflows/e2e.yml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -156,6 +156,18 @@ jobs:
156156
if: github.event_name != 'pull_request' || steps.e2e-marker.outputs.cache-hit != 'true'
157157
run: TEST_FILE=restart-persistence.test USE_MOCK=true xvfb-run -a pnpm --filter @roo-code/vscode-e2e exec node ./out/runTest.js
158158

159+
- name: Upload restart-persistence failure artifacts
160+
# runTest.js preserves a failed scenario workspace (phase results,
161+
# VS Code logs, task files) under apps/vscode-e2e/test-failures/
162+
# instead of deleting it, so the interrupted-write state from
163+
# issue #2130-style failures stays inspectable.
164+
if: failure()
165+
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
166+
with:
167+
name: restart-persistence-failure
168+
path: apps/vscode-e2e/test-failures/
169+
if-no-files-found: ignore
170+
159171
- name: Explain skipped mocked E2E pass marker
160172
if: steps.e2e-marker.outputs.cache-hit != 'true' && steps.run-e2e.outcome == 'success' && steps.vscode-fallback.outputs.used == 'true'
161173
run: echo "Skipping mocked E2E pass marker because tests ran against a stale cached VS Code binary (VS Code download endpoints unreachable)."

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ AGENTS.local.md
2727
.vscode-test/
2828
apps/vscode-e2e/playwright-report-electron/
2929
apps/vscode-e2e/test-results-electron/
30+
apps/vscode-e2e/test-failures/
3031

3132
# Docs
3233
docs/_site/

‎apps/vscode-e2e/src/restart/scenarioWorkspace.ts‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,3 +56,56 @@ export async function removeScenarioWorkspace(scenarioWorkspace: ScenarioWorkspa
5656

5757
await fs.rm(root, { recursive: true, force: true })
5858
}
59+
60+
const FAILURE_ARTIFACTS_DIR = "test-failures"
61+
62+
/**
63+
* Copies the diagnosable parts of a failed scenario workspace into a stable
64+
* directory inside the checkout, then removes the temporary scenario root.
65+
* CI uploads that directory as a run artifact, so the failure that used to be
66+
* deleted in the cleanup path stays inspectable: phase results, VS Code logs,
67+
* and the extension's global storage with the task files.
68+
*
69+
* The directory is keyed by the unique scenario root name and cleared before
70+
* copying, so retried runs cannot mix artifacts from different failures.
71+
*/
72+
export async function preserveScenarioWorkspace(scenarioWorkspace: ScenarioWorkspace): Promise<string> {
73+
const destinationRoot = path.resolve(
74+
__dirname,
75+
"../../",
76+
FAILURE_ARTIFACTS_DIR,
77+
path.basename(scenarioWorkspace.root),
78+
)
79+
await fs.rm(destinationRoot, { recursive: true, force: true })
80+
await fs.mkdir(destinationRoot, { recursive: true })
81+
82+
const selections: Array<{ label: string; source: string; destination: string }> = [
83+
{
84+
label: "phase results",
85+
source: scenarioWorkspace.results,
86+
destination: path.join(destinationRoot, "phase-results"),
87+
},
88+
{
89+
label: "VS Code logs",
90+
source: path.join(scenarioWorkspace.userData, "logs"),
91+
destination: path.join(destinationRoot, "vscode-logs"),
92+
},
93+
{
94+
label: "task files",
95+
source: path.join(scenarioWorkspace.userData, "User", "globalStorage"),
96+
destination: path.join(destinationRoot, "global-storage"),
97+
},
98+
]
99+
100+
for (const selection of selections) {
101+
try {
102+
await fs.cp(selection.source, selection.destination, { recursive: true, force: true })
103+
} catch (error) {
104+
// A missing subdirectory must not hide the other artifacts.
105+
console.error(`Failed to preserve ${selection.label} from ${selection.source}:`, error)
106+
}
107+
}
108+
109+
await removeScenarioWorkspace(scenarioWorkspace)
110+
return destinationRoot
111+
}

‎apps/vscode-e2e/src/runTest.ts‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,11 @@ import { addSearchFilesResultFixtures } from "./fixtures/search-files"
2121
import { addSubtaskFixtures } from "./fixtures/subtasks"
2222
import { addUseMcpToolResultFixtures } from "./fixtures/use-mcp-tool"
2323
import { addWriteToFileResultFixtures } from "./fixtures/write-to-file"
24-
import { createScenarioWorkspace, removeScenarioWorkspace } from "./restart/scenarioWorkspace"
24+
import {
25+
createScenarioWorkspace,
26+
preserveScenarioWorkspace,
27+
removeScenarioWorkspace,
28+
} from "./restart/scenarioWorkspace"
2529
import { runRestartScenario } from "./restart/vscodeCoordinator"
2630

2731
function getCliFlagValue(flag: string) {
@@ -95,6 +99,7 @@ async function main() {
9599

96100
let testWorkspace: string | undefined
97101
let scenarioWorkspace: Awaited<ReturnType<typeof createScenarioWorkspace>> | undefined
102+
let runFailed = false
98103

99104
try {
100105
// Create a temporary workspace folder for regular tests. Restart scenarios own
@@ -218,14 +223,27 @@ async function main() {
218223
})
219224
}
220225
} catch (error) {
226+
runFailed = true
221227
console.error("Failed to run tests", error)
222228
process.exitCode = 1
223229
} finally {
224230
if (testWorkspace) {
225231
await fs.rm(testWorkspace, { recursive: true, force: true })
226232
}
227233
if (scenarioWorkspace) {
228-
await removeScenarioWorkspace(scenarioWorkspace)
234+
// A failed restart scenario must stay inspectable: the phase
235+
// results, VS Code logs, and task files are copied to a stable
236+
// directory for CI artifact upload before the temp root goes away.
237+
if (runFailed) {
238+
try {
239+
const preservedAt = await preserveScenarioWorkspace(scenarioWorkspace)
240+
console.error(`Restart scenario workspace preserved for failure inspection: ${preservedAt}`)
241+
} catch (preserveError) {
242+
console.error("Failed to preserve the restart scenario workspace:", preserveError)
243+
}
244+
} else {
245+
await removeScenarioWorkspace(scenarioWorkspace)
246+
}
229247
}
230248
await mock?.stop()
231249
}
Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
import { describe, it, expect, vi, afterEach } from "vitest"
2+
3+
import { setPersistenceTraceSink, tracePersistenceEvent } from "../persistenceTrace"
4+
5+
function parseTraceLine(line: string): Record<string, unknown> {
6+
return JSON.parse(line.replace("[persistence-trace] ", "")) as Record<string, unknown>
7+
}
8+
9+
describe("persistenceTrace", () => {
10+
afterEach(() => {
11+
setPersistenceTraceSink(undefined)
12+
vi.restoreAllMocks()
13+
})
14+
15+
it("emits a JSON line with the timestamp, process ID, stage, and fields to the sink", () => {
16+
const lines: string[] = []
17+
setPersistenceTraceSink((line) => lines.push(line))
18+
19+
tracePersistenceEvent("ui-messages-write-start", { taskId: "task-1", filePath: "/tmp/ui_messages.json" })
20+
21+
expect(lines).toHaveLength(1)
22+
const event = parseTraceLine(lines[0])
23+
expect(event.stage).toBe("ui-messages-write-start")
24+
expect(event.taskId).toBe("task-1")
25+
expect(event.pid).toBe(process.pid)
26+
expect(typeof event.ts).toBe("string")
27+
expect(event.filePath).toBe("/tmp/ui_messages.json")
28+
})
29+
30+
it("marks the marker-check result", () => {
31+
const lines: string[] = []
32+
setPersistenceTraceSink((line) => lines.push(line))
33+
34+
tracePersistenceEvent("ui-messages-marker-check", { taskId: "task-2", result: "match" })
35+
36+
const event = parseTraceLine(lines[0])
37+
expect(event.stage).toBe("ui-messages-marker-check")
38+
expect(event.result).toBe("match")
39+
})
40+
41+
it("falls back to the console when no sink is installed", () => {
42+
const consoleSpy = vi.spyOn(console, "log").mockImplementation(() => {})
43+
44+
tracePersistenceEvent("shutdown")
45+
46+
expect(consoleSpy).toHaveBeenCalledTimes(1)
47+
const line = String(consoleSpy.mock.calls[0]?.[0])
48+
expect(line).toContain("[persistence-trace]")
49+
expect(parseTraceLine(line).stage).toBe("shutdown")
50+
})
51+
52+
it("never lets a failing sink break the caller", () => {
53+
setPersistenceTraceSink(() => {
54+
throw new Error("sink failed")
55+
})
56+
57+
expect(() => tracePersistenceEvent("ui-messages-marker-check", { result: "match" })).not.toThrow()
58+
})
59+
})

‎src/core/task-persistence/__tests__/taskMessages.spec.ts‎

Lines changed: 40 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ vi.mock("../../../utils/safeWriteJson", () => ({
2121
// Import after mocks
2222
import { saveTaskMessages, readTaskMessages } from "../taskMessages"
2323
import { formatErrorDetail } from "../readFileWithMissingRetry"
24+
import { setPersistenceTraceSink } from "../persistenceTrace"
2425

2526
let tmpBaseDir: string
2627

@@ -35,6 +36,7 @@ beforeEach(async () => {
3536
afterEach(() => {
3637
vi.useRealTimers()
3738
vi.restoreAllMocks()
39+
setPersistenceTraceSink(undefined)
3840
})
3941

4042
describe("taskMessages.saveTaskMessages", () => {
@@ -63,7 +65,11 @@ describe("taskMessages.saveTaskMessages", () => {
6365
expect(hoisted.safeWriteJsonMock).toHaveBeenCalledTimes(1)
6466
const [, persisted] = hoisted.safeWriteJsonMock.mock.calls[0]
6567
expect(persisted).toEqual(messages)
66-
expect(hoisted.safeWriteJsonMock.mock.calls[0][2]).toBeUndefined()
68+
const options = hoisted.safeWriteJsonMock.mock.calls[0][2] as { merge?: unknown; trace?: unknown } | undefined
69+
expect(options?.merge).toBeUndefined()
70+
// The trace callback is always installed so restart logs can follow
71+
// the write stages.
72+
expect(options?.trace).toBeTypeOf("function")
6773
})
6874

6975
it("persists messages without modification when no metadata", async () => {
@@ -101,6 +107,39 @@ describe("taskMessages.saveTaskMessages", () => {
101107
])
102108
})
103109

110+
it("emits persistence trace events with the task ID for the write stages", async () => {
111+
const traceLines: string[] = []
112+
hoisted.safeWriteJsonMock.mockImplementationOnce(async (_path, _data, options) => {
113+
for (const stage of ["recover", "write-start", "backup-rename", "replacement-rename", "write-complete"]) {
114+
options.trace({ stage, detail: stage === "backup-rename" ? "eperm-fallback" : undefined })
115+
}
116+
})
117+
setPersistenceTraceSink((line) => traceLines.push(line))
118+
119+
await saveTaskMessages({
120+
messages: [{ ts: 1, type: "say", say: "text", text: "traced" }],
121+
taskId: "task-trace",
122+
globalStoragePath: tmpBaseDir,
123+
})
124+
125+
const events = traceLines.map(
126+
(line) => JSON.parse(line.replace("[persistence-trace] ", "")) as Record<string, unknown>,
127+
)
128+
expect(events.map((event) => event.stage)).toEqual([
129+
"ui-messages-recovery",
130+
"ui-messages-write-start",
131+
"ui-messages-backup-rename",
132+
"ui-messages-replacement-rename",
133+
"ui-messages-write-complete",
134+
])
135+
for (const event of events) {
136+
expect(event.taskId).toBe("task-trace")
137+
expect(event.pid).toBe(process.pid)
138+
}
139+
const backupRename = events.find((event) => event.stage === "ui-messages-backup-rename")
140+
expect(backupRename?.detail).toBe("eperm-fallback")
141+
})
142+
104143
it("propagates a forced write failure without swallowing or replacing it", async () => {
105144
// Fault injection: simulate an interrupted/failed final write (e.g. the
106145
// extension host quitting mid-save or the disk refusing the write). The
Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
/**
2+
* Structured trace points for task-persistence file writes and shutdown.
3+
*
4+
* The restart-persistence investigation in issue #2130 could not tell which
5+
* writer touched ui_messages.json or when, because nothing logged the
6+
* replacement-protocol steps. These trace lines record the write stages, the
7+
* marker check, and shutdown with the task ID and the process ID, so CI logs
8+
* and extension logs can order writer activity across a restart.
9+
*
10+
* The default sink is the console. Extension activation installs a sink that
11+
* also mirrors every line into the extension output channel; tests can
12+
* install their own sink to capture events. Tracing must never break
13+
* persistence, so sink failures are swallowed.
14+
*/
15+
16+
export type PersistenceTraceStage =
17+
| "ui-messages-recovery"
18+
| "ui-messages-write-start"
19+
| "ui-messages-backup-rename"
20+
| "ui-messages-replacement-rename"
21+
| "ui-messages-write-complete"
22+
| "ui-messages-marker-check"
23+
| "shutdown"
24+
25+
export type PersistenceTraceFields = {
26+
taskId?: string
27+
filePath?: string
28+
result?: string
29+
detail?: string
30+
}
31+
32+
type PersistenceTraceSink = (line: string) => void
33+
34+
let traceSink: PersistenceTraceSink | undefined
35+
36+
/**
37+
* Installs the sink used for every subsequent trace event. Pass undefined to
38+
* restore the default console sink.
39+
*/
40+
export function setPersistenceTraceSink(sink: PersistenceTraceSink | undefined): void {
41+
traceSink = sink
42+
}
43+
44+
export function tracePersistenceEvent(stage: PersistenceTraceStage, fields: PersistenceTraceFields = {}): void {
45+
const line = `[persistence-trace] ${JSON.stringify({
46+
ts: new Date().toISOString(),
47+
pid: process.pid,
48+
stage,
49+
...fields,
50+
})}`
51+
try {
52+
if (traceSink) {
53+
traceSink(line)
54+
} else {
55+
console.log(line)
56+
}
57+
} catch {
58+
// A failing sink must never break the write, read, or shutdown path.
59+
}
60+
}

‎src/core/task-persistence/readFileWithMissingRetry.ts‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
11
import * as fs from "fs/promises"
22

3-
// safeWriteJson replaces a file in two renames (target to backup, new file to
4-
// target) because a single rename over an open file fails with EPERM on
5-
// Windows. A reader can see ENOENT between the two renames, so retry once.
3+
// safeWriteJson commits a replacement with a single atomic rename. Only the
4+
// Windows EPERM fallback replaces a file in two renames (target to backup,
5+
// new file to target), because a single rename over an open file fails with
6+
// EPERM on Windows. A reader can see ENOENT between those two renames, so
7+
// retry once.
68
const READ_RETRY_MIN_MS = 1
79
const READ_RETRY_MAX_MS = 10
810
const READ_RETRY_RANGE_MS = READ_RETRY_MAX_MS - READ_RETRY_MIN_MS + 1

‎src/core/task-persistence/taskMessages.ts‎

Lines changed: 26 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,20 @@ import * as path from "path"
22

33
import type { ClineMessage } from "@roo-code/types"
44

5-
import { safeWriteJson } from "../../utils/safeWriteJson"
5+
import { safeWriteJson, type SafeWriteJsonTraceStage } from "../../utils/safeWriteJson"
66
import { GlobalFileNames } from "../../shared/globalFileNames"
77
import { getTaskDirectoryPath } from "../../utils/storage"
88
import { ensureMessageIdentifiers, mergeClineMessageSnapshots } from "./mergeMessageSnapshots"
99
import { formatErrorDetail, getErrorCode, readFileWithMissingRetry } from "./readFileWithMissingRetry"
10+
import { tracePersistenceEvent, type PersistenceTraceStage } from "./persistenceTrace"
11+
12+
const uiMessagesTraceStages: Record<SafeWriteJsonTraceStage, PersistenceTraceStage> = {
13+
recover: "ui-messages-recovery",
14+
"write-start": "ui-messages-write-start",
15+
"backup-rename": "ui-messages-backup-rename",
16+
"replacement-rename": "ui-messages-replacement-rename",
17+
"write-complete": "ui-messages-write-complete",
18+
}
1019

1120
export type TaskMessagesReadErrorKind = "not_found" | "invalid" | "io_error"
1221

@@ -83,17 +92,21 @@ export async function saveTaskMessages({
8392
const taskDir = await getTaskDirectoryPath(globalStoragePath, taskId)
8493
const filePath = path.join(taskDir, GlobalFileNames.uiMessages)
8594
let savedMessages = messages
86-
await safeWriteJson(
87-
filePath,
88-
messages,
89-
merge
90-
? {
91-
merge: (existing, incoming) => {
92-
savedMessages = mergeClineMessageSnapshots(existing, incoming) as ClineMessage[]
93-
return savedMessages
94-
},
95-
}
96-
: undefined,
97-
)
95+
await safeWriteJson(filePath, messages, {
96+
...(merge && {
97+
merge: (existing, incoming) => {
98+
savedMessages = mergeClineMessageSnapshots(existing, incoming) as ClineMessage[]
99+
return savedMessages
100+
},
101+
}),
102+
// Trace the replacement-protocol stages with the task ID so restart
103+
// logs can order UI-message writer activity across processes.
104+
trace: (event) =>
105+
tracePersistenceEvent(uiMessagesTraceStages[event.stage], {
106+
taskId,
107+
filePath,
108+
detail: event.detail,
109+
}),
110+
})
98111
return savedMessages
99112
}

0 commit comments

Comments
 (0)