Skip to content

Commit e6ab2fb

Browse files
authored
Merge pull request #1306 from taras/codex/repl-reopen-large-history
Reopen REPL histories beyond the aggregate string limit
2 parents 230c735 + a893ab8 commit e6ab2fb

5 files changed

Lines changed: 118 additions & 100 deletions

File tree

‎packages/cli/src/repl/journal.ts‎

Lines changed: 10 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,7 @@
1616
* Reading is also incremental. The file is read a record at a time and never
1717
* held as one string, and the records keep one instance of each large string
1818
* they repeat — a root close hands on every value it inherited, so the same
19-
* captured output recurs in every later close. A history past
20-
* {@link HISTORY_LIMIT} is refused by name rather than by the engine.
19+
* captured output recurs in every later close.
2120
*
2221
* Nothing here chooses a runtime. Reading, existence, kind and directory
2322
* creation are `@effectionx/fs` operations; the two this repository's portable
@@ -196,16 +195,8 @@ export function* openExecutionFile(root: string, execution: string): Operation<R
196195
return { id: execution, stream: new RetainedReplStream(path, yield* readRecords(path)) };
197196
}
198197

199-
/**
200-
* The most text a history may decode to and still be reopened, in UTF-16 code units.
201-
*
202-
* V8's longest string, which is what reading a history as one string used to
203-
* run into. Reading no longer builds that string, but this stays the supported
204-
* bound: a history past it is refused with a sentence that names it, rather
205-
* than read into a heap it may not fit. It bounds reopening only — a running
206-
* history can grow past it, and it is this reader that then refuses.
207-
*/
208-
export const HISTORY_LIMIT = 0x1fffffe8;
198+
/** V8's maximum string length, applied to a single serialized record. */
199+
const MAX_RECORD_UNITS = 0x1fffffe8;
209200

210201
/**
211202
* Every record one file holds, in append order.
@@ -214,25 +205,23 @@ export const HISTORY_LIMIT = 0x1fffffe8;
214205
* record was still being written, and reading around it would present a prefix
215206
* as the whole of what happened.
216207
*
217-
* `limit` is {@link HISTORY_LIMIT} everywhere but a test that has to reach it
218-
* without writing half a gigabyte.
208+
* The limit can be set lower to exercise the record boundary in tests.
219209
*/
220210
export function* readRecords(
221211
path: string,
222-
limit: number = HISTORY_LIMIT,
212+
maxRecordUnits = MAX_RECORD_UNITS,
223213
): Operation<DurableEvent[]> {
224214
const events: DurableEvent[] = [];
225215
const share = stringSharing();
226216
yield* forEachLine(
227217
path,
228218
{
229-
limit: {
230-
units: limit,
231-
exceeded: () =>
219+
recordLimit: {
220+
units: maxRecordUnits,
221+
exceeded: (record) =>
232222
new ReplJournalError(
233-
`this execution's history exceeds the supported limit of ` +
234-
`${limit.toLocaleString("en-US")} UTF-16 code units. Start a new execution with ` +
235-
"`xmd repl` for further work; this history was left unchanged.",
223+
`this execution's history holds a record that exceeds ` +
224+
`${maxRecordUnits.toLocaleString("en-US")} UTF-16 code units (record ${record}).`,
236225
),
237226
},
238227
refusals: {

‎packages/cli/src/repl/record-file.ts‎

Lines changed: 7 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -91,29 +91,14 @@ export interface LineRefusals {
9191
undecodable?(): Error;
9292
}
9393

94-
/**
95-
* How much text a read may decode, and the refusal that names the bound.
96-
*
97-
* The two together, because a bound nobody can state is a bound nobody can be
98-
* told about: a reader with no bound to state passes none, and then carries no
99-
* sentence for a refusal that cannot happen.
100-
*/
94+
/** A bound on one decoded line, including its newline. */
10195
export interface LineBound {
102-
/**
103-
* The most UTF-16 code units the file may decode to, newlines included.
104-
*
105-
* Counted as the file is read and checked before each piece of a line joins
106-
* the rest, so a file past the bound is refused without building a string
107-
* past it. Code units rather than bytes, because the ceiling this guards is a
108-
* string length and a byte count is a different number for the same text.
109-
*/
11096
readonly units: number;
111-
exceeded(): Error;
97+
exceeded(record: number): Error;
11298
}
11399

114100
export interface LineOptions {
115-
/** What the decoded text may not exceed, where the caller has a bound. */
116-
readonly limit?: LineBound;
101+
readonly recordLimit?: LineBound;
117102
/** Refuse bytes that are not UTF-8, rather than replacing them. */
118103
readonly strict?: boolean;
119104
readonly refusals: LineRefusals;
@@ -143,7 +128,7 @@ export function forEachLine(
143128
fatal: options.strict === true,
144129
ignoreBOM: true,
145130
});
146-
const bound = options.limit;
131+
const bound = options.recordLimit;
147132
let pieces: string[] = [];
148133
let units = 0;
149134
let lines = 0;
@@ -158,7 +143,7 @@ export function forEachLine(
158143
}
159144
units += text.length;
160145
if (bound !== undefined && units > bound.units) {
161-
throw bound.exceeded();
146+
throw bound.exceeded(lines + 1);
162147
}
163148
if (text.length > 0) {
164149
pieces.push(text);
@@ -174,10 +159,11 @@ export function forEachLine(
174159
decode(chunk.subarray(start, end), false);
175160
units += 1;
176161
if (bound !== undefined && units > bound.units) {
177-
throw bound.exceeded();
162+
throw bound.exceeded(lines + 1);
178163
}
179164
const line = pieces.join("");
180165
pieces = [];
166+
units = 0;
181167
lines += 1;
182168
unfinished = false;
183169
start = end + 1;

‎packages/cli/src/repl/sidekick/history.ts‎

Lines changed: 47 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ import type { EntryPlacement } from "../entry-placement.ts";
5858
* reach the old path after the published prefix was chosen.
5959
*/
6060

61-
import { ensureDir, exists, readTextFile, stat } from "@effectionx/fs";
61+
import { ensureDir, exists, stat } from "@effectionx/fs";
6262
import { join } from "node:path";
6363
import {
6464
parseDurableEvent,
@@ -883,28 +883,38 @@ export function* readSidekickRecords(path: string): Operation<SidekickRecord[]>
883883
if (!target.isFile()) {
884884
throw new SidekickHistoryError("this execution's chat history is not a file.");
885885
}
886-
const body = yield* readTextFile(path);
887-
if (body.length === 0) {
888-
return [];
889-
}
890-
if (!body.endsWith("\n")) {
886+
const records: SidekickRecord[] = [];
887+
let malformed = false;
888+
yield* forEachLine(
889+
path,
890+
{
891+
refusals: {
892+
unterminated: () =>
893+
new SidekickHistoryError(
894+
"this execution's chat history ends in a record that was never finished writing.",
895+
),
896+
},
897+
},
898+
// deno-lint-ignore require-yield
899+
function* (line): Operation<void> {
900+
// An unfinished tail takes precedence over a malformed earlier record.
901+
if (malformed) {
902+
return;
903+
}
904+
const parsed = parseSidekickRecord(line);
905+
if (parsed === undefined) {
906+
malformed = true;
907+
} else {
908+
records.push(parsed);
909+
}
910+
},
911+
);
912+
if (malformed) {
891913
throw new SidekickHistoryError(
892-
"this execution's chat history ends in a record that was never finished writing.",
914+
`this execution's chat history holds a record this version cannot read (record ` +
915+
`${records.length + 1}).`,
893916
);
894917
}
895-
const records: SidekickRecord[] = [];
896-
for (const line of body.slice(0, -1).split("\n")) {
897-
const parsed = parseSidekickRecord(line);
898-
if (parsed === undefined) {
899-
// Its text stays out of this. What is in one of these files is a
900-
// conversation somebody had.
901-
throw new SidekickHistoryError(
902-
`this execution's chat history holds a record this version cannot read (record ` +
903-
`${records.length + 1}).`,
904-
);
905-
}
906-
records.push(parsed);
907-
}
908918
for (const [index, record] of records.entries()) {
909919
if (record.kind !== "recovery-decision" && record.kind !== "human-stop") {
910920
continue;
@@ -1468,17 +1478,20 @@ const COPY_BATCH = 1 << 22;
14681478
* order and `serializeDurableEvent` preserves whatever order it is handed, so
14691479
* a round trip produces *a* valid encoding of the same history and not the
14701480
* encoding the origin's digest describes.
1471-
*
1472-
* No bound is passed. The file this reads is one `readRecords` has already
1473-
* opened under the supported limit, so a bound here would be a sentence
1474-
* nothing could reach.
14751481
*/
14761482
function* copyEntryRecords(from: string, to: string, expected: number): Operation<RetainedOrigin> {
14771483
const hash = createHash("sha256");
14781484
let bytes = 0;
14791485
let batch: string[] = [];
14801486
let batched = 0;
14811487
let parsed = 0;
1488+
function* flush(): Operation<void> {
1489+
if (batched > 0) {
1490+
yield* ReplHost.operations.appendRecord(to, batch.join(""));
1491+
batch = [];
1492+
batched = 0;
1493+
}
1494+
}
14821495
const records = yield* forEachLine(
14831496
from,
14841497
{
@@ -1507,18 +1520,19 @@ function* copyEntryRecords(from: string, to: string, expected: number): Operatio
15071520
);
15081521
}
15091522
parsed += 1;
1510-
batch.push(line, "\n");
1511-
batched += line.length + 1;
1512-
if (batched >= COPY_BATCH) {
1513-
yield* ReplHost.operations.appendRecord(to, batch.join(""));
1514-
batch = [];
1515-
batched = 0;
1523+
const size = line.length + 1;
1524+
if (batched + size > COPY_BATCH) {
1525+
yield* flush();
15161526
}
1527+
if (size > COPY_BATCH) {
1528+
yield* ReplHost.operations.appendRecord(to, line + "\n");
1529+
return;
1530+
}
1531+
batch.push(line, "\n");
1532+
batched += size;
15171533
},
15181534
);
1519-
if (batched > 0) {
1520-
yield* ReplHost.operations.appendRecord(to, batch.join(""));
1521-
}
1535+
yield* flush();
15221536
if (records !== expected) {
15231537
throw new SidekickHistoryError(
15241538
"this execution's entry history no longer holds the records this process retained, so " +

‎packages/cli/tests/repl-retained-history.test.ts‎

Lines changed: 49 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,8 @@
1414
* physical stream keep the same graph rather than one copy each.
1515
* - **Incremental reading.** Records are read a chunk at a time, split at the
1616
* newline byte and decoded per line, so multi-byte text split across a chunk
17-
* boundary reads exactly as a whole-file read did. A history past the
18-
* supported length is refused by a sentence naming the limit.
17+
* boundary reads exactly as a whole-file read did. The decoded limit applies
18+
* to each record independently.
1919
* - **Exact bytes.** The Sidekick's prefix bindings and the single-file-to-pair
2020
* upgrade measure and copy the file's own bytes through the same reader.
2121
* - **Bounded reading of a long value.** What the screen says a binding is
@@ -51,12 +51,7 @@ import type { ReplRenderer } from "../src/repl/renderer.ts";
5151
import type { ReplTerminalSize } from "../src/repl/terminal.ts";
5252
import { committedContext, useMeasuringRenderer } from "./fixtures/repl/presentation.ts";
5353
import { ReplHost } from "../src/repl/host.ts";
54-
import {
55-
HISTORY_LIMIT,
56-
openExecutionFile,
57-
readRecords,
58-
RetainedReplStream,
59-
} from "../src/repl/journal.ts";
54+
import { openExecutionFile, readRecords, RetainedReplStream } from "../src/repl/journal.ts";
6055
import { EntrySegmentStream, ROOT_IMPORT } from "../src/repl/entries.ts";
6156
import { projectRepl } from "../src/repl/model.ts";
6257
import { READ_CHUNK } from "../src/repl/record-file.ts";
@@ -420,25 +415,23 @@ describe("#992 — a history is read a record at a time", () => {
420415
);
421416
});
422417

423-
it("H3: the limit is in UTF-16 code units with newlines, and refusing it names it", function* () {
418+
it("H3: the decoded limit applies to each record, including its newline", function* () {
424419
const directory = yield* useTempDirectory("xmd-limit-");
425420
const path = join(directory, "limit.jsonl");
426-
// Three-byte characters, so a byte count and a code-unit count disagree and
427-
// a limit that confused them would refuse a file that fits.
428-
const text = line(record("one", "—".repeat(400))) + line(record("two", "x".repeat(300)));
421+
const short = line(record("one", "x".repeat(300)));
422+
const long = line(record("two", "—".repeat(400)));
423+
const text = short + long;
429424
yield* until(writeFile(path, text));
430-
expect(bytesOf(text)).toBeGreaterThan(text.length);
425+
expect(bytesOf(long)).toBeGreaterThan(long.length);
426+
expect(short.length).toBeLessThan(long.length);
431427

432-
expect(yield* readRecords(path, text.length)).toEqual(wholeFile(text));
433-
const refused = yield* refusal(() => readRecords(path, text.length - 1));
428+
expect(yield* readRecords(path, long.length)).toEqual(wholeFile(text));
429+
const refused = yield* refusal(() => readRecords(path, long.length - 1));
434430
expect(refused).toBe(
435-
`this execution's history exceeds the supported limit of ` +
436-
`${(text.length - 1).toLocaleString("en-US")} UTF-16 code units. Start a new execution ` +
437-
"with `xmd repl` for further work; this history was left unchanged.",
431+
`this execution's history holds a record that exceeds ` +
432+
`${(long.length - 1).toLocaleString("en-US")} UTF-16 code units (record 2).`,
438433
);
439-
// Refusing leaves the history exactly as it was.
440434
expect(yield* until(readFile(path, "utf8"))).toBe(text);
441-
expect(HISTORY_LIMIT).toBe(0x1fffffe8);
442435
});
443436
});
444437

@@ -461,6 +454,40 @@ function linesOf(text: string): string[] {
461454
.map((each) => `${each}\n`);
462455
}
463456

457+
describe("#992 — Sidekick histories are read one record at a time", () => {
458+
it("reads a UTF-8 character split between chunks and preserves record order", function* () {
459+
const directory = yield* useTempDirectory("xmd-sidekick-chunks-");
460+
const path = join(directory, SIDEKICK_FILE);
461+
const owner = { kind: "owner", version: 1, execution: "chat" };
462+
const ready = { kind: "ready" };
463+
const prefix = `${JSON.stringify(owner)}\n${JSON.stringify(ready)}\n`;
464+
const conversation = {
465+
kind: "conversation",
466+
conversation: "chat",
467+
phase: "prepared",
468+
agent: "MARK",
469+
};
470+
const at = prefix.length + JSON.stringify(conversation).indexOf("MARK");
471+
conversation.agent = "x".repeat(READ_CHUNK - at - 1) + "😀";
472+
yield* until(writeFile(path, `${prefix}${JSON.stringify(conversation)}\n`));
473+
expect(yield* readSidekickRecords(path)).toEqual([owner, ready, conversation]);
474+
});
475+
476+
it("preserves the refusal for an unfinished tail after a malformed record", function* () {
477+
const directory = yield* useTempDirectory("xmd-sidekick-tail-");
478+
const path = join(directory, SIDEKICK_FILE);
479+
yield* until(writeFile(path, '{"kind":"ready"}\n{"secret":"do not repeat"}\n{'));
480+
expect(yield* refusal(() => readSidekickRecords(path))).toBe(
481+
"this execution's chat history ends in a record that was never finished writing.",
482+
);
483+
484+
yield* until(appendFile(path, "}\n"));
485+
expect(yield* refusal(() => readSidekickRecords(path))).toBe(
486+
"this execution's chat history holds a record this version cannot read (record 2).",
487+
);
488+
});
489+
});
490+
464491
describe("#992 — Sidekick prefixes and the upgrade use the file's own bytes", () => {
465492
it("H4: a bound and a measured prefix are the digest of exactly those bytes", function* () {
466493
const directory = yield* useTempDirectory("xmd-prefix-");
@@ -507,7 +534,8 @@ describe("#992 — Sidekick prefixes and the upgrade use the file's own bytes",
507534
const reordered =
508535
'{"result":{"status":"ok","value":"— reordered"},"coroutineId":"root",' +
509536
'"description":{"name":"reordered","type":"test_effect"},"type":"yield"}\n';
510-
const text = longHistory() + reordered;
537+
const largeRecord = line(record("large", "x".repeat(5 * READ_CHUNK)));
538+
const text = longHistory() + largeRecord + reordered;
511539
const legacy = join(root, "long-legacy.jsonl");
512540
yield* until(writeFile(legacy, text));
513541
const repository = pairedReplRepository(root);

‎specs/repl-spec.md‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -851,10 +851,11 @@ back to the engine to continue and may perform unrecorded work. Passive Source
851851
and Journal inspection does not open execution. It preserves uncertainty when
852852
no live host observation remains and never reconstructs an old joined verdict. A location naming a history
853853
this version cannot read, or naming an entry its selected prefix never admitted,
854-
refuses whole — with nothing appended and no execution started. So does a
855-
history longer than 536,870,888 UTF-16 code units, newlines included, and its
856-
refusal names that limit. A running execution can grow its history past it;
857-
the history is left as it is, and further work starts in a new execution.
854+
refuses whole, with nothing appended and no execution started. The entry and
855+
Sidekick histories are read record by record, with no limit on their combined
856+
decoded length or on the decoded length of either file. An entry record longer
857+
than 536,870,888 UTF-16 code units including its newline refuses with the
858+
record number and the limit.
858859

859860
You may supply the include options again when reopening a location. Completed
860861
entries show their retained output and submitted entry source even after included

0 commit comments

Comments
 (0)