Skip to content

Commit b4e514e

Browse files
Prithvi-Webclaude
andcommitted
fix(query): a node of the query walk lets go of its folder once its own path is built - so a path term down a deep chain no longer keeps every folder above, and the whole copy of its path the term read, alive; and a test holds the deep hit's walk to a few hundred bytes per folder, so a walk that holds the depth squared fails on every machine, not only on CI's 2 GB heap (CI4)
The second way the query walk held the square of a deep chain's depth. QueryNode kept its folder after its path was built, though the folder is only there to build that path. A path term reads every node's path, which copies it whole; each node then held its folder, which held its folder, up to the root - every one with a whole copy of its path. With the store fixed (the commit before), `path:leaf` down 10,000 folders still held 291 MB; with the folder let go it holds 0.2 MB at 30,000. The guard (tests/queryWalkWork.test.ts, "CI4: a deep hit costs memory in proportion to its depth, not its square"): down an 8,000-deep chain, `name:leaf` and `path:leaf` each hold at most 2 KiB per folder at the moment the deep path is built (the heap after a full collection, tests/fixtures/heapProbe.ts; measured 243 and 16 bytes per folder - a whole copy of every folder's path is 23 KB). The deep path is still built once, from its folder's, without a stack frame per folder, and is exactly right. The F1.2 chain builder is shared with it, unchanged. Mutants, each RED for the right reason, restored and checked by SHA-256: the folder kept (path:leaf: 184.7 MB held, 23091 bytes per folder); the store's endsWith rule back (name:leaf: 184.8 MB, 23099 bytes per folder); the path built by store.path() instead of from its folder's (the deep path is built once, from its folder's: 0 builds - the probe never measured). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent 35e14ba commit b4e514e

2 files changed

Lines changed: 77 additions & 10 deletions

File tree

‎src/services/query/execute.ts‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -417,7 +417,14 @@ class QueryNode implements EvalNode {
417417
constructor(
418418
private readonly store: ScanStore,
419419
readonly id: number,
420-
private readonly folder: QueryNode | null,
420+
/**
421+
* Held only to build this node's path, and let go once it is built (CI4):
422+
* kept after, a node held every folder above it alive, and each folder's
423+
* path with it — a whole copy once a path term had read it — so a walk
424+
* down a deep chain held the square of its depth. Null for the root, and
425+
* for a node whose path is known.
426+
*/
427+
private folder: QueryNode | null,
421428
readonly depth: number,
422429
childlessKnown: boolean,
423430
) {
@@ -449,6 +456,7 @@ class QueryNode implements EvalNode {
449456
for (let i = unbuilt.length - 1; i >= 0; i--) {
450457
const node = unbuilt[i];
451458
path = node.pathRead = this.store.childPath(node.id, path);
459+
node.folder = null;
452460
}
453461
return path;
454462
}

‎tests/queryWalkWork.test.ts‎

Lines changed: 68 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import { createScanRecord } from '../src/services/diskScanner';
1313
import { buildStoreFromTree, Flag, PackedScanStore, type ScanStore } from '../src/services/scanStore';
1414
import type { FileNode, ScanResult } from '../src/models/types';
1515
import type { Ast } from '../src/services/query/types';
16+
import { collector, heldBytes } from './fixtures/heapProbe';
1617

1718
/**
1819
* The query walk's work, counted (design pass G07).
@@ -268,27 +269,85 @@ test('G07: a tree that changes during a slice\'s turn is walked again from the s
268269
assert.ok(!out.hits.some((h) => h.path === victim), 'the deleted file is not among the hits');
269270
});
270271

271-
test('G07 review F1.2: a hit at the bottom of a 30k-deep chain is answered, its path built without a stack frame per folder', async () => {
272-
// The store's own standing case (tests/packedStore.test.ts: a 30k-deep
273-
// chain, nothing recurses on pathological depth). A query whose terms read
274-
// no path during the walk (name:, ext:, size) leaves every folder's path
275-
// unbuilt, so the first path read is the deep hit's, for the page.
276-
const DEPTH = 30_000;
272+
/** A completed scan of `depth` folders, one inside the next under /deep, with one file at the bottom. */
273+
function deepChain(depth: number): { scan: ScanResult; store: PackedScanStore; leaf: number; leafPath: string } {
277274
const store = new PackedScanStore('/deep', '/', { name: 'deep', isDir: true, size: 0, modifiedAt: 1, isHidden: false });
278275
let parent = store.rootId;
279-
for (let i = 0; i < DEPTH; i++) parent = store.addNode(parent, { name: `d${i}`, isDir: true, size: 0, modifiedAt: 1, isHidden: false });
276+
for (let i = 0; i < depth; i++) parent = store.addNode(parent, { name: `d${i}`, isDir: true, size: 0, modifiedAt: 1, isHidden: false });
280277
store.addNode(parent, { name: 'leaf.bin', isDir: false, size: 7, modifiedAt: 1, isHidden: false, extension: 'bin' });
281278
store.finalize();
282279
store.sumSizes();
283280
const scan = createScanRecord('/deep');
284281
scan.store = store;
285282
scan.status = 'complete';
286283
scan.engine = 'walker';
284+
// Found by walking down after finalize, which may renumber.
285+
let leaf = store.rootId;
286+
while (store.isDir(leaf)) leaf = store.childIds(leaf)[0];
287287
let deepest = '/deep';
288-
for (let i = 0; i < DEPTH; i++) deepest += `/d${i}`;
288+
for (let i = 0; i < depth; i++) deepest += `/d${i}`;
289+
return { scan, store, leaf, leafPath: `${deepest}/leaf.bin` };
290+
}
291+
292+
test('G07 review F1.2: a hit at the bottom of a 30k-deep chain is answered, its path built without a stack frame per folder', async () => {
293+
// The store's own standing case (tests/packedStore.test.ts: a 30k-deep
294+
// chain, nothing recurses on pathological depth). A query whose terms read
295+
// no path during the walk (name:, ext:, size) leaves every folder's path
296+
// unbuilt, so the first path read is the deep hit's, for the page.
297+
const DEPTH = 30_000;
298+
const { scan, leafPath } = deepChain(DEPTH);
289299
for (const q of ['name:leaf', 'ext:bin', 'size>5 type:file']) {
290300
const out = await run(scan, q);
291301
assert.equal(out.total, 1, `${q}: the one file matches`);
292-
assert.equal(out.hits[0].path, `${deepest}/leaf.bin`, `${q}: its path, all ${DEPTH} folders of it`);
302+
assert.equal(out.hits[0].path, leafPath, `${q}: its path, all ${DEPTH} folders of it`);
303+
}
304+
});
305+
306+
/**
307+
* What a deep hit's path may hold while it is built: a few hundred bytes per
308+
* folder on its way down (the node the walk holds, and the folder's share of
309+
* a path joined from its folder's) — never a whole copy of every folder's
310+
* path, which costs half the deep path's length per folder: 26 KB at 8,000
311+
* folders deep.
312+
*/
313+
const HELD_PER_FOLDER = 2048;
314+
315+
test('CI4: a deep hit costs memory in proportion to its depth, not its square — with a path term or without, no folder on its way down keeps a whole copy of its path', async () => {
316+
// CI run 39095045009: the 30k-deep chain above ran macOS's 2 GB heap out
317+
// (it passed on this Mac's 4 GB). Each folder's path, built from its
318+
// folder's, was read whole as the next was joined (the store's childPath
319+
// asked whether it ended with the separator), so every folder kept a flat
320+
// copy of its own path: the depth squared, about 2.7 GB at 30k. A path
321+
// term reads every node's path whole anyway, and a folder kept reachable
322+
// from below after its path was built then held that copy alive.
323+
// Measured where the deep path is built: the most the walk holds at once.
324+
const DEPTH = 8_000;
325+
const gc = collector();
326+
for (const q of ['name:leaf', 'path:leaf']) {
327+
const { scan, store, leaf, leafPath } = deepChain(DEPTH);
328+
await run(scan, q); // whatever the first query loads is not the walk's
329+
let probes = 0;
330+
let held = -1;
331+
const base = heldBytes(gc);
332+
scan.store = new Proxy(store, {
333+
get(target, prop, receiver) {
334+
const value = Reflect.get(target, prop, receiver);
335+
if (typeof value !== 'function') return value;
336+
if (prop !== 'childPath') return (value as (...a: unknown[]) => unknown).bind(target);
337+
return (child: number, parentPath: string): string => {
338+
const built = (value as (c: number, p: string) => string).call(target, child, parentPath);
339+
if (child === leaf) { probes++; held = heldBytes(gc) - base; }
340+
return built;
341+
};
342+
},
343+
}) as ScanStore;
344+
const out = await run(scan, q);
345+
assert.equal(out.total, 1, `${q}: the one file matches`);
346+
assert.equal(out.hits[0].path, leafPath, `${q}: its path, all ${DEPTH} folders of it`);
347+
assert.equal(probes, 1, `${q}: the deep path is built once, from its folder's`);
348+
assert.ok(
349+
held < HELD_PER_FOLDER * DEPTH,
350+
`${q}: ${(held / 1e6).toFixed(1)} MB held as the deep path is built, ${Math.round(held / DEPTH)} bytes per folder of ${DEPTH} (at most ${HELD_PER_FOLDER}; a whole copy of every folder's path is about ${Math.round(leafPath.length / 2)})`,
351+
);
293352
}
294353
});

0 commit comments

Comments
 (0)