fix(sessions): Search Sessions searches every session and whole conversations, title matches first

Three review findings in History search:

- Each conversation was cut to its first 400,000 characters before matching, so a phrase later in
  a long or often-resumed session was never found. The whole conversation is matched now; the
  cache is bounded by the characters it holds in all (20M, least recently used out first) rather
  than by cutting each session.
- Only the newest 150 sessions were searched, title matches included, and the quick pick had no
  way to reach the rest. searchStep(query, {from, readMax}) now reads conversations a batch at a
  time (50) and returns where to go on. Its first step reports every title or preview match in
  the index with no read. search() runs every step; the quick pick runs them in the background,
  re-ranks as each lands, stops when the query changes or the pick closes, and says "no session
  mentions" only after every session was searched.
- The limit was applied while scanning, before title matches were put first, so 40 newer matches
  in the conversation pushed out an older exact title match. rankSearch ranks first, then cuts.

Tests over a store of 163 sessions: a match in the second-oldest conversation, a phrase past the
first 400,000 characters, an older title match ahead of 40 newer ones under limit 5, steps that
add up to search(); and the quick pick reaching past the first batch and dropping a superseded
query's scan. The test that pinned scanMax is replaced.

Author: ndemianc

extensions/levelcode-ai/extension.js

@@ -1649,20 +1649,36 @@ function searchSessionsQuickPick() {
 	qp.placeholder = 'Search this project\'s sessions — titles and what was said';
 	qp.matchOnDescription = false; qp.matchOnDetail = false;
 	let timer = null;
+	let gen = 0;   // bumped by every keystroke and by closing: a scan still going for an older query stops
 	const when = (iso) => { const t = Date.parse(iso || ''); return Number.isFinite(t) ? new Date(t).toLocaleDateString(undefined, { month: 'short', day: 'numeric' }) : ''; };
+	// Every session is searched, a step at a time (searchStep reads a batch of conversations), so a project
+	// with years of history never blocks the editor. The list is re-ranked as each step lands, and says
+	// "no session mentions" only once every session has been searched.
 	const run = (q) => {
-		let results = [];
-		try { results = m.search(q, { limit: 40 }); } catch (e) { results = []; }
-		dbg('sessions.search', { chars: q.length, results: results.length });
-		// alwaysShow: these items are already the matches; the quick pick's own label filter would drop the
-		// ones that matched inside the conversation rather than in the title.
-		qp.items = results.length
-			? results.map((r) => ({ label: r.title || 'Untitled', description: [when(r.updatedAt), r.lifecycle === 'archived' ? 'done' : '', r.pinned ? 'pinned' : ''].filter(Boolean).join(' · '), detail: r.snippet || '', alwaysShow: true, id: r.id }))
-			: [{ label: 'No session mentions “' + q + '”', alwaysShow: true, id: '' }];
-		qp.busy = false;
+		const mine = gen;
+		const found = [];
+		let from = 0;
+		const step = () => {
+			if (mine !== gen) { return; }
+			let r;
+			try { r = m.searchStep(q, { from }); } catch (e) { r = { matches: [], next: null }; }
+			found.push(...r.matches);
+			const done = r.next == null;
+			const results = m.rankSearch(found, 40);
+			if (done) { dbg('sessions.search', { chars: q.length, results: results.length }); }
+			// alwaysShow: these items are already the matches; the quick pick's own label filter would drop the
+			// ones that matched inside the conversation rather than in the title.
+			qp.items = results.length
+				? results.map((x) => ({ label: x.title || 'Untitled', description: [when(x.updatedAt), x.lifecycle === 'archived' ? 'done' : '', x.pinned ? 'pinned' : ''].filter(Boolean).join(' · '), detail: x.snippet || '', alwaysShow: true, id: x.id }))
+				: (done ? [{ label: 'No session mentions “' + q + '”', alwaysShow: true, id: '' }] : []);
+			qp.busy = !done;
+			if (!done) { from = r.next; setTimeout(step, 0); }
+		};
+		step();
 	};
 	qp.onDidChangeValue((v) => {
 		clearTimeout(timer);
+		gen++;
 		const q = String(v || '').trim();
 		if (q.length < 2) { qp.items = []; qp.busy = false; return; }
 		qp.busy = true;
@@ -1674,7 +1690,7 @@ function searchSessionsQuickPick() {
 		qp.hide();
 		await handleSessionAction('resume', pick.id);
 	});
-	qp.onDidHide(() => { clearTimeout(timer); qp.dispose(); });
+	qp.onDidHide(() => { clearTimeout(timer); gen++; qp.dispose(); });
 	qp.show();
 }
 

extensions/levelcode-ai/sessions.js

@@ -21,9 +21,26 @@ const planner = require('./sessionResume');
 const memory = require('./sessionMemory');
 const fs = require('fs');
 
-/** History search bounds: characters of one session kept for matching, and sessions kept in the cache. */
-const SEARCH_MAX_CHARS = 400000;
-const SEARCH_CACHE_MAX = 200;
+/** History search bounds: characters of conversation the cache holds in all (one larger session is still
+ *  searched whole, and is then the only one kept), and conversations read from disk per searchStep. */
+const SEARCH_CACHE_CHARS = 20000000;
+const SEARCH_READ_BATCH = 50;
+
+/**
+ * Search results in the order they are shown, at most `limit`: a session whose title has every word first,
+ * then the rest, each newest first. One result per session; a later one with a snippet replaces one without.
+ * Ranked BEFORE the cut, so an older title match is never pushed out by newer matches in the conversation.
+ */
+function rankSearch(results, limit) {
+	const byId = new Map();
+	for (const r of (Array.isArray(results) ? results : [])) {
+		if (!r || !r.id) { continue; }
+		const had = byId.get(r.id);
+		if (!had || (!had.snippet && r.snippet)) { byId.set(r.id, r); }
+	}
+	const out = [...byId.values()].sort((a, b) => (Number(b.inTitle) - Number(a.inTitle)) || String(b.updatedAt || '').localeCompare(String(a.updatedAt || '')));
+	return Number.isFinite(limit) && limit > 0 ? out.slice(0, limit) : out;
+}
 const imageStore = require('./imageStore');
 
 /**
@@ -388,59 +405,92 @@ function createSessions(opts) {
 	function list() { try { return store.loadIndex(root, slug).entries; } catch (e) { return []; } }
 
 	// What each session SAYS, as one lowercased string plus its readable turns, cached by file size + mtime:
-	// History search runs on every pause in typing, and a session file only grows by appends.
+	// History search runs on every pause in typing, and a session file only grows by appends. The whole
+	// conversation is kept: a phrase in its last turn must be found like one in its first. The cache is
+	// bounded by the characters it holds in all, least recently used out first.
 	const searchCache = new Map();
+	let searchCacheChars = 0;
 	function searchable(id) {
 		const file = store.sessionFile(root, slug, id);
 		let st;
 		try { st = fs.statSync(file); } catch (e) { return null; }
 		const hit = searchCache.get(file);
-		if (hit && hit.size === st.size && hit.mtimeMs === st.mtimeMs) { return hit; }
+		if (hit) { searchCache.delete(file); }
+		if (hit && hit.size === st.size && hit.mtimeMs === st.mtimeMs) { searchCache.set(file, hit); return hit; }
+		if (hit) { searchCacheChars -= hit.hay.length; }
 		let turns = [];
 		try { turns = events.toDisplayTurns(events.eventsToMessages(store.readSession(file).events)); } catch (e) { turns = []; }
-		const hay = turns.map((t) => String((t && t.text) || '')).join('\n').toLowerCase().slice(0, SEARCH_MAX_CHARS);
+		const hay = turns.map((t) => String((t && t.text) || '')).join('\n').toLowerCase();
 		const v = { size: st.size, mtimeMs: st.mtimeMs, turns, hay };
 		searchCache.set(file, v);
-		if (searchCache.size > SEARCH_CACHE_MAX) { searchCache.delete(searchCache.keys().next().value); }
+		searchCacheChars += hay.length;
+		while (searchCacheChars > SEARCH_CACHE_CHARS && searchCache.size > 1) {
+			const [k, old] = searchCache.entries().next().value;
+			searchCache.delete(k);
+			searchCacheChars -= old.hay.length;
+		}
 		return v;
 	}
 	/**
-	 * History search: the sessions whose title or conversation contains EVERY word of the query, newest first,
-	 * each with a cited snippet. Archived sessions are included (finding old work is the point); trashed ones
-	 * are not. Reads at most `scanMax` sessions' transcripts. Empty for an empty query or on error.
-	 * @returns {Array<{id:string, title:string|null, updatedAt:string, lifecycle:string, pinned:boolean, snippet:string, inTitle:boolean}>}
+	 * One step of History search, newest first. Every session's title and preview are in the index, so the
+	 * first step reports every session whose title or preview has every word, with no read. Each step then
+	 * reads the conversations of the next `readMax` sessions from `from` (cached ones cost nothing) and
+	 * reports those that have every word, in the title or what was said, each with a snippet. Pass `next`
+	 * back as `from` until it is null; rankSearch puts the steps together. Archived sessions are included
+	 * (finding old work is the point); trashed ones are not.
+	 * @returns {{ matches: Array<{id:string, title:string|null, updatedAt:string, lifecycle:string, pinned:boolean, snippet:string, inTitle:boolean}>, next: number|null }}
 	 */
-	function search(query, opts) {
+	function searchStep(query, opts) {
 		const o = opts || {};
-		const limit = Number.isFinite(o.limit) && o.limit > 0 ? o.limit : 30;
-		const scanMax = Number.isFinite(o.scanMax) && o.scanMax > 0 ? o.scanMax : 150;
+		const from = Number.isFinite(o.from) && o.from > 0 ? Math.floor(o.from) : 0;
+		const readMax = Number.isFinite(o.readMax) && o.readMax > 0 ? Math.floor(o.readMax) : SEARCH_READ_BATCH;
 		const terms = memory.queryTerms(query);
-		if (!terms.length) { return []; }
+		if (!terms.length) { return { matches: [], next: null }; }
 		const phrase = String(query || '').toLowerCase().replace(/\s+/g, ' ').trim();
-		const out = [];
+		const matches = [];
+		const head = (e) => ((e.title || '') + ' ' + (e.preview || '')).toLowerCase();
+		const result = (e, body) => ({ id: e.id, title: e.title || null, updatedAt: e.updatedAt || '', lifecycle: e.lifecycle || 'active', pinned: !!e.pinned,
+			// The snippet shows the whole phrase when the conversation has it, else the first word that is there.
+			snippet: body ? memory.snippetFor(body.turns, [phrase].concat(terms)) : '',
+			inTitle: terms.every((t) => (e.title || '').toLowerCase().indexOf(t) >= 0) });
 		try {
 			const entries = list().filter((e) => e && e.lifecycle !== 'trashed')
-				.sort((a, b) => String(b.updatedAt || '').localeCompare(String(a.updatedAt || '')))
-				.slice(0, scanMax);
-			for (const e of entries) {
-				const head = ((e.title || '') + ' ' + (e.preview || '')).toLowerCase();
+				.sort((a, b) => String(b.updatedAt || '').localeCompare(String(a.updatedAt || '')));
+			const end = Math.min(entries.length, from + readMax);
+			if (from === 0) {
+				for (let i = end; i < entries.length; i++) { const h = head(entries[i]); if (terms.every((t) => h.indexOf(t) >= 0)) { matches.push(result(entries[i], null)); } }
+			}
+			for (let i = from; i < end; i++) {
+				const e = entries[i];
 				const body = searchable(e.id);
-				const hay = head + '\n' + (body ? body.hay : '');
-				if (!terms.every((t) => hay.indexOf(t) >= 0)) { continue; }
-				// The snippet shows the whole phrase when the conversation has it, else the first word that is there.
-				const snippet = body ? (memory.snippetFor(body.turns, [phrase].concat(terms))) : '';
-				out.push({ id: e.id, title: e.title || null, updatedAt: e.updatedAt || '', lifecycle: e.lifecycle || 'active',
-					pinned: !!e.pinned, snippet, inTitle: terms.every((t) => (e.title || '').toLowerCase().indexOf(t) >= 0) });
-				if (out.length >= limit) { break; }
+				const hay = head(e) + '\n' + (body ? body.hay : '');
+				if (terms.every((t) => hay.indexOf(t) >= 0)) { matches.push(result(e, body)); }
 			}
-		} catch (e) { return out; }
-		// Title matches first, then the rest, each newest first (the scan order).
-		return out.filter((r) => r.inTitle).concat(out.filter((r) => !r.inTitle));
+			return { matches, next: end < entries.length ? end : null };
+		} catch (e) { return { matches, next: null }; }
+	}
+	/**
+	 * History search, whole: every session not in the trash whose title or conversation has EVERY word of the
+	 * query, ranked (rankSearch), at most `limit`. Reads every conversation it has not cached; the quick pick
+	 * uses searchStep instead, so a large project never blocks it. Empty for an empty query or on error.
+	 */
+	function search(query, opts) {
+		const o = opts || {};
+		const limit = Number.isFinite(o.limit) && o.limit > 0 ? o.limit : 30;
+		const all = [];
+		let from = 0;
+		for (;;) {
+			const r = searchStep(query, { from, readMax: o.readMax });
+			all.push(...r.matches);
+			if (r.next == null) { break; }
+			from = r.next;
+		}
+		return rankSearch(all, limit);
 	}
 
 	function liveId() { return live ? live.id : null; }
 
-	return { ensure, recordTurn, seal, resume, fork, archive, trash, restore, setPinned, rename, autoArchiveStale, digest, consolidate, transcript, diagrams, refineSummary, recall, recallFacts, memoryItems, forget, recordFacts, rememberNote, factsList, factAction, supersedeFact, memoryPaths, mediaRoot, sweepMedia, list, search, liveId };
+	return { ensure, recordTurn, seal, resume, fork, archive, trash, restore, setPinned, rename, autoArchiveStale, digest, consolidate, transcript, diagrams, refineSummary, recall, recallFacts, memoryItems, forget, recordFacts, rememberNote, factsList, factAction, supersedeFact, memoryPaths, mediaRoot, sweepMedia, list, search, searchStep, rankSearch, liveId };
 }
 
-module.exports = { createSessions };
+module.exports = { createSessions, rankSearch };

extensions/levelcode-ai/test/memoryNotes.test.js

@@ -233,10 +233,51 @@ test('SEARCH: a session that grows is searched as it is now', () => {
 	assert.deepStrictEqual(m.search('pool pagination').map((x) => x.id), [idPool]);
 	m.seal('done');
 });
-test('SEARCH: an empty or one-letter query finds nothing; scanMax bounds the reads', () => {
+test('SEARCH: an empty or one-letter query finds nothing; a step reads at most readMax conversations and says where to go on', () => {
 	assert.deepStrictEqual(m.search(''), []);
 	assert.deepStrictEqual(m.search('a'), []);
-	assert.strictEqual(m.search('redis', { scanMax: 1 }).length, 1);
+	assert.deepStrictEqual(m.searchStep('', {}), { matches: [], next: null });
+	const steps = []; let from = 0;
+	for (;;) { const r = m.searchStep('redis', { from, readMax: 1 }); steps.push(r); if (r.next == null) { break; } assert.strictEqual(r.next, from + 1); from = r.next; }
+	assert.strictEqual(steps.length, 3, 'three sessions not in the trash, one read a step');
+	assert.deepStrictEqual(m.rankSearch(steps.flatMap((r) => r.matches)).map((x) => x.id), m.search('redis').map((x) => x.id));
+});
+
+// A project with more history than one step reads: 160 sessions, the oldest ones holding the matches.
+const tmpBig = fs.mkdtempSync(path.join(os.tmpdir(), 'lc-search-'));
+process.on('exit', () => { try { fs.rmSync(tmpBig, { recursive: true, force: true }); } catch { /* best effort */ } });
+let tb = NOW;
+const big = createSessions({ root: tmpBig, slug: 'big', projectPath: '/big', memory: true, now: () => new Date(tb += 60000) });
+big.recordTurn(turn('Quokka migration plan', 'Planned in three steps.'), 'm'); const idQuokkaTitle = big.liveId(); big.seal('done');
+big.recordTurn(turn('Look at the old importer', 'Its retry loop is fine. The wombat flag is never read.'), 'm'); const idWombat = big.liveId(); big.seal('done');
+// one long conversation: the phrase is in its last turn, past the first 400,000 characters
+const long = []; for (let i = 0; i < 9; i++) { long.push(...turn('Step ' + i + ' of the long refactor', 'x'.repeat(50000))); }
+long.push(...turn('One more thing', 'The numbat cache is warmed at boot.'));
+big.recordTurn(long, 'm'); const idNumbat = big.liveId(); big.seal('done');
+for (let i = 0; i < 160; i++) { big.recordTurn(turn('Routine task ' + i, i < 40 ? 'Touched the quokka adapter again.' : 'Nothing notable.'), 'm'); big.seal('done'); }
+test('SEARCH: every session is searched, not only the newest 150', () => {
+	assert.deepStrictEqual(big.search('wombat').map((x) => x.id), [idWombat], 'a match in the conversation of the second-oldest of 163');
+	assert.match(big.search('wombat')[0].snippet, /wombat flag is never read/);
+});
+test('SEARCH: the whole conversation is searched, not only its first 400,000 characters', () => {
+	assert.deepStrictEqual(big.search('numbat').map((x) => x.id), [idNumbat]);
+});
+test('SEARCH: an older title match outranks newer matches in the conversation, before the limit', () => {
+	// 40 newer sessions mention quokka; the one titled with it is the oldest of all.
+	const r = big.search('quokka', { limit: 5 });
+	assert.strictEqual(r.length, 5);
+	assert.deepStrictEqual([r[0].id, r[0].inTitle], [idQuokkaTitle, true]);
+	assert.strictEqual(big.search('quokka', { limit: 100 }).length, 41);
+});
+test('SEARCH: the first step reports a title match anywhere, with no read; a later step adds its snippet', () => {
+	const first = big.searchStep('quokka migration', { readMax: 10 });
+	assert.deepStrictEqual(first.matches.map((x) => [x.id, x.snippet]), [[idQuokkaTitle, '']]);
+	assert.strictEqual(first.next, 10);
+	let r = first, all = first.matches.slice();
+	while (r.next != null) { r = big.searchStep('quokka migration', { from: r.next, readMax: 10 }); all = all.concat(r.matches); }
+	const [top] = big.rankSearch(all);
+	assert.strictEqual(top.id, idQuokkaTitle);
+	assert.match(top.snippet, /Quokka migration plan/i);
 });
 
 // ---- rememberNote, through the manager --------------------------------------------------------------

extensions/levelcode-ai/test/memoryNotesHost.test.js

@@ -251,6 +251,31 @@ const host = new Function('env', [
 		assert.deepStrictEqual(ui.resumed, [refundId]);
 		assert.strictEqual(qp.disposed, true);
 	});
+	await testAsync('SEARCH: the quick pick searches every session, a batch at a time; a new keystroke stops the older scan', async () => {
+		sessions.recordTurn([{ role: 'user', content: 'Check the platypus exporter' }, { role: 'assistant', content: 'The platypus exporter writes CSV.' }], 'm');
+		const oldId = sessions.liveId(); sessions.seal('done');
+		for (let i = 0; i < 60; i++) { sessions.recordTurn([{ role: 'user', content: 'Chore ' + i }, { role: 'assistant', content: 'Done.' }], 'm'); sessions.seal('done'); }
+		/** @type {string[]} */ const asked = [];
+		ui.manager = Object.assign({}, sessions, { searchStep: (q, o) => { asked.push(q); return sessions.searchStep(q, o); } });
+		host.searchSessionsQuickPick();
+		const qp = ui.quickPick;
+		const settled = async () => { const end = Date.now() + 3000; while (Date.now() < end) { await new Promise((r) => setTimeout(r, 20)); if (!qp.busy) { return; } } throw new Error('the search never finished'); };
+		qp._h.change('platypus exporter');
+		await settled();
+		assert.deepStrictEqual(qp.items.map((x) => x.id), [oldId], 'the oldest of 60+ sessions, past the first batch');
+		assert.match(qp.items[0].detail, /you: Check the platypus exporter/);
+		assert.ok(asked.length >= 2, 'one step read everything: ' + asked.length);
+		// type again while a scan is running: the old query's scan stops after the step it is in
+		asked.length = 0;
+		qp._h.change('chore');
+		const end = Date.now() + 3000; while (!asked.length && Date.now() < end) { await new Promise((r) => setTimeout(r, 5)); }
+		qp._h.change('platypus');
+		await settled();
+		assert.strictEqual(asked.filter((q) => q === 'chore').length, 1, 'the scan for "chore" went on after the next keystroke');
+		assert.deepStrictEqual(qp.items.map((x) => x.id), [oldId]);
+		qp.hide();
+		ui.manager = undefined;
+	});
 	await testAsync('SEARCH: no match says so, and picking that line does nothing', async () => {
 		host.searchSessionsQuickPick();
 		const qp = ui.quickPick;