Batch git.log to cap memory on large repos

Fetches commits in batches of 5,000 instead of
loading the entire history at once. Reduces peak
memory ~10x for repos with 1M+ commits (from
~600 MB to ~60 MB). Also adds cancellation
support and live commit count during loading.

Author: vdavid

src/lib/CLAUDE.md

@@ -21,7 +21,10 @@ run client-side in a Web Worker.
 - `git/clone.ts` — Clone/fetch using isomorphic-git + lightning-fs, default branch detection via
   `listServerRefs` (protocol v2), abortable HTTP wrapper for signal support, size-warning emission
   for repos >1 GB
-- `git/history.ts` — Commit log grouped by date, consecutive date generation, gap filling
+- `git/history.ts` — Commit log grouped by date, consecutive date generation, gap filling.
+  `getCommitsByDate` fetches commits in batches (via `git.log` with `depth` parameter) to bound peak
+  memory on large repos. A `seenOids` Set deduplicates commits across batches (merges can cause overlap).
+  Supports `signal` for cancellation and `onProgress` for reporting processed commit counts.
 - `git/count.ts` — Line counting per commit tree, prod/test classification, blob dedup caching.
   Files with unrecognized extensions are counted under the `'other'` bucket (with test-dir detection).
   Blob reads are parallelized with a concurrency limit of 8 using an inline `createLimiter` utility

src/lib/git/history.ts

@@ -7,37 +7,72 @@ export interface DailyCommit {
 	messages: string[];
 }
 
+const commitBatchSize = 5000;
+
 /**
  * Get the commit log for a branch and group by date.
  * Returns entries in chronological order (oldest first).
  * Keeps the latest commit hash per day but collects all messages.
+ *
+ * Fetches commits in batches to bound peak memory on large repos.
  */
 export const getCommitsByDate = async (options: {
 	fs: FsClient;
 	dir: string;
 	ref: string;
 	gitCache?: object;
+	signal?: AbortSignal;
+	onProgress?: (processedCommits: number) => void;
 }): Promise<DailyCommit[]> => {
-	const { fs, dir, ref, gitCache } = options;
-
-	const commits: ReadCommitResult[] = await git.log({ fs, dir, ref, cache: gitCache });
+	const { fs, dir, ref, gitCache, signal, onProgress } = options;
 
-	// git.log returns newest-first; group by date
 	const byDate = new Map<string, DailyCommit>();
+	const seenOids = new Set<string>();
+	let currentRef: string = ref;
+	let totalProcessed = 0;
+
+	while (true) {
+		if (signal?.aborted) throw new Error('Cancelled');
+
+		const batch: ReadCommitResult[] = await git.log({
+			fs,
+			dir,
+			ref: currentRef,
+			depth: commitBatchSize,
+			cache: gitCache
+		});
+
+		if (batch.length === 0) break;
 
-	for (const commit of commits) {
-		const date = formatDate(commit.commit.author.timestamp);
-		const existing = byDate.get(date);
-		if (!existing) {
-			// First commit for this date (latest chronologically since log is reverse order)
-			byDate.set(date, {
-				date,
-				hash: commit.oid,
-				messages: [commit.commit.message.trim()]
-			});
-		} else {
-			existing.messages.push(commit.commit.message.trim());
+		for (const commit of batch) {
+			if (seenOids.has(commit.oid)) continue;
+			seenOids.add(commit.oid);
+			totalProcessed++;
+
+			const date = formatDate(commit.commit.author.timestamp);
+			const existing = byDate.get(date);
+			if (!existing) {
+				byDate.set(date, {
+					date,
+					hash: commit.oid,
+					messages: [commit.commit.message.trim()]
+				});
+			} else {
+				existing.messages.push(commit.commit.message.trim());
+			}
 		}
+
+		onProgress?.(totalProcessed);
+
+		// Stop if this batch was smaller than requested (end of history)
+		if (batch.length < commitBatchSize) break;
+
+		// Stop if the oldest commit has no parents (root commit)
+		const oldest = batch[batch.length - 1];
+		if (oldest.commit.parent.length === 0) break;
+
+		// Continue from the first parent of the oldest commit
+		currentRef = oldest.commit.parent[0];
 	}
 
 	// Sort chronologically (oldest first)

src/lib/worker/analyzer.worker.ts

@@ -129,7 +129,21 @@ const analyzerApi = {
 			// Step 3: Get commit history grouped by date
 			onProgress({ type: 'process', current: 0, total: 0, date: 'Loading history...' });
 			const gitCache = {};
-			const dailyCommits = await getCommitsByDate({ fs, dir, ref: defaultBranch, gitCache });
+			const dailyCommits = await getCommitsByDate({
+				fs,
+				dir,
+				ref: defaultBranch,
+				gitCache,
+				signal,
+				onProgress: (processed) => {
+					onProgress({
+						type: 'process',
+						current: 0,
+						total: 0,
+						date: `Loading history... (${processed.toLocaleString()} commits)`
+					});
+				}
+			});
 
 			if (dailyCommits.length === 0) {
 				throw new Error('No commits found in repository');
@@ -282,7 +296,21 @@ const analyzerApi = {
 			// Step 2: Get full commit history
 			onProgress({ type: 'process', current: 0, total: 0, date: 'Loading history...' });
 			const gitCache = {};
-			const dailyCommits = await getCommitsByDate({ fs, dir, ref: defaultBranch, gitCache });
+			const dailyCommits = await getCommitsByDate({
+				fs,
+				dir,
+				ref: defaultBranch,
+				gitCache,
+				signal,
+				onProgress: (processed) => {
+					onProgress({
+						type: 'process',
+						current: 0,
+						total: 0,
+						date: `Loading history... (${processed.toLocaleString()} commits)`
+					});
+				}
+			});
 
 			if (dailyCommits.length === 0) {
 				throw new Error('No commits found in repository');