Merge pull request #271 from closedloop-ai/feat/fea-1554

FEA-1554: Close session data collection gaps across all 5 harness parsers

Author: mikeangstadt

apps/desktop/package.json

@@ -1,6 +1,6 @@
 {
   "name": "desktop",
-  "version": "0.15.114",
+  "version": "0.15.115",
   "description": "ClosedLoop Desktop",
   "author": "ClosedLoop AI <support@closedloop.ai>",
   "private": true,

apps/desktop/src/main/agent-session-sync-service.ts

@@ -833,7 +833,11 @@ export function sanitizeSessionForSync(
   };
 }
 
-const STRIPPED_LEAF_KEYS = new Set(["prompt", "content", "stdout", "stderr"]);
+const STRIPPED_LEAF_KEYS = new Set([
+  "prompt", "content", "stdout", "stderr",
+  "text", "output", "reasoning",
+  "old_string", "new_string", "patch", "command", "arguments",
+]);
 
 function stripDataContent(data: SyncJsonValue | undefined): SyncJsonValue | undefined {
   if (data === undefined || data === null) {

apps/desktop/src/main/collectors/claude/claude-parser.ts

@@ -12,12 +12,16 @@ import path from "node:path";
 import readline from "node:readline";
 import type {
   NormalizedApiError,
+  NormalizedDiffStats,
+  NormalizedMessage,
   NormalizedSession,
   NormalizedTokenCounts,
+  NormalizedTokenRecord,
   NormalizedToolResultError,
   NormalizedToolUse,
   NormalizedTurnDuration,
 } from "../types.js";
+import { truncateText, computeLineDelta, collectArtifacts } from "../parser-utils.js";
 
 /** Mirror the vendor's lenient timestamp handling: epoch number → ISO, string as-is. */
 function isoTs(ts: unknown): string | null {
@@ -72,6 +76,19 @@ export async function parseSessionFile(filePath: string): Promise<NormalizedSess
   const speeds = new Set<string>();
   const inferenceGeos = new Set<string>();
 
+  // CR-1: ordered messages
+  const messages: NormalizedMessage[] = [];
+  // CR-2: per-turn token time-series
+  const tokenSeries: NormalizedTokenRecord[] = [];
+  // CR-4: aggregate diff stats
+  let totalAdded = 0;
+  let totalRemoved = 0;
+  const diffFiles = new Set<string>();
+  // CR-7: slash commands
+  const slashCommands: Array<{ name: string; timestamp: string }> = [];
+  // CR-3: map tool_use_id → index in toolUses for back-linking tool results
+  const toolUseIdIndex = new Map<string, number>();
+
   try {
     for await (const line of rl) {
       if (!line.trim()) continue;
@@ -142,6 +159,52 @@ export async function parseSessionFile(filePath: string): Promise<NormalizedSess
 
       if (entry.type === "user") {
         userMessageCount++;
+
+        // CR-1: Build NormalizedMessage for user messages.
+        const userMsg = asRecord(entry.message);
+        const userContent = Array.isArray(userMsg.content) ? userMsg.content : [];
+        const userTextParts: string[] = [];
+        for (const raw of userContent) {
+          const block = asRecord(raw);
+          if (block.type === "text" && typeof block.text === "string") {
+            userTextParts.push(block.text);
+          }
+          // CR-3: Capture tool_result content and back-link to the originating tool_use.
+          if (block.type === "tool_result" && typeof block.tool_use_id === "string") {
+            const resultContent = Array.isArray(block.content) ? block.content : [];
+            const resultTextParts: string[] = [];
+            for (const rc of resultContent) {
+              const rcBlock = asRecord(rc);
+              if (typeof rcBlock.text === "string") resultTextParts.push(rcBlock.text);
+            }
+            // Also handle string content directly
+            if (typeof block.content === "string") resultTextParts.push(block.content);
+            const resultText = resultTextParts.join("\n");
+            const tuIdx = toolUseIdIndex.get(block.tool_use_id as string);
+            if (tuIdx !== undefined && toolUses[tuIdx]) {
+              toolUses[tuIdx].output = truncateText(resultText);
+              if (block.is_error) toolUses[tuIdx].isError = true;
+            }
+          }
+        }
+        const userTextJoined = userTextParts.join("\n");
+        messages.push({
+          role: "human",
+          timestamp: isoTs(entry.timestamp),
+          text: truncateText(userTextJoined) || null,
+        });
+
+        // CR-7: Scan user message text for <command-name> XML tags (slash commands).
+        const cmdRe = /<command-name>([^<]+)<\/command-name>/g;
+        let cmdMatch: RegExpExecArray | null;
+        const entryIso = isoTs(entry.timestamp);
+        while ((cmdMatch = cmdRe.exec(userTextJoined)) !== null) {
+          if (entryIso) {
+            slashCommands.push({ name: cmdMatch[1].trim(), timestamp: entryIso });
+          }
+        }
+
+        // Existing: toolUseResult error tracking (top-level shorthand).
         const toolUseResult = entry.toolUseResult;
         if (toolUseResult && typeof toolUseResult === "object") {
           const tur = toolUseResult as Record<string, unknown>;
@@ -171,6 +234,18 @@ export async function parseSessionFile(filePath: string): Promise<NormalizedSess
           tokensByModel[msgModel].output += num(usage.output_tokens);
           tokensByModel[msgModel].cacheRead += num(usage.cache_read_input_tokens);
           tokensByModel[msgModel].cacheWrite += num(usage.cache_creation_input_tokens);
+
+          // CR-2: Push per-turn token record for time-series.
+          if (iso) {
+            tokenSeries.push({
+              timestamp: iso,
+              model: msgModel,
+              input: num(usage.input_tokens),
+              output: num(usage.output_tokens),
+              cacheRead: num(usage.cache_read_input_tokens),
+              cacheWrite: num(usage.cache_creation_input_tokens),
+            });
+          }
         }
         if (msg.usage) {
           if (typeof usage.service_tier === "string") serviceTiers.add(usage.service_tier);
@@ -183,17 +258,90 @@ export async function parseSessionFile(filePath: string): Promise<NormalizedSess
           }
         }
         const content = msg.content;
+        // CR-1: Collect text blocks for the assistant NormalizedMessage.
+        const assistantTextParts: string[] = [];
         if (Array.isArray(content)) {
           for (const raw of content) {
             const block = asRecord(raw);
+            if (block.type === "text" && typeof block.text === "string") {
+              assistantTextParts.push(block.text);
+            }
             if (block.type === "tool_use" && typeof block.name === "string") {
-              toolUses.push({
-                name: block.name,
+              const toolName = block.name as string;
+              const toolInput = block.input ?? null;
+              const tu: NormalizedToolUse = {
+                name: toolName,
                 timestamp: iso || firstTimestamp,
-                input: block.input ?? null,
+                input: toolInput,
+              };
+
+              // CR-8: Extract skill name from Skill tool.
+              if (toolName === "Skill") {
+                const inp = asRecord(toolInput);
+                if (typeof inp.skill === "string") tu.skillName = inp.skill;
+              }
+
+              // CR-4: Compute diffDelta for Edit and Write tool uses.
+              if (toolName === "Edit") {
+                const inp = asRecord(toolInput);
+                const oldStr = typeof inp.old_string === "string" ? inp.old_string : null;
+                const newStr = typeof inp.new_string === "string" ? inp.new_string : null;
+                tu.diffDelta = computeLineDelta(oldStr, newStr);
+                totalAdded += tu.diffDelta.add;
+                totalRemoved += tu.diffDelta.del;
+                if (typeof inp.file_path === "string") diffFiles.add(inp.file_path);
+              }
+              if (toolName === "Write") {
+                const inp = asRecord(toolInput);
+                const fileContent = typeof inp.content === "string" ? inp.content : "";
+                const addLines = fileContent.split("\n").length;
+                tu.diffDelta = { add: addLines, del: 0 };
+                totalAdded += addLines;
+                if (typeof inp.file_path === "string") diffFiles.add(inp.file_path);
+              }
+
+              // CR-3: Track tool_use_id for back-linking tool results.
+              if (typeof block.id === "string") {
+                toolUseIdIndex.set(block.id as string, toolUses.length);
+              }
+              toolUses.push(tu);
+            }
+            if (block.type === "thinking") {
+              thinkingBlockCount++;
+              // CR-1: Emit a NormalizedMessage for thinking blocks (text redacted).
+              messages.push({
+                role: "assistant",
+                timestamp: iso,
+                text: null,
+                model: msgModel,
+                isThinking: true,
               });
             }
-            if (block.type === "thinking") thinkingBlockCount++;
+          }
+        }
+        // CR-1: Build main assistant NormalizedMessage.
+        const assistantText = assistantTextParts.join("\n");
+        messages.push({
+          role: "assistant",
+          timestamp: iso,
+          text: truncateText(assistantText) || null,
+          model: msgModel,
+          tokens: msg.usage
+            ? {
+                input: num(usage.input_tokens),
+                output: num(usage.output_tokens),
+                cacheRead: num(usage.cache_read_input_tokens),
+                cacheWrite: num(usage.cache_creation_input_tokens),
+              }
+            : undefined,
+        });
+
+        // CR-7: Scan assistant text for <command-name> tags too.
+        const cmdRe = /<command-name>([^<]+)<\/command-name>/g;
+        let cmdMatch: RegExpExecArray | null;
+        while ((cmdMatch = cmdRe.exec(assistantText)) !== null) {
+          if (iso) {
+            slashCommands.push({ name: cmdMatch[1].trim(), timestamp: iso });
           }
         }
       }
@@ -218,6 +366,15 @@ export async function parseSessionFile(filePath: string): Promise<NormalizedSess
     /* non-fatal */
   }
 
+  // CR-4: Build aggregate diffStats (null when no edits were made).
+  const diffStats: NormalizedDiffStats | null =
+    diffFiles.size > 0
+      ? { filesChanged: diffFiles.size, linesAdded: totalAdded, linesRemoved: totalRemoved }
+      : null;
+
+  // CR-13: Collect artifact references from tool uses.
+  const artifacts = collectArtifacts(toolUses, cwd);
+
   return {
     sessionId,
     name: sessionName,
@@ -247,5 +404,15 @@ export async function parseSessionFile(filePath: string): Promise<NormalizedSess
       speeds: [...speeds],
       inference_geos: [...inferenceGeos],
     },
+    // CR-1: Ordered messages with text content.
+    messages,
+    // CR-2: Per-turn token time-series.
+    tokenSeries,
+    // CR-4: Aggregate diff stats.
+    diffStats,
+    // CR-7: Extracted slash commands.
+    slashCommands,
+    // CR-13: Structured artifact references.
+    artifacts,
   };
 }