feat: enhance diagnostics handling and output streaming

- Introduced a diagnostics collector to track unexpected issues during runs, including rate limits and provider errors.
- Updated the session management to feed agent output into the diagnostics collector, ensuring no loss of critical information.
- Enhanced the watcher spawning logic to support output streaming, allowing real-time diagnostics feedback.
- Modified the run and stats routes to include diagnostics in the response, improving user visibility on issues.
- Added a new status meta file to centralize status definitions and improve consistency across the application.
- Updated the frontend to display diagnostics in the history and matrix views, providing users with immediate feedback on potential issues.
- Improved error handling in tests to strip ANSI codes for better readability.

Author: dkalinovInfra

.githooks/pre-commit

@@ -1,4 +1,5 @@
 #!/bin/sh
-# Pre-commit gate: block commits that don't type-check.
+# Pre-commit gate: block commits that don't type-check or fail the diagnostics fixtures.
 # One-time setup per clone:  git config core.hooksPath .githooks
-exec npm run --silent typecheck
+npm run --silent typecheck || exit 1
+exec npm run --silent diagnostics:test

CLAUDE.md

@@ -7,6 +7,9 @@
   "scripts": {
     "start": "node src/server.ts",
     "typecheck": "tsc -p tsconfig.json && tsc -p web/tsconfig.json",
+    "diagnostics:test": "node scripts/diagnostics-fixtures.mjs && node scripts/loop-store-check.mjs",
+    "diagnostics:replay": "node scripts/replay-diagnostics.mjs",
+    "exec:smoke": "node scripts/exec-smoke.mjs",
     "build:web": "esbuild web/main.ts --bundle --format=esm --outfile=public/vendor/app.js",
     "dev:web": "esbuild web/main.ts --bundle --format=esm --outfile=public/vendor/app.js --watch"
   },

package.json

@@ -244,7 +244,32 @@ igc-grid.runs-grid igc-grid-toolbar-exporter {
 .pill.success { background:rgba(43,179,104,.15); color:var(--green); }
 .pill.error { background:rgba(224,106,85,.16); color:var(--red); }
 .pill.build-error { background:rgba(202,162,60,.18); color:var(--amber); }
-.pill.rate-limited { background:rgba(202,162,60,.18); color:var(--amber); }
+/* Provider-side outcomes: the run didn't fail, the provider did — amber, not red, so
+   they read as "not your code" at a glance. Kept in sync with src/status-meta.ts. */
+.pill.rate-limited,
+.pill.provider-down,
+.pill.no-credits,
+.pill.auth,
+.pill.timed-out { background:rgba(202,162,60,.18); color:var(--amber); }
+.pill.other { background:rgba(142,166,164,.15); color:var(--steel); }
+
+/* Diagnostics. A resolved one is good news and a superseded one is not, so they must
+   not look alike — muted-green vs muted-grey with a strikethrough title. */
+.diag { border-left:3px solid var(--amber); padding:.35rem .6rem; margin:.35rem 0; background:rgba(202,162,60,.07); }
+.diag .diag-title { color:var(--amber); font-weight:600; }
+.diag .diag-advice { color:var(--ink); margin-top:.15rem; }
+.diag .diag-detail { color:var(--steel); font-family:var(--mono); font-size:.72rem; margin-top:.25rem; overflow-wrap:anywhere; white-space:pre-wrap; }
+.diag .diag-meta { color:var(--steel); font-size:.7rem; margin-top:.2rem; }
+.diag.suspected { border-left-style:dashed; }
+.diag.resolved { border-left-color:var(--green); background:rgba(43,179,104,.06); opacity:.75; }
+.diag.resolved .diag-title { color:var(--green); }
+.diag.superseded { border-left-color:var(--steel); background:transparent; opacity:.6; }
+.diag.superseded .diag-title { color:var(--steel); text-decoration:line-through; }
+.diag-chip { color:var(--amber); margin-left:.35rem; }
+
+/* Matrix-level aggregate banner. */
+.mx-banner { border:1px solid rgba(202,162,60,.5); background:rgba(202,162,60,.12); color:var(--amber);
+  padding:.5rem .75rem; border-radius:6px; margin:.5rem 0; font-size:.8rem; }
 .pill.test-failed { background:rgba(224,106,85,.16); color:var(--red); }
 .pill.running { background:rgba(202,162,60,.16); color:var(--amber); }
 .detail { display:grid; grid-template-columns:repeat(3,minmax(0,1fr)); gap:1rem 1.6rem;

public/css/app.css

@@ -167,6 +167,11 @@ $envFlags = @()
 foreach ($v in $keyVars) {
   if (Test-Path "env:$v") { $envFlags += @('-e', $v) }
 }
+# Non-secret tunables: diagnostics tunables (src/config.ts): forwarded when set so a run can be tuned, and so DIAGNOSTICS_STREAM_DEBUG=1 can be used to answer which stream provider errors actually arrive on. Values are plain numbers/flags, not secrets.
+$diagVars = @('DIAGNOSTICS_STREAM_DEBUG', 'AGENT_STALL_MS', 'AGENT_LOOP_REPEATS', 'DIAGNOSTIC_AGGREGATE_THRESHOLD', 'AGENT_TIMEOUT_MS')
+foreach ($v in $diagVars) {
+  if (Test-Path "env:$v") { $envFlags += @('-e', "$v=$((Get-Item "env:$v").Value)") }
+}
 if ($mcAbs) { $envFlags += @('-e', 'MATRIX_CONFIG=/matrix-config.json') }
 if ($Validate) { $envFlags += @('-e', 'MATRIX_VALIDATE=1') }
 

run.ps1

@@ -147,7 +147,12 @@ KEY_VARS=(ANTHROPIC_API_KEY OPENAI_API_KEY OPENROUTER_API_KEY GOOGLE_GENERATIVE_
 # A matrix config may name a custom env var to hold its key via "apiKeyEnv"; forward
 # that var too (else loadMatrixConfig can't resolve it and the run goes out keyless).
 if [[ -n "$MATRIX_CONFIG_FILE" ]]; then
-  CUSTOM_KEY_ENV="$(grep -oE '"apiKeyEnv"[[:space:]]*:[[:space:]]*"[^"]+"' "$MATRIX_CONFIG_FILE" | head -1 | sed -E 's/.*:[[:space:]]*"([^"]+)".*/\1/')"
+  # `|| true` is load-bearing: apiKeyEnv is OPTIONAL, and under `set -euo pipefail` a
+  # grep that matches nothing exits 1, pipefail propagates it out of the substitution
+  # and the script dies right here with no output at all. Every committed example
+  # config happens to carry the field, which is why this hid — a config that omits it
+  # (entirely legal: apiKey > apiKeyEnv > PROVIDER_ENV) made run.sh exit 1 silently.
+  CUSTOM_KEY_ENV="$(grep -oE '"apiKeyEnv"[[:space:]]*:[[:space:]]*"[^"]+"' "$MATRIX_CONFIG_FILE" | head -1 | sed -E 's/.*:[[:space:]]*"([^"]+)".*/\1/' || true)"
   if [[ -n "$CUSTOM_KEY_ENV" ]]; then
     if [[ "$CUSTOM_KEY_ENV" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]]; then
       dup=0; for v in "${KEY_VARS[@]}"; do [[ "$v" == "$CUSTOM_KEY_ENV" ]] && dup=1; done
@@ -163,6 +168,11 @@ ENVFLAGS=()
 for v in "${KEY_VARS[@]}"; do
   [[ -n "${!v:-}" ]] && ENVFLAGS+=(-e "$v")
 done
+# Non-secret tunables: diagnostics tunables (src/config.ts): forwarded when set so a run can be tuned, and so DIAGNOSTICS_STREAM_DEBUG=1 can be used to answer which stream provider errors actually arrive on. Values are plain numbers/flags, not secrets.
+DIAG_VARS=(DIAGNOSTICS_STREAM_DEBUG AGENT_STALL_MS AGENT_LOOP_REPEATS DIAGNOSTIC_AGGREGATE_THRESHOLD AGENT_TIMEOUT_MS)
+for v in "${DIAG_VARS[@]}"; do
+  [[ -n "${!v:-}" ]] && ENVFLAGS+=(-e "$v=${!v}")
+done
 [[ -n "$MATRIX_CONFIG_FILE" ]] && ENVFLAGS+=(-e "MATRIX_CONFIG=/matrix-config.json")
 [[ "$VALIDATE" == 1 ]] && ENVFLAGS+=(-e "MATRIX_VALIDATE=1")