fix(agent-teams): keep long-running teams recoverable (#1426)

Long Agent Teams runs lost members for good: a truncated provider stream
ended a member's turn with nobody to wake it, the desktop Stop button and
every lead restart killed all members and marked the plan interrupted,
mail sent to a stopped member landed in an inbox nothing read, and a lead
kept inside one long turn never saw member reports. Aligned with the
official CLI 2.1.284 and verified with DeepSeek Flash through a
fault-injecting proxy.

Stream recovery
- Re-send a stream that breaks before any tool ran (proxy truncation,
  transport errors), with the existing retry budget and backoff; the
  desktop drops the discarded attempt's tool cards and todo update.

Desktop team runtime (teamPlanRuntime)
- The server supervises members: a stopped member restarts from its own
  transcript when messaged; transient failures continue automatically
  (15s/45s/2m/5m/10m) and only exhausted retries reach the lead; ready
  dependent tasks wake their owner; a crash-loop guard ignores user stops.
- Stop pauses the team instead of ending it; the lead's next user message
  is followed by a notice listing the stopped members and their open
  tasks. Lead restarts (model/permission switch, crash) keep members;
  server restarts re-own the team. Teams end on /clear or session delete.
- Approving a plan no longer races a concurrent plan read into
  "Launch ownership was lost".

Mailbox and messaging
- Atomic inbox writes, identity-based read marking, read history files,
  idle notifications with result/failureReason, and write failures
  reported instead of "Message sent". External builds keep the official
  between-turn delivery to the lead.
- SendMessage resumes non-running in-process teammates from their
  transcript, notes restarting desktop members, queues mail for members
  of a plan awaiting approval, and rejects unknown names.

CLI in-process teammates
- Compaction uses the teammate's own controller and real history and no
  longer kills it on error; failed turns are classified and continued;
  the turn-end mailbox drains as one batch; one durable transcript per
  teammate.

Lead behaviour
- An unmet /goal ends the lead turn while members work, so member reports
  arrive; WaitSessions on own team members returns immediately.

Desktop UI
- Member states for stopped, auto-retrying and failed, with reason,
  countdown and recovery hint in all five locales.

Tests and tooling
- Regression tests for every behaviour above; module mocks in four test
  files are restored after use so the single-process coverage run is not
  polluted; the desktop smoke asserts the new Stop semantics.

Author: NanmiCoder

desktop/src/components/agentTeams/AgentTeamsCanvas.test.tsx

@@ -401,4 +401,49 @@ describe('AgentTeamsCanvas', () => {
     const lead = screen.queryByTestId('agent-teams-canvas-member-model-team-lead@canvas-team')
     expect(lead).toBeNull()
   })
+
+  it('labels stopped, retrying and failed members and keeps their failure on hover', () => {
+    const current = snapshot('current')
+    const recovering: TeamWorkbenchSnapshot = {
+      ...current,
+      team: {
+        ...current.team,
+        members: current.team.members.map(member => (
+          member.agentId === 'builder@canvas-team'
+            ? {
+                ...member,
+                status: 'idle' as const,
+                activity: 'idle' as const,
+                lastError: 'API Error: 529 overloaded',
+                autoRetry: { attempt: 2, max: 5, nextAt: Date.parse('2026-08-12T02:30:45.000Z') },
+              }
+            : member.agentId === 'reviewer@canvas-team'
+              ? { ...member, status: 'idle' as const, activity: 'stopped' as const }
+              : member.agentId === 'qa@canvas-team'
+                ? { ...member, status: 'error' as const, activity: 'idle' as const, lastError: 'Credit balance is too low' }
+                : member
+        )),
+      },
+    }
+
+    render(<AgentTeamsCanvas {...props({
+      snapshots: [recovering],
+      selectedIndex: 0,
+      snapshot: recovering,
+      previousSnapshot: undefined,
+      activeMessageId: null,
+    })} />)
+
+    const builder = screen.getByTestId('agent-teams-canvas-member-builder@canvas-team')
+    expect(builder.getAttribute('data-member-state')).toBe('retrying')
+    expect(screen.getByText('Auto-retry 2/5').getAttribute('title')).toBe('API Error: 529 overloaded')
+
+    const reviewer = screen.getByTestId('agent-teams-canvas-member-reviewer@canvas-team')
+    expect(reviewer.getAttribute('data-member-state')).toBe('stopped')
+    expect(reviewer.textContent).toContain('Stopped')
+
+    const qa = screen.getByTestId('agent-teams-canvas-member-qa@canvas-team')
+    expect(qa.getAttribute('data-member-state')).toBe('error')
+    expect(screen.getByText('Error').getAttribute('title')).toBe('Credit balance is too low')
+  })
 })

desktop/src/components/agentTeams/AgentTeamsCanvas.tsx

@@ -117,11 +117,24 @@ function memberStatusColor(state: MemberWorkState): string {
   return 'var(--color-text-secondary)'
 }
 
+function memberDotColor(state: MemberWorkState, accent: string): string {
+  if (state === 'working') return accent
+  // The raw warning accent is too light for label text, but reads as a fill.
+  if (state === 'retrying') return 'var(--color-warning)'
+  return memberStatusColor(state)
+}
+
 function taskStateLabel(state: WorkbenchTaskState, t: TranslationFn): string {
   return t(`agentTeams.task.${state}` as TranslationKey)
 }
 
-function memberStateLabel(state: MemberWorkState, t: TranslationFn): string {
+function memberStateLabel(state: MemberWorkState, member: TeamMember, t: TranslationFn): string {
+  if (state === 'retrying') {
+    return t('agentTeams.member.retrying', {
+      attempt: member.autoRetry?.attempt ?? '?',
+      max: member.autoRetry?.max ?? '?',
+    })
+  }
   return t(`agentTeams.member.${state}` as TranslationKey)
 }
 
@@ -466,10 +479,10 @@ function MemberNode({
           : waitingDependency
             ? t('agentTeams.member.waitingForDependency', { task: waitingDependency })
             : t('agentTeams.member.waitingForTask')
-        : memberStateLabel(state, t)
+        : memberStateLabel(state, member, t)
   const characterClass = state === 'working'
     ? 'agent-teams-character-working'
-    : state === 'idle'
+    : state === 'idle' || state === 'retrying'
       ? 'agent-teams-character-idle'
       : state === 'exited'
         ? 'agent-teams-character-archived'
@@ -547,10 +560,15 @@ function MemberNode({
         {!isLead ? (
           <span
             className="h-1.5 w-1.5 shrink-0 rounded-full"
-            style={{ backgroundColor: state === 'working' ? accent : memberStatusColor(state) }}
+            style={{ backgroundColor: memberDotColor(state, accent) }}
           />
         ) : null}
-        <span className="truncate">{stateLabel}</span>
+        <span
+          className="truncate"
+          title={state === 'stopped' || state === 'retrying' || state === 'error' ? member.lastError : undefined}
+        >
+          {stateLabel}
+        </span>
         {isLead ? (
           <span className="shrink-0 text-[var(--color-text-tertiary)]">
             · {t('agentTeams.member.inbox', { count: position.inbox })}

desktop/src/components/agentTeams/AgentTeamsMemberInspector.test.tsx

@@ -238,4 +238,141 @@ describe('AgentTeamsMemberInspector', () => {
     expect(cell.textContent).toBe('Unknown')
     expect(cell.textContent).not.toContain('undefined')
   })
+
+  describe('recovery states', () => {
+    const now = Date.parse('2026-08-08T07:00:00.000Z')
+
+    function renderMember(member: TeamMember, clock?: number) {
+      const base = snapshot('2026-08-08T07:00:00.000Z', 'in_progress')
+      const frame: TeamWorkbenchSnapshot = {
+        ...base,
+        team: { ...base.team, members: [member, reviewer] },
+      }
+      return render(
+        <AgentTeamsMemberInspector
+          snapshots={[frame]}
+          selectedIndex={0}
+          snapshot={frame}
+          member={member}
+          isLead={false}
+          leadIsStreaming={false}
+          now={clock}
+          onBack={vi.fn()}
+          onClose={vi.fn()}
+          onOpenExecution={vi.fn()}
+        />,
+      )
+    }
+
+    it('says a stopped member resumes from its saved conversation when messaged', () => {
+      const onOpenExecution = vi.fn()
+      const frame = snapshot('2026-08-08T07:00:00.000Z', 'in_progress')
+      const stopped: TeamMember = { ...builder, status: 'idle', activity: 'stopped' }
+      render(
+        <AgentTeamsMemberInspector
+          snapshots={[frame]}
+          selectedIndex={0}
+          snapshot={frame}
+          member={stopped}
+          isLead={false}
+          leadIsStreaming={false}
+          onBack={vi.fn()}
+          onClose={vi.fn()}
+          onOpenExecution={onOpenExecution}
+        />,
+      )
+
+      expect(screen.getByText('Stopped')).toBeTruthy()
+      const notice = screen.getByTestId('agent-teams-member-recovery')
+      expect(notice.getAttribute('data-member-state')).toBe('stopped')
+      expect(screen.getByTestId('agent-teams-member-recovery-hint').textContent)
+        .toBe('Send a message to resume it from its saved conversation')
+      expect(screen.queryByTestId('agent-teams-member-last-error')).toBeNull()
+
+      // The way back is the member's own conversation, where the composer lives.
+      const execution = screen.getByRole('button', { name: 'View builder execution' }) as HTMLButtonElement
+      expect(execution.disabled).toBe(false)
+      fireEvent.click(execution)
+      expect(onOpenExecution).toHaveBeenCalledOnce()
+    })
+
+    it('counts down to the next automatic retry and names the failure behind it', () => {
+      const retrying: TeamMember = {
+        ...builder,
+        status: 'idle',
+        activity: 'idle',
+        lastError: 'API Error: 529 overloaded',
+        autoRetry: { attempt: 2, max: 5, nextAt: now + 45_000 },
+      }
+      const view = renderMember(retrying, now)
+
+      expect(screen.getByText('Auto-retry 2/5')).toBeTruthy()
+      expect(screen.getByTestId('agent-teams-member-recovery').getAttribute('data-member-state'))
+        .toBe('retrying')
+      expect(screen.getByTestId('agent-teams-member-last-error').textContent)
+        .toBe('API Error: 529 overloaded')
+      expect(screen.getByTestId('agent-teams-member-recovery-hint').textContent)
+        .toBe('Retrying automatically in 45s')
+
+      view.rerender(
+        <AgentTeamsMemberInspector
+          snapshots={[]}
+          selectedIndex={0}
+          snapshot={snapshot('2026-08-08T07:00:00.000Z', 'in_progress')}
+          member={{ ...retrying, autoRetry: { attempt: 3, max: 5, nextAt: now + 125_000 } }}
+          isLead={false}
+          leadIsStreaming={false}
+          now={now}
+          onBack={vi.fn()}
+          onClose={vi.fn()}
+          onOpenExecution={vi.fn()}
+        />,
+      )
+      expect(screen.getByTestId('agent-teams-member-recovery-hint').textContent)
+        .toBe('Retrying automatically in about 2 min')
+      view.unmount()
+
+      // Past its schedule, the retry is due rather than counting negative.
+      renderMember(retrying, now + 46_000)
+      expect(screen.getByTestId('agent-teams-member-recovery-hint').textContent)
+        .toBe('Retrying automatically now')
+    })
+
+    it('shows no countdown while replaying, where the present clock means nothing', () => {
+      renderMember({
+        ...builder,
+        status: 'idle',
+        activity: 'idle',
+        lastError: 'API Error: 529 overloaded',
+        autoRetry: { attempt: 1, max: 5, nextAt: now + 15_000 },
+      })
+
+      expect(screen.getByText('Auto-retry 1/5')).toBeTruthy()
+      expect(screen.getByTestId('agent-teams-member-last-error').textContent)
+        .toBe('API Error: 529 overloaded')
+      expect(screen.queryByTestId('agent-teams-member-recovery-hint')).toBeNull()
+    })
+
+    it('keeps a long failure reason on one line with the full text on hover', () => {
+      const reason = 'API Error: 400 {"type":"error","error":{"type":"invalid_request_error","message":"prompt is too long: 215000 tokens > 200000 maximum"}}'
+      renderMember({ ...builder, status: 'error', activity: 'idle', lastError: reason })
+
+      expect(screen.getByText('Error')).toBeTruthy()
+      expect(screen.getByTestId('agent-teams-member-recovery').getAttribute('data-member-state'))
+        .toBe('error')
+      const lastError = screen.getByTestId('agent-teams-member-last-error')
+      expect(lastError.textContent).toBe(reason)
+      expect(lastError.getAttribute('title')).toBe(reason)
+      expect(lastError.className).toContain('truncate')
+      expect(screen.getByTestId('agent-teams-member-recovery-hint').textContent)
+        .toBe('Once the problem is fixed, send a message to let it continue')
+    })
+
+    it('drops the failure notice while a failed member works its way back', () => {
+      renderMember({ ...builder, status: 'error', activity: 'active', lastError: 'Credit balance is too low' })
+
+      expect(screen.getByText('Executing #7')).toBeTruthy()
+      expect(screen.queryByTestId('agent-teams-member-recovery')).toBeNull()
+    })
+  })
 })

desktop/src/components/agentTeams/AgentTeamsMemberInspector.tsx

@@ -23,6 +23,7 @@ import { IconButton } from '@/components/ui/IconButton'
 import { useTranslation, type TranslationKey } from '@/i18n'
 import type {
   TeamMember,
+  TeamMemberAutoRetry,
   TeamWorkbenchMessage,
   TeamWorkbenchSnapshot,
   TeamWorkbenchTask,
@@ -35,6 +36,11 @@ export type AgentTeamsMemberInspectorProps = {
   member: TeamMember
   isLead: boolean
   leadIsStreaming: boolean
+  /**
+   * The live clock, for the time left until an automatic retry. Omitted while
+   * replaying, where a countdown from the present would describe nothing.
+   */
+  now?: number
   onBack: () => void
   onClose: () => void
   onOpenExecution: () => void
@@ -190,10 +196,25 @@ function taskTone(state: WorkbenchTaskState): Tone {
 function memberTone(state: MemberWorkState): Tone {
   if (state === 'working') return 'brand'
   if (state === 'error') return 'danger'
+  if (state === 'retrying') return 'warning'
   if (state === 'exited' || state === 'stopped') return 'neutral'
   return 'info'
 }
 
+/** Container and foreground pairs, the same ones `Badge` uses for these tones. */
+function recoveryNoticeClasses(state: MemberWorkState): string {
+  if (state === 'error') return 'bg-[var(--color-error-container)] text-[var(--color-on-error-container)]'
+  if (state === 'retrying') return 'bg-[var(--color-warning-container)] text-[var(--color-on-warning-container)]'
+  return 'bg-[var(--color-surface-container)] text-[var(--color-text-secondary)]'
+}
+
+function retryCountdown(autoRetry: TeamMemberAutoRetry, now: number, t: TranslationFn): string {
+  const seconds = Math.ceil((autoRetry.nextAt - now) / 1000)
+  if (seconds <= 0) return t('agentTeams.member.retryNow')
+  if (seconds < 60) return t('agentTeams.member.retryInSeconds', { n: seconds })
+  return t('agentTeams.member.retryInMinutes', { n: Math.round(seconds / 60) })
+}
+
 function leadStatusLabel(snapshot: TeamWorkbenchSnapshot, t: TranslationFn): string {
   if (snapshot.deletedAt) return t('agentTeams.lead.archived')
   if (snapshot.tasks.length === 0) return t('agentTeams.lead.forming')
@@ -283,6 +304,7 @@ export function AgentTeamsMemberInspector({
   member,
   isLead,
   leadIsStreaming,
+  now,
   onBack,
   onClose,
   onOpenExecution,
@@ -324,7 +346,27 @@ export function AgentTeamsMemberInspector({
           ? t('agentTeams.member.waitingForDependency', { task: waitingDependency })
           : workState === 'idle'
             ? t('agentTeams.member.waitingForTask')
-            : t(`agentTeams.member.${workState}` as TranslationKey)
+            : workState === 'retrying'
+              ? t('agentTeams.member.retrying', {
+                  attempt: member.autoRetry?.attempt ?? '?',
+                  max: member.autoRetry?.max ?? '?',
+                })
+              : t(`agentTeams.member.${workState}` as TranslationKey)
+  // Stopped, retrying and failed members all come back on their own or through
+  // a message; say why they paused and what brings them back.
+  const awaitsRecovery = !isLead && (
+    workState === 'stopped' || workState === 'retrying' || workState === 'error'
+  )
+  const failureReason = awaitsRecovery ? member.lastError : undefined
+  const recoveryHint = !awaitsRecovery
+    ? undefined
+    : workState === 'stopped'
+      ? t('agentTeams.member.stoppedHint')
+      : workState === 'error'
+        ? t('agentTeams.member.errorHint')
+        : member.autoRetry && now !== undefined
+          ? retryCountdown(member.autoRetry, now, t)
+          : undefined
 
   return (
     <section
@@ -424,6 +466,32 @@ export function AgentTeamsMemberInspector({
             <dd className="mt-0.5 truncate font-extrabold" data-testid="agent-teams-member-provider">{member.providerName || member.providerId || t('teamPlan.official')}</dd>
           </div>}
         </dl>
+
+        {failureReason || recoveryHint ? (
+          <div
+            data-testid="agent-teams-member-recovery"
+            data-member-state={workState}
+            className={`mt-3 min-w-0 rounded-[var(--radius-md)] px-2.5 py-2 text-[11px] leading-[1.45] ${recoveryNoticeClasses(workState)}`}
+          >
+            {failureReason ? (
+              <p className="flex min-w-0 items-baseline gap-1.5">
+                <span className="shrink-0 font-semibold">{t('agentTeams.inspector.failureReason')}</span>
+                <span
+                  data-testid="agent-teams-member-last-error"
+                  className="min-w-0 truncate font-mono"
+                  title={failureReason}
+                >
+                  {failureReason}
+                </span>
+              </p>
+            ) : null}
+            {recoveryHint ? (
+              <p data-testid="agent-teams-member-recovery-hint" className={failureReason ? 'mt-0.5' : undefined}>
+                {recoveryHint}
+              </p>
+            ) : null}
+          </div>
+        ) : null}
       </header>
 
       <div className="min-h-0 flex-1 overflow-y-auto overscroll-contain">

desktop/src/components/agentTeams/AgentTeamsWorkbench.tsx

@@ -518,6 +518,7 @@ export function AgentTeamsWorkbench({ sessionId }: { sessionId: string }) {
                 member={selectedMember}
                 isLead={selectedMemberIsLead}
                 leadIsStreaming={leadIsStreaming}
+                now={followingLive ? now : undefined}
                 onBack={() => setSelectedMemberId(null)}
                 onClose={closeCommunication}
                 onOpenExecution={openSelectedExecution}