auth: a vault call that finds the agent gone brings it back

hush's agent exits when its TTL runs out, and that TTL is absolute: counted
from the agent's start, never extended by use. figaro sets it to a day, so
every agent dies exactly one day after it was born (the log on the author's
box shows it nightly: 01:36, 01:41, 01:42, creeping by the keep-alive's
delay). Until something starts a new one, every vault call fails with
"dial unix .../agent.sock: connect: no such file or directory".

Two things used to start a new one: any CLI command (ensureHush on the way
to the daemon), and keepHushAlive, on a ticker of up to five minutes. A turn
that starts without a CLI command (the next round of a turn already running,
a queued message, a prompt typed into a long-lived pager, a message from
another aria or a bridge) landed in the gap and failed. Since the Copilot
session token stopped being cached in figaro (c59f63a), every Copilot round
asks the agent, so every round in the gap failed:

  copilot: resolve token: dial unix /tmp/figaro-hush/agent.sock: connect:
  no such file or directory

seen on two boxes, and then explained to the user as a rate-limited token
exchange, which it was not: the credential never left the vault.

The repair is at the point of use. auth.WithAgent runs a vault call and, if
it failed because no agent serves the socket (ENOENT, ECONNREFUSED: the real
client's error, matched with errors.Is), revives the agent and asks once
more. The OAuth strategy (get and refresh) and the encrypted API key take a
Revive, wired to ensureHushAgent, which serializes EnsureReady: managed's
own ping-then-spawn has no lock, and the keep-alive and every provider
goroutine now arrive at it together. A failed revival is reported as such
(ReviveError), and authFailureHint names the vault for it, and for the raw
dial error from older builds, instead of the exchange endpoint.

Reproduced end to end in an isolated box (own vault via FIGARO_HUSH_DIR, a
45 s FIGARO_HUSH_TTL, a fake Copilot playing both the exchange and the API
through base_url, no CLI command during the wait): a turn whose tool sleeps
past the agent's deadline. On v0.38.2 round two failed with the message
above, every time; on this build the agent was revived at the moment of the
call, ten seconds before the keep-alive's tick, and the turn finished.

Unit tests use the real hush client against a socket with nothing behind it,
so AgentGone is checked against the shape production sees; the strategy's
revival and the hint were each canaried.

Author: gluck

internal/auth/revive.go

@@ -0,0 +1,55 @@
+package auth
+
+import (
+	"errors"
+	"syscall"
+)
+
+// THE AGENT IS MORTAL, AND IT DIES ON A SCHEDULE.
+//
+// hush's agent exits when its TTL runs out, and that TTL is ABSOLUTE: counted
+// from the moment it started, never extended by use. figaro sets it to a day,
+// so every agent dies exactly one day after it was born, and every request in
+// flight against its socket then fails with "connect: no such file or
+// directory" until something starts a new one. The daemon's keep-alive did,
+// on a ticker of up to five minutes, which made a daily outage of up to five
+// minutes in which every turn of every aria whose credential lives in the
+// vault failed (Gluck, 2026-10-12, two boxes). It reaches the agent dying of
+// anything else the same way: a crash, a stop in a dev shell sharing the
+// surface.
+//
+// The repair is at the point of use. A vault call that finds no agent brings
+// one up and asks once more. Starting the embedded agent is a spawn and an
+// unlock from the keyring or the passphrase file, so the turn pays well under
+// a second, once a day, instead of failing.
+
+// AgentGone reports whether a hush call failed because no agent is serving
+// the socket: the socket file is missing, or nothing is listening on it. Any
+// other failure is the agent's answer and is not retried.
+func AgentGone(err error) bool {
+	return errors.Is(err, syscall.ENOENT) || errors.Is(err, syscall.ECONNREFUSED)
+}
+
+// WithAgent runs call, and if it failed because the agent is gone, runs
+// revive and call once more. A nil revive, or a revive that fails, returns
+// the original error: the caller's own message stays the diagnosis, with the
+// revival failure joined to it.
+func WithAgent[T any](revive func() error, call func() (T, error)) (T, error) {
+	out, err := call()
+	if err == nil || revive == nil || !AgentGone(err) {
+		return out, err
+	}
+	if rerr := revive(); rerr != nil {
+		var zero T
+		return zero, errors.Join(err, &ReviveError{Err: rerr})
+	}
+	return call()
+}
+
+// ReviveError is a failure to bring the agent back. It is its own type so a
+// message for the user can say "the vault agent would not start" instead of
+// blaming whatever the call was trying to do.
+type ReviveError struct{ Err error }
+
+func (e *ReviveError) Error() string { return "hush agent would not start: " + e.Err.Error() }
+func (e *ReviveError) Unwrap() error { return e.Err }

internal/auth/revive_test.go

@@ -0,0 +1,116 @@
+package auth
+
+import (
+	"errors"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	hush "github.com/jack-work/hush/client"
+)
+
+// goneErr is the error a real hush client returns when no agent serves its
+// socket: the exact shape Gluck's turns failed with, produced by the real
+// client rather than spelled by hand.
+func goneErr(t *testing.T) error {
+	t.Helper()
+	c := hush.NewWithSocket(filepath.Join(t.TempDir(), "agent.sock"))
+	_, _, err := c.OAuthGetFull("copilot")
+	if err == nil {
+		t.Fatal("a client with no agent behind it answered")
+	}
+	if !strings.Contains(err.Error(), "no such file or directory") {
+		t.Fatalf("unexpected shape for a missing agent: %v", err)
+	}
+	return err
+}
+
+func TestAgentGoneRecognisesTheRealClientError(t *testing.T) {
+	if !AgentGone(goneErr(t)) {
+		t.Fatal("the error a missing agent produces is not recognised as a missing agent")
+	}
+	for _, other := range []error{
+		hush.ErrOAuthNotFound,
+		errors.New("oauth refresh rejected"),
+		nil,
+	} {
+		if AgentGone(other) {
+			t.Fatalf("%v was taken for a missing agent", other)
+		}
+	}
+}
+
+// A call that finds the agent gone revives it ONCE and asks again; the turn
+// sees only the second answer.
+func TestWithAgentRevivesAndRetriesOnce(t *testing.T) {
+	gone := goneErr(t)
+	calls, revives := 0, 0
+	got, err := WithAgent(func() error { revives++; return nil }, func() (string, error) {
+		calls++
+		if calls == 1 {
+			return "", gone
+		}
+		return "token", nil
+	})
+	if err != nil || got != "token" {
+		t.Fatalf("got (%q, %v), want the token from the retry", got, err)
+	}
+	if calls != 2 || revives != 1 {
+		t.Fatalf("%d calls and %d revives, want 2 and 1", calls, revives)
+	}
+}
+
+// Anything that is not a missing agent is the agent's answer, and is not
+// second-guessed; nor is a missing agent retried with nothing to revive it.
+func TestWithAgentLeavesOtherFailuresAlone(t *testing.T) {
+	revives := 0
+	revive := func() error { revives++; return nil }
+	_, err := WithAgent(revive, func() (string, error) { return "", hush.ErrOAuthNotFound })
+	if !errors.Is(err, hush.ErrOAuthNotFound) || revives != 0 {
+		t.Fatalf("a real answer was retried (revives=%d, err=%v)", revives, err)
+	}
+	gone := goneErr(t)
+	calls := 0
+	_, err = WithAgent[string](nil, func() (string, error) { calls++; return "", gone })
+	if calls != 1 || !AgentGone(err) {
+		t.Fatalf("with no reviver: %d calls, err %v; want one call and the original error", calls, err)
+	}
+}
+
+// A revival that fails keeps the original error AND says the agent would not
+// start, so the message for the user can name the vault, not the provider.
+func TestWithAgentReportsAFailedRevival(t *testing.T) {
+	gone := goneErr(t)
+	_, err := WithAgent(func() error { return errors.New("unlock figaro: keyring locked") }, func() (string, error) {
+		return "", gone
+	})
+	var rev *ReviveError
+	if !errors.As(err, &rev) || !AgentGone(err) {
+		t.Fatalf("want both the dial error and a ReviveError, got %v", err)
+	}
+	if !strings.Contains(err.Error(), "hush agent would not start: unlock figaro: keyring locked") {
+		t.Fatalf("message does not name the revival: %v", err)
+	}
+}
+
+// The strategy itself, end to end through the real client: the first ask
+// finds no agent, Revive is called, and the error that reaches the caller
+// after a failed revival is the one the hint recognises.
+func TestOAuthRevivesThroughTheRealClient(t *testing.T) {
+	revives := 0
+	o := &OAuth{
+		Hush:   hush.NewWithSocket(filepath.Join(t.TempDir(), "agent.sock")),
+		Name:   "copilot",
+		Revive: func() error { revives++; return errors.New("no identity") },
+	}
+	_, ok, err := o.TryResolve()
+	if ok || err == nil {
+		t.Fatalf("resolved with no agent: ok=%v err=%v", ok, err)
+	}
+	if revives != 1 {
+		t.Fatalf("the strategy tried to revive %d times, want 1", revives)
+	}
+	if !strings.Contains(err.Error(), "hush agent would not start") {
+		t.Fatalf("the failed revival is not in the error: %v", err)
+	}
+}

internal/auth/strategy.go

@@ -96,6 +96,10 @@ type OAuth struct {
 	Hush *hush.Client
 	Name string
 
+	// Revive brings the agent back when a call finds it gone (see
+	// WithAgent). Nil means a missing agent is an error, as it always was.
+	Revive func() error
+
 	// endpoint is the routing minted alongside the token, for credentials
 	// whose exchange chooses the host (Copilot). Published atomically
 	// because Resolve runs on provider goroutines and Endpoint is read by
@@ -107,7 +111,15 @@ func (o *OAuth) TryResolve() (string, bool, error) {
 	if o.Hush == nil || o.Name == "" {
 		return "", false, nil
 	}
-	tok, meta, err := o.Hush.OAuthGetFull(o.Name)
+	type full struct {
+		tok  string
+		meta map[string]string
+	}
+	got, err := WithAgent(o.Revive, func() (full, error) {
+		tok, meta, err := o.Hush.OAuthGetFull(o.Name)
+		return full{tok, meta}, err
+	})
+	tok, meta := got.tok, got.meta
 	if err != nil {
 		if errors.Is(err, hush.ErrOAuthNotFound) {
 			return "", false, nil
@@ -151,7 +163,10 @@ func (o *OAuth) Invalidate(token string) error {
 	if o.Hush == nil || o.Name == "" {
 		return nil
 	}
-	_, meta, err := o.Hush.OAuthRefreshFull(o.Name)
+	meta, err := WithAgent(o.Revive, func() (map[string]string, error) {
+		_, meta, err := o.Hush.OAuthRefreshFull(o.Name)
+		return meta, err
+	})
 	if err == nil {
 		o.rememberEndpoint(meta)
 		return nil

internal/cli/angelus.go

@@ -8,6 +8,7 @@ import (
 	"os/signal"
 	"path/filepath"
 	"runtime/debug"
+	"sync"
 	"syscall"
 	"time"
 
@@ -59,6 +60,23 @@ func incumbentAnswers() bool {
 }
 
 // runAngelus runs the supervisor side of the binary.
+// hushEnsureMu serializes bringing the agent up. managed.EnsureReady pings
+// and then spawns, with no lock of its own, and since a dead agent is now
+// revived by whichever caller finds it dead (every provider goroutine
+// resolving a token, as well as the keep-alive), several of them arrive at
+// once. Serialized, the first spawns and the rest find it answering.
+var hushEnsureMu sync.Mutex
+
+// ensureHushAgent makes sure the embedded agent is up, starting it if not. It
+// is the Revive of every vault-backed credential strategy (auth.WithAgent),
+// so a turn that lands in the gap after the agent's TTL brings the agent back
+// rather than failing until the keep-alive's next tick.
+func ensureHushAgent() error {
+	hushEnsureMu.Lock()
+	defer hushEnsureMu.Unlock()
+	return mustHush().EnsureReady()
+}
+
 // keepHushAlive pings the embedded hush agent on an interval and respawns it
 // if it has died (EnsureReady), so the token-refresh machinery survives for
 // the whole daemon session. Interval is well under the agent TTL so a dead
@@ -77,7 +95,7 @@ func keepHushAlive(ctx context.Context) {
 	// `figaro stop` acts on, and a claim written before there is anything
 	// to claim would name a socket that never existed.
 	ensure := func(stage string) {
-		if err := h.EnsureReady(); err != nil {
+		if err := ensureHushAgent(); err != nil {
 			slog.Warn("hush keep-alive: "+stage+" ensure failed", "err", err)
 			return
 		}

internal/cli/outfit_helpers.go

@@ -21,6 +21,9 @@ import (
 type encryptedAPIKey struct {
 	Hush       *hush.Client
 	ConfigPath string
+	// Revive brings the agent back when a decrypt finds it gone; see
+	// auth.WithAgent.
+	Revive func() error
 
 	mu       sync.Mutex
 	cached   string
@@ -52,7 +55,9 @@ func (e *encryptedAPIKey) TryResolve() (string, bool, error) {
 	if !strings.HasPrefix(pa.APIKey, "AGE-ENC[") {
 		return "", false, nil
 	}
-	res, err := e.Hush.Decrypt(map[string]string{"v": pa.APIKey})
+	res, err := auth.WithAgent(e.Revive, func() (map[string]string, error) {
+		return e.Hush.Decrypt(map[string]string{"v": pa.APIKey})
+	})
 	if err != nil {
 		return "", false, fmt.Errorf("hush decrypt %s: %w", e.ConfigPath, err)
 	}

internal/cli/provider.go

@@ -54,7 +54,7 @@ func buildResolver(loaded *config.Loaded, providerName string) (auth.TokenResolv
 		}
 		// Returned bare, not wrapped in an Aggregate: it is the only
 		// source, and the provider reads the routing off it directly.
-		return &auth.OAuth{Hush: hushClient, Name: providerName}, nil
+		return &auth.OAuth{Hush: hushClient, Name: providerName, Revive: ensureHushAgent}, nil
 	}
 
 	strategies := environmentStrategies(reg)
@@ -70,12 +70,14 @@ func buildResolver(loaded *config.Loaded, providerName string) (auth.TokenResolv
 		&encryptedAPIKey{
 			Hush:       hushClient,
 			ConfigPath: loaded.ProviderAuthPath(providerName),
+			Revive:     ensureHushAgent,
 		},
 	)
 	if hasOAuth {
 		strategies = append(strategies, &auth.OAuth{
-			Hush: hushClient,
-			Name: providerName,
+			Hush:   hushClient,
+			Name:   providerName,
+			Revive: ensureHushAgent,
 		})
 	}
 	return &auth.Aggregate{Strategies: strategies}, nil
@@ -135,10 +137,43 @@ func sessionExchangeHint(reason string) string {
 	return b.String()
 }
 
-// authFailureHint picks the right explanation for a failed turn: no
-// credential at all, or a credential that could not be exchanged.
+// vaultAgentHint explains a credential that could not be READ, because the
+// hush agent that holds it was not running and could not be started. It used
+// to fall through to sessionExchangeHint, which told the user an exchange
+// endpoint was rate-limiting them while the credential had never left the
+// vault.
+func vaultAgentHint(reason string) string {
+	var b strings.Builder
+	b.WriteString(reason)
+	if !strings.HasSuffix(reason, "\n") {
+		b.WriteString("\n")
+	}
+	b.WriteString("\nfigaro's vault agent (hush) was not running, so the stored credential\n")
+	b.WriteString("could not be read, and starting it again failed. Nothing was sent to the\n")
+	b.WriteString("provider. To see why, and to start it by hand:\n")
+	b.WriteString("  figaro vault status     (identity, unlock method, agent)\n")
+	b.WriteString("  figaro vault unlock\n")
+	return b.String()
+}
+
+// vaultAgentDown reports whether a failure reason is the vault agent being
+// unreachable rather than anything the provider said: the revival failing,
+// or (from a build that did not revive) the raw dial error on its socket.
+func vaultAgentDown(reason string) bool {
+	if strings.Contains(reason, "hush agent would not start") {
+		return true
+	}
+	return strings.Contains(reason, "agent.sock") &&
+		(strings.Contains(reason, "no such file or directory") || strings.Contains(reason, "connection refused"))
+}
+
+// authFailureHint picks the right explanation for a failed turn: the vault
+// agent unreachable, no credential at all, or a credential that could not be
+// exchanged.
 func authFailureHint(reason string) (string, bool) {
 	switch {
+	case vaultAgentDown(reason):
+		return vaultAgentHint(reason), true
 	case strings.Contains(reason, "resolve token"), strings.Contains(reason, "token exchange"):
 		return sessionExchangeHint(reason), true
 	case strings.Contains(reason, "no credential"):

internal/cli/provider_hint_test.go

@@ -1,6 +1,9 @@
 package cli
 
-import "testing"
+import (
+	"strings"
+	"testing"
+)
 
 // The bug this splits: a 403 from GitHub's Copilot token-exchange endpoint
 // arrived as "resolve token: copilot token exchange 403: <html>", figaro
@@ -45,3 +48,30 @@ func contains(hay, needle string) bool {
 		return false
 	})()
 }
+
+// THE MESSAGE GLUCK GOT, verbatim: the vault agent's socket was missing, and
+// the hint blamed a rate-limited exchange endpoint for a credential that never
+// left the vault. It names the vault now, as does a failed revival.
+func TestAuthFailureHintNamesTheVaultWhenTheAgentIsGone(t *testing.T) {
+	for _, reason := range []string{
+		"copilot: resolve token: dial unix /tmp/figaro-hush/agent.sock: connect: no such file or directory",
+		"resolve token: no credential available; first strategy error: dial unix /tmp/figaro-hush/agent.sock: connect: connection refused",
+		"copilot: resolve token: dial unix /tmp/figaro-hush/agent.sock: connect: no such file or directory\nhush agent would not start: unlock figaro: keyring locked",
+	} {
+		hint, ok := authFailureHint(reason)
+		if !ok {
+			t.Fatalf("no hint for %q", reason)
+		}
+		if strings.Contains(hint, "rate-limits") || strings.Contains(hint, "exchanging it for a session token failed") {
+			t.Fatalf("the vault agent's absence was blamed on the exchange endpoint:\n%s", hint)
+		}
+		if !strings.Contains(hint, "figaro vault status") {
+			t.Fatalf("the hint does not point at the vault:\n%s", hint)
+		}
+	}
+	// And a genuine exchange failure keeps the exchange hint.
+	hint, _ := authFailureHint("copilot: resolve token: token exchange: 429 Too Many Requests")
+	if !strings.Contains(hint, "exchanging it for a session token failed") {
+		t.Fatalf("an exchange failure lost its hint:\n%s", hint)
+	}
+}