diff --git a/internal/figaro/agent.go b/internal/figaro/agent.go index 341ca235..9ac2a839 100644 --- a/internal/figaro/agent.go +++ b/internal/figaro/agent.go @@ -5,6 +5,7 @@ import ( "encoding/json" "log/slog" "runtime" + "runtime/pprof" "slices" "sync" "sync/atomic" @@ -308,7 +309,20 @@ func NewAgent(cfg Config) *Agent { a.resumeStudies() a.publishMetadata() - go a.runWithRecovery(ctx) + // Label the agent goroutine with its aria id. + // + // A hung daemon is diagnosed from a goroutine dump, and until now that + // dump was unattributable: eighty goroutines, a dozen of them agents, + // every stack identical, and no way to tell which one belonged to the + // aria that was not answering. The label costs a map on one goroutine + // and is INHERITED by every goroutine this one spawns - the turn loop, + // the tool dispatcher, the provider's stream reader - so the whole + // subtree of a stuck turn names itself. + // + // Labels show in `/debug/pprof/goroutine?debug=1` and in `go tool pprof + // -tagfocus`; the debug=2 full stack dump does not render them, so reach + // for debug=1 first when asking "which aria". + go pprof.Do(ctx, pprof.Labels("figaro.aria", a.id), a.runWithRecovery) return a } diff --git a/internal/figaro/pprof_labels_test.go b/internal/figaro/pprof_labels_test.go new file mode 100644 index 00000000..644dd804 --- /dev/null +++ b/internal/figaro/pprof_labels_test.go @@ -0,0 +1,53 @@ +package figaro + +import ( + "bytes" + "context" + "runtime/pprof" + "strings" + "testing" + "time" +) + +// A hung daemon is diagnosed from a goroutine dump, and an unlabeled dump +// cannot answer the only question worth asking: WHICH aria is stuck. This +// test asserts the label reaches the profile, and that it is inherited by +// goroutines the agent spawns - the turn loop and the provider stream reader +// are children, and they are where a hang actually lives. +func TestAgentGoroutineLabelsCarryAriaID(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + const ariaID = "94f0752b" + started := make(chan struct{}) + childStarted := make(chan struct{}) + + go pprof.Do(ctx, pprof.Labels("figaro.aria", ariaID), func(ctx context.Context) { + // A child goroutine, as the turn loop and tool dispatch are. + go func() { + close(childStarted) + <-ctx.Done() + }() + close(started) + <-ctx.Done() + }) + + <-started + <-childStarted + // Give the child a moment to be observable in the profile. + time.Sleep(10 * time.Millisecond) + + var buf bytes.Buffer + // debug=1 is the format that renders labels; debug=2 (the full stack + // dump) drops them, which is worth knowing before you go looking. + if err := pprof.Lookup("goroutine").WriteTo(&buf, 1); err != nil { + t.Fatal(err) + } + got := buf.String() + if !strings.Contains(got, ariaID) { + t.Fatalf("goroutine profile carries no aria label; a dump of this daemon is unattributable") + } + if n := strings.Count(got, ariaID); n < 2 { + t.Errorf("label appears %d time(s); it must be inherited by child goroutines too", n) + } +}