diag: label agent goroutines with their aria id

A hung daemon is diagnosed from a goroutine dump. Ours was unattributable:
eighty goroutines, a dozen of them agents, every agent stack byte-identical
(actor.Queue.Recv <- Inbox.Recv <- act <- actProtected <- runWithRecovery),
and no way to tell which one belonged to the aria that was not answering.
Diagnosing a real stall meant guessing from goroutine ages and correlating
against timestamps in another file.

pprof.Do costs a map on one goroutine, and the label is INHERITED by every
goroutine that one spawns: the turn loop, the tool dispatcher, the provider's
stream reader. So the entire subtree of a stuck turn names itself, which is
the case that matters - the interesting frame is never the agent loop, it is
whatever the agent is blocked inside three levels down.

Labels render in /debug/pprof/goroutine?debug=1 and in `go tool pprof
-tagfocus=figaro.aria`. They do NOT render in the debug=2 full stack dump,
which is the one people reach for first; the comment says so.

Author: gluck

internal/figaro/agent.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"log/slog"
 	"runtime"
+	"runtime/pprof"
 	"slices"
 	"sync"
 	"sync/atomic"
@@ -308,7 +309,20 @@ func NewAgent(cfg Config) *Agent {
 
 	a.resumeStudies()
 	a.publishMetadata()
-	go a.runWithRecovery(ctx)
+	// Label the agent goroutine with its aria id.
+	//
+	// A hung daemon is diagnosed from a goroutine dump, and until now that
+	// dump was unattributable: eighty goroutines, a dozen of them agents,
+	// every stack identical, and no way to tell which one belonged to the
+	// aria that was not answering. The label costs a map on one goroutine
+	// and is INHERITED by every goroutine this one spawns - the turn loop,
+	// the tool dispatcher, the provider's stream reader - so the whole
+	// subtree of a stuck turn names itself.
+	//
+	// Labels show in `/debug/pprof/goroutine?debug=1` and in `go tool pprof
+	// -tagfocus`; the debug=2 full stack dump does not render them, so reach
+	// for debug=1 first when asking "which aria".
+	go pprof.Do(ctx, pprof.Labels("figaro.aria", a.id), a.runWithRecovery)
 	return a
 }
 

internal/figaro/pprof_labels_test.go

@@ -0,0 +1,53 @@
+package figaro
+
+import (
+	"bytes"
+	"context"
+	"runtime/pprof"
+	"strings"
+	"testing"
+	"time"
+)
+
+// A hung daemon is diagnosed from a goroutine dump, and an unlabeled dump
+// cannot answer the only question worth asking: WHICH aria is stuck. This
+// test asserts the label reaches the profile, and that it is inherited by
+// goroutines the agent spawns - the turn loop and the provider stream reader
+// are children, and they are where a hang actually lives.
+func TestAgentGoroutineLabelsCarryAriaID(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	const ariaID = "94f0752b"
+	started := make(chan struct{})
+	childStarted := make(chan struct{})
+
+	go pprof.Do(ctx, pprof.Labels("figaro.aria", ariaID), func(ctx context.Context) {
+		// A child goroutine, as the turn loop and tool dispatch are.
+		go func() {
+			close(childStarted)
+			<-ctx.Done()
+		}()
+		close(started)
+		<-ctx.Done()
+	})
+
+	<-started
+	<-childStarted
+	// Give the child a moment to be observable in the profile.
+	time.Sleep(10 * time.Millisecond)
+
+	var buf bytes.Buffer
+	// debug=1 is the format that renders labels; debug=2 (the full stack
+	// dump) drops them, which is worth knowing before you go looking.
+	if err := pprof.Lookup("goroutine").WriteTo(&buf, 1); err != nil {
+		t.Fatal(err)
+	}
+	got := buf.String()
+	if !strings.Contains(got, ariaID) {
+		t.Fatalf("goroutine profile carries no aria label; a dump of this daemon is unattributable")
+	}
+	if n := strings.Count(got, ariaID); n < 2 {
+		t.Errorf("label appears %d time(s); it must be inherited by child goroutines too", n)
+	}
+}