From 6acb1bdd580ac8676ed067eb2941414ad5256906 Mon Sep 17 00:00:00 2001 From: Algis Dumbris Date: Tue, 20 Oct 2026 14:37:59 +0300 Subject: [PATCH 1/2] fix(upstream): make the late-callback test deterministic under CI load Related #1586 TestManager_LateCallbacks_ReachExistingClients timed out on every PR Build run since it merged, and today on the required Unit Tests job too. The cause is in mcp-go v1.0.0's streamable-HTTP server: in-request notifications are forwarded to the SSE response by a goroutine, and when the tool handler returns the request goroutine takes the stream mutex, drains the queue and closes done, so the forwarder discards the one notification it had already dequeued. Which of the three notifications is lost depends on scheduling, so the single warmup did not cover it; under CPU contention (2x ncpu busy loops) the test failed 9 of 20 runs locally. The test now repeats the notify call until both late-registered callbacks have fired (500 ms settle per attempt, 10 s deadline) and the callbacks are non-blocking signals so a repeat can never block core's notification goroutine. Under the same contention it passed 25/25 plain and 20/20 with -race. Co-Authored-By: Claude Fable 5.1 --- .../upstream/manager_late_callback_test.go | 78 ++++++++++++++----- 1 file changed, 57 insertions(+), 21 deletions(-) diff --git a/internal/upstream/manager_late_callback_test.go b/internal/upstream/manager_late_callback_test.go index b71570ead..39adf38ba 100644 --- a/internal/upstream/manager_late_callback_test.go +++ b/internal/upstream/manager_late_callback_test.go @@ -33,9 +33,10 @@ func TestManager_LateCallbacks_ReachExistingClients(t *testing.T) { ) upstream.AddTool(mcp.NewTool("notify"), func(ctx context.Context, _ mcp.CallToolRequest) (*mcp.CallToolResult, error) { srv := mcpserver.ServerFromContext(ctx) - // The first in-request notification on a streamable-HTTP response is - // consumed while the response upgrades to SSE and never reaches the - // client handler, so lead with a sacrificial one that core ignores. + // mcp-go v1.0.0's streamable-HTTP server can drop ONE in-request + // notification per call (see the retry loop below), so lead with a + // sacrificial one that core ignores; it absorbs the loss most of the + // time, and the retry loop covers the rest. if err := srv.SendNotificationToClient(ctx, "notifications/test/warmup", nil); err != nil { return nil, err } @@ -59,35 +60,70 @@ func TestManager_LateCallbacks_ReachExistingClients(t *testing.T) { })) client, ok := m.GetClient("srv-late") require.True(t, ok) - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() require.NoError(t, client.Connect(ctx)) // 2. Callbacks registered afterwards, as runtime.LoadConfiguredServers does. - toolsFired := make(chan string, 4) - promptsFired := make(chan string, 4) + // The callbacks signal rather than count: a non-blocking send keeps a + // repeated notify call from ever blocking core's notification goroutine + // (and with it the SSE reader the CallTool response rides on). + toolsFired := make(chan string, 1) + promptsFired := make(chan string, 1) m.SetToolDiscoveryCallback(func(_ context.Context, serverName string) error { - toolsFired <- serverName + select { + case toolsFired <- serverName: + default: + } return nil }) m.SetPromptsChangedCallback(func(serverName string) { - promptsFired <- serverName + select { + case promptsFired <- serverName: + default: + } }) // 3. Upstream sends tools/list_changed + prompts/list_changed. - _, err := client.CallTool(ctx, "notify", nil) - require.NoError(t, err) + // + // mcp-go v1.0.0's streamable-HTTP server (server/streamable_http.go, + // handlePost) forwards in-request notifications to the SSE response from + // a separate goroutine. When the tool handler returns, the request + // goroutine takes the stream mutex, drains whatever is still queued, + // closes `done`, and the forwarder then discards the notification it had + // already dequeued but not yet written (the non-resumable branch). So any + // ONE of the notifications a call sends can be lost, and which one is a + // matter of scheduling: on a loaded CI runner it is the tools or prompts + // notification often enough to fail every run. The subject here is the + // manager → client callback plumbing, not transport delivery, so the call + // is repeated until both callbacks have fired. Each attempt delivers at + // least two of its three notifications, so two attempts nearly always + // suffice; the deadline is generous for a loaded runner. + deadline := time.Now().Add(10 * time.Second) + gotTools, gotPrompts := false, false + attempts := 0 + for !gotTools || !gotPrompts { + if time.Now().After(deadline) { + t.Fatalf("after %d notify calls the late-registered callbacks had not both fired: tools=%v prompts=%v", + attempts, gotTools, gotPrompts) + } + attempts++ + _, err := client.CallTool(ctx, "notify", nil) + require.NoError(t, err) - select { - case name := <-toolsFired: - require.Equal(t, "late-server", name) - case <-time.After(5 * time.Second): - t.Fatal("tool discovery callback set after client creation was not invoked on tools/list_changed") - } - select { - case name := <-promptsFired: - require.Equal(t, "late-server", name) - case <-time.After(5 * time.Second): - t.Fatal("prompts-changed callback set after client creation was not invoked on prompts/list_changed") + settle := time.After(500 * time.Millisecond) + collect: + for !gotTools || !gotPrompts { + select { + case name := <-toolsFired: + require.Equal(t, "late-server", name) + gotTools = true + case name := <-promptsFired: + require.Equal(t, "late-server", name) + gotPrompts = true + case <-settle: + break collect + } + } } } From 21017169e9e813173fadb99127d1b4c9f491556d Mon Sep 17 00:00:00 2001 From: Algis Dumbris Date: Tue, 20 Oct 2026 14:39:53 +0300 Subject: [PATCH 2/2] test(upstream): qualify the per-call delivery note in the late-callback test A forwarder left over from an earlier request can discard a later call's notification as well, so the comment no longer claims a fixed minimum per attempt (opencode Sol 6.1 review, low). Co-Authored-By: Claude Fable 5.1 --- internal/upstream/manager_late_callback_test.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/internal/upstream/manager_late_callback_test.go b/internal/upstream/manager_late_callback_test.go index 39adf38ba..559171ee9 100644 --- a/internal/upstream/manager_late_callback_test.go +++ b/internal/upstream/manager_late_callback_test.go @@ -94,11 +94,13 @@ func TestManager_LateCallbacks_ReachExistingClients(t *testing.T) { // already dequeued but not yet written (the non-resumable branch). So any // ONE of the notifications a call sends can be lost, and which one is a // matter of scheduling: on a loaded CI runner it is the tools or prompts - // notification often enough to fail every run. The subject here is the - // manager → client callback plumbing, not transport delivery, so the call - // is repeated until both callbacks have fired. Each attempt delivers at - // least two of its three notifications, so two attempts nearly always - // suffice; the deadline is generous for a loaded runner. + // notification often enough to fail every run. (A forwarder from an + // earlier request that has not yet observed its own `done` can discard a + // later call's notification too, so no fixed per-call minimum holds.) The + // subject here is the manager → client callback plumbing, not transport + // delivery, so the call is repeated until both callbacks have fired; in + // practice one or two attempts suffice, and the deadline is generous for + // a loaded runner. deadline := time.Now().Add(10 * time.Second) gotTools, gotPrompts := false, false attempts := 0