From d24d48362d521e4fd3893c13b416fe4519012e4d Mon Sep 17 00:00:00 2001 From: Kyrill <1942093+poolski@users.noreply.github.com> Date: Sun, 1 Nov 2026 00:37:34 +0000 Subject: [PATCH 1/2] perf(upstream): list tools from servers concurrently during discovery A discovery sweep listed each connected server's tools one after another, so one slow or large upstream delayed every other server's tools/list by up to its own timeout. The sweep now lists up to 8 servers at once (discoveryListConcurrency) and records the results in the same order and with the same logging as before. A server's concurrent tools/list calls were already coalesced into one request, and each capture carries its own sequence, so the order in which the lists finish does not affect how their results are applied. --- internal/upstream/discovery_parallel_test.go | 97 ++++++++++++++++++++ internal/upstream/manager.go | 40 ++++++-- 2 files changed, 131 insertions(+), 6 deletions(-) create mode 100644 internal/upstream/discovery_parallel_test.go diff --git a/internal/upstream/discovery_parallel_test.go b/internal/upstream/discovery_parallel_test.go new file mode 100644 index 000000000..95c02e5ab --- /dev/null +++ b/internal/upstream/discovery_parallel_test.go @@ -0,0 +1,97 @@ +package upstream + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/mark3labs/mcp-go/mcp" + mcpserver "github.com/mark3labs/mcp-go/server" + servertest "github.com/mark3labs/mcp-go/server/servertest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/smart-mcp-proxy/mcpproxy-go/internal/config" +) + +// inFlightGauge counts concurrent tools/list requests across upstreams. Each +// request waits until another one is in flight (or a timeout passes), so a +// sweep that lists servers one at a time never reaches a peak above 1. +type inFlightGauge struct { + mu sync.Mutex + cur, max int + overlap chan struct{} + once sync.Once + armed atomic.Bool +} + +func (g *inFlightGauge) enter() { + g.mu.Lock() + g.cur++ + if g.cur > g.max { + g.max = g.cur + } + if g.cur >= 2 { + g.once.Do(func() { close(g.overlap) }) + } + g.mu.Unlock() + select { + case <-g.overlap: + case <-time.After(2 * time.Second): + } + g.mu.Lock() + g.cur-- + g.mu.Unlock() +} + +func newGaugedUpstream(t *testing.T, g *inFlightGauge, tool string) *mcpserver.MCPServer { + t.Helper() + hooks := &mcpserver.Hooks{} + hooks.AddBeforeListTools(func(context.Context, any, *mcp.ListToolsRequest) { + // Connect lists tools too; only the sweep's lists are measured. + if g.armed.Load() { + g.enter() + } + }) + srv := mcpserver.NewMCPServer("gauged", "0.0.1", mcpserver.WithToolCapabilities(true), mcpserver.WithHooks(hooks)) + srv.AddTool(mcp.NewTool(tool, mcp.WithDescription("test tool")), + func(context.Context, mcp.CallToolRequest) (*mcp.CallToolResult, error) { + return mcp.NewToolResultText("ok"), nil + }) + return srv +} + +// A slow upstream must not hold up the tools/list of every other server: the +// sweep lists servers concurrently. +func TestDiscoverToolsListsServersConcurrently(t *testing.T) { + m, _ := newObservedManager(t) + t.Setenv("MCPPROXY_DISABLE_OAUTH", "true") + + g := &inFlightGauge{overlap: make(chan struct{})} + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + for i := 0; i < 3; i++ { + ts := servertest.NewTestStreamableHTTPServer(newGaugedUpstream(t, g, fmt.Sprintf("tool_%d", i))) + t.Cleanup(ts.Close) + id := fmt.Sprintf("srv-%d", i) + require.NoError(t, m.AddServerConfig(id, &config.ServerConfig{ + Name: fmt.Sprintf("server-%d", i), Protocol: "streamable-http", URL: ts.URL, Enabled: true, + })) + client, ok := m.GetClient(id) + require.True(t, ok) + require.NoError(t, client.Connect(ctx)) + } + g.armed.Store(true) + + tools, listed, err := m.DiscoverToolsReport(ctx, false) + require.NoError(t, err) + assert.Len(t, tools, 3) + assert.Len(t, listed, 3) + g.mu.Lock() + peak := g.max + g.mu.Unlock() + assert.GreaterOrEqual(t, peak, 2, "tools/list requests never overlapped: the sweep lists servers one at a time") +} diff --git a/internal/upstream/manager.go b/internal/upstream/manager.go index bc5505b67..2582acb3f 100644 --- a/internal/upstream/manager.go +++ b/internal/upstream/manager.go @@ -1196,6 +1196,10 @@ func (m *Manager) DiscoverToolsDue(ctx context.Context) ([]*config.ToolMetadata, return tools, err } +// discoveryListConcurrency bounds how many upstream tools/list requests one +// discovery sweep runs at once. +const discoveryListConcurrency = 8 + func (m *Manager) discoverTools(ctx context.Context, dueOnly bool) ([]*config.ToolMetadata, []string, map[string]managed.InventoryCapture, error) { type clientSnapshot struct { id string @@ -1247,6 +1251,7 @@ func (m *Manager) discoverTools(ctx context.Context, dueOnly bool) ([]*config.To connectedCount := 0 skippedNotDue := 0 known := make(map[string]struct{}, len(snapshots)) + var due []clientSnapshot for _, snapshot := range snapshots { client := snapshot.client @@ -1293,9 +1298,32 @@ func (m *Manager) discoverTools(ctx context.Context, dueOnly bool) ([]*config.To } connectedCount++ + due = append(due, snapshot) + } - tools, capture, err := client.ListToolsCaptured(ctx) - if err != nil { + // List the due servers concurrently: one slow or large upstream must not + // hold up every other server's tools/list until its own timeout. + type listResult struct { + tools []*config.ToolMetadata + capture managed.InventoryCapture + err error + } + results := make([]listResult, len(due)) + sem := make(chan struct{}, discoveryListConcurrency) + var wg sync.WaitGroup + for i, snapshot := range due { + wg.Go(func() { + sem <- struct{}{} + defer func() { <-sem }() + tools, capture, err := snapshot.client.ListToolsCaptured(ctx) + results[i] = listResult{tools: tools, capture: capture, err: err} + }) + } + wg.Wait() + + for i, snapshot := range due { + res := results[i] + if res.err != nil { // Warn, not Error: markSwept below is deliberately skipped so the // next sweep retries this server. A failure the code already plans // to retry is not an error — and on a healthy install with several @@ -1304,7 +1332,7 @@ func (m *Manager) discoverTools(ctx context.Context, dueOnly bool) ([]*config.To m.logger.Warn("Failed to list tools from client", zap.String("id", snapshot.id), zap.String("server", snapshot.name), - zap.Error(err)) + zap.Error(res.err)) continue } @@ -1313,11 +1341,11 @@ func (m *Manager) discoverTools(ctx context.Context, dueOnly bool) ([]*config.To m.markSwept(snapshot.name, now) if snapshot.name != "" { listed = append(listed, snapshot.name) - captures[snapshot.name] = capture + captures[snapshot.name] = res.capture } - if tools != nil { - allTools = append(allTools, tools...) + if res.tools != nil { + allTools = append(allTools, res.tools...) } } From de923de59729cd28949c73d878596c38aed48882 Mon Sep 17 00:00:00 2001 From: Algis Dumbris Date: Mon, 2 Nov 2026 19:27:39 +0200 Subject: [PATCH 2/2] test(upstream): assert the discovery list concurrency bound Register 12 servers whose tools/list holds until 8 are in flight, then assert the peak never exceeds discoveryListConcurrency; removing the semaphore now fails. Co-Authored-By: Claude Fable 5.1 --- internal/upstream/discovery_parallel_test.go | 86 ++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/internal/upstream/discovery_parallel_test.go b/internal/upstream/discovery_parallel_test.go index 95c02e5ab..6479fa10c 100644 --- a/internal/upstream/discovery_parallel_test.go +++ b/internal/upstream/discovery_parallel_test.go @@ -95,3 +95,89 @@ func TestDiscoverToolsListsServersConcurrently(t *testing.T) { g.mu.Unlock() assert.GreaterOrEqual(t, peak, 2, "tools/list requests never overlapped: the sweep lists servers one at a time") } + +// slotGauge holds each tools/list request until `want` requests are in flight +// (or a timeout passes), then keeps it a little longer so any request beyond +// the concurrency bound would overlap with the full set and raise the peak. +type slotGauge struct { + mu sync.Mutex + cur, max int + want int + full chan struct{} + once sync.Once + armed atomic.Bool +} + +func (g *slotGauge) enter() { + g.mu.Lock() + g.cur++ + if g.cur > g.max { + g.max = g.cur + } + if g.cur >= g.want { + g.once.Do(func() { close(g.full) }) + } + g.mu.Unlock() + select { + case <-g.full: + case <-time.After(2 * time.Second): + } + // Hold the slot so requests an unbounded sweep launches meanwhile are + // counted against the peak. + time.Sleep(50 * time.Millisecond) + g.mu.Lock() + g.cur-- + g.mu.Unlock() +} + +func newSlotGaugedUpstream(t *testing.T, g *slotGauge, tool string) *mcpserver.MCPServer { + t.Helper() + hooks := &mcpserver.Hooks{} + hooks.AddBeforeListTools(func(context.Context, any, *mcp.ListToolsRequest) { + if g.armed.Load() { + g.enter() + } + }) + srv := mcpserver.NewMCPServer("slot-gauged", "0.0.1", mcpserver.WithToolCapabilities(true), mcpserver.WithHooks(hooks)) + srv.AddTool(mcp.NewTool(tool, mcp.WithDescription("test tool")), + func(context.Context, mcp.CallToolRequest) (*mcp.CallToolResult, error) { + return mcp.NewToolResultText("ok"), nil + }) + return srv +} + +// The sweep lists concurrently but never more than discoveryListConcurrency +// servers at once, however many are due. +func TestDiscoverToolsBoundsConcurrentLists(t *testing.T) { + m, _ := newObservedManager(t) + t.Setenv("MCPPROXY_DISABLE_OAUTH", "true") + + const serverCount = 12 + require.Greater(t, serverCount, discoveryListConcurrency) + + g := &slotGauge{want: discoveryListConcurrency, full: make(chan struct{})} + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + for i := 0; i < serverCount; i++ { + ts := servertest.NewTestStreamableHTTPServer(newSlotGaugedUpstream(t, g, fmt.Sprintf("tool_%d", i))) + t.Cleanup(ts.Close) + id := fmt.Sprintf("srv-%d", i) + require.NoError(t, m.AddServerConfig(id, &config.ServerConfig{ + Name: fmt.Sprintf("server-%d", i), Protocol: "streamable-http", URL: ts.URL, Enabled: true, + })) + client, ok := m.GetClient(id) + require.True(t, ok) + require.NoError(t, client.Connect(ctx)) + } + g.armed.Store(true) + + tools, listed, err := m.DiscoverToolsReport(ctx, false) + require.NoError(t, err) + assert.Len(t, tools, serverCount) + assert.Len(t, listed, serverCount) + g.mu.Lock() + peak := g.max + g.mu.Unlock() + assert.GreaterOrEqual(t, peak, 2, "tools/list requests never overlapped") + assert.LessOrEqual(t, peak, discoveryListConcurrency, "more servers were listed at once than discoveryListConcurrency allows") +}