Skip to content

Commit 764a1bf

Browse files
poolskiDumbrisclaude
authored
perf(upstream): list tools from servers concurrently during discovery (#1814)
A discovery sweep listed each connected server's tools one after another, so one slow or large upstream delayed every other server's `tools/list` by up to its own timeout. The sweep now lists up to 8 servers at once (`discoveryListConcurrency`). Results are recorded in the original order with the same logging, `markSwept` and capture handling. A server's concurrent `tools/list` calls were already coalesced into one request, and each capture carries its own process-global sequence, so the order in which lists finish does not change how their results are applied. `TestDiscoverToolsListsServersConcurrently` uses three upstreams whose `tools/list` waits for another list to be in flight and requires the sweep to overlap them. ## Testing - [x] `go test -race ./internal/upstream/...` - [x] `scripts/test-api-e2e.sh` (70/70) - [x] `golangci-lint` with `.github/.golangci.yml`, bare and `--build-tags server`: no new issues --------- Co-authored-by: Algis Dumbris <a.dumbris@gmail.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
1 parent 866adef commit 764a1bf

2 files changed

Lines changed: 217 additions & 6 deletions

File tree

Lines changed: 183 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,183 @@
1+
package upstream
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"sync"
7+
"sync/atomic"
8+
"testing"
9+
"time"
10+
11+
"github.com/mark3labs/mcp-go/mcp"
12+
mcpserver "github.com/mark3labs/mcp-go/server"
13+
servertest "github.com/mark3labs/mcp-go/server/servertest"
14+
"github.com/stretchr/testify/assert"
15+
"github.com/stretchr/testify/require"
16+
17+
"github.com/smart-mcp-proxy/mcpproxy-go/internal/config"
18+
)
19+
20+
// inFlightGauge counts concurrent tools/list requests across upstreams. Each
21+
// request waits until another one is in flight (or a timeout passes), so a
22+
// sweep that lists servers one at a time never reaches a peak above 1.
23+
type inFlightGauge struct {
24+
mu sync.Mutex
25+
cur, max int
26+
overlap chan struct{}
27+
once sync.Once
28+
armed atomic.Bool
29+
}
30+
31+
func (g *inFlightGauge) enter() {
32+
g.mu.Lock()
33+
g.cur++
34+
if g.cur > g.max {
35+
g.max = g.cur
36+
}
37+
if g.cur >= 2 {
38+
g.once.Do(func() { close(g.overlap) })
39+
}
40+
g.mu.Unlock()
41+
select {
42+
case <-g.overlap:
43+
case <-time.After(2 * time.Second):
44+
}
45+
g.mu.Lock()
46+
g.cur--
47+
g.mu.Unlock()
48+
}
49+
50+
func newGaugedUpstream(t *testing.T, g *inFlightGauge, tool string) *mcpserver.MCPServer {
51+
t.Helper()
52+
hooks := &mcpserver.Hooks{}
53+
hooks.AddBeforeListTools(func(context.Context, any, *mcp.ListToolsRequest) {
54+
// Connect lists tools too; only the sweep's lists are measured.
55+
if g.armed.Load() {
56+
g.enter()
57+
}
58+
})
59+
srv := mcpserver.NewMCPServer("gauged", "0.0.1", mcpserver.WithToolCapabilities(true), mcpserver.WithHooks(hooks))
60+
srv.AddTool(mcp.NewTool(tool, mcp.WithDescription("test tool")),
61+
func(context.Context, mcp.CallToolRequest) (*mcp.CallToolResult, error) {
62+
return mcp.NewToolResultText("ok"), nil
63+
})
64+
return srv
65+
}
66+
67+
// A slow upstream must not hold up the tools/list of every other server: the
68+
// sweep lists servers concurrently.
69+
func TestDiscoverToolsListsServersConcurrently(t *testing.T) {
70+
m, _ := newObservedManager(t)
71+
t.Setenv("MCPPROXY_DISABLE_OAUTH", "true")
72+
73+
g := &inFlightGauge{overlap: make(chan struct{})}
74+
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
75+
defer cancel()
76+
for i := 0; i < 3; i++ {
77+
ts := servertest.NewTestStreamableHTTPServer(newGaugedUpstream(t, g, fmt.Sprintf("tool_%d", i)))
78+
t.Cleanup(ts.Close)
79+
id := fmt.Sprintf("srv-%d", i)
80+
require.NoError(t, m.AddServerConfig(id, &config.ServerConfig{
81+
Name: fmt.Sprintf("server-%d", i), Protocol: "streamable-http", URL: ts.URL, Enabled: true,
82+
}))
83+
client, ok := m.GetClient(id)
84+
require.True(t, ok)
85+
require.NoError(t, client.Connect(ctx))
86+
}
87+
g.armed.Store(true)
88+
89+
tools, listed, err := m.DiscoverToolsReport(ctx, false)
90+
require.NoError(t, err)
91+
assert.Len(t, tools, 3)
92+
assert.Len(t, listed, 3)
93+
g.mu.Lock()
94+
peak := g.max
95+
g.mu.Unlock()
96+
assert.GreaterOrEqual(t, peak, 2, "tools/list requests never overlapped: the sweep lists servers one at a time")
97+
}
98+
99+
// slotGauge holds each tools/list request until `want` requests are in flight
100+
// (or a timeout passes), then keeps it a little longer so any request beyond
101+
// the concurrency bound would overlap with the full set and raise the peak.
102+
type slotGauge struct {
103+
mu sync.Mutex
104+
cur, max int
105+
want int
106+
full chan struct{}
107+
once sync.Once
108+
armed atomic.Bool
109+
}
110+
111+
func (g *slotGauge) enter() {
112+
g.mu.Lock()
113+
g.cur++
114+
if g.cur > g.max {
115+
g.max = g.cur
116+
}
117+
if g.cur >= g.want {
118+
g.once.Do(func() { close(g.full) })
119+
}
120+
g.mu.Unlock()
121+
select {
122+
case <-g.full:
123+
case <-time.After(2 * time.Second):
124+
}
125+
// Hold the slot so requests an unbounded sweep launches meanwhile are
126+
// counted against the peak.
127+
time.Sleep(50 * time.Millisecond)
128+
g.mu.Lock()
129+
g.cur--
130+
g.mu.Unlock()
131+
}
132+
133+
func newSlotGaugedUpstream(t *testing.T, g *slotGauge, tool string) *mcpserver.MCPServer {
134+
t.Helper()
135+
hooks := &mcpserver.Hooks{}
136+
hooks.AddBeforeListTools(func(context.Context, any, *mcp.ListToolsRequest) {
137+
if g.armed.Load() {
138+
g.enter()
139+
}
140+
})
141+
srv := mcpserver.NewMCPServer("slot-gauged", "0.0.1", mcpserver.WithToolCapabilities(true), mcpserver.WithHooks(hooks))
142+
srv.AddTool(mcp.NewTool(tool, mcp.WithDescription("test tool")),
143+
func(context.Context, mcp.CallToolRequest) (*mcp.CallToolResult, error) {
144+
return mcp.NewToolResultText("ok"), nil
145+
})
146+
return srv
147+
}
148+
149+
// The sweep lists concurrently but never more than discoveryListConcurrency
150+
// servers at once, however many are due.
151+
func TestDiscoverToolsBoundsConcurrentLists(t *testing.T) {
152+
m, _ := newObservedManager(t)
153+
t.Setenv("MCPPROXY_DISABLE_OAUTH", "true")
154+
155+
const serverCount = 12
156+
require.Greater(t, serverCount, discoveryListConcurrency)
157+
158+
g := &slotGauge{want: discoveryListConcurrency, full: make(chan struct{})}
159+
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
160+
defer cancel()
161+
for i := 0; i < serverCount; i++ {
162+
ts := servertest.NewTestStreamableHTTPServer(newSlotGaugedUpstream(t, g, fmt.Sprintf("tool_%d", i)))
163+
t.Cleanup(ts.Close)
164+
id := fmt.Sprintf("srv-%d", i)
165+
require.NoError(t, m.AddServerConfig(id, &config.ServerConfig{
166+
Name: fmt.Sprintf("server-%d", i), Protocol: "streamable-http", URL: ts.URL, Enabled: true,
167+
}))
168+
client, ok := m.GetClient(id)
169+
require.True(t, ok)
170+
require.NoError(t, client.Connect(ctx))
171+
}
172+
g.armed.Store(true)
173+
174+
tools, listed, err := m.DiscoverToolsReport(ctx, false)
175+
require.NoError(t, err)
176+
assert.Len(t, tools, serverCount)
177+
assert.Len(t, listed, serverCount)
178+
g.mu.Lock()
179+
peak := g.max
180+
g.mu.Unlock()
181+
assert.GreaterOrEqual(t, peak, 2, "tools/list requests never overlapped")
182+
assert.LessOrEqual(t, peak, discoveryListConcurrency, "more servers were listed at once than discoveryListConcurrency allows")
183+
}

‎internal/upstream/manager.go‎

Lines changed: 34 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1196,6 +1196,10 @@ func (m *Manager) DiscoverToolsDue(ctx context.Context) ([]*config.ToolMetadata,
11961196
return tools, err
11971197
}
11981198

1199+
// discoveryListConcurrency bounds how many upstream tools/list requests one
1200+
// discovery sweep runs at once.
1201+
const discoveryListConcurrency = 8
1202+
11991203
func (m *Manager) discoverTools(ctx context.Context, dueOnly bool) ([]*config.ToolMetadata, []string, map[string]managed.InventoryCapture, error) {
12001204
type clientSnapshot struct {
12011205
id string
@@ -1247,6 +1251,7 @@ func (m *Manager) discoverTools(ctx context.Context, dueOnly bool) ([]*config.To
12471251
connectedCount := 0
12481252
skippedNotDue := 0
12491253
known := make(map[string]struct{}, len(snapshots))
1254+
var due []clientSnapshot
12501255

12511256
for _, snapshot := range snapshots {
12521257
client := snapshot.client
@@ -1293,9 +1298,32 @@ func (m *Manager) discoverTools(ctx context.Context, dueOnly bool) ([]*config.To
12931298
}
12941299

12951300
connectedCount++
1301+
due = append(due, snapshot)
1302+
}
12961303

1297-
tools, capture, err := client.ListToolsCaptured(ctx)
1298-
if err != nil {
1304+
// List the due servers concurrently: one slow or large upstream must not
1305+
// hold up every other server's tools/list until its own timeout.
1306+
type listResult struct {
1307+
tools []*config.ToolMetadata
1308+
capture managed.InventoryCapture
1309+
err error
1310+
}
1311+
results := make([]listResult, len(due))
1312+
sem := make(chan struct{}, discoveryListConcurrency)
1313+
var wg sync.WaitGroup
1314+
for i, snapshot := range due {
1315+
wg.Go(func() {
1316+
sem <- struct{}{}
1317+
defer func() { <-sem }()
1318+
tools, capture, err := snapshot.client.ListToolsCaptured(ctx)
1319+
results[i] = listResult{tools: tools, capture: capture, err: err}
1320+
})
1321+
}
1322+
wg.Wait()
1323+
1324+
for i, snapshot := range due {
1325+
res := results[i]
1326+
if res.err != nil {
12991327
// Warn, not Error: markSwept below is deliberately skipped so the
13001328
// next sweep retries this server. A failure the code already plans
13011329
// to retry is not an error — and on a healthy install with several
@@ -1304,7 +1332,7 @@ func (m *Manager) discoverTools(ctx context.Context, dueOnly bool) ([]*config.To
13041332
m.logger.Warn("Failed to list tools from client",
13051333
zap.String("id", snapshot.id),
13061334
zap.String("server", snapshot.name),
1307-
zap.Error(err))
1335+
zap.Error(res.err))
13081336
continue
13091337
}
13101338

@@ -1313,11 +1341,11 @@ func (m *Manager) discoverTools(ctx context.Context, dueOnly bool) ([]*config.To
13131341
m.markSwept(snapshot.name, now)
13141342
if snapshot.name != "" {
13151343
listed = append(listed, snapshot.name)
1316-
captures[snapshot.name] = capture
1344+
captures[snapshot.name] = res.capture
13171345
}
13181346

1319-
if tools != nil {
1320-
allTools = append(allTools, tools...)
1347+
if res.tools != nil {
1348+
allTools = append(allTools, res.tools...)
13211349
}
13221350
}
13231351

0 commit comments

Comments
 (0)