Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
183 changes: 183 additions & 0 deletions internal/upstream/discovery_parallel_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,183 @@
package upstream

import (
"context"
"fmt"
"sync"
"sync/atomic"
"testing"
"time"

"github.com/mark3labs/mcp-go/mcp"
mcpserver "github.com/mark3labs/mcp-go/server"
servertest "github.com/mark3labs/mcp-go/server/servertest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"github.com/smart-mcp-proxy/mcpproxy-go/internal/config"
)

// inFlightGauge counts concurrent tools/list requests across upstreams. Each
// request waits until another one is in flight (or a timeout passes), so a
// sweep that lists servers one at a time never reaches a peak above 1.
type inFlightGauge struct {
mu sync.Mutex
cur, max int
overlap chan struct{}
once sync.Once
armed atomic.Bool
}

func (g *inFlightGauge) enter() {
g.mu.Lock()
g.cur++
if g.cur > g.max {
g.max = g.cur
}
if g.cur >= 2 {
g.once.Do(func() { close(g.overlap) })
}
g.mu.Unlock()
select {
case <-g.overlap:
case <-time.After(2 * time.Second):
}
g.mu.Lock()
g.cur--
g.mu.Unlock()
}

func newGaugedUpstream(t *testing.T, g *inFlightGauge, tool string) *mcpserver.MCPServer {
t.Helper()
hooks := &mcpserver.Hooks{}
hooks.AddBeforeListTools(func(context.Context, any, *mcp.ListToolsRequest) {
// Connect lists tools too; only the sweep's lists are measured.
if g.armed.Load() {
g.enter()
}
})
srv := mcpserver.NewMCPServer("gauged", "0.0.1", mcpserver.WithToolCapabilities(true), mcpserver.WithHooks(hooks))
srv.AddTool(mcp.NewTool(tool, mcp.WithDescription("test tool")),
func(context.Context, mcp.CallToolRequest) (*mcp.CallToolResult, error) {
return mcp.NewToolResultText("ok"), nil
})
return srv
}

// A slow upstream must not hold up the tools/list of every other server: the
// sweep lists servers concurrently.
func TestDiscoverToolsListsServersConcurrently(t *testing.T) {
m, _ := newObservedManager(t)
t.Setenv("MCPPROXY_DISABLE_OAUTH", "true")

g := &inFlightGauge{overlap: make(chan struct{})}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
for i := 0; i < 3; i++ {
ts := servertest.NewTestStreamableHTTPServer(newGaugedUpstream(t, g, fmt.Sprintf("tool_%d", i)))
t.Cleanup(ts.Close)
id := fmt.Sprintf("srv-%d", i)
require.NoError(t, m.AddServerConfig(id, &config.ServerConfig{
Name: fmt.Sprintf("server-%d", i), Protocol: "streamable-http", URL: ts.URL, Enabled: true,
}))
client, ok := m.GetClient(id)
require.True(t, ok)
require.NoError(t, client.Connect(ctx))
}
g.armed.Store(true)

tools, listed, err := m.DiscoverToolsReport(ctx, false)
require.NoError(t, err)
assert.Len(t, tools, 3)
assert.Len(t, listed, 3)
g.mu.Lock()
peak := g.max
g.mu.Unlock()
assert.GreaterOrEqual(t, peak, 2, "tools/list requests never overlapped: the sweep lists servers one at a time")
}

// slotGauge holds each tools/list request until `want` requests are in flight
// (or a timeout passes), then keeps it a little longer so any request beyond
// the concurrency bound would overlap with the full set and raise the peak.
type slotGauge struct {
mu sync.Mutex
cur, max int
want int
full chan struct{}
once sync.Once
armed atomic.Bool
}

func (g *slotGauge) enter() {
g.mu.Lock()
g.cur++
if g.cur > g.max {
g.max = g.cur
}
if g.cur >= g.want {
g.once.Do(func() { close(g.full) })
}
g.mu.Unlock()
select {
case <-g.full:
case <-time.After(2 * time.Second):
}
// Hold the slot so requests an unbounded sweep launches meanwhile are
// counted against the peak.
time.Sleep(50 * time.Millisecond)
g.mu.Lock()
g.cur--
g.mu.Unlock()
}

func newSlotGaugedUpstream(t *testing.T, g *slotGauge, tool string) *mcpserver.MCPServer {
t.Helper()
hooks := &mcpserver.Hooks{}
hooks.AddBeforeListTools(func(context.Context, any, *mcp.ListToolsRequest) {
if g.armed.Load() {
g.enter()
}
})
srv := mcpserver.NewMCPServer("slot-gauged", "0.0.1", mcpserver.WithToolCapabilities(true), mcpserver.WithHooks(hooks))
srv.AddTool(mcp.NewTool(tool, mcp.WithDescription("test tool")),
func(context.Context, mcp.CallToolRequest) (*mcp.CallToolResult, error) {
return mcp.NewToolResultText("ok"), nil
})
return srv
}

// The sweep lists concurrently but never more than discoveryListConcurrency
// servers at once, however many are due.
func TestDiscoverToolsBoundsConcurrentLists(t *testing.T) {
m, _ := newObservedManager(t)
t.Setenv("MCPPROXY_DISABLE_OAUTH", "true")

const serverCount = 12
require.Greater(t, serverCount, discoveryListConcurrency)

g := &slotGauge{want: discoveryListConcurrency, full: make(chan struct{})}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
for i := 0; i < serverCount; i++ {
ts := servertest.NewTestStreamableHTTPServer(newSlotGaugedUpstream(t, g, fmt.Sprintf("tool_%d", i)))
t.Cleanup(ts.Close)
id := fmt.Sprintf("srv-%d", i)
require.NoError(t, m.AddServerConfig(id, &config.ServerConfig{
Name: fmt.Sprintf("server-%d", i), Protocol: "streamable-http", URL: ts.URL, Enabled: true,
}))
client, ok := m.GetClient(id)
require.True(t, ok)
require.NoError(t, client.Connect(ctx))
}
g.armed.Store(true)

tools, listed, err := m.DiscoverToolsReport(ctx, false)
require.NoError(t, err)
assert.Len(t, tools, serverCount)
assert.Len(t, listed, serverCount)
g.mu.Lock()
peak := g.max
g.mu.Unlock()
assert.GreaterOrEqual(t, peak, 2, "tools/list requests never overlapped")
assert.LessOrEqual(t, peak, discoveryListConcurrency, "more servers were listed at once than discoveryListConcurrency allows")
}
40 changes: 34 additions & 6 deletions internal/upstream/manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -1196,6 +1196,10 @@ func (m *Manager) DiscoverToolsDue(ctx context.Context) ([]*config.ToolMetadata,
return tools, err
}

// discoveryListConcurrency bounds how many upstream tools/list requests one
// discovery sweep runs at once.
const discoveryListConcurrency = 8

func (m *Manager) discoverTools(ctx context.Context, dueOnly bool) ([]*config.ToolMetadata, []string, map[string]managed.InventoryCapture, error) {
type clientSnapshot struct {
id string
Expand Down Expand Up @@ -1247,6 +1251,7 @@ func (m *Manager) discoverTools(ctx context.Context, dueOnly bool) ([]*config.To
connectedCount := 0
skippedNotDue := 0
known := make(map[string]struct{}, len(snapshots))
var due []clientSnapshot

for _, snapshot := range snapshots {
client := snapshot.client
Expand Down Expand Up @@ -1293,9 +1298,32 @@ func (m *Manager) discoverTools(ctx context.Context, dueOnly bool) ([]*config.To
}

connectedCount++
due = append(due, snapshot)
}

tools, capture, err := client.ListToolsCaptured(ctx)
if err != nil {
// List the due servers concurrently: one slow or large upstream must not
// hold up every other server's tools/list until its own timeout.
type listResult struct {
tools []*config.ToolMetadata
capture managed.InventoryCapture
err error
}
results := make([]listResult, len(due))
sem := make(chan struct{}, discoveryListConcurrency)
var wg sync.WaitGroup
for i, snapshot := range due {
wg.Go(func() {
sem <- struct{}{}
defer func() { <-sem }()
tools, capture, err := snapshot.client.ListToolsCaptured(ctx)
results[i] = listResult{tools: tools, capture: capture, err: err}
})
}
wg.Wait()

for i, snapshot := range due {
res := results[i]
if res.err != nil {
// Warn, not Error: markSwept below is deliberately skipped so the
// next sweep retries this server. A failure the code already plans
// to retry is not an error — and on a healthy install with several
Expand All @@ -1304,7 +1332,7 @@ func (m *Manager) discoverTools(ctx context.Context, dueOnly bool) ([]*config.To
m.logger.Warn("Failed to list tools from client",
zap.String("id", snapshot.id),
zap.String("server", snapshot.name),
zap.Error(err))
zap.Error(res.err))
continue
}

Expand All @@ -1313,11 +1341,11 @@ func (m *Manager) discoverTools(ctx context.Context, dueOnly bool) ([]*config.To
m.markSwept(snapshot.name, now)
if snapshot.name != "" {
listed = append(listed, snapshot.name)
captures[snapshot.name] = capture
captures[snapshot.name] = res.capture
}

if tools != nil {
allTools = append(allTools, tools...)
if res.tools != nil {
allTools = append(allTools, res.tools...)
}
}

Expand Down
Loading