Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78 changes: 78 additions & 0 deletions internal/runtime/supervisor/retry_after_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
package supervisor

import (
"testing"
"time"

"github.com/stretchr/testify/require"
"go.uber.org/zap"

"github.com/smart-mcp-proxy/mcpproxy-go/internal/config"
"github.com/smart-mcp-proxy/mcpproxy-go/internal/runtime/configsvc"
"github.com/smart-mcp-proxy/mcpproxy-go/internal/upstream/types"
)

// TestSupervisor_Reconcile_RespectsRetryAfter is the supervisor half of #1040's
// acceptance: while a rate-limited upstream's Retry-After window is open the
// periodic reconciliation must plan no ActionConnect, even though the ordinary
// exponential-backoff window has long since elapsed.
func TestSupervisor_Reconcile_RespectsRetryAfter(t *testing.T) {
cfg := &config.Config{
Listen: "127.0.0.1:8080",
Servers: []*config.ServerConfig{
{Name: "rate-limited-server", Enabled: true},
},
}

configSvc := configsvc.NewService(cfg, "/tmp/config.json", zap.NewNop())
defer configSvc.Close()

mockUpstream := NewMockUpstreamAdapter()
defer mockUpstream.Close()

supervisor := New(configSvc, mockUpstream, zap.NewNop())

require.NoError(t, supervisor.reconcile(configSvc.Current()))
supervisor.actionWg.Wait()

setConnectionState := func(info *types.ConnectionInfo) {
mockUpstream.mu.Lock()
defer mockUpstream.mu.Unlock()
mockUpstream.connected["rate-limited-server"] = false
if state, ok := mockUpstream.states["rate-limited-server"]; ok {
state.Connected = false
state.ConnectionInfo = info
}
}
isConnected := func() bool {
mockUpstream.mu.Lock()
defer mockUpstream.mu.Unlock()
return mockUpstream.connected["rate-limited-server"]
}
reconcileAndDrain := func() {
t.Helper()
require.NoError(t, supervisor.reconcile(configSvc.Current()))
supervisor.actionWg.Wait()
}

// `Retry-After: 3600` on a 429: the ladder says "dial now" (its window
// elapsed long ago), the upstream says "not for an hour". The upstream wins.
setConnectionState(&types.ConnectionInfo{
State: types.StateError,
RetryCount: 1,
LastRetryTime: time.Now().Add(-time.Hour),
RetryAfter: time.Now().Add(time.Hour),
})
reconcileAndDrain()
require.False(t, isConnected(), "supervisor re-dialed a server inside its Retry-After window")

// Once the window has passed the normal ladder takes over again.
setConnectionState(&types.ConnectionInfo{
State: types.StateError,
RetryCount: 1,
LastRetryTime: time.Now().Add(-time.Hour),
RetryAfter: time.Now().Add(-time.Second),
})
reconcileAndDrain()
require.True(t, isConnected(), "supervisor never came back after the Retry-After window elapsed")
}
184 changes: 97 additions & 87 deletions internal/transport/http.go
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,36 @@ type HTTPTransportConfig struct {
// credential broker can drive injection without this package importing it.
BrokeredAuth *BrokeredAuth
TraceEnabled bool // Enable detailed HTTP/SSE frame tracing
// RetryAfter, when set, receives the `Retry-After` hints observed on this
// upstream's rate-limited responses (#1040). mcp-go flattens non-2xx
// responses into strings, so a RoundTripper is the last place the header
// still exists. nil disables the wrapper entirely (unchanged behaviour).
RetryAfter *RetryAfterRecorder
}

// upstreamRoundTripper composes the outbound transport wrappers for an upstream
// HTTP/SSE client: frame tracing (opt-in) and the Retry-After recorder (#1040).
// It always WRAPS the given base rather than replacing it, and it sits strictly
// below mcp-go's OAuth handling — the OAuth authorize/token wrappers live above
// this layer and are untouched.
func (cfg *HTTPTransportConfig) upstreamRoundTripper(base http.RoundTripper, logger *zap.Logger) http.RoundTripper {
rt := base
if rt == nil {
rt = http.DefaultTransport
}
if cfg.TraceEnabled {
rt = NewLoggingTransport(rt, logger)
}
if cfg.RetryAfter != nil {
rt = NewRetryAfterTransport(rt, cfg.RetryAfter, logger)
}
return rt
}

// needsCustomTransport reports whether this config requires us to hand mcp-go
// our own *http.Client instead of letting it build the default one.
func (cfg *HTTPTransportConfig) needsCustomTransport() bool {
return cfg.TraceEnabled || cfg.RetryAfter != nil
}

// effectiveHeaders returns the outbound header set, applying brokered per-user
Expand Down Expand Up @@ -202,7 +232,20 @@ func CreateHTTPClient(cfg *HTTPTransportConfig) (*client.Client, error) {
zap.String("note", "Extra parameters will be injected into OAuth requests"))
}

client, err := client.NewOAuthStreamableHttpClient(cfg.URL, *cfg.OAuthConfig)
// mcp-go builds the OAuth client's transport itself; OAuthConfig.HTTPClient
// only covers the OAuth metadata/DCR/token requests, NOT the MCP requests.
// Passing our own basic client is therefore the only way the Retry-After
// recorder (#1040) sees a 429 on the MCP endpoint. No Timeout is set, to
// preserve mcp-go's default for this branch (OAuth flows rely on the
// caller's context deadline, which can be up to 30 minutes).
var oauthOpts []transport.StreamableHTTPCOption
if cfg.needsCustomTransport() {
oauthOpts = append(oauthOpts, transport.WithHTTPBasicClient(&http.Client{
Transport: cfg.upstreamRoundTripper(http.DefaultTransport, logger),
}))
}

client, err := client.NewOAuthStreamableHttpClient(cfg.URL, *cfg.OAuthConfig, oauthOpts...)
if err != nil {
logger.Error("Failed to create OAuth client", zap.Error(err))
return nil, fmt.Errorf("failed to create OAuth client: %w", err)
Expand All @@ -219,49 +262,40 @@ func CreateHTTPClient(cfg *HTTPTransportConfig) (*client.Client, error) {
// inbound gateway/IdP token (FR-017).
headers := cfg.effectiveHeaders()

// If tracing is enabled, create HTTP client with logging transport
if cfg.TraceEnabled {
logger.Info("🔍 HTTP TRACE MODE ENABLED - All HTTP traffic will be logged")
baseTransport := &http.Transport{
MaxIdleConns: 10,
IdleConnTimeout: 90 * time.Second,
ResponseHeaderTimeout: 30 * time.Second,
}
loggingTransport := NewLoggingTransport(baseTransport, logger)
httpClient := &http.Client{
Transport: loggingTransport,
Timeout: 180 * time.Second,
}

var httpTransport transport.Interface
var err error
if len(headers) > 0 {
httpTransport, err = transport.NewStreamableHTTP(cfg.URL,
transport.WithHTTPHeaders(headers),
transport.WithHTTPBasicClient(httpClient))
} else {
httpTransport, err = transport.NewStreamableHTTP(cfg.URL,
transport.WithHTTPBasicClient(httpClient))
}
if err != nil {
return nil, fmt.Errorf("failed to create HTTP transport with tracing: %w", err)
}
return client.NewClient(httpTransport), nil
}

// Use regular HTTP client
opts := []transport.StreamableHTTPCOption{}
if len(headers) > 0 {
logger.Debug("Adding HTTP headers", zap.Int("header_count", len(headers)))
httpTransport, err := transport.NewStreamableHTTP(cfg.URL,
transport.WithHTTPHeaders(headers))
if err != nil {
return nil, fmt.Errorf("failed to create HTTP transport: %w", err)
opts = append(opts, transport.WithHTTPHeaders(headers))
}

switch {
case cfg.needsCustomTransport():
// Timeout policy is preserved exactly as it was before #1040: the trace
// client and the header-less client carry a 180s request timeout, while
// the headers variant has always run on mcp-go's default (no timeout),
// which long-running tool calls depend on.
base := http.RoundTripper(http.DefaultTransport)
timeout := time.Duration(0)
if cfg.TraceEnabled {
logger.Info("🔍 HTTP TRACE MODE ENABLED - All HTTP traffic will be logged")
base = &http.Transport{
MaxIdleConns: 10,
IdleConnTimeout: 90 * time.Second,
ResponseHeaderTimeout: 30 * time.Second,
}
timeout = 180 * time.Second
} else if len(headers) == 0 {
timeout = 180 * time.Second
}
return client.NewClient(httpTransport), nil
opts = append(opts, transport.WithHTTPBasicClient(&http.Client{
Transport: cfg.upstreamRoundTripper(base, logger),
Timeout: timeout,
}))
case len(headers) == 0:
opts = append(opts, transport.WithHTTPTimeout(180*time.Second)) // Increased timeout for HTTP connections
}

httpTransport, err := transport.NewStreamableHTTP(cfg.URL,
transport.WithHTTPTimeout(180*time.Second)) // Increased timeout for HTTP connections
httpTransport, err := transport.NewStreamableHTTP(cfg.URL, opts...)
if err != nil {
return nil, fmt.Errorf("failed to create HTTP transport: %w", err)
}
Expand Down Expand Up @@ -313,7 +347,18 @@ func CreateSSEClient(cfg *HTTPTransportConfig) (*client.Client, error) {
zap.String("note", "Extra parameters will be injected into OAuth requests"))
}

client, err := client.NewOAuthSSEClient(cfg.URL, *cfg.OAuthConfig)
// As in the streamable-HTTP OAuth branch: OAuthConfig.HTTPClient covers
// only the OAuth metadata/DCR/token calls, so the MCP requests need our
// own client for the Retry-After recorder to see a 429 (#1040). No
// Timeout — SSE streams are long-lived by design.
var oauthOpts []transport.ClientOption
if cfg.needsCustomTransport() {
oauthOpts = append(oauthOpts, client.WithHTTPClient(&http.Client{
Transport: cfg.upstreamRoundTripper(http.DefaultTransport, logger),
}))
}

client, err := client.NewOAuthSSEClient(cfg.URL, *cfg.OAuthConfig, oauthOpts...)
if err != nil {
logger.Error("Failed to create OAuth SSE client", zap.Error(err))
return nil, fmt.Errorf("failed to create OAuth SSE client: %w", err)
Expand All @@ -330,48 +375,6 @@ func CreateSSEClient(cfg *HTTPTransportConfig) (*client.Client, error) {
// inbound gateway/IdP token (FR-017).
headers := cfg.effectiveHeaders()

// Use regular SSE client
if len(headers) > 0 {
logger.Debug("Adding SSE headers", zap.Int("header_count", len(headers)))
// Create custom HTTP client for SSE - NO Timeout field to allow indefinite streaming
// The Timeout field covers the entire request duration, which kills long-lived SSE streams
// Instead, we rely on IdleConnTimeout to detect dead connections
baseTransport := &http.Transport{
MaxIdleConns: 10,
IdleConnTimeout: 300 * time.Second, // 5 minutes idle before closing
DisableCompression: false,
DisableKeepAlives: false, // Enable keep-alives for SSE stability
MaxIdleConnsPerHost: 5,
// ResponseHeaderTimeout can be used to timeout initial connection, but not ongoing stream
ResponseHeaderTimeout: 30 * time.Second,
}

var transport http.RoundTripper = baseTransport
if cfg.TraceEnabled {
logger.Info("🔍 SSE TRACE MODE ENABLED - All HTTP traffic and SSE frames will be logged")
transport = NewLoggingTransport(baseTransport, logger)
}

httpClient := &http.Client{
Transport: transport,
}

logger.Info("Creating SSE MCP client with indefinite timeout for long-lived streams",
zap.String("url", cfg.URL),
zap.Duration("idle_timeout", 300*time.Second),
zap.Duration("header_timeout", 30*time.Second),
zap.Int("header_count", len(headers)),
zap.String("note", "Removed http.Client.Timeout to allow SSE streams longer than 3 minutes"))

sseClient, err := client.NewSSEMCPClient(cfg.URL,
client.WithHTTPClient(httpClient),
client.WithHeaders(headers))
if err != nil {
return nil, fmt.Errorf("failed to create SSE client: %w", err)
}
return sseClient, nil
}

// Create custom HTTP client for SSE - NO Timeout field to allow indefinite streaming
// The Timeout field covers the entire request duration, which kills long-lived SSE streams
// Instead, we rely on IdleConnTimeout to detect dead connections
Expand All @@ -385,20 +388,22 @@ func CreateSSEClient(cfg *HTTPTransportConfig) (*client.Client, error) {
ResponseHeaderTimeout: 30 * time.Second,
}

var transport http.RoundTripper = baseTransport
if cfg.TraceEnabled {
logger.Info("🔍 SSE TRACE MODE ENABLED - All HTTP traffic and SSE frames will be logged")
transport = NewLoggingTransport(baseTransport, logger)
}

// Tracing and the Retry-After recorder (#1040) both wrap the base transport;
// the SSE-specific tuning above and the deliberately absent Client.Timeout
// are unchanged.
httpClient := &http.Client{
Transport: transport,
Transport: cfg.upstreamRoundTripper(baseTransport, logger),
}

logger.Info("Creating SSE MCP client with indefinite timeout for long-lived streams",
zap.String("url", cfg.URL),
zap.Duration("idle_timeout", 300*time.Second),
zap.Duration("header_timeout", 30*time.Second),
zap.Int("header_count", len(headers)),
zap.String("note", "Removed http.Client.Timeout to allow SSE streams longer than 3 minutes"))

// Enhanced trace-level debugging for SSE transport
Expand All @@ -411,8 +416,13 @@ func CreateSSEClient(cfg *HTTPTransportConfig) (*client.Client, error) {
zap.String("debug_note", "SSE client will establish persistent connection for JSON-RPC over SSE with no overall timeout"))
}

sseClient, err := client.NewSSEMCPClient(cfg.URL,
client.WithHTTPClient(httpClient))
sseOpts := []transport.ClientOption{client.WithHTTPClient(httpClient)}
if len(headers) > 0 {
logger.Debug("Adding SSE headers", zap.Int("header_count", len(headers)))
sseOpts = append(sseOpts, client.WithHeaders(headers))
}

sseClient, err := client.NewSSEMCPClient(cfg.URL, sseOpts...)
if err != nil {
return nil, fmt.Errorf("failed to create SSE client: %w", err)
}
Expand Down
Loading
Loading