From 750977d27a6f4fbed0911e44a021df3158431fb1 Mon Sep 17 00:00:00 2001 From: Algis Dumbris Date: Tue, 25 Aug 2026 06:53:11 +0300 Subject: [PATCH 1/3] feat(upstream): honor Retry-After on 429 via transport RoundTripper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit mcp-go flattens every non-2xx upstream response into an error string, so a 429's Retry-After header was gone before any connection-state code ran and the reconnect ladder paced a rate-limited upstream on its own schedule. Capture the hint where the *http.Response still exists: a RetryAfterTransport RoundTripper installed under every HTTP/SSE client we hand to mcp-go, feeding a per-server RetryAfterRecorder. The managed client stamps the resulting deadline onto the state machine, and ConnectionInfo.ShouldAutoReconnect / ShouldRetry treat it as a floor under the existing backoff — effective delay is max(backoff, Retry-After), capped at one hour. - Both RFC 7231 forms parse (delta-seconds and HTTP-date); a malformed, absent, or already-elapsed value falls back to the existing ladder untouched. - 503 counts only when it carries a parseable hint; a bare 503 is an outage the ladder already paces. - The wrapper composes with the existing tracing transport and sits below mcp-go's OAuth handling, so the three authorize paths are unchanged. The two OAuth client constructors now receive our basic client explicitly, since OAuthConfig.HTTPClient covers only metadata/DCR/token requests. - The window is cleared on a successful connect and on Reset, so a stale hint cannot hold back a later, unrelated reconnect. Closes #1040. Refs #1013, #1039. --- .../runtime/supervisor/retry_after_test.go | 78 ++++++++ internal/transport/http.go | 184 +++++++++-------- internal/transport/retry_after.go | 188 ++++++++++++++++++ internal/transport/retry_after_test.go | 181 +++++++++++++++++ internal/upstream/core/client.go | 33 +++ internal/upstream/core/connection.go | 4 + internal/upstream/core/connection_http.go | 6 +- internal/upstream/core/connection_oauth.go | 10 +- internal/upstream/managed/client.go | 13 ++ internal/upstream/retry_after_connect_test.go | 176 ++++++++++++++++ internal/upstream/types/retry_after_test.go | 102 ++++++++++ internal/upstream/types/types.go | 57 ++++++ 12 files changed, 937 insertions(+), 95 deletions(-) create mode 100644 internal/runtime/supervisor/retry_after_test.go create mode 100644 internal/transport/retry_after.go create mode 100644 internal/transport/retry_after_test.go create mode 100644 internal/upstream/retry_after_connect_test.go create mode 100644 internal/upstream/types/retry_after_test.go diff --git a/internal/runtime/supervisor/retry_after_test.go b/internal/runtime/supervisor/retry_after_test.go new file mode 100644 index 000000000..491914704 --- /dev/null +++ b/internal/runtime/supervisor/retry_after_test.go @@ -0,0 +1,78 @@ +package supervisor + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + "github.com/smart-mcp-proxy/mcpproxy-go/internal/config" + "github.com/smart-mcp-proxy/mcpproxy-go/internal/runtime/configsvc" + "github.com/smart-mcp-proxy/mcpproxy-go/internal/upstream/types" +) + +// TestSupervisor_Reconcile_RespectsRetryAfter is the supervisor half of #1040's +// acceptance: while a rate-limited upstream's Retry-After window is open the +// periodic reconciliation must plan no ActionConnect, even though the ordinary +// exponential-backoff window has long since elapsed. +func TestSupervisor_Reconcile_RespectsRetryAfter(t *testing.T) { + cfg := &config.Config{ + Listen: "127.0.0.1:8080", + Servers: []*config.ServerConfig{ + {Name: "rate-limited-server", Enabled: true}, + }, + } + + configSvc := configsvc.NewService(cfg, "/tmp/config.json", zap.NewNop()) + defer configSvc.Close() + + mockUpstream := NewMockUpstreamAdapter() + defer mockUpstream.Close() + + supervisor := New(configSvc, mockUpstream, zap.NewNop()) + + require.NoError(t, supervisor.reconcile(configSvc.Current())) + supervisor.actionWg.Wait() + + setConnectionState := func(info *types.ConnectionInfo) { + mockUpstream.mu.Lock() + defer mockUpstream.mu.Unlock() + mockUpstream.connected["rate-limited-server"] = false + if state, ok := mockUpstream.states["rate-limited-server"]; ok { + state.Connected = false + state.ConnectionInfo = info + } + } + isConnected := func() bool { + mockUpstream.mu.Lock() + defer mockUpstream.mu.Unlock() + return mockUpstream.connected["rate-limited-server"] + } + reconcileAndDrain := func() { + t.Helper() + require.NoError(t, supervisor.reconcile(configSvc.Current())) + supervisor.actionWg.Wait() + } + + // `Retry-After: 3600` on a 429: the ladder says "dial now" (its window + // elapsed long ago), the upstream says "not for an hour". The upstream wins. + setConnectionState(&types.ConnectionInfo{ + State: types.StateError, + RetryCount: 1, + LastRetryTime: time.Now().Add(-time.Hour), + RetryAfter: time.Now().Add(time.Hour), + }) + reconcileAndDrain() + require.False(t, isConnected(), "supervisor re-dialed a server inside its Retry-After window") + + // Once the window has passed the normal ladder takes over again. + setConnectionState(&types.ConnectionInfo{ + State: types.StateError, + RetryCount: 1, + LastRetryTime: time.Now().Add(-time.Hour), + RetryAfter: time.Now().Add(-time.Second), + }) + reconcileAndDrain() + require.True(t, isConnected(), "supervisor never came back after the Retry-After window elapsed") +} diff --git a/internal/transport/http.go b/internal/transport/http.go index b4a931299..f22d6d3f4 100644 --- a/internal/transport/http.go +++ b/internal/transport/http.go @@ -137,6 +137,36 @@ type HTTPTransportConfig struct { // credential broker can drive injection without this package importing it. BrokeredAuth *BrokeredAuth TraceEnabled bool // Enable detailed HTTP/SSE frame tracing + // RetryAfter, when set, receives the `Retry-After` hints observed on this + // upstream's rate-limited responses (#1040). mcp-go flattens non-2xx + // responses into strings, so a RoundTripper is the last place the header + // still exists. nil disables the wrapper entirely (unchanged behaviour). + RetryAfter *RetryAfterRecorder +} + +// upstreamRoundTripper composes the outbound transport wrappers for an upstream +// HTTP/SSE client: frame tracing (opt-in) and the Retry-After recorder (#1040). +// It always WRAPS the given base rather than replacing it, and it sits strictly +// below mcp-go's OAuth handling — the OAuth authorize/token wrappers live above +// this layer and are untouched. +func (cfg *HTTPTransportConfig) upstreamRoundTripper(base http.RoundTripper, logger *zap.Logger) http.RoundTripper { + rt := base + if rt == nil { + rt = http.DefaultTransport + } + if cfg.TraceEnabled { + rt = NewLoggingTransport(rt, logger) + } + if cfg.RetryAfter != nil { + rt = NewRetryAfterTransport(rt, cfg.RetryAfter, logger) + } + return rt +} + +// needsCustomTransport reports whether this config requires us to hand mcp-go +// our own *http.Client instead of letting it build the default one. +func (cfg *HTTPTransportConfig) needsCustomTransport() bool { + return cfg.TraceEnabled || cfg.RetryAfter != nil } // effectiveHeaders returns the outbound header set, applying brokered per-user @@ -202,7 +232,20 @@ func CreateHTTPClient(cfg *HTTPTransportConfig) (*client.Client, error) { zap.String("note", "Extra parameters will be injected into OAuth requests")) } - client, err := client.NewOAuthStreamableHttpClient(cfg.URL, *cfg.OAuthConfig) + // mcp-go builds the OAuth client's transport itself; OAuthConfig.HTTPClient + // only covers the OAuth metadata/DCR/token requests, NOT the MCP requests. + // Passing our own basic client is therefore the only way the Retry-After + // recorder (#1040) sees a 429 on the MCP endpoint. No Timeout is set, to + // preserve mcp-go's default for this branch (OAuth flows rely on the + // caller's context deadline, which can be up to 30 minutes). + var oauthOpts []transport.StreamableHTTPCOption + if cfg.needsCustomTransport() { + oauthOpts = append(oauthOpts, transport.WithHTTPBasicClient(&http.Client{ + Transport: cfg.upstreamRoundTripper(http.DefaultTransport, logger), + })) + } + + client, err := client.NewOAuthStreamableHttpClient(cfg.URL, *cfg.OAuthConfig, oauthOpts...) if err != nil { logger.Error("Failed to create OAuth client", zap.Error(err)) return nil, fmt.Errorf("failed to create OAuth client: %w", err) @@ -219,49 +262,40 @@ func CreateHTTPClient(cfg *HTTPTransportConfig) (*client.Client, error) { // inbound gateway/IdP token (FR-017). headers := cfg.effectiveHeaders() - // If tracing is enabled, create HTTP client with logging transport - if cfg.TraceEnabled { - logger.Info("🔍 HTTP TRACE MODE ENABLED - All HTTP traffic will be logged") - baseTransport := &http.Transport{ - MaxIdleConns: 10, - IdleConnTimeout: 90 * time.Second, - ResponseHeaderTimeout: 30 * time.Second, - } - loggingTransport := NewLoggingTransport(baseTransport, logger) - httpClient := &http.Client{ - Transport: loggingTransport, - Timeout: 180 * time.Second, - } - - var httpTransport transport.Interface - var err error - if len(headers) > 0 { - httpTransport, err = transport.NewStreamableHTTP(cfg.URL, - transport.WithHTTPHeaders(headers), - transport.WithHTTPBasicClient(httpClient)) - } else { - httpTransport, err = transport.NewStreamableHTTP(cfg.URL, - transport.WithHTTPBasicClient(httpClient)) - } - if err != nil { - return nil, fmt.Errorf("failed to create HTTP transport with tracing: %w", err) - } - return client.NewClient(httpTransport), nil - } - - // Use regular HTTP client + opts := []transport.StreamableHTTPCOption{} if len(headers) > 0 { logger.Debug("Adding HTTP headers", zap.Int("header_count", len(headers))) - httpTransport, err := transport.NewStreamableHTTP(cfg.URL, - transport.WithHTTPHeaders(headers)) - if err != nil { - return nil, fmt.Errorf("failed to create HTTP transport: %w", err) + opts = append(opts, transport.WithHTTPHeaders(headers)) + } + + switch { + case cfg.needsCustomTransport(): + // Timeout policy is preserved exactly as it was before #1040: the trace + // client and the header-less client carry a 180s request timeout, while + // the headers variant has always run on mcp-go's default (no timeout), + // which long-running tool calls depend on. + base := http.RoundTripper(http.DefaultTransport) + timeout := time.Duration(0) + if cfg.TraceEnabled { + logger.Info("🔍 HTTP TRACE MODE ENABLED - All HTTP traffic will be logged") + base = &http.Transport{ + MaxIdleConns: 10, + IdleConnTimeout: 90 * time.Second, + ResponseHeaderTimeout: 30 * time.Second, + } + timeout = 180 * time.Second + } else if len(headers) == 0 { + timeout = 180 * time.Second } - return client.NewClient(httpTransport), nil + opts = append(opts, transport.WithHTTPBasicClient(&http.Client{ + Transport: cfg.upstreamRoundTripper(base, logger), + Timeout: timeout, + })) + case len(headers) == 0: + opts = append(opts, transport.WithHTTPTimeout(180*time.Second)) // Increased timeout for HTTP connections } - httpTransport, err := transport.NewStreamableHTTP(cfg.URL, - transport.WithHTTPTimeout(180*time.Second)) // Increased timeout for HTTP connections + httpTransport, err := transport.NewStreamableHTTP(cfg.URL, opts...) if err != nil { return nil, fmt.Errorf("failed to create HTTP transport: %w", err) } @@ -313,7 +347,18 @@ func CreateSSEClient(cfg *HTTPTransportConfig) (*client.Client, error) { zap.String("note", "Extra parameters will be injected into OAuth requests")) } - client, err := client.NewOAuthSSEClient(cfg.URL, *cfg.OAuthConfig) + // As in the streamable-HTTP OAuth branch: OAuthConfig.HTTPClient covers + // only the OAuth metadata/DCR/token calls, so the MCP requests need our + // own client for the Retry-After recorder to see a 429 (#1040). No + // Timeout — SSE streams are long-lived by design. + var oauthOpts []transport.ClientOption + if cfg.needsCustomTransport() { + oauthOpts = append(oauthOpts, client.WithHTTPClient(&http.Client{ + Transport: cfg.upstreamRoundTripper(http.DefaultTransport, logger), + })) + } + + client, err := client.NewOAuthSSEClient(cfg.URL, *cfg.OAuthConfig, oauthOpts...) if err != nil { logger.Error("Failed to create OAuth SSE client", zap.Error(err)) return nil, fmt.Errorf("failed to create OAuth SSE client: %w", err) @@ -330,48 +375,6 @@ func CreateSSEClient(cfg *HTTPTransportConfig) (*client.Client, error) { // inbound gateway/IdP token (FR-017). headers := cfg.effectiveHeaders() - // Use regular SSE client - if len(headers) > 0 { - logger.Debug("Adding SSE headers", zap.Int("header_count", len(headers))) - // Create custom HTTP client for SSE - NO Timeout field to allow indefinite streaming - // The Timeout field covers the entire request duration, which kills long-lived SSE streams - // Instead, we rely on IdleConnTimeout to detect dead connections - baseTransport := &http.Transport{ - MaxIdleConns: 10, - IdleConnTimeout: 300 * time.Second, // 5 minutes idle before closing - DisableCompression: false, - DisableKeepAlives: false, // Enable keep-alives for SSE stability - MaxIdleConnsPerHost: 5, - // ResponseHeaderTimeout can be used to timeout initial connection, but not ongoing stream - ResponseHeaderTimeout: 30 * time.Second, - } - - var transport http.RoundTripper = baseTransport - if cfg.TraceEnabled { - logger.Info("🔍 SSE TRACE MODE ENABLED - All HTTP traffic and SSE frames will be logged") - transport = NewLoggingTransport(baseTransport, logger) - } - - httpClient := &http.Client{ - Transport: transport, - } - - logger.Info("Creating SSE MCP client with indefinite timeout for long-lived streams", - zap.String("url", cfg.URL), - zap.Duration("idle_timeout", 300*time.Second), - zap.Duration("header_timeout", 30*time.Second), - zap.Int("header_count", len(headers)), - zap.String("note", "Removed http.Client.Timeout to allow SSE streams longer than 3 minutes")) - - sseClient, err := client.NewSSEMCPClient(cfg.URL, - client.WithHTTPClient(httpClient), - client.WithHeaders(headers)) - if err != nil { - return nil, fmt.Errorf("failed to create SSE client: %w", err) - } - return sseClient, nil - } - // Create custom HTTP client for SSE - NO Timeout field to allow indefinite streaming // The Timeout field covers the entire request duration, which kills long-lived SSE streams // Instead, we rely on IdleConnTimeout to detect dead connections @@ -385,20 +388,22 @@ func CreateSSEClient(cfg *HTTPTransportConfig) (*client.Client, error) { ResponseHeaderTimeout: 30 * time.Second, } - var transport http.RoundTripper = baseTransport if cfg.TraceEnabled { logger.Info("🔍 SSE TRACE MODE ENABLED - All HTTP traffic and SSE frames will be logged") - transport = NewLoggingTransport(baseTransport, logger) } + // Tracing and the Retry-After recorder (#1040) both wrap the base transport; + // the SSE-specific tuning above and the deliberately absent Client.Timeout + // are unchanged. httpClient := &http.Client{ - Transport: transport, + Transport: cfg.upstreamRoundTripper(baseTransport, logger), } logger.Info("Creating SSE MCP client with indefinite timeout for long-lived streams", zap.String("url", cfg.URL), zap.Duration("idle_timeout", 300*time.Second), zap.Duration("header_timeout", 30*time.Second), + zap.Int("header_count", len(headers)), zap.String("note", "Removed http.Client.Timeout to allow SSE streams longer than 3 minutes")) // Enhanced trace-level debugging for SSE transport @@ -411,8 +416,13 @@ func CreateSSEClient(cfg *HTTPTransportConfig) (*client.Client, error) { zap.String("debug_note", "SSE client will establish persistent connection for JSON-RPC over SSE with no overall timeout")) } - sseClient, err := client.NewSSEMCPClient(cfg.URL, - client.WithHTTPClient(httpClient)) + sseOpts := []transport.ClientOption{client.WithHTTPClient(httpClient)} + if len(headers) > 0 { + logger.Debug("Adding SSE headers", zap.Int("header_count", len(headers))) + sseOpts = append(sseOpts, client.WithHeaders(headers)) + } + + sseClient, err := client.NewSSEMCPClient(cfg.URL, sseOpts...) if err != nil { return nil, fmt.Errorf("failed to create SSE client: %w", err) } diff --git a/internal/transport/retry_after.go b/internal/transport/retry_after.go new file mode 100644 index 000000000..653fa032f --- /dev/null +++ b/internal/transport/retry_after.go @@ -0,0 +1,188 @@ +package transport + +import ( + "net/http" + "strconv" + "strings" + "sync" + "time" + + "go.uber.org/zap" +) + +// MaxRetryAfterDelay caps how long an upstream-supplied Retry-After may park a +// server. RFC 7231 puts no ceiling on the value, so a misconfigured (or hostile) +// upstream could hand us days. An hour is long enough to respect a real vendor +// rate-limit window and short enough that a bogus hint self-heals without a +// human having to hit "reconnect". +const MaxRetryAfterDelay = time.Hour + +// RetryAfterRecorder captures the rate-limit hints observed on one upstream's +// HTTP responses. +// +// It exists because mcp-go flattens every non-2xx response into a bare string +// (`request failed with status %d: %s` in client/transport/streamable_http.go), +// so by the time a connect error reaches the state machine the `Retry-After` +// header is gone. The only layer that still holds the *http.Response is the +// RoundTripper underneath the MCP client — hence RetryAfterTransport below, +// which feeds this recorder (#1040). +// +// A nil *RetryAfterRecorder is usable and inert, so callers that do not care +// about rate-limit hints can pass nil everywhere. +type RetryAfterRecorder struct { + mu sync.Mutex + deadline time.Time + status int +} + +// NewRetryAfterRecorder returns an empty recorder. +func NewRetryAfterRecorder() *RetryAfterRecorder { + return &RetryAfterRecorder{} +} + +// Record notes that the upstream asked us to wait `delay` as of `now`. The +// delay is clamped to MaxRetryAfterDelay and only ever extends the current +// park window: a later, shorter hint must not shorten a wait the upstream +// already asked for. It returns the effective deadline. +func (r *RetryAfterRecorder) Record(now time.Time, delay time.Duration, status int) time.Time { + if r == nil || delay <= 0 { + return time.Time{} + } + if delay > MaxRetryAfterDelay { + delay = MaxRetryAfterDelay + } + + deadline := now.Add(delay) + + r.mu.Lock() + defer r.mu.Unlock() + if deadline.After(r.deadline) { + r.deadline = deadline + r.status = status + } + return r.deadline +} + +// Deadline returns the instant before which no automatic reconnect should be +// attempted, or the zero time when no hint has been observed. +func (r *RetryAfterRecorder) Deadline() time.Time { + if r == nil { + return time.Time{} + } + r.mu.Lock() + defer r.mu.Unlock() + return r.deadline +} + +// Status returns the HTTP status that produced the current deadline (0 when +// there is none). Used for logging. +func (r *RetryAfterRecorder) Status() int { + if r == nil { + return 0 + } + r.mu.Lock() + defer r.mu.Unlock() + return r.status +} + +// Clear drops any recorded hint. Called once a connection succeeds so a stale +// window cannot hold back a later, unrelated reconnect. +func (r *RetryAfterRecorder) Clear() { + if r == nil { + return + } + r.mu.Lock() + defer r.mu.Unlock() + r.deadline = time.Time{} + r.status = 0 +} + +// ParseRetryAfter parses an RFC 7231 `Retry-After` value — either delta-seconds +// or an HTTP-date — into a delay relative to now. +// +// ok is false for an absent, malformed, or already-elapsed value; the caller +// then falls back to its normal exponential backoff. +func ParseRetryAfter(value string, now time.Time) (time.Duration, bool) { + value = strings.TrimSpace(value) + if value == "" { + return 0, false + } + + // delta-seconds + if seconds, err := strconv.Atoi(value); err == nil { + if seconds <= 0 { + return 0, false + } + return time.Duration(seconds) * time.Second, true + } + + // HTTP-date + if t, err := http.ParseTime(value); err == nil { + if delay := t.Sub(now); delay > 0 { + return delay, true + } + } + + return 0, false +} + +// RetryAfterTransport is an http.RoundTripper wrapper that records `Retry-After` +// hints from rate-limited upstream responses into a RetryAfterRecorder. It never +// alters the request or the response — it only observes the status line and one +// header, leaving the body untouched for mcp-go to consume. +type RetryAfterTransport struct { + base http.RoundTripper + recorder *RetryAfterRecorder + logger *zap.Logger +} + +// NewRetryAfterTransport wraps base so 429 (and 503-with-a-hint) responses feed +// recorder. A nil base means http.DefaultTransport; a nil logger is tolerated. +func NewRetryAfterTransport(base http.RoundTripper, recorder *RetryAfterRecorder, logger *zap.Logger) *RetryAfterTransport { + if base == nil { + base = http.DefaultTransport + } + if logger == nil { + logger = zap.NewNop() + } + return &RetryAfterTransport{ + base: base, + recorder: recorder, + logger: logger.Named("retry-after"), + } +} + +// RoundTrip implements http.RoundTripper. +func (t *RetryAfterTransport) RoundTrip(req *http.Request) (*http.Response, error) { + resp, err := t.base.RoundTrip(req) + if err != nil || resp == nil { + return resp, err + } + + switch resp.StatusCode { + case http.StatusTooManyRequests: + // A 429 is always a rate limit, with or without a parseable hint. + case http.StatusServiceUnavailable: + // 503 only counts when the upstream actually tells us how long to wait; + // a bare 503 is an outage, which the normal backoff ladder already paces. + default: + return resp, nil + } + + now := time.Now() + delay, ok := ParseRetryAfter(resp.Header.Get("Retry-After"), now) + if !ok { + return resp, nil + } + + deadline := t.recorder.Record(now, delay, resp.StatusCode) + if !deadline.IsZero() { + t.logger.Info("Upstream asked us to back off", + zap.Int("status", resp.StatusCode), + zap.String("url", req.URL.Redacted()), + zap.Duration("retry_after", delay), + zap.Time("retry_not_before", deadline)) + } + + return resp, nil +} diff --git a/internal/transport/retry_after_test.go b/internal/transport/retry_after_test.go new file mode 100644 index 000000000..e85f99c14 --- /dev/null +++ b/internal/transport/retry_after_test.go @@ -0,0 +1,181 @@ +package transport + +import ( + "net/http" + "net/http/httptest" + "testing" + "time" + + "go.uber.org/zap" +) + +func TestParseRetryAfter(t *testing.T) { + now := time.Date(2026, 8, 25, 12, 0, 0, 0, time.UTC) + + tests := []struct { + name string + value string + want time.Duration + ok bool + }{ + {"empty", "", 0, false}, + {"seconds", "3600", time.Hour, true}, + {"seconds with padding", " 30 ", 30 * time.Second, true}, + {"zero seconds", "0", 0, false}, + {"negative seconds", "-5", 0, false}, + {"http date in future", now.Add(90 * time.Second).Format(http.TimeFormat), 90 * time.Second, true}, + {"http date in past", now.Add(-time.Minute).Format(http.TimeFormat), 0, false}, + {"malformed", "soon please", 0, false}, + {"float seconds are malformed per RFC 7231", "12.5", 0, false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, ok := ParseRetryAfter(tt.value, now) + if ok != tt.ok { + t.Fatalf("ParseRetryAfter(%q) ok = %v, want %v", tt.value, ok, tt.ok) + } + if ok && got != tt.want { + t.Fatalf("ParseRetryAfter(%q) = %v, want %v", tt.value, got, tt.want) + } + }) + } +} + +func TestRetryAfterRecorder_CapsAndKeepsFurthestDeadline(t *testing.T) { + now := time.Date(2026, 8, 25, 12, 0, 0, 0, time.UTC) + rec := NewRetryAfterRecorder() + + if !rec.Deadline().IsZero() { + t.Fatalf("fresh recorder should have no deadline, got %v", rec.Deadline()) + } + + rec.Record(now, 30*time.Second, http.StatusTooManyRequests) + if got, want := rec.Deadline(), now.Add(30*time.Second); !got.Equal(want) { + t.Fatalf("deadline = %v, want %v", got, want) + } + + // A shorter hint must not shorten an outstanding park window. + rec.Record(now, 5*time.Second, http.StatusTooManyRequests) + if got, want := rec.Deadline(), now.Add(30*time.Second); !got.Equal(want) { + t.Fatalf("deadline after shorter hint = %v, want %v", got, want) + } + + // Anything beyond the cap is clamped. + rec.Record(now, 72*time.Hour, http.StatusTooManyRequests) + if got, want := rec.Deadline(), now.Add(MaxRetryAfterDelay); !got.Equal(want) { + t.Fatalf("deadline after oversized hint = %v, want %v", got, want) + } + + rec.Clear() + if !rec.Deadline().IsZero() { + t.Fatalf("Clear should drop the deadline, got %v", rec.Deadline()) + } +} + +func TestRetryAfterRecorder_NilSafe(t *testing.T) { + var rec *RetryAfterRecorder + rec.Record(time.Now(), time.Minute, http.StatusTooManyRequests) + rec.Clear() + if !rec.Deadline().IsZero() { + t.Fatal("nil recorder must report a zero deadline") + } +} + +func TestRetryAfterTransport_RecordsRateLimitHints(t *testing.T) { + tests := []struct { + name string + status int + header string + wantRecord bool + wantDelay time.Duration + }{ + {"429 with delta-seconds", http.StatusTooManyRequests, "3600", true, time.Hour}, + {"429 with http-date", http.StatusTooManyRequests, "", true, 120 * time.Second}, // filled in below + {"429 with malformed header", http.StatusTooManyRequests, "later", false, 0}, + {"429 with no header", http.StatusTooManyRequests, "", false, 0}, + {"503 with header", http.StatusServiceUnavailable, "45", true, 45 * time.Second}, + {"503 without header", http.StatusServiceUnavailable, "", false, 0}, + {"200 with header is ignored", http.StatusOK, "3600", false, 0}, + {"401 with header is ignored", http.StatusUnauthorized, "3600", false, 0}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + header := tt.header + if tt.name == "429 with http-date" { + header = time.Now().Add(120 * time.Second).UTC().Format(http.TimeFormat) + } + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if header != "" { + w.Header().Set("Retry-After", header) + } + w.WriteHeader(tt.status) + _, _ = w.Write([]byte(`{"error":"slow down"}`)) + })) + defer srv.Close() + + rec := NewRetryAfterRecorder() + httpClient := &http.Client{ + Transport: NewRetryAfterTransport(http.DefaultTransport, rec, zap.NewNop()), + } + + before := time.Now() + resp, err := httpClient.Get(srv.URL) + if err != nil { + t.Fatalf("request failed: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != tt.status { + t.Fatalf("status = %d, want %d (the wrapper must pass the response through untouched)", resp.StatusCode, tt.status) + } + + deadline := rec.Deadline() + if !tt.wantRecord { + if !deadline.IsZero() { + t.Fatalf("expected no recorded hint, got deadline %v", deadline) + } + return + } + if deadline.IsZero() { + t.Fatal("expected a recorded deadline, got none") + } + // Allow generous slack: HTTP-date has one-second granularity. + gotDelay := deadline.Sub(before) + if gotDelay < tt.wantDelay-2*time.Second || gotDelay > tt.wantDelay+2*time.Second { + t.Fatalf("recorded delay = %v, want ~%v", gotDelay, tt.wantDelay) + } + }) + } +} + +// TestRetryAfterTransport_BodyIsIntact guards the one thing a response-inspecting +// RoundTripper can easily break: the body must still be readable downstream. +func TestRetryAfterTransport_BodyIsIntact(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Retry-After", "60") + w.WriteHeader(http.StatusTooManyRequests) + _, _ = w.Write([]byte("rate limited")) + })) + defer srv.Close() + + rec := NewRetryAfterRecorder() + httpClient := &http.Client{Transport: NewRetryAfterTransport(nil, rec, nil)} + + resp, err := httpClient.Get(srv.URL) + if err != nil { + t.Fatalf("request failed: %v", err) + } + defer resp.Body.Close() + + buf := make([]byte, 64) + n, _ := resp.Body.Read(buf) + if string(buf[:n]) != "rate limited" { + t.Fatalf("body = %q, want %q", string(buf[:n]), "rate limited") + } + if rec.Deadline().IsZero() { + t.Fatal("expected the 429 hint to be recorded") + } +} diff --git a/internal/upstream/core/client.go b/internal/upstream/core/client.go index eca45036c..8c9282939 100644 --- a/internal/upstream/core/client.go +++ b/internal/upstream/core/client.go @@ -85,6 +85,13 @@ type Client struct { // behaviour). brokeredAuth *proxytransport.BrokeredAuth + // retryAfter collects the `Retry-After` hints this upstream's HTTP/SSE + // responses carry. mcp-go flattens a 429 into an error string long before + // the connection state machine sees it, so the hint is captured by a + // RoundTripper installed under the MCP client and read back here via + // RetryAfterDeadline (#1040). + retryAfter *proxytransport.RetryAfterRecorder + // Transport type and stderr access (for stdio) transportType string stderr io.Reader @@ -173,6 +180,7 @@ func NewClientWithOptions(id string, serverConfig *config.ServerConfig, logger * globalConfig: globalConfig, storage: storage, secretResolver: secretResolver, // Store resolver for future use + retryAfter: proxytransport.NewRetryAfterRecorder(), logger: logger.With( zap.String("upstream_id", id), zap.String("upstream_name", serverConfig.Name), @@ -522,6 +530,31 @@ func (c *Client) GetConnectionInfo() types.ConnectionInfo { } } +// httpTransportConfig builds the transport config for this client's HTTP/SSE +// connections, threading the per-server Retry-After recorder (#1040) into every +// mcp-go client we construct. Every HTTP/SSE connect path must go through it — +// a branch that calls proxytransport.CreateHTTPTransportConfig directly would +// silently lose the rate-limit hint for that auth strategy. +func (c *Client) httpTransportConfig(serverConfig *config.ServerConfig, oauthConfig *client.OAuthConfig) *proxytransport.HTTPTransportConfig { + cfg := proxytransport.CreateHTTPTransportConfig(serverConfig, oauthConfig) + cfg.RetryAfter = c.retryAfter + return cfg +} + +// RetryAfterDeadline reports the instant before which this upstream asked us not +// to come back (from a `Retry-After` on a 429/503), or the zero time when it gave +// no such hint. The managed client stamps it onto the state machine so both +// reconnect gates honour it (#1040). +func (c *Client) RetryAfterDeadline() time.Time { + return c.retryAfter.Deadline() +} + +// ClearRetryAfter drops any recorded rate-limit hint. Called once a connection +// succeeds so a stale window cannot hold back a later, unrelated reconnect. +func (c *Client) ClearRetryAfter() { + c.retryAfter.Clear() +} + // GetServerInfo returns server information from initialization func (c *Client) GetServerInfo() *mcp.InitializeResult { c.mu.RLock() diff --git a/internal/upstream/core/connection.go b/internal/upstream/core/connection.go index f37e16b90..8e8995be9 100644 --- a/internal/upstream/core/connection.go +++ b/internal/upstream/core/connection.go @@ -236,6 +236,10 @@ func (c *Client) Connect(ctx context.Context) error { c.connected = true + // The upstream answered, so any rate-limit window we were holding is spent. + // Dropping it here keeps a stale hint from parking a future reconnect (#1040). + c.retryAfter.Clear() + // If we had an OAuth flow in progress and connection succeeded, mark OAuth as complete if c.isOAuthInProgress() { c.logger.Info("✅ OAuth flow completed successfully - connection established with token", diff --git a/internal/upstream/core/connection_http.go b/internal/upstream/core/connection_http.go index 4869fa9d6..f2854295e 100644 --- a/internal/upstream/core/connection_http.go +++ b/internal/upstream/core/connection_http.go @@ -162,7 +162,7 @@ func (c *Client) canUseHeadersStrategy() bool { // strategy, threading the per-user brokered credential through so the transport // layer injects it (spec 074 FR-016/FR-017). func (c *Client) brokeredHTTPConfig() *transport.HTTPTransportConfig { - httpConfig := transport.CreateHTTPTransportConfig(c.config, nil) + httpConfig := c.httpTransportConfig(c.config, nil) httpConfig.BrokeredAuth = c.brokeredAuth return httpConfig } @@ -201,7 +201,7 @@ func (c *Client) tryNoAuth(ctx context.Context) error { configNoAuth := *c.config configNoAuth.Headers = nil - httpConfig := transport.CreateHTTPTransportConfig(&configNoAuth, nil) + httpConfig := c.httpTransportConfig(&configNoAuth, nil) httpClient, err := transport.CreateHTTPClient(httpConfig) if err != nil { return fmt.Errorf("failed to create HTTP client without auth: %w", err) @@ -270,7 +270,7 @@ func (c *Client) trySSENoAuth(ctx context.Context) error { configNoAuth := *c.config configNoAuth.Headers = nil - httpConfig := transport.CreateHTTPTransportConfig(&configNoAuth, nil) + httpConfig := c.httpTransportConfig(&configNoAuth, nil) sseClient, err := transport.CreateSSEClient(httpConfig) if err != nil { return fmt.Errorf("failed to create SSE client without auth: %w", err) diff --git a/internal/upstream/core/connection_oauth.go b/internal/upstream/core/connection_oauth.go index 8f4819a1e..a748021b1 100644 --- a/internal/upstream/core/connection_oauth.go +++ b/internal/upstream/core/connection_oauth.go @@ -245,7 +245,7 @@ func (c *Client) tryOAuthAuth(ctx context.Context) error { // Create HTTP transport config with OAuth c.logger.Debug("🛠️ Creating HTTP transport config for OAuth") - httpConfig := transport.CreateHTTPTransportConfig(c.config, oauthConfig) + httpConfig := c.httpTransportConfig(c.config, oauthConfig) c.logger.Debug("🔨 Calling transport.CreateHTTPClient with OAuth config") httpClient, err := transport.CreateHTTPClient(httpConfig) @@ -679,7 +679,7 @@ func (c *Client) trySSEOAuthAuth(ctx context.Context) error { // Create SSE transport config with OAuth c.logger.Debug("🛠️ Creating SSE transport config for OAuth") - httpConfig := transport.CreateHTTPTransportConfig(c.config, oauthConfig) + httpConfig := c.httpTransportConfig(c.config, oauthConfig) c.logger.Debug("🔨 Calling transport.CreateSSEClient with OAuth config") sseClient, err := transport.CreateSSEClient(httpConfig) @@ -1839,7 +1839,7 @@ func (c *Client) StartOAuthFlowQuick(ctx context.Context) (*OAuthStartResult, er // Returns the URL, OAuth handler, code verifier, and state for later use. func (c *Client) getAuthorizationURLQuick(ctx context.Context, oauthConfig *client.OAuthConfig, extraParams map[string]string, correlationID string) (string, *uptransport.OAuthHandler, string, string, error) { // Create transport config with OAuth - httpConfig := transport.CreateHTTPTransportConfig(c.config, oauthConfig) + httpConfig := c.httpTransportConfig(c.config, oauthConfig) // Create OAuth-enabled HTTP client httpClient, err := transport.CreateHTTPClient(httpConfig) @@ -2174,7 +2174,7 @@ func (c *Client) forceHTTPOAuthFlowWithResult(ctx context.Context) (*OAuthStartR zap.Int("extra_params_count", len(extraParams))) // Create HTTP transport config with OAuth - httpConfig := transport.CreateHTTPTransportConfig(c.config, oauthConfig) + httpConfig := c.httpTransportConfig(c.config, oauthConfig) // Create OAuth-enabled HTTP client using transport layer httpClient, err := transport.CreateHTTPClient(httpConfig) @@ -2239,7 +2239,7 @@ func (c *Client) forceSSEOAuthFlowWithResult(ctx context.Context) (*OAuthStartRe zap.Int("extra_params_count", len(extraParams))) // Create SSE transport config with OAuth - httpConfig := transport.CreateHTTPTransportConfig(c.config, oauthConfig) + httpConfig := c.httpTransportConfig(c.config, oauthConfig) // Create OAuth-enabled SSE client using transport layer sseClient, err := transport.CreateSSEClient(httpConfig) diff --git a/internal/upstream/managed/client.go b/internal/upstream/managed/client.go index 64b66e058..c3dc94d7a 100644 --- a/internal/upstream/managed/client.go +++ b/internal/upstream/managed/client.go @@ -314,6 +314,19 @@ func (mc *Client) Connect(ctx context.Context) error { defer mc.mu.Unlock() if connectErr != nil { + // A rate-limited upstream (429, or 503 with a hint) told us when to come + // back. mcp-go flattened that response into connectErr's string, so the + // deadline comes from the recorder the transport RoundTripper fed (#1040). + // Stamp it BEFORE the SetError/SetOAuthError branches below so the + // ConnectionInfo their callbacks publish already carries the window. + if deadline := mc.coreClient.RetryAfterDeadline(); !deadline.IsZero() { + mc.logger.Warn("Upstream rate-limited us; parking reconnects until it says we may return", + zap.String("server", mc.GetConfig().Name), + zap.Time("retry_not_before", deadline), + zap.Duration("retry_after", time.Until(deadline))) + mc.StateManager.SetRetryAfter(deadline) + } + // Check if this is a deferred OAuth requirement (pending user action) if core.IsOAuthPending(connectErr) { mc.logger.Info("⏳ OAuth authentication pending user action", diff --git a/internal/upstream/retry_after_connect_test.go b/internal/upstream/retry_after_connect_test.go new file mode 100644 index 000000000..726e6bef5 --- /dev/null +++ b/internal/upstream/retry_after_connect_test.go @@ -0,0 +1,176 @@ +package upstream + +import ( + "context" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + "github.com/smart-mcp-proxy/mcpproxy-go/internal/config" + "github.com/smart-mcp-proxy/mcpproxy-go/internal/secret" + "github.com/smart-mcp-proxy/mcpproxy-go/internal/upstream/managed" + "github.com/smart-mcp-proxy/mcpproxy-go/internal/upstream/types" +) + +// rateLimitedServer answers every request with the given status and, when +// retryAfter is non-empty, a Retry-After header. It counts the requests it saw +// so a test can prove the reconnect gate actually stopped the redials. +func rateLimitedServer(t *testing.T, status int, retryAfter string, hits *atomic.Int32) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + hits.Add(1) + if retryAfter != "" { + w.Header().Set("Retry-After", retryAfter) + } + w.WriteHeader(status) + _, _ = w.Write([]byte(`{"error":"rate limit exceeded"}`)) + })) + t.Cleanup(srv.Close) + return srv +} + +func connectToRateLimited(t *testing.T, url, protocol string, headers ...map[string]string) *managed.Client { + t.Helper() + t.Setenv("MCPPROXY_DISABLE_OAUTH", "true") + + cfg := &config.ServerConfig{ + Name: "rate-limited-upstream", + URL: url, + Protocol: protocol, + Enabled: true, + Created: time.Now(), + } + if len(headers) > 0 { + cfg.Headers = headers[0] + } + + client, err := managed.NewClient("retry-after-test", cfg, zap.NewNop(), nil, nil, nil, secret.NewResolver()) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + + err = client.Connect(ctx) + require.Error(t, err, "a 429 upstream must not produce a successful connection") + return client +} + +// TestConnect_429WithRetryAfter_ParksReconnects is the acceptance case for +// #1040: mcp-go flattens the 429 response into an error string, so without the +// transport RoundTripper the Retry-After header is gone before any state code +// runs and the supervisor keeps redialing on its own ladder. +func TestConnect_429WithRetryAfter_ParksReconnects(t *testing.T) { + var hits atomic.Int32 + srv := rateLimitedServer(t, http.StatusTooManyRequests, "3600", &hits) + + client := connectToRateLimited(t, srv.URL+"/mcp", "http") + + info := client.GetConnectionInfo() + require.False(t, info.RetryAfter.IsZero(), "the Retry-After hint must reach ConnectionInfo") + assert.WithinDuration(t, time.Now().Add(time.Hour), info.RetryAfter, 30*time.Second, + "Retry-After: 3600 must park the server for an hour") + + // The gate the supervisor's reconcile and Manager.ConnectAll both consult. + assert.False(t, info.ShouldAutoReconnect(time.Now()), + "no ActionConnect may be planned inside the Retry-After window") + assert.False(t, info.ShouldAutoReconnect(time.Now().Add(59*time.Minute)), + "the window must still hold nearly an hour later") + assert.True(t, info.ShouldAutoReconnect(time.Now().Add(61*time.Minute)), + "once the window elapses the normal ladder takes over again") +} + +// TestConnect_429WithHTTPDateRetryAfter covers the second RFC 7231 form. +func TestConnect_429WithHTTPDateRetryAfter(t *testing.T) { + var hits atomic.Int32 + when := time.Now().Add(20 * time.Minute).UTC().Format(http.TimeFormat) + srv := rateLimitedServer(t, http.StatusTooManyRequests, when, &hits) + + client := connectToRateLimited(t, srv.URL+"/mcp", "http") + + info := client.GetConnectionInfo() + require.False(t, info.RetryAfter.IsZero(), "an HTTP-date Retry-After must parse") + assert.WithinDuration(t, time.Now().Add(20*time.Minute), info.RetryAfter, time.Minute) + assert.False(t, info.ShouldAutoReconnect(time.Now())) +} + +// TestConnect_429MalformedRetryAfter_FallsBackToBackoff pins the fallback: a +// value we cannot parse must leave the existing exponential ladder in charge +// rather than parking the server on a guess. +func TestConnect_429MalformedRetryAfter_FallsBackToBackoff(t *testing.T) { + var hits atomic.Int32 + srv := rateLimitedServer(t, http.StatusTooManyRequests, "when we feel like it", &hits) + + client := connectToRateLimited(t, srv.URL+"/mcp", "http") + + info := client.GetConnectionInfo() + assert.True(t, info.RetryAfter.IsZero(), "a malformed hint must not park the server") + assert.Equal(t, types.StateError, info.State) + assert.GreaterOrEqual(t, info.RetryCount, 1, "the normal retry ladder must have advanced instead") +} + +// TestConnect_429OversizedRetryAfter_IsCapped keeps a hostile or misconfigured +// upstream from parking a server for days. +func TestConnect_429OversizedRetryAfter_IsCapped(t *testing.T) { + var hits atomic.Int32 + srv := rateLimitedServer(t, http.StatusTooManyRequests, "604800", &hits) // one week + + client := connectToRateLimited(t, srv.URL+"/mcp", "http") + + info := client.GetConnectionInfo() + require.False(t, info.RetryAfter.IsZero()) + assert.WithinDuration(t, time.Now().Add(time.Hour), info.RetryAfter, time.Minute, + "an oversized hint must clamp to the one-hour cap") +} + +// TestConnect_429RetryAfter_HeadersAndSSETransports walks the other two +// client-construction branches touched by #1040 — the headers variant of the +// streamable-HTTP client and the SSE client — because each builds its +// *http.Client differently and could silently drop the RoundTripper. +func TestConnect_429RetryAfter_HeadersAndSSETransports(t *testing.T) { + tests := []struct { + name string + protocol string + path string + headers map[string]string + }{ + {"streamable-http with headers", "http", "/mcp", map[string]string{"Authorization": "Bearer test-token"}}, + {"sse", "sse", "/sse", nil}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var hits atomic.Int32 + srv := rateLimitedServer(t, http.StatusTooManyRequests, "900", &hits) + + var client *managed.Client + if tt.headers != nil { + client = connectToRateLimited(t, srv.URL+tt.path, tt.protocol, tt.headers) + } else { + client = connectToRateLimited(t, srv.URL+tt.path, tt.protocol) + } + + info := client.GetConnectionInfo() + require.False(t, info.RetryAfter.IsZero(), "the %s branch must also record the hint", tt.name) + assert.WithinDuration(t, time.Now().Add(15*time.Minute), info.RetryAfter, time.Minute) + assert.False(t, info.ShouldAutoReconnect(time.Now())) + }) + } +} + +// TestConnect_503WithoutRetryAfter_LeavesLadderAlone guards the deliberately +// narrow 503 handling: a bare 503 is an outage, which the ladder already paces. +func TestConnect_503WithoutRetryAfter_LeavesLadderAlone(t *testing.T) { + var hits atomic.Int32 + srv := rateLimitedServer(t, http.StatusServiceUnavailable, "", &hits) + + client := connectToRateLimited(t, srv.URL+"/mcp", "http") + + info := client.GetConnectionInfo() + assert.True(t, info.RetryAfter.IsZero(), "a bare 503 carries no hint to honour") +} diff --git a/internal/upstream/types/retry_after_test.go b/internal/upstream/types/retry_after_test.go new file mode 100644 index 000000000..f0f59e3fc --- /dev/null +++ b/internal/upstream/types/retry_after_test.go @@ -0,0 +1,102 @@ +package types + +import ( + "errors" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +// TestConnectionInfo_ShouldAutoReconnect_RetryAfter pins the 429 park window +// (#1040): an upstream-supplied Retry-After is a FLOOR under the retry ladder, +// so the effective delay is max(backoff, Retry-After). +func TestConnectionInfo_ShouldAutoReconnect_RetryAfter(t *testing.T) { + now := time.Now() + + tests := []struct { + name string + info *ConnectionInfo + expected bool + }{ + { + name: "retry-after window still open beats an elapsed backoff", + info: &ConnectionInfo{State: StateError, RetryCount: 1, LastRetryTime: now.Add(-time.Hour), RetryAfter: now.Add(30 * time.Minute)}, + expected: false, + }, + { + name: "retry-after window elapsed falls back to the ladder", + info: &ConnectionInfo{State: StateError, RetryCount: 1, LastRetryTime: now.Add(-time.Hour), RetryAfter: now.Add(-time.Second)}, + expected: true, + }, + { + name: "no retry-after leaves the ladder untouched", + info: &ConnectionInfo{State: StateError, RetryCount: 1, LastRetryTime: now.Add(-time.Hour)}, + expected: true, + }, + { + name: "backoff still open even though retry-after elapsed", + info: &ConnectionInfo{State: StateError, RetryCount: 5, LastRetryTime: now, RetryAfter: now.Add(-time.Minute)}, + expected: false, + }, + { + name: "a disconnected server is parked too while the window is open", + info: &ConnectionInfo{State: StateDisconnected, RetryAfter: now.Add(10 * time.Minute)}, + expected: false, + }, + { + name: "a give-up probe is still held back by the window", + info: &ConnectionInfo{State: StateError, GaveUp: true, LastRetryTime: now.Add(-time.Hour), RetryAfter: now.Add(15 * time.Minute)}, + expected: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expected, tt.info.ShouldAutoReconnect(now)) + }) + } +} + +// TestStateManager_SetRetryAfter covers the state-manager side of the 429 park: +// the hint reaches ConnectionInfo and both reconnect gates, extends rather than +// shortens, and is dropped once the server actually connects. +func TestStateManager_SetRetryAfter(t *testing.T) { + sm := NewStateManager() + sm.TransitionTo(StateConnecting) + + deadline := time.Now().Add(time.Hour) + sm.SetRetryAfter(deadline) + sm.SetError(errors.New("request failed with status 429: rate limited")) + + info := sm.GetConnectionInfo() + assert.WithinDuration(t, deadline, info.RetryAfter, time.Second, "the hint must reach ConnectionInfo") + assert.False(t, info.ShouldAutoReconnect(time.Now()), "the supervisor gate must park the server") + assert.False(t, sm.ShouldRetry(), "ConnectAll/health must not redial inside the window either") + + // A shorter hint must not shorten an outstanding window. + sm.SetRetryAfter(time.Now().Add(time.Second)) + assert.WithinDuration(t, deadline, sm.GetConnectionInfo().RetryAfter, time.Second) + + // A successful connection clears it — a stale window must not hold back a + // later, unrelated reconnect. + sm.TransitionTo(StateConnecting) + sm.TransitionTo(StateReady) + assert.True(t, sm.GetConnectionInfo().RetryAfter.IsZero(), "Ready must clear the park window") + + // Reset (manual reconnect) clears it as well. + sm.SetRetryAfter(time.Now().Add(time.Hour)) + sm.Reset() + assert.True(t, sm.GetConnectionInfo().RetryAfter.IsZero(), "Reset must clear the park window") +} + +// TestStateManager_SetRetryAfter_IgnoresPastDeadlines guards against a +// misparsed/expired hint permanently poisoning the gate. +func TestStateManager_SetRetryAfter_IgnoresPastDeadlines(t *testing.T) { + sm := NewStateManager() + sm.SetRetryAfter(time.Time{}) + assert.True(t, sm.GetConnectionInfo().RetryAfter.IsZero()) + + sm.SetRetryAfter(time.Now().Add(-time.Hour)) + assert.True(t, sm.GetConnectionInfo().RetryAfter.IsZero(), "an already-elapsed hint is not worth storing") +} diff --git a/internal/upstream/types/types.go b/internal/upstream/types/types.go index faf8a0b40..e733f2f33 100644 --- a/internal/upstream/types/types.go +++ b/internal/upstream/types/types.go @@ -65,6 +65,11 @@ type ConnectionInfo struct { OAuthRetryCount int `json:"oauth_retry_count"` IsOAuthError bool `json:"is_oauth_error"` GaveUp bool `json:"gave_up"` // True when max retries exceeded + // RetryAfter is the instant before which the upstream asked us not to come + // back, captured from a `Retry-After` header on a rate-limited response + // (#1040). Zero when the upstream gave no hint. It acts as a floor under the + // retry ladder — the effective delay is max(backoff, Retry-After). + RetryAfter time.Time `json:"retry_after,omitempty"` } // RetryBackoffDuration returns the exponential backoff to wait after the given @@ -124,6 +129,13 @@ func (ci *ConnectionInfo) ShouldAutoReconnect(now time.Time) bool { if ci == nil { return true } + // A `Retry-After` from a rate-limited upstream is a floor under every other + // rung of the ladder: the vendor told us in so many words when to come back, + // and dialing earlier only burns quota. It applies whatever the state, + // because a 429 can arrive on a tool call as easily as on connect (#1040). + if !ci.RetryAfter.IsZero() && now.Before(ci.RetryAfter) { + return false + } switch ci.State { case StatePendingAuth: return false @@ -169,6 +181,10 @@ type StateManager struct { oauthRetryCount int isOAuthError bool userLoggedOut bool // When true, prevents auto-reconnection until user explicitly logs in + // retryAfter is the upstream-supplied park deadline from a rate-limited + // response (#1040), set by SetRetryAfter and cleared on a successful + // connection or an explicit Reset. + retryAfter time.Time // Callbacks for state transitions onStateChange func(oldState, newState ConnectionState, info *ConnectionInfo) @@ -217,6 +233,7 @@ func (sm *StateManager) GetConnectionInfo() ConnectionInfo { LastOAuthAttempt: sm.lastOAuthAttempt, OAuthRetryCount: sm.oauthRetryCount, IsOAuthError: sm.isOAuthError, + RetryAfter: sm.retryAfter, GaveUp: sm.retryCount >= MaxConnectionRetries, } } @@ -242,6 +259,7 @@ func (sm *StateManager) TransitionTo(newState ConnectionState) { sm.isOAuthError = false sm.oauthRetryCount = 0 sm.userLoggedOut = false // Clear logout flag on successful connection + sm.retryAfter = time.Time{} } info := ConnectionInfo{ @@ -254,6 +272,7 @@ func (sm *StateManager) TransitionTo(newState ConnectionState) { LastOAuthAttempt: sm.lastOAuthAttempt, OAuthRetryCount: sm.oauthRetryCount, IsOAuthError: sm.isOAuthError, + RetryAfter: sm.retryAfter, } callback := sm.onStateChange @@ -286,6 +305,7 @@ func (sm *StateManager) SetError(err error) { LastOAuthAttempt: sm.lastOAuthAttempt, OAuthRetryCount: sm.oauthRetryCount, IsOAuthError: sm.isOAuthError, + RetryAfter: sm.retryAfter, } callback := sm.onStateChange @@ -325,6 +345,7 @@ func (sm *StateManager) SetPendingAuth(err error) { LastOAuthAttempt: sm.lastOAuthAttempt, OAuthRetryCount: sm.oauthRetryCount, IsOAuthError: sm.isOAuthError, + RetryAfter: sm.retryAfter, GaveUp: sm.retryCount >= MaxConnectionRetries, } @@ -343,6 +364,32 @@ func (sm *StateManager) SetPendingAuth(err error) { } } +// SetRetryAfter records an upstream-supplied park deadline captured from a +// `Retry-After` header on a rate-limited (429 / 503) response (#1040). +// +// It only ever EXTENDS the window: a later, shorter hint must not shorten a wait +// the upstream already asked for, and an already-elapsed deadline is dropped +// rather than stored. Call it BEFORE SetError/SetOAuthError so the resulting +// ConnectionInfo the state-change callback publishes already carries the window. +func (sm *StateManager) SetRetryAfter(deadline time.Time) { + if deadline.IsZero() || !deadline.After(time.Now()) { + return + } + sm.mu.Lock() + defer sm.mu.Unlock() + if deadline.After(sm.retryAfter) { + sm.retryAfter = deadline + } +} + +// RetryAfter returns the current park deadline, or the zero time when the +// upstream has given no rate-limit hint. +func (sm *StateManager) RetryAfter() time.Time { + sm.mu.RLock() + defer sm.mu.RUnlock() + return sm.retryAfter +} + // SetServerInfo sets the server information func (sm *StateManager) SetServerInfo(name, version string) { sm.mu.Lock() @@ -376,6 +423,12 @@ func (sm *StateManager) ShouldRetry() bool { return false } + // Honour an upstream-supplied Retry-After window (#1040) — the same floor + // ConnectionInfo.ShouldAutoReconnect applies for the supervisor. + if !sm.retryAfter.IsZero() && time.Now().Before(sm.retryAfter) { + return false + } + if sm.currentState != StateError { return false } @@ -456,6 +509,7 @@ func (sm *StateManager) Reset() { sm.lastOAuthAttempt = time.Time{} sm.oauthRetryCount = 0 sm.isOAuthError = false + sm.retryAfter = time.Time{} info := ConnectionInfo{ State: sm.currentState, @@ -467,6 +521,7 @@ func (sm *StateManager) Reset() { LastOAuthAttempt: sm.lastOAuthAttempt, OAuthRetryCount: sm.oauthRetryCount, IsOAuthError: sm.isOAuthError, + RetryAfter: sm.retryAfter, } callback := sm.onStateChange @@ -501,6 +556,7 @@ func (sm *StateManager) ResetForReconnect() { LastOAuthAttempt: sm.lastOAuthAttempt, OAuthRetryCount: sm.oauthRetryCount, IsOAuthError: sm.isOAuthError, + RetryAfter: sm.retryAfter, } callback := sm.onStateChange @@ -532,6 +588,7 @@ func (sm *StateManager) SetOAuthError(err error) { LastOAuthAttempt: sm.lastOAuthAttempt, OAuthRetryCount: sm.oauthRetryCount, IsOAuthError: sm.isOAuthError, + RetryAfter: sm.retryAfter, } callback := sm.onStateChange From 6331a61ae9357eeb754f34dee72fd5755ecc47ac Mon Sep 17 00:00:00 2001 From: Algis Dumbris Date: Tue, 25 Aug 2026 07:07:39 +0300 Subject: [PATCH 2/3] fix(upstream): scope the Retry-After window to its attempt and gate the OAuth ladder MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cross-model review of #1042 found four real holes: - The hint only reached the state machine after a failed Connect, so a 429 answered to the health-check ping was recorded and then ignored. syncRetryAfterFromTransport is now called from every path that turns an upstream failure into Error state. - The recorder was cleared only on success, so a window from an earlier attempt could be re-attributed to a later, unrelated failure — re-parking a server the upstream had not rate-limited again (notably right after a manual reconnect reset the state machine). Each attempt now starts with a clean slate. - ShouldRetryOAuth (the monitoring loop's reconnect path) ignored the window, and its rungs can be shorter than what the upstream asked for. - Converting delta-seconds to a Duration before capping overflowed int64 nanoseconds past ~292 years, turning a huge header into a tiny or negative delay instead of the cap. The bound check now precedes the multiplication. Tests: OAuth streamable-HTTP and SSE clients now have wire-level coverage (a pre-seeded token store makes the request actually go out, so the 429 is observable) — verified to fail when either constructor loses the basic-client option; plus per-attempt window scoping at the core level, the OAuth-ladder gate, and the two overflow-shaped Retry-After inputs. Refs #1040. --- internal/transport/retry_after.go | 10 +- internal/transport/retry_after_oauth_test.go | 124 ++++++++++++++++++ internal/transport/retry_after_test.go | 25 ++++ internal/upstream/core/connection.go | 7 + internal/upstream/managed/client.go | 36 ++++- internal/upstream/retry_after_connect_test.go | 37 ++++++ internal/upstream/types/retry_after_test.go | 20 +++ internal/upstream/types/types.go | 7 + 8 files changed, 257 insertions(+), 9 deletions(-) create mode 100644 internal/transport/retry_after_oauth_test.go diff --git a/internal/transport/retry_after.go b/internal/transport/retry_after.go index 653fa032f..cbfaa8eb0 100644 --- a/internal/transport/retry_after.go +++ b/internal/transport/retry_after.go @@ -108,11 +108,17 @@ func ParseRetryAfter(value string, now time.Time) (time.Duration, bool) { return 0, false } - // delta-seconds - if seconds, err := strconv.Atoi(value); err == nil { + // delta-seconds. The bound check happens BEFORE the multiplication: RFC 7231 + // puts no ceiling on the value, and `time.Duration(seconds) * time.Second` + // overflows int64 nanoseconds past ~292 years, which would turn a huge (but + // syntactically valid) header into a tiny or negative delay instead of the cap. + if seconds, err := strconv.ParseInt(value, 10, 64); err == nil { if seconds <= 0 { return 0, false } + if seconds >= int64(MaxRetryAfterDelay/time.Second) { + return MaxRetryAfterDelay, true + } return time.Duration(seconds) * time.Second, true } diff --git a/internal/transport/retry_after_oauth_test.go b/internal/transport/retry_after_oauth_test.go new file mode 100644 index 000000000..ae31748a2 --- /dev/null +++ b/internal/transport/retry_after_oauth_test.go @@ -0,0 +1,124 @@ +package transport + +import ( + "context" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + mcpclient "github.com/mark3labs/mcp-go/client" + mcptransport "github.com/mark3labs/mcp-go/client/transport" + "github.com/mark3labs/mcp-go/mcp" +) + +// oauthTestConfig builds an OAuth config whose token store already holds a live +// token, so the mcp-go OAuth handler attaches an Authorization header and +// actually issues the MCP request instead of short-circuiting into "no token". +// That is what lets the test observe the upstream's 429 on the MCP endpoint. +func oauthTestConfig() *mcpclient.OAuthConfig { + store := mcptransport.NewMemoryTokenStore() + _ = store.SaveToken(context.Background(), &mcptransport.Token{ + AccessToken: "test-access-token", + TokenType: "Bearer", + ExpiresIn: 3600, + ExpiresAt: time.Now().Add(time.Hour), + }) + return &mcpclient.OAuthConfig{ + ClientID: "test-client-id", + RedirectURI: "http://127.0.0.1:0/callback", + Scopes: []string{"mcp"}, + TokenStore: store, + PKCEEnabled: true, + } +} + +// TestOAuthClients_RecordRetryAfter covers the branch the non-OAuth tests cannot +// reach: mcp-go builds the OAuth client's transport itself, and OAuthConfig.HTTPClient +// governs only the metadata/DCR/token calls. If the basic-client option were +// dropped from either OAuth constructor, a 429 on the MCP endpoint of an +// OAuth'd upstream would go unrecorded and the server would keep being redialed +// on the plain ladder (#1040). +func TestOAuthClients_RecordRetryAfter(t *testing.T) { + tests := []struct { + name string + create func(cfg *HTTPTransportConfig) (*mcpclient.Client, error) + }{ + {"streamable-http", CreateHTTPClient}, + {"sse", CreateSSEClient}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var sawAuthHeader atomic.Bool + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != "" { + sawAuthHeader.Store(true) + } + w.Header().Set("Retry-After", "1800") + w.WriteHeader(http.StatusTooManyRequests) + _, _ = w.Write([]byte(`{"error":"rate limit exceeded"}`)) + })) + defer srv.Close() + + recorder := NewRetryAfterRecorder() + cfg := &HTTPTransportConfig{ + URL: srv.URL + "/mcp", + OAuthConfig: oauthTestConfig(), + UseOAuth: true, + RetryAfter: recorder, + } + + mcpClient, err := tt.create(cfg) + if err != nil { + t.Fatalf("failed to create OAuth %s client: %v", tt.name, err) + } + defer mcpClient.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + // Both Start and Initialize are expected to fail against a 429 upstream; + // what matters is that the request went out through OUR transport. + if err := mcpClient.Start(ctx); err == nil { + _, _ = mcpClient.Initialize(ctx, mcp.InitializeRequest{}) + } + + if !sawAuthHeader.Load() { + t.Fatal("the OAuth handler never attached a token, so the request under test never left the client") + } + deadline := recorder.Deadline() + if deadline.IsZero() { + t.Fatalf("the OAuth %s client dropped the Retry-After recorder", tt.name) + } + if got := time.Until(deadline); got < 25*time.Minute || got > 35*time.Minute { + t.Fatalf("recorded window = %v, want ~30m", got) + } + }) + } +} + +// TestOAuthClients_NoRecorder_KeepsMcpGoDefaults pins the opt-out: with no +// recorder and no tracing we must hand mcp-go no client at all, leaving its own +// defaults (and the OAuth wiring) exactly as they were. +func TestOAuthClients_NoRecorder_KeepsMcpGoDefaults(t *testing.T) { + cfg := &HTTPTransportConfig{ + URL: "http://127.0.0.1:1/mcp", + OAuthConfig: oauthTestConfig(), + UseOAuth: true, + } + if cfg.needsCustomTransport() { + t.Fatal("a config with neither tracing nor a recorder must not force a custom transport") + } + if c, err := CreateHTTPClient(cfg); err != nil { + t.Fatalf("CreateHTTPClient: %v", err) + } else { + c.Close() + } + if c, err := CreateSSEClient(cfg); err != nil { + t.Fatalf("CreateSSEClient: %v", err) + } else { + c.Close() + } +} diff --git a/internal/transport/retry_after_test.go b/internal/transport/retry_after_test.go index e85f99c14..7f064c80e 100644 --- a/internal/transport/retry_after_test.go +++ b/internal/transport/retry_after_test.go @@ -27,6 +27,11 @@ func TestParseRetryAfter(t *testing.T) { {"http date in past", now.Add(-time.Minute).Format(http.TimeFormat), 0, false}, {"malformed", "soon please", 0, false}, {"float seconds are malformed per RFC 7231", "12.5", 0, false}, + // RFC 7231 puts no ceiling on delta-seconds, and the naive + // `time.Duration(seconds) * time.Second` overflows int64 nanoseconds past + // ~292 years — turning a huge header into a tiny or negative delay. + {"seconds beyond the cap clamp instead of overflowing", "99999999999999", MaxRetryAfterDelay, true}, + {"seconds beyond int64 are malformed", "99999999999999999999999999", 0, false}, } for _, tt := range tests { @@ -73,6 +78,26 @@ func TestRetryAfterRecorder_CapsAndKeepsFurthestDeadline(t *testing.T) { } } +// TestParseRetryAfter_DistantHTTPDateStaysSane guards the other overflow-shaped +// input: a date centuries out saturates time.Duration rather than wrapping, and +// the recorder still clamps it to the cap. +func TestParseRetryAfter_DistantHTTPDateStaysSane(t *testing.T) { + now := time.Now() + delay, ok := ParseRetryAfter("Mon, 01 Jan 2999 00:00:00 GMT", now) + if !ok { + t.Fatal("a valid future HTTP-date must parse") + } + if delay <= 0 { + t.Fatalf("delay = %v, want a positive duration", delay) + } + + rec := NewRetryAfterRecorder() + rec.Record(now, delay, http.StatusTooManyRequests) + if got, want := rec.Deadline(), now.Add(MaxRetryAfterDelay); !got.Equal(want) { + t.Fatalf("deadline = %v, want the capped %v", got, want) + } +} + func TestRetryAfterRecorder_NilSafe(t *testing.T) { var rec *RetryAfterRecorder rec.Record(time.Now(), time.Minute, http.StatusTooManyRequests) diff --git a/internal/upstream/core/connection.go b/internal/upstream/core/connection.go index 8e8995be9..d54b42eb3 100644 --- a/internal/upstream/core/connection.go +++ b/internal/upstream/core/connection.go @@ -87,6 +87,13 @@ func (c *Client) Connect(ctx context.Context) error { } } + // Start each attempt with an empty rate-limit slate (#1040). Without this a + // hint recorded during an earlier attempt outlives it: a manual reconnect + // that then fails for an unrelated reason (connection refused, DNS) would be + // re-parked for the remainder of a window the upstream never repeated. + // Whatever this attempt observes is recorded again, on its own merits. + c.retryAfter.Clear() + c.logger.Info("Connecting to upstream MCP server", zap.String("server", c.config.Name), zap.String("url", c.config.URL), diff --git a/internal/upstream/managed/client.go b/internal/upstream/managed/client.go index c3dc94d7a..7f1c39168 100644 --- a/internal/upstream/managed/client.go +++ b/internal/upstream/managed/client.go @@ -319,13 +319,7 @@ func (mc *Client) Connect(ctx context.Context) error { // deadline comes from the recorder the transport RoundTripper fed (#1040). // Stamp it BEFORE the SetError/SetOAuthError branches below so the // ConnectionInfo their callbacks publish already carries the window. - if deadline := mc.coreClient.RetryAfterDeadline(); !deadline.IsZero() { - mc.logger.Warn("Upstream rate-limited us; parking reconnects until it says we may return", - zap.String("server", mc.GetConfig().Name), - zap.Time("retry_not_before", deadline), - zap.Duration("retry_after", time.Until(deadline))) - mc.StateManager.SetRetryAfter(deadline) - } + mc.syncRetryAfterFromTransport() // Check if this is a deferred OAuth requirement (pending user action) if core.IsOAuthPending(connectErr) { @@ -1279,6 +1273,10 @@ func (mc *Client) performHealthCheck() { zap.String("server", mc.GetConfig().Name), zap.Int("consecutive_failures", mc.consecutiveHealthFailures), zap.Error(err)) + // A 429 answered to the ping carries the same "come back later" + // as one answered to connect (#1040); stamp it before SetError + // so the published ConnectionInfo already carries the window. + mc.syncRetryAfterFromTransport() mc.StateManager.SetError(err) } else { mc.logger.Info("Health check failed transiently, tolerating below threshold", @@ -1310,6 +1308,30 @@ func (mc *Client) performHealthCheck() { // failures (connection refused, host unreachable, DNS gone) trigger Error // immediately because waiting buys nothing — the server is genuinely // unreachable and the user should see that. +// syncRetryAfterFromTransport copies any rate-limit hint the transport +// RoundTripper recorded for this upstream into the state machine, where both +// reconnect gates can see it (#1040). +// +// It is called from every path that turns an upstream failure into Error state, +// not just connect: a 429 answered to a tools/call or to the health-check ping +// is just as much a "come back later" as one answered to initialize, and the +// recorder is the only place that hint exists — mcp-go has already flattened +// the response into a string by the time the error reaches us. +func (mc *Client) syncRetryAfterFromTransport() { + if mc.coreClient == nil { + return + } + deadline := mc.coreClient.RetryAfterDeadline() + if deadline.IsZero() { + return + } + mc.logger.Warn("Upstream rate-limited us; parking reconnects until it says we may return", + zap.String("server", mc.GetConfig().Name), + zap.Time("retry_not_before", deadline), + zap.Duration("retry_after", time.Until(deadline))) + mc.StateManager.SetRetryAfter(deadline) +} + func (mc *Client) recordHealthCheckFailure(err error) bool { mc.consecutiveHealthFailures++ if !isTransientHealthCheckError(err) { diff --git a/internal/upstream/retry_after_connect_test.go b/internal/upstream/retry_after_connect_test.go index 726e6bef5..7fb0f2799 100644 --- a/internal/upstream/retry_after_connect_test.go +++ b/internal/upstream/retry_after_connect_test.go @@ -14,6 +14,7 @@ import ( "github.com/smart-mcp-proxy/mcpproxy-go/internal/config" "github.com/smart-mcp-proxy/mcpproxy-go/internal/secret" + "github.com/smart-mcp-proxy/mcpproxy-go/internal/upstream/core" "github.com/smart-mcp-proxy/mcpproxy-go/internal/upstream/managed" "github.com/smart-mcp-proxy/mcpproxy-go/internal/upstream/types" ) @@ -163,6 +164,42 @@ func TestConnect_429RetryAfter_HeadersAndSSETransports(t *testing.T) { } } +// TestConnect_RetryAfterIsPerAttempt pins the lifetime of a recorded hint: it +// belongs to the attempt that saw it. Carrying it forward would let a window the +// upstream never repeated re-park a server on the back of an unrelated later +// failure (connection refused, DNS) — including right after a manual reconnect +// cleared the state machine. +func TestConnect_RetryAfterIsPerAttempt(t *testing.T) { + t.Setenv("MCPPROXY_DISABLE_OAUTH", "true") + + var hits atomic.Int32 + srv := rateLimitedServer(t, http.StatusTooManyRequests, "3600", &hits) + + cfg := &config.ServerConfig{ + Name: "rate-limited-upstream", + URL: srv.URL + "/mcp", + Protocol: "http", + Enabled: true, + Created: time.Now(), + } + + client, err := core.NewClient("retry-after-attempt-test", cfg, zap.NewNop(), nil, nil, nil, secret.NewResolver()) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + + require.Error(t, client.Connect(ctx), "a 429 upstream must not connect") + require.False(t, client.RetryAfterDeadline().IsZero(), "the rate-limited attempt must record its hint") + + // The upstream goes away entirely: the next attempt fails for a completely + // different reason and must not inherit the previous window. + srv.Close() + require.Error(t, client.Connect(ctx)) + assert.True(t, client.RetryAfterDeadline().IsZero(), + "a stale hint from an earlier attempt must not be attributed to this one") +} + // TestConnect_503WithoutRetryAfter_LeavesLadderAlone guards the deliberately // narrow 503 handling: a bare 503 is an outage, which the ladder already paces. func TestConnect_503WithoutRetryAfter_LeavesLadderAlone(t *testing.T) { diff --git a/internal/upstream/types/retry_after_test.go b/internal/upstream/types/retry_after_test.go index f0f59e3fc..24c68341c 100644 --- a/internal/upstream/types/retry_after_test.go +++ b/internal/upstream/types/retry_after_test.go @@ -6,6 +6,7 @@ import ( "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) // TestConnectionInfo_ShouldAutoReconnect_RetryAfter pins the 429 park window @@ -90,6 +91,25 @@ func TestStateManager_SetRetryAfter(t *testing.T) { assert.True(t, sm.GetConnectionInfo().RetryAfter.IsZero(), "Reset must clear the park window") } +// TestStateManager_SetRetryAfter_GatesTheOAuthLadderToo: the OAuth ladder is a +// separate reconnect path (the monitoring loop consults ShouldRetryOAuth), and +// its rungs can be shorter than the window a rate-limited upstream asked for. +func TestStateManager_SetRetryAfter_GatesTheOAuthLadderToo(t *testing.T) { + // An OAuth failure whose ladder rung (5 min for the first retry) has long + // since elapsed — built directly so the test does not have to wait it out. + sm := &StateManager{ + currentState: StateError, + lastError: errors.New("request failed with status 429: rate limited during token exchange"), + isOAuthError: true, + oauthRetryCount: 1, + lastOAuthAttempt: time.Now().Add(-time.Hour), + } + require.True(t, sm.ShouldRetryOAuth(), "precondition: the OAuth ladder has elapsed and would retry") + + sm.SetRetryAfter(time.Now().Add(time.Hour)) + assert.False(t, sm.ShouldRetryOAuth(), "the rate-limit window outranks the OAuth ladder") +} + // TestStateManager_SetRetryAfter_IgnoresPastDeadlines guards against a // misparsed/expired hint permanently poisoning the gate. func TestStateManager_SetRetryAfter_IgnoresPastDeadlines(t *testing.T) { diff --git a/internal/upstream/types/types.go b/internal/upstream/types/types.go index e733f2f33..ee9c84d66 100644 --- a/internal/upstream/types/types.go +++ b/internal/upstream/types/types.go @@ -610,6 +610,13 @@ func (sm *StateManager) ShouldRetryOAuth() bool { return false } + // A rate-limit window outranks the OAuth ladder too (#1040): an upstream + // that answered 429 will answer 429 to the token exchange as well, and the + // OAuth ladder can be shorter than the window the upstream asked for. + if !sm.retryAfter.IsZero() && time.Now().Before(sm.retryAfter) { + return false + } + if !sm.isOAuthError || sm.currentState != StateError { return false } From 8d7b0c88b9c69d49281b0bfc3992dea3d1b25e9d Mon Sep 17 00:00:00 2001 From: Algis Dumbris Date: Tue, 25 Aug 2026 07:16:33 +0300 Subject: [PATCH 3/3] fix(upstream): make the 429 sync reachable from the request paths and swap recorders per attempt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round 2 of the cross-model review found the round-1 health-path fix was dead code and the per-attempt clear was too blunt: - performHealthCheck only reached the sync after isConnectionError, which matches transport failures ("connection refused", "deadline exceeded") and not HTTP statuses — so a ping answered 429 never handed its window to the state machine. The sync now runs on any failed ping, before classification; it does not change the health verdict, it only stops the automatic reconnect paths from returning early. The same sync now also runs on a failed tools/call, the other surface that can be rate-limited with no connect attempt in sight. - Clearing one shared recorder at the start of an attempt let a request still in flight on a superseded transport write into the new attempt's slate (and discarded a hint recorded moments before the clear). The core client now holds a generation pointer and swaps in a fresh recorder per attempt, so each transport keeps the recorder it was built with. Tests: the health-path test is verified to fail when the sync moves back behind isConnectionError, plus tools/call coverage. Refs #1040. --- internal/upstream/core/client.go | 32 +++++-- internal/upstream/core/connection.go | 6 +- internal/upstream/managed/client.go | 19 +++- internal/upstream/managed/retry_after_test.go | 93 +++++++++++++++++++ 4 files changed, 138 insertions(+), 12 deletions(-) create mode 100644 internal/upstream/managed/retry_after_test.go diff --git a/internal/upstream/core/client.go b/internal/upstream/core/client.go index 8c9282939..ca7da50b6 100644 --- a/internal/upstream/core/client.go +++ b/internal/upstream/core/client.go @@ -90,7 +90,14 @@ type Client struct { // the connection state machine sees it, so the hint is captured by a // RoundTripper installed under the MCP client and read back here via // RetryAfterDeadline (#1040). - retryAfter *proxytransport.RetryAfterRecorder + // + // It is a generation pointer, not a fixed recorder: each connect attempt + // swaps in a fresh one (beginRetryAfterGeneration). Transports built for an + // earlier attempt keep the recorder they captured at construction, so a + // request still in flight on a superseded client cannot write into the + // current attempt's slate — and the current attempt starts empty by + // construction rather than by racing a Clear. + retryAfter atomic.Pointer[proxytransport.RetryAfterRecorder] // Transport type and stderr access (for stdio) transportType string @@ -180,13 +187,13 @@ func NewClientWithOptions(id string, serverConfig *config.ServerConfig, logger * globalConfig: globalConfig, storage: storage, secretResolver: secretResolver, // Store resolver for future use - retryAfter: proxytransport.NewRetryAfterRecorder(), logger: logger.With( zap.String("upstream_id", id), zap.String("upstream_name", serverConfig.Name), ), } c.exposePrompts.Store(resolvedServerConfig.ExposePrompts) + c.retryAfter.Store(proxytransport.NewRetryAfterRecorder()) // Create secure environment manager var envConfig *secureenv.EnvConfig @@ -537,22 +544,35 @@ func (c *Client) GetConnectionInfo() types.ConnectionInfo { // silently lose the rate-limit hint for that auth strategy. func (c *Client) httpTransportConfig(serverConfig *config.ServerConfig, oauthConfig *client.OAuthConfig) *proxytransport.HTTPTransportConfig { cfg := proxytransport.CreateHTTPTransportConfig(serverConfig, oauthConfig) - cfg.RetryAfter = c.retryAfter + // The transport captures THIS generation's recorder. A later attempt swaps + // the pointer, and this client keeps writing to the recorder it was built + // with — which is exactly what keeps generations from bleeding into each + // other (#1040). + cfg.RetryAfter = c.retryAfter.Load() return cfg } +// beginRetryAfterGeneration retires the current recorder and installs a fresh +// one for a new connect attempt. Swapping rather than clearing means a request +// still in flight on a superseded transport writes into the retired recorder, +// where it can no longer be mistaken for something this attempt observed. +func (c *Client) beginRetryAfterGeneration() { + c.retryAfter.Store(proxytransport.NewRetryAfterRecorder()) +} + // RetryAfterDeadline reports the instant before which this upstream asked us not // to come back (from a `Retry-After` on a 429/503), or the zero time when it gave // no such hint. The managed client stamps it onto the state machine so both // reconnect gates honour it (#1040). func (c *Client) RetryAfterDeadline() time.Time { - return c.retryAfter.Deadline() + return c.retryAfter.Load().Deadline() } // ClearRetryAfter drops any recorded rate-limit hint. Called once a connection -// succeeds so a stale window cannot hold back a later, unrelated reconnect. +// succeeds so a hint observed by an auth strategy that was superseded by a +// working one cannot hold back a later, unrelated reconnect. func (c *Client) ClearRetryAfter() { - c.retryAfter.Clear() + c.retryAfter.Load().Clear() } // GetServerInfo returns server information from initialization diff --git a/internal/upstream/core/connection.go b/internal/upstream/core/connection.go index d54b42eb3..a4a8e8226 100644 --- a/internal/upstream/core/connection.go +++ b/internal/upstream/core/connection.go @@ -92,7 +92,9 @@ func (c *Client) Connect(ctx context.Context) error { // that then fails for an unrelated reason (connection refused, DNS) would be // re-parked for the remainder of a window the upstream never repeated. // Whatever this attempt observes is recorded again, on its own merits. - c.retryAfter.Clear() + // A swap, not a Clear: a request still in flight on the superseded client + // keeps the recorder it was built with instead of writing into this attempt. + c.beginRetryAfterGeneration() c.logger.Info("Connecting to upstream MCP server", zap.String("server", c.config.Name), @@ -245,7 +247,7 @@ func (c *Client) Connect(ctx context.Context) error { // The upstream answered, so any rate-limit window we were holding is spent. // Dropping it here keeps a stale hint from parking a future reconnect (#1040). - c.retryAfter.Clear() + c.ClearRetryAfter() // If we had an OAuth flow in progress and connection succeeded, mark OAuth as complete if c.isOAuthInProgress() { diff --git a/internal/upstream/managed/client.go b/internal/upstream/managed/client.go index 7f1c39168..81fd40b94 100644 --- a/internal/upstream/managed/client.go +++ b/internal/upstream/managed/client.go @@ -784,6 +784,12 @@ func (mc *Client) CallTool(ctx context.Context, toolName string, args map[string result, err := invoker.CallTool(ctx, toolName, args) if err != nil { mc.recordCallToolOAuthSignal(toolName, err) + // A 429 answered to a tools/call is the same instruction as one answered + // to connect (#1040). Recording it here does NOT mark the server + // unhealthy — the classification below owns that — it only makes sure + // the hint survives into the state machine, where the reconnect gates + // can honour it if this upstream later needs redialing. + mc.syncRetryAfterFromTransport() // GH #965: a canceled or timed-out CALL is not a dead SERVER. SetError // flips the whole upstream to Error and burns a retry, evicting it for // every other client — so only hard evidence of a broken transport may @@ -1266,6 +1272,15 @@ func (mc *Client) performHealthCheck() { err := prober.Ping(ctx) if err != nil { + // Pick up any rate-limit hint this ping's response carried, BEFORE the + // classification below (#1040). A 429 does not read as a connection + // error — isConnectionError matches transport-level failures, not HTTP + // statuses — so gating the sync on that branch would make it + // unreachable for exactly the case it exists for. Recording the window + // does not change the health verdict; it only stops the automatic + // reconnect paths from returning before the upstream said we may. + mc.syncRetryAfterFromTransport() + // Only mark as error if it's a real connection issue, not timeout during high activity if mc.isConnectionError(err) { if mc.recordHealthCheckFailure(err) { @@ -1273,10 +1288,6 @@ func (mc *Client) performHealthCheck() { zap.String("server", mc.GetConfig().Name), zap.Int("consecutive_failures", mc.consecutiveHealthFailures), zap.Error(err)) - // A 429 answered to the ping carries the same "come back later" - // as one answered to connect (#1040); stamp it before SetError - // so the published ConnectionInfo already carries the window. - mc.syncRetryAfterFromTransport() mc.StateManager.SetError(err) } else { mc.logger.Info("Health check failed transiently, tolerating below threshold", diff --git a/internal/upstream/managed/retry_after_test.go b/internal/upstream/managed/retry_after_test.go new file mode 100644 index 000000000..d368444ee --- /dev/null +++ b/internal/upstream/managed/retry_after_test.go @@ -0,0 +1,93 @@ +package managed + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + "github.com/smart-mcp-proxy/mcpproxy-go/internal/config" + "github.com/smart-mcp-proxy/mcpproxy-go/internal/secret" + "github.com/smart-mcp-proxy/mcpproxy-go/internal/upstream/core" + "github.com/smart-mcp-proxy/mcpproxy-go/internal/upstream/types" +) + +// coreClientThatWasRateLimited returns a core client whose transport has just +// observed a 429 with the given Retry-After — the state the recorder is in when +// any later failure path asks it for a hint (#1040). +func coreClientThatWasRateLimited(t *testing.T, retryAfter string) *core.Client { + t.Helper() + t.Setenv("MCPPROXY_DISABLE_OAUTH", "true") + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Retry-After", retryAfter) + w.WriteHeader(http.StatusTooManyRequests) + _, _ = w.Write([]byte(`{"error":"rate limit exceeded"}`)) + })) + t.Cleanup(srv.Close) + + cc, err := core.NewClient("retry-after-health-test", &config.ServerConfig{ + Name: "rate-limited-upstream", + URL: srv.URL + "/mcp", + Protocol: "http", + Enabled: true, + Created: time.Now(), + }, zap.NewNop(), nil, nil, nil, secret.NewResolver()) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + require.Error(t, cc.Connect(ctx), "a 429 upstream must not connect") + require.False(t, cc.RetryAfterDeadline().IsZero(), "precondition: the transport recorded the hint") + return cc +} + +// TestPerformHealthCheck_SyncsRetryAfter pins reachability, which is the whole +// point of this path: a 429 is NOT a connection error (isConnectionError matches +// transport failures, not HTTP statuses), so a sync gated behind that +// classification would never run for the case it exists for. The window must +// reach the state machine even though the health verdict itself is unchanged. +func TestPerformHealthCheck_SyncsRetryAfter(t *testing.T) { + mc := newTestClientForHealth(t) + mc.coreClient = coreClientThatWasRateLimited(t, "1800") + mc.healthProbe = &fakeProber{ + pingErr: errors.New("request failed with status 429: rate limit exceeded"), + } + + mc.performHealthCheck() + + info := mc.StateManager.GetConnectionInfo() + require.False(t, info.RetryAfter.IsZero(), + "a rate-limited ping must hand its window to the state machine") + assert.WithinDuration(t, time.Now().Add(30*time.Minute), info.RetryAfter, time.Minute) + assert.False(t, info.ShouldAutoReconnect(time.Now()), + "no automatic reconnect may happen inside the window") + + // The health verdict is unchanged: a 429 does not mean the transport died. + assert.Equal(t, types.StateReady, mc.StateManager.GetState(), + "recording a rate-limit window must not itself mark the server unhealthy") +} + +// TestCallTool_SyncsRetryAfter covers the other request surface that can be +// rate-limited without any connect attempt being involved. +func TestCallTool_SyncsRetryAfter(t *testing.T) { + mc := newTestClientForHealth(t) + mc.coreClient = coreClientThatWasRateLimited(t, "900") + mc.toolInvoker = &fakeToolCaller{ + err: errors.New("request failed with status 429: rate limit exceeded"), + } + + _, err := mc.CallTool(context.Background(), "some_tool", map[string]interface{}{}) + require.Error(t, err) + + info := mc.StateManager.GetConnectionInfo() + require.False(t, info.RetryAfter.IsZero(), + "a rate-limited tools/call must hand its window to the state machine") + assert.WithinDuration(t, time.Now().Add(15*time.Minute), info.RetryAfter, time.Minute) +}