From 0ad8d1d8a675108486ed8c25ba0de09ff5c7bebc Mon Sep 17 00:00:00 2001 From: Algis Dumbris Date: Mon, 24 Aug 2026 23:24:21 +0300 Subject: [PATCH 1/4] fix(upstream): close the remaining reconnect-storm holes after #1014 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #1014 gated the supervisor's 30s reconcile on the client's retry policy, but three holes in that gate remain — two of which leave the original storm intact: - OAuth-classified failures are recorded by SetOAuthError, which bumps OAuthRetryCount and never RetryCount. ShouldAutoReconnect read them as "RetryCount == 0 → no failures yet" and re-dialed on every tick, forever. ShouldAutoReconnect now also honors the OAuth ladder (5min → 24h), extracted from ShouldRetryOAuth into OAuthRetryBackoffDuration so both paths share one schedule. - The gate's StatePendingAuth branch was dead code: managed.Connect parked a login-blocked server with TransitionTo(StatePendingAuth) followed by SetError, and SetError forces StateError unconditionally, so the park lasted microseconds. StateManager.SetPendingAuth parks it for real; the supervisor, ConnectAll and the health check all leave it alone until a human acts. The wake paths are wired to match: Connect clears the stale core client from PendingAuth too, Manager.scanForNewTokens accepts a parked client when a token appears, PendingAuth is a valid transition source/target, health reports Sign-in required instead of falling through to healthy, the notifier emits the login prompt, and monitorConnectionStatus answers immediately instead of stalling until its timeout. - After MaxConnectionRetries the gate said "never again", so any outage longer than the ladder (~1h: sleep, VPN, overnight maintenance) left the upstream silently dead until someone reconnected it by hand — recovery the supervisor used to provide by re-dialing unconditionally. A given-up server is now probed once per GaveUpProbeInterval (30min): ~48 requests/day instead of ~2880, and it self-heals. Refs #1013, #1014 --- internal/health/calculator.go | 13 ++ internal/health/calculator_test.go | 32 +++++ .../runtime/supervisor/supervisor_test.go | 31 ++++- internal/server/mcp.go | 5 + internal/upstream/managed/client.go | 10 +- internal/upstream/manager.go | 6 +- internal/upstream/notifications.go | 6 + internal/upstream/types/types.go | 121 ++++++++++++++---- internal/upstream/types/types_test.go | 71 +++++++++- 9 files changed, 262 insertions(+), 33 deletions(-) diff --git a/internal/health/calculator.go b/internal/health/calculator.go index 8cfb6f2ae..afb9829e4 100644 --- a/internal/health/calculator.go +++ b/internal/health/calculator.go @@ -170,6 +170,19 @@ func CalculateHealth(input HealthCalculatorInput, cfg *HealthCalculatorConfig) * Detail: input.LastError, Action: action, } + case "pending auth", "pending_auth": + // Parked awaiting user login (#1013): the client stopped redialing on + // purpose, so this never "resolves on its own" — it is always an + // attention item with a Sign-in CTA, regardless of OAuthRequired (a + // header-auth server whose token expired is parked the same way). + level, action, summary := oauthAttentionState(input.LastError) + return &contracts.HealthStatus{ + Level: level, + AdminState: StateEnabled, + Summary: summary, + Detail: input.LastError, + Action: action, + } case "connecting", "idle": return &contracts.HealthStatus{ Level: LevelHealthy, diff --git a/internal/health/calculator_test.go b/internal/health/calculator_test.go index cf7aa2e7c..8996ebce9 100644 --- a/internal/health/calculator_test.go +++ b/internal/health/calculator_test.go @@ -221,6 +221,38 @@ func TestCalculateHealth_OAuthLoginRequired(t *testing.T) { } } +// TestCalculateHealth_PendingAuthParked verifies that a server parked in +// PendingAuth (the client stopped redialing until a human signs in, #1013) is an +// attention item with a Sign-in CTA — not a silently healthy server. The state +// string arrives as ConnectionState.String() ("Pending Auth"), possibly +// lowercased by the supervisor's stateview. +func TestCalculateHealth_PendingAuthParked(t *testing.T) { + loginErr := "OAuth authentication required for github: login available via Web UI, system tray menu, or 'mcpproxy auth login' CLI command" + + for _, state := range []string{"Pending Auth", "pending auth", "pending_auth"} { + // First-time sign-in: amber, actionable. + result := CalculateHealth(HealthCalculatorInput{ + Name: "test-server", + Enabled: true, + State: state, + LastError: loginErr, + }, nil) + assert.Equal(t, LevelDegraded, result.Level, "state=%s", state) + assert.Equal(t, "Sign-in required", result.Summary, "state=%s", state) + assert.Equal(t, ActionLogin, result.Action, "state=%s", state) + + // A broken stored token stays red. + reauth := CalculateHealth(HealthCalculatorInput{ + Name: "test-server", + Enabled: true, + State: state, + LastError: "OAuth authentication required for slack: server error with stored token - re-login available via Web UI", + }, nil) + assert.Equal(t, LevelUnhealthy, reauth.Level, "state=%s", state) + assert.Equal(t, ActionLogin, reauth.Action, "state=%s", state) + } +} + // TestCalculateHealth_OAuthReauthRequired verifies that a server whose stored // token broke (re-auth needed) stays unhealthy/red with a login action — it was // working before, so this is a regression the user should treat as red (MCP-1820). diff --git a/internal/runtime/supervisor/supervisor_test.go b/internal/runtime/supervisor/supervisor_test.go index ffb3a924a..82bffdb28 100644 --- a/internal/runtime/supervisor/supervisor_test.go +++ b/internal/runtime/supervisor/supervisor_test.go @@ -1160,17 +1160,31 @@ func TestSupervisor_Reconcile_RespectsRetryBackoff(t *testing.T) { time.Sleep(50 * time.Millisecond) require.False(t, isConnected(), "supervisor re-dialed a failed server inside its backoff window") - // A server that gave up after max retries must not be re-dialed either. + // A server that gave up after max retries must not be re-dialed either, + // until the give-up probe interval has elapsed (see below). setConnectionState(false, &types.ConnectionInfo{ State: types.StateError, RetryCount: types.MaxConnectionRetries, GaveUp: true, - LastRetryTime: time.Now().Add(-time.Hour), + LastRetryTime: time.Now().Add(-time.Minute), }) require.NoError(t, supervisor.reconcile(configSvc.Current())) time.Sleep(50 * time.Millisecond) require.False(t, isConnected(), "supervisor re-dialed a server that gave up after max retries") + // An OAuth-classified failure is paced by the OAuth ladder, which bumps + // OAuthRetryCount and never RetryCount — without that gate it reads as + // "no failures yet" and is re-dialed on every tick forever (#1013). + setConnectionState(false, &types.ConnectionInfo{ + State: types.StateError, + IsOAuthError: true, + OAuthRetryCount: 2, + LastOAuthAttempt: time.Now().Add(-time.Minute), + }) + require.NoError(t, supervisor.reconcile(configSvc.Current())) + time.Sleep(50 * time.Millisecond) + require.False(t, isConnected(), "supervisor re-dialed a server inside its OAuth backoff window") + // A server parked in PendingAuth (waiting for user OAuth login) must not be // re-dialed - each attempt fires real requests at the upstream and cannot // succeed until the user completes the login. @@ -1190,4 +1204,17 @@ func TestSupervisor_Reconcile_RespectsRetryBackoff(t *testing.T) { require.NoError(t, supervisor.reconcile(configSvc.Current())) time.Sleep(50 * time.Millisecond) require.True(t, isConnected(), "supervisor did not reconnect after the backoff window elapsed") + + // A given-up server is still probed once per GaveUpProbeInterval, so an + // outage longer than the retry ladder (sleep, VPN, maintenance) self-heals + // instead of leaving the upstream silently dead until a human notices. + setConnectionState(false, &types.ConnectionInfo{ + State: types.StateError, + RetryCount: types.MaxConnectionRetries, + GaveUp: true, + LastRetryTime: time.Now().Add(-types.GaveUpProbeInterval - time.Minute), + }) + require.NoError(t, supervisor.reconcile(configSvc.Current())) + time.Sleep(50 * time.Millisecond) + require.True(t, isConnected(), "supervisor never probes a given-up server again") } diff --git a/internal/server/mcp.go b/internal/server/mcp.go index 34071f561..07c20b0b8 100644 --- a/internal/server/mcp.go +++ b/internal/server/mcp.go @@ -5723,6 +5723,11 @@ func (p *MCPProxyServer) monitorConnectionStatus(ctx context.Context, serverName return "ready", "Server connected and ready" case types.StateError: return "error", fmt.Sprintf("Server connection failed: %v", connectionInfo.LastError) + case types.StatePendingAuth: + // Parked awaiting user login (#1013): waiting out the monitor + // timeout would just stall the caller — nothing will change + // until a human signs in. + return statusError, fmt.Sprintf("Server connection failed: %v", connectionInfo.LastError) case types.StateDisconnected: // If server is explicitly disconnected and enabled is false, return disabled for _, serverConfig := range p.config.Servers { diff --git a/internal/upstream/managed/client.go b/internal/upstream/managed/client.go index 618fbb54d..64b66e058 100644 --- a/internal/upstream/managed/client.go +++ b/internal/upstream/managed/client.go @@ -274,7 +274,7 @@ func (mc *Client) Connect(ctx context.Context) error { // that died silently (e.g., HTTP server timeout). Without this, core client // rejects the connect attempt with "client already connected" error. currentState := mc.StateManager.GetState() - if currentState == types.StateError || currentState == types.StateDisconnected { + if currentState == types.StateError || currentState == types.StateDisconnected || currentState == types.StatePendingAuth { mc.logger.Debug("Disconnecting core client before reconnect to clear stale state", zap.String("server", mc.GetConfig().Name), zap.String("from_state", currentState.String())) @@ -318,9 +318,11 @@ func (mc *Client) Connect(ctx context.Context) error { if core.IsOAuthPending(connectErr) { mc.logger.Info("⏳ OAuth authentication pending user action", zap.String("server", mc.GetConfig().Name)) - // Transition to PendingAuth state instead of Error - mc.StateManager.TransitionTo(types.StatePendingAuth) - mc.StateManager.SetError(connectErr) + // Park in PendingAuth. SetPendingAuth (not TransitionTo + SetError: + // SetError forces StateError and would immediately undo the park, so + // the supervisor kept redialing a login-blocked server every 30s — + // #1013). + mc.StateManager.SetPendingAuth(connectErr) return fmt.Errorf("OAuth authentication pending: %w", connectErr) } // Check if this is an OAuth authorization requirement (not an error) diff --git a/internal/upstream/manager.go b/internal/upstream/manager.go index 875c1a109..c2f686252 100644 --- a/internal/upstream/manager.go +++ b/internal/upstream/manager.go @@ -2156,8 +2156,10 @@ func (m *Manager) scanForNewTokens() { } state := c.GetState() - // Focus on Error state likely due to OAuth/authorization - if state != types.StateError { + // Focus on states that a freshly persisted token can unblock: a plain + // Error (likely OAuth/authorization) and PendingAuth, where the client is + // parked waiting for exactly this token (#1013). + if state != types.StateError && state != types.StatePendingAuth { continue } diff --git a/internal/upstream/notifications.go b/internal/upstream/notifications.go index 18862166d..7c7090ef7 100644 --- a/internal/upstream/notifications.go +++ b/internal/upstream/notifications.go @@ -162,6 +162,12 @@ func StateChangeNotifier(nm *NotificationManager, serverName string) func(oldSta if oldState == types.StateConnecting { nm.NotifyOAuthRequired(serverName) } + case types.StatePendingAuth: + // Parked awaiting user login: the connection attempt is over and only a + // human can move it forward, so surface the login prompt once. + if oldState != types.StatePendingAuth { + nm.NotifyOAuthRequired(serverName) + } case types.StateDisconnected: if oldState == types.StateReady { nm.NotifyServerDisconnected(serverName, info.LastError) diff --git a/internal/upstream/types/types.go b/internal/upstream/types/types.go index 1ec05f3fd..a6ef77359 100644 --- a/internal/upstream/types/types.go +++ b/internal/upstream/types/types.go @@ -86,13 +86,40 @@ func RetryBackoffDuration(retryCount int) time.Duration { return backoffDuration } +// OAuthRetryBackoffDuration returns the (much longer) backoff to wait after the +// given number of consecutive OAuth failures: 5min, 15min, 1h, 4h, then 24h. +// An OAuth failure cannot be resolved by redialing quickly — it needs a token +// refresh or a human completing a login — so its ladder is deliberately coarse. +func OAuthRetryBackoffDuration(oauthRetryCount int) time.Duration { + switch { + case oauthRetryCount <= 1: + return 5 * time.Minute + case oauthRetryCount <= 2: + return 15 * time.Minute + case oauthRetryCount <= 3: + return 1 * time.Hour + case oauthRetryCount <= 4: + return 4 * time.Hour + default: + return 24 * time.Hour // Max backoff for OAuth: 24 hours + } +} + +// GaveUpProbeInterval is how often a given-up server is still probed by the +// periodic reconciliation. After MaxConnectionRetries consecutive failures the +// client stops its own retry ladder, but "never again" would leave an upstream +// silently dead after any outage longer than the ladder (laptop sleep, VPN drop, +// overnight maintenance) until a human notices and reconnects by hand — a failed +// upstream is invisible to the operator (#1013). One probe every 30 minutes keeps +// that self-healing at ~48 requests/day instead of ~2880. +const GaveUpProbeInterval = 30 * time.Minute + // ShouldAutoReconnect reports whether an automatic (supervisor-driven) reconnect // attempt is appropriate given the connection's failure history. It returns false -// while the exponential backoff window from the last failure has not elapsed, -// after the client has given up (MaxConnectionRetries), and for servers parked in -// PendingAuth — redialing cannot succeed until the user completes the OAuth login, -// and each attempt costs real requests against the upstream. Manual reconnects, -// login flows, and reconnect-on-use are not subject to this policy. +// while a backoff window from the last failure has not elapsed and for servers +// parked in PendingAuth — redialing cannot succeed until the user completes the +// OAuth login, and each attempt costs real requests against the upstream. Manual +// reconnects, login flows, and reconnect-on-use are not subject to this policy. func (ci *ConnectionInfo) ShouldAutoReconnect(now time.Time) bool { if ci == nil { return true @@ -101,9 +128,17 @@ func (ci *ConnectionInfo) ShouldAutoReconnect(now time.Time) bool { case StatePendingAuth: return false case StateError: - if ci.GaveUp || ci.RetryCount >= MaxConnectionRetries { + // OAuth-classified failures are paced by SetOAuthError's coarse ladder, + // which bumps OAuthRetryCount and NOT RetryCount. Without this branch such + // a server reads as RetryCount==0 ("no failures yet") and is re-dialed on + // every 30s reconcile tick forever — exactly the storm this gate exists to + // stop, for the error class (#1013) that triggered it. + if ci.IsOAuthError && !ci.oauthBackoffElapsed(now) { return false } + if ci.GaveUp || ci.RetryCount >= MaxConnectionRetries { + return now.Sub(ci.LastRetryTime) >= GaveUpProbeInterval + } if ci.RetryCount == 0 { return true } @@ -113,6 +148,14 @@ func (ci *ConnectionInfo) ShouldAutoReconnect(now time.Time) bool { } } +// oauthBackoffElapsed reports whether the OAuth ladder allows another attempt. +func (ci *ConnectionInfo) oauthBackoffElapsed(now time.Time) bool { + if ci.OAuthRetryCount == 0 { + return true + } + return now.Sub(ci.LastOAuthAttempt) >= OAuthRetryBackoffDuration(ci.OAuthRetryCount) +} + // StateManager manages the state transitions for an upstream connection type StateManager struct { mu sync.RWMutex @@ -253,6 +296,48 @@ func (sm *StateManager) SetError(err error) { } } +// SetPendingAuth parks the connection in StatePendingAuth with the deferred-OAuth +// error attached. It exists because the obvious spelling — TransitionTo(StatePendingAuth) +// followed by SetError(err) — silently undoes itself: SetError forces StateError +// unconditionally, so the PendingAuth state survived microseconds and every consumer +// (health, notifications, the supervisor's reconnect gate) saw a plain error and +// kept redialing a server that cannot connect until a human logs in (#1013). +// +// RetryCount is deliberately NOT bumped: a parked server is not retrying, so there +// is no ladder to advance and nothing to "give up" on. The wake paths are explicit +// user action (login, manual reconnect, reconnect-on-use) and the persisted-token +// scan in Manager.scanForNewTokens. +func (sm *StateManager) SetPendingAuth(err error) { + sm.mu.Lock() + + oldState := sm.currentState + sm.currentState = StatePendingAuth + sm.lastError = err + sm.lastRetryTime = time.Now() + + info := ConnectionInfo{ + State: sm.currentState, + LastError: sm.lastError, + RetryCount: sm.retryCount, + LastRetryTime: sm.lastRetryTime, + ServerName: sm.serverName, + ServerVersion: sm.serverVersion, + LastOAuthAttempt: sm.lastOAuthAttempt, + OAuthRetryCount: sm.oauthRetryCount, + IsOAuthError: sm.isOAuthError, + GaveUp: sm.retryCount >= MaxConnectionRetries, + } + + callback := sm.onStateChange + sm.mu.Unlock() + + // Dispatched in a goroutine like SetError's: callers hold the managed client's + // mutex across this call, and consumers may call back into the client. + if callback != nil { + go callback(oldState, StatePendingAuth, &info) + } +} + // SetServerInfo sets the server information func (sm *StateManager) SetServerInfo(name, version string) { sm.mu.Lock() @@ -326,11 +411,15 @@ func (sm *StateManager) ValidateTransition(from, to ConnectionState) error { // Define valid transitions validTransitions := map[ConnectionState][]ConnectionState{ StateDisconnected: {StateConnecting}, - StateConnecting: {StateAuthenticating, StateDiscovering, StateReady, StateError, StateDisconnected}, // Allow direct to Ready for OAuth flows - StateAuthenticating: {StateConnecting, StateDiscovering, StateReady, StateError, StateDisconnected}, + StateConnecting: {StateAuthenticating, StateDiscovering, StateReady, StateError, StateDisconnected, StatePendingAuth}, // Allow direct to Ready for OAuth flows + StateAuthenticating: {StateConnecting, StateDiscovering, StateReady, StateError, StateDisconnected, StatePendingAuth}, StateDiscovering: {StateReady, StateError, StateDisconnected}, StateReady: {StateError, StateDisconnected}, StateError: {StateConnecting, StateDisconnected}, + // A parked server leaves PendingAuth when the user logs in / reconnects + // (StateConnecting), is disabled (StateDisconnected), or a later attempt + // fails for a non-auth reason (StateError). + StatePendingAuth: {StateConnecting, StateDisconnected, StateError}, } allowed, exists := validTransitions[from] @@ -468,21 +557,7 @@ func (sm *StateManager) ShouldRetryOAuth() bool { } // OAuth has much longer backoff intervals: 5min, 15min, 1h, 4h, 24h - var backoffDuration time.Duration - switch { - case sm.oauthRetryCount <= 1: - backoffDuration = 5 * time.Minute - case sm.oauthRetryCount <= 2: - backoffDuration = 15 * time.Minute - case sm.oauthRetryCount <= 3: - backoffDuration = 1 * time.Hour - case sm.oauthRetryCount <= 4: - backoffDuration = 4 * time.Hour - default: - backoffDuration = 24 * time.Hour // Max backoff for OAuth: 24 hours - } - - return time.Since(sm.lastOAuthAttempt) >= backoffDuration + return time.Since(sm.lastOAuthAttempt) >= OAuthRetryBackoffDuration(sm.oauthRetryCount) } // IsOAuthError returns true if the last error was OAuth-related diff --git a/internal/upstream/types/types_test.go b/internal/upstream/types/types_test.go index ff9737b9c..fd11d0339 100644 --- a/internal/upstream/types/types_test.go +++ b/internal/upstream/types/types_test.go @@ -220,8 +220,20 @@ func TestConnectionInfo_ShouldAutoReconnect(t *testing.T) { {"error within backoff window", &ConnectionInfo{State: StateError, RetryCount: 5, LastRetryTime: now.Add(-1 * time.Second)}, false}, {"error with backoff elapsed", &ConnectionInfo{State: StateError, RetryCount: 3, LastRetryTime: now.Add(-10 * time.Second)}, true}, {"error no failures yet", &ConnectionInfo{State: StateError, RetryCount: 0}, true}, - {"gave up flag", &ConnectionInfo{State: StateError, GaveUp: true, LastRetryTime: now.Add(-time.Hour)}, false}, - {"retry count at max", &ConnectionInfo{State: StateError, RetryCount: MaxConnectionRetries, LastRetryTime: now.Add(-time.Hour)}, false}, + {"gave up flag", &ConnectionInfo{State: StateError, GaveUp: true, LastRetryTime: now.Add(-time.Minute)}, false}, + {"retry count at max", &ConnectionInfo{State: StateError, RetryCount: MaxConnectionRetries, LastRetryTime: now.Add(-time.Minute)}, false}, + // Given up, but the probe interval has elapsed: one attempt is allowed so a + // long outage (sleep, VPN, maintenance) still self-heals without a human. + {"gave up, probe interval elapsed", &ConnectionInfo{State: StateError, GaveUp: true, LastRetryTime: now.Add(-GaveUpProbeInterval - time.Second)}, true}, + {"retry count at max, probe interval elapsed", &ConnectionInfo{State: StateError, RetryCount: MaxConnectionRetries, LastRetryTime: now.Add(-time.Hour)}, true}, + // OAuth failures are paced by the OAuth ladder, not RetryCount (which + // SetOAuthError never bumps) — the 30s storm's remaining hole (#1013). + {"oauth error, ladder window open", &ConnectionInfo{State: StateError, IsOAuthError: true, OAuthRetryCount: 1, LastOAuthAttempt: now.Add(-time.Minute)}, false}, + {"oauth error, ladder elapsed", &ConnectionInfo{State: StateError, IsOAuthError: true, OAuthRetryCount: 1, LastOAuthAttempt: now.Add(-6 * time.Minute)}, true}, + {"oauth error, long ladder still open", &ConnectionInfo{State: StateError, IsOAuthError: true, OAuthRetryCount: 5, LastOAuthAttempt: now.Add(-5 * time.Hour)}, false}, + {"oauth error, no oauth attempt recorded", &ConnectionInfo{State: StateError, IsOAuthError: true}, true}, + // Both ladders apply to an OAuth failure that also bumped RetryCount. + {"oauth ladder elapsed but plain backoff open", &ConnectionInfo{State: StateError, IsOAuthError: true, OAuthRetryCount: 1, LastOAuthAttempt: now.Add(-time.Hour), RetryCount: 8, LastRetryTime: now}, false}, } for _, tt := range tests { @@ -230,3 +242,58 @@ func TestConnectionInfo_ShouldAutoReconnect(t *testing.T) { }) } } + +// TestOAuthRetryBackoffDuration pins the coarse OAuth ladder shared by +// StateManager.ShouldRetryOAuth and the supervisor's reconnect gate. +func TestOAuthRetryBackoffDuration(t *testing.T) { + tests := []struct { + oauthRetryCount int + expected time.Duration + }{ + {0, 5 * time.Minute}, + {1, 5 * time.Minute}, + {2, 15 * time.Minute}, + {3, 1 * time.Hour}, + {4, 4 * time.Hour}, + {5, 24 * time.Hour}, + {50, 24 * time.Hour}, + } + + for _, tt := range tests { + assert.Equal(t, tt.expected, OAuthRetryBackoffDuration(tt.oauthRetryCount), "oauthRetryCount=%d", tt.oauthRetryCount) + } +} + +// TestSetPendingAuth verifies that parking a connection in PendingAuth sticks: +// the pre-existing spelling (TransitionTo + SetError) silently forced StateError, +// so every consumer saw a plain error and kept redialing (#1013). +func TestSetPendingAuth(t *testing.T) { + sm := NewStateManager() + sm.TransitionTo(StateConnecting) + + stateChanges := make(chan ConnectionState, 4) + sm.SetStateChangeCallback(func(_, newState ConnectionState, _ *ConnectionInfo) { + stateChanges <- newState + }) + + pendingErr := errors.New("OAuth authentication required for test-server: login available via Web UI") + sm.SetPendingAuth(pendingErr) + + info := sm.GetConnectionInfo() + assert.Equal(t, StatePendingAuth, info.State, "PendingAuth must survive - it is the parked state") + assert.Equal(t, pendingErr, info.LastError, "the deferred-OAuth error must stay attached") + assert.Equal(t, 0, info.RetryCount, "a parked server is not retrying, so no ladder is advanced") + assert.False(t, info.ShouldAutoReconnect(time.Now()), "a parked server must not be auto-redialed") + assert.False(t, sm.ShouldRetry(), "ConnectAll must not redial a parked server either") + + select { + case got := <-stateChanges: + assert.Equal(t, StatePendingAuth, got, "consumers must be told about the park") + case <-time.After(2 * time.Second): + t.Fatal("no state-change callback fired for SetPendingAuth") + } + + // The wake path (user login / manual reconnect) must be a legal transition. + assert.NoError(t, sm.ValidateTransition(StatePendingAuth, StateConnecting)) + assert.NoError(t, sm.ValidateTransition(StateConnecting, StatePendingAuth)) +} From bcf6b77bc8a09aab21dd4e27fd290283c957cb02 Mon Sep 17 00:00:00 2001 From: Algis Dumbris Date: Mon, 24 Aug 2026 23:49:50 +0300 Subject: [PATCH 2/4] fix(upstream): make the PendingAuth park reachable and stop the token-scan redial loop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cross-model review of the previous commit surfaced four defects, three of them in code it newly depended on: - `IsOAuthPending` type-asserted instead of unwrapping, but the deferred-OAuth error is wrapped twice on its way out of Connect ("all authentication strategies failed, last error: %w" then "failed to connect: %w"). The check never fired in production, so the park — and every consumer keyed to it — was unreachable. Now `errors.As`, with the production wrapping shape pinned in a test. - `Manager.scanForNewTokens` triggered a reconnect whenever *any* persisted token existed. A failing server usually still holds the expired/revoked token it just failed with, so the 5s scan redialed it every pass (rate-limited only to 10s/server) — a tighter loop than the supervisor's and one that walks straight through the park. It now records a fingerprint of the token it retried with and fires only when a login or refresh writes a different one. - `ConnectAll` gated on `StateError && !ShouldRetry()`, so a config reload redialed parked and OAuth-backed-off servers. It now consults the same `ShouldAutoReconnect` policy as the supervisor. A config change for the server itself still reconnects it via ActionReconnect. - `SetPendingAuth` dispatched its state-change callback detached, which can be scheduled after a later Connecting/Ready callback and surface a stale "sign in required" prompt right after a successful login. It is synchronous now, matching the TransitionTo that used to deliver this state. Also replaces the reconcile test's fixed sleeps with an actionWg barrier so the negative assertions cannot pass before an erroneous redial would have run. Refs #1013, #1014 --- internal/runtime/supervisor/supervisor.go | 9 +- .../runtime/supervisor/supervisor_test.go | 29 ++--- internal/upstream/core/connection_oauth.go | 13 ++- internal/upstream/core/oauth_error_test.go | 14 ++- internal/upstream/manager.go | 71 ++++++++++--- internal/upstream/manager_oauth_test.go | 100 +++++++++++++++++- internal/upstream/types/types.go | 11 +- 7 files changed, 206 insertions(+), 41 deletions(-) diff --git a/internal/runtime/supervisor/supervisor.go b/internal/runtime/supervisor/supervisor.go index 2bd4e7189..f1b2410a4 100644 --- a/internal/runtime/supervisor/supervisor.go +++ b/internal/runtime/supervisor/supervisor.go @@ -490,10 +490,11 @@ func (s *Supervisor) computeReconcilePlan(configSnapshot *configsvc.Snapshot, ac plan.Actions[name] = ActionNone } else if actual, ok := actualStates[name]; ok && !actual.ConnectionInfo.ShouldAutoReconnect(time.Now()) { // Respect the client's retry policy: exponential backoff after - // consecutive failures, gave-up after MaxConnectionRetries, and - // PendingAuth servers waiting on user OAuth login. Without this - // gate the periodic 30s reconciliation re-dials a dead upstream - // forever, hammering the remote server (~3 requests per tick). + // consecutive failures, the coarse OAuth ladder, half-hourly + // probes once the client gave up, and PendingAuth servers + // parked waiting on user OAuth login. Without this gate the + // periodic 30s reconciliation re-dials a dead upstream forever, + // hammering the remote server (~3 requests per tick). s.logger.Debug("Skipping auto-reconnect (backoff/pending-auth)", zap.String("server", name), zap.String("state", actual.ConnectionInfo.State.String()), diff --git a/internal/runtime/supervisor/supervisor_test.go b/internal/runtime/supervisor/supervisor_test.go index 82bffdb28..abb7ecc37 100644 --- a/internal/runtime/supervisor/supervisor_test.go +++ b/internal/runtime/supervisor/supervisor_test.go @@ -1132,7 +1132,7 @@ func TestSupervisor_Reconcile_RespectsRetryBackoff(t *testing.T) { // First reconciliation - server is added and connected require.NoError(t, supervisor.reconcile(configSvc.Current())) - time.Sleep(50 * time.Millisecond) + supervisor.actionWg.Wait() setConnectionState := func(connected bool, info *types.ConnectionInfo) { mockUpstream.mu.Lock() @@ -1148,6 +1148,15 @@ func TestSupervisor_Reconcile_RespectsRetryBackoff(t *testing.T) { defer mockUpstream.mu.Unlock() return mockUpstream.connected["flaky-server"] } + // reconcile dispatches its actions into goroutines tracked by actionWg, so + // draining that group is an exact barrier: after it returns, either the + // connect ran or none was planned. A fixed sleep would let the negative + // assertions below pass before an erroneous redial had a chance to execute. + reconcileAndDrain := func() { + t.Helper() + require.NoError(t, supervisor.reconcile(configSvc.Current())) + supervisor.actionWg.Wait() + } // Simulate a connection failure with the backoff window still open: // reconciliation must NOT re-dial. @@ -1156,8 +1165,7 @@ func TestSupervisor_Reconcile_RespectsRetryBackoff(t *testing.T) { RetryCount: 5, LastRetryTime: time.Now(), }) - require.NoError(t, supervisor.reconcile(configSvc.Current())) - time.Sleep(50 * time.Millisecond) + reconcileAndDrain() require.False(t, isConnected(), "supervisor re-dialed a failed server inside its backoff window") // A server that gave up after max retries must not be re-dialed either, @@ -1168,8 +1176,7 @@ func TestSupervisor_Reconcile_RespectsRetryBackoff(t *testing.T) { GaveUp: true, LastRetryTime: time.Now().Add(-time.Minute), }) - require.NoError(t, supervisor.reconcile(configSvc.Current())) - time.Sleep(50 * time.Millisecond) + reconcileAndDrain() require.False(t, isConnected(), "supervisor re-dialed a server that gave up after max retries") // An OAuth-classified failure is paced by the OAuth ladder, which bumps @@ -1181,8 +1188,7 @@ func TestSupervisor_Reconcile_RespectsRetryBackoff(t *testing.T) { OAuthRetryCount: 2, LastOAuthAttempt: time.Now().Add(-time.Minute), }) - require.NoError(t, supervisor.reconcile(configSvc.Current())) - time.Sleep(50 * time.Millisecond) + reconcileAndDrain() require.False(t, isConnected(), "supervisor re-dialed a server inside its OAuth backoff window") // A server parked in PendingAuth (waiting for user OAuth login) must not be @@ -1191,8 +1197,7 @@ func TestSupervisor_Reconcile_RespectsRetryBackoff(t *testing.T) { setConnectionState(false, &types.ConnectionInfo{ State: types.StatePendingAuth, }) - require.NoError(t, supervisor.reconcile(configSvc.Current())) - time.Sleep(50 * time.Millisecond) + reconcileAndDrain() require.False(t, isConnected(), "supervisor re-dialed a server pending OAuth login") // Once the backoff window has elapsed, reconciliation reconnects as before. @@ -1201,8 +1206,7 @@ func TestSupervisor_Reconcile_RespectsRetryBackoff(t *testing.T) { RetryCount: 3, LastRetryTime: time.Now().Add(-10 * time.Second), // backoff for 3 failures is 4s }) - require.NoError(t, supervisor.reconcile(configSvc.Current())) - time.Sleep(50 * time.Millisecond) + reconcileAndDrain() require.True(t, isConnected(), "supervisor did not reconnect after the backoff window elapsed") // A given-up server is still probed once per GaveUpProbeInterval, so an @@ -1214,7 +1218,6 @@ func TestSupervisor_Reconcile_RespectsRetryBackoff(t *testing.T) { GaveUp: true, LastRetryTime: time.Now().Add(-types.GaveUpProbeInterval - time.Minute), }) - require.NoError(t, supervisor.reconcile(configSvc.Current())) - time.Sleep(50 * time.Millisecond) + reconcileAndDrain() require.True(t, isConnected(), "supervisor never probes a given-up server again") } diff --git a/internal/upstream/core/connection_oauth.go b/internal/upstream/core/connection_oauth.go index 7e3122633..8f4819a1e 100644 --- a/internal/upstream/core/connection_oauth.go +++ b/internal/upstream/core/connection_oauth.go @@ -69,10 +69,17 @@ func (e *ErrOAuthPending) Code() diagnostics.Code { return diagnostics.OAuthLoginRequired } -// IsOAuthPending checks if an error is an ErrOAuthPending +// IsOAuthPending checks if an error is (or wraps) an ErrOAuthPending. +// +// It MUST unwrap: the pending error is raised inside an auth strategy and then +// wrapped twice on its way out — connectHTTP/connectSSE add "all authentication +// strategies failed, last error: %w" and Connect adds "failed to connect: %w". +// With a bare type assertion the check therefore never fired in production, and +// every login-blocked server fell through to the generic error path and was +// re-dialed instead of parked (#1013). func IsOAuthPending(err error) bool { - _, ok := err.(*ErrOAuthPending) - return ok + var pending *ErrOAuthPending + return errors.As(err, &pending) } // OAuthStartResult contains the result of initiating an OAuth flow. diff --git a/internal/upstream/core/oauth_error_test.go b/internal/upstream/core/oauth_error_test.go index 880a8832e..430bc875f 100644 --- a/internal/upstream/core/oauth_error_test.go +++ b/internal/upstream/core/oauth_error_test.go @@ -2,6 +2,7 @@ package core import ( "errors" + "fmt" "testing" "github.com/smart-mcp-proxy/mcpproxy-go/internal/diagnostics" @@ -217,12 +218,23 @@ func TestIsOAuthPending(t *testing.T) { expected: false, }, { - name: "wrapped ErrOAuthPending returns false", + name: "flattened-to-string ErrOAuthPending returns false", err: errors.New("wrapped: " + (&ErrOAuthPending{ ServerName: "slack", }).Error()), expected: false, }, + { + // The production chain: connectHTTP/connectSSE wrap the strategy's + // error and Connect wraps that again. A bare type assertion made the + // pending check unreachable, so login-blocked servers were re-dialed + // forever instead of parked (#1013). + name: "%w-wrapped ErrOAuthPending returns true", + err: fmt.Errorf("failed to connect: %w", + fmt.Errorf("all authentication strategies failed, last error: %w", + &ErrOAuthPending{ServerName: "slack", Message: "login available via Web UI"})), + expected: true, + }, } for _, tt := range tests { diff --git a/internal/upstream/manager.go b/internal/upstream/manager.go index c2f686252..db758eeab 100644 --- a/internal/upstream/manager.go +++ b/internal/upstream/manager.go @@ -2,6 +2,8 @@ package upstream import ( "context" + "crypto/sha256" + "encoding/hex" "errors" "fmt" "maps" @@ -12,6 +14,7 @@ import ( "sync/atomic" "time" + uptransport "github.com/mark3labs/mcp-go/client/transport" "go.uber.org/zap" "github.com/smart-mcp-proxy/mcpproxy-go/internal/config" @@ -113,6 +116,12 @@ type Manager struct { // cannot write due to DB lock). Prevents rapid retrigger loops. tokenReconnect map[string]time.Time + // tokenFingerprints records which persisted token each server was last + // retried with, so the scan fires on a NEW token rather than on the mere + // presence of the stale one it already failed with (#1013). Same goroutine + // as tokenReconnect (the OAuth event monitor), so it needs no extra lock. + tokenFingerprints map[string]string + // Context for shutdown coordination shutdownCtx context.Context shutdownCancel context.CancelFunc @@ -185,17 +194,18 @@ func cloneServerConfig(cfg *config.ServerConfig) *config.ServerConfig { func NewManager(logger *zap.Logger, globalConfig *config.Config, boltStorage *storage.BoltDB, secretResolver *secret.Resolver, storageMgr *storage.Manager) *Manager { shutdownCtx, shutdownCancel := context.WithCancel(context.Background()) manager := &Manager{ - clients: make(map[string]*managed.Client), - logger: logger, - storage: boltStorage, - notificationMgr: NewNotificationManager(), - secretResolver: secretResolver, - tokenReconnect: make(map[string]time.Time), - lastSweptAt: make(map[string]time.Time), - shutdownCtx: shutdownCtx, - shutdownCancel: shutdownCancel, - storageMgr: storageMgr, - limiters: limiter.NewRegistry(), + clients: make(map[string]*managed.Client), + logger: logger, + storage: boltStorage, + notificationMgr: NewNotificationManager(), + secretResolver: secretResolver, + tokenReconnect: make(map[string]time.Time), + tokenFingerprints: make(map[string]string), + lastSweptAt: make(map[string]time.Time), + shutdownCtx: shutdownCtx, + shutdownCancel: shutdownCancel, + storageMgr: storageMgr, + limiters: limiter.NewRegistry(), } manager.globalConfig.Store(globalConfig) // Spec 093: publish the initial limit generation. With no limits configured @@ -1469,11 +1479,18 @@ func (m *Manager) ConnectAll(ctx context.Context) error { continue } - if client.GetState() == types.StateError && !client.ShouldRetry() { - info := client.GetConnectionInfo() + // Same retry policy the supervisor's reconcile honors: plain backoff, + // the coarse OAuth ladder, half-hourly probes after give-up, and no + // redial at all for a server parked awaiting login. ConnectAll runs on + // every config reload, so gating only on StateError+ShouldRetry (as it + // used to) let a reload re-dial servers their own client had paused + // (#1013). A user-driven config change for THIS server still reconnects + // it — the supervisor plans ActionReconnect, which bypasses the gate. + if info := client.GetConnectionInfo(); !info.ShouldAutoReconnect(time.Now()) { m.logger.Debug("Client backoff active, skipping connect attempt", zap.String("id", id), zap.String("name", client.GetConfig().Name), + zap.String("state", info.State.String()), zap.Int("retry_count", info.RetryCount), zap.Time("last_retry_time", info.LastRetryTime)) continue @@ -2175,16 +2192,40 @@ func (m *Manager) scanForNewTokens() { continue } - m.logger.Info("Detected persisted OAuth token; triggering reconnect", + // Only a token we have NOT already retried with is news. The failing + // server usually still has its (expired/revoked/rejected) token in the + // store, so triggering on mere presence redialed it every scan — a 5s + // loop that outpaces the supervisor's and defeats the PendingAuth park + // (#1013). The fingerprint changes exactly when a login/refresh writes a + // new token, which is the event this scan exists to catch. + fingerprint := tokenFingerprint(tok) + if seen, ok := m.tokenFingerprints[id]; ok && seen == fingerprint { + continue + } + + m.logger.Info("Detected new persisted OAuth token; triggering reconnect", zap.String("server", cfg.Name), zap.Time("token_expires_at", tok.ExpiresAt)) - // Remember trigger time and retry connection + // Remember trigger time + which token we tried, then retry connection m.tokenReconnect[id] = now + m.tokenFingerprints[id] = fingerprint _ = m.RetryConnection(cfg.Name) } } +// tokenFingerprint identifies a persisted OAuth token without retaining it: a +// truncated SHA-256 over the access token, refresh token and expiry, so a +// refreshed or re-issued token compares different while the same stale token +// compares equal. Never log or persist the raw token to make this comparison. +func tokenFingerprint(tok *uptransport.Token) string { + if tok == nil { + return "" + } + sum := sha256.Sum256([]byte(tok.AccessToken + "\x00" + tok.RefreshToken + "\x00" + tok.ExpiresAt.UTC().Format(time.RFC3339Nano))) + return hex.EncodeToString(sum[:8]) +} + // StartManualOAuth performs an in-process OAuth flow for the given server. // This avoids cross-process DB locking by using the daemon's storage directly. func (m *Manager) StartManualOAuth(serverName string, force bool) error { diff --git a/internal/upstream/manager_oauth_test.go b/internal/upstream/manager_oauth_test.go index c935574b9..89639b45b 100644 --- a/internal/upstream/manager_oauth_test.go +++ b/internal/upstream/manager_oauth_test.go @@ -1,9 +1,11 @@ package upstream import ( + "errors" "testing" "time" + uptransport "github.com/mark3labs/mcp-go/client/transport" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/zap" @@ -54,8 +56,8 @@ func TestRefreshOAuthToken_DynamicOAuthDiscovery(t *testing.T) { // Store an OAuth token for the server (as if it had authenticated previously) // The ServerName field is used as the storage key (must match GenerateServerKey output) token := &storage.OAuthTokenRecord{ - ServerName: serverKey, // Key used for storage lookup (hash-based) - DisplayName: "test-dynamic-oauth", // Human-readable name for RefreshManager + ServerName: serverKey, // Key used for storage lookup (hash-based) + DisplayName: "test-dynamic-oauth", // Human-readable name for RefreshManager AccessToken: "expired-access-token", RefreshToken: "valid-refresh-token", TokenType: "Bearer", @@ -176,3 +178,97 @@ func TestRefreshOAuthToken_ServerNotFound(t *testing.T) { require.Error(t, err) assert.Contains(t, err.Error(), "server not found") } + +// TestTokenFingerprint verifies the identity used to decide whether a persisted +// token is NEW: the same token must compare equal (so the scan does not redial), +// a refreshed/re-issued one must not. +func TestTokenFingerprint(t *testing.T) { + expires := time.Now().Add(time.Hour).UTC() + base := &uptransport.Token{AccessToken: "at", RefreshToken: "rt", ExpiresAt: expires} + + assert.Equal(t, tokenFingerprint(base), tokenFingerprint(&uptransport.Token{ + AccessToken: "at", RefreshToken: "rt", ExpiresAt: expires, + }), "same token must fingerprint the same") + + assert.NotEqual(t, tokenFingerprint(base), tokenFingerprint(&uptransport.Token{ + AccessToken: "at2", RefreshToken: "rt", ExpiresAt: expires, + }), "a new access token must fingerprint differently") + + assert.NotEqual(t, tokenFingerprint(base), tokenFingerprint(&uptransport.Token{ + AccessToken: "at", RefreshToken: "rt", ExpiresAt: expires.Add(time.Minute), + }), "a refreshed expiry must fingerprint differently") + + assert.NotContains(t, tokenFingerprint(base), "at", "the fingerprint must not carry the token") + assert.Empty(t, tokenFingerprint(nil)) +} + +// TestScanForNewTokens_OnlyOnNewToken pins the fix for the 5s redial loop: the +// scan must fire when a login/refresh writes a NEW token, not on every pass just +// because the server still holds the stale token it already failed with (#1013). +func TestScanForNewTokens_OnlyOnNewToken(t *testing.T) { + logger := zap.NewNop() + tempDir := t.TempDir() + db, err := storage.NewBoltDB(tempDir, logger.Sugar()) + require.NoError(t, err) + defer db.Close() + + serverConfig := &config.ServerConfig{ + Name: "parked-server", + URL: "http://127.0.0.1:1/mcp", // refused immediately; no real upstream needed + Protocol: "http", + Enabled: true, + Created: time.Now(), + } + serverKey := oauth.GenerateServerKey(serverConfig.Name, serverConfig.URL) + saveToken := func(access string) { + require.NoError(t, db.SaveOAuthToken(&storage.OAuthTokenRecord{ + ServerName: serverKey, + DisplayName: serverConfig.Name, + AccessToken: access, + TokenType: "Bearer", + ExpiresAt: time.Now().Add(time.Hour), + Created: time.Now(), + Updated: time.Now(), + })) + } + saveToken("stale-token") + + manager := &Manager{ + clients: make(map[string]*managed.Client), + logger: logger, + storage: db, + secretResolver: secret.NewResolver(), + tokenReconnect: make(map[string]time.Time), + tokenFingerprints: make(map[string]string), + } + + client, err := managed.NewClient("parked-server", serverConfig, logger, nil, &config.Config{}, db, secret.NewResolver()) + require.NoError(t, err) + manager.clients["parked-server"] = client + + // Park the client exactly as a deferred-OAuth connect failure would. + client.StateManager.SetPendingAuth(errors.New("OAuth authentication required for parked-server: login available via Web UI")) + + // Pretend the per-server rate limit has expired so it cannot mask the result. + expireRateLimit := func() { manager.tokenReconnect["parked-server"] = time.Now().Add(-time.Minute) } + + expireRateLimit() + manager.scanForNewTokens() + require.WithinDuration(t, time.Now(), manager.tokenReconnect["parked-server"], time.Second, + "first scan must retry with the stored token") + require.NotEmpty(t, manager.tokenFingerprints["parked-server"]) + + // Same token, rate limit expired: must NOT redial again. + expireRateLimit() + before := manager.tokenReconnect["parked-server"] + manager.scanForNewTokens() + require.Equal(t, before, manager.tokenReconnect["parked-server"], + "scan redialed a parked server for a token it had already tried") + + // A fresh token (login completed) must wake it. + saveToken("fresh-token") + expireRateLimit() + manager.scanForNewTokens() + require.WithinDuration(t, time.Now(), manager.tokenReconnect["parked-server"], time.Second, + "scan did not wake the parked server when a new token appeared") +} diff --git a/internal/upstream/types/types.go b/internal/upstream/types/types.go index a6ef77359..faf8a0b40 100644 --- a/internal/upstream/types/types.go +++ b/internal/upstream/types/types.go @@ -331,10 +331,15 @@ func (sm *StateManager) SetPendingAuth(err error) { callback := sm.onStateChange sm.mu.Unlock() - // Dispatched in a goroutine like SetError's: callers hold the managed client's - // mutex across this call, and consumers may call back into the client. + // Dispatched synchronously, like TransitionTo (which delivered this very + // state before) and unlike SetError: a detached goroutine can be scheduled + // after a subsequent Connecting/Ready callback, which would surface a stale + // "sign in required" prompt right after a successful login. The registered + // consumers only log and hand off to notification handlers that are + // themselves goroutine-dispatched, and they never take the managed client's + // mutex — the same reason TransitionTo can already run under it. if callback != nil { - go callback(oldState, StatePendingAuth, &info) + callback(oldState, StatePendingAuth, &info) } } From 040410e2f7cca1fdbb77dabad031086a060ffc0f Mon Sep 17 00:00:00 2001 From: Algis Dumbris Date: Mon, 24 Aug 2026 23:54:44 +0300 Subject: [PATCH 3/4] fix(upstream): fingerprint the token write, not just its bytes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round-2 review: a provider that re-issues a byte-identical access token (same refresh token, same expiry) left the scan's fingerprint unchanged, permanently suppressing the wake for a parked server — and when a CLI login cannot persist its OAuth completion event, this scan is the only wake left. The record's persisted Updated stamp is now part of the fingerprint, so any new write counts as news while an untouched token still does not. Refs #1013 --- internal/upstream/manager.go | 28 +++++++++++++++---- internal/upstream/manager_oauth_test.go | 37 ++++++++++++++++++------- 2 files changed, 49 insertions(+), 16 deletions(-) diff --git a/internal/upstream/manager.go b/internal/upstream/manager.go index db758eeab..dfdad177e 100644 --- a/internal/upstream/manager.go +++ b/internal/upstream/manager.go @@ -2198,7 +2198,17 @@ func (m *Manager) scanForNewTokens() { // loop that outpaces the supervisor's and defeats the PendingAuth park // (#1013). The fingerprint changes exactly when a login/refresh writes a // new token, which is the event this scan exists to catch. - fingerprint := tokenFingerprint(tok) + // + // The write timestamp is part of the identity, not just the token + // content: a provider re-issuing a byte-identical token would otherwise + // leave the fingerprint unchanged and permanently suppress this wake — + // and for a CLI login that could not persist its completion event, this + // scan is the only one left. + var writtenAt time.Time + if rec, recErr := m.storage.GetOAuthToken(oauth.GenerateServerKey(cfg.Name, cfg.URL)); recErr == nil && rec != nil { + writtenAt = rec.Updated + } + fingerprint := tokenFingerprint(tok, writtenAt) if seen, ok := m.tokenFingerprints[id]; ok && seen == fingerprint { continue } @@ -2215,14 +2225,20 @@ func (m *Manager) scanForNewTokens() { } // tokenFingerprint identifies a persisted OAuth token without retaining it: a -// truncated SHA-256 over the access token, refresh token and expiry, so a -// refreshed or re-issued token compares different while the same stale token -// compares equal. Never log or persist the raw token to make this comparison. -func tokenFingerprint(tok *uptransport.Token) string { +// truncated SHA-256 over the access token, refresh token, expiry and the time +// the record was last written, so any re-issue compares different while the +// same untouched token compares equal. Never log or persist the raw token to +// make this comparison. +func tokenFingerprint(tok *uptransport.Token, writtenAt time.Time) string { if tok == nil { return "" } - sum := sha256.Sum256([]byte(tok.AccessToken + "\x00" + tok.RefreshToken + "\x00" + tok.ExpiresAt.UTC().Format(time.RFC3339Nano))) + sum := sha256.Sum256([]byte(strings.Join([]string{ + tok.AccessToken, + tok.RefreshToken, + tok.ExpiresAt.UTC().Format(time.RFC3339Nano), + writtenAt.UTC().Format(time.RFC3339Nano), + }, "\x00"))) return hex.EncodeToString(sum[:8]) } diff --git a/internal/upstream/manager_oauth_test.go b/internal/upstream/manager_oauth_test.go index 89639b45b..d85d53a34 100644 --- a/internal/upstream/manager_oauth_test.go +++ b/internal/upstream/manager_oauth_test.go @@ -180,26 +180,33 @@ func TestRefreshOAuthToken_ServerNotFound(t *testing.T) { } // TestTokenFingerprint verifies the identity used to decide whether a persisted -// token is NEW: the same token must compare equal (so the scan does not redial), -// a refreshed/re-issued one must not. +// token is NEW: the same untouched token must compare equal (so the scan does +// not redial), any re-issue must not. func TestTokenFingerprint(t *testing.T) { expires := time.Now().Add(time.Hour).UTC() + written := time.Now().UTC() base := &uptransport.Token{AccessToken: "at", RefreshToken: "rt", ExpiresAt: expires} + fp := tokenFingerprint(base, written) - assert.Equal(t, tokenFingerprint(base), tokenFingerprint(&uptransport.Token{ + assert.Equal(t, fp, tokenFingerprint(&uptransport.Token{ AccessToken: "at", RefreshToken: "rt", ExpiresAt: expires, - }), "same token must fingerprint the same") + }, written), "same token, same write: must fingerprint the same") - assert.NotEqual(t, tokenFingerprint(base), tokenFingerprint(&uptransport.Token{ + assert.NotEqual(t, fp, tokenFingerprint(&uptransport.Token{ AccessToken: "at2", RefreshToken: "rt", ExpiresAt: expires, - }), "a new access token must fingerprint differently") + }, written), "a new access token must fingerprint differently") - assert.NotEqual(t, tokenFingerprint(base), tokenFingerprint(&uptransport.Token{ + assert.NotEqual(t, fp, tokenFingerprint(&uptransport.Token{ AccessToken: "at", RefreshToken: "rt", ExpiresAt: expires.Add(time.Minute), - }), "a refreshed expiry must fingerprint differently") + }, written), "a refreshed expiry must fingerprint differently") - assert.NotContains(t, tokenFingerprint(base), "at", "the fingerprint must not carry the token") - assert.Empty(t, tokenFingerprint(nil)) + // A provider may re-issue a byte-identical token; the write stamp is what + // keeps that from silently suppressing the wake. + assert.NotEqual(t, fp, tokenFingerprint(base, written.Add(time.Second)), + "a token rewritten identically must still fingerprint differently") + + assert.NotContains(t, fp, "at", "the fingerprint must not carry the token") + assert.Empty(t, tokenFingerprint(nil, written)) } // TestScanForNewTokens_OnlyOnNewToken pins the fix for the 5s redial loop: the @@ -271,4 +278,14 @@ func TestScanForNewTokens_OnlyOnNewToken(t *testing.T) { manager.scanForNewTokens() require.WithinDuration(t, time.Now(), manager.tokenReconnect["parked-server"], time.Second, "scan did not wake the parked server when a new token appeared") + + // A re-login that happens to persist a byte-identical token is still a new + // write, and this scan is the fallback wake when the CLI could not record an + // OAuth completion event — it must not be swallowed. + time.Sleep(2 * time.Millisecond) // ensure a distinct Updated stamp + saveToken("fresh-token") + expireRateLimit() + manager.scanForNewTokens() + require.WithinDuration(t, time.Now(), manager.tokenReconnect["parked-server"], time.Second, + "scan ignored an identically-rewritten token") } From 5b8e0a328cca410643a03d555ff505c61bba032b Mon Sep 17 00:00:00 2001 From: Algis Dumbris Date: Tue, 25 Aug 2026 00:03:40 +0300 Subject: [PATCH 4/4] test(upstream): wait for the client to settle between token-scan legs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The scan test's later legs ran while the reconnect triggered by the previous leg was still in flight, so the client could be Connecting — a state the scan skips for a reason the test is not about — and the assertion failed in CI. Each leg now waits for the client to settle back into PendingAuth or Error first. Refs #1013 --- internal/upstream/manager_oauth_test.go | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/internal/upstream/manager_oauth_test.go b/internal/upstream/manager_oauth_test.go index d85d53a34..c3456cb1f 100644 --- a/internal/upstream/manager_oauth_test.go +++ b/internal/upstream/manager_oauth_test.go @@ -15,6 +15,7 @@ import ( "github.com/smart-mcp-proxy/mcpproxy-go/internal/secret" "github.com/smart-mcp-proxy/mcpproxy-go/internal/storage" "github.com/smart-mcp-proxy/mcpproxy-go/internal/upstream/managed" + "github.com/smart-mcp-proxy/mcpproxy-go/internal/upstream/types" ) // TestRefreshOAuthToken_DynamicOAuthDiscovery tests that RefreshOAuthToken works @@ -256,8 +257,24 @@ func TestScanForNewTokens_OnlyOnNewToken(t *testing.T) { // Park the client exactly as a deferred-OAuth connect failure would. client.StateManager.SetPendingAuth(errors.New("OAuth authentication required for parked-server: login available via Web UI")) + // A triggered scan calls RetryConnection, which reconnects in the background + // (the dial is refused immediately, but not synchronously). Wait for the + // client to settle back into a state the scan considers before the next leg, + // otherwise a leg can land while it is still Connecting and be skipped for a + // reason the test is not about. + settled := func() { + t.Helper() + require.Eventually(t, func() bool { + st := client.GetState() + return st == types.StatePendingAuth || st == types.StateError + }, 30*time.Second, 10*time.Millisecond, "client never settled into a scannable state") + } // Pretend the per-server rate limit has expired so it cannot mask the result. - expireRateLimit := func() { manager.tokenReconnect["parked-server"] = time.Now().Add(-time.Minute) } + expireRateLimit := func() { + t.Helper() + settled() + manager.tokenReconnect["parked-server"] = time.Now().Add(-time.Minute) + } expireRateLimit() manager.scanForNewTokens() @@ -281,8 +298,9 @@ func TestScanForNewTokens_OnlyOnNewToken(t *testing.T) { // A re-login that happens to persist a byte-identical token is still a new // write, and this scan is the fallback wake when the CLI could not record an - // OAuth completion event — it must not be swallowed. - time.Sleep(2 * time.Millisecond) // ensure a distinct Updated stamp + // OAuth completion event — it must not be swallowed. SaveOAuthToken stamps + // Updated with the wall clock, so give it a distinct instant. + time.Sleep(2 * time.Millisecond) saveToken("fresh-token") expireRateLimit() manager.scanForNewTokens()