diff --git a/internal/health/calculator.go b/internal/health/calculator.go index 8cfb6f2ae..afb9829e4 100644 --- a/internal/health/calculator.go +++ b/internal/health/calculator.go @@ -170,6 +170,19 @@ func CalculateHealth(input HealthCalculatorInput, cfg *HealthCalculatorConfig) * Detail: input.LastError, Action: action, } + case "pending auth", "pending_auth": + // Parked awaiting user login (#1013): the client stopped redialing on + // purpose, so this never "resolves on its own" — it is always an + // attention item with a Sign-in CTA, regardless of OAuthRequired (a + // header-auth server whose token expired is parked the same way). + level, action, summary := oauthAttentionState(input.LastError) + return &contracts.HealthStatus{ + Level: level, + AdminState: StateEnabled, + Summary: summary, + Detail: input.LastError, + Action: action, + } case "connecting", "idle": return &contracts.HealthStatus{ Level: LevelHealthy, diff --git a/internal/health/calculator_test.go b/internal/health/calculator_test.go index cf7aa2e7c..8996ebce9 100644 --- a/internal/health/calculator_test.go +++ b/internal/health/calculator_test.go @@ -221,6 +221,38 @@ func TestCalculateHealth_OAuthLoginRequired(t *testing.T) { } } +// TestCalculateHealth_PendingAuthParked verifies that a server parked in +// PendingAuth (the client stopped redialing until a human signs in, #1013) is an +// attention item with a Sign-in CTA — not a silently healthy server. The state +// string arrives as ConnectionState.String() ("Pending Auth"), possibly +// lowercased by the supervisor's stateview. +func TestCalculateHealth_PendingAuthParked(t *testing.T) { + loginErr := "OAuth authentication required for github: login available via Web UI, system tray menu, or 'mcpproxy auth login' CLI command" + + for _, state := range []string{"Pending Auth", "pending auth", "pending_auth"} { + // First-time sign-in: amber, actionable. + result := CalculateHealth(HealthCalculatorInput{ + Name: "test-server", + Enabled: true, + State: state, + LastError: loginErr, + }, nil) + assert.Equal(t, LevelDegraded, result.Level, "state=%s", state) + assert.Equal(t, "Sign-in required", result.Summary, "state=%s", state) + assert.Equal(t, ActionLogin, result.Action, "state=%s", state) + + // A broken stored token stays red. + reauth := CalculateHealth(HealthCalculatorInput{ + Name: "test-server", + Enabled: true, + State: state, + LastError: "OAuth authentication required for slack: server error with stored token - re-login available via Web UI", + }, nil) + assert.Equal(t, LevelUnhealthy, reauth.Level, "state=%s", state) + assert.Equal(t, ActionLogin, reauth.Action, "state=%s", state) + } +} + // TestCalculateHealth_OAuthReauthRequired verifies that a server whose stored // token broke (re-auth needed) stays unhealthy/red with a login action — it was // working before, so this is a regression the user should treat as red (MCP-1820). diff --git a/internal/runtime/supervisor/supervisor.go b/internal/runtime/supervisor/supervisor.go index 2bd4e7189..f1b2410a4 100644 --- a/internal/runtime/supervisor/supervisor.go +++ b/internal/runtime/supervisor/supervisor.go @@ -490,10 +490,11 @@ func (s *Supervisor) computeReconcilePlan(configSnapshot *configsvc.Snapshot, ac plan.Actions[name] = ActionNone } else if actual, ok := actualStates[name]; ok && !actual.ConnectionInfo.ShouldAutoReconnect(time.Now()) { // Respect the client's retry policy: exponential backoff after - // consecutive failures, gave-up after MaxConnectionRetries, and - // PendingAuth servers waiting on user OAuth login. Without this - // gate the periodic 30s reconciliation re-dials a dead upstream - // forever, hammering the remote server (~3 requests per tick). + // consecutive failures, the coarse OAuth ladder, half-hourly + // probes once the client gave up, and PendingAuth servers + // parked waiting on user OAuth login. Without this gate the + // periodic 30s reconciliation re-dials a dead upstream forever, + // hammering the remote server (~3 requests per tick). s.logger.Debug("Skipping auto-reconnect (backoff/pending-auth)", zap.String("server", name), zap.String("state", actual.ConnectionInfo.State.String()), diff --git a/internal/runtime/supervisor/supervisor_test.go b/internal/runtime/supervisor/supervisor_test.go index ffb3a924a..abb7ecc37 100644 --- a/internal/runtime/supervisor/supervisor_test.go +++ b/internal/runtime/supervisor/supervisor_test.go @@ -1132,7 +1132,7 @@ func TestSupervisor_Reconcile_RespectsRetryBackoff(t *testing.T) { // First reconciliation - server is added and connected require.NoError(t, supervisor.reconcile(configSvc.Current())) - time.Sleep(50 * time.Millisecond) + supervisor.actionWg.Wait() setConnectionState := func(connected bool, info *types.ConnectionInfo) { mockUpstream.mu.Lock() @@ -1148,6 +1148,15 @@ func TestSupervisor_Reconcile_RespectsRetryBackoff(t *testing.T) { defer mockUpstream.mu.Unlock() return mockUpstream.connected["flaky-server"] } + // reconcile dispatches its actions into goroutines tracked by actionWg, so + // draining that group is an exact barrier: after it returns, either the + // connect ran or none was planned. A fixed sleep would let the negative + // assertions below pass before an erroneous redial had a chance to execute. + reconcileAndDrain := func() { + t.Helper() + require.NoError(t, supervisor.reconcile(configSvc.Current())) + supervisor.actionWg.Wait() + } // Simulate a connection failure with the backoff window still open: // reconciliation must NOT re-dial. @@ -1156,29 +1165,39 @@ func TestSupervisor_Reconcile_RespectsRetryBackoff(t *testing.T) { RetryCount: 5, LastRetryTime: time.Now(), }) - require.NoError(t, supervisor.reconcile(configSvc.Current())) - time.Sleep(50 * time.Millisecond) + reconcileAndDrain() require.False(t, isConnected(), "supervisor re-dialed a failed server inside its backoff window") - // A server that gave up after max retries must not be re-dialed either. + // A server that gave up after max retries must not be re-dialed either, + // until the give-up probe interval has elapsed (see below). setConnectionState(false, &types.ConnectionInfo{ State: types.StateError, RetryCount: types.MaxConnectionRetries, GaveUp: true, - LastRetryTime: time.Now().Add(-time.Hour), + LastRetryTime: time.Now().Add(-time.Minute), }) - require.NoError(t, supervisor.reconcile(configSvc.Current())) - time.Sleep(50 * time.Millisecond) + reconcileAndDrain() require.False(t, isConnected(), "supervisor re-dialed a server that gave up after max retries") + // An OAuth-classified failure is paced by the OAuth ladder, which bumps + // OAuthRetryCount and never RetryCount — without that gate it reads as + // "no failures yet" and is re-dialed on every tick forever (#1013). + setConnectionState(false, &types.ConnectionInfo{ + State: types.StateError, + IsOAuthError: true, + OAuthRetryCount: 2, + LastOAuthAttempt: time.Now().Add(-time.Minute), + }) + reconcileAndDrain() + require.False(t, isConnected(), "supervisor re-dialed a server inside its OAuth backoff window") + // A server parked in PendingAuth (waiting for user OAuth login) must not be // re-dialed - each attempt fires real requests at the upstream and cannot // succeed until the user completes the login. setConnectionState(false, &types.ConnectionInfo{ State: types.StatePendingAuth, }) - require.NoError(t, supervisor.reconcile(configSvc.Current())) - time.Sleep(50 * time.Millisecond) + reconcileAndDrain() require.False(t, isConnected(), "supervisor re-dialed a server pending OAuth login") // Once the backoff window has elapsed, reconciliation reconnects as before. @@ -1187,7 +1206,18 @@ func TestSupervisor_Reconcile_RespectsRetryBackoff(t *testing.T) { RetryCount: 3, LastRetryTime: time.Now().Add(-10 * time.Second), // backoff for 3 failures is 4s }) - require.NoError(t, supervisor.reconcile(configSvc.Current())) - time.Sleep(50 * time.Millisecond) + reconcileAndDrain() require.True(t, isConnected(), "supervisor did not reconnect after the backoff window elapsed") + + // A given-up server is still probed once per GaveUpProbeInterval, so an + // outage longer than the retry ladder (sleep, VPN, maintenance) self-heals + // instead of leaving the upstream silently dead until a human notices. + setConnectionState(false, &types.ConnectionInfo{ + State: types.StateError, + RetryCount: types.MaxConnectionRetries, + GaveUp: true, + LastRetryTime: time.Now().Add(-types.GaveUpProbeInterval - time.Minute), + }) + reconcileAndDrain() + require.True(t, isConnected(), "supervisor never probes a given-up server again") } diff --git a/internal/server/mcp.go b/internal/server/mcp.go index 34071f561..07c20b0b8 100644 --- a/internal/server/mcp.go +++ b/internal/server/mcp.go @@ -5723,6 +5723,11 @@ func (p *MCPProxyServer) monitorConnectionStatus(ctx context.Context, serverName return "ready", "Server connected and ready" case types.StateError: return "error", fmt.Sprintf("Server connection failed: %v", connectionInfo.LastError) + case types.StatePendingAuth: + // Parked awaiting user login (#1013): waiting out the monitor + // timeout would just stall the caller — nothing will change + // until a human signs in. + return statusError, fmt.Sprintf("Server connection failed: %v", connectionInfo.LastError) case types.StateDisconnected: // If server is explicitly disconnected and enabled is false, return disabled for _, serverConfig := range p.config.Servers { diff --git a/internal/upstream/core/connection_oauth.go b/internal/upstream/core/connection_oauth.go index 7e3122633..8f4819a1e 100644 --- a/internal/upstream/core/connection_oauth.go +++ b/internal/upstream/core/connection_oauth.go @@ -69,10 +69,17 @@ func (e *ErrOAuthPending) Code() diagnostics.Code { return diagnostics.OAuthLoginRequired } -// IsOAuthPending checks if an error is an ErrOAuthPending +// IsOAuthPending checks if an error is (or wraps) an ErrOAuthPending. +// +// It MUST unwrap: the pending error is raised inside an auth strategy and then +// wrapped twice on its way out — connectHTTP/connectSSE add "all authentication +// strategies failed, last error: %w" and Connect adds "failed to connect: %w". +// With a bare type assertion the check therefore never fired in production, and +// every login-blocked server fell through to the generic error path and was +// re-dialed instead of parked (#1013). func IsOAuthPending(err error) bool { - _, ok := err.(*ErrOAuthPending) - return ok + var pending *ErrOAuthPending + return errors.As(err, &pending) } // OAuthStartResult contains the result of initiating an OAuth flow. diff --git a/internal/upstream/core/oauth_error_test.go b/internal/upstream/core/oauth_error_test.go index 880a8832e..430bc875f 100644 --- a/internal/upstream/core/oauth_error_test.go +++ b/internal/upstream/core/oauth_error_test.go @@ -2,6 +2,7 @@ package core import ( "errors" + "fmt" "testing" "github.com/smart-mcp-proxy/mcpproxy-go/internal/diagnostics" @@ -217,12 +218,23 @@ func TestIsOAuthPending(t *testing.T) { expected: false, }, { - name: "wrapped ErrOAuthPending returns false", + name: "flattened-to-string ErrOAuthPending returns false", err: errors.New("wrapped: " + (&ErrOAuthPending{ ServerName: "slack", }).Error()), expected: false, }, + { + // The production chain: connectHTTP/connectSSE wrap the strategy's + // error and Connect wraps that again. A bare type assertion made the + // pending check unreachable, so login-blocked servers were re-dialed + // forever instead of parked (#1013). + name: "%w-wrapped ErrOAuthPending returns true", + err: fmt.Errorf("failed to connect: %w", + fmt.Errorf("all authentication strategies failed, last error: %w", + &ErrOAuthPending{ServerName: "slack", Message: "login available via Web UI"})), + expected: true, + }, } for _, tt := range tests { diff --git a/internal/upstream/managed/client.go b/internal/upstream/managed/client.go index 618fbb54d..64b66e058 100644 --- a/internal/upstream/managed/client.go +++ b/internal/upstream/managed/client.go @@ -274,7 +274,7 @@ func (mc *Client) Connect(ctx context.Context) error { // that died silently (e.g., HTTP server timeout). Without this, core client // rejects the connect attempt with "client already connected" error. currentState := mc.StateManager.GetState() - if currentState == types.StateError || currentState == types.StateDisconnected { + if currentState == types.StateError || currentState == types.StateDisconnected || currentState == types.StatePendingAuth { mc.logger.Debug("Disconnecting core client before reconnect to clear stale state", zap.String("server", mc.GetConfig().Name), zap.String("from_state", currentState.String())) @@ -318,9 +318,11 @@ func (mc *Client) Connect(ctx context.Context) error { if core.IsOAuthPending(connectErr) { mc.logger.Info("⏳ OAuth authentication pending user action", zap.String("server", mc.GetConfig().Name)) - // Transition to PendingAuth state instead of Error - mc.StateManager.TransitionTo(types.StatePendingAuth) - mc.StateManager.SetError(connectErr) + // Park in PendingAuth. SetPendingAuth (not TransitionTo + SetError: + // SetError forces StateError and would immediately undo the park, so + // the supervisor kept redialing a login-blocked server every 30s — + // #1013). + mc.StateManager.SetPendingAuth(connectErr) return fmt.Errorf("OAuth authentication pending: %w", connectErr) } // Check if this is an OAuth authorization requirement (not an error) diff --git a/internal/upstream/manager.go b/internal/upstream/manager.go index 875c1a109..dfdad177e 100644 --- a/internal/upstream/manager.go +++ b/internal/upstream/manager.go @@ -2,6 +2,8 @@ package upstream import ( "context" + "crypto/sha256" + "encoding/hex" "errors" "fmt" "maps" @@ -12,6 +14,7 @@ import ( "sync/atomic" "time" + uptransport "github.com/mark3labs/mcp-go/client/transport" "go.uber.org/zap" "github.com/smart-mcp-proxy/mcpproxy-go/internal/config" @@ -113,6 +116,12 @@ type Manager struct { // cannot write due to DB lock). Prevents rapid retrigger loops. tokenReconnect map[string]time.Time + // tokenFingerprints records which persisted token each server was last + // retried with, so the scan fires on a NEW token rather than on the mere + // presence of the stale one it already failed with (#1013). Same goroutine + // as tokenReconnect (the OAuth event monitor), so it needs no extra lock. + tokenFingerprints map[string]string + // Context for shutdown coordination shutdownCtx context.Context shutdownCancel context.CancelFunc @@ -185,17 +194,18 @@ func cloneServerConfig(cfg *config.ServerConfig) *config.ServerConfig { func NewManager(logger *zap.Logger, globalConfig *config.Config, boltStorage *storage.BoltDB, secretResolver *secret.Resolver, storageMgr *storage.Manager) *Manager { shutdownCtx, shutdownCancel := context.WithCancel(context.Background()) manager := &Manager{ - clients: make(map[string]*managed.Client), - logger: logger, - storage: boltStorage, - notificationMgr: NewNotificationManager(), - secretResolver: secretResolver, - tokenReconnect: make(map[string]time.Time), - lastSweptAt: make(map[string]time.Time), - shutdownCtx: shutdownCtx, - shutdownCancel: shutdownCancel, - storageMgr: storageMgr, - limiters: limiter.NewRegistry(), + clients: make(map[string]*managed.Client), + logger: logger, + storage: boltStorage, + notificationMgr: NewNotificationManager(), + secretResolver: secretResolver, + tokenReconnect: make(map[string]time.Time), + tokenFingerprints: make(map[string]string), + lastSweptAt: make(map[string]time.Time), + shutdownCtx: shutdownCtx, + shutdownCancel: shutdownCancel, + storageMgr: storageMgr, + limiters: limiter.NewRegistry(), } manager.globalConfig.Store(globalConfig) // Spec 093: publish the initial limit generation. With no limits configured @@ -1469,11 +1479,18 @@ func (m *Manager) ConnectAll(ctx context.Context) error { continue } - if client.GetState() == types.StateError && !client.ShouldRetry() { - info := client.GetConnectionInfo() + // Same retry policy the supervisor's reconcile honors: plain backoff, + // the coarse OAuth ladder, half-hourly probes after give-up, and no + // redial at all for a server parked awaiting login. ConnectAll runs on + // every config reload, so gating only on StateError+ShouldRetry (as it + // used to) let a reload re-dial servers their own client had paused + // (#1013). A user-driven config change for THIS server still reconnects + // it — the supervisor plans ActionReconnect, which bypasses the gate. + if info := client.GetConnectionInfo(); !info.ShouldAutoReconnect(time.Now()) { m.logger.Debug("Client backoff active, skipping connect attempt", zap.String("id", id), zap.String("name", client.GetConfig().Name), + zap.String("state", info.State.String()), zap.Int("retry_count", info.RetryCount), zap.Time("last_retry_time", info.LastRetryTime)) continue @@ -2156,8 +2173,10 @@ func (m *Manager) scanForNewTokens() { } state := c.GetState() - // Focus on Error state likely due to OAuth/authorization - if state != types.StateError { + // Focus on states that a freshly persisted token can unblock: a plain + // Error (likely OAuth/authorization) and PendingAuth, where the client is + // parked waiting for exactly this token (#1013). + if state != types.StateError && state != types.StatePendingAuth { continue } @@ -2173,16 +2192,56 @@ func (m *Manager) scanForNewTokens() { continue } - m.logger.Info("Detected persisted OAuth token; triggering reconnect", + // Only a token we have NOT already retried with is news. The failing + // server usually still has its (expired/revoked/rejected) token in the + // store, so triggering on mere presence redialed it every scan — a 5s + // loop that outpaces the supervisor's and defeats the PendingAuth park + // (#1013). The fingerprint changes exactly when a login/refresh writes a + // new token, which is the event this scan exists to catch. + // + // The write timestamp is part of the identity, not just the token + // content: a provider re-issuing a byte-identical token would otherwise + // leave the fingerprint unchanged and permanently suppress this wake — + // and for a CLI login that could not persist its completion event, this + // scan is the only one left. + var writtenAt time.Time + if rec, recErr := m.storage.GetOAuthToken(oauth.GenerateServerKey(cfg.Name, cfg.URL)); recErr == nil && rec != nil { + writtenAt = rec.Updated + } + fingerprint := tokenFingerprint(tok, writtenAt) + if seen, ok := m.tokenFingerprints[id]; ok && seen == fingerprint { + continue + } + + m.logger.Info("Detected new persisted OAuth token; triggering reconnect", zap.String("server", cfg.Name), zap.Time("token_expires_at", tok.ExpiresAt)) - // Remember trigger time and retry connection + // Remember trigger time + which token we tried, then retry connection m.tokenReconnect[id] = now + m.tokenFingerprints[id] = fingerprint _ = m.RetryConnection(cfg.Name) } } +// tokenFingerprint identifies a persisted OAuth token without retaining it: a +// truncated SHA-256 over the access token, refresh token, expiry and the time +// the record was last written, so any re-issue compares different while the +// same untouched token compares equal. Never log or persist the raw token to +// make this comparison. +func tokenFingerprint(tok *uptransport.Token, writtenAt time.Time) string { + if tok == nil { + return "" + } + sum := sha256.Sum256([]byte(strings.Join([]string{ + tok.AccessToken, + tok.RefreshToken, + tok.ExpiresAt.UTC().Format(time.RFC3339Nano), + writtenAt.UTC().Format(time.RFC3339Nano), + }, "\x00"))) + return hex.EncodeToString(sum[:8]) +} + // StartManualOAuth performs an in-process OAuth flow for the given server. // This avoids cross-process DB locking by using the daemon's storage directly. func (m *Manager) StartManualOAuth(serverName string, force bool) error { diff --git a/internal/upstream/manager_oauth_test.go b/internal/upstream/manager_oauth_test.go index c935574b9..c3456cb1f 100644 --- a/internal/upstream/manager_oauth_test.go +++ b/internal/upstream/manager_oauth_test.go @@ -1,9 +1,11 @@ package upstream import ( + "errors" "testing" "time" + uptransport "github.com/mark3labs/mcp-go/client/transport" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/zap" @@ -13,6 +15,7 @@ import ( "github.com/smart-mcp-proxy/mcpproxy-go/internal/secret" "github.com/smart-mcp-proxy/mcpproxy-go/internal/storage" "github.com/smart-mcp-proxy/mcpproxy-go/internal/upstream/managed" + "github.com/smart-mcp-proxy/mcpproxy-go/internal/upstream/types" ) // TestRefreshOAuthToken_DynamicOAuthDiscovery tests that RefreshOAuthToken works @@ -54,8 +57,8 @@ func TestRefreshOAuthToken_DynamicOAuthDiscovery(t *testing.T) { // Store an OAuth token for the server (as if it had authenticated previously) // The ServerName field is used as the storage key (must match GenerateServerKey output) token := &storage.OAuthTokenRecord{ - ServerName: serverKey, // Key used for storage lookup (hash-based) - DisplayName: "test-dynamic-oauth", // Human-readable name for RefreshManager + ServerName: serverKey, // Key used for storage lookup (hash-based) + DisplayName: "test-dynamic-oauth", // Human-readable name for RefreshManager AccessToken: "expired-access-token", RefreshToken: "valid-refresh-token", TokenType: "Bearer", @@ -176,3 +179,131 @@ func TestRefreshOAuthToken_ServerNotFound(t *testing.T) { require.Error(t, err) assert.Contains(t, err.Error(), "server not found") } + +// TestTokenFingerprint verifies the identity used to decide whether a persisted +// token is NEW: the same untouched token must compare equal (so the scan does +// not redial), any re-issue must not. +func TestTokenFingerprint(t *testing.T) { + expires := time.Now().Add(time.Hour).UTC() + written := time.Now().UTC() + base := &uptransport.Token{AccessToken: "at", RefreshToken: "rt", ExpiresAt: expires} + fp := tokenFingerprint(base, written) + + assert.Equal(t, fp, tokenFingerprint(&uptransport.Token{ + AccessToken: "at", RefreshToken: "rt", ExpiresAt: expires, + }, written), "same token, same write: must fingerprint the same") + + assert.NotEqual(t, fp, tokenFingerprint(&uptransport.Token{ + AccessToken: "at2", RefreshToken: "rt", ExpiresAt: expires, + }, written), "a new access token must fingerprint differently") + + assert.NotEqual(t, fp, tokenFingerprint(&uptransport.Token{ + AccessToken: "at", RefreshToken: "rt", ExpiresAt: expires.Add(time.Minute), + }, written), "a refreshed expiry must fingerprint differently") + + // A provider may re-issue a byte-identical token; the write stamp is what + // keeps that from silently suppressing the wake. + assert.NotEqual(t, fp, tokenFingerprint(base, written.Add(time.Second)), + "a token rewritten identically must still fingerprint differently") + + assert.NotContains(t, fp, "at", "the fingerprint must not carry the token") + assert.Empty(t, tokenFingerprint(nil, written)) +} + +// TestScanForNewTokens_OnlyOnNewToken pins the fix for the 5s redial loop: the +// scan must fire when a login/refresh writes a NEW token, not on every pass just +// because the server still holds the stale token it already failed with (#1013). +func TestScanForNewTokens_OnlyOnNewToken(t *testing.T) { + logger := zap.NewNop() + tempDir := t.TempDir() + db, err := storage.NewBoltDB(tempDir, logger.Sugar()) + require.NoError(t, err) + defer db.Close() + + serverConfig := &config.ServerConfig{ + Name: "parked-server", + URL: "http://127.0.0.1:1/mcp", // refused immediately; no real upstream needed + Protocol: "http", + Enabled: true, + Created: time.Now(), + } + serverKey := oauth.GenerateServerKey(serverConfig.Name, serverConfig.URL) + saveToken := func(access string) { + require.NoError(t, db.SaveOAuthToken(&storage.OAuthTokenRecord{ + ServerName: serverKey, + DisplayName: serverConfig.Name, + AccessToken: access, + TokenType: "Bearer", + ExpiresAt: time.Now().Add(time.Hour), + Created: time.Now(), + Updated: time.Now(), + })) + } + saveToken("stale-token") + + manager := &Manager{ + clients: make(map[string]*managed.Client), + logger: logger, + storage: db, + secretResolver: secret.NewResolver(), + tokenReconnect: make(map[string]time.Time), + tokenFingerprints: make(map[string]string), + } + + client, err := managed.NewClient("parked-server", serverConfig, logger, nil, &config.Config{}, db, secret.NewResolver()) + require.NoError(t, err) + manager.clients["parked-server"] = client + + // Park the client exactly as a deferred-OAuth connect failure would. + client.StateManager.SetPendingAuth(errors.New("OAuth authentication required for parked-server: login available via Web UI")) + + // A triggered scan calls RetryConnection, which reconnects in the background + // (the dial is refused immediately, but not synchronously). Wait for the + // client to settle back into a state the scan considers before the next leg, + // otherwise a leg can land while it is still Connecting and be skipped for a + // reason the test is not about. + settled := func() { + t.Helper() + require.Eventually(t, func() bool { + st := client.GetState() + return st == types.StatePendingAuth || st == types.StateError + }, 30*time.Second, 10*time.Millisecond, "client never settled into a scannable state") + } + // Pretend the per-server rate limit has expired so it cannot mask the result. + expireRateLimit := func() { + t.Helper() + settled() + manager.tokenReconnect["parked-server"] = time.Now().Add(-time.Minute) + } + + expireRateLimit() + manager.scanForNewTokens() + require.WithinDuration(t, time.Now(), manager.tokenReconnect["parked-server"], time.Second, + "first scan must retry with the stored token") + require.NotEmpty(t, manager.tokenFingerprints["parked-server"]) + + // Same token, rate limit expired: must NOT redial again. + expireRateLimit() + before := manager.tokenReconnect["parked-server"] + manager.scanForNewTokens() + require.Equal(t, before, manager.tokenReconnect["parked-server"], + "scan redialed a parked server for a token it had already tried") + + // A fresh token (login completed) must wake it. + saveToken("fresh-token") + expireRateLimit() + manager.scanForNewTokens() + require.WithinDuration(t, time.Now(), manager.tokenReconnect["parked-server"], time.Second, + "scan did not wake the parked server when a new token appeared") + + // A re-login that happens to persist a byte-identical token is still a new + // write, and this scan is the fallback wake when the CLI could not record an + // OAuth completion event — it must not be swallowed. SaveOAuthToken stamps + // Updated with the wall clock, so give it a distinct instant. + time.Sleep(2 * time.Millisecond) + saveToken("fresh-token") + expireRateLimit() + manager.scanForNewTokens() + require.WithinDuration(t, time.Now(), manager.tokenReconnect["parked-server"], time.Second, + "scan ignored an identically-rewritten token") +} diff --git a/internal/upstream/notifications.go b/internal/upstream/notifications.go index 18862166d..7c7090ef7 100644 --- a/internal/upstream/notifications.go +++ b/internal/upstream/notifications.go @@ -162,6 +162,12 @@ func StateChangeNotifier(nm *NotificationManager, serverName string) func(oldSta if oldState == types.StateConnecting { nm.NotifyOAuthRequired(serverName) } + case types.StatePendingAuth: + // Parked awaiting user login: the connection attempt is over and only a + // human can move it forward, so surface the login prompt once. + if oldState != types.StatePendingAuth { + nm.NotifyOAuthRequired(serverName) + } case types.StateDisconnected: if oldState == types.StateReady { nm.NotifyServerDisconnected(serverName, info.LastError) diff --git a/internal/upstream/types/types.go b/internal/upstream/types/types.go index 1ec05f3fd..faf8a0b40 100644 --- a/internal/upstream/types/types.go +++ b/internal/upstream/types/types.go @@ -86,13 +86,40 @@ func RetryBackoffDuration(retryCount int) time.Duration { return backoffDuration } +// OAuthRetryBackoffDuration returns the (much longer) backoff to wait after the +// given number of consecutive OAuth failures: 5min, 15min, 1h, 4h, then 24h. +// An OAuth failure cannot be resolved by redialing quickly — it needs a token +// refresh or a human completing a login — so its ladder is deliberately coarse. +func OAuthRetryBackoffDuration(oauthRetryCount int) time.Duration { + switch { + case oauthRetryCount <= 1: + return 5 * time.Minute + case oauthRetryCount <= 2: + return 15 * time.Minute + case oauthRetryCount <= 3: + return 1 * time.Hour + case oauthRetryCount <= 4: + return 4 * time.Hour + default: + return 24 * time.Hour // Max backoff for OAuth: 24 hours + } +} + +// GaveUpProbeInterval is how often a given-up server is still probed by the +// periodic reconciliation. After MaxConnectionRetries consecutive failures the +// client stops its own retry ladder, but "never again" would leave an upstream +// silently dead after any outage longer than the ladder (laptop sleep, VPN drop, +// overnight maintenance) until a human notices and reconnects by hand — a failed +// upstream is invisible to the operator (#1013). One probe every 30 minutes keeps +// that self-healing at ~48 requests/day instead of ~2880. +const GaveUpProbeInterval = 30 * time.Minute + // ShouldAutoReconnect reports whether an automatic (supervisor-driven) reconnect // attempt is appropriate given the connection's failure history. It returns false -// while the exponential backoff window from the last failure has not elapsed, -// after the client has given up (MaxConnectionRetries), and for servers parked in -// PendingAuth — redialing cannot succeed until the user completes the OAuth login, -// and each attempt costs real requests against the upstream. Manual reconnects, -// login flows, and reconnect-on-use are not subject to this policy. +// while a backoff window from the last failure has not elapsed and for servers +// parked in PendingAuth — redialing cannot succeed until the user completes the +// OAuth login, and each attempt costs real requests against the upstream. Manual +// reconnects, login flows, and reconnect-on-use are not subject to this policy. func (ci *ConnectionInfo) ShouldAutoReconnect(now time.Time) bool { if ci == nil { return true @@ -101,9 +128,17 @@ func (ci *ConnectionInfo) ShouldAutoReconnect(now time.Time) bool { case StatePendingAuth: return false case StateError: - if ci.GaveUp || ci.RetryCount >= MaxConnectionRetries { + // OAuth-classified failures are paced by SetOAuthError's coarse ladder, + // which bumps OAuthRetryCount and NOT RetryCount. Without this branch such + // a server reads as RetryCount==0 ("no failures yet") and is re-dialed on + // every 30s reconcile tick forever — exactly the storm this gate exists to + // stop, for the error class (#1013) that triggered it. + if ci.IsOAuthError && !ci.oauthBackoffElapsed(now) { return false } + if ci.GaveUp || ci.RetryCount >= MaxConnectionRetries { + return now.Sub(ci.LastRetryTime) >= GaveUpProbeInterval + } if ci.RetryCount == 0 { return true } @@ -113,6 +148,14 @@ func (ci *ConnectionInfo) ShouldAutoReconnect(now time.Time) bool { } } +// oauthBackoffElapsed reports whether the OAuth ladder allows another attempt. +func (ci *ConnectionInfo) oauthBackoffElapsed(now time.Time) bool { + if ci.OAuthRetryCount == 0 { + return true + } + return now.Sub(ci.LastOAuthAttempt) >= OAuthRetryBackoffDuration(ci.OAuthRetryCount) +} + // StateManager manages the state transitions for an upstream connection type StateManager struct { mu sync.RWMutex @@ -253,6 +296,53 @@ func (sm *StateManager) SetError(err error) { } } +// SetPendingAuth parks the connection in StatePendingAuth with the deferred-OAuth +// error attached. It exists because the obvious spelling — TransitionTo(StatePendingAuth) +// followed by SetError(err) — silently undoes itself: SetError forces StateError +// unconditionally, so the PendingAuth state survived microseconds and every consumer +// (health, notifications, the supervisor's reconnect gate) saw a plain error and +// kept redialing a server that cannot connect until a human logs in (#1013). +// +// RetryCount is deliberately NOT bumped: a parked server is not retrying, so there +// is no ladder to advance and nothing to "give up" on. The wake paths are explicit +// user action (login, manual reconnect, reconnect-on-use) and the persisted-token +// scan in Manager.scanForNewTokens. +func (sm *StateManager) SetPendingAuth(err error) { + sm.mu.Lock() + + oldState := sm.currentState + sm.currentState = StatePendingAuth + sm.lastError = err + sm.lastRetryTime = time.Now() + + info := ConnectionInfo{ + State: sm.currentState, + LastError: sm.lastError, + RetryCount: sm.retryCount, + LastRetryTime: sm.lastRetryTime, + ServerName: sm.serverName, + ServerVersion: sm.serverVersion, + LastOAuthAttempt: sm.lastOAuthAttempt, + OAuthRetryCount: sm.oauthRetryCount, + IsOAuthError: sm.isOAuthError, + GaveUp: sm.retryCount >= MaxConnectionRetries, + } + + callback := sm.onStateChange + sm.mu.Unlock() + + // Dispatched synchronously, like TransitionTo (which delivered this very + // state before) and unlike SetError: a detached goroutine can be scheduled + // after a subsequent Connecting/Ready callback, which would surface a stale + // "sign in required" prompt right after a successful login. The registered + // consumers only log and hand off to notification handlers that are + // themselves goroutine-dispatched, and they never take the managed client's + // mutex — the same reason TransitionTo can already run under it. + if callback != nil { + callback(oldState, StatePendingAuth, &info) + } +} + // SetServerInfo sets the server information func (sm *StateManager) SetServerInfo(name, version string) { sm.mu.Lock() @@ -326,11 +416,15 @@ func (sm *StateManager) ValidateTransition(from, to ConnectionState) error { // Define valid transitions validTransitions := map[ConnectionState][]ConnectionState{ StateDisconnected: {StateConnecting}, - StateConnecting: {StateAuthenticating, StateDiscovering, StateReady, StateError, StateDisconnected}, // Allow direct to Ready for OAuth flows - StateAuthenticating: {StateConnecting, StateDiscovering, StateReady, StateError, StateDisconnected}, + StateConnecting: {StateAuthenticating, StateDiscovering, StateReady, StateError, StateDisconnected, StatePendingAuth}, // Allow direct to Ready for OAuth flows + StateAuthenticating: {StateConnecting, StateDiscovering, StateReady, StateError, StateDisconnected, StatePendingAuth}, StateDiscovering: {StateReady, StateError, StateDisconnected}, StateReady: {StateError, StateDisconnected}, StateError: {StateConnecting, StateDisconnected}, + // A parked server leaves PendingAuth when the user logs in / reconnects + // (StateConnecting), is disabled (StateDisconnected), or a later attempt + // fails for a non-auth reason (StateError). + StatePendingAuth: {StateConnecting, StateDisconnected, StateError}, } allowed, exists := validTransitions[from] @@ -468,21 +562,7 @@ func (sm *StateManager) ShouldRetryOAuth() bool { } // OAuth has much longer backoff intervals: 5min, 15min, 1h, 4h, 24h - var backoffDuration time.Duration - switch { - case sm.oauthRetryCount <= 1: - backoffDuration = 5 * time.Minute - case sm.oauthRetryCount <= 2: - backoffDuration = 15 * time.Minute - case sm.oauthRetryCount <= 3: - backoffDuration = 1 * time.Hour - case sm.oauthRetryCount <= 4: - backoffDuration = 4 * time.Hour - default: - backoffDuration = 24 * time.Hour // Max backoff for OAuth: 24 hours - } - - return time.Since(sm.lastOAuthAttempt) >= backoffDuration + return time.Since(sm.lastOAuthAttempt) >= OAuthRetryBackoffDuration(sm.oauthRetryCount) } // IsOAuthError returns true if the last error was OAuth-related diff --git a/internal/upstream/types/types_test.go b/internal/upstream/types/types_test.go index ff9737b9c..fd11d0339 100644 --- a/internal/upstream/types/types_test.go +++ b/internal/upstream/types/types_test.go @@ -220,8 +220,20 @@ func TestConnectionInfo_ShouldAutoReconnect(t *testing.T) { {"error within backoff window", &ConnectionInfo{State: StateError, RetryCount: 5, LastRetryTime: now.Add(-1 * time.Second)}, false}, {"error with backoff elapsed", &ConnectionInfo{State: StateError, RetryCount: 3, LastRetryTime: now.Add(-10 * time.Second)}, true}, {"error no failures yet", &ConnectionInfo{State: StateError, RetryCount: 0}, true}, - {"gave up flag", &ConnectionInfo{State: StateError, GaveUp: true, LastRetryTime: now.Add(-time.Hour)}, false}, - {"retry count at max", &ConnectionInfo{State: StateError, RetryCount: MaxConnectionRetries, LastRetryTime: now.Add(-time.Hour)}, false}, + {"gave up flag", &ConnectionInfo{State: StateError, GaveUp: true, LastRetryTime: now.Add(-time.Minute)}, false}, + {"retry count at max", &ConnectionInfo{State: StateError, RetryCount: MaxConnectionRetries, LastRetryTime: now.Add(-time.Minute)}, false}, + // Given up, but the probe interval has elapsed: one attempt is allowed so a + // long outage (sleep, VPN, maintenance) still self-heals without a human. + {"gave up, probe interval elapsed", &ConnectionInfo{State: StateError, GaveUp: true, LastRetryTime: now.Add(-GaveUpProbeInterval - time.Second)}, true}, + {"retry count at max, probe interval elapsed", &ConnectionInfo{State: StateError, RetryCount: MaxConnectionRetries, LastRetryTime: now.Add(-time.Hour)}, true}, + // OAuth failures are paced by the OAuth ladder, not RetryCount (which + // SetOAuthError never bumps) — the 30s storm's remaining hole (#1013). + {"oauth error, ladder window open", &ConnectionInfo{State: StateError, IsOAuthError: true, OAuthRetryCount: 1, LastOAuthAttempt: now.Add(-time.Minute)}, false}, + {"oauth error, ladder elapsed", &ConnectionInfo{State: StateError, IsOAuthError: true, OAuthRetryCount: 1, LastOAuthAttempt: now.Add(-6 * time.Minute)}, true}, + {"oauth error, long ladder still open", &ConnectionInfo{State: StateError, IsOAuthError: true, OAuthRetryCount: 5, LastOAuthAttempt: now.Add(-5 * time.Hour)}, false}, + {"oauth error, no oauth attempt recorded", &ConnectionInfo{State: StateError, IsOAuthError: true}, true}, + // Both ladders apply to an OAuth failure that also bumped RetryCount. + {"oauth ladder elapsed but plain backoff open", &ConnectionInfo{State: StateError, IsOAuthError: true, OAuthRetryCount: 1, LastOAuthAttempt: now.Add(-time.Hour), RetryCount: 8, LastRetryTime: now}, false}, } for _, tt := range tests { @@ -230,3 +242,58 @@ func TestConnectionInfo_ShouldAutoReconnect(t *testing.T) { }) } } + +// TestOAuthRetryBackoffDuration pins the coarse OAuth ladder shared by +// StateManager.ShouldRetryOAuth and the supervisor's reconnect gate. +func TestOAuthRetryBackoffDuration(t *testing.T) { + tests := []struct { + oauthRetryCount int + expected time.Duration + }{ + {0, 5 * time.Minute}, + {1, 5 * time.Minute}, + {2, 15 * time.Minute}, + {3, 1 * time.Hour}, + {4, 4 * time.Hour}, + {5, 24 * time.Hour}, + {50, 24 * time.Hour}, + } + + for _, tt := range tests { + assert.Equal(t, tt.expected, OAuthRetryBackoffDuration(tt.oauthRetryCount), "oauthRetryCount=%d", tt.oauthRetryCount) + } +} + +// TestSetPendingAuth verifies that parking a connection in PendingAuth sticks: +// the pre-existing spelling (TransitionTo + SetError) silently forced StateError, +// so every consumer saw a plain error and kept redialing (#1013). +func TestSetPendingAuth(t *testing.T) { + sm := NewStateManager() + sm.TransitionTo(StateConnecting) + + stateChanges := make(chan ConnectionState, 4) + sm.SetStateChangeCallback(func(_, newState ConnectionState, _ *ConnectionInfo) { + stateChanges <- newState + }) + + pendingErr := errors.New("OAuth authentication required for test-server: login available via Web UI") + sm.SetPendingAuth(pendingErr) + + info := sm.GetConnectionInfo() + assert.Equal(t, StatePendingAuth, info.State, "PendingAuth must survive - it is the parked state") + assert.Equal(t, pendingErr, info.LastError, "the deferred-OAuth error must stay attached") + assert.Equal(t, 0, info.RetryCount, "a parked server is not retrying, so no ladder is advanced") + assert.False(t, info.ShouldAutoReconnect(time.Now()), "a parked server must not be auto-redialed") + assert.False(t, sm.ShouldRetry(), "ConnectAll must not redial a parked server either") + + select { + case got := <-stateChanges: + assert.Equal(t, StatePendingAuth, got, "consumers must be told about the park") + case <-time.After(2 * time.Second): + t.Fatal("no state-change callback fired for SetPendingAuth") + } + + // The wake path (user login / manual reconnect) must be a legal transition. + assert.NoError(t, sm.ValidateTransition(StatePendingAuth, StateConnecting)) + assert.NoError(t, sm.ValidateTransition(StateConnecting, StatePendingAuth)) +}