Skip to content

Commit bcf6b77

Browse files
committed
fix(upstream): make the PendingAuth park reachable and stop the token-scan redial loop
Cross-model review of the previous commit surfaced four defects, three of them in code it newly depended on: - `IsOAuthPending` type-asserted instead of unwrapping, but the deferred-OAuth error is wrapped twice on its way out of Connect ("all authentication strategies failed, last error: %w" then "failed to connect: %w"). The check never fired in production, so the park — and every consumer keyed to it — was unreachable. Now `errors.As`, with the production wrapping shape pinned in a test. - `Manager.scanForNewTokens` triggered a reconnect whenever *any* persisted token existed. A failing server usually still holds the expired/revoked token it just failed with, so the 5s scan redialed it every pass (rate-limited only to 10s/server) — a tighter loop than the supervisor's and one that walks straight through the park. It now records a fingerprint of the token it retried with and fires only when a login or refresh writes a different one. - `ConnectAll` gated on `StateError && !ShouldRetry()`, so a config reload redialed parked and OAuth-backed-off servers. It now consults the same `ShouldAutoReconnect` policy as the supervisor. A config change for the server itself still reconnects it via ActionReconnect. - `SetPendingAuth` dispatched its state-change callback detached, which can be scheduled after a later Connecting/Ready callback and surface a stale "sign in required" prompt right after a successful login. It is synchronous now, matching the TransitionTo that used to deliver this state. Also replaces the reconcile test's fixed sleeps with an actionWg barrier so the negative assertions cannot pass before an erroneous redial would have run. Refs #1013, #1014
1 parent 0ad8d1d commit bcf6b77

7 files changed

Lines changed: 206 additions & 41 deletions

File tree

‎internal/runtime/supervisor/supervisor.go‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -490,10 +490,11 @@ func (s *Supervisor) computeReconcilePlan(configSnapshot *configsvc.Snapshot, ac
490490
plan.Actions[name] = ActionNone
491491
} else if actual, ok := actualStates[name]; ok && !actual.ConnectionInfo.ShouldAutoReconnect(time.Now()) {
492492
// Respect the client's retry policy: exponential backoff after
493-
// consecutive failures, gave-up after MaxConnectionRetries, and
494-
// PendingAuth servers waiting on user OAuth login. Without this
495-
// gate the periodic 30s reconciliation re-dials a dead upstream
496-
// forever, hammering the remote server (~3 requests per tick).
493+
// consecutive failures, the coarse OAuth ladder, half-hourly
494+
// probes once the client gave up, and PendingAuth servers
495+
// parked waiting on user OAuth login. Without this gate the
496+
// periodic 30s reconciliation re-dials a dead upstream forever,
497+
// hammering the remote server (~3 requests per tick).
497498
s.logger.Debug("Skipping auto-reconnect (backoff/pending-auth)",
498499
zap.String("server", name),
499500
zap.String("state", actual.ConnectionInfo.State.String()),

‎internal/runtime/supervisor/supervisor_test.go‎

Lines changed: 16 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1132,7 +1132,7 @@ func TestSupervisor_Reconcile_RespectsRetryBackoff(t *testing.T) {
11321132

11331133
// First reconciliation - server is added and connected
11341134
require.NoError(t, supervisor.reconcile(configSvc.Current()))
1135-
time.Sleep(50 * time.Millisecond)
1135+
supervisor.actionWg.Wait()
11361136

11371137
setConnectionState := func(connected bool, info *types.ConnectionInfo) {
11381138
mockUpstream.mu.Lock()
@@ -1148,6 +1148,15 @@ func TestSupervisor_Reconcile_RespectsRetryBackoff(t *testing.T) {
11481148
defer mockUpstream.mu.Unlock()
11491149
return mockUpstream.connected["flaky-server"]
11501150
}
1151+
// reconcile dispatches its actions into goroutines tracked by actionWg, so
1152+
// draining that group is an exact barrier: after it returns, either the
1153+
// connect ran or none was planned. A fixed sleep would let the negative
1154+
// assertions below pass before an erroneous redial had a chance to execute.
1155+
reconcileAndDrain := func() {
1156+
t.Helper()
1157+
require.NoError(t, supervisor.reconcile(configSvc.Current()))
1158+
supervisor.actionWg.Wait()
1159+
}
11511160

11521161
// Simulate a connection failure with the backoff window still open:
11531162
// reconciliation must NOT re-dial.
@@ -1156,8 +1165,7 @@ func TestSupervisor_Reconcile_RespectsRetryBackoff(t *testing.T) {
11561165
RetryCount: 5,
11571166
LastRetryTime: time.Now(),
11581167
})
1159-
require.NoError(t, supervisor.reconcile(configSvc.Current()))
1160-
time.Sleep(50 * time.Millisecond)
1168+
reconcileAndDrain()
11611169
require.False(t, isConnected(), "supervisor re-dialed a failed server inside its backoff window")
11621170

11631171
// A server that gave up after max retries must not be re-dialed either,
@@ -1168,8 +1176,7 @@ func TestSupervisor_Reconcile_RespectsRetryBackoff(t *testing.T) {
11681176
GaveUp: true,
11691177
LastRetryTime: time.Now().Add(-time.Minute),
11701178
})
1171-
require.NoError(t, supervisor.reconcile(configSvc.Current()))
1172-
time.Sleep(50 * time.Millisecond)
1179+
reconcileAndDrain()
11731180
require.False(t, isConnected(), "supervisor re-dialed a server that gave up after max retries")
11741181

11751182
// An OAuth-classified failure is paced by the OAuth ladder, which bumps
@@ -1181,8 +1188,7 @@ func TestSupervisor_Reconcile_RespectsRetryBackoff(t *testing.T) {
11811188
OAuthRetryCount: 2,
11821189
LastOAuthAttempt: time.Now().Add(-time.Minute),
11831190
})
1184-
require.NoError(t, supervisor.reconcile(configSvc.Current()))
1185-
time.Sleep(50 * time.Millisecond)
1191+
reconcileAndDrain()
11861192
require.False(t, isConnected(), "supervisor re-dialed a server inside its OAuth backoff window")
11871193

11881194
// A server parked in PendingAuth (waiting for user OAuth login) must not be
@@ -1191,8 +1197,7 @@ func TestSupervisor_Reconcile_RespectsRetryBackoff(t *testing.T) {
11911197
setConnectionState(false, &types.ConnectionInfo{
11921198
State: types.StatePendingAuth,
11931199
})
1194-
require.NoError(t, supervisor.reconcile(configSvc.Current()))
1195-
time.Sleep(50 * time.Millisecond)
1200+
reconcileAndDrain()
11961201
require.False(t, isConnected(), "supervisor re-dialed a server pending OAuth login")
11971202

11981203
// Once the backoff window has elapsed, reconciliation reconnects as before.
@@ -1201,8 +1206,7 @@ func TestSupervisor_Reconcile_RespectsRetryBackoff(t *testing.T) {
12011206
RetryCount: 3,
12021207
LastRetryTime: time.Now().Add(-10 * time.Second), // backoff for 3 failures is 4s
12031208
})
1204-
require.NoError(t, supervisor.reconcile(configSvc.Current()))
1205-
time.Sleep(50 * time.Millisecond)
1209+
reconcileAndDrain()
12061210
require.True(t, isConnected(), "supervisor did not reconnect after the backoff window elapsed")
12071211

12081212
// A given-up server is still probed once per GaveUpProbeInterval, so an
@@ -1214,7 +1218,6 @@ func TestSupervisor_Reconcile_RespectsRetryBackoff(t *testing.T) {
12141218
GaveUp: true,
12151219
LastRetryTime: time.Now().Add(-types.GaveUpProbeInterval - time.Minute),
12161220
})
1217-
require.NoError(t, supervisor.reconcile(configSvc.Current()))
1218-
time.Sleep(50 * time.Millisecond)
1221+
reconcileAndDrain()
12191222
require.True(t, isConnected(), "supervisor never probes a given-up server again")
12201223
}

‎internal/upstream/core/connection_oauth.go‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -69,10 +69,17 @@ func (e *ErrOAuthPending) Code() diagnostics.Code {
6969
return diagnostics.OAuthLoginRequired
7070
}
7171

72-
// IsOAuthPending checks if an error is an ErrOAuthPending
72+
// IsOAuthPending checks if an error is (or wraps) an ErrOAuthPending.
73+
//
74+
// It MUST unwrap: the pending error is raised inside an auth strategy and then
75+
// wrapped twice on its way out — connectHTTP/connectSSE add "all authentication
76+
// strategies failed, last error: %w" and Connect adds "failed to connect: %w".
77+
// With a bare type assertion the check therefore never fired in production, and
78+
// every login-blocked server fell through to the generic error path and was
79+
// re-dialed instead of parked (#1013).
7380
func IsOAuthPending(err error) bool {
74-
_, ok := err.(*ErrOAuthPending)
75-
return ok
81+
var pending *ErrOAuthPending
82+
return errors.As(err, &pending)
7683
}
7784

7885
// OAuthStartResult contains the result of initiating an OAuth flow.

‎internal/upstream/core/oauth_error_test.go‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package core
22

33
import (
44
"errors"
5+
"fmt"
56
"testing"
67

78
"github.com/smart-mcp-proxy/mcpproxy-go/internal/diagnostics"
@@ -217,12 +218,23 @@ func TestIsOAuthPending(t *testing.T) {
217218
expected: false,
218219
},
219220
{
220-
name: "wrapped ErrOAuthPending returns false",
221+
name: "flattened-to-string ErrOAuthPending returns false",
221222
err: errors.New("wrapped: " + (&ErrOAuthPending{
222223
ServerName: "slack",
223224
}).Error()),
224225
expected: false,
225226
},
227+
{
228+
// The production chain: connectHTTP/connectSSE wrap the strategy's
229+
// error and Connect wraps that again. A bare type assertion made the
230+
// pending check unreachable, so login-blocked servers were re-dialed
231+
// forever instead of parked (#1013).
232+
name: "%w-wrapped ErrOAuthPending returns true",
233+
err: fmt.Errorf("failed to connect: %w",
234+
fmt.Errorf("all authentication strategies failed, last error: %w",
235+
&ErrOAuthPending{ServerName: "slack", Message: "login available via Web UI"})),
236+
expected: true,
237+
},
226238
}
227239

228240
for _, tt := range tests {

‎internal/upstream/manager.go‎

Lines changed: 56 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ package upstream
22

33
import (
44
"context"
5+
"crypto/sha256"
6+
"encoding/hex"
57
"errors"
68
"fmt"
79
"maps"
@@ -12,6 +14,7 @@ import (
1214
"sync/atomic"
1315
"time"
1416

17+
uptransport "github.com/mark3labs/mcp-go/client/transport"
1518
"go.uber.org/zap"
1619

1720
"github.com/smart-mcp-proxy/mcpproxy-go/internal/config"
@@ -113,6 +116,12 @@ type Manager struct {
113116
// cannot write due to DB lock). Prevents rapid retrigger loops.
114117
tokenReconnect map[string]time.Time
115118

119+
// tokenFingerprints records which persisted token each server was last
120+
// retried with, so the scan fires on a NEW token rather than on the mere
121+
// presence of the stale one it already failed with (#1013). Same goroutine
122+
// as tokenReconnect (the OAuth event monitor), so it needs no extra lock.
123+
tokenFingerprints map[string]string
124+
116125
// Context for shutdown coordination
117126
shutdownCtx context.Context
118127
shutdownCancel context.CancelFunc
@@ -185,17 +194,18 @@ func cloneServerConfig(cfg *config.ServerConfig) *config.ServerConfig {
185194
func NewManager(logger *zap.Logger, globalConfig *config.Config, boltStorage *storage.BoltDB, secretResolver *secret.Resolver, storageMgr *storage.Manager) *Manager {
186195
shutdownCtx, shutdownCancel := context.WithCancel(context.Background())
187196
manager := &Manager{
188-
clients: make(map[string]*managed.Client),
189-
logger: logger,
190-
storage: boltStorage,
191-
notificationMgr: NewNotificationManager(),
192-
secretResolver: secretResolver,
193-
tokenReconnect: make(map[string]time.Time),
194-
lastSweptAt: make(map[string]time.Time),
195-
shutdownCtx: shutdownCtx,
196-
shutdownCancel: shutdownCancel,
197-
storageMgr: storageMgr,
198-
limiters: limiter.NewRegistry(),
197+
clients: make(map[string]*managed.Client),
198+
logger: logger,
199+
storage: boltStorage,
200+
notificationMgr: NewNotificationManager(),
201+
secretResolver: secretResolver,
202+
tokenReconnect: make(map[string]time.Time),
203+
tokenFingerprints: make(map[string]string),
204+
lastSweptAt: make(map[string]time.Time),
205+
shutdownCtx: shutdownCtx,
206+
shutdownCancel: shutdownCancel,
207+
storageMgr: storageMgr,
208+
limiters: limiter.NewRegistry(),
199209
}
200210
manager.globalConfig.Store(globalConfig)
201211
// Spec 093: publish the initial limit generation. With no limits configured
@@ -1469,11 +1479,18 @@ func (m *Manager) ConnectAll(ctx context.Context) error {
14691479
continue
14701480
}
14711481

1472-
if client.GetState() == types.StateError && !client.ShouldRetry() {
1473-
info := client.GetConnectionInfo()
1482+
// Same retry policy the supervisor's reconcile honors: plain backoff,
1483+
// the coarse OAuth ladder, half-hourly probes after give-up, and no
1484+
// redial at all for a server parked awaiting login. ConnectAll runs on
1485+
// every config reload, so gating only on StateError+ShouldRetry (as it
1486+
// used to) let a reload re-dial servers their own client had paused
1487+
// (#1013). A user-driven config change for THIS server still reconnects
1488+
// it — the supervisor plans ActionReconnect, which bypasses the gate.
1489+
if info := client.GetConnectionInfo(); !info.ShouldAutoReconnect(time.Now()) {
14741490
m.logger.Debug("Client backoff active, skipping connect attempt",
14751491
zap.String("id", id),
14761492
zap.String("name", client.GetConfig().Name),
1493+
zap.String("state", info.State.String()),
14771494
zap.Int("retry_count", info.RetryCount),
14781495
zap.Time("last_retry_time", info.LastRetryTime))
14791496
continue
@@ -2175,16 +2192,40 @@ func (m *Manager) scanForNewTokens() {
21752192
continue
21762193
}
21772194

2178-
m.logger.Info("Detected persisted OAuth token; triggering reconnect",
2195+
// Only a token we have NOT already retried with is news. The failing
2196+
// server usually still has its (expired/revoked/rejected) token in the
2197+
// store, so triggering on mere presence redialed it every scan — a 5s
2198+
// loop that outpaces the supervisor's and defeats the PendingAuth park
2199+
// (#1013). The fingerprint changes exactly when a login/refresh writes a
2200+
// new token, which is the event this scan exists to catch.
2201+
fingerprint := tokenFingerprint(tok)
2202+
if seen, ok := m.tokenFingerprints[id]; ok && seen == fingerprint {
2203+
continue
2204+
}
2205+
2206+
m.logger.Info("Detected new persisted OAuth token; triggering reconnect",
21792207
zap.String("server", cfg.Name),
21802208
zap.Time("token_expires_at", tok.ExpiresAt))
21812209

2182-
// Remember trigger time and retry connection
2210+
// Remember trigger time + which token we tried, then retry connection
21832211
m.tokenReconnect[id] = now
2212+
m.tokenFingerprints[id] = fingerprint
21842213
_ = m.RetryConnection(cfg.Name)
21852214
}
21862215
}
21872216

2217+
// tokenFingerprint identifies a persisted OAuth token without retaining it: a
2218+
// truncated SHA-256 over the access token, refresh token and expiry, so a
2219+
// refreshed or re-issued token compares different while the same stale token
2220+
// compares equal. Never log or persist the raw token to make this comparison.
2221+
func tokenFingerprint(tok *uptransport.Token) string {
2222+
if tok == nil {
2223+
return ""
2224+
}
2225+
sum := sha256.Sum256([]byte(tok.AccessToken + "\x00" + tok.RefreshToken + "\x00" + tok.ExpiresAt.UTC().Format(time.RFC3339Nano)))
2226+
return hex.EncodeToString(sum[:8])
2227+
}
2228+
21882229
// StartManualOAuth performs an in-process OAuth flow for the given server.
21892230
// This avoids cross-process DB locking by using the daemon's storage directly.
21902231
func (m *Manager) StartManualOAuth(serverName string, force bool) error {

‎internal/upstream/manager_oauth_test.go‎

Lines changed: 98 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
package upstream
22

33
import (
4+
"errors"
45
"testing"
56
"time"
67

8+
uptransport "github.com/mark3labs/mcp-go/client/transport"
79
"github.com/stretchr/testify/assert"
810
"github.com/stretchr/testify/require"
911
"go.uber.org/zap"
@@ -54,8 +56,8 @@ func TestRefreshOAuthToken_DynamicOAuthDiscovery(t *testing.T) {
5456
// Store an OAuth token for the server (as if it had authenticated previously)
5557
// The ServerName field is used as the storage key (must match GenerateServerKey output)
5658
token := &storage.OAuthTokenRecord{
57-
ServerName: serverKey, // Key used for storage lookup (hash-based)
58-
DisplayName: "test-dynamic-oauth", // Human-readable name for RefreshManager
59+
ServerName: serverKey, // Key used for storage lookup (hash-based)
60+
DisplayName: "test-dynamic-oauth", // Human-readable name for RefreshManager
5961
AccessToken: "expired-access-token",
6062
RefreshToken: "valid-refresh-token",
6163
TokenType: "Bearer",
@@ -176,3 +178,97 @@ func TestRefreshOAuthToken_ServerNotFound(t *testing.T) {
176178
require.Error(t, err)
177179
assert.Contains(t, err.Error(), "server not found")
178180
}
181+
182+
// TestTokenFingerprint verifies the identity used to decide whether a persisted
183+
// token is NEW: the same token must compare equal (so the scan does not redial),
184+
// a refreshed/re-issued one must not.
185+
func TestTokenFingerprint(t *testing.T) {
186+
expires := time.Now().Add(time.Hour).UTC()
187+
base := &uptransport.Token{AccessToken: "at", RefreshToken: "rt", ExpiresAt: expires}
188+
189+
assert.Equal(t, tokenFingerprint(base), tokenFingerprint(&uptransport.Token{
190+
AccessToken: "at", RefreshToken: "rt", ExpiresAt: expires,
191+
}), "same token must fingerprint the same")
192+
193+
assert.NotEqual(t, tokenFingerprint(base), tokenFingerprint(&uptransport.Token{
194+
AccessToken: "at2", RefreshToken: "rt", ExpiresAt: expires,
195+
}), "a new access token must fingerprint differently")
196+
197+
assert.NotEqual(t, tokenFingerprint(base), tokenFingerprint(&uptransport.Token{
198+
AccessToken: "at", RefreshToken: "rt", ExpiresAt: expires.Add(time.Minute),
199+
}), "a refreshed expiry must fingerprint differently")
200+
201+
assert.NotContains(t, tokenFingerprint(base), "at", "the fingerprint must not carry the token")
202+
assert.Empty(t, tokenFingerprint(nil))
203+
}
204+
205+
// TestScanForNewTokens_OnlyOnNewToken pins the fix for the 5s redial loop: the
206+
// scan must fire when a login/refresh writes a NEW token, not on every pass just
207+
// because the server still holds the stale token it already failed with (#1013).
208+
func TestScanForNewTokens_OnlyOnNewToken(t *testing.T) {
209+
logger := zap.NewNop()
210+
tempDir := t.TempDir()
211+
db, err := storage.NewBoltDB(tempDir, logger.Sugar())
212+
require.NoError(t, err)
213+
defer db.Close()
214+
215+
serverConfig := &config.ServerConfig{
216+
Name: "parked-server",
217+
URL: "http://127.0.0.1:1/mcp", // refused immediately; no real upstream needed
218+
Protocol: "http",
219+
Enabled: true,
220+
Created: time.Now(),
221+
}
222+
serverKey := oauth.GenerateServerKey(serverConfig.Name, serverConfig.URL)
223+
saveToken := func(access string) {
224+
require.NoError(t, db.SaveOAuthToken(&storage.OAuthTokenRecord{
225+
ServerName: serverKey,
226+
DisplayName: serverConfig.Name,
227+
AccessToken: access,
228+
TokenType: "Bearer",
229+
ExpiresAt: time.Now().Add(time.Hour),
230+
Created: time.Now(),
231+
Updated: time.Now(),
232+
}))
233+
}
234+
saveToken("stale-token")
235+
236+
manager := &Manager{
237+
clients: make(map[string]*managed.Client),
238+
logger: logger,
239+
storage: db,
240+
secretResolver: secret.NewResolver(),
241+
tokenReconnect: make(map[string]time.Time),
242+
tokenFingerprints: make(map[string]string),
243+
}
244+
245+
client, err := managed.NewClient("parked-server", serverConfig, logger, nil, &config.Config{}, db, secret.NewResolver())
246+
require.NoError(t, err)
247+
manager.clients["parked-server"] = client
248+
249+
// Park the client exactly as a deferred-OAuth connect failure would.
250+
client.StateManager.SetPendingAuth(errors.New("OAuth authentication required for parked-server: login available via Web UI"))
251+
252+
// Pretend the per-server rate limit has expired so it cannot mask the result.
253+
expireRateLimit := func() { manager.tokenReconnect["parked-server"] = time.Now().Add(-time.Minute) }
254+
255+
expireRateLimit()
256+
manager.scanForNewTokens()
257+
require.WithinDuration(t, time.Now(), manager.tokenReconnect["parked-server"], time.Second,
258+
"first scan must retry with the stored token")
259+
require.NotEmpty(t, manager.tokenFingerprints["parked-server"])
260+
261+
// Same token, rate limit expired: must NOT redial again.
262+
expireRateLimit()
263+
before := manager.tokenReconnect["parked-server"]
264+
manager.scanForNewTokens()
265+
require.Equal(t, before, manager.tokenReconnect["parked-server"],
266+
"scan redialed a parked server for a token it had already tried")
267+
268+
// A fresh token (login completed) must wake it.
269+
saveToken("fresh-token")
270+
expireRateLimit()
271+
manager.scanForNewTokens()
272+
require.WithinDuration(t, time.Now(), manager.tokenReconnect["parked-server"], time.Second,
273+
"scan did not wake the parked server when a new token appeared")
274+
}

0 commit comments

Comments
 (0)