Preserve config-level outgoing auth across backend watcher reconciles (#6513)

The Kubernetes backend watcher rebuilt each backend from its resource references alone, so outgoingAuth.default and outgoingAuth.backends entries applied at startup were silently dropped on the first reconcile.
The resolution logic moves to config.OutgoingAuthConfig.ApplyToBackend and the reconciler now applies it before every upsert, mirroring startup discovery exactly.
NewBackendWatcher accepts the outgoing auth configuration and NewKubernetesBackendRegistry exposes it through the WithOutgoingAuth option.

Author: alex-feel

pkg/vmcp/aggregator/discoverer.go

@@ -220,49 +220,11 @@ func (d *backendDiscoverer) Discover(ctx context.Context, groupRef string) (back
 // applyAuthConfigToBackend applies authentication configuration to a backend based on the source mode.
 // It determines whether to use discovered auth from the MCPServer or auth from the vMCP config.
 //
-// Auth resolution logic:
-// - "discovered" mode: Use discovered auth if available, otherwise fall back to Default or backend-specific config
-// - "inline" mode (or ""): Always use config-based auth, ignore discovered auth
-// - unknown mode: Default to config-based auth for safety
-//
-// When useDiscoveredAuth is false, ResolveForBackend is called which handles:
-// 1. Backend-specific config (d.authConfig.Backends[backendName])
-// 2. Default config fallback (d.authConfig.Default)
-// 3. No auth if neither is configured
+// The resolution logic lives in config.OutgoingAuthConfig.ApplyToBackend so the
+// Kubernetes backend watcher's reconciler applies the exact same semantics when it
+// rebuilds a backend (see pkg/vmcp/k8s).
 func (d *backendDiscoverer) applyAuthConfigToBackend(backend *vmcp.Backend, backendName string) {
-	if d.authConfig == nil {
-		return
-	}
-
-	// Determine if we should use discovered auth or config-based auth
-	var useDiscoveredAuth bool
-	switch d.authConfig.Source {
-	case "discovered":
-		// In discovered mode, use auth discovered from MCPServer (if any exists)
-		// If no auth is discovered, fall back to config-based auth via ResolveForBackend
-		// which will use backend-specific config, then Default, then no auth
-		useDiscoveredAuth = backend.AuthConfig != nil
-	case "inline", "":
-		// For inline mode or empty source, always use config-based auth
-		// Ignore any discovered auth from backends
-		useDiscoveredAuth = false
-	default:
-		// Unknown source mode - default to config-based auth for safety
-		slog.Warn("unknown auth source mode, defaulting to config-based auth", "source", d.authConfig.Source)
-		useDiscoveredAuth = false
-	}
-
-	if useDiscoveredAuth {
-		// Keep the auth discovered from MCPServer (already populated in backend)
-		slog.Debug("backend using discovered auth strategy", "backend", backendName, "strategy", backend.AuthConfig.Type)
-	} else {
-		// Use auth from config (inline mode)
-		authConfig := d.authConfig.ResolveForBackend(backendName)
-		if authConfig != nil {
-			backend.AuthConfig = authConfig
-			slog.Debug("backend configured with auth strategy from config", "backend", backendName, "strategy", authConfig.Type)
-		}
-	}
+	d.authConfig.ApplyToBackend(backend, backendName)
 }
 
 // discoverFromStaticConfig converts pre-configured static backends into vmcp.Backend objects

pkg/vmcp/backendregistry/registry.go

@@ -28,13 +28,15 @@ import (
 	"k8s.io/client-go/rest"
 
 	"github.com/stacklok/toolhive/pkg/vmcp"
+	"github.com/stacklok/toolhive/pkg/vmcp/config"
 	"github.com/stacklok/toolhive/pkg/vmcp/k8s"
 	"github.com/stacklok/toolhive/pkg/vmcp/server"
 )
 
 // options holds the optional settings for NewKubernetesBackendRegistry.
 type options struct {
-	restConfig *rest.Config
+	restConfig   *rest.Config
+	outgoingAuth *config.OutgoingAuthConfig
 }
 
 // Option configures NewKubernetesBackendRegistry.
@@ -51,6 +53,17 @@ func WithRESTConfig(cfg *rest.Config) Option {
 	}
 }
 
+// WithOutgoingAuth supplies the vMCP config-level outgoing auth configuration.
+// The watcher applies it to every reconciled backend with the same precedence
+// startup discovery uses (discovered CR-side auth first, then backends[<name>],
+// then Default). The default (no option) is nil: reconciled backends keep only
+// the auth discovered from their own resource references.
+func WithOutgoingAuth(authConfig *config.OutgoingAuthConfig) Option {
+	return func(o *options) {
+		o.outgoingAuth = authConfig
+	}
+}
+
 // NewKubernetesBackendRegistry builds a live, Kubernetes-populated backend
 // registry for an embedder, hiding the pkg/vmcp/k8s watch substrate.
 //
@@ -127,7 +140,7 @@ func NewKubernetesBackendRegistry(
 	// Start empty; the watcher's initial informer sync populates the registry.
 	dynamicRegistry := vmcp.NewDynamicRegistry(nil)
 
-	watcher, err := k8s.NewBackendWatcher(restConfig, namespace, group, dynamicRegistry)
+	watcher, err := k8s.NewBackendWatcher(restConfig, namespace, group, dynamicRegistry, o.outgoingAuth)
 	if err != nil {
 		return nil, nil, fmt.Errorf("failed to create backend watcher: %w", err)
 	}

pkg/vmcp/cli/serve.go

@@ -230,7 +230,7 @@ func Serve(ctx context.Context, cfg ServeConfig) error {
 			return fmt.Errorf("VMCP_NAMESPACE environment variable not set")
 		}
 
-		backendWatcher, err = k8s.NewBackendWatcher(restConfig, namespace, vmcpCfg.Group, dynamicRegistry)
+		backendWatcher, err = k8s.NewBackendWatcher(restConfig, namespace, vmcpCfg.Group, dynamicRegistry, vmcpCfg.OutgoingAuth)
 		if err != nil {
 			return fmt.Errorf("failed to create backend watcher: %w", err)
 		}

pkg/vmcp/config/config.go

@@ -11,6 +11,7 @@ package config
 import (
 	"encoding/json"
 	"fmt"
+	"log/slog"
 	"time"
 
 	"github.com/stacklok/toolhive/pkg/audit"
@@ -427,6 +428,54 @@ func (c *OutgoingAuthConfig) ResolveForBackend(backendID string) *authtypes.Back
 	return nil
 }
 
+// ApplyToBackend applies this outgoing auth configuration to a backend, choosing
+// between the backend's own discovered auth and config-based auth. It is the single
+// implementation shared by startup discovery (pkg/vmcp/aggregator) and the Kubernetes
+// backend watcher's reconciler (pkg/vmcp/k8s), so both paths resolve auth identically.
+//
+// Auth resolution logic:
+//   - "discovered" mode: keep discovered auth if the backend has any, otherwise fall
+//     back to config-based auth via ResolveForBackend (backend-specific entry, then
+//     Default, then no auth)
+//   - "inline" mode (or ""): always use config-based auth, ignore discovered auth
+//   - unknown mode: default to config-based auth for safety
+//
+// A nil receiver is a no-op: the backend keeps whatever auth it already carries.
+func (c *OutgoingAuthConfig) ApplyToBackend(backend *vmcp.Backend, backendName string) {
+	if c == nil {
+		return
+	}
+
+	// Determine if we should use discovered auth or config-based auth
+	var useDiscoveredAuth bool
+	switch c.Source {
+	case "discovered":
+		// In discovered mode, use auth discovered from MCPServer (if any exists)
+		// If no auth is discovered, fall back to config-based auth via ResolveForBackend
+		// which will use backend-specific config, then Default, then no auth
+		useDiscoveredAuth = backend.AuthConfig != nil
+	case "inline", "":
+		// For inline mode or empty source, always use config-based auth
+		// Ignore any discovered auth from backends
+		useDiscoveredAuth = false
+	default:
+		// Unknown source mode - default to config-based auth for safety
+		slog.Warn("unknown auth source mode, defaulting to config-based auth", "source", c.Source)
+		useDiscoveredAuth = false
+	}
+
+	if useDiscoveredAuth {
+		// Keep the auth discovered from MCPServer (already populated in backend)
+		slog.Debug("backend using discovered auth strategy", "backend", backendName, "strategy", backend.AuthConfig.Type)
+	} else {
+		// Use auth from config (inline mode)
+		if authConfig := c.ResolveForBackend(backendName); authConfig != nil {
+			backend.AuthConfig = authConfig
+			slog.Debug("backend configured with auth strategy from config", "backend", backendName, "strategy", authConfig.Type)
+		}
+	}
+}
+
 // AggregationConfig defines tool aggregation, filtering, and conflict resolution strategies.
 //
 // Tool Visibility vs Routing:

pkg/vmcp/k8s/backend_reconciler.go

@@ -18,6 +18,7 @@ import (
 
 	mcpv1beta1 "github.com/stacklok/toolhive/cmd/thv-operator/api/v1beta1"
 	"github.com/stacklok/toolhive/pkg/vmcp"
+	"github.com/stacklok/toolhive/pkg/vmcp/config"
 	"github.com/stacklok/toolhive/pkg/vmcp/workloads"
 )
 
@@ -55,7 +56,8 @@ const (
 //  3. If groupRef doesn't match → Remove from registry (moved to different group)
 //  4. Convert to vmcp.Backend using discoverer
 //  5. If conversion fails or returns nil (auth failed) → Remove from registry
-//  6. Upsert backend to registry (triggers version increment + cache invalidation)
+//  6. Apply config-level outgoing auth (same semantics as startup discovery)
+//  7. Upsert backend to registry (triggers version increment + cache invalidation)
 type BackendReconciler struct {
 	client.Client
 
@@ -70,6 +72,13 @@ type BackendReconciler struct {
 
 	// Discoverer converts K8s resources to vmcp.Backend (reuses existing code)
 	Discoverer workloads.Discoverer
+
+	// OutgoingAuth is the vMCP config-level outgoing auth configuration. It is
+	// applied to every reconciled backend with the same precedence startup
+	// discovery uses (discovered CR-side auth first, then backends[<name>],
+	// then Default), so a reconcile does not strip auth that only exists in
+	// the config. May be nil: backends then keep only their discovered auth.
+	OutgoingAuth *config.OutgoingAuthConfig
 }
 
 // SetupIndexes registers field indexes required by the reconciler's watch handlers.
@@ -283,6 +292,13 @@ func (r *BackendReconciler) convertAndUpsertBackend(
 		return r.removeBackendFromRegistry(ctx, backendID, "Auth failure or no URL")
 	}
 
+	// Apply config-level outgoing auth with the same precedence startup discovery
+	// uses (discovered CR-side auth first, then backends[<name>], then Default).
+	// The discoverer above only resolves auth from the resource's own references,
+	// so without this step a reconcile would strip auth that exists only in the
+	// vMCP config (outgoingAuth.default / outgoingAuth.backends).
+	r.OutgoingAuth.ApplyToBackend(backend, resourceInfo.Name)
+
 	// Upsert backend to registry (triggers version increment + cache invalidation)
 	if err := r.Registry.Upsert(*backend); err != nil {
 		ctxLogger.Error(err, "Failed to upsert backend to registry", "backendID", backend.ID)

pkg/vmcp/k8s/backend_reconciler_test.go

@@ -20,6 +20,8 @@ import (
 	mcpv1beta1 "github.com/stacklok/toolhive/cmd/thv-operator/api/v1beta1"
 	"github.com/stacklok/toolhive/cmd/thv-operator/api/v1beta1/v1beta1test"
 	"github.com/stacklok/toolhive/pkg/vmcp"
+	authtypes "github.com/stacklok/toolhive/pkg/vmcp/auth/types"
+	"github.com/stacklok/toolhive/pkg/vmcp/config"
 	"github.com/stacklok/toolhive/pkg/vmcp/k8s"
 	"github.com/stacklok/toolhive/pkg/vmcp/workloads"
 )
@@ -645,3 +647,140 @@ func TestMapAuthConfigToEntries(t *testing.T) {
 		})
 	}
 }
+
+// newOutgoingAuthTestFixture creates the fake client, discoverer, and registry
+// shared by the outgoing-auth reconciliation tests below. The MCPServer carries
+// no ExternalAuthConfigRef, so any auth on the upserted backend must come from
+// the reconciler's OutgoingAuth config (or from discoveredAuth when non-nil,
+// simulating auth resolved from the resource's own references).
+func newOutgoingAuthTestFixture(
+	t *testing.T,
+	discoveredAuth *authtypes.BackendAuthStrategy,
+) (client.Client, *mockDiscoverer, *mockRegistry) {
+	t.Helper()
+
+	scheme := runtime.NewScheme()
+	require.NoError(t, mcpv1beta1.AddToScheme(scheme))
+
+	mcpServer := &mcpv1beta1.MCPServer{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-server",
+			Namespace: "default",
+		},
+		Spec: mcpv1beta1.MCPServerSpec{
+			GroupRef: &mcpv1beta1.MCPGroupRef{Name: "test-group"},
+		},
+	}
+
+	k8sClient := fake.NewClientBuilder().
+		WithScheme(scheme).
+		WithObjects(mcpServer).
+		Build()
+
+	mockBackend := &vmcp.Backend{
+		ID:         "test-server",
+		Name:       "test-server",
+		BaseURL:    "http://test-server:8080",
+		AuthConfig: discoveredAuth,
+	}
+
+	return k8sClient, &mockDiscoverer{backend: mockBackend}, &mockRegistry{}
+}
+
+// reconcileTestServer runs one reconcile of the "test-server" MCPServer and
+// returns the single upserted backend.
+func reconcileTestServer(t *testing.T, reconciler *k8s.BackendReconciler, mockReg *mockRegistry) vmcp.Backend {
+	t.Helper()
+
+	req := ctrl.Request{
+		NamespacedName: types.NamespacedName{
+			Name:      "test-server",
+			Namespace: "default",
+		},
+	}
+
+	result, err := reconciler.Reconcile(context.Background(), req)
+	require.NoError(t, err)
+	assert.Equal(t, ctrl.Result{}, result)
+	require.Len(t, mockReg.upsertedBackends, 1, "Backend should be upserted to registry")
+	return mockReg.upsertedBackends[0]
+}
+
+// TestReconcile_OutgoingAuthBackendsEntry verifies that a config-level
+// outgoingAuth.backends.<name> entry survives reconciliation for a backend whose
+// own resource carries no discovered auth. Regression test for the watcher path
+// silently dropping config-level outgoing auth (#6454).
+func TestReconcile_OutgoingAuthBackendsEntry(t *testing.T) {
+	t.Parallel()
+
+	k8sClient, mockDisc, mockReg := newOutgoingAuthTestFixture(t, nil)
+
+	wantStrategy := &authtypes.BackendAuthStrategy{Type: authtypes.StrategyTypeUpstreamInject}
+	reconciler := newTestReconciler(k8sClient, "default", "test-group", mockReg, mockDisc)
+	reconciler.OutgoingAuth = &config.OutgoingAuthConfig{
+		Source: "discovered",
+		Backends: map[string]*authtypes.BackendAuthStrategy{
+			"test-server": wantStrategy,
+		},
+	}
+
+	upserted := reconcileTestServer(t, reconciler, mockReg)
+	require.NotNil(t, upserted.AuthConfig, "backends[<name>] auth config must survive reconciliation")
+	assert.Equal(t, authtypes.StrategyTypeUpstreamInject, upserted.AuthConfig.Type)
+}
+
+// TestReconcile_OutgoingAuthDefault verifies that the config-level
+// outgoingAuth.default survives reconciliation for a backend whose own resource
+// carries no discovered auth. Regression test for #6454.
+func TestReconcile_OutgoingAuthDefault(t *testing.T) {
+	t.Parallel()
+
+	k8sClient, mockDisc, mockReg := newOutgoingAuthTestFixture(t, nil)
+
+	reconciler := newTestReconciler(k8sClient, "default", "test-group", mockReg, mockDisc)
+	reconciler.OutgoingAuth = &config.OutgoingAuthConfig{
+		Source:  "discovered",
+		Default: &authtypes.BackendAuthStrategy{Type: authtypes.StrategyTypeHeaderInjection},
+	}
+
+	upserted := reconcileTestServer(t, reconciler, mockReg)
+	require.NotNil(t, upserted.AuthConfig, "default auth config must survive reconciliation")
+	assert.Equal(t, authtypes.StrategyTypeHeaderInjection, upserted.AuthConfig.Type)
+}
+
+// TestReconcile_OutgoingAuthDiscoveredWins verifies that in discovered mode the
+// auth resolved from the backend's own resource references takes precedence over
+// a conflicting config-level backends entry — the same precedence startup
+// discovery applies.
+func TestReconcile_OutgoingAuthDiscoveredWins(t *testing.T) {
+	t.Parallel()
+
+	discovered := &authtypes.BackendAuthStrategy{Type: authtypes.StrategyTypeUpstreamInject}
+	k8sClient, mockDisc, mockReg := newOutgoingAuthTestFixture(t, discovered)
+
+	reconciler := newTestReconciler(k8sClient, "default", "test-group", mockReg, mockDisc)
+	reconciler.OutgoingAuth = &config.OutgoingAuthConfig{
+		Source: "discovered",
+		Backends: map[string]*authtypes.BackendAuthStrategy{
+			"test-server": {Type: authtypes.StrategyTypeHeaderInjection},
+		},
+	}
+
+	upserted := reconcileTestServer(t, reconciler, mockReg)
+	require.NotNil(t, upserted.AuthConfig)
+	assert.Equal(t, authtypes.StrategyTypeUpstreamInject, upserted.AuthConfig.Type,
+		"discovered auth must win over the config-level entry in discovered mode")
+}
+
+// TestReconcile_OutgoingAuthNilConfig verifies that a nil OutgoingAuth leaves
+// the reconciled backend untouched (the pre-existing behavior).
+func TestReconcile_OutgoingAuthNilConfig(t *testing.T) {
+	t.Parallel()
+
+	k8sClient, mockDisc, mockReg := newOutgoingAuthTestFixture(t, nil)
+
+	reconciler := newTestReconciler(k8sClient, "default", "test-group", mockReg, mockDisc)
+
+	upserted := reconcileTestServer(t, reconciler, mockReg)
+	assert.Nil(t, upserted.AuthConfig, "backend without discovered auth stays unauthenticated when no config is supplied")
+}