Skip to content

Commit 55e18d6

Browse files
feat: introduce ErrHeartbeatDeregistered to allow controllers to distinguish heartbeat failures from permanent deregistration
1 parent 5564962 commit 55e18d6

4 files changed

Lines changed: 27 additions & 8 deletions

File tree

‎internal/controller/agentdeployment_controller.go‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ package controller
1919

2020
import (
2121
"context"
22+
"errors"
2223
"fmt"
2324
"sync"
2425
"time"
@@ -625,7 +626,7 @@ func (r *AgentDeploymentReconciler) reconcileMCPRegistration(ctx context.Context
625626
if ad.Status.Registered {
626627
if err := r.Registrar.Deregister(ctx, ad); err != nil {
627628
SetCondition(ad, agentraxv1alpha1.ConditionMCPHandshakeFailed, metav1.ConditionTrue, "DeregisterFailed", err.Error())
628-
return 0
629+
return 5 * time.Second
629630
}
630631
ad.Status.Registered = false
631632
}
@@ -644,8 +645,10 @@ func (r *AgentDeploymentReconciler) reconcileMCPRegistration(ctx context.Context
644645
// If already registered, perform heartbeat probe.
645646
if ad.Status.Registered {
646647
if err := r.Registrar.Heartbeat(ctx, ad); err != nil {
647-
// After 3 strikes, Heartbeat deregisters the agent.
648-
ad.Status.Registered = false
648+
// Only mark unregistered if Heartbeat confirms deregistration (after 3 consecutive failures).
649+
if errors.Is(err, registry.ErrHeartbeatDeregistered) {
650+
ad.Status.Registered = false
651+
}
649652
SetCondition(ad, agentraxv1alpha1.ConditionMCPHandshakeFailed, metav1.ConditionTrue, "HeartbeatFailed", err.Error())
650653
return requeueInterval
651654
}

‎internal/controller/mcp_registration_test.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"context"
2121
"errors"
2222
"fmt"
23+
"sync/atomic"
2324
"time"
2425

2526
. "github.com/onsi/ginkgo/v2"
@@ -306,12 +307,12 @@ var _ = Describe("MCP Registration Integration", func() {
306307
}, timeout, interval).Should(Succeed())
307308

308309
// Install a Deregister hook that verifies Service still exists.
309-
var serviceExistedDuringDeregister bool
310+
var serviceExistedDuringDeregister atomic.Bool
310311
testReconciler.SetDeregister(func(ctx context.Context, ad *agentraxv1alpha1.AgentDeployment) error {
311312
svc := &corev1.Service{}
312313
svcKey := types.NamespacedName{Name: adName, Namespace: ns}
313314
err := k8sClient.Get(ctx, svcKey, svc)
314-
serviceExistedDuringDeregister = (err == nil)
315+
serviceExistedDuringDeregister.Store(err == nil)
315316
return testRegistrar.Deregister(ctx, ad)
316317
})
317318
defer testReconciler.SetDeregister(nil)
@@ -328,7 +329,7 @@ var _ = Describe("MCP Registration Integration", func() {
328329
}, timeout, interval).Should(BeTrue())
329330

330331
// Verify deregistration occurred while Service was present
331-
Expect(serviceExistedDuringDeregister).To(BeTrue(), "Service should exist during deregistration")
332+
Expect(serviceExistedDuringDeregister.Load()).To(BeTrue(), "Service should exist during deregistration")
332333

333334
// Entry must be gone from registry
334335
_, ok := testRegistry.Get(ns, adName)

‎internal/registry/mcp_registrar.go‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ package registry
1818

1919
import (
2020
"context"
21+
"errors"
2122
"fmt"
2223
"strings"
2324
"sync"
@@ -30,6 +31,9 @@ const (
3031
MaxConsecutiveHeartbeatFailures = 3
3132
)
3233

34+
// ErrHeartbeatDeregistered is returned when an agent is deregistered due to reaching MaxConsecutiveHeartbeatFailures.
35+
var ErrHeartbeatDeregistered = errors.New("deregistered after consecutive heartbeat failures")
36+
3337
// Registrar coordinates MCP initialize handshakes and updates the discovery Registry.
3438
type Registrar struct {
3539
// Registry is the underlying registry store and HTTP server handler.
@@ -130,9 +134,11 @@ func (r *Registrar) Heartbeat(ctx context.Context, ad *agentraxv1alpha1.AgentDep
130134
if failCount >= MaxConsecutiveHeartbeatFailures {
131135
deregErr := r.Registry.Deregister(ctx, ad.Namespace, ad.Name)
132136
if deregErr != nil {
133-
return fmt.Errorf("deregistered after %d consecutive heartbeat failures (latest error: %w); deregister error: %v", failCount, err, deregErr)
137+
return fmt.Errorf("%w: deregistered after %d consecutive heartbeat failures (latest error: %w); deregister error: %v",
138+
ErrHeartbeatDeregistered, failCount, err, deregErr)
134139
}
135-
return fmt.Errorf("deregistered after %d consecutive heartbeat failures (latest error: %w)", failCount, err)
140+
return fmt.Errorf("%w: deregistered after %d consecutive heartbeat failures (latest error: %w)",
141+
ErrHeartbeatDeregistered, failCount, err)
136142
}
137143
return fmt.Errorf("heartbeat probe failed (%d/%d): %w", failCount, MaxConsecutiveHeartbeatFailures, err)
138144
}

‎internal/registry/registry_test.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -429,6 +429,9 @@ func TestRegistrar_RegisterAndHeartbeat(t *testing.T) {
429429
if err == nil {
430430
t.Errorf("expected error on strike 1")
431431
}
432+
if errors.Is(err, ErrHeartbeatDeregistered) {
433+
t.Errorf("strike 1 should not return ErrHeartbeatDeregistered")
434+
}
432435
if _, ok := reg.Get("tenant-x", "test-agent"); !ok {
433436
t.Errorf("entry should still exist after 1 strike")
434437
}
@@ -438,6 +441,9 @@ func TestRegistrar_RegisterAndHeartbeat(t *testing.T) {
438441
if err == nil {
439442
t.Errorf("expected error on strike 2")
440443
}
444+
if errors.Is(err, ErrHeartbeatDeregistered) {
445+
t.Errorf("strike 2 should not return ErrHeartbeatDeregistered")
446+
}
441447
if _, ok := reg.Get("tenant-x", "test-agent"); !ok {
442448
t.Errorf("entry should still exist after 2 strikes")
443449
}
@@ -447,6 +453,9 @@ func TestRegistrar_RegisterAndHeartbeat(t *testing.T) {
447453
if err == nil {
448454
t.Errorf("expected error on strike 3")
449455
}
456+
if !errors.Is(err, ErrHeartbeatDeregistered) {
457+
t.Errorf("strike 3 should return ErrHeartbeatDeregistered, got %v", err)
458+
}
450459
if _, ok := reg.Get("tenant-x", "test-agent"); ok {
451460
t.Errorf("entry should be removed after 3 consecutive failures")
452461
}

0 commit comments

Comments
 (0)