Skip to content

Commit 4dccba9

Browse files
stuggiclaude
andcommitted
Wait for webhook certificate secrets before checking endpoints
Fixes operator crash-loop when webhook certificate secrets don't exist yet. Operators with ENABLE_WEBHOOKS=true mount certificate secrets as volumes. If these secrets don't exist when the pod starts (cert-manager creates them asynchronously), the pod fails with FailedMount and crash-loops. This was surfaced by the new OLM-based CI flow which deploys operators and immediately proceeds to deploy OpenStackControlPlane, not giving cert-manager enough time to create the certificate secrets. The controller now checks for required webhook certificate secrets before checking service endpoints, preventing the controller itself from crash-looping while waiting for cert-manager to finish. This makes the operator robust to edge cases like certificate regeneration or slow cert-manager processing. Additional improvements: - Filter webhook cert secrets to only check enabled operators, skipping operators disabled via OperatorOverrides (replicas: 0) - Add unit tests for cert validation logic and disabled operator filtering Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com> Signed-off-by: Martin Schuppert <mschuppert@redhat.com>
1 parent 9692cf3 commit 4dccba9

3 files changed

Lines changed: 248 additions & 3 deletions

File tree

‎config/operator/rbac/role.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,14 @@ rules:
1414
- list
1515
- update
1616
- watch
17+
- apiGroups:
18+
- ""
19+
resources:
20+
- secrets
21+
verbs:
22+
- get
23+
- list
24+
- watch
1725
- apiGroups:
1826
- ""
1927
resources:

‎internal/controller/operator/openstack_controller.go‎

Lines changed: 104 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -57,8 +57,9 @@ import (
5757
// OpenStackReconciler reconciles a OpenStack object
5858
type OpenStackReconciler struct {
5959
client.Client
60-
Scheme *runtime.Scheme
61-
Kclient kubernetes.Interface
60+
Scheme *runtime.Scheme
61+
Kclient kubernetes.Interface
62+
webhookCertSecretNames []string // cached list of webhook certificate secret names, populated once at startup
6263
}
6364

6465
// GetLogger returns a logger object with a prefix of "controller.name" and additional controller context fields
@@ -116,6 +117,7 @@ func SetupEnv() {
116117
// +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete;
117118
// +kubebuilder:rbac:groups="",resources=serviceaccounts,verbs=get;list;watch;update;delete
118119
// +kubebuilder:rbac:groups="",resources=namespaces,verbs=get;list;watch;create;update
120+
// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch
119121
// +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;delete
120122
// +kubebuilder:rbac:groups=discovery.k8s.io,resources=endpointslices,verbs=get;list;watch;
121123
// +kubebuilder:rbac:groups=cert-manager.io,resources=issuers,verbs=get;list;watch;create;update;patch;delete;
@@ -333,6 +335,24 @@ func (r *OpenStackReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
333335
err))
334336
return ctrl.Result{}, err
335337
}
338+
339+
// Check if webhook certificate secrets exist before checking deployment status
340+
// Operators with webhooks enabled will fail to start if these secrets don't exist yet,
341+
// causing deployments to not run. Check for missing secrets first so we can requeue
342+
// with the appropriate delay instead of waiting indefinitely.
343+
ctrlResult, err := r.checkWebhookCertificates(ctx, instance)
344+
if err != nil {
345+
instance.Status.Conditions.Set(condition.FalseCondition(
346+
operatorv1beta1.OpenStackOperatorReadyCondition,
347+
condition.ErrorReason,
348+
condition.SeverityWarning,
349+
operatorv1beta1.OpenStackOperatorErrorMessage,
350+
err))
351+
return ctrl.Result{}, err
352+
} else if (ctrlResult != ctrl.Result{}) {
353+
return ctrlResult, nil
354+
}
355+
336356
if deploymentsRunning < *instance.Status.EnabledOperatorCount {
337357
Log.Info("Waiting for all deployments to be running", "current", deploymentsRunning, "expected", *instance.Status.EnabledOperatorCount, "pending", deploymentsPending)
338358
instance.Status.Conditions.Set(condition.FalseCondition(
@@ -349,7 +369,7 @@ func (r *OpenStackReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
349369
operatorv1beta1.OpenStackOperatorDeploymentsReadyMessage)
350370

351371
// Check if Services are running and have an endpoint
352-
ctrlResult, err := r.checkServiceEndpoints(ctx, instance)
372+
ctrlResult, err = r.checkServiceEndpoints(ctx, instance)
353373
if err != nil {
354374
instance.Status.Conditions.Set(condition.FalseCondition(
355375
operatorv1beta1.OpenStackOperatorReadyCondition,
@@ -561,6 +581,85 @@ func isWebhookEndpoint(name string) bool {
561581
return false
562582
}
563583

584+
// loadWebhookCertSecretNames inspects bindata/operator/*-webhooks.yaml and returns the expected
585+
// webhook certificate secret names. Called once at startup in SetupWithManager.
586+
func loadWebhookCertSecretNames(bindir string) []string {
587+
secrets := []string{"webhook-server-cert"} // openstack-operator
588+
589+
pattern := filepath.Join(bindir, "operator", "*-webhooks.yaml")
590+
matches, _ := filepath.Glob(pattern)
591+
for _, path := range matches {
592+
base := filepath.Base(path)
593+
// Extract operator name: "infra-operator-webhooks.yaml" -> "infra-operator"
594+
opName := strings.TrimSuffix(base, "-webhooks.yaml")
595+
if opName != "openstack-operator" { // already added above
596+
secrets = append(secrets, opName+"-webhook-server-cert")
597+
}
598+
}
599+
return secrets
600+
}
601+
602+
// checkWebhookCertificates checks if webhook certificate secrets exist for operators with webhooks enabled.
603+
// Operator pods with ENABLE_WEBHOOKS=true mount these secrets and fail to start if they don't exist yet.
604+
// This prevents the controller from crash-looping while waiting for cert-manager to create the certificates.
605+
func (r *OpenStackReconciler) checkWebhookCertificates(ctx context.Context, instance *operatorv1beta1.OpenStack) (ctrl.Result, error) {
606+
Log := r.GetLogger(ctx)
607+
608+
// Filter webhook cert secrets to only include enabled operators
609+
// An operator is disabled if OperatorOverrides sets replicas to 0
610+
enabledSecrets := make([]string, 0, len(r.webhookCertSecretNames))
611+
for _, secretName := range r.webhookCertSecretNames {
612+
// Extract operator name from secret name
613+
// "webhook-server-cert" -> "openstack"
614+
// "infra-operator-webhook-server-cert" -> "infra"
615+
var operatorName string
616+
if secretName == "webhook-server-cert" {
617+
operatorName = "openstack"
618+
} else {
619+
operatorName = strings.TrimSuffix(secretName, "-operator-webhook-server-cert")
620+
}
621+
622+
// Check if this operator is explicitly disabled
623+
disabled := false
624+
for _, op := range instance.Spec.OperatorOverrides {
625+
if op.Name == operatorName && op.Replicas != nil && *op.Replicas == 0 {
626+
disabled = true
627+
Log.Info("Skipping webhook certificate check for disabled operator", "operator", operatorName, "secret", secretName)
628+
break
629+
}
630+
}
631+
632+
if !disabled {
633+
enabledSecrets = append(enabledSecrets, secretName)
634+
}
635+
}
636+
637+
secretList := &corev1.SecretList{}
638+
err := r.List(ctx, secretList, &client.ListOptions{Namespace: instance.Namespace})
639+
if err != nil {
640+
return ctrl.Result{}, err
641+
}
642+
643+
existingSecrets := make(map[string]bool)
644+
for _, secret := range secretList.Items {
645+
existingSecrets[secret.Name] = true
646+
}
647+
648+
var missingSecrets []string
649+
for _, secretName := range enabledSecrets {
650+
if !existingSecrets[secretName] {
651+
missingSecrets = append(missingSecrets, secretName)
652+
}
653+
}
654+
655+
if len(missingSecrets) > 0 {
656+
Log.Info("Waiting for webhook certificate secrets to be created by cert-manager", "missing", missingSecrets)
657+
return ctrl.Result{RequeueAfter: time.Duration(5) * time.Second}, nil
658+
}
659+
660+
return ctrl.Result{}, nil
661+
}
662+
564663
// checkServiceEndpoints -
565664
func (r *OpenStackReconciler) checkServiceEndpoints(ctx context.Context, instance *operatorv1beta1.OpenStack) (ctrl.Result, error) {
566665
Log := r.GetLogger(ctx)
@@ -1178,6 +1277,8 @@ func (r *OpenStackReconciler) postCleanupObsoleteResources(ctx context.Context,
11781277

11791278
// SetupWithManager sets up the controller with the Manager.
11801279
func (r *OpenStackReconciler) SetupWithManager(mgr ctrl.Manager) error {
1280+
bindir := util.GetEnvVar("BASE_BINDATA", "/bindata")
1281+
r.webhookCertSecretNames = loadWebhookCertSecretNames(bindir)
11811282
return ctrl.NewControllerManagedBy(mgr).
11821283
Owns(&appsv1.Deployment{}).
11831284
For(&operatorv1beta1.OpenStack{}).

‎internal/controller/operator/openstack_controller_test.go‎

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121
"testing"
2222

2323
. "github.com/onsi/gomega" //revive:disable:dot-imports
24+
"k8s.io/utils/ptr"
2425

2526
operatorv1beta1 "github.com/openstack-k8s-operators/openstack-operator/api/operator/v1beta1"
2627
corev1 "k8s.io/api/core/v1"
@@ -85,3 +86,138 @@ func TestAdoptAwayServiceAccounts(t *testing.T) {
8586
// Idempotent: calling it again with the reference already gone must not error.
8687
g.Expect(r.adoptAwayServiceAccounts(context.Background(), instance)).To(Succeed())
8788
}
89+
90+
func TestCheckWebhookCertificates(t *testing.T) {
91+
s := newTestScheme(t)
92+
ctx := context.Background()
93+
94+
instance := &operatorv1beta1.OpenStack{
95+
ObjectMeta: metav1.ObjectMeta{Name: "openstack", Namespace: "openstack-operators"},
96+
Spec: operatorv1beta1.OpenStackSpec{},
97+
}
98+
99+
t.Run("returns requeue when certificate secret is missing", func(t *testing.T) {
100+
g := NewWithT(t)
101+
102+
fakeClient := fakeclient.NewClientBuilder().
103+
WithScheme(s).
104+
WithObjects(instance).
105+
Build()
106+
107+
r := &OpenStackReconciler{
108+
Client: fakeClient,
109+
Scheme: s,
110+
webhookCertSecretNames: []string{"webhook-server-cert"},
111+
}
112+
113+
result, err := r.checkWebhookCertificates(ctx, instance)
114+
g.Expect(err).NotTo(HaveOccurred())
115+
g.Expect(result.RequeueAfter).NotTo(BeZero(), "should requeue when cert is missing")
116+
})
117+
118+
t.Run("proceeds when all certificate secrets exist", func(t *testing.T) {
119+
g := NewWithT(t)
120+
121+
cert := &corev1.Secret{
122+
ObjectMeta: metav1.ObjectMeta{
123+
Name: "webhook-server-cert",
124+
Namespace: "openstack-operators",
125+
},
126+
}
127+
128+
fakeClient := fakeclient.NewClientBuilder().
129+
WithScheme(s).
130+
WithObjects(instance, cert).
131+
Build()
132+
133+
r := &OpenStackReconciler{
134+
Client: fakeClient,
135+
Scheme: s,
136+
webhookCertSecretNames: []string{"webhook-server-cert"},
137+
}
138+
139+
result, err := r.checkWebhookCertificates(ctx, instance)
140+
g.Expect(err).NotTo(HaveOccurred())
141+
g.Expect(result.RequeueAfter).To(BeZero(), "should not requeue when all certs exist")
142+
})
143+
144+
t.Run("skips disabled operators", func(t *testing.T) {
145+
g := NewWithT(t)
146+
147+
instanceWithDisabled := &operatorv1beta1.OpenStack{
148+
ObjectMeta: metav1.ObjectMeta{Name: "openstack", Namespace: "openstack-operators"},
149+
Spec: operatorv1beta1.OpenStackSpec{
150+
OperatorOverrides: []operatorv1beta1.OperatorSpec{
151+
{
152+
Name: "infra",
153+
Replicas: ptr.To(int32(0)),
154+
},
155+
},
156+
},
157+
}
158+
159+
// Only create openstack-operator cert, not infra-operator cert
160+
cert := &corev1.Secret{
161+
ObjectMeta: metav1.ObjectMeta{
162+
Name: "webhook-server-cert",
163+
Namespace: "openstack-operators",
164+
},
165+
}
166+
167+
fakeClient := fakeclient.NewClientBuilder().
168+
WithScheme(s).
169+
WithObjects(instanceWithDisabled, cert).
170+
Build()
171+
172+
r := &OpenStackReconciler{
173+
Client: fakeClient,
174+
Scheme: s,
175+
// Both certs are in the discovered list
176+
webhookCertSecretNames: []string{
177+
"webhook-server-cert",
178+
"infra-operator-webhook-server-cert",
179+
},
180+
}
181+
182+
result, err := r.checkWebhookCertificates(ctx, instanceWithDisabled)
183+
g.Expect(err).NotTo(HaveOccurred())
184+
// Should not requeue - infra cert is not required because infra is disabled
185+
g.Expect(result.RequeueAfter).To(BeZero(), "should not wait for disabled operator's cert")
186+
})
187+
188+
t.Run("waits for enabled operators even when others disabled", func(t *testing.T) {
189+
g := NewWithT(t)
190+
191+
instanceWithDisabled := &operatorv1beta1.OpenStack{
192+
ObjectMeta: metav1.ObjectMeta{Name: "openstack", Namespace: "openstack-operators"},
193+
Spec: operatorv1beta1.OpenStackSpec{
194+
OperatorOverrides: []operatorv1beta1.OperatorSpec{
195+
{
196+
Name: "infra",
197+
Replicas: ptr.To(int32(0)),
198+
},
199+
},
200+
},
201+
}
202+
203+
// No certs exist
204+
fakeClient := fakeclient.NewClientBuilder().
205+
WithScheme(s).
206+
WithObjects(instanceWithDisabled).
207+
Build()
208+
209+
r := &OpenStackReconciler{
210+
Client: fakeClient,
211+
Scheme: s,
212+
webhookCertSecretNames: []string{
213+
"webhook-server-cert", // needed (openstack operator)
214+
"infra-operator-webhook-server-cert", // not needed (disabled)
215+
},
216+
}
217+
218+
result, err := r.checkWebhookCertificates(ctx, instanceWithDisabled)
219+
g.Expect(err).NotTo(HaveOccurred())
220+
// Should requeue - still need openstack-operator cert even though infra is disabled
221+
g.Expect(result.RequeueAfter).NotTo(BeZero(), "should wait for enabled operator's cert")
222+
})
223+
}

0 commit comments

Comments
 (0)