Skip to content

Commit b441207

Browse files
stuggiclaude
andcommitted
Wait for webhook certificate secrets before checking endpoints
Fixes operator crash-loop when webhook certificate secrets don't exist yet. Operators with ENABLE_WEBHOOKS=true mount certificate secrets as volumes. If these secrets don't exist when the pod starts (cert-manager creates them asynchronously), the pod fails with FailedMount and crash-loops. This was surfaced by the new OLM-based CI flow which deploys operators and immediately proceeds to deploy OpenStackControlPlane, not giving cert-manager enough time to create the certificate secrets. The controller now checks for required webhook certificate secrets before checking service endpoints, preventing the controller itself from crash-looping while waiting for cert-manager to finish. This makes the operator robust to edge cases like certificate regeneration or slow cert-manager processing. Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com> Signed-off-by: Martin Schuppert <mschuppert@redhat.com>
1 parent f3a5bdb commit b441207

1 file changed

Lines changed: 71 additions & 3 deletions

File tree

‎internal/controller/operator/openstack_controller.go‎

Lines changed: 71 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -57,8 +57,9 @@ import (
5757
// OpenStackReconciler reconciles a OpenStack object
5858
type OpenStackReconciler struct {
5959
client.Client
60-
Scheme *runtime.Scheme
61-
Kclient kubernetes.Interface
60+
Scheme *runtime.Scheme
61+
Kclient kubernetes.Interface
62+
webhookCertSecretNames []string // cached list of webhook certificate secret names, populated once at startup
6263
}
6364

6465
// GetLogger returns a logger object with a prefix of "controller.name" and additional controller context fields
@@ -348,8 +349,23 @@ func (r *OpenStackReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
348349
operatorv1beta1.OpenStackOperatorDeploymentsReadyCondition,
349350
operatorv1beta1.OpenStackOperatorDeploymentsReadyMessage)
350351

352+
// Check if webhook certificate secrets exist before checking endpoints
353+
// Operators with webhooks enabled will fail to start if these secrets don't exist yet
354+
ctrlResult, err := r.checkWebhookCertificates(ctx, instance)
355+
if err != nil {
356+
instance.Status.Conditions.Set(condition.FalseCondition(
357+
operatorv1beta1.OpenStackOperatorReadyCondition,
358+
condition.ErrorReason,
359+
condition.SeverityWarning,
360+
operatorv1beta1.OpenStackOperatorErrorMessage,
361+
err))
362+
return ctrl.Result{}, err
363+
} else if (ctrlResult != ctrl.Result{}) {
364+
return ctrlResult, nil
365+
}
366+
351367
// Check if Services are running and have an endpoint
352-
ctrlResult, err := r.checkServiceEndpoints(ctx, instance)
368+
ctrlResult, err = r.checkServiceEndpoints(ctx, instance)
353369
if err != nil {
354370
instance.Status.Conditions.Set(condition.FalseCondition(
355371
operatorv1beta1.OpenStackOperatorReadyCondition,
@@ -561,6 +577,56 @@ func isWebhookEndpoint(name string) bool {
561577
return false
562578
}
563579

580+
// loadWebhookCertSecretNames inspects bindata/operator/*-webhooks.yaml and returns the expected
581+
// webhook certificate secret names. Called once at startup in SetupWithManager.
582+
func loadWebhookCertSecretNames(bindir string) []string {
583+
secrets := []string{"webhook-server-cert"} // openstack-operator
584+
585+
pattern := filepath.Join(bindir, "operator", "*-webhooks.yaml")
586+
matches, _ := filepath.Glob(pattern)
587+
for _, path := range matches {
588+
base := filepath.Base(path)
589+
// Extract operator name: "infra-operator-webhooks.yaml" -> "infra-operator"
590+
opName := strings.TrimSuffix(base, "-webhooks.yaml")
591+
if opName != "openstack-operator" { // already added above
592+
secrets = append(secrets, opName+"-webhook-server-cert")
593+
}
594+
}
595+
return secrets
596+
}
597+
598+
// checkWebhookCertificates checks if webhook certificate secrets exist for operators with webhooks enabled.
599+
// Operator pods with ENABLE_WEBHOOKS=true mount these secrets and fail to start if they don't exist yet.
600+
// This prevents the controller from crash-looping while waiting for cert-manager to create the certificates.
601+
func (r *OpenStackReconciler) checkWebhookCertificates(ctx context.Context, instance *operatorv1beta1.OpenStack) (ctrl.Result, error) {
602+
Log := r.GetLogger(ctx)
603+
604+
secretList := &corev1.SecretList{}
605+
err := r.List(ctx, secretList, &client.ListOptions{Namespace: instance.Namespace})
606+
if err != nil {
607+
return ctrl.Result{}, err
608+
}
609+
610+
existingSecrets := make(map[string]bool)
611+
for _, secret := range secretList.Items {
612+
existingSecrets[secret.Name] = true
613+
}
614+
615+
var missingSecrets []string
616+
for _, secretName := range r.webhookCertSecretNames {
617+
if !existingSecrets[secretName] {
618+
missingSecrets = append(missingSecrets, secretName)
619+
}
620+
}
621+
622+
if len(missingSecrets) > 0 {
623+
Log.Info("Waiting for webhook certificate secrets to be created by cert-manager", "missing", missingSecrets)
624+
return ctrl.Result{RequeueAfter: time.Duration(5) * time.Second}, nil
625+
}
626+
627+
return ctrl.Result{}, nil
628+
}
629+
564630
// checkServiceEndpoints -
565631
func (r *OpenStackReconciler) checkServiceEndpoints(ctx context.Context, instance *operatorv1beta1.OpenStack) (ctrl.Result, error) {
566632
Log := r.GetLogger(ctx)
@@ -1178,6 +1244,8 @@ func (r *OpenStackReconciler) postCleanupObsoleteResources(ctx context.Context,
11781244

11791245
// SetupWithManager sets up the controller with the Manager.
11801246
func (r *OpenStackReconciler) SetupWithManager(mgr ctrl.Manager) error {
1247+
bindir := util.GetEnvVar("BASE_BINDATA", "/bindata")
1248+
r.webhookCertSecretNames = loadWebhookCertSecretNames(bindir)
11811249
return ctrl.NewControllerManagedBy(mgr).
11821250
Owns(&appsv1.Deployment{}).
11831251
For(&operatorv1beta1.OpenStack{}).

0 commit comments

Comments
 (0)