@@ -113,18 +113,10 @@ func (r *OpenStackDataPlaneDeploymentReconciler) Reconcile(ctx context.Context,
113113 return ctrl.Result {}, nil
114114 }
115115
116- // If the deployment has failed with backoff limit exceeded, do not reconcile
117- // even if nodesets or other watched resources change. The deployment is in a
118- // terminal failure state and should not be retried.
119- if instance .Status .Conditions != nil {
120- deploymentCondition := instance .Status .Conditions .Get (condition .DeploymentReadyCondition )
121- if deploymentCondition != nil &&
122- deploymentCondition .Severity == condition .SeverityError &&
123- deploymentCondition .Reason == condition .JobReasonBackoffLimitExceeded {
124- Log .Info ("Deployment has failed with backoff limit exceeded, skipping reconciliation" ,
125- "deployment" , instance .Name )
126- return ctrl.Result {}, nil
127- }
116+ if terminalFailure (instance ) {
117+ Log .Info ("Deployment is terminal, skipping reconciliation" ,
118+ "deployment" , instance .Name )
119+ return ctrl.Result {}, nil
128120 }
129121
130122 // initialize status if Conditions is nil, but do not reset if it already
@@ -255,7 +247,10 @@ func (r *OpenStackDataPlaneDeploymentReconciler) Reconcile(ctx context.Context,
255247 haveError := false
256248 deploymentErrMsg := ""
257249 var servicePlan * deployment.ServicePlan
258- backoffLimitReached := false
250+ var finishedNodeSets []dataplanev1.OpenStackDataPlaneNodeSet
251+ if instance .Status .FailedNodeSets == nil {
252+ instance .Status .FailedNodeSets = map [string ]string {}
253+ }
259254
260255 if servicePlan , err = deployment .DedupeServices (ctx , helper , nodeSets .Items ,
261256 instance .Spec .ServicesOverride ); err != nil {
@@ -383,6 +378,21 @@ func (r *OpenStackDataPlaneDeploymentReconciler) Reconcile(ctx context.Context,
383378 for i := range nodeSets .Items {
384379 nodeSet := & nodeSets .Items [i ]
385380
381+ if failure , ok := instance .Status .FailedNodeSets [nodeSet .Name ]; ok {
382+ Log .Info ("NodeSet already terminated by a permanent failure" , "nodeSet" , nodeSet .Name )
383+ nsConditions := instance .Status .NodeSetConditions [nodeSet .Name ]
384+ nsConditions .MarkFalse (
385+ dataplanev1 .NodeSetDeploymentReadyCondition ,
386+ condition .JobReasonBackoffLimitExceeded ,
387+ condition .SeverityError ,
388+ "%s" , failure )
389+ instance .Status .NodeSetConditions [nodeSet .Name ] = nsConditions
390+ haveError = true
391+ deploymentErrMsg = appendDeploymentErrMsg (deploymentErrMsg ,
392+ fmt .Sprintf ("nodeSet: %s error: %s" , nodeSet .Name , failure ))
393+ continue
394+ }
395+
386396 Log .Info ("Deploying NodeSet" , "nodeSet" , nodeSet .Name )
387397 Log .Info ("Set Status.Deployed to false" , "instance" , instance )
388398 instance .Status .Deployed = false
@@ -413,14 +423,12 @@ func (r *OpenStackDataPlaneDeploymentReconciler) Reconcile(ctx context.Context,
413423 util .LogErrorForObject (helper , err , fmt .Sprintf ("OpenStackDeployment error for NodeSet %s" , nodeSet .Name ), instance )
414424 Log .Info ("Set NodeSetDeploymentReadyCondition false" , "nodeSet" , nodeSet .Name )
415425 haveError = true
416- errMsg := fmt . Sprintf ( "nodeSet: %s error: %s" , nodeSet . Name , err . Error ())
417- if len ( deploymentErrMsg ) == 0 {
418- deploymentErrMsg = errMsg
419- } else {
420- deploymentErrMsg = fmt . Sprintf ( "%s & %s" , deploymentErrMsg , errMsg )
426+ deploymentErrMsg = appendDeploymentErrMsg ( deploymentErrMsg ,
427+ fmt . Sprintf ( "nodeSet: %s error: %s" , nodeSet . Name , err . Error ()))
428+ if nsConditions . Get ( dataplanev1 . NodeSetDeploymentReadyCondition ) != nil &&
429+ nsConditions . Get ( dataplanev1 . NodeSetDeploymentReadyCondition ). Reason == condition . JobReasonBackoffLimitExceeded {
430+ instance . Status . FailedNodeSets [ nodeSet . Name ] = err . Error ( )
421431 }
422- errorReason := nsConditions .Get (dataplanev1 .NodeSetDeploymentReadyCondition ).Reason
423- backoffLimitReached = errorReason == condition .JobReasonBackoffLimitExceeded
424432 }
425433
426434 if deployResult != nil {
@@ -431,16 +439,26 @@ func (r *OpenStackDataPlaneDeploymentReconciler) Reconcile(ctx context.Context,
431439 nsConditions .MarkTrue (
432440 dataplanev1 .NodeSetDeploymentReadyCondition ,
433441 condition .DeploymentReadyMessage )
434- instance .Status .NodeSetHashes [nodeSet .Name ] = nodeSet .Status .ConfigHash
435- instance .Status .BmhRefHashes [nodeSet .Name ] = nodeSet .Status .BmhRefHash
442+ finishedNodeSets = append (finishedNodeSets , * nodeSet )
443+ }
444+ }
445+
446+ // A nodeSet that finished alongside a failure must still get its hashes, or the
447+ // NodeSet controller re-deploys it because its inputs look changed.
448+ if len (finishedNodeSets ) > 0 {
449+ err = r .setHashes (ctx , helper , instance ,
450+ dataplanev1.OpenStackDataPlaneNodeSetList {Items : finishedNodeSets })
451+ if err != nil {
452+ return ctrl.Result {}, err
436453 }
437454 }
438455
439456 if haveError {
457+ settled := terminalFailure (instance )
440458 var reason condition.Reason
441459 reason = condition .ErrorReason
442460 severity := condition .SeverityWarning
443- if backoffLimitReached {
461+ if settled {
444462 reason = condition .JobReasonBackoffLimitExceeded
445463 severity = condition .SeverityError
446464 }
@@ -450,7 +468,7 @@ func (r *OpenStackDataPlaneDeploymentReconciler) Reconcile(ctx context.Context,
450468 severity ,
451469 condition .DeploymentReadyErrorMessage ,
452470 deploymentErrMsg )
453- if backoffLimitReached {
471+ if settled {
454472 return ctrl.Result {}, nil
455473 }
456474 return ctrl.Result {}, fmt .Errorf ("%s" , deploymentErrMsg )
@@ -468,10 +486,6 @@ func (r *OpenStackDataPlaneDeploymentReconciler) Reconcile(ctx context.Context,
468486 if version != nil {
469487 instance .Status .DeployedVersion = version .Spec .TargetVersion
470488 }
471- err = r .setHashes (ctx , helper , instance , * nodeSets )
472- if err != nil {
473- Log .Error (err , "Error setting service hashes" )
474- }
475489 return ctrl.Result {}, nil
476490}
477491
@@ -600,14 +614,19 @@ func (r *OpenStackDataPlaneDeploymentReconciler) setHashes(
600614 }
601615
602616 for _ , nodeSet := range nodeSets .Items {
603- instance .Status .NodeSetHashes [nodeSet .Name ] = nodeSet .Status .ConfigHash
604-
605617 err = setNodeSetAnsibleVarsFromHashes (ctx , helper , & nodeSet , instance .Status .ConfigMapHashes , instance .Status .SecretHashes )
606618 if err != nil {
607619 return err
608620 }
609621 }
610622
623+ // Split from the loop above: terminalFailure reads a NodeSetHashes entry as proof
624+ // the nodeSet's input hashes are recorded, so nothing above may fail after it.
625+ for _ , nodeSet := range nodeSets .Items {
626+ instance .Status .NodeSetHashes [nodeSet .Name ] = nodeSet .Status .ConfigHash
627+ instance .Status .BmhRefHashes [nodeSet .Name ] = nodeSet .Status .BmhRefHash
628+ }
629+
611630 return nil
612631}
613632
@@ -703,6 +722,39 @@ func (r *OpenStackDataPlaneDeploymentReconciler) listNodeSets(ctx context.Contex
703722 return & nodeSets , err
704723}
705724
725+ // terminalFailure reports whether the deployment has a permanently failed
726+ // nodeSet and nothing left to wait for. A nodeSet still in flight keeps this
727+ // false even when another one already failed. Deployments that failed before
728+ // FailedNodeSets existed carry the failure only on the top-level condition, so
729+ // honor it as well: their backing jobs are garbage collected and must not be
730+ // recreated.
731+ func terminalFailure (instance * dataplanev1.OpenStackDataPlaneDeployment ) bool {
732+ if c := instance .Status .Conditions .Get (condition .DeploymentReadyCondition ); c != nil &&
733+ c .Severity == condition .SeverityError &&
734+ c .Reason == condition .JobReasonBackoffLimitExceeded {
735+ return true
736+ }
737+ if len (instance .Spec .NodeSets ) == 0 || len (instance .Status .FailedNodeSets ) == 0 {
738+ return false
739+ }
740+ for _ , nodeSet := range instance .Spec .NodeSets {
741+ if _ , failed := instance .Status .FailedNodeSets [nodeSet ]; failed {
742+ continue
743+ }
744+ if _ , done := instance .Status .NodeSetHashes [nodeSet ]; ! done {
745+ return false
746+ }
747+ }
748+ return true
749+ }
750+
751+ func appendDeploymentErrMsg (current string , errMsg string ) string {
752+ if len (current ) == 0 {
753+ return errMsg
754+ }
755+ return fmt .Sprintf ("%s & %s" , current , errMsg )
756+ }
757+
706758func setNodeSetAnsibleVarsFromHashes (
707759 ctx context.Context ,
708760 helper * helper.Helper ,
0 commit comments