Skip to content

Commit 632d454

Browse files
committed
Track failure per nodeset
A failed service job (backoff limit exceeded) stalls whole deployment and reconcile returned before the other nodeSets could be advanced, so their in-flight jobs were never collected and their remaining services never started. InitConditions() wipes NodeSetConditions every pass and the AEE job backing the failure is garbage collected once its TTL expires, so once it is gone the failure can no longer be derived. Signed-off-by: Rabi Mishra <ramishra@redhat.com>
1 parent 4d7ca3a commit 632d454

7 files changed

Lines changed: 463 additions & 30 deletions

File tree

‎api/bases/dataplane.openstack.org_openstackdataplanedeployments.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -236,6 +236,14 @@ spec:
236236
deployedVersion:
237237
description: DeployedVersion
238238
type: string
239+
failedNodeSets:
240+
additionalProperties:
241+
type: string
242+
description: |-
243+
FailedNodeSets maps a nodeSet name to the message of the permanent failure
244+
that terminated it. Terminality cannot live in the conditions because the AEE
245+
job backing the failure is garbage collected once the deployment settles.
246+
type: object
239247
nodeSetConditions:
240248
additionalProperties:
241249
description: Conditions provide observations of the operational

‎api/dataplane/v1beta1/openstackdataplanedeployment_types.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,11 @@ type OpenStackDataPlaneDeploymentStatus struct {
112112
// NodeSetConditions
113113
NodeSetConditions map[string]condition.Conditions `json:"nodeSetConditions,omitempty" optional:"true"`
114114

115+
// FailedNodeSets maps a nodeSet name to the message of the permanent failure
116+
// that terminated it. Terminality cannot live in the conditions because the AEE
117+
// job backing the failure is garbage collected once the deployment settles.
118+
FailedNodeSets map[string]string `json:"failedNodeSets,omitempty" optional:"true"`
119+
115120
// AnsibleEEHashes
116121
AnsibleEEHashes map[string]string `json:"ansibleEEHashes,omitempty" optional:"true"`
117122

‎api/dataplane/v1beta1/zz_generated.deepcopy.go‎

Lines changed: 7 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎bindata/crds/crds.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20863,6 +20863,14 @@ spec:
2086320863
deployedVersion:
2086420864
description: DeployedVersion
2086520865
type: string
20866+
failedNodeSets:
20867+
additionalProperties:
20868+
type: string
20869+
description: |-
20870+
FailedNodeSets maps a nodeSet name to the message of the permanent failure
20871+
that terminated it. Terminality cannot live in the conditions because the AEE
20872+
job backing the failure is garbage collected once the deployment settles.
20873+
type: object
2086620874
nodeSetConditions:
2086720875
additionalProperties:
2086820876
description: Conditions provide observations of the operational

‎config/crd/bases/dataplane.openstack.org_openstackdataplanedeployments.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -236,6 +236,14 @@ spec:
236236
deployedVersion:
237237
description: DeployedVersion
238238
type: string
239+
failedNodeSets:
240+
additionalProperties:
241+
type: string
242+
description: |-
243+
FailedNodeSets maps a nodeSet name to the message of the permanent failure
244+
that terminated it. Terminality cannot live in the conditions because the AEE
245+
job backing the failure is garbage collected once the deployment settles.
246+
type: object
239247
nodeSetConditions:
240248
additionalProperties:
241249
description: Conditions provide observations of the operational

‎internal/controller/dataplane/openstackdataplanedeployment_controller.go‎

Lines changed: 82 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -113,18 +113,10 @@ func (r *OpenStackDataPlaneDeploymentReconciler) Reconcile(ctx context.Context,
113113
return ctrl.Result{}, nil
114114
}
115115

116-
// If the deployment has failed with backoff limit exceeded, do not reconcile
117-
// even if nodesets or other watched resources change. The deployment is in a
118-
// terminal failure state and should not be retried.
119-
if instance.Status.Conditions != nil {
120-
deploymentCondition := instance.Status.Conditions.Get(condition.DeploymentReadyCondition)
121-
if deploymentCondition != nil &&
122-
deploymentCondition.Severity == condition.SeverityError &&
123-
deploymentCondition.Reason == condition.JobReasonBackoffLimitExceeded {
124-
Log.Info("Deployment has failed with backoff limit exceeded, skipping reconciliation",
125-
"deployment", instance.Name)
126-
return ctrl.Result{}, nil
127-
}
116+
if terminalFailure(instance) {
117+
Log.Info("Deployment is terminal, skipping reconciliation",
118+
"deployment", instance.Name)
119+
return ctrl.Result{}, nil
128120
}
129121

130122
// initialize status if Conditions is nil, but do not reset if it already
@@ -255,7 +247,10 @@ func (r *OpenStackDataPlaneDeploymentReconciler) Reconcile(ctx context.Context,
255247
haveError := false
256248
deploymentErrMsg := ""
257249
var servicePlan *deployment.ServicePlan
258-
backoffLimitReached := false
250+
var finishedNodeSets []dataplanev1.OpenStackDataPlaneNodeSet
251+
if instance.Status.FailedNodeSets == nil {
252+
instance.Status.FailedNodeSets = map[string]string{}
253+
}
259254

260255
if servicePlan, err = deployment.DedupeServices(ctx, helper, nodeSets.Items,
261256
instance.Spec.ServicesOverride); err != nil {
@@ -383,6 +378,21 @@ func (r *OpenStackDataPlaneDeploymentReconciler) Reconcile(ctx context.Context,
383378
for i := range nodeSets.Items {
384379
nodeSet := &nodeSets.Items[i]
385380

381+
if failure, ok := instance.Status.FailedNodeSets[nodeSet.Name]; ok {
382+
Log.Info("NodeSet already terminated by a permanent failure", "nodeSet", nodeSet.Name)
383+
nsConditions := instance.Status.NodeSetConditions[nodeSet.Name]
384+
nsConditions.MarkFalse(
385+
dataplanev1.NodeSetDeploymentReadyCondition,
386+
condition.JobReasonBackoffLimitExceeded,
387+
condition.SeverityError,
388+
"%s", failure)
389+
instance.Status.NodeSetConditions[nodeSet.Name] = nsConditions
390+
haveError = true
391+
deploymentErrMsg = appendDeploymentErrMsg(deploymentErrMsg,
392+
fmt.Sprintf("nodeSet: %s error: %s", nodeSet.Name, failure))
393+
continue
394+
}
395+
386396
Log.Info("Deploying NodeSet", "nodeSet", nodeSet.Name)
387397
Log.Info("Set Status.Deployed to false", "instance", instance)
388398
instance.Status.Deployed = false
@@ -413,14 +423,12 @@ func (r *OpenStackDataPlaneDeploymentReconciler) Reconcile(ctx context.Context,
413423
util.LogErrorForObject(helper, err, fmt.Sprintf("OpenStackDeployment error for NodeSet %s", nodeSet.Name), instance)
414424
Log.Info("Set NodeSetDeploymentReadyCondition false", "nodeSet", nodeSet.Name)
415425
haveError = true
416-
errMsg := fmt.Sprintf("nodeSet: %s error: %s", nodeSet.Name, err.Error())
417-
if len(deploymentErrMsg) == 0 {
418-
deploymentErrMsg = errMsg
419-
} else {
420-
deploymentErrMsg = fmt.Sprintf("%s & %s", deploymentErrMsg, errMsg)
426+
deploymentErrMsg = appendDeploymentErrMsg(deploymentErrMsg,
427+
fmt.Sprintf("nodeSet: %s error: %s", nodeSet.Name, err.Error()))
428+
if nsConditions.Get(dataplanev1.NodeSetDeploymentReadyCondition) != nil &&
429+
nsConditions.Get(dataplanev1.NodeSetDeploymentReadyCondition).Reason == condition.JobReasonBackoffLimitExceeded {
430+
instance.Status.FailedNodeSets[nodeSet.Name] = err.Error()
421431
}
422-
errorReason := nsConditions.Get(dataplanev1.NodeSetDeploymentReadyCondition).Reason
423-
backoffLimitReached = errorReason == condition.JobReasonBackoffLimitExceeded
424432
}
425433

426434
if deployResult != nil {
@@ -431,16 +439,26 @@ func (r *OpenStackDataPlaneDeploymentReconciler) Reconcile(ctx context.Context,
431439
nsConditions.MarkTrue(
432440
dataplanev1.NodeSetDeploymentReadyCondition,
433441
condition.DeploymentReadyMessage)
434-
instance.Status.NodeSetHashes[nodeSet.Name] = nodeSet.Status.ConfigHash
435-
instance.Status.BmhRefHashes[nodeSet.Name] = nodeSet.Status.BmhRefHash
442+
finishedNodeSets = append(finishedNodeSets, *nodeSet)
443+
}
444+
}
445+
446+
// A nodeSet that finished alongside a failure must still get its hashes, or the
447+
// NodeSet controller re-deploys it because its inputs look changed.
448+
if len(finishedNodeSets) > 0 {
449+
err = r.setHashes(ctx, helper, instance,
450+
dataplanev1.OpenStackDataPlaneNodeSetList{Items: finishedNodeSets})
451+
if err != nil {
452+
return ctrl.Result{}, err
436453
}
437454
}
438455

439456
if haveError {
457+
settled := terminalFailure(instance)
440458
var reason condition.Reason
441459
reason = condition.ErrorReason
442460
severity := condition.SeverityWarning
443-
if backoffLimitReached {
461+
if settled {
444462
reason = condition.JobReasonBackoffLimitExceeded
445463
severity = condition.SeverityError
446464
}
@@ -450,7 +468,7 @@ func (r *OpenStackDataPlaneDeploymentReconciler) Reconcile(ctx context.Context,
450468
severity,
451469
condition.DeploymentReadyErrorMessage,
452470
deploymentErrMsg)
453-
if backoffLimitReached {
471+
if settled {
454472
return ctrl.Result{}, nil
455473
}
456474
return ctrl.Result{}, fmt.Errorf("%s", deploymentErrMsg)
@@ -468,10 +486,6 @@ func (r *OpenStackDataPlaneDeploymentReconciler) Reconcile(ctx context.Context,
468486
if version != nil {
469487
instance.Status.DeployedVersion = version.Spec.TargetVersion
470488
}
471-
err = r.setHashes(ctx, helper, instance, *nodeSets)
472-
if err != nil {
473-
Log.Error(err, "Error setting service hashes")
474-
}
475489
return ctrl.Result{}, nil
476490
}
477491

@@ -600,14 +614,19 @@ func (r *OpenStackDataPlaneDeploymentReconciler) setHashes(
600614
}
601615

602616
for _, nodeSet := range nodeSets.Items {
603-
instance.Status.NodeSetHashes[nodeSet.Name] = nodeSet.Status.ConfigHash
604-
605617
err = setNodeSetAnsibleVarsFromHashes(ctx, helper, &nodeSet, instance.Status.ConfigMapHashes, instance.Status.SecretHashes)
606618
if err != nil {
607619
return err
608620
}
609621
}
610622

623+
// Split from the loop above: terminalFailure reads a NodeSetHashes entry as proof
624+
// the nodeSet's input hashes are recorded, so nothing above may fail after it.
625+
for _, nodeSet := range nodeSets.Items {
626+
instance.Status.NodeSetHashes[nodeSet.Name] = nodeSet.Status.ConfigHash
627+
instance.Status.BmhRefHashes[nodeSet.Name] = nodeSet.Status.BmhRefHash
628+
}
629+
611630
return nil
612631
}
613632

@@ -703,6 +722,39 @@ func (r *OpenStackDataPlaneDeploymentReconciler) listNodeSets(ctx context.Contex
703722
return &nodeSets, err
704723
}
705724

725+
// terminalFailure reports whether the deployment has a permanently failed
726+
// nodeSet and nothing left to wait for. A nodeSet still in flight keeps this
727+
// false even when another one already failed. Deployments that failed before
728+
// FailedNodeSets existed carry the failure only on the top-level condition, so
729+
// honor it as well: their backing jobs are garbage collected and must not be
730+
// recreated.
731+
func terminalFailure(instance *dataplanev1.OpenStackDataPlaneDeployment) bool {
732+
if c := instance.Status.Conditions.Get(condition.DeploymentReadyCondition); c != nil &&
733+
c.Severity == condition.SeverityError &&
734+
c.Reason == condition.JobReasonBackoffLimitExceeded {
735+
return true
736+
}
737+
if len(instance.Spec.NodeSets) == 0 || len(instance.Status.FailedNodeSets) == 0 {
738+
return false
739+
}
740+
for _, nodeSet := range instance.Spec.NodeSets {
741+
if _, failed := instance.Status.FailedNodeSets[nodeSet]; failed {
742+
continue
743+
}
744+
if _, done := instance.Status.NodeSetHashes[nodeSet]; !done {
745+
return false
746+
}
747+
}
748+
return true
749+
}
750+
751+
func appendDeploymentErrMsg(current string, errMsg string) string {
752+
if len(current) == 0 {
753+
return errMsg
754+
}
755+
return fmt.Sprintf("%s & %s", current, errMsg)
756+
}
757+
706758
func setNodeSetAnsibleVarsFromHashes(
707759
ctx context.Context,
708760
helper *helper.Helper,

0 commit comments

Comments
 (0)