Skip to content

Commit 266396c

Browse files
Merge pull request #2140 from rabi/multiple_nodesets
Track failure per nodeset
2 parents 3d1d668 + 632d454 commit 266396c

7 files changed

Lines changed: 463 additions & 30 deletions

File tree

‎api/bases/dataplane.openstack.org_openstackdataplanedeployments.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -236,6 +236,14 @@ spec:
236236
deployedVersion:
237237
description: DeployedVersion
238238
type: string
239+
failedNodeSets:
240+
additionalProperties:
241+
type: string
242+
description: |-
243+
FailedNodeSets maps a nodeSet name to the message of the permanent failure
244+
that terminated it. Terminality cannot live in the conditions because the AEE
245+
job backing the failure is garbage collected once the deployment settles.
246+
type: object
239247
nodeSetConditions:
240248
additionalProperties:
241249
description: Conditions provide observations of the operational

‎api/dataplane/v1beta1/openstackdataplanedeployment_types.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,11 @@ type OpenStackDataPlaneDeploymentStatus struct {
112112
// NodeSetConditions
113113
NodeSetConditions map[string]condition.Conditions `json:"nodeSetConditions,omitempty" optional:"true"`
114114

115+
// FailedNodeSets maps a nodeSet name to the message of the permanent failure
116+
// that terminated it. Terminality cannot live in the conditions because the AEE
117+
// job backing the failure is garbage collected once the deployment settles.
118+
FailedNodeSets map[string]string `json:"failedNodeSets,omitempty" optional:"true"`
119+
115120
// AnsibleEEHashes
116121
AnsibleEEHashes map[string]string `json:"ansibleEEHashes,omitempty" optional:"true"`
117122

‎api/dataplane/v1beta1/zz_generated.deepcopy.go‎

Lines changed: 7 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎bindata/crds/crds.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20863,6 +20863,14 @@ spec:
2086320863
deployedVersion:
2086420864
description: DeployedVersion
2086520865
type: string
20866+
failedNodeSets:
20867+
additionalProperties:
20868+
type: string
20869+
description: |-
20870+
FailedNodeSets maps a nodeSet name to the message of the permanent failure
20871+
that terminated it. Terminality cannot live in the conditions because the AEE
20872+
job backing the failure is garbage collected once the deployment settles.
20873+
type: object
2086620874
nodeSetConditions:
2086720875
additionalProperties:
2086820876
description: Conditions provide observations of the operational

‎config/crd/bases/dataplane.openstack.org_openstackdataplanedeployments.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -236,6 +236,14 @@ spec:
236236
deployedVersion:
237237
description: DeployedVersion
238238
type: string
239+
failedNodeSets:
240+
additionalProperties:
241+
type: string
242+
description: |-
243+
FailedNodeSets maps a nodeSet name to the message of the permanent failure
244+
that terminated it. Terminality cannot live in the conditions because the AEE
245+
job backing the failure is garbage collected once the deployment settles.
246+
type: object
239247
nodeSetConditions:
240248
additionalProperties:
241249
description: Conditions provide observations of the operational

‎internal/controller/dataplane/openstackdataplanedeployment_controller.go‎

Lines changed: 82 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -113,18 +113,10 @@ func (r *OpenStackDataPlaneDeploymentReconciler) Reconcile(ctx context.Context,
113113
return ctrl.Result{}, nil
114114
}
115115

116-
// If the deployment has failed with backoff limit exceeded, do not reconcile
117-
// even if nodesets or other watched resources change. The deployment is in a
118-
// terminal failure state and should not be retried.
119-
if instance.Status.Conditions != nil {
120-
deploymentCondition := instance.Status.Conditions.Get(condition.DeploymentReadyCondition)
121-
if deploymentCondition != nil &&
122-
deploymentCondition.Severity == condition.SeverityError &&
123-
deploymentCondition.Reason == condition.JobReasonBackoffLimitExceeded {
124-
Log.Info("Deployment has failed with backoff limit exceeded, skipping reconciliation",
125-
"deployment", instance.Name)
126-
return ctrl.Result{}, nil
127-
}
116+
if terminalFailure(instance) {
117+
Log.Info("Deployment is terminal, skipping reconciliation",
118+
"deployment", instance.Name)
119+
return ctrl.Result{}, nil
128120
}
129121

130122
// initialize status if Conditions is nil, but do not reset if it already
@@ -255,7 +247,10 @@ func (r *OpenStackDataPlaneDeploymentReconciler) Reconcile(ctx context.Context,
255247
haveError := false
256248
deploymentErrMsg := ""
257249
var servicePlan *deployment.ServicePlan
258-
backoffLimitReached := false
250+
var finishedNodeSets []dataplanev1.OpenStackDataPlaneNodeSet
251+
if instance.Status.FailedNodeSets == nil {
252+
instance.Status.FailedNodeSets = map[string]string{}
253+
}
259254

260255
if servicePlan, err = deployment.DedupeServices(ctx, helper, nodeSets.Items,
261256
instance.Spec.ServicesOverride); err != nil {
@@ -383,6 +378,21 @@ func (r *OpenStackDataPlaneDeploymentReconciler) Reconcile(ctx context.Context,
383378
for i := range nodeSets.Items {
384379
nodeSet := &nodeSets.Items[i]
385380

381+
if failure, ok := instance.Status.FailedNodeSets[nodeSet.Name]; ok {
382+
Log.Info("NodeSet already terminated by a permanent failure", "nodeSet", nodeSet.Name)
383+
nsConditions := instance.Status.NodeSetConditions[nodeSet.Name]
384+
nsConditions.MarkFalse(
385+
dataplanev1.NodeSetDeploymentReadyCondition,
386+
condition.JobReasonBackoffLimitExceeded,
387+
condition.SeverityError,
388+
"%s", failure)
389+
instance.Status.NodeSetConditions[nodeSet.Name] = nsConditions
390+
haveError = true
391+
deploymentErrMsg = appendDeploymentErrMsg(deploymentErrMsg,
392+
fmt.Sprintf("nodeSet: %s error: %s", nodeSet.Name, failure))
393+
continue
394+
}
395+
386396
Log.Info("Deploying NodeSet", "nodeSet", nodeSet.Name)
387397
Log.Info("Set Status.Deployed to false", "instance", instance)
388398
instance.Status.Deployed = false
@@ -413,14 +423,12 @@ func (r *OpenStackDataPlaneDeploymentReconciler) Reconcile(ctx context.Context,
413423
util.LogErrorForObject(helper, err, fmt.Sprintf("OpenStackDeployment error for NodeSet %s", nodeSet.Name), instance)
414424
Log.Info("Set NodeSetDeploymentReadyCondition false", "nodeSet", nodeSet.Name)
415425
haveError = true
416-
errMsg := fmt.Sprintf("nodeSet: %s error: %s", nodeSet.Name, err.Error())
417-
if len(deploymentErrMsg) == 0 {
418-
deploymentErrMsg = errMsg
419-
} else {
420-
deploymentErrMsg = fmt.Sprintf("%s & %s", deploymentErrMsg, errMsg)
426+
deploymentErrMsg = appendDeploymentErrMsg(deploymentErrMsg,
427+
fmt.Sprintf("nodeSet: %s error: %s", nodeSet.Name, err.Error()))
428+
if nsConditions.Get(dataplanev1.NodeSetDeploymentReadyCondition) != nil &&
429+
nsConditions.Get(dataplanev1.NodeSetDeploymentReadyCondition).Reason == condition.JobReasonBackoffLimitExceeded {
430+
instance.Status.FailedNodeSets[nodeSet.Name] = err.Error()
421431
}
422-
errorReason := nsConditions.Get(dataplanev1.NodeSetDeploymentReadyCondition).Reason
423-
backoffLimitReached = errorReason == condition.JobReasonBackoffLimitExceeded
424432
}
425433

426434
if deployResult != nil {
@@ -431,16 +439,26 @@ func (r *OpenStackDataPlaneDeploymentReconciler) Reconcile(ctx context.Context,
431439
nsConditions.MarkTrue(
432440
dataplanev1.NodeSetDeploymentReadyCondition,
433441
condition.DeploymentReadyMessage)
434-
instance.Status.NodeSetHashes[nodeSet.Name] = nodeSet.Status.ConfigHash
435-
instance.Status.BmhRefHashes[nodeSet.Name] = nodeSet.Status.BmhRefHash
442+
finishedNodeSets = append(finishedNodeSets, *nodeSet)
443+
}
444+
}
445+
446+
// A nodeSet that finished alongside a failure must still get its hashes, or the
447+
// NodeSet controller re-deploys it because its inputs look changed.
448+
if len(finishedNodeSets) > 0 {
449+
err = r.setHashes(ctx, helper, instance,
450+
dataplanev1.OpenStackDataPlaneNodeSetList{Items: finishedNodeSets})
451+
if err != nil {
452+
return ctrl.Result{}, err
436453
}
437454
}
438455

439456
if haveError {
457+
settled := terminalFailure(instance)
440458
var reason condition.Reason
441459
reason = condition.ErrorReason
442460
severity := condition.SeverityWarning
443-
if backoffLimitReached {
461+
if settled {
444462
reason = condition.JobReasonBackoffLimitExceeded
445463
severity = condition.SeverityError
446464
}
@@ -450,7 +468,7 @@ func (r *OpenStackDataPlaneDeploymentReconciler) Reconcile(ctx context.Context,
450468
severity,
451469
condition.DeploymentReadyErrorMessage,
452470
deploymentErrMsg)
453-
if backoffLimitReached {
471+
if settled {
454472
return ctrl.Result{}, nil
455473
}
456474
return ctrl.Result{}, fmt.Errorf("%s", deploymentErrMsg)
@@ -468,10 +486,6 @@ func (r *OpenStackDataPlaneDeploymentReconciler) Reconcile(ctx context.Context,
468486
if version != nil {
469487
instance.Status.DeployedVersion = version.Spec.TargetVersion
470488
}
471-
err = r.setHashes(ctx, helper, instance, *nodeSets)
472-
if err != nil {
473-
Log.Error(err, "Error setting service hashes")
474-
}
475489
return ctrl.Result{}, nil
476490
}
477491

@@ -600,14 +614,19 @@ func (r *OpenStackDataPlaneDeploymentReconciler) setHashes(
600614
}
601615

602616
for _, nodeSet := range nodeSets.Items {
603-
instance.Status.NodeSetHashes[nodeSet.Name] = nodeSet.Status.ConfigHash
604-
605617
err = setNodeSetAnsibleVarsFromHashes(ctx, helper, &nodeSet, instance.Status.ConfigMapHashes, instance.Status.SecretHashes)
606618
if err != nil {
607619
return err
608620
}
609621
}
610622

623+
// Split from the loop above: terminalFailure reads a NodeSetHashes entry as proof
624+
// the nodeSet's input hashes are recorded, so nothing above may fail after it.
625+
for _, nodeSet := range nodeSets.Items {
626+
instance.Status.NodeSetHashes[nodeSet.Name] = nodeSet.Status.ConfigHash
627+
instance.Status.BmhRefHashes[nodeSet.Name] = nodeSet.Status.BmhRefHash
628+
}
629+
611630
return nil
612631
}
613632

@@ -703,6 +722,39 @@ func (r *OpenStackDataPlaneDeploymentReconciler) listNodeSets(ctx context.Contex
703722
return &nodeSets, err
704723
}
705724

725+
// terminalFailure reports whether the deployment has a permanently failed
726+
// nodeSet and nothing left to wait for. A nodeSet still in flight keeps this
727+
// false even when another one already failed. Deployments that failed before
728+
// FailedNodeSets existed carry the failure only on the top-level condition, so
729+
// honor it as well: their backing jobs are garbage collected and must not be
730+
// recreated.
731+
func terminalFailure(instance *dataplanev1.OpenStackDataPlaneDeployment) bool {
732+
if c := instance.Status.Conditions.Get(condition.DeploymentReadyCondition); c != nil &&
733+
c.Severity == condition.SeverityError &&
734+
c.Reason == condition.JobReasonBackoffLimitExceeded {
735+
return true
736+
}
737+
if len(instance.Spec.NodeSets) == 0 || len(instance.Status.FailedNodeSets) == 0 {
738+
return false
739+
}
740+
for _, nodeSet := range instance.Spec.NodeSets {
741+
if _, failed := instance.Status.FailedNodeSets[nodeSet]; failed {
742+
continue
743+
}
744+
if _, done := instance.Status.NodeSetHashes[nodeSet]; !done {
745+
return false
746+
}
747+
}
748+
return true
749+
}
750+
751+
func appendDeploymentErrMsg(current string, errMsg string) string {
752+
if len(current) == 0 {
753+
return errMsg
754+
}
755+
return fmt.Sprintf("%s & %s", current, errMsg)
756+
}
757+
706758
func setNodeSetAnsibleVarsFromHashes(
707759
ctx context.Context,
708760
helper *helper.Helper,

0 commit comments

Comments
 (0)