Skip to content

Commit 330ae3d

Browse files
committed
Collect dataplane AnsibleEE jobs only when deployment has finished
An OpenStackDataPlaneDeployment re-derives every service outcome from the AnsibleEE Job of that service: the deployment conditions are rebuilt on each reconcile pass, and lib-common does not recreate a Job whose hash is unchanged. With preserveJobs: false the Jobs are deleted ten minutes after they finish, so a deployment still running once that happens waits forever for an execution it can neither find nor re-run, and reports no error. Create the Jobs with cleanup deferred while the deployment is in flight, and apply the cleanup deadline only once the deployment reaches a finished state: deployed, or failed with the backoff limit exceeded. Signed-off-by: Rabi Mishra <ramishra@redhat.com>
1 parent 4d7ca3a commit 330ae3d

8 files changed

Lines changed: 360 additions & 20 deletions

File tree

‎api/bases/dataplane.openstack.org_openstackdataplanedeployments.yaml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,11 @@ spec:
105105
preserveJobs:
106106
default: true
107107
description: |-
108-
PreserveJobs - do not delete jobs after they finished e.g. to check logs
108+
PreserveJobs - do not delete jobs after they finished e.g. to check logs.
109+
When false the jobs are deleted after a delay once the deployment is
110+
finished - deployed, or failed with the backoff limit exceeded - and not
111+
when an individual job finishes. While a deployment is in flight its
112+
jobs are what tells the operator which services already ran.
109113
PreserveJobs default: true
110114
enum:
111115
- true

‎api/dataplane/v1beta1/openstackdataplanedeployment_types.go‎

Lines changed: 41 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,11 @@ type OpenStackDataPlaneDeploymentSpec struct {
3939
// +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors={"urn:alm:descriptor:com.tectonic.ui:number"}
4040
BackoffLimit *int32 `json:"backoffLimit,omitempty"`
4141

42-
// PreserveJobs - do not delete jobs after they finished e.g. to check logs
42+
// PreserveJobs - do not delete jobs after they finished e.g. to check logs.
43+
// When false the jobs are deleted after a delay once the deployment is
44+
// finished - deployed, or failed with the backoff limit exceeded - and not
45+
// when an individual job finishes. While a deployment is in flight its
46+
// jobs are what tells the operator which services already ran.
4347
// PreserveJobs default: true
4448
// +kubebuilder:validation:Enum:=true;false
4549
// +kubebuilder:default:=true
@@ -184,6 +188,42 @@ func (instance OpenStackDataPlaneDeployment) IsReady() bool {
184188
return instance.Status.Conditions.IsTrue(condition.ReadyCondition)
185189
}
186190

191+
// IsFinished reports whether the deployment reached a state it will not advance
192+
// from. The failure verdict is only readable before InitConditions resets the
193+
// conditions.
194+
func (instance *OpenStackDataPlaneDeployment) IsFinished() bool {
195+
return instance.Status.Deployed || instance.isTerminalFailure()
196+
}
197+
198+
// FinishedAt reports when the deployment reached that state, or nil while it can
199+
// still progress. LastTransitionTime only moves when the condition state
200+
// changes, so the value is stable across reconciles.
201+
func (instance *OpenStackDataPlaneDeployment) FinishedAt() *metav1.Time {
202+
if instance.Status.Conditions == nil {
203+
return nil
204+
}
205+
deploymentCondition := instance.Status.Conditions.Get(condition.DeploymentReadyCondition)
206+
if deploymentCondition == nil || deploymentCondition.LastTransitionTime.IsZero() {
207+
return nil
208+
}
209+
if !instance.Status.Deployed && !instance.isTerminalFailure() {
210+
return nil
211+
}
212+
213+
finishedAt := deploymentCondition.LastTransitionTime
214+
return &finishedAt
215+
}
216+
217+
func (instance *OpenStackDataPlaneDeployment) isTerminalFailure() bool {
218+
if instance.Status.Conditions == nil {
219+
return false
220+
}
221+
deploymentCondition := instance.Status.Conditions.Get(condition.DeploymentReadyCondition)
222+
return deploymentCondition != nil &&
223+
deploymentCondition.Severity == condition.SeverityError &&
224+
deploymentCondition.Reason == condition.JobReasonBackoffLimitExceeded
225+
}
226+
187227
// InitConditions - Initializes Status Conditons
188228
func (instance *OpenStackDataPlaneDeployment) InitConditions() {
189229
instance.Status.Conditions = condition.Conditions{}

‎bindata/crds/crds.yaml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20732,7 +20732,11 @@ spec:
2073220732
preserveJobs:
2073320733
default: true
2073420734
description: |-
20735-
PreserveJobs - do not delete jobs after they finished e.g. to check logs
20735+
PreserveJobs - do not delete jobs after they finished e.g. to check logs.
20736+
When false the jobs are deleted after a delay once the deployment is
20737+
finished - deployed, or failed with the backoff limit exceeded - and not
20738+
when an individual job finishes. While a deployment is in flight its
20739+
jobs are what tells the operator which services already ran.
2073620740
PreserveJobs default: true
2073720741
enum:
2073820742
- true

‎config/crd/bases/dataplane.openstack.org_openstackdataplanedeployments.yaml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,11 @@ spec:
105105
preserveJobs:
106106
default: true
107107
description: |-
108-
PreserveJobs - do not delete jobs after they finished e.g. to check logs
108+
PreserveJobs - do not delete jobs after they finished e.g. to check logs.
109+
When false the jobs are deleted after a delay once the deployment is
110+
finished - deployed, or failed with the backoff limit exceeded - and not
111+
when an individual job finishes. While a deployment is in flight its
112+
jobs are what tells the operator which services already ran.
109113
PreserveJobs default: true
110114
enum:
111115
- true

‎docs/assemblies/dataplane_resources.adoc‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ OpenStackDataPlaneDeploymentSpec defines the desired state of OpenStackDataPlane
9393
| false
9494
9595
| preserveJobs
96-
| PreserveJobs - do not delete jobs after they finished e.g. to check logs PreserveJobs default: true
96+
| PreserveJobs - do not delete jobs after they finished e.g. to check logs. When false the jobs are deleted after a delay once the deployment is finished - deployed, or failed with the backoff limit exceeded - and not when an individual job finishes. While a deployment is in flight its jobs are what tells the operator which services already ran. PreserveJobs default: true
9797
| bool
9898
| false
9999

‎internal/controller/dataplane/openstackdataplanedeployment_controller.go‎

Lines changed: 116 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121
"context"
2222
"errors"
2323
"fmt"
24+
"math"
2425
"slices"
2526
"time"
2627

@@ -29,6 +30,7 @@ import (
2930
batchv1 "k8s.io/api/batch/v1"
3031
corev1 "k8s.io/api/core/v1"
3132
k8s_errors "k8s.io/apimachinery/pkg/api/errors"
33+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
3234
"k8s.io/apimachinery/pkg/runtime"
3335
"k8s.io/apimachinery/pkg/types"
3436
"k8s.io/client-go/kubernetes"
@@ -107,24 +109,34 @@ func (r *OpenStackDataPlaneDeploymentReconciler) Reconcile(ctx context.Context,
107109
Log,
108110
)
109111

112+
// Whether Job cleanup may be scheduled: a verdict freshly computed by this
113+
// reconcile is only trustworthy once the status patch below persists it.
114+
finishedOnEntry := instance.IsFinished()
115+
statusPersisted := false
116+
117+
// Registered ahead of the status patch defer so it runs after the verdict persists.
118+
defer func() {
119+
if err := r.reconcileJobTTL(ctx, helper, instance, finishedOnEntry || statusPersisted); err != nil {
120+
Log.Error(err, "Failed to reconcile AnsibleEE job cleanup", "deployment", instance.Name)
121+
if _err == nil {
122+
_err = err
123+
}
124+
}
125+
}()
126+
110127
// If the deploy is already done, return immediately.
111128
if instance.Status.Deployed {
112129
Log.Info("Already deployed", "instance.Status.Deployed", instance.Status.Deployed)
113130
return ctrl.Result{}, nil
114131
}
115132

116133
// If the deployment has failed with backoff limit exceeded, do not reconcile
117-
// even if nodesets or other watched resources change. The deployment is in a
118-
// terminal failure state and should not be retried.
119-
if instance.Status.Conditions != nil {
120-
deploymentCondition := instance.Status.Conditions.Get(condition.DeploymentReadyCondition)
121-
if deploymentCondition != nil &&
122-
deploymentCondition.Severity == condition.SeverityError &&
123-
deploymentCondition.Reason == condition.JobReasonBackoffLimitExceeded {
124-
Log.Info("Deployment has failed with backoff limit exceeded, skipping reconciliation",
125-
"deployment", instance.Name)
126-
return ctrl.Result{}, nil
127-
}
134+
// even if nodesets or other watched resources change. The deployment has
135+
// failed and should not be retried.
136+
if instance.IsFinished() {
137+
Log.Info("Deployment has failed with backoff limit exceeded, skipping reconciliation",
138+
"deployment", instance.Name)
139+
return ctrl.Result{}, nil
128140
}
129141

130142
// initialize status if Conditions is nil, but do not reset if it already
@@ -170,6 +182,7 @@ func (r *OpenStackDataPlaneDeploymentReconciler) Reconcile(ctx context.Context,
170182
_err = err
171183
return
172184
}
185+
statusPersisted = true
173186
}()
174187

175188
// Ensure NodeSets
@@ -533,6 +546,98 @@ func (r *OpenStackDataPlaneDeploymentReconciler) deployGlobalServices(
533546
return nil
534547
}
535548

549+
// Mirrors the delay lib-common applies to finished Jobs.
550+
const jobCleanupDelaySeconds int32 = 10 * 60
551+
552+
// reconcileJobTTL defers Job cleanup until the deployment is finished: while it
553+
// runs the Jobs are the only record of which services already ran. scheduleCleanup
554+
// is false while the terminal verdict exists only in memory.
555+
func (r *OpenStackDataPlaneDeploymentReconciler) reconcileJobTTL(
556+
ctx context.Context,
557+
helper *helper.Helper,
558+
instance *dataplanev1.OpenStackDataPlaneDeployment,
559+
scheduleCleanup bool,
560+
) error {
561+
if instance.Spec.PreserveJobs {
562+
return nil
563+
}
564+
565+
jobList := &batchv1.JobList{}
566+
err := helper.GetClient().List(
567+
ctx,
568+
jobList,
569+
client.InNamespace(instance.Namespace),
570+
client.MatchingLabels{dataplaneutil.AnsibleEEDeploymentLabel: instance.Name},
571+
)
572+
if err != nil {
573+
return err
574+
}
575+
576+
finished := scheduleCleanup && instance.IsFinished()
577+
finishedAt := instance.FinishedAt()
578+
579+
for i := range jobList.Items {
580+
job := &jobList.Items[i]
581+
if finished {
582+
// Ageing starts at the Job's own completion time, so one schedule per Job
583+
// is enough and the value must not be re-derived afterwards.
584+
if job.Spec.TTLSecondsAfterFinished != nil {
585+
continue
586+
}
587+
patch := client.MergeFrom(job.DeepCopy())
588+
ttl := jobCleanupDelay(job, finishedAt)
589+
job.Spec.TTLSecondsAfterFinished = &ttl
590+
591+
err := helper.GetClient().Patch(ctx, job, patch)
592+
if err != nil && !k8s_errors.IsNotFound(err) {
593+
return err
594+
}
595+
continue
596+
}
597+
598+
// An older operator may have left a deadline on a running deployment.
599+
if job.Spec.TTLSecondsAfterFinished == nil {
600+
continue
601+
}
602+
patch := client.MergeFrom(job.DeepCopy())
603+
job.Spec.TTLSecondsAfterFinished = nil
604+
605+
err := helper.GetClient().Patch(ctx, job, patch)
606+
if err != nil && !k8s_errors.IsNotFound(err) {
607+
return err
608+
}
609+
}
610+
return nil
611+
}
612+
613+
// jobCleanupDelay returns the TTLSecondsAfterFinished that makes the Job expire
614+
// jobCleanupDelaySeconds after the deployment finished. Ageing starts at the
615+
// Job's own completion time, so a Job that finished while the deployment still
616+
// ran gets that elapsed time added back - a flat delay would delete it at once.
617+
func jobCleanupDelay(job *batchv1.Job, finishedAt *metav1.Time) int32 {
618+
finishTime := jobFinishTime(job)
619+
if finishedAt == nil || finishTime == nil || !finishedAt.After(finishTime.Time) {
620+
return jobCleanupDelaySeconds
621+
}
622+
623+
delay := int64(jobCleanupDelaySeconds) + int64(math.Floor(finishedAt.Time.Sub(finishTime.Time).Seconds()))
624+
return int32(min(delay, math.MaxInt32))
625+
}
626+
627+
// jobFinishTime mirrors the TTL-after-finished controller, which measures from
628+
// the Complete/Failed transition rather than CompletionTime.
629+
func jobFinishTime(job *batchv1.Job) *metav1.Time {
630+
for _, jobCondition := range job.Status.Conditions {
631+
isFinished := (jobCondition.Type == batchv1.JobComplete || jobCondition.Type == batchv1.JobFailed) &&
632+
jobCondition.Status == corev1.ConditionTrue
633+
if isFinished && !jobCondition.LastTransitionTime.IsZero() {
634+
finishTime := jobCondition.LastTransitionTime
635+
return &finishTime
636+
}
637+
}
638+
return job.Status.CompletionTime
639+
}
640+
536641
// GetService retrieves a service for the OpenStackDataPlaneDeployment
537642
func (r *OpenStackDataPlaneDeploymentReconciler) GetService(
538643
ctx context.Context,

‎internal/dataplane/util/ansible_execution.go‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -243,6 +243,13 @@ func ParseAnsibleExecutionSummaryFromPod(
243243
return nil, nil
244244
}
245245

246+
// Labels the AnsibleEE Jobs carry, to build the Job spec and find them again.
247+
const (
248+
AnsibleEEServiceLabel = "openstackdataplaneservice"
249+
AnsibleEEDeploymentLabel = "openstackdataplanedeployment"
250+
AnsibleEENodeSetLabel = "openstackdataplanenodeset"
251+
)
252+
246253
// GetAnsibleExecutionNameAndLabels Name and Labels of AnsibleEE
247254
func GetAnsibleExecutionNameAndLabels(service *dataplanev1.OpenStackDataPlaneService,
248255
deploymentName string,
@@ -263,7 +270,7 @@ func GetAnsibleExecutionNameAndLabels(service *dataplanev1.OpenStackDataPlaneSer
263270
}
264271

265272
labels := GetGlobalAnsibleExecutionLabels(service.Name, deploymentName)
266-
labels["openstackdataplanenodeset"] = nodeSetName
273+
labels[AnsibleEENodeSetLabel] = nodeSetName
267274
return executionName, labels
268275
}
269276

@@ -272,8 +279,8 @@ func GetAnsibleExecutionNameAndLabels(service *dataplanev1.OpenStackDataPlaneSer
272279
// selector must not carry the nodeset label.
273280
func GetGlobalAnsibleExecutionLabels(serviceName string, deploymentName string) map[string]string {
274281
return map[string]string{
275-
"openstackdataplaneservice": serviceName,
276-
"openstackdataplanedeployment": deploymentName,
282+
AnsibleEEServiceLabel: serviceName,
283+
AnsibleEEDeploymentLabel: deploymentName,
277284
}
278285
}
279286

@@ -302,7 +309,9 @@ func (a *EEJob) BuildAeeJobSpec(
302309
}
303310

304311
a.BackoffLimit = deployment.Spec.BackoffLimit
305-
a.PreserveJobs = deployment.Spec.PreserveJobs
312+
// Cleanup must not start while the deployment still runs: lib-common never
313+
// recreates a Job whose hash is unchanged.
314+
a.PreserveJobs = deployment.Spec.PreserveJobs || !deployment.IsFinished()
306315
a.FormatAEECmdLineArguments(aeeSpec)
307316
a.FormatAEEExtraVars(aeeSpec, service, deployment, nodeSet)
308317
a.DetermineAeeImage(aeeSpec)

0 commit comments

Comments
 (0)