Skip to content

Commit 1303f3e

Browse files
committed
Collect dataplane AnsibleEE jobs only when deployment has finished
An OpenStackDataPlaneDeployment re-derives every service outcome from the AnsibleEE Job of that service: the deployment conditions are rebuilt on each reconcile pass, and lib-common does not recreate a Job whose hash is unchanged. With preserveJobs: false the Jobs are deleted ten minutes after they finish, so a deployment still running once that happens waits forever for an execution it can neither find nor re-run, and reports no error. Create the Jobs with cleanup deferred while the deployment is in flight, and apply the cleanup deadline only once the deployment reaches a finished state: deployed, or failed with the backoff limit exceeded. Signed-off-by: Rabi Mishra <ramishra@redhat.com>
1 parent b703e9c commit 1303f3e

8 files changed

Lines changed: 360 additions & 20 deletions

File tree

‎api/bases/dataplane.openstack.org_openstackdataplanedeployments.yaml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,11 @@ spec:
105105
preserveJobs:
106106
default: true
107107
description: |-
108-
PreserveJobs - do not delete jobs after they finished e.g. to check logs
108+
PreserveJobs - do not delete jobs after they finished e.g. to check logs.
109+
When false the jobs are deleted after a delay once the deployment is
110+
finished - deployed, or failed with the backoff limit exceeded - and not
111+
when an individual job finishes. While a deployment is in flight its
112+
jobs are what tells the operator which services already ran.
109113
PreserveJobs default: true
110114
enum:
111115
- true

‎api/dataplane/v1beta1/openstackdataplanedeployment_types.go‎

Lines changed: 41 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,11 @@ type OpenStackDataPlaneDeploymentSpec struct {
3939
// +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors={"urn:alm:descriptor:com.tectonic.ui:number"}
4040
BackoffLimit *int32 `json:"backoffLimit,omitempty"`
4141

42-
// PreserveJobs - do not delete jobs after they finished e.g. to check logs
42+
// PreserveJobs - do not delete jobs after they finished e.g. to check logs.
43+
// When false the jobs are deleted after a delay once the deployment is
44+
// finished - deployed, or failed with the backoff limit exceeded - and not
45+
// when an individual job finishes. While a deployment is in flight its
46+
// jobs are what tells the operator which services already ran.
4347
// PreserveJobs default: true
4448
// +kubebuilder:validation:Enum:=true;false
4549
// +kubebuilder:default:=true
@@ -184,6 +188,42 @@ func (instance OpenStackDataPlaneDeployment) IsReady() bool {
184188
return instance.Status.Conditions.IsTrue(condition.ReadyCondition)
185189
}
186190

191+
// IsFinished reports whether the deployment reached a state it will not advance
192+
// from. The failure verdict is only readable before InitConditions resets the
193+
// conditions.
194+
func (instance *OpenStackDataPlaneDeployment) IsFinished() bool {
195+
return instance.Status.Deployed || instance.isTerminalFailure()
196+
}
197+
198+
// FinishedAt reports when the deployment reached that state, or nil while it can
199+
// still progress. LastTransitionTime only moves when the condition state
200+
// changes, so the value is stable across reconciles.
201+
func (instance *OpenStackDataPlaneDeployment) FinishedAt() *metav1.Time {
202+
if instance.Status.Conditions == nil {
203+
return nil
204+
}
205+
deploymentCondition := instance.Status.Conditions.Get(condition.DeploymentReadyCondition)
206+
if deploymentCondition == nil || deploymentCondition.LastTransitionTime.IsZero() {
207+
return nil
208+
}
209+
if !instance.Status.Deployed && !instance.isTerminalFailure() {
210+
return nil
211+
}
212+
213+
finishedAt := deploymentCondition.LastTransitionTime
214+
return &finishedAt
215+
}
216+
217+
func (instance *OpenStackDataPlaneDeployment) isTerminalFailure() bool {
218+
if instance.Status.Conditions == nil {
219+
return false
220+
}
221+
deploymentCondition := instance.Status.Conditions.Get(condition.DeploymentReadyCondition)
222+
return deploymentCondition != nil &&
223+
deploymentCondition.Severity == condition.SeverityError &&
224+
deploymentCondition.Reason == condition.JobReasonBackoffLimitExceeded
225+
}
226+
187227
// InitConditions - Initializes Status Conditons
188228
func (instance *OpenStackDataPlaneDeployment) InitConditions() {
189229
instance.Status.Conditions = condition.Conditions{}

‎bindata/crds/crds.yaml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20732,7 +20732,11 @@ spec:
2073220732
preserveJobs:
2073320733
default: true
2073420734
description: |-
20735-
PreserveJobs - do not delete jobs after they finished e.g. to check logs
20735+
PreserveJobs - do not delete jobs after they finished e.g. to check logs.
20736+
When false the jobs are deleted after a delay once the deployment is
20737+
finished - deployed, or failed with the backoff limit exceeded - and not
20738+
when an individual job finishes. While a deployment is in flight its
20739+
jobs are what tells the operator which services already ran.
2073620740
PreserveJobs default: true
2073720741
enum:
2073820742
- true

‎config/crd/bases/dataplane.openstack.org_openstackdataplanedeployments.yaml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,11 @@ spec:
105105
preserveJobs:
106106
default: true
107107
description: |-
108-
PreserveJobs - do not delete jobs after they finished e.g. to check logs
108+
PreserveJobs - do not delete jobs after they finished e.g. to check logs.
109+
When false the jobs are deleted after a delay once the deployment is
110+
finished - deployed, or failed with the backoff limit exceeded - and not
111+
when an individual job finishes. While a deployment is in flight its
112+
jobs are what tells the operator which services already ran.
109113
PreserveJobs default: true
110114
enum:
111115
- true

‎docs/assemblies/dataplane_resources.adoc‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ OpenStackDataPlaneDeploymentSpec defines the desired state of OpenStackDataPlane
9393
| false
9494
9595
| preserveJobs
96-
| PreserveJobs - do not delete jobs after they finished e.g. to check logs PreserveJobs default: true
96+
| PreserveJobs - do not delete jobs after they finished e.g. to check logs. When false the jobs are deleted after a delay once the deployment is finished - deployed, or failed with the backoff limit exceeded - and not when an individual job finishes. While a deployment is in flight its jobs are what tells the operator which services already ran. PreserveJobs default: true
9797
| bool
9898
| false
9999

‎internal/controller/dataplane/openstackdataplanedeployment_controller.go‎

Lines changed: 116 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,15 @@ import (
2121
"context"
2222
"errors"
2323
"fmt"
24+
"math"
2425
"time"
2526

2627
"github.com/go-playground/validator/v10"
2728
"github.com/iancoleman/strcase"
2829
batchv1 "k8s.io/api/batch/v1"
2930
corev1 "k8s.io/api/core/v1"
3031
k8s_errors "k8s.io/apimachinery/pkg/api/errors"
32+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
3133
"k8s.io/apimachinery/pkg/runtime"
3234
"k8s.io/apimachinery/pkg/types"
3335
"k8s.io/client-go/kubernetes"
@@ -106,24 +108,34 @@ func (r *OpenStackDataPlaneDeploymentReconciler) Reconcile(ctx context.Context,
106108
Log,
107109
)
108110

111+
// Whether Job cleanup may be scheduled: a verdict freshly computed by this
112+
// reconcile is only trustworthy once the status patch below persists it.
113+
finishedOnEntry := instance.IsFinished()
114+
statusPersisted := false
115+
116+
// Registered ahead of the status patch defer so it runs after the verdict persists.
117+
defer func() {
118+
if err := r.reconcileJobTTL(ctx, helper, instance, finishedOnEntry || statusPersisted); err != nil {
119+
Log.Error(err, "Failed to reconcile AnsibleEE job cleanup", "deployment", instance.Name)
120+
if _err == nil {
121+
_err = err
122+
}
123+
}
124+
}()
125+
109126
// If the deploy is already done, return immediately.
110127
if instance.Status.Deployed {
111128
Log.Info("Already deployed", "instance.Status.Deployed", instance.Status.Deployed)
112129
return ctrl.Result{}, nil
113130
}
114131

115132
// If the deployment has failed with backoff limit exceeded, do not reconcile
116-
// even if nodesets or other watched resources change. The deployment is in a
117-
// terminal failure state and should not be retried.
118-
if instance.Status.Conditions != nil {
119-
deploymentCondition := instance.Status.Conditions.Get(condition.DeploymentReadyCondition)
120-
if deploymentCondition != nil &&
121-
deploymentCondition.Severity == condition.SeverityError &&
122-
deploymentCondition.Reason == condition.JobReasonBackoffLimitExceeded {
123-
Log.Info("Deployment has failed with backoff limit exceeded, skipping reconciliation",
124-
"deployment", instance.Name)
125-
return ctrl.Result{}, nil
126-
}
133+
// even if nodesets or other watched resources change. The deployment has
134+
// failed and should not be retried.
135+
if instance.IsFinished() {
136+
Log.Info("Deployment has failed with backoff limit exceeded, skipping reconciliation",
137+
"deployment", instance.Name)
138+
return ctrl.Result{}, nil
127139
}
128140

129141
// initialize status if Conditions is nil, but do not reset if it already
@@ -169,6 +181,7 @@ func (r *OpenStackDataPlaneDeploymentReconciler) Reconcile(ctx context.Context,
169181
_err = err
170182
return
171183
}
184+
statusPersisted = true
172185
}()
173186

174187
// Ensure NodeSets
@@ -445,6 +458,98 @@ func (r *OpenStackDataPlaneDeploymentReconciler) Reconcile(ctx context.Context,
445458
return ctrl.Result{}, nil
446459
}
447460

461+
// Mirrors the delay lib-common applies to finished Jobs.
462+
const jobCleanupDelaySeconds int32 = 10 * 60
463+
464+
// reconcileJobTTL defers Job cleanup until the deployment is finished: while it
465+
// runs the Jobs are the only record of which services already ran. scheduleCleanup
466+
// is false while the terminal verdict exists only in memory.
467+
func (r *OpenStackDataPlaneDeploymentReconciler) reconcileJobTTL(
468+
ctx context.Context,
469+
helper *helper.Helper,
470+
instance *dataplanev1.OpenStackDataPlaneDeployment,
471+
scheduleCleanup bool,
472+
) error {
473+
if instance.Spec.PreserveJobs {
474+
return nil
475+
}
476+
477+
jobList := &batchv1.JobList{}
478+
err := helper.GetClient().List(
479+
ctx,
480+
jobList,
481+
client.InNamespace(instance.Namespace),
482+
client.MatchingLabels{dataplaneutil.AnsibleEEDeploymentLabel: instance.Name},
483+
)
484+
if err != nil {
485+
return err
486+
}
487+
488+
finished := scheduleCleanup && instance.IsFinished()
489+
finishedAt := instance.FinishedAt()
490+
491+
for i := range jobList.Items {
492+
job := &jobList.Items[i]
493+
if finished {
494+
// Ageing starts at the Job's own completion time, so one schedule per Job
495+
// is enough and the value must not be re-derived afterwards.
496+
if job.Spec.TTLSecondsAfterFinished != nil {
497+
continue
498+
}
499+
patch := client.MergeFrom(job.DeepCopy())
500+
ttl := jobCleanupDelay(job, finishedAt)
501+
job.Spec.TTLSecondsAfterFinished = &ttl
502+
503+
err := helper.GetClient().Patch(ctx, job, patch)
504+
if err != nil && !k8s_errors.IsNotFound(err) {
505+
return err
506+
}
507+
continue
508+
}
509+
510+
// An older operator may have left a deadline on a running deployment.
511+
if job.Spec.TTLSecondsAfterFinished == nil {
512+
continue
513+
}
514+
patch := client.MergeFrom(job.DeepCopy())
515+
job.Spec.TTLSecondsAfterFinished = nil
516+
517+
err := helper.GetClient().Patch(ctx, job, patch)
518+
if err != nil && !k8s_errors.IsNotFound(err) {
519+
return err
520+
}
521+
}
522+
return nil
523+
}
524+
525+
// jobCleanupDelay returns the TTLSecondsAfterFinished that makes the Job expire
526+
// jobCleanupDelaySeconds after the deployment finished. Ageing starts at the
527+
// Job's own completion time, so a Job that finished while the deployment still
528+
// ran gets that elapsed time added back - a flat delay would delete it at once.
529+
func jobCleanupDelay(job *batchv1.Job, finishedAt *metav1.Time) int32 {
530+
finishTime := jobFinishTime(job)
531+
if finishedAt == nil || finishTime == nil || !finishedAt.After(finishTime.Time) {
532+
return jobCleanupDelaySeconds
533+
}
534+
535+
delay := int64(jobCleanupDelaySeconds) + int64(math.Floor(finishedAt.Time.Sub(finishTime.Time).Seconds()))
536+
return int32(min(delay, math.MaxInt32))
537+
}
538+
539+
// jobFinishTime mirrors the TTL-after-finished controller, which measures from
540+
// the Complete/Failed transition rather than CompletionTime.
541+
func jobFinishTime(job *batchv1.Job) *metav1.Time {
542+
for _, jobCondition := range job.Status.Conditions {
543+
isFinished := (jobCondition.Type == batchv1.JobComplete || jobCondition.Type == batchv1.JobFailed) &&
544+
jobCondition.Status == corev1.ConditionTrue
545+
if isFinished && !jobCondition.LastTransitionTime.IsZero() {
546+
finishTime := jobCondition.LastTransitionTime
547+
return &finishTime
548+
}
549+
}
550+
return job.Status.CompletionTime
551+
}
552+
448553
// GetService retrieves a service for the OpenStackDataPlaneDeployment
449554
func (r *OpenStackDataPlaneDeploymentReconciler) GetService(
450555
ctx context.Context,

‎internal/dataplane/util/ansible_execution.go‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -243,6 +243,13 @@ func ParseAnsibleExecutionSummaryFromPod(
243243
return nil, nil
244244
}
245245

246+
// Labels the AnsibleEE Jobs carry, to build the Job spec and find them again.
247+
const (
248+
AnsibleEEServiceLabel = "openstackdataplaneservice"
249+
AnsibleEEDeploymentLabel = "openstackdataplanedeployment"
250+
AnsibleEENodeSetLabel = "openstackdataplanenodeset"
251+
)
252+
246253
// GetAnsibleExecutionNameAndLabels Name and Labels of AnsibleEE
247254
func GetAnsibleExecutionNameAndLabels(service *dataplanev1.OpenStackDataPlaneService,
248255
deploymentName string,
@@ -263,9 +270,9 @@ func GetAnsibleExecutionNameAndLabels(service *dataplanev1.OpenStackDataPlaneSer
263270
}
264271

265272
labels := map[string]string{
266-
"openstackdataplaneservice": service.Name,
267-
"openstackdataplanedeployment": deploymentName,
268-
"openstackdataplanenodeset": nodeSetName,
273+
AnsibleEEServiceLabel: service.Name,
274+
AnsibleEEDeploymentLabel: deploymentName,
275+
AnsibleEENodeSetLabel: nodeSetName,
269276
}
270277
return executionName, labels
271278
}
@@ -295,7 +302,9 @@ func (a *EEJob) BuildAeeJobSpec(
295302
}
296303

297304
a.BackoffLimit = deployment.Spec.BackoffLimit
298-
a.PreserveJobs = deployment.Spec.PreserveJobs
305+
// Cleanup must not start while the deployment still runs: lib-common never
306+
// recreates a Job whose hash is unchanged.
307+
a.PreserveJobs = deployment.Spec.PreserveJobs || !deployment.IsFinished()
299308
a.FormatAEECmdLineArguments(aeeSpec)
300309
a.FormatAEEExtraVars(aeeSpec, service, deployment, nodeSet)
301310
a.DetermineAeeImage(aeeSpec)

0 commit comments

Comments
 (0)