Skip to content

Commit 9d6a910

Browse files
committed
Gate nodesets on their deployOnAllNodeSets service jobs
A deployOnAllNodeSets service runs as a single AnsibleEE Job for the whole deployment. DedupeServices kept such a service in the first nodeSet's service list only, so every later nodeSet lost its ordering relationship to it: dependsOn resolves against the nodeSet's own list and silently skipped the missing entry. Those nodeSets advanced past the shared Job while it was still mutating their hosts. Keep the service in every listing nodeSet's plan, where it acts as a barrier. The deployment starts its Job before the nodeSet loop, once every service ordered before it in each listing nodeSet has completed, and a nodeSet blocks until that Job succeeded. Readiness is read from the Job rather than from conditions, which are rebuilt every reconcile, so a restart mid-deployment resumes from cluster state. A nodeSet that does not list the service neither waits for it nor delays it. Applies to both serial and parallel execution. Related: OSPRH-37768 Signed-off-by: Rabi Mishra <ramishra@redhat.com>
1 parent b703e9c commit 9d6a910

8 files changed

Lines changed: 663 additions & 123 deletions

File tree

‎docs/assemblies/proc_creating-a-custom-service.adoc‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,9 @@ spec:
9292
+
9393
A mount is created for each `ConfigMap` and `Secret` in the `OpenStackAnsibleEE` pod with a filename that matches the resource value. The mounts are created under `/var/lib/openstack/configs/<service name>`.
9494

95-
. Optional: It may be necessary to run some services on all nodesets at the same time. These services need to have their `deployOnAllNodeSets` field set to true. If these services are repated in multiple nodeset specs included in a deployment, they would be ignored from subsequent nodeset services and would be run only once.
95+
. Optional: It may be necessary to run some services on all nodesets at the same time. These services need to have their `deployOnAllNodeSets` field set to true. Such a service produces a single `OpenStackAnsibleEE` job for the whole deployment, targeting the hosts of every nodeset, even nodesets that do not list the service.
96+
+
97+
A deployment scoped service is an execution barrier for the nodesets that list it: its job starts only once every service ordered before it in each of those nodesets has completed, and none of those nodesets starts a service ordered after it until the job completed. A nodeset that does not list the service neither waits for it nor delays it, so list the service in every nodeset whose deployment ordering depends on it.
9698
+
9799
----
98100
apiVersion: dataplane.openstack.org/v1beta1

‎internal/controller/dataplane/openstackdataplanedeployment_controller.go‎

Lines changed: 113 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121
"context"
2222
"errors"
2323
"fmt"
24+
"slices"
2425
"time"
2526

2627
"github.com/go-playground/validator/v10"
@@ -253,11 +254,10 @@ func (r *OpenStackDataPlaneDeploymentReconciler) Reconcile(ctx context.Context,
253254
shouldRequeue := false
254255
haveError := false
255256
deploymentErrMsg := ""
256-
var nodesetServiceMap map[string][]string
257-
var serviceCache *deployment.ServiceCache
257+
var servicePlan *deployment.ServicePlan
258258
backoffLimitReached := false
259259

260-
if nodesetServiceMap, serviceCache, err = deployment.DedupeServices(ctx, helper, nodeSets.Items,
260+
if servicePlan, err = deployment.DedupeServices(ctx, helper, nodeSets.Items,
261261
instance.Spec.ServicesOverride); err != nil {
262262
util.LogErrorForObject(helper, err, "OpenStackDeployment error for deployment", instance)
263263
var missingServiceErr *deployment.MissingServiceError
@@ -304,22 +304,14 @@ func (r *OpenStackDataPlaneDeploymentReconciler) Reconcile(ctx context.Context,
304304
return ctrl.Result{}, err
305305
}
306306

307-
// Deploy each nodeSet
308-
// The loop starts and checks NodeSet deployments sequentially. However, after they
309-
// are started, they are running in parallel, since the loop does not wait
310-
// for the first started NodeSet to finish before starting the next.
311-
if !instance.Spec.UseParallelExecution {
312-
Log.Info("useParallelExecution is not set to true, services will run sequentially in list order and service dependencies (dependsOn) are ignored")
313-
}
314-
for _, nodeSet := range nodeSets.Items {
307+
// Built up front: the deployment starts its deployment scoped services itself,
308+
// independent of which nodeset reconciles first.
309+
deployers := make(map[string]*deployment.Deployer, len(nodeSets.Items))
310+
serviceLevels := make(map[string][][]string, len(nodeSets.Items))
311+
312+
for i := range nodeSets.Items {
313+
nodeSet := &nodeSets.Items[i]
315314

316-
Log.Info("Deploying NodeSet", "nodeSet", nodeSet.Name)
317-
Log.Info("Set Status.Deployed to false", "instance", instance)
318-
instance.Status.Deployed = false
319-
Log.Info("Set DeploymentReadyCondition false")
320-
instance.Status.Conditions.MarkFalse(
321-
condition.DeploymentReadyCondition, condition.RequestedReason,
322-
condition.SeverityInfo, condition.DeploymentReadyRunningMessage)
323315
ansibleEESpec := nodeSet.GetAnsibleEESpec()
324316
ansibleEESpec.AnsibleTags = instance.Spec.AnsibleTags
325317
ansibleEESpec.AnsibleSkipTags = instance.Spec.AnsibleSkipTags
@@ -334,22 +326,22 @@ func (r *OpenStackDataPlaneDeploymentReconciler) Reconcile(ctx context.Context,
334326
}
335327
}
336328

337-
deployer := deployment.Deployer{
329+
deployers[nodeSet.Name] = &deployment.Deployer{
338330
Ctx: ctx,
339331
Helper: helper,
340-
NodeSet: &nodeSet,
332+
NodeSet: nodeSet,
341333
Deployment: instance,
342334
Status: &instance.Status,
343335
AeeSpec: &ansibleEESpec,
344336
InventorySecrets: globalInventorySecrets,
345337
AnsibleSSHPrivateKeySecrets: globalSSHKeySecrets,
346-
ServiceCache: serviceCache,
338+
ServiceCache: servicePlan.Cache,
347339
Version: version,
348340
}
349341

350-
var serviceLevels [][]string
351342
if instance.Spec.UseParallelExecution {
352-
serviceLevels, err = deployment.BuildServiceLevels(ctx, helper, serviceCache, nodesetServiceMap[nodeSet.Name])
343+
serviceLevels[nodeSet.Name], err = deployment.BuildServiceLevels(
344+
ctx, helper, servicePlan.Cache, servicePlan.Services[nodeSet.Name])
353345
if err != nil {
354346
util.LogErrorForObject(helper, err, fmt.Sprintf("error building service dependency graph for NodeSet %s", nodeSet.Name), instance)
355347
graphErrMsg := fmt.Sprintf("nodeSet: %s error: %s", nodeSet.Name, err.Error())
@@ -370,14 +362,52 @@ func (r *OpenStackDataPlaneDeploymentReconciler) Reconcile(ctx context.Context,
370362
return ctrl.Result{}, fmt.Errorf("error building service dependency graph for nodeset %s: %w", nodeSet.Name, err)
371363
}
372364
} else {
373-
serviceLevels = deployment.SerialServiceLevels(nodesetServiceMap[nodeSet.Name])
365+
serviceLevels[nodeSet.Name] = deployment.SerialServiceLevels(servicePlan.Services[nodeSet.Name])
374366
}
367+
deployers[nodeSet.Name].AllServices = slices.Concat(serviceLevels[nodeSet.Name]...)
368+
}
369+
370+
// Start them only once every listing nodeset reached its barrier, else a
371+
// nodeset advances while the shared Job still runs against its hosts.
372+
if err := r.deployGlobalServices(ctx, helper, instance, servicePlan, serviceLevels, deployers); err != nil {
373+
return ctrl.Result{}, err
374+
}
375+
376+
// Deploy each nodeSet
377+
// The loop starts and checks NodeSet deployments sequentially. However, after they
378+
// are started, they are running in parallel, since the loop does not wait
379+
// for the first started NodeSet to finish before starting the next.
380+
if !instance.Spec.UseParallelExecution {
381+
Log.Info("useParallelExecution is not set to true, services will run sequentially in list order and service dependencies (dependsOn) are ignored")
382+
}
383+
for i := range nodeSets.Items {
384+
nodeSet := &nodeSets.Items[i]
385+
386+
Log.Info("Deploying NodeSet", "nodeSet", nodeSet.Name)
387+
Log.Info("Set Status.Deployed to false", "instance", instance)
388+
instance.Status.Deployed = false
389+
Log.Info("Set DeploymentReadyCondition false")
390+
instance.Status.Conditions.MarkFalse(
391+
condition.DeploymentReadyCondition, condition.RequestedReason,
392+
condition.SeverityInfo, condition.DeploymentReadyRunningMessage)
375393

376394
var deployResult *ctrl.Result
377-
deployResult, err = deployer.Deploy(serviceLevels)
395+
deployResult, err = deployers[nodeSet.Name].Deploy(serviceLevels[nodeSet.Name])
378396

379397
nsConditions := instance.Status.NodeSetConditions[nodeSet.Name]
380398
nsConditions.Set(nsConditions.Mirror(dataplanev1.NodeSetDeploymentReadyCondition))
399+
// A barrier wait sets no service condition, so surface it on the nodeset.
400+
if err == nil {
401+
if blockedOn := deployers[nodeSet.Name].BlockedOn; blockedOn != "" {
402+
nsConditions.Set(condition.FalseCondition(
403+
dataplanev1.NodeSetDeploymentReadyCondition,
404+
condition.RequestedReason,
405+
condition.SeverityInfo,
406+
"%s", fmt.Sprintf("%s: it runs once for the whole deployment",
407+
fmt.Sprintf(dataplanev1.NodeSetServiceDeploymentReadyWaitingMessage, blockedOn))))
408+
}
409+
}
410+
instance.Status.NodeSetConditions[nodeSet.Name] = nsConditions
381411

382412
if err != nil {
383413
util.LogErrorForObject(helper, err, fmt.Sprintf("OpenStackDeployment error for NodeSet %s", nodeSet.Name), instance)
@@ -445,6 +475,64 @@ func (r *OpenStackDataPlaneDeploymentReconciler) Reconcile(ctx context.Context,
445475
return ctrl.Result{}, nil
446476
}
447477

478+
// deployGlobalServices starts the single AnsibleEE Job of every deployment
479+
// scoped service whose barrier has cleared.
480+
func (r *OpenStackDataPlaneDeploymentReconciler) deployGlobalServices(
481+
ctx context.Context,
482+
helper *helper.Helper,
483+
instance *dataplanev1.OpenStackDataPlaneDeployment,
484+
servicePlan *deployment.ServicePlan,
485+
serviceLevels map[string][][]string,
486+
deployers map[string]*deployment.Deployer,
487+
) error {
488+
Log := r.GetLogger(ctx)
489+
jobState := deployment.JobState{}
490+
491+
for _, serviceName := range servicePlan.GlobalServices {
492+
gateOpen, err := deployment.GlobalServiceGateOpen(
493+
ctx, helper, servicePlan, instance, serviceName, serviceLevels, jobState)
494+
if err != nil {
495+
return err
496+
}
497+
if !gateOpen {
498+
Log.Info("Deployment scoped service is behind a barrier that has not cleared",
499+
"service", serviceName, "nodeSets", servicePlan.NodesetsWithGlobalService[serviceName])
500+
continue
501+
}
502+
503+
foundService, err := servicePlan.Cache.Get(ctx, helper, serviceName)
504+
if err != nil {
505+
instance.Status.Conditions.MarkFalse(
506+
condition.InputReadyCondition,
507+
condition.ErrorReason,
508+
condition.SeverityError,
509+
dataplanev1.ServiceErrorMessage,
510+
err.Error())
511+
return err
512+
}
513+
514+
listingNodeSets := servicePlan.NodesetsWithGlobalService[serviceName]
515+
if len(listingNodeSets) == 0 {
516+
continue
517+
}
518+
// Any listing nodeset works; it only supplies the Job labels and mounts.
519+
if err := deployers[listingNodeSets[0]].DeployGlobalService(foundService); err != nil {
520+
util.LogErrorForObject(helper, err,
521+
fmt.Sprintf("Unable to execute deployment scoped service %s", serviceName), instance)
522+
instance.Status.Conditions.MarkFalse(
523+
condition.DeploymentReadyCondition,
524+
condition.ErrorReason,
525+
condition.SeverityError,
526+
condition.DeploymentReadyErrorMessage,
527+
err.Error())
528+
return err
529+
}
530+
Log.Info("Started deployment scoped service", "service", serviceName, "nodeSets", listingNodeSets)
531+
}
532+
533+
return nil
534+
}
535+
448536
// GetService retrieves a service for the OpenStackDataPlaneDeployment
449537
func (r *OpenStackDataPlaneDeploymentReconciler) GetService(
450538
ctx context.Context,

‎internal/dataplane/deployment.go‎

Lines changed: 110 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,12 @@ type Deployer struct {
5757
AnsibleSSHPrivateKeySecrets map[string]string
5858
ServiceCache *ServiceCache
5959
Version *openstackv1.OpenStackVersion
60-
allServices []string
60+
// AllServices is the nodeset's flat service plan, used to resolve cert
61+
// mounts. Set it before any Job is built, including the deployment scoped
62+
// ones the deployment starts outside the nodeset loop.
63+
AllServices []string
64+
// BlockedOn names the barrier a nodeset is stalled on, for its status.
65+
BlockedOn string
6166
}
6267

6368
// Deploy runs the nodeset's leveled service plan, waiting for each level
@@ -67,9 +72,6 @@ func (d *Deployer) Deploy(serviceLevels [][]string) (*ctrl.Result, error) {
6772
d.ServiceCache = NewServiceCache()
6873
}
6974

70-
// Flat list for cert-mount lookups
71-
d.allServices = slices.Concat(serviceLevels...)
72-
7375
for levelIdx, level := range serviceLevels {
7476
d.Helper.GetLogger().Info("Deploying service level", "level", levelIdx, "services", level)
7577

@@ -78,7 +80,11 @@ func (d *Deployer) Deploy(serviceLevels [][]string) (*ctrl.Result, error) {
7880
return &ctrl.Result{}, err
7981
}
8082

81-
if !d.isLevelReady(levelIdx, level) {
83+
ready, err := d.isLevelReady(levelIdx, level)
84+
if err != nil {
85+
return &ctrl.Result{}, err
86+
}
87+
if !ready {
8288
return &ctrl.Result{}, nil
8389
}
8490
}
@@ -112,6 +118,11 @@ func (d *Deployer) startLevel(levelIdx int, level []string) error {
112118
continue
113119
}
114120

121+
// Deployment scoped Jobs are started by the deployment; isLevelReady waits.
122+
if foundService.Spec.DeployOnAllNodeSets {
123+
continue
124+
}
125+
115126
serviceAeeSpec, err := d.buildServiceAeeSpec(foundService)
116127
if err != nil {
117128
d.setServiceError(readyCondition, readyErrorMessage, err)
@@ -136,17 +147,107 @@ func (d *Deployer) startLevel(levelIdx int, level []string) error {
136147
}
137148

138149
// isLevelReady reports whether every service in the level completed.
139-
func (d *Deployer) isLevelReady(levelIdx int, level []string) bool {
150+
func (d *Deployer) isLevelReady(levelIdx int, level []string) (bool, error) {
140151
log := d.Helper.GetLogger()
141152
nsConditions := d.Status.NodeSetConditions[d.NodeSet.Name]
142153
for _, service := range level {
154+
foundService, err := d.ServiceCache.Get(d.Ctx, d.Helper, service)
155+
if err != nil {
156+
return false, err
157+
}
158+
159+
if foundService.Spec.DeployOnAllNodeSets {
160+
completed, err := d.globalServiceCompleted(foundService)
161+
if err != nil {
162+
return false, err
163+
}
164+
if !completed {
165+
d.BlockedOn = service
166+
log.Info("Waiting for deployment scoped service",
167+
"service", service, "level", levelIdx, "nodeSet", d.NodeSet.Name)
168+
return false, nil
169+
}
170+
continue
171+
}
172+
143173
readyCondition := d.serviceReadyCondition(service)
144174
if !nsConditions.IsTrue(readyCondition) {
145175
log.Info("Condition not ready in service level, waiting", "condition", readyCondition, "level", levelIdx)
146-
return false
176+
return false, nil
177+
}
178+
}
179+
return true, nil
180+
}
181+
182+
// globalServiceCompleted reports whether the Job of a deployment scoped service
183+
// finished successfully. A Job past its backoff limit is recorded on the nodeset
184+
// conditions, so it fails the same way a nodeset owned Job would.
185+
func (d *Deployer) globalServiceCompleted(
186+
service dataplanev1.OpenStackDataPlaneService,
187+
) (bool, error) {
188+
log := d.Helper.GetLogger()
189+
190+
ansibleJob, err := dataplaneutil.GetAnsibleExecution(
191+
d.Ctx, d.Helper, d.Deployment,
192+
dataplaneutil.GetGlobalAnsibleExecutionLabels(service.Name, d.Deployment.Name))
193+
if err != nil {
194+
if k8s_errors.IsNotFound(err) {
195+
return false, nil
196+
}
197+
return false, err
198+
}
199+
200+
if ansibleJob.Status.Succeeded > 0 {
201+
d.storeExecutionSummary(ansibleJob)
202+
d.recordContainerImages(service)
203+
return true, nil
204+
}
205+
206+
if ansibleJob.Status.Failed > *ansibleJob.Spec.BackoffLimit {
207+
reason := condition.Reason(condition.ErrorReason)
208+
for _, jobCondition := range ansibleJob.Status.Conditions {
209+
if jobCondition.Type == batchv1.JobFailed {
210+
reason = condition.Reason(jobCondition.Reason)
211+
}
212+
}
213+
d.storeExecutionSummary(ansibleJob)
214+
errorMsg := fmt.Sprintf("execution.name %s execution.namespace %s failed pods: %d",
215+
ansibleJob.Name, ansibleJob.Namespace, ansibleJob.Status.Failed)
216+
if reason == condition.JobReasonBackoffLimitExceeded {
217+
errorMsg = fmt.Sprintf("backoff limit reached for execution.name %s execution.namespace %s",
218+
ansibleJob.Name, ansibleJob.Namespace)
147219
}
220+
log.Info("Deployment scoped service failed", "service", service.Name, "execution", ansibleJob.Name)
221+
222+
nsConditions := d.Status.NodeSetConditions[d.NodeSet.Name]
223+
nsConditions.Set(condition.FalseCondition(
224+
dataplanev1.NodeSetDeploymentReadyCondition,
225+
reason,
226+
condition.SeverityError,
227+
"%s", errorMsg))
228+
d.Status.NodeSetConditions[d.NodeSet.Name] = nsConditions
229+
return false, fmt.Errorf("%s", errorMsg)
148230
}
149-
return true
231+
232+
return false, nil
233+
}
234+
235+
// DeployGlobalService creates the single AnsibleEE Job of a deployment scoped
236+
// service. The bound nodeset only supplies the Job labels and the mounts; the
237+
// play targets every nodeset's hosts.
238+
func (d *Deployer) DeployGlobalService(
239+
service dataplanev1.OpenStackDataPlaneService,
240+
) error {
241+
if !service.Spec.DeployOnAllNodeSets {
242+
return fmt.Errorf("service %s is not deployment scoped", service.Name)
243+
}
244+
245+
serviceAeeSpec, err := d.buildServiceAeeSpec(service)
246+
if err != nil {
247+
return err
248+
}
249+
250+
return d.DeployService(service, serviceAeeSpec)
150251
}
151252

152253
// serviceReadyCondition returns the per-service readiness condition type.
@@ -314,7 +415,7 @@ func (d *Deployer) buildServiceAeeSpec(
314415
}
315416

316417
if foundService.Spec.AddCertMounts {
317-
if err := d.addCertMounts(serviceAeeSpec, d.allServices); err != nil {
418+
if err := d.addCertMounts(serviceAeeSpec, d.AllServices); err != nil {
318419
return nil, err
319420
}
320421
} else if len(foundService.Spec.CACerts) > 0 {

0 commit comments

Comments
 (0)