Skip to content

Commit 53a2ebc

Browse files
slagleslagle
committed
Configure Ansible EE resources and CPU-based forks
Raises the default ansible forks to 8 if not otherwise set. Add NodeSet EE resource requests and limits and derive the default fork count from CPU allocation while preserving explicit environment overrides. Jira: https://redhat.atlassian.net/browse/OSPRH-18954 Assisted-by: OpenCode GPT-6 Sol Signed-off-by: James Slagle <jslagle@redhat.com>
1 parent d5ac89d commit 53a2ebc

31 files changed

Lines changed: 655 additions & 4 deletions

‎api/bases/dataplane.openstack.org_openstackdataplanenodesets.yaml‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,67 @@ spec:
5959
description: OpenStackDataPlaneNodeSetSpec defines the desired state of
6060
OpenStackDataPlaneNodeSet
6161
properties:
62+
ansibleEEResources:
63+
description: |-
64+
AnsibleEEResources specifies CPU and memory requests and limits for the
65+
main Ansible execution environment Job container. No resources are set by default.
66+
properties:
67+
claims:
68+
description: |-
69+
Claims lists the names of resources, defined in spec.resourceClaims,
70+
that are used by this container.
71+
72+
This is an alpha field and requires enabling the
73+
DynamicResourceAllocation feature gate.
74+
75+
This field is immutable. It can only be set for containers.
76+
items:
77+
description: ResourceClaim references one entry in PodSpec.ResourceClaims.
78+
properties:
79+
name:
80+
description: |-
81+
Name must match the name of one entry in pod.spec.resourceClaims of
82+
the Pod where this field is used. It makes that resource available
83+
inside a container.
84+
type: string
85+
request:
86+
description: |-
87+
Request is the name chosen for a request in the referenced claim.
88+
If empty, everything from the claim is made available, otherwise
89+
only the result of this request.
90+
type: string
91+
required:
92+
- name
93+
type: object
94+
type: array
95+
x-kubernetes-list-map-keys:
96+
- name
97+
x-kubernetes-list-type: map
98+
limits:
99+
additionalProperties:
100+
anyOf:
101+
- type: integer
102+
- type: string
103+
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
104+
x-kubernetes-int-or-string: true
105+
description: |-
106+
Limits describes the maximum amount of compute resources allowed.
107+
More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
108+
type: object
109+
requests:
110+
additionalProperties:
111+
anyOf:
112+
- type: integer
113+
- type: string
114+
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
115+
x-kubernetes-int-or-string: true
116+
description: |-
117+
Requests describes the minimum amount of compute resources required.
118+
If Requests is omitted for a container, it defaults to Limits if that is explicitly specified,
119+
otherwise to an implementation-defined value. Requests cannot exceed Limits.
120+
More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
121+
type: object
122+
type: object
62123
baremetalSetTemplate:
63124
description: BaremetalSetTemplate Template for BaremetalSet for the
64125
NodeSet

‎api/dataplane/v1beta1/common.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -179,6 +179,8 @@ type AnsibleEESpec struct {
179179
ExtraMounts []storage.VolMounts `json:"extraMounts,omitempty"`
180180
// Env is a list containing the environment variables to pass to the pod
181181
Env []corev1.EnvVar `json:"env,omitempty"`
182+
// AnsibleEEResources is applied to the main execution container.
183+
AnsibleEEResources corev1.ResourceRequirements `json:"ansibleEEResources,omitempty"`
182184
// ExtraVars for ansible execution
183185
ExtraVars map[string]json.RawMessage `json:"extraVars,omitempty"`
184186
// DNSConfig for setting dnsservers

‎api/dataplane/v1beta1/openstackdataplanenodeset_types.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,11 @@ type OpenStackDataPlaneNodeSetSpec struct {
5555
// +kubebuilder:validation:Optional
5656
Env []corev1.EnvVar `json:"env,omitempty"`
5757

58+
// AnsibleEEResources specifies CPU and memory requests and limits for the
59+
// main Ansible execution environment Job container. No resources are set by default.
60+
// +kubebuilder:validation:Optional
61+
AnsibleEEResources corev1.ResourceRequirements `json:"ansibleEEResources,omitempty"`
62+
5863
// +kubebuilder:validation:Optional
5964
// NetworkAttachments is a list of NetworkAttachment resource names to pass to the ansibleee resource
6065
// which allows to connect the ansibleee runner to the given network
@@ -216,6 +221,7 @@ func (instance OpenStackDataPlaneNodeSet) GetAnsibleEESpec() AnsibleEESpec {
216221
NetworkAttachments: instance.Spec.NetworkAttachments,
217222
ExtraMounts: instance.Spec.NodeTemplate.ExtraMounts,
218223
Env: instance.Spec.Env,
224+
AnsibleEEResources: instance.Spec.AnsibleEEResources,
219225
ServiceAccountName: instance.Name,
220226
}
221227
}

‎api/dataplane/v1beta1/zz_generated.deepcopy.go‎

Lines changed: 2 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎bindata/crds/crds.yaml‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20994,6 +20994,67 @@ spec:
2099420994
description: OpenStackDataPlaneNodeSetSpec defines the desired state of
2099520995
OpenStackDataPlaneNodeSet
2099620996
properties:
20997+
ansibleEEResources:
20998+
description: |-
20999+
AnsibleEEResources specifies CPU and memory requests and limits for the
21000+
main Ansible execution environment Job container. No resources are set by default.
21001+
properties:
21002+
claims:
21003+
description: |-
21004+
Claims lists the names of resources, defined in spec.resourceClaims,
21005+
that are used by this container.
21006+
21007+
This is an alpha field and requires enabling the
21008+
DynamicResourceAllocation feature gate.
21009+
21010+
This field is immutable. It can only be set for containers.
21011+
items:
21012+
description: ResourceClaim references one entry in PodSpec.ResourceClaims.
21013+
properties:
21014+
name:
21015+
description: |-
21016+
Name must match the name of one entry in pod.spec.resourceClaims of
21017+
the Pod where this field is used. It makes that resource available
21018+
inside a container.
21019+
type: string
21020+
request:
21021+
description: |-
21022+
Request is the name chosen for a request in the referenced claim.
21023+
If empty, everything from the claim is made available, otherwise
21024+
only the result of this request.
21025+
type: string
21026+
required:
21027+
- name
21028+
type: object
21029+
type: array
21030+
x-kubernetes-list-map-keys:
21031+
- name
21032+
x-kubernetes-list-type: map
21033+
limits:
21034+
additionalProperties:
21035+
anyOf:
21036+
- type: integer
21037+
- type: string
21038+
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
21039+
x-kubernetes-int-or-string: true
21040+
description: |-
21041+
Limits describes the maximum amount of compute resources allowed.
21042+
More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
21043+
type: object
21044+
requests:
21045+
additionalProperties:
21046+
anyOf:
21047+
- type: integer
21048+
- type: string
21049+
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
21050+
x-kubernetes-int-or-string: true
21051+
description: |-
21052+
Requests describes the minimum amount of compute resources required.
21053+
If Requests is omitted for a container, it defaults to Limits if that is explicitly specified,
21054+
otherwise to an implementation-defined value. Requests cannot exceed Limits.
21055+
More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
21056+
type: object
21057+
type: object
2099721058
baremetalSetTemplate:
2099821059
description: BaremetalSetTemplate Template for BaremetalSet for the
2099921060
NodeSet

‎config/crd/bases/dataplane.openstack.org_openstackdataplanenodesets.yaml‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,67 @@ spec:
5959
description: OpenStackDataPlaneNodeSetSpec defines the desired state of
6060
OpenStackDataPlaneNodeSet
6161
properties:
62+
ansibleEEResources:
63+
description: |-
64+
AnsibleEEResources specifies CPU and memory requests and limits for the
65+
main Ansible execution environment Job container. No resources are set by default.
66+
properties:
67+
claims:
68+
description: |-
69+
Claims lists the names of resources, defined in spec.resourceClaims,
70+
that are used by this container.
71+
72+
This is an alpha field and requires enabling the
73+
DynamicResourceAllocation feature gate.
74+
75+
This field is immutable. It can only be set for containers.
76+
items:
77+
description: ResourceClaim references one entry in PodSpec.ResourceClaims.
78+
properties:
79+
name:
80+
description: |-
81+
Name must match the name of one entry in pod.spec.resourceClaims of
82+
the Pod where this field is used. It makes that resource available
83+
inside a container.
84+
type: string
85+
request:
86+
description: |-
87+
Request is the name chosen for a request in the referenced claim.
88+
If empty, everything from the claim is made available, otherwise
89+
only the result of this request.
90+
type: string
91+
required:
92+
- name
93+
type: object
94+
type: array
95+
x-kubernetes-list-map-keys:
96+
- name
97+
x-kubernetes-list-type: map
98+
limits:
99+
additionalProperties:
100+
anyOf:
101+
- type: integer
102+
- type: string
103+
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
104+
x-kubernetes-int-or-string: true
105+
description: |-
106+
Limits describes the maximum amount of compute resources allowed.
107+
More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
108+
type: object
109+
requests:
110+
additionalProperties:
111+
anyOf:
112+
- type: integer
113+
- type: string
114+
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
115+
x-kubernetes-int-or-string: true
116+
description: |-
117+
Requests describes the minimum amount of compute resources required.
118+
If Requests is omitted for a container, it defaults to Limits if that is explicitly specified,
119+
otherwise to an implementation-defined value. Requests cannot exceed Limits.
120+
More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
121+
type: object
122+
type: object
62123
baremetalSetTemplate:
63124
description: BaremetalSetTemplate Template for BaremetalSet for the
64125
NodeSet

‎config/samples/dataplane_v1beta1_openstackdataplanenodeset.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,15 @@ metadata:
44
name: openstack-edpm
55
spec:
66
tlsEnabled: true
7+
# Optional: resources for the main Ansible EE Job container.
8+
# 1500m CPU request (below the 2 CPU limit) gives 2 default forks.
9+
ansibleEEResources:
10+
requests:
11+
cpu: 1500m
12+
memory: 512Mi
13+
limits:
14+
cpu: "2"
15+
memory: 2Gi
716
env:
817
- name: ANSIBLE_FORCE_COLOR
918
value: "True"

‎docs/assemblies/dataplane_performance_tuning_large_scale.adoc‎

Lines changed: 31 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -217,7 +217,12 @@ simultaneously.
217217
*Default behavior* (from edpm-ansible playbooks):
218218

219219
* *Strategy*: `linear` (waits for all hosts to complete a task before moving to next task)
220-
* *Forks*: Defaults to 5 (can be overridden with `ANSIBLE_FORKS`)
220+
* *Forks*: The operator sets `ANSIBLE_FORKS` from the EE container CPU request,
221+
capped by a lower CPU limit. With only one CPU setting it uses that setting;
222+
with neither it sets 8. Fractional CPUs round up (250m -> 1, 1500m -> 2).
223+
Memory settings do not affect fork count. This does not create implicit
224+
requests or limits. Kubernetes still requires the CPU request to be no
225+
greater than the CPU limit for a schedulable Job.
221226

222227
== Ansible Performance Tuning
223228

@@ -239,7 +244,7 @@ spec:
239244
- name: ANSIBLE_FORCE_COLOR
240245
value: "True"
241246
242-
# Increase parallel execution (default: 5)
247+
# Override CPU-derived parallelism when needed (fallback without CPU: 8)
243248
# Set based on your control plane resources
244249
- name: ANSIBLE_FORKS
245250
value: "50"
@@ -290,7 +295,7 @@ The `ANSIBLE_FORKS` setting is the most impactful tuning parameter.
290295

291296
*Considerations:*
292297

293-
* *Control plane resources*: More forks require more CPU/memory in the ansible-runner pod
298+
* *EE Job resources*: More forks require more CPU/memory in the ansible-runner pod
294299
* *Network capacity*: More simultaneous SSH connections
295300
* *Target node capacity*: Nodes must handle concurrent configuration tasks
296301

@@ -301,6 +306,29 @@ The `ANSIBLE_FORKS` setting is the most impactful tuning parameter.
301306
* Large deployments (50-100 nodes): `ANSIBLE_FORKS=50-75`
302307
* Very large (100+ nodes): Consider multiple NodeSets instead of very high fork count
303308

309+
[source,yaml]
310+
----
311+
ansibleEEResources:
312+
requests:
313+
cpu: 1500m
314+
memory: 512Mi
315+
limits:
316+
cpu: "2"
317+
memory: 2Gi
318+
# Without an explicit override this produces ANSIBLE_FORKS=2.
319+
# Set resources under OpenStackDataPlaneNodeSet.spec.
320+
----
321+
322+
An explicit `ANSIBLE_FORKS` in NodeSet `spec.env` takes precedence over the
323+
Deployment-selected `spec.ansibleEEEnvConfigMapName` value, which takes
324+
precedence over the computed default. Explicit `-f` / `--forks` supplied to
325+
Ansible by the runner command line takes precedence over the environment.
326+
Even an empty or invalid explicit `ANSIBLE_FORKS` is passed through unchanged;
327+
Ansible determines how to handle it. A missing selected ConfigMap is optional
328+
and does not prevent using the computed default. ConfigMap changes do not
329+
update existing Jobs. Deployments are immutable and completed Jobs are not
330+
rerun; create a new Deployment to apply changed resources or fork settings.
331+
304332
[source,yaml]
305333
----
306334
env:

‎docs/assemblies/dataplane_resources.adoc‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -276,6 +276,11 @@ OpenStackDataPlaneNodeSetSpec defines the desired state of OpenStackDataPlaneNod
276276
| []corev1.EnvVar
277277
| false
278278
279+
| ansibleEEResources
280+
| CPU and memory requests and limits for the main Ansible EE Job container; unset by default. The CPU request capped by a lower limit determines default forks (ceil fractional cores; 8 with no CPU).
281+
| corev1.ResourceRequirements
282+
| false
283+
279284
| networkAttachments
280285
| NetworkAttachments is a list of NetworkAttachment resource names to pass to the ansibleee resource which allows to connect the ansibleee runner to the given network
281286
| []string

‎internal/dataplane/util/ansible_execution.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,11 @@ func AnsibleExecution(
109109

110110
ansibleEE.ExtraMounts = append(aeeSpec.ExtraMounts, []storage.VolMounts{ansibleEEMounts}...)
111111
ansibleEE.Env = aeeSpec.Env
112+
ansibleEE.Resources = *aeeSpec.AnsibleEEResources.DeepCopy()
112113
ansibleEE.NodeSelector = deployment.Spec.AnsibleJobNodeSelector
114+
if err := ansibleEE.addDefaultAnsibleForks(ctx, helper.GetClient()); err != nil {
115+
return err
116+
}
113117

114118
currentJobHash := deployment.Status.AnsibleEEHashes[ansibleEE.Name]
115119
jobDef, err := ansibleEE.JobForOpenStackAnsibleEE(helper)

0 commit comments

Comments
 (0)