Skip to content

Commit 4a77d00

Browse files
Merge pull request #6 from gitcommitankit/phase-3
feat: implement prometheus-based autoscaling logic and custom metrics…
2 parents f9a16f1 + 5ef8d02 commit 4a77d00

27 files changed

Lines changed: 3291 additions & 106 deletions

‎api/v1alpha1/agentdeployment_types.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -234,6 +234,7 @@ type AgentDeploymentList struct {
234234
Items []AgentDeployment `json:"items"`
235235
}
236236

237+
// init registers AgentDeployment and AgentDeploymentList types with the SchemeBuilder.
237238
func init() {
238239
SchemeBuilder.Register(&AgentDeployment{}, &AgentDeploymentList{})
239240
}

‎api/v1alpha1/error_rate_test.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import (
2222
agentraxv1alpha1 "github.com/gitcommitankit/agentrax/api/v1alpha1"
2323
)
2424

25+
// TestParseErrorRate verifies percentage string parsing across valid, invalid, and boundary inputs.
2526
func TestParseErrorRate(t *testing.T) {
2627
t.Parallel()
2728
tests := []struct {
@@ -62,6 +63,7 @@ func TestParseErrorRate(t *testing.T) {
6263
}
6364
}
6465

66+
// abs returns the absolute value of a float64.
6567
func abs(f float64) float64 {
6668
if f < 0 {
6769
return -f

‎api/v1alpha1/tenantquota_types.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,7 @@ type TenantQuotaList struct {
8686
Items []TenantQuota `json:"items"`
8787
}
8888

89+
// init registers TenantQuota and TenantQuotaList types with the SchemeBuilder.
8990
func init() {
9091
SchemeBuilder.Register(&TenantQuota{}, &TenantQuotaList{})
9192
}

‎cmd/main.go‎

Lines changed: 29 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,8 @@ import (
2626
// to ensure that exec-entrypoint and run can make use of them.
2727
_ "k8s.io/client-go/plugin/pkg/client/auth"
2828

29+
autoscalingv2 "k8s.io/api/autoscaling/v2"
30+
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
2931
"k8s.io/apimachinery/pkg/runtime"
3032
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
3133
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
@@ -36,6 +38,8 @@ import (
3638
metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
3739
"sigs.k8s.io/controller-runtime/pkg/webhook"
3840

41+
monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
42+
3943
agentraxv1alpha1 "github.com/gitcommitankit/agentrax/api/v1alpha1"
4044
"github.com/gitcommitankit/agentrax/internal/controller"
4145
"github.com/gitcommitankit/agentrax/internal/quota"
@@ -48,13 +52,18 @@ var (
4852
setupLog = ctrl.Log.WithName("setup")
4953
)
5054

55+
// init registers all Kubernetes core, CRD, and monitoring schemes.
5156
func init() {
5257
utilruntime.Must(clientgoscheme.AddToScheme(scheme))
58+
utilruntime.Must(autoscalingv2.AddToScheme(scheme))
59+
utilruntime.Must(apiextensionsv1.AddToScheme(scheme))
60+
utilruntime.Must(monitoringv1.AddToScheme(scheme))
5361

5462
utilruntime.Must(agentraxv1alpha1.AddToScheme(scheme))
5563
// +kubebuilder:scaffold:scheme
5664
}
5765

66+
// main is the entrypoint for the Agentrax controller manager binary.
5867
func main() {
5968
var metricsAddr string
6069
var enableLeaderElection bool
@@ -98,9 +107,18 @@ func main() {
98107
tlsOpts = append(tlsOpts, disableHTTP2)
99108
}
100109

101-
webhookServer := webhook.NewServer(webhook.Options{
102-
TLSOpts: tlsOpts,
103-
})
110+
// Resolve the webhook-enabled flag once so both the server creation and
111+
// handler registration use the same value. Log it explicitly so operators
112+
// can confirm the resolved state at startup.
113+
enableWebhooks := os.Getenv("ENABLE_WEBHOOKS") != "false"
114+
setupLog.Info("webhook state resolved", "enabled", enableWebhooks)
115+
116+
var webhookServer webhook.Server
117+
if enableWebhooks {
118+
webhookServer = webhook.NewServer(webhook.Options{
119+
TLSOpts: tlsOpts,
120+
})
121+
}
104122

105123
// Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server.
106124
// More info:
@@ -154,8 +172,9 @@ func main() {
154172
quotaEnforcer := quota.NewEnforcer(gpuResourceName)
155173

156174
if err = (&controller.AgentDeploymentReconciler{
157-
Client: mgr.GetClient(),
158-
Scheme: mgr.GetScheme(),
175+
Client: mgr.GetClient(),
176+
Scheme: mgr.GetScheme(),
177+
GPUResourceName: gpuResourceName,
159178
}).SetupWithManager(mgr); err != nil {
160179
setupLog.Error(err, "unable to create controller", "controller", "AgentDeployment")
161180
os.Exit(1)
@@ -168,9 +187,11 @@ func main() {
168187
setupLog.Error(err, "unable to create controller", "controller", "TenantQuota")
169188
os.Exit(1)
170189
}
171-
if err = agentraxwebhook.SetupAgentDeploymentWebhookWithManager(mgr, quotaEnforcer); err != nil {
172-
setupLog.Error(err, "unable to register webhook", "webhook", "AgentDeployment")
173-
os.Exit(1)
190+
if enableWebhooks {
191+
if err = agentraxwebhook.SetupAgentDeploymentWebhookWithManager(mgr, quotaEnforcer); err != nil {
192+
setupLog.Error(err, "unable to register webhook", "webhook", "AgentDeployment")
193+
os.Exit(1)
194+
}
174195
}
175196
// +kubebuilder:scaffold:builder
176197

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
# Prometheus Adapter custom metrics configuration for Agentrax.
2+
#
3+
# This ConfigMap is consumed by the prometheus-adapter deployment (typically in
4+
# the monitoring namespace). It maps PromQL queries to named custom metrics that
5+
# the HorizontalPodAutoscaler can target via the custom.metrics.k8s.io API.
6+
#
7+
# Deploy with:
8+
# kubectl apply -f config/prometheus-adapter/custom-metrics-config.yaml
9+
# kubectl rollout restart deployment/prometheus-adapter -n monitoring
10+
#
11+
# Verify metrics are registered:
12+
# kubectl get --raw /apis/external.metrics.k8s.io/v1beta1 | jq .
13+
#
14+
apiVersion: v1
15+
kind: ConfigMap
16+
metadata:
17+
# Use a distinct name so this ConfigMap does not collide with or overwrite
18+
# the upstream prometheus-adapter ConfigMap (commonly named adapter-config).
19+
# Reference this name in your prometheus-adapter Deployment via
20+
# --config=/etc/adapter/config.yaml mounted from this ConfigMap.
21+
name: agentrax-custom-metrics
22+
namespace: monitoring
23+
labels:
24+
app.kubernetes.io/name: prometheus-adapter
25+
app.kubernetes.io/managed-by: agentrax
26+
data:
27+
config.yaml: |
28+
externalRules:
29+
# ── queueDepth ────────────────────────────────────────────────────────────
30+
# Exposes agentrax_queue_depth via external.metrics.k8s.io, which is the
31+
# API group queried by HPAs using ExternalMetricSourceType (the type
32+
# BuildHPA produces). The <<.LabelMatchers>> template is populated by the
33+
# Prometheus Adapter from the HPA metric selector. Prometheus sanitizes
34+
# label names (app.kubernetes.io/name → app_kubernetes_io_name), so the
35+
# query references the sanitized forms that actually exist in storage.
36+
- seriesQuery: 'agentrax_queue_depth{namespace!=""}'
37+
name:
38+
matches: "^agentrax_queue_depth$"
39+
as: "agentrax_queue_depth"
40+
metricsQuery: 'sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>)'
41+
42+
# ── gpuUtilization ────────────────────────────────────────────────────────
43+
# Exposes agentrax_gpu_utilization via external.metrics.k8s.io.
44+
# If your GPU device plugin exposes a different metric name (e.g., from DCGM),
45+
# update the seriesQuery and the as: name here; the HPA target in the
46+
# AgentDeployment spec references the as: name, which stays stable.
47+
- seriesQuery: 'agentrax_gpu_utilization{namespace!=""}'
48+
name:
49+
matches: "^agentrax_gpu_utilization$"
50+
as: "agentrax_gpu_utilization"
51+
metricsQuery: 'sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>)'
Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
apiVersion: kustomize.config.k8s.io/v1beta1
2+
kind: Kustomization
3+
4+
# Prometheus Adapter custom metrics configuration for Agentrax.
5+
#
6+
# Apply this overlay after the Prometheus Adapter base is installed:
7+
# kubectl apply -k config/prometheus-adapter/
8+
#
9+
# Note: This kustomization targets the monitoring namespace where the
10+
# prometheus-adapter deployment is expected to run. If your cluster uses
11+
# a different namespace, update the namespace field below.
12+
namespace: monitoring
13+
14+
resources:
15+
- custom-metrics-config.yaml
16+
17+
# To apply this configuration, either:
18+
# 1. Include the prometheus-adapter Deployment base in resources above, then
19+
# uncomment the patches section below, OR
20+
# 2. Manually configure your existing prometheus-adapter Deployment to mount
21+
# this ConfigMap at /etc/adapter/config.yaml and pass --config=/etc/adapter/config.yaml
22+
#
23+
# patches:
24+
# - target:
25+
# kind: Deployment
26+
# name: prometheus-adapter
27+
# patch: |-
28+
# - op: add
29+
# path: /spec/template/spec/volumes/-
30+
# value:
31+
# name: adapter-config
32+
# configMap:
33+
# name: agentrax-custom-metrics
34+
# - op: add
35+
# path: /spec/template/spec/containers/0/volumeMounts/-
36+
# value:
37+
# name: adapter-config
38+
# mountPath: /etc/adapter
39+
# - op: add
40+
# path: /spec/template/spec/containers/0/args/-
41+
# value: --config=/etc/adapter/config.yaml

‎config/rbac/role.yaml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,18 @@ rules:
5959
- patch
6060
- update
6161
- watch
62+
- apiGroups:
63+
- autoscaling
64+
resources:
65+
- horizontalpodautoscalers
66+
verbs:
67+
- create
68+
- delete
69+
- get
70+
- list
71+
- patch
72+
- update
73+
- watch
6274
- apiGroups:
6375
- ""
6476
resources:

‎config/samples/agentrax_v1alpha1_agentdeployment.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,10 @@ spec:
3131
maxTotalReplicas: 10
3232
# Maximum number of distinct AgentDeployments allowed.
3333
maxAgents: 5
34+
# Maximum GPU count allowed across all agents in this namespace.
35+
maxGPUs: 2
36+
# Maximum replica count allowed for any single AgentDeployment.
37+
maxReplicasPerAgent: 5
3438

3539
---
3640
# 3. AgentDeployment — the main resource.

‎config/webhook/manifests.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,3 +50,4 @@ webhooks:
5050
resources:
5151
- agentdeployments
5252
sideEffects: None
53+
timeoutSeconds: 10

‎internal/controller/agentdeployment_builder_test.go‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ func makeAD(name, image string, port int32, minReplicas int32) *agentraxv1alpha1
6262

6363
// ── desiredDeployment ─────────────────────────────────────────────────────────
6464

65+
// TestDesiredDeployment_Image verifies that the desired Deployment container image matches spec.image.
6566
func TestDesiredDeployment_Image(t *testing.T) {
6667
r := &AgentDeploymentReconciler{}
6768
ad := makeAD("my-agent", "registry.io/agent:v1", 8080, 1)
@@ -72,6 +73,7 @@ func TestDesiredDeployment_Image(t *testing.T) {
7273
}
7374
}
7475

76+
// TestDesiredDeployment_Port verifies container port configuration and default fallback.
7577
func TestDesiredDeployment_Port(t *testing.T) {
7678
tests := []struct {
7779
name string
@@ -94,6 +96,7 @@ func TestDesiredDeployment_Port(t *testing.T) {
9496
}
9597
}
9698

99+
// TestDesiredDeployment_Replicas verifies that desired Deployment replicas match spec.replicas.min.
97100
func TestDesiredDeployment_Replicas(t *testing.T) {
98101
r := &AgentDeploymentReconciler{}
99102
ad := makeAD(testDefaultAgent, testDefaultImage, 8080, 3)
@@ -104,6 +107,7 @@ func TestDesiredDeployment_Replicas(t *testing.T) {
104107
}
105108
}
106109

110+
// TestDesiredDeployment_EnvAndArgs verifies propagation of environment variables and container arguments.
107111
func TestDesiredDeployment_EnvAndArgs(t *testing.T) {
108112
r := &AgentDeploymentReconciler{}
109113
ad := makeAD(testDefaultAgent, testDefaultImage, 8080, 1)
@@ -121,6 +125,7 @@ func TestDesiredDeployment_EnvAndArgs(t *testing.T) {
121125
}
122126
}
123127

128+
// TestDesiredDeployment_Resources verifies container CPU and Memory resource requests/limits propagation.
124129
func TestDesiredDeployment_Resources(t *testing.T) {
125130
r := &AgentDeploymentReconciler{}
126131
ad := makeAD(testDefaultAgent, testDefaultImage, 8080, 1)
@@ -153,6 +158,7 @@ func TestDesiredDeployment_Resources(t *testing.T) {
153158
}
154159
}
155160

161+
// TestDesiredDeployment_Labels verifies required standard labels on Deployment and Pod template.
156162
func TestDesiredDeployment_Labels(t *testing.T) {
157163
r := &AgentDeploymentReconciler{}
158164
ad := makeAD(testAgentName, testDefaultImage, 8080, 1)
@@ -173,6 +179,7 @@ func TestDesiredDeployment_Labels(t *testing.T) {
173179
}
174180
}
175181

182+
// TestDesiredDeployment_SelectorMatchesPodLabels verifies Deployment selector matches Pod template labels.
176183
func TestDesiredDeployment_SelectorMatchesPodLabels(t *testing.T) {
177184
r := &AgentDeploymentReconciler{}
178185
ad := makeAD(testDefaultAgent, testDefaultImage, 8080, 1)
@@ -187,6 +194,7 @@ func TestDesiredDeployment_SelectorMatchesPodLabels(t *testing.T) {
187194

188195
// ── desiredService ────────────────────────────────────────────────────────────
189196

197+
// TestDesiredService_Port verifies Service port configuration and default fallback.
190198
func TestDesiredService_Port(t *testing.T) {
191199
tests := []struct {
192200
name string
@@ -208,6 +216,7 @@ func TestDesiredService_Port(t *testing.T) {
208216
}
209217
}
210218

219+
// TestDesiredService_Selector verifies Service selector targets the agent pod label.
211220
func TestDesiredService_Selector(t *testing.T) {
212221
r := &AgentDeploymentReconciler{}
213222
ad := makeAD(testAgentName, testDefaultImage, 8080, 1)
@@ -218,6 +227,7 @@ func TestDesiredService_Selector(t *testing.T) {
218227
}
219228
}
220229

230+
// TestDesiredService_ClusterIPType verifies that the created Service is of type ClusterIP.
221231
func TestDesiredService_ClusterIPType(t *testing.T) {
222232
r := &AgentDeploymentReconciler{}
223233
ad := makeAD(testDefaultAgent, testDefaultImage, 8080, 1)
@@ -230,6 +240,7 @@ func TestDesiredService_ClusterIPType(t *testing.T) {
230240

231241
// ── agentLabels ───────────────────────────────────────────────────────────────
232242

243+
// TestAgentLabels verifies standard label generation for an AgentDeployment.
233244
func TestAgentLabels(t *testing.T) {
234245
ad := &agentraxv1alpha1.AgentDeployment{
235246
ObjectMeta: metav1.ObjectMeta{Name: "foo"},
@@ -251,6 +262,7 @@ func TestAgentLabels(t *testing.T) {
251262

252263
// ── condition helpers ─────────────────────────────────────────────────────────
253264

265+
// TestSetAndGetCondition verifies setting and reading status conditions on AgentDeployment.
254266
func TestSetAndGetCondition(t *testing.T) {
255267
ad := &agentraxv1alpha1.AgentDeployment{}
256268

@@ -268,6 +280,7 @@ func TestSetAndGetCondition(t *testing.T) {
268280
}
269281
}
270282

283+
// TestSetCondition_Overwrite verifies that updating an existing condition updates in-place without duplicates.
271284
func TestSetCondition_Overwrite(t *testing.T) {
272285
ad := &agentraxv1alpha1.AgentDeployment{}
273286

@@ -284,6 +297,7 @@ func TestSetCondition_Overwrite(t *testing.T) {
284297
}
285298
}
286299

300+
// TestRemoveCondition verifies condition removal from the status condition slice.
287301
func TestRemoveCondition(t *testing.T) {
288302
ad := &agentraxv1alpha1.AgentDeployment{}
289303

@@ -300,6 +314,7 @@ func TestRemoveCondition(t *testing.T) {
300314
}
301315
}
302316

317+
// TestRemoveCondition_NonExistent verifies that removing a non-existent condition is a safe no-op.
303318
func TestRemoveCondition_NonExistent(t *testing.T) {
304319
ad := &agentraxv1alpha1.AgentDeployment{}
305320
// Should be a no-op, not panic.
@@ -309,6 +324,7 @@ func TestRemoveCondition_NonExistent(t *testing.T) {
309324
}
310325
}
311326

327+
// TestGetCondition_Absent verifies that querying an un-set condition returns nil.
312328
func TestGetCondition_Absent(t *testing.T) {
313329
ad := &agentraxv1alpha1.AgentDeployment{}
314330
c := GetCondition(ad, agentraxv1alpha1.ConditionReady)
@@ -319,6 +335,7 @@ func TestGetCondition_Absent(t *testing.T) {
319335

320336
// ── desiredServiceMonitor ─────────────────────────────────────────────────────
321337

338+
// TestDesiredServiceMonitor_Endpoint verifies ServiceMonitor metrics endpoint configuration.
322339
func TestDesiredServiceMonitor_Endpoint(t *testing.T) {
323340
r := &AgentDeploymentReconciler{}
324341
ad := makeAD(testDefaultAgent, testDefaultImage, 8080, 1)
@@ -336,6 +353,7 @@ func TestDesiredServiceMonitor_Endpoint(t *testing.T) {
336353
}
337354
}
338355

356+
// TestDesiredServiceMonitor_SelectorMatchesLabels verifies ServiceMonitor selector matches agent labels.
339357
func TestDesiredServiceMonitor_SelectorMatchesLabels(t *testing.T) {
340358
r := &AgentDeploymentReconciler{}
341359
ad := makeAD(testAgentName, testDefaultImage, 8080, 1)

0 commit comments

Comments
 (0)