Skip to content

Commit edf788a

Browse files
gitcommitankitgitcommitankit
authored
Merge pull request #15 from gitcommitankit/phase-5
feat: implement prometheus metrics for reconcile duration and quota u…
2 parents fdddfd0 + d5e554b commit edf788a

11 files changed

Lines changed: 428 additions & 2 deletions

File tree

‎config/grafana/dashboard-configmap.yaml‎

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
apiVersion: v1
2+
kind: ConfigMap
3+
metadata:
4+
name: agentrax-grafana-dashboard
5+
namespace: agentrax-system
6+
labels:
7+
app.kubernetes.io/name: agentrax
8+
app.kubernetes.io/component: grafana-dashboard
9+
# kube-prometheus-stack's Grafana sidecar watches for this label and
10+
# auto-imports the JSON payload below as a Grafana dashboard.
11+
grafana_dashboard: "1"
12+
data:
13+
agentrax-dashboard.json: |
14+
{
15+
"title": "Agentrax Operator — RED Dashboard",
16+
"uid": "agentrax-red",
17+
"tags": ["agentrax", "operator", "kubernetes"],
18+
"timezone": "browser",
19+
"schemaVersion": 38,
20+
"refresh": "30s",
21+
"templating": {
22+
"list": [
23+
{
24+
"name": "tenant",
25+
"label": "Tenant",
26+
"type": "query",
27+
"datasource": "Prometheus",
28+
"query": "label_values(agentrax_tenant_quota_usage_ratio, tenant)",
29+
"includeAll": true,
30+
"allValue": ".*",
31+
"refresh": 1,
32+
"multi": true,
33+
"current": { "text": "All", "value": "$__all" }
34+
}
35+
]
36+
},
37+
"panels": [
38+
{
39+
"id": 1,
40+
"title": "Rate — Reconcile Operations / sec",
41+
"type": "timeseries",
42+
"gridPos": { "x": 0, "y": 0, "w": 12, "h": 8 },
43+
"datasource": "Prometheus",
44+
"targets": [
45+
{
46+
"expr": "sum(rate(agentrax_reconcile_duration_seconds_count{tenant=~\"$tenant\"}[5m])) by (tenant)",
47+
"legendFormat": "{{ tenant }}"
48+
}
49+
],
50+
"fieldConfig": {
51+
"defaults": {
52+
"unit": "reqps",
53+
"custom": { "lineWidth": 2 }
54+
}
55+
}
56+
},
57+
{
58+
"id": 2,
59+
"title": "Errors — Reconcile Error Rate / sec",
60+
"type": "timeseries",
61+
"gridPos": { "x": 12, "y": 0, "w": 12, "h": 8 },
62+
"datasource": "Prometheus",
63+
"targets": [
64+
{
65+
"expr": "sum(rate(controller_runtime_reconcile_errors_total{controller=\"agentdeployment\"}[5m]))",
66+
"legendFormat": "errors/s"
67+
}
68+
],
69+
"fieldConfig": {
70+
"defaults": {
71+
"unit": "reqps",
72+
"color": { "fixedColor": "red", "mode": "fixed" },
73+
"custom": { "lineWidth": 2 }
74+
}
75+
}
76+
},
77+
{
78+
"id": 3,
79+
"title": "Duration — Reconcile P99 Latency (seconds)",
80+
"type": "timeseries",
81+
"gridPos": { "x": 0, "y": 8, "w": 12, "h": 8 },
82+
"datasource": "Prometheus",
83+
"targets": [
84+
{
85+
"expr": "histogram_quantile(0.99, sum(rate(agentrax_reconcile_duration_seconds_bucket{tenant=~\"$tenant\"}[5m])) by (le, tenant))",
86+
"legendFormat": "P99 {{ tenant }}"
87+
},
88+
{
89+
"expr": "histogram_quantile(0.50, sum(rate(agentrax_reconcile_duration_seconds_bucket{tenant=~\"$tenant\"}[5m])) by (le, tenant))",
90+
"legendFormat": "P50 {{ tenant }}"
91+
}
92+
],
93+
"fieldConfig": {
94+
"defaults": {
95+
"unit": "s",
96+
"custom": { "lineWidth": 2 },
97+
"thresholds": {
98+
"mode": "absolute",
99+
"steps": [
100+
{ "color": "green", "value": null },
101+
{ "color": "yellow", "value": 1 },
102+
{ "color": "red", "value": 2 }
103+
]
104+
}
105+
}
106+
}
107+
},
108+
{
109+
"id": 4,
110+
"title": "Quota — Tenant Replica Usage Ratio",
111+
"type": "gauge",
112+
"gridPos": { "x": 12, "y": 8, "w": 12, "h": 8 },
113+
"datasource": "Prometheus",
114+
"targets": [
115+
{
116+
"expr": "agentrax_tenant_quota_usage_ratio{tenant=~\"$tenant\"}",
117+
"legendFormat": "{{ tenant }}"
118+
}
119+
],
120+
"fieldConfig": {
121+
"defaults": {
122+
"unit": "percentunit",
123+
"min": 0,
124+
"max": 1,
125+
"thresholds": {
126+
"mode": "absolute",
127+
"steps": [
128+
{ "color": "green", "value": null },
129+
{ "color": "yellow", "value": 0.7 },
130+
{ "color": "red", "value": 0.9 }
131+
]
132+
}
133+
}
134+
}
135+
}
136+
]
137+
}

‎config/prometheus/alerting-rules.yaml‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
apiVersion: monitoring.coreos.com/v1
2+
kind: PrometheusRule
3+
metadata:
4+
name: agentrax-alerts
5+
namespace: agentrax-system
6+
labels:
7+
app.kubernetes.io/name: agentrax
8+
app.kubernetes.io/component: alerting
9+
# kube-prometheus-stack discovers PrometheusRule resources by this label.
10+
release: kube-prometheus-stack
11+
spec:
12+
groups:
13+
14+
# -------------------------------------------------------------------------
15+
# Reconcile latency — fires when the P99 reconcile time sustained > 2s.
16+
# This indicates the operator is struggling to converge: slow API server,
17+
# contention on the TenantQuota lock, or an unhealthy admission webhook.
18+
# -------------------------------------------------------------------------
19+
- name: agentrax.reconcile
20+
rules:
21+
- alert: AgentraxReconcileLatencyHigh
22+
expr: |
23+
histogram_quantile(0.99,
24+
sum(rate(agentrax_reconcile_duration_seconds_bucket[5m]))
25+
by (le, controller, tenant)
26+
) > 2
27+
for: 5m
28+
labels:
29+
severity: critical
30+
annotations:
31+
summary: >-
32+
Reconcile P99 > 2s for controller={{ $labels.controller }}
33+
tenant={{ $labels.tenant }}
34+
description: >-
35+
The AgentDeployment reconciler P99 latency has exceeded 2 seconds
36+
for 5 minutes. Check kube-apiserver latency, webhook response times,
37+
and operator logs for slow database or registry calls.
38+
runbook_url: "https://github.com/gitcommitankit/agentrax/docs/ARCHITECTURE.md"
39+
40+
# -------------------------------------------------------------------------
41+
# Quota saturation — fires when a tenant's replica usage ratio exceeds 90%.
42+
# Early warning before the hard cap blocks new AgentDeployment creates.
43+
# -------------------------------------------------------------------------
44+
- name: agentrax.quota
45+
rules:
46+
- alert: AgentraxTenantQuotaHigh
47+
expr: agentrax_tenant_quota_usage_ratio > 0.9
48+
for: 2m
49+
labels:
50+
severity: warning
51+
annotations:
52+
summary: >-
53+
Tenant {{ $labels.tenant }} replica quota usage > 90%
54+
description: >-
55+
Tenant {{ $labels.tenant }} is using more than 90% of its
56+
maxTotalReplicas quota. New AgentDeployment creates will be
57+
rejected by the admission webhook when the limit is reached.
58+
Consider raising maxTotalReplicas in the TenantQuota spec.
59+
runbook_url: "https://github.com/gitcommitankit/agentrax/docs/ARCHITECTURE.md"

‎config/prometheus/kustomization.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,3 @@
11
resources:
22
- monitor.yaml
3+
- alerting-rules.yaml

‎config/prometheus/monitor.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ metadata:
66
control-plane: controller-manager
77
app.kubernetes.io/name: agentrax
88
app.kubernetes.io/managed-by: kustomize
9+
release: kube-prometheus-stack
910
name: controller-manager-metrics-monitor
1011
namespace: system
1112
spec:

‎docs/ARCHITECTURE.md‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -435,6 +435,40 @@ The `.github/workflows/terraform-lint.yml` workflow runs on every PR touching `i
435435

436436
---
437437

438+
### 4.10 Custom Prometheus Metrics, Alerting, & Grafana Dashboard
439+
440+
Agentrax exports two custom metrics via `internal/observability/metrics.go`, both registered against the controller-runtime shared Prometheus registry (exposed on `/metrics`):
441+
442+
| Metric | Type | Labels | What it measures |
443+
| :--- | :--- | :--- | :--- |
444+
| `agentrax_reconcile_duration_seconds` | Histogram | `controller`, `tenant` | Wall-clock duration of every `AgentDeployment` reconcile loop |
445+
| `agentrax_tenant_quota_usage_ratio` | Gauge | `tenant` | `usedTotalReplicas / maxTotalReplicas` — approaches 1.0 at the quota ceiling |
446+
447+
#### Instrumentation Points
448+
449+
- **`internal/controller/agentdeployment_controller.go`**: A deferred closure at the top of `Reconcile()` records the histogram observation on every exit path — including early returns, errors, and requeues — so no reconcile path is missed.
450+
- **`internal/controller/tenantquota_controller.go`**: The quota gauge is updated after `ComputeUsage()` runs, guarded against divide-by-zero when `MaxTotalReplicas` is zero.
451+
452+
#### Alerting Rules (`config/prometheus/alerting-rules.yaml`)
453+
454+
A `PrometheusRule` CRD with two alert groups, auto-discovered by the Prometheus Operator:
455+
456+
| Alert | Expression | Duration | Severity |
457+
| :--- | :--- | :--- | :--- |
458+
| `AgentraxReconcileLatencyHigh` | P99 reconcile latency > 2s | 5 min | critical |
459+
| `AgentraxTenantQuotaHigh` | Quota usage ratio > 0.9 | 2 min | warning |
460+
461+
#### Grafana RED Dashboard (`config/grafana/dashboard-configmap.yaml`)
462+
463+
A `ConfigMap` labelled `grafana_dashboard: "1"` auto-imported by the `kube-prometheus-stack` Grafana sidecar. Contains 4 panels:
464+
465+
- **Rate**: `rate(agentrax_reconcile_duration_seconds_count[5m])` per tenant
466+
- **Errors**: `rate(controller_runtime_reconcile_errors_total[5m])` for the `agentdeployment` controller
467+
- **Duration**: P99 and P50 latency time-series with red/yellow/green threshold colouring at 2s/1s
468+
- **Quota**: Gauge panel per tenant with a `$tenant` template variable; turns red above 90%
469+
470+
---
471+
438472
## 5. Architectural Decision Records (ADRs) & Trade-Offs
439473

440474
| Decision | Alternative Considered | Trade-Off & Rationale for Agentrax |

‎go.mod‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,9 @@ require (
77
github.com/onsi/ginkgo/v2 v2.19.0
88
github.com/onsi/gomega v1.33.1
99
github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.75.0
10+
github.com/prometheus/client_golang v1.19.1
11+
github.com/prometheus/client_model v0.6.1
12+
github.com/stretchr/testify v1.10.0
1013
k8s.io/api v0.31.0
1114
k8s.io/apiextensions-apiserver v0.31.0
1215
k8s.io/apimachinery v0.31.0
@@ -53,8 +56,7 @@ require (
5356
github.com/modern-go/reflect2 v1.0.2 // indirect
5457
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
5558
github.com/pkg/errors v0.9.1 // indirect
56-
github.com/prometheus/client_golang v1.19.1 // indirect
57-
github.com/prometheus/client_model v0.6.1 // indirect
59+
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
5860
github.com/prometheus/common v0.55.0 // indirect
5961
github.com/prometheus/procfs v0.15.1 // indirect
6062
github.com/spf13/cobra v1.8.1 // indirect

‎internal/controller/agentdeployment_controller.go‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ import (
4343
apimeta "k8s.io/apimachinery/pkg/api/meta"
4444

4545
agentraxv1alpha1 "github.com/gitcommitankit/agentrax/api/v1alpha1"
46+
"github.com/gitcommitankit/agentrax/internal/observability"
4647
"github.com/gitcommitankit/agentrax/internal/registry"
4748
"github.com/gitcommitankit/agentrax/internal/rollout"
4849
"github.com/gitcommitankit/agentrax/internal/scaling"
@@ -126,6 +127,16 @@ type AgentDeploymentReconciler struct {
126127
func (r *AgentDeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
127128
logger := log.FromContext(ctx)
128129

130+
// Observe reconcile wall-clock duration on every exit path, including early
131+
// returns, errors, and requeues. The tenant label uses the request namespace
132+
// because each namespace maps 1:1 to a tenant in Agentrax.
133+
start := time.Now()
134+
defer func() {
135+
observability.ReconcileDuration.
136+
WithLabelValues("agentdeployment", req.Namespace).
137+
Observe(time.Since(start).Seconds())
138+
}()
139+
129140
// 1. Fetch the AgentDeployment; return immediately if it has been deleted.
130141
ad := &agentraxv1alpha1.AgentDeployment{}
131142
if err := r.Get(ctx, req.NamespacedName, ad); err != nil {

‎internal/controller/tenantquota_controller.go‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ import (
3232
"sigs.k8s.io/controller-runtime/pkg/log"
3333

3434
agentraxv1alpha1 "github.com/gitcommitankit/agentrax/api/v1alpha1"
35+
"github.com/gitcommitankit/agentrax/internal/observability"
3536
"github.com/gitcommitankit/agentrax/internal/quota"
3637
)
3738

@@ -57,6 +58,7 @@ func (r *TenantQuotaReconciler) Reconcile(ctx context.Context, req ctrl.Request)
5758
tq := &agentraxv1alpha1.TenantQuota{}
5859
if err := r.Get(ctx, req.NamespacedName, tq); err != nil {
5960
if apierrors.IsNotFound(err) {
61+
observability.QuotaUsageRatio.DeleteLabelValues(req.Namespace)
6062
return ctrl.Result{}, nil
6163
}
6264
return ctrl.Result{}, fmt.Errorf("fetching TenantQuota: %w", err)
@@ -89,6 +91,7 @@ func (r *TenantQuotaReconciler) Reconcile(ctx context.Context, req ctrl.Request)
8991
latest := &agentraxv1alpha1.TenantQuota{}
9092
if err := r.Get(ctx, req.NamespacedName, latest); err != nil {
9193
if apierrors.IsNotFound(err) {
94+
observability.QuotaUsageRatio.DeleteLabelValues(req.Namespace)
9295
return ctrl.Result{}, nil
9396
}
9497
return ctrl.Result{}, fmt.Errorf("re-fetching TenantQuota for status update: %w", err)
@@ -100,6 +103,16 @@ func (r *TenantQuotaReconciler) Reconcile(ctx context.Context, req ctrl.Request)
100103
latest.Status.UsedGPUs = usage.UsedGPUs
101104
latest.Status.UsedTotalReplicas = usage.UsedTotalReplicas
102105

106+
// Emit quota usage ratio metric so Grafana and alerting rules can track
107+
// how close this tenant is to its replica ceiling in real time.
108+
// Guard against divide-by-zero: if MaxTotalReplicas is zero the ratio is 0.
109+
if latest.Spec.MaxTotalReplicas > 0 {
110+
ratio := float64(usage.UsedTotalReplicas) / float64(latest.Spec.MaxTotalReplicas)
111+
observability.QuotaUsageRatio.WithLabelValues(latest.Namespace).Set(ratio)
112+
} else {
113+
observability.QuotaUsageRatio.WithLabelValues(latest.Namespace).Set(0)
114+
}
115+
103116
// 5. Set or clear the OverQuota condition based on whether usage exceeds spec.
104117
// Use latest.Spec (re-fetched) rather than tq.Spec (first fetch) to avoid
105118
// evaluating against a ceiling that may have changed between the two Gets.

‎internal/controller/tenantquota_controller_test.go‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,9 @@ import (
3030
"sigs.k8s.io/controller-runtime/pkg/client"
3131

3232
agentraxv1alpha1 "github.com/gitcommitankit/agentrax/api/v1alpha1"
33+
"github.com/gitcommitankit/agentrax/internal/observability"
34+
"github.com/prometheus/client_golang/prometheus/testutil"
35+
crmetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
3336
)
3437

3538
var _ = Describe("TenantQuota Controller", func() {
@@ -155,6 +158,45 @@ var _ = Describe("TenantQuota Controller", func() {
155158
g.Expect(tqFetched.Status.UsedAgents).To(BeNumerically("==", 0))
156159
}, timeout, interval).Should(Succeed())
157160
})
161+
162+
It("updates and cleans up QuotaUsageRatio metric keyed by tenant namespace", func() {
163+
tq := makeTQ("tq-metric", tqNS, 6, 4, 10, 6)
164+
Expect(k8sClient.Create(ctx, tq)).To(Succeed())
165+
166+
ad := makeBasicAD("ad-metric-1", tqNS, "tq-metric", 2)
167+
Expect(k8sClient.Create(ctx, ad)).To(Succeed())
168+
169+
Eventually(func(g Gomega) {
170+
val := testutil.ToFloat64(observability.QuotaUsageRatio.WithLabelValues(tqNS))
171+
g.Expect(val).To(BeNumerically("~", 0.2, 1e-4))
172+
}, timeout, interval).Should(Succeed())
173+
174+
// Deleting the TenantQuota should trigger reconcile NotFound path and remove the series.
175+
Expect(k8sClient.Delete(ctx, tq)).To(Succeed())
176+
Eventually(func() bool {
177+
err := k8sClient.Get(ctx, namespacedName("tq-metric", tqNS), &agentraxv1alpha1.TenantQuota{})
178+
return apierrors.IsNotFound(err)
179+
}, timeout, interval).Should(BeTrue())
180+
181+
Eventually(func(g Gomega) {
182+
mfs, err := crmetrics.Registry.Gather()
183+
g.Expect(err).NotTo(HaveOccurred())
184+
found := false
185+
for _, mf := range mfs {
186+
if mf.GetName() == "agentrax_tenant_quota_usage_ratio" {
187+
for _, m := range mf.GetMetric() {
188+
for _, lbl := range m.GetLabel() {
189+
if lbl.GetName() == "tenant" && lbl.GetValue() == tqNS {
190+
found = true
191+
break
192+
}
193+
}
194+
}
195+
}
196+
}
197+
g.Expect(found).To(BeFalse(), "expected no metric sample labeled tenant=%q to remain", tqNS)
198+
}, timeout, interval).Should(Succeed())
199+
})
158200
})
159201

160202
// ── OverQuota condition ───────────────────────────────────────────────────

0 commit comments

Comments
 (0)