refactor: key QuotaUsageRatio metric by namespace, implement label cleanup, and update Grafana dashboard queries to support filtering

Signed-off-by: Ankit Kr. Chowdhury <rakesh856100@gmail.com>

Author: gitcommitankit

config/grafana/dashboard-configmap.yaml

@@ -43,7 +43,7 @@ data:
           "datasource": "Prometheus",
           "targets": [
             {
-              "expr": "sum(rate(agentrax_reconcile_duration_seconds_count[5m])) by (tenant)",
+              "expr": "sum(rate(agentrax_reconcile_duration_seconds_count{tenant=~\"$tenant\"}[5m])) by (tenant)",
               "legendFormat": "{{ tenant }}"
             }
           ],
@@ -82,11 +82,11 @@ data:
           "datasource": "Prometheus",
           "targets": [
             {
-              "expr": "histogram_quantile(0.99, sum(rate(agentrax_reconcile_duration_seconds_bucket[5m])) by (le, tenant))",
+              "expr": "histogram_quantile(0.99, sum(rate(agentrax_reconcile_duration_seconds_bucket{tenant=~\"$tenant\"}[5m])) by (le, tenant))",
               "legendFormat": "P99 {{ tenant }}"
             },
             {
-              "expr": "histogram_quantile(0.50, sum(rate(agentrax_reconcile_duration_seconds_bucket[5m])) by (le, tenant))",
+              "expr": "histogram_quantile(0.50, sum(rate(agentrax_reconcile_duration_seconds_bucket{tenant=~\"$tenant\"}[5m])) by (le, tenant))",
               "legendFormat": "P50 {{ tenant }}"
             }
           ],

internal/controller/tenantquota_controller.go

@@ -58,6 +58,7 @@ func (r *TenantQuotaReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 	tq := &agentraxv1alpha1.TenantQuota{}
 	if err := r.Get(ctx, req.NamespacedName, tq); err != nil {
 		if apierrors.IsNotFound(err) {
+			observability.QuotaUsageRatio.DeleteLabelValues(req.Namespace)
 			return ctrl.Result{}, nil
 		}
 		return ctrl.Result{}, fmt.Errorf("fetching TenantQuota: %w", err)
@@ -90,6 +91,7 @@ func (r *TenantQuotaReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 	latest := &agentraxv1alpha1.TenantQuota{}
 	if err := r.Get(ctx, req.NamespacedName, latest); err != nil {
 		if apierrors.IsNotFound(err) {
+			observability.QuotaUsageRatio.DeleteLabelValues(req.Namespace)
 			return ctrl.Result{}, nil
 		}
 		return ctrl.Result{}, fmt.Errorf("re-fetching TenantQuota for status update: %w", err)
@@ -106,9 +108,9 @@ func (r *TenantQuotaReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 	// Guard against divide-by-zero: if MaxTotalReplicas is zero the ratio is 0.
 	if latest.Spec.MaxTotalReplicas > 0 {
 		ratio := float64(usage.UsedTotalReplicas) / float64(latest.Spec.MaxTotalReplicas)
-		observability.QuotaUsageRatio.WithLabelValues(latest.Name).Set(ratio)
+		observability.QuotaUsageRatio.WithLabelValues(latest.Namespace).Set(ratio)
 	} else {
-		observability.QuotaUsageRatio.WithLabelValues(latest.Name).Set(0)
+		observability.QuotaUsageRatio.WithLabelValues(latest.Namespace).Set(0)
 	}
 
 	// 5. Set or clear the OverQuota condition based on whether usage exceeds spec.

internal/controller/tenantquota_controller_test.go

@@ -30,6 +30,8 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	agentraxv1alpha1 "github.com/gitcommitankit/agentrax/api/v1alpha1"
+	"github.com/gitcommitankit/agentrax/internal/observability"
+	"github.com/prometheus/client_golang/prometheus/testutil"
 )
 
 var _ = Describe("TenantQuota Controller", func() {
@@ -155,6 +157,31 @@ var _ = Describe("TenantQuota Controller", func() {
 				g.Expect(tqFetched.Status.UsedAgents).To(BeNumerically("==", 0))
 			}, timeout, interval).Should(Succeed())
 		})
+
+		It("updates and cleans up QuotaUsageRatio metric keyed by tenant namespace", func() {
+			tq := makeTQ("tq-metric", tqNS, 6, 4, 10, 6)
+			Expect(k8sClient.Create(ctx, tq)).To(Succeed())
+
+			ad := makeBasicAD("ad-metric-1", tqNS, "tq-metric", 2)
+			Expect(k8sClient.Create(ctx, ad)).To(Succeed())
+
+			Eventually(func(g Gomega) {
+				val := testutil.ToFloat64(observability.QuotaUsageRatio.WithLabelValues(tqNS))
+				g.Expect(val).To(BeNumerically("~", 0.2, 1e-4))
+			}, timeout, interval).Should(Succeed())
+
+			// Deleting the TenantQuota should trigger reconcile NotFound path and remove the series.
+			Expect(k8sClient.Delete(ctx, tq)).To(Succeed())
+			Eventually(func() bool {
+				err := k8sClient.Get(ctx, namespacedName("tq-metric", tqNS), &agentraxv1alpha1.TenantQuota{})
+				return apierrors.IsNotFound(err)
+			}, timeout, interval).Should(BeTrue())
+
+			Eventually(func(g Gomega) {
+				// DeleteLabelValues returns false when the series is already gone.
+				g.Expect(observability.QuotaUsageRatio.DeleteLabelValues(tqNS)).To(BeFalse())
+			}, timeout, interval).Should(Succeed())
+		})
 	})
 
 	// ── OverQuota condition ───────────────────────────────────────────────────

internal/observability/metrics.go

@@ -25,7 +25,7 @@ var ReconcileDuration = prometheus.NewHistogramVec(
 // QuotaUsageRatio tracks the current replica usage as a fraction of maxTotalReplicas.
 // A value of 1.0 means the tenant is at the hard replica cap.
 // Labels:
-//   - tenant: the TenantQuota name (and namespace)
+//   - tenant: the Kubernetes namespace (which maps 1:1 to a tenant in Agentrax)
 var QuotaUsageRatio = prometheus.NewGaugeVec(
 	prometheus.GaugeOpts{
 		Name: "agentrax_tenant_quota_usage_ratio",

internal/observability/metrics_test.go

@@ -1,6 +1,7 @@
 package observability_test
 
 import (
+	"fmt"
 	"strings"
 	"testing"
 
@@ -16,39 +17,36 @@ import (
 // observations and that the histogram is correctly exported via text format.
 func TestReconcileDurationObserve(t *testing.T) {
 	reg := prometheus.NewRegistry()
+	observability.ReconcileDuration.Reset()
+	reg.MustRegister(observability.ReconcileDuration)
 
-	hist := prometheus.NewHistogramVec(
-		prometheus.HistogramOpts{
-			Name:    "agentrax_reconcile_duration_seconds",
-			Help:    "Duration of AgentDeployment reconcile loops in seconds.",
-			Buckets: prometheus.DefBuckets,
-		},
-		[]string{"controller", "tenant"},
-	)
-	reg.MustRegister(hist)
+	observability.ReconcileDuration.WithLabelValues("agentdeployment", "tenant-test").Observe(0.5)
+	observability.ReconcileDuration.WithLabelValues("agentdeployment", "tenant-test").Observe(1.5)
 
-	hist.WithLabelValues("agentdeployment", "tenant-test").Observe(0.5)
-	hist.WithLabelValues("agentdeployment", "tenant-test").Observe(1.5)
-
-	// Verify via text format: sample_count should be 2.
 	expected := `
 		# HELP agentrax_reconcile_duration_seconds Duration of AgentDeployment reconcile loops in seconds.
 		# TYPE agentrax_reconcile_duration_seconds histogram
+		agentrax_reconcile_duration_seconds_bucket{controller="agentdeployment",tenant="tenant-test",le="0.005"} 0
+		agentrax_reconcile_duration_seconds_bucket{controller="agentdeployment",tenant="tenant-test",le="0.01"} 0
+		agentrax_reconcile_duration_seconds_bucket{controller="agentdeployment",tenant="tenant-test",le="0.025"} 0
+		agentrax_reconcile_duration_seconds_bucket{controller="agentdeployment",tenant="tenant-test",le="0.05"} 0
+		agentrax_reconcile_duration_seconds_bucket{controller="agentdeployment",tenant="tenant-test",le="0.1"} 0
+		agentrax_reconcile_duration_seconds_bucket{controller="agentdeployment",tenant="tenant-test",le="0.25"} 0
+		agentrax_reconcile_duration_seconds_bucket{controller="agentdeployment",tenant="tenant-test",le="0.5"} 1
+		agentrax_reconcile_duration_seconds_bucket{controller="agentdeployment",tenant="tenant-test",le="1"} 1
+		agentrax_reconcile_duration_seconds_bucket{controller="agentdeployment",tenant="tenant-test",le="2.5"} 2
+		agentrax_reconcile_duration_seconds_bucket{controller="agentdeployment",tenant="tenant-test",le="5"} 2
+		agentrax_reconcile_duration_seconds_bucket{controller="agentdeployment",tenant="tenant-test",le="10"} 2
+		agentrax_reconcile_duration_seconds_bucket{controller="agentdeployment",tenant="tenant-test",le="+Inf"} 2
+		agentrax_reconcile_duration_seconds_sum{controller="agentdeployment",tenant="tenant-test"} 2
+		agentrax_reconcile_duration_seconds_count{controller="agentdeployment",tenant="tenant-test"} 2
 	`
-	err := testutil.GatherAndCompare(reg, strings.NewReader(expected),
-		"agentrax_reconcile_duration_seconds")
-	// We only check prefix presence, not exact bucket values — a full comparison
-	// would require listing all bucket boundaries. Use Count instead.
-	_ = err
+	err := testutil.GatherAndCompare(reg, strings.NewReader(expected), "agentrax_reconcile_duration_seconds")
+	require.NoError(t, err)
 
 	count, err := testutil.GatherAndCount(reg, "agentrax_reconcile_duration_seconds")
 	require.NoError(t, err)
-	// GatherAndCount returns the number of MetricFamily + all their metrics
-	// (buckets + sum + count); assert > 0 to prove the histogram emitted output.
 	assert.Greater(t, count, 0, "expected histogram to produce at least one metric series")
-
-	// Verify the package-level var is registered and usable.
-	_ = observability.ReconcileDuration
 }
 
 // TestQuotaUsageRatioSet verifies that QuotaUsageRatio records the expected
@@ -68,22 +66,20 @@ func TestQuotaUsageRatioSet(t *testing.T) {
 	for _, tc := range tests {
 		t.Run(tc.name, func(t *testing.T) {
 			reg := prometheus.NewRegistry()
-			gauge := prometheus.NewGaugeVec(
-				prometheus.GaugeOpts{
-					Name: "agentrax_tenant_quota_usage_ratio",
-					Help: "Current replica usage as a fraction of maxTotalReplicas.",
-				},
-				[]string{"tenant"},
-			)
-			reg.MustRegister(gauge)
+			observability.QuotaUsageRatio.Reset()
+			reg.MustRegister(observability.QuotaUsageRatio)
 
-			gauge.WithLabelValues(tc.tenant).Set(tc.value)
+			observability.QuotaUsageRatio.WithLabelValues(tc.tenant).Set(tc.value)
 
-			got := testutil.ToFloat64(gauge.WithLabelValues(tc.tenant))
+			expected := fmt.Sprintf(`
+				# HELP agentrax_tenant_quota_usage_ratio Current replica usage as a fraction of maxTotalReplicas (0.0–1.0) per tenant.
+				# TYPE agentrax_tenant_quota_usage_ratio gauge
+				agentrax_tenant_quota_usage_ratio{tenant="%s"} %g
+			`, tc.tenant, tc.value)
+			require.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(expected), "agentrax_tenant_quota_usage_ratio"))
+
+			got := testutil.ToFloat64(observability.QuotaUsageRatio.WithLabelValues(tc.tenant))
 			assert.InDelta(t, tc.expected, got, 1e-9)
 		})
 	}
-
-	// Verify the package-level var is registered and usable.
-	_ = observability.QuotaUsageRatio
 }