From 7cd21ca58cfbab789c3ef2bf91f81991410217ec Mon Sep 17 00:00:00 2001 From: Roi Glinik Date: Thu, 29 Oct 2026 09:37:49 +0200 Subject: [PATCH 1/7] ROB-1291 Send the Prometheus tenant from the namespace label With PROMETHEUS_TENANT_LABEL set, every Prometheus call of an action sends the value of that label on the action's namespace in PROMETHEUS_TENANT_HEADER (default X-Scope-OrgID). prometheus_enricher takes an optional namespace param for UI graphs. Also sets the namespace on HPA alert subjects. Co-Authored-By: Claude Opus 5.5 --- src/robusta/core/model/base_params.py | 35 +++++ src/robusta/core/model/env_vars.py | 4 + .../playbooks/playbooks_event_handler_impl.py | 1 + src/robusta/integrations/prometheus/models.py | 1 + src/robusta/integrations/prometheus/tenant.py | 20 +++ tests/test_prometheus_tenant.py | 125 ++++++++++++++++++ 6 files changed, 186 insertions(+) create mode 100644 src/robusta/integrations/prometheus/tenant.py create mode 100644 tests/test_prometheus_tenant.py diff --git a/src/robusta/core/model/base_params.py b/src/robusta/core/model/base_params.py index 70da1c0f4..4d090b3c9 100644 --- a/src/robusta/core/model/base_params.py +++ b/src/robusta/core/model/base_params.py @@ -4,6 +4,7 @@ from pydantic.v1 import BaseModel, SecretStr, validator, Field +from robusta.core.model.env_vars import PROMETHEUS_TENANT_HEADER, PROMETHEUS_TENANT_LABEL from robusta.integrations import openshift from robusta.utils.documented_pydantic import DocumentedModel @@ -68,6 +69,12 @@ def post_initialization(self): """ pass + def post_event_initialization(self, event) -> None: + """ + Runs right before the action, with the event it runs on + """ + pass + pass @@ -366,6 +373,28 @@ def auto_openshift_token(cls, v: Optional[SecretStr]): return None + def _tenant_namespace(self, event) -> Optional[str]: + return event.get_subject().namespace + + def post_event_initialization(self, event) -> None: + """ + With PROMETHEUS_TENANT_LABEL set, sends the tenant of the action's namespace in PROMETHEUS_TENANT_HEADER. + Every Prometheus call made with these params gets it, since they all build their connection from them. + With no namespace or no label, the configured prometheus_additional_headers are used as they are. + """ + if not PROMETHEUS_TENANT_LABEL: + return + from robusta.integrations.prometheus.tenant import resolve_tenant + + tenant = resolve_tenant(PROMETHEUS_TENANT_LABEL, self._tenant_namespace(event)) + if not tenant: + return + # A new dict, set last: it replaces any tenant header the request itself sent, and never changes a shared one + self.prometheus_additional_headers = { + **(self.prometheus_additional_headers or {}), + PROMETHEUS_TENANT_HEADER: tenant, + } + class PrometheusDuration(BaseModel): """ @@ -401,11 +430,17 @@ class PrometheusQueryParams(PrometheusParams): :var promql_query: the prometheusql query you want to run :var duration: the duration of the query :var step: (str) Query resolution step width in duration format or float number of seconds - i.e 100s, 3d, 2w, 170.3 + :var namespace: the namespace the query is about. With PROMETHEUS_TENANT_LABEL set, its label picks the tenant """ promql_query: str duration: Union[PrometheusDateRange, PrometheusDuration] step: Optional[str] + namespace: Optional[str] = None + + def _tenant_namespace(self, event) -> Optional[str]: + # prometheus_enricher runs on an event with no subject, so the UI names the namespace in the params + return self.namespace or super()._tenant_namespace(event) class TimedPrometheusParams(PrometheusParams): diff --git a/src/robusta/core/model/env_vars.py b/src/robusta/core/model/env_vars.py index 3b07363bc..a8d74ca04 100644 --- a/src/robusta/core/model/env_vars.py +++ b/src/robusta/core/model/env_vars.py @@ -49,6 +49,10 @@ def load_bool(env_var, default: bool): PROMETHEUS_ENABLED = os.environ.get("PROMETHEUS_ENABLED", "false").lower() == "true" MANAGED_CONFIGURATION_ENABLED = os.environ.get("MANAGED_CONFIGURATION_ENABLED", "false").lower() == "true" PROMETHEUS_SSL_ENABLED = os.environ.get("PROMETHEUS_SSL_ENABLED", "false").lower() == "true" +# Multi-tenant Prometheus (e.g. Mimir): label on the action's namespace, sent as the tenant header. +# Env vars, not global_config, so a request from the UI can't override them. +PROMETHEUS_TENANT_LABEL = os.environ.get("PROMETHEUS_TENANT_LABEL") +PROMETHEUS_TENANT_HEADER = os.environ.get("PROMETHEUS_TENANT_HEADER", "X-Scope-OrgID") INCOMING_REQUEST_TIME_WINDOW_SECONDS = int(os.environ.get("INCOMING_REQUEST_TIME_WINDOW_SECONDS", 3600)) diff --git a/src/robusta/core/playbooks/playbooks_event_handler_impl.py b/src/robusta/core/playbooks/playbooks_event_handler_impl.py index 110190b4a..1c0b95e6d 100644 --- a/src/robusta/core/playbooks/playbooks_event_handler_impl.py +++ b/src/robusta/core/playbooks/playbooks_event_handler_impl.py @@ -239,6 +239,7 @@ def __run_playbook_actions( continue try: if action_with_params: + params.post_event_initialization(execution_event) registered_action.func(execution_event, params) else: registered_action.func(execution_event) diff --git a/src/robusta/integrations/prometheus/models.py b/src/robusta/integrations/prometheus/models.py index d169f4a1a..48a4ecd42 100644 --- a/src/robusta/integrations/prometheus/models.py +++ b/src/robusta/integrations/prometheus/models.py @@ -193,6 +193,7 @@ def get_alert_subject(self) -> FindingSubject: elif self.hpa: subject_type = FindingSubjectType.TYPE_HPA name = self.hpa.metadata.name + namespace = self.hpa.metadata.namespace labels = self.hpa.metadata.labels annotations = self.hpa.metadata.annotations diff --git a/src/robusta/integrations/prometheus/tenant.py b/src/robusta/integrations/prometheus/tenant.py new file mode 100644 index 000000000..73872f0af --- /dev/null +++ b/src/robusta/integrations/prometheus/tenant.py @@ -0,0 +1,20 @@ +import logging +from typing import Optional + +from robusta.integrations.kubernetes.api_client_utils import get_namespace_labels + + +def resolve_tenant(key: str, namespace: Optional[str]) -> Optional[str]: + """ + Returns the value of the `key` label of a namespace, read from the cached namespace data (NAMESPACE_DATA_TTL). + Returns None when there is no namespace or no such label, so the caller keeps its configured default tenant. + """ + if not namespace: + return None + try: + tenant = (get_namespace_labels(namespace) or {}).get(key) + except KeyError: # a namespace created after the last reload, or deleted + tenant = None + if not tenant: + logging.warning(f"No '{key}' label on namespace {namespace}, using the default Prometheus tenant") + return tenant diff --git a/tests/test_prometheus_tenant.py b/tests/test_prometheus_tenant.py new file mode 100644 index 000000000..d7598f138 --- /dev/null +++ b/tests/test_prometheus_tenant.py @@ -0,0 +1,125 @@ +from types import SimpleNamespace +from unittest.mock import patch + +from robusta.core.model import base_params +from robusta.core.model.base_params import PrometheusParams, PrometheusQueryParams +from robusta.core.reporting.base import FindingSubject +from robusta.core.reporting.consts import FindingSubjectType +from robusta.integrations.prometheus.tenant import resolve_tenant + +NAMESPACE_LABELS = "robusta.integrations.prometheus.tenant.get_namespace_labels" + + +def _namespaces(labels_by_namespace): + return lambda namespace: labels_by_namespace[namespace] + + +def test_tenant_from_namespace_label(): + with patch(NAMESPACE_LABELS, side_effect=_namespaces({"shop": {"team": "corporate"}})): + assert resolve_tenant("team", "shop") == "corporate" + + +def test_no_namespace_reads_nothing(): + with patch(NAMESPACE_LABELS) as labels: + assert resolve_tenant("team", None) is None + labels.assert_not_called() + + +def test_missing_label_returns_none(): + with patch(NAMESPACE_LABELS, side_effect=_namespaces({"shop": {"app": "web"}})): + assert resolve_tenant("team", "shop") is None + + +def test_namespace_with_no_labels_returns_none(): + with patch(NAMESPACE_LABELS, side_effect=_namespaces({"shop": None})): + assert resolve_tenant("team", "shop") is None + + +def test_unknown_namespace_returns_none(): + with patch(NAMESPACE_LABELS, side_effect=KeyError("shop")): + assert resolve_tenant("team", "shop") is None + + +def _event(subject: FindingSubject): + return SimpleNamespace(get_subject=lambda: subject) + + +# The pod and alert labels carry another value on purpose: only the namespace label counts +POD_SUBJECT = FindingSubject( + name="web", subject_type=FindingSubjectType.TYPE_POD, namespace="shop", labels={"team": "from-pod"} +) +NAMESPACE_ONLY_SUBJECT = FindingSubject(name="Unresolved", subject_type=FindingSubjectType.TYPE_NONE, namespace="shop") +UNRESOLVED_SUBJECT = FindingSubject(name="Unresolved", subject_type=FindingSubjectType.TYPE_NONE) +NAMESPACES = {"shop": {"team": "corporate"}, "billing": {"team": "retail"}} + + +def test_no_tenant_label_configured_changes_nothing(): + params = PrometheusParams(prometheus_additional_headers={"X-Scope-OrgID": "sent"}) + with patch.object(base_params, "PROMETHEUS_TENANT_LABEL", None), patch(NAMESPACE_LABELS) as labels: + params.post_event_initialization(_event(POD_SUBJECT)) + assert params.prometheus_additional_headers == {"X-Scope-OrgID": "sent"} + labels.assert_not_called() + + +def test_tenant_from_subject_namespace_ignores_resource_labels(): + params = PrometheusParams() + with patch.object(base_params, "PROMETHEUS_TENANT_LABEL", "team"), patch( + NAMESPACE_LABELS, side_effect=_namespaces(NAMESPACES) + ): + params.post_event_initialization(_event(POD_SUBJECT)) + assert params.prometheus_additional_headers == {"X-Scope-OrgID": "corporate"} + + +def test_namespace_only_alert_gets_tenant(): + params = PrometheusParams() + with patch.object(base_params, "PROMETHEUS_TENANT_LABEL", "team"), patch( + NAMESPACE_LABELS, side_effect=_namespaces(NAMESPACES) + ): + params.post_event_initialization(_event(NAMESPACE_ONLY_SUBJECT)) + assert params.prometheus_additional_headers == {"X-Scope-OrgID": "corporate"} + + +def test_tenant_header_replaces_sent_one_and_keeps_others(): + params = PrometheusParams(prometheus_additional_headers={"X-Scope-OrgID": "forged", "X-Other": "1"}) + with patch.object(base_params, "PROMETHEUS_TENANT_LABEL", "team"), patch( + NAMESPACE_LABELS, side_effect=_namespaces(NAMESPACES) + ): + params.post_event_initialization(_event(POD_SUBJECT)) + assert params.prometheus_additional_headers == {"X-Scope-OrgID": "corporate", "X-Other": "1"} + + +def test_no_namespace_keeps_default_tenant(): + params = PrometheusParams(prometheus_additional_headers={"X-Scope-OrgID": "default"}) + with patch.object(base_params, "PROMETHEUS_TENANT_LABEL", "team"), patch(NAMESPACE_LABELS) as labels: + params.post_event_initialization(_event(UNRESOLVED_SUBJECT)) + assert params.prometheus_additional_headers == {"X-Scope-OrgID": "default"} + labels.assert_not_called() + + +def test_query_params_namespace_wins_over_event_subject(): + params = PrometheusQueryParams(promql_query="up", duration={"duration_minutes": 5}, namespace="billing") + with patch.object(base_params, "PROMETHEUS_TENANT_LABEL", "team"), patch( + NAMESPACE_LABELS, side_effect=_namespaces(NAMESPACES) + ): + params.post_event_initialization(_event(POD_SUBJECT)) + assert params.prometheus_additional_headers == {"X-Scope-OrgID": "retail"} + + +def test_requests_do_not_share_headers(): + # Two requests built from the same global config, as merge_global_params does, for different tenants + global_config = {"prometheus_additional_headers": {"X-Scope-OrgID": "default"}} + with patch.object(base_params, "PROMETHEUS_TENANT_LABEL", "team"), patch( + NAMESPACE_LABELS, side_effect=_namespaces(NAMESPACES) + ): + first, second = [ + PrometheusQueryParams( + **global_config.copy(), promql_query="up", duration={"duration_minutes": 5}, namespace=namespace + ) + for namespace in ("shop", "billing") + ] + first.post_event_initialization(_event(UNRESOLVED_SUBJECT)) + second.post_event_initialization(_event(UNRESOLVED_SUBJECT)) + + assert first.prometheus_additional_headers == {"X-Scope-OrgID": "corporate"} + assert second.prometheus_additional_headers == {"X-Scope-OrgID": "retail"} + assert global_config == {"prometheus_additional_headers": {"X-Scope-OrgID": "default"}} From b595f0ec17b3f2d418d4c59e034f9fd1503e4812 Mon Sep 17 00:00:00 2001 From: Roi Glinik Date: Thu, 29 Oct 2026 11:00:51 +0200 Subject: [PATCH 2/7] ROB-1291 Prometheus tenant from the namespace param (UI graphs only) PrometheusParams gets a namespace field; generate_prometheus_config sends that namespace's PROMETHEUS_TENANT_LABEL label in PROMETHEUS_TENANT_HEADER. No event hook: only requests that name a namespace (the UI graphs) get a tenant. Co-Authored-By: Claude Opus 5.5 --- src/robusta/core/model/base_params.py | 37 +---- .../playbooks/playbooks_event_handler_impl.py | 1 - src/robusta/integrations/prometheus/models.py | 1 - src/robusta/integrations/prometheus/tenant.py | 20 --- src/robusta/integrations/prometheus/utils.py | 27 +++- tests/test_prometheus_tenant.py | 139 +++++------------- 6 files changed, 62 insertions(+), 163 deletions(-) delete mode 100644 src/robusta/integrations/prometheus/tenant.py diff --git a/src/robusta/core/model/base_params.py b/src/robusta/core/model/base_params.py index 4d090b3c9..6294fd996 100644 --- a/src/robusta/core/model/base_params.py +++ b/src/robusta/core/model/base_params.py @@ -4,7 +4,6 @@ from pydantic.v1 import BaseModel, SecretStr, validator, Field -from robusta.core.model.env_vars import PROMETHEUS_TENANT_HEADER, PROMETHEUS_TENANT_LABEL from robusta.integrations import openshift from robusta.utils.documented_pydantic import DocumentedModel @@ -69,12 +68,6 @@ def post_initialization(self): """ pass - def post_event_initialization(self, event) -> None: - """ - Runs right before the action, with the event it runs on - """ - pass - pass @@ -328,6 +321,7 @@ class PrometheusParams(ActionParams): :var prometheus_additional_headers: additional HTTP headers (if defined) to add to every prometheus query :var prometheus_additional_labels: A dictionary of additional labels needed for multi-cluster prometheus :var add_additional_labels: adds the additional labels (if defined) to the query + :var namespace: the namespace the queries are about. With PROMETHEUS_TENANT_LABEL set, its label picks the tenant :example prometheus_url: "http://prometheus-k8s.monitoring.svc.cluster.local:9090" :example prometheus_auth: Basic YWRtaW46cGFzc3dvcmQ= @@ -344,6 +338,7 @@ class PrometheusParams(ActionParams): prometheus_additional_headers: Optional[Dict[str, str]] = None prometheus_additional_labels: Optional[Dict[str, str]] = None add_additional_labels: bool = True + namespace: Optional[str] = None prometheus_graphs_overrides: Optional[List[OverrideGraph]] = None @validator("prometheus_url", allow_reuse=True) @@ -373,28 +368,6 @@ def auto_openshift_token(cls, v: Optional[SecretStr]): return None - def _tenant_namespace(self, event) -> Optional[str]: - return event.get_subject().namespace - - def post_event_initialization(self, event) -> None: - """ - With PROMETHEUS_TENANT_LABEL set, sends the tenant of the action's namespace in PROMETHEUS_TENANT_HEADER. - Every Prometheus call made with these params gets it, since they all build their connection from them. - With no namespace or no label, the configured prometheus_additional_headers are used as they are. - """ - if not PROMETHEUS_TENANT_LABEL: - return - from robusta.integrations.prometheus.tenant import resolve_tenant - - tenant = resolve_tenant(PROMETHEUS_TENANT_LABEL, self._tenant_namespace(event)) - if not tenant: - return - # A new dict, set last: it replaces any tenant header the request itself sent, and never changes a shared one - self.prometheus_additional_headers = { - **(self.prometheus_additional_headers or {}), - PROMETHEUS_TENANT_HEADER: tenant, - } - class PrometheusDuration(BaseModel): """ @@ -430,17 +403,11 @@ class PrometheusQueryParams(PrometheusParams): :var promql_query: the prometheusql query you want to run :var duration: the duration of the query :var step: (str) Query resolution step width in duration format or float number of seconds - i.e 100s, 3d, 2w, 170.3 - :var namespace: the namespace the query is about. With PROMETHEUS_TENANT_LABEL set, its label picks the tenant """ promql_query: str duration: Union[PrometheusDateRange, PrometheusDuration] step: Optional[str] - namespace: Optional[str] = None - - def _tenant_namespace(self, event) -> Optional[str]: - # prometheus_enricher runs on an event with no subject, so the UI names the namespace in the params - return self.namespace or super()._tenant_namespace(event) class TimedPrometheusParams(PrometheusParams): diff --git a/src/robusta/core/playbooks/playbooks_event_handler_impl.py b/src/robusta/core/playbooks/playbooks_event_handler_impl.py index 1c0b95e6d..110190b4a 100644 --- a/src/robusta/core/playbooks/playbooks_event_handler_impl.py +++ b/src/robusta/core/playbooks/playbooks_event_handler_impl.py @@ -239,7 +239,6 @@ def __run_playbook_actions( continue try: if action_with_params: - params.post_event_initialization(execution_event) registered_action.func(execution_event, params) else: registered_action.func(execution_event) diff --git a/src/robusta/integrations/prometheus/models.py b/src/robusta/integrations/prometheus/models.py index 48a4ecd42..d169f4a1a 100644 --- a/src/robusta/integrations/prometheus/models.py +++ b/src/robusta/integrations/prometheus/models.py @@ -193,7 +193,6 @@ def get_alert_subject(self) -> FindingSubject: elif self.hpa: subject_type = FindingSubjectType.TYPE_HPA name = self.hpa.metadata.name - namespace = self.hpa.metadata.namespace labels = self.hpa.metadata.labels annotations = self.hpa.metadata.annotations diff --git a/src/robusta/integrations/prometheus/tenant.py b/src/robusta/integrations/prometheus/tenant.py deleted file mode 100644 index 73872f0af..000000000 --- a/src/robusta/integrations/prometheus/tenant.py +++ /dev/null @@ -1,20 +0,0 @@ -import logging -from typing import Optional - -from robusta.integrations.kubernetes.api_client_utils import get_namespace_labels - - -def resolve_tenant(key: str, namespace: Optional[str]) -> Optional[str]: - """ - Returns the value of the `key` label of a namespace, read from the cached namespace data (NAMESPACE_DATA_TTL). - Returns None when there is no namespace or no such label, so the caller keeps its configured default tenant. - """ - if not namespace: - return None - try: - tenant = (get_namespace_labels(namespace) or {}).get(key) - except KeyError: # a namespace created after the last reload, or deleted - tenant = None - if not tenant: - logging.warning(f"No '{key}' label on namespace {namespace}, using the default Prometheus tenant") - return tenant diff --git a/src/robusta/integrations/prometheus/utils.py b/src/robusta/integrations/prometheus/utils.py index b1f23ee09..2413d7043 100644 --- a/src/robusta/integrations/prometheus/utils.py +++ b/src/robusta/integrations/prometheus/utils.py @@ -19,8 +19,11 @@ DISABLE_PROMETHEUS_DISCOVERY, HOLMES_DISCOVERY_NAMESPACE, PROMETHEUS_SSL_ENABLED, + PROMETHEUS_TENANT_HEADER, + PROMETHEUS_TENANT_LABEL, SERVICE_CACHE_TTL_SEC, ) +from robusta.integrations.kubernetes.api_client_utils import get_namespace_labels from robusta.utils.service_discovery import find_service_url AZURE_RESOURCE = os.environ.get("AZURE_RESOURCE", "https://prometheus.monitor.azure.com") @@ -39,6 +42,22 @@ VICTORIA_METRICS_CONFIGURED = os.environ.get("VICTORIA_METRICS_CONFIGURED", "false").lower() == "true" +def get_prometheus_tenant(namespace: Optional[str]) -> Optional[str]: + """ + Multi-tenant Prometheus (e.g. Mimir): the value of the PROMETHEUS_TENANT_LABEL label of the namespace, read from + the cached namespace data (NAMESPACE_DATA_TTL). None when the feature is off, or there's no namespace or label. + """ + if not PROMETHEUS_TENANT_LABEL or not namespace: + return None + try: + tenant = (get_namespace_labels(namespace) or {}).get(PROMETHEUS_TENANT_LABEL) + except KeyError: # a namespace created after the last reload, or deleted + tenant = None + if not tenant: + logging.debug(f"No '{PROMETHEUS_TENANT_LABEL}' label on namespace {namespace}, using the default tenant") + return tenant + + def generate_prometheus_config(prometheus_params: PrometheusParams) -> PrometheusConfig: is_victoria_metrics = VICTORIA_METRICS_CONFIGURED url: Optional[str] = ( @@ -60,8 +79,12 @@ def generate_prometheus_config(prometheus_params: PrometheusParams) -> Prometheu if prometheus_params.prometheus_auth: baseconfig["prometheus_auth"] = prometheus_params.prometheus_auth.get_secret_value() - if prometheus_params.prometheus_additional_headers: - baseconfig["headers"] = prometheus_params.prometheus_additional_headers + headers = dict(prometheus_params.prometheus_additional_headers or {}) + tenant = get_prometheus_tenant(prometheus_params.namespace) + if tenant: # replaces any tenant header from global_config or the request + headers[PROMETHEUS_TENANT_HEADER] = tenant + if headers: + baseconfig["headers"] = headers # aws config if AWS_REGION: diff --git a/tests/test_prometheus_tenant.py b/tests/test_prometheus_tenant.py index d7598f138..59434b07d 100644 --- a/tests/test_prometheus_tenant.py +++ b/tests/test_prometheus_tenant.py @@ -1,125 +1,56 @@ -from types import SimpleNamespace from unittest.mock import patch -from robusta.core.model import base_params -from robusta.core.model.base_params import PrometheusParams, PrometheusQueryParams -from robusta.core.reporting.base import FindingSubject -from robusta.core.reporting.consts import FindingSubjectType -from robusta.integrations.prometheus.tenant import resolve_tenant +import pytest -NAMESPACE_LABELS = "robusta.integrations.prometheus.tenant.get_namespace_labels" +from robusta.core.model.base_params import PrometheusParams +from robusta.integrations.prometheus import utils +from robusta.integrations.prometheus.utils import generate_prometheus_config +NAMESPACES = {"shop": {"team": "corporate"}, "billing": {"team": "retail"}, "unlabeled": {"app": "web"}} -def _namespaces(labels_by_namespace): - return lambda namespace: labels_by_namespace[namespace] +@pytest.fixture +def tenant_label(): + def labels(namespace): + return NAMESPACES[namespace] -def test_tenant_from_namespace_label(): - with patch(NAMESPACE_LABELS, side_effect=_namespaces({"shop": {"team": "corporate"}})): - assert resolve_tenant("team", "shop") == "corporate" + with patch.object(utils, "PROMETHEUS_TENANT_LABEL", "team"), patch.object( + utils, "get_namespace_labels", side_effect=labels + ) as get_labels: + yield get_labels -def test_no_namespace_reads_nothing(): - with patch(NAMESPACE_LABELS) as labels: - assert resolve_tenant("team", None) is None - labels.assert_not_called() +def _headers(**params): + return generate_prometheus_config(PrometheusParams(prometheus_url="http://prometheus:9090", **params)).headers -def test_missing_label_returns_none(): - with patch(NAMESPACE_LABELS, side_effect=_namespaces({"shop": {"app": "web"}})): - assert resolve_tenant("team", "shop") is None +def test_tenant_from_namespace_label(tenant_label): + assert _headers(namespace="shop") == {"X-Scope-OrgID": "corporate"} -def test_namespace_with_no_labels_returns_none(): - with patch(NAMESPACE_LABELS, side_effect=_namespaces({"shop": None})): - assert resolve_tenant("team", "shop") is None +def test_tenant_replaces_configured_one_and_keeps_other_headers(tenant_label): + headers = _headers(namespace="billing", prometheus_additional_headers={"X-Scope-OrgID": "default", "X-Other": "1"}) + assert headers == {"X-Scope-OrgID": "retail", "X-Other": "1"} -def test_unknown_namespace_returns_none(): - with patch(NAMESPACE_LABELS, side_effect=KeyError("shop")): - assert resolve_tenant("team", "shop") is None +def test_no_namespace_keeps_default_tenant(tenant_label): + assert _headers(prometheus_additional_headers={"X-Scope-OrgID": "default"}) == {"X-Scope-OrgID": "default"} + tenant_label.assert_not_called() -def _event(subject: FindingSubject): - return SimpleNamespace(get_subject=lambda: subject) +@pytest.mark.parametrize("namespace", ["unlabeled", "missing"]) +def test_unlabeled_or_unknown_namespace_keeps_default_tenant(tenant_label, namespace): + headers = _headers(namespace=namespace, prometheus_additional_headers={"X-Scope-OrgID": "default"}) + assert headers == {"X-Scope-OrgID": "default"} -# The pod and alert labels carry another value on purpose: only the namespace label counts -POD_SUBJECT = FindingSubject( - name="web", subject_type=FindingSubjectType.TYPE_POD, namespace="shop", labels={"team": "from-pod"} -) -NAMESPACE_ONLY_SUBJECT = FindingSubject(name="Unresolved", subject_type=FindingSubjectType.TYPE_NONE, namespace="shop") -UNRESOLVED_SUBJECT = FindingSubject(name="Unresolved", subject_type=FindingSubjectType.TYPE_NONE) -NAMESPACES = {"shop": {"team": "corporate"}, "billing": {"team": "retail"}} +def test_feature_off_reads_nothing(tenant_label): + with patch.object(utils, "PROMETHEUS_TENANT_LABEL", None): + assert not _headers(namespace="shop") + tenant_label.assert_not_called() -def test_no_tenant_label_configured_changes_nothing(): - params = PrometheusParams(prometheus_additional_headers={"X-Scope-OrgID": "sent"}) - with patch.object(base_params, "PROMETHEUS_TENANT_LABEL", None), patch(NAMESPACE_LABELS) as labels: - params.post_event_initialization(_event(POD_SUBJECT)) - assert params.prometheus_additional_headers == {"X-Scope-OrgID": "sent"} - labels.assert_not_called() - - -def test_tenant_from_subject_namespace_ignores_resource_labels(): - params = PrometheusParams() - with patch.object(base_params, "PROMETHEUS_TENANT_LABEL", "team"), patch( - NAMESPACE_LABELS, side_effect=_namespaces(NAMESPACES) - ): - params.post_event_initialization(_event(POD_SUBJECT)) - assert params.prometheus_additional_headers == {"X-Scope-OrgID": "corporate"} - - -def test_namespace_only_alert_gets_tenant(): - params = PrometheusParams() - with patch.object(base_params, "PROMETHEUS_TENANT_LABEL", "team"), patch( - NAMESPACE_LABELS, side_effect=_namespaces(NAMESPACES) - ): - params.post_event_initialization(_event(NAMESPACE_ONLY_SUBJECT)) - assert params.prometheus_additional_headers == {"X-Scope-OrgID": "corporate"} - - -def test_tenant_header_replaces_sent_one_and_keeps_others(): - params = PrometheusParams(prometheus_additional_headers={"X-Scope-OrgID": "forged", "X-Other": "1"}) - with patch.object(base_params, "PROMETHEUS_TENANT_LABEL", "team"), patch( - NAMESPACE_LABELS, side_effect=_namespaces(NAMESPACES) - ): - params.post_event_initialization(_event(POD_SUBJECT)) - assert params.prometheus_additional_headers == {"X-Scope-OrgID": "corporate", "X-Other": "1"} - - -def test_no_namespace_keeps_default_tenant(): - params = PrometheusParams(prometheus_additional_headers={"X-Scope-OrgID": "default"}) - with patch.object(base_params, "PROMETHEUS_TENANT_LABEL", "team"), patch(NAMESPACE_LABELS) as labels: - params.post_event_initialization(_event(UNRESOLVED_SUBJECT)) - assert params.prometheus_additional_headers == {"X-Scope-OrgID": "default"} - labels.assert_not_called() - - -def test_query_params_namespace_wins_over_event_subject(): - params = PrometheusQueryParams(promql_query="up", duration={"duration_minutes": 5}, namespace="billing") - with patch.object(base_params, "PROMETHEUS_TENANT_LABEL", "team"), patch( - NAMESPACE_LABELS, side_effect=_namespaces(NAMESPACES) - ): - params.post_event_initialization(_event(POD_SUBJECT)) - assert params.prometheus_additional_headers == {"X-Scope-OrgID": "retail"} - - -def test_requests_do_not_share_headers(): - # Two requests built from the same global config, as merge_global_params does, for different tenants - global_config = {"prometheus_additional_headers": {"X-Scope-OrgID": "default"}} - with patch.object(base_params, "PROMETHEUS_TENANT_LABEL", "team"), patch( - NAMESPACE_LABELS, side_effect=_namespaces(NAMESPACES) - ): - first, second = [ - PrometheusQueryParams( - **global_config.copy(), promql_query="up", duration={"duration_minutes": 5}, namespace=namespace - ) - for namespace in ("shop", "billing") - ] - first.post_event_initialization(_event(UNRESOLVED_SUBJECT)) - second.post_event_initialization(_event(UNRESOLVED_SUBJECT)) - - assert first.prometheus_additional_headers == {"X-Scope-OrgID": "corporate"} - assert second.prometheus_additional_headers == {"X-Scope-OrgID": "retail"} - assert global_config == {"prometheus_additional_headers": {"X-Scope-OrgID": "default"}} +def test_configured_headers_are_not_changed(tenant_label): + configured = {"X-Scope-OrgID": "default"} + _headers(namespace="shop", prometheus_additional_headers=configured) + assert configured == {"X-Scope-OrgID": "default"} From e7ce27d2cc0fa800b4b3206d03845dfc0afe947b Mon Sep 17 00:00:00 2001 From: Roi Glinik Date: Thu, 29 Oct 2026 11:01:29 +0200 Subject: [PATCH 3/7] ROB-1291 Alert graphs pick the Prometheus tenant from their namespace label create_graph_enrichment fills the params' namespace from the labels it puts in the query, on a copy, so alert and resource graphs query their namespace's tenant. Co-Authored-By: Claude Opus 5.5 --- .../playbooks/prometheus_enrichment_utils.py | 3 +++ tests/test_prometheus_tenant.py | 22 +++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/src/robusta/core/playbooks/prometheus_enrichment_utils.py b/src/robusta/core/playbooks/prometheus_enrichment_utils.py index 85fea41fe..87c21e244 100644 --- a/src/robusta/core/playbooks/prometheus_enrichment_utils.py +++ b/src/robusta/core/playbooks/prometheus_enrichment_utils.py @@ -496,6 +496,9 @@ def create_graph_enrichment( metrics_legends_labels: Optional[List[str]] = None, ) -> GraphBlock: promql_query = __prepare_promql_query(labels, promql_query) + if not prometheus_params.namespace and labels.get("namespace"): + # The graph is about the namespace it fills into the query, so it picks the tenant. A copy: params are shared + prometheus_params = prometheus_params.copy(update={"namespace": labels["namespace"]}) chart, prom_block = create_chart_from_prometheus_query( prometheus_params, promql_query, diff --git a/tests/test_prometheus_tenant.py b/tests/test_prometheus_tenant.py index 59434b07d..ca7c16f20 100644 --- a/tests/test_prometheus_tenant.py +++ b/tests/test_prometheus_tenant.py @@ -1,8 +1,10 @@ +from datetime import datetime from unittest.mock import patch import pytest from robusta.core.model.base_params import PrometheusParams +from robusta.core.playbooks import prometheus_enrichment_utils from robusta.integrations.prometheus import utils from robusta.integrations.prometheus.utils import generate_prometheus_config @@ -50,6 +52,26 @@ def test_feature_off_reads_nothing(tenant_label): tenant_label.assert_not_called() +@pytest.mark.parametrize( + "params_namespace, labels, expected", + [ + (None, {"namespace": "shop", "pod": "web"}, "shop"), # alert graph: namespace from the labels + ("billing", {"namespace": "shop"}, "billing"), # an explicit namespace wins + (None, {"node": "node-1"}, None), # node graph: default tenant + ], +) +def test_graph_enrichment_namespace(params_namespace, labels, expected): + params = PrometheusParams(namespace=params_namespace) + with patch.object(prometheus_enrichment_utils, "create_chart_from_prometheus_query") as create_chart: + create_chart.side_effect = Exception("stop") + with pytest.raises(Exception, match="stop"): + prometheus_enrichment_utils.create_graph_enrichment( + datetime.now(), labels, "up", params, 60, None, None + ) + assert create_chart.call_args.args[0].namespace == expected + assert params.namespace == params_namespace + + def test_configured_headers_are_not_changed(tenant_label): configured = {"X-Scope-OrgID": "default"} _headers(namespace="shop", prometheus_additional_headers=configured) From d31e176043b25c4b4d2fca14617e61d59b450df6 Mon Sep 17 00:00:00 2001 From: Roi Glinik Date: Thu, 29 Oct 2026 11:24:26 +0200 Subject: [PATCH 4/7] ROB-1291 Gate the tenant on PROMETHEUS_TENANT_LABEL, name it get_namespace_tenant With the label unset, generate_prometheus_config and create_graph_enrichment run exactly as before. Co-Authored-By: Claude Opus 5.5 --- .../playbooks/prometheus_enrichment_utils.py | 11 ++++++--- src/robusta/integrations/prometheus/utils.py | 23 ++++++++++--------- tests/test_prometheus_tenant.py | 15 +++++++----- 3 files changed, 29 insertions(+), 20 deletions(-) diff --git a/src/robusta/core/playbooks/prometheus_enrichment_utils.py b/src/robusta/core/playbooks/prometheus_enrichment_utils.py index 87c21e244..4b4ef9b87 100644 --- a/src/robusta/core/playbooks/prometheus_enrichment_utils.py +++ b/src/robusta/core/playbooks/prometheus_enrichment_utils.py @@ -16,7 +16,11 @@ ResourceChartItemType, ResourceChartResourceType, ) -from robusta.core.model.env_vars import FLOAT_PRECISION_LIMIT, PROMETHEUS_REQUEST_TIMEOUT_SECONDS +from robusta.core.model.env_vars import ( + FLOAT_PRECISION_LIMIT, + PROMETHEUS_REQUEST_TIMEOUT_SECONDS, + PROMETHEUS_TENANT_LABEL, +) from robusta.core.reporting.blocks import GraphBlock, PrometheusBlock, PrometheusBlockLineData from robusta.core.reporting.charts import XYChart from robusta.core.reporting.custom_rendering import charts_style @@ -496,8 +500,9 @@ def create_graph_enrichment( metrics_legends_labels: Optional[List[str]] = None, ) -> GraphBlock: promql_query = __prepare_promql_query(labels, promql_query) - if not prometheus_params.namespace and labels.get("namespace"): - # The graph is about the namespace it fills into the query, so it picks the tenant. A copy: params are shared + if PROMETHEUS_TENANT_LABEL and not prometheus_params.namespace and labels.get("namespace"): + # Multi-tenant Prometheus: the graph is about the namespace it fills into the query, so that namespace picks + # the tenant. Set on a copy, since an action reuses its params for several graphs prometheus_params = prometheus_params.copy(update={"namespace": labels["namespace"]}) chart, prom_block = create_chart_from_prometheus_query( prometheus_params, diff --git a/src/robusta/integrations/prometheus/utils.py b/src/robusta/integrations/prometheus/utils.py index 2413d7043..45e4b9896 100644 --- a/src/robusta/integrations/prometheus/utils.py +++ b/src/robusta/integrations/prometheus/utils.py @@ -42,13 +42,12 @@ VICTORIA_METRICS_CONFIGURED = os.environ.get("VICTORIA_METRICS_CONFIGURED", "false").lower() == "true" -def get_prometheus_tenant(namespace: Optional[str]) -> Optional[str]: +def get_namespace_tenant(namespace: str) -> Optional[str]: """ - Multi-tenant Prometheus (e.g. Mimir): the value of the PROMETHEUS_TENANT_LABEL label of the namespace, read from - the cached namespace data (NAMESPACE_DATA_TTL). None when the feature is off, or there's no namespace or label. + Multi-tenant Prometheus (e.g. Mimir): the tenant of a namespace, which is the value of its PROMETHEUS_TENANT_LABEL + label, read from the cached namespace data (NAMESPACE_DATA_TTL). None when the namespace has no such label. + Call it only with PROMETHEUS_TENANT_LABEL set. """ - if not PROMETHEUS_TENANT_LABEL or not namespace: - return None try: tenant = (get_namespace_labels(namespace) or {}).get(PROMETHEUS_TENANT_LABEL) except KeyError: # a namespace created after the last reload, or deleted @@ -79,12 +78,14 @@ def generate_prometheus_config(prometheus_params: PrometheusParams) -> Prometheu if prometheus_params.prometheus_auth: baseconfig["prometheus_auth"] = prometheus_params.prometheus_auth.get_secret_value() - headers = dict(prometheus_params.prometheus_additional_headers or {}) - tenant = get_prometheus_tenant(prometheus_params.namespace) - if tenant: # replaces any tenant header from global_config or the request - headers[PROMETHEUS_TENANT_HEADER] = tenant - if headers: - baseconfig["headers"] = headers + if prometheus_params.prometheus_additional_headers: + baseconfig["headers"] = prometheus_params.prometheus_additional_headers + + if PROMETHEUS_TENANT_LABEL and prometheus_params.namespace: + tenant = get_namespace_tenant(prometheus_params.namespace) + if tenant: + # A new dict that replaces any tenant header from global_config or the request, leaving theirs unchanged + baseconfig["headers"] = {**baseconfig.get("headers", {}), PROMETHEUS_TENANT_HEADER: tenant} # aws config if AWS_REGION: diff --git a/tests/test_prometheus_tenant.py b/tests/test_prometheus_tenant.py index ca7c16f20..6f964ea8f 100644 --- a/tests/test_prometheus_tenant.py +++ b/tests/test_prometheus_tenant.py @@ -53,16 +53,19 @@ def test_feature_off_reads_nothing(tenant_label): @pytest.mark.parametrize( - "params_namespace, labels, expected", + "tenant_label_env, params_namespace, labels, expected", [ - (None, {"namespace": "shop", "pod": "web"}, "shop"), # alert graph: namespace from the labels - ("billing", {"namespace": "shop"}, "billing"), # an explicit namespace wins - (None, {"node": "node-1"}, None), # node graph: default tenant + ("team", None, {"namespace": "shop", "pod": "web"}, "shop"), # alert graph: namespace from the labels + ("team", "billing", {"namespace": "shop"}, "billing"), # an explicit namespace wins + ("team", None, {"node": "node-1"}, None), # node graph: default tenant + (None, None, {"namespace": "shop"}, None), # feature off: params untouched ], ) -def test_graph_enrichment_namespace(params_namespace, labels, expected): +def test_graph_enrichment_namespace(tenant_label_env, params_namespace, labels, expected): params = PrometheusParams(namespace=params_namespace) - with patch.object(prometheus_enrichment_utils, "create_chart_from_prometheus_query") as create_chart: + with patch.object(prometheus_enrichment_utils, "PROMETHEUS_TENANT_LABEL", tenant_label_env), patch.object( + prometheus_enrichment_utils, "create_chart_from_prometheus_query" + ) as create_chart: create_chart.side_effect = Exception("stop") with pytest.raises(Exception, match="stop"): prometheus_enrichment_utils.create_graph_enrichment( From 48ca03afc1a4ad28b963ca533e5171b3df0d9e06 Mon Sep 17 00:00:00 2001 From: Roi Glinik Date: Thu, 29 Oct 2026 11:29:56 +0200 Subject: [PATCH 5/7] ROB-1291 Tenant only for graphs about a namespace; graph the alert expression from its tenant create_graph_enrichment takes the namespace only when its query fills in $namespace, so node graphs keep the default tenant. graph_enricher uses the alert's namespace. Both go through with_namespace_tenant. Co-Authored-By: Claude Opus 5.5 --- .../robusta_playbooks/alerts_integration.py | 4 +- .../playbooks/prometheus_enrichment_utils.py | 21 +++++++++-- tests/test_prometheus_tenant.py | 37 +++++++++++++++---- 3 files changed, 50 insertions(+), 12 deletions(-) diff --git a/playbooks/robusta_playbooks/alerts_integration.py b/playbooks/robusta_playbooks/alerts_integration.py index 021ebac52..935aace5a 100644 --- a/playbooks/robusta_playbooks/alerts_integration.py +++ b/playbooks/robusta_playbooks/alerts_integration.py @@ -45,6 +45,7 @@ get_node_internal_ip, ) from robusta.core.playbooks.oom_killer_utils import logs_enricher, start_log_enrichment +from robusta.core.playbooks.prometheus_enrichment_utils import with_namespace_tenant from robusta.core.reporting import FindingSubject from robusta.core.reporting.base import Link, LinkType from robusta.core.reporting.blocks import TableBlockFormat @@ -305,7 +306,8 @@ def graph_enricher(alert: PrometheusKubernetesAlert, params: GraphEnricherParams """ promql_query = alert.get_prometheus_query() chart, prom_block = create_chart_from_prometheus_query( - params, + # The alert fired about its namespace, so its expression is graphed from that namespace's tenant + with_namespace_tenant(params, alert.alert.labels.get("namespace")), promql_query, alert.alert.startsAt, include_x_axis=False, diff --git a/src/robusta/core/playbooks/prometheus_enrichment_utils.py b/src/robusta/core/playbooks/prometheus_enrichment_utils.py index 4b4ef9b87..fa0dbc8bd 100644 --- a/src/robusta/core/playbooks/prometheus_enrichment_utils.py +++ b/src/robusta/core/playbooks/prometheus_enrichment_utils.py @@ -57,6 +57,9 @@ def __init__( self.dasharray = dasharray +NAMESPACE_PLACEHOLDER = re.compile(r"\$(namespace\b|\{namespace\})") + + def __prepare_promql_query(provided_labels: Dict[Any, Any], promql_query_template: str) -> str: labels: Dict[Any, Any] = defaultdict(lambda: "") labels.update(provided_labels) @@ -65,6 +68,16 @@ def __prepare_promql_query(provided_labels: Dict[Any, Any], promql_query_templat return promql_query +def with_namespace_tenant(prometheus_params: PrometheusParams, namespace: Optional[str]) -> PrometheusParams: + """ + Multi-tenant Prometheus: params whose queries go to the tenant of `namespace`, unless they already name one. + Returns a copy, since an action reuses its params for several graphs. + """ + if not PROMETHEUS_TENANT_LABEL or prometheus_params.namespace or not namespace: + return prometheus_params + return prometheus_params.copy(update={"namespace": namespace}) + + def get_node_internal_ip(node: Node) -> str: internal_ip = next(addr.address for addr in node.status.addresses if addr.type == "InternalIP") return internal_ip @@ -499,11 +512,11 @@ def create_graph_enrichment( hide_legends: Optional[bool] = False, metrics_legends_labels: Optional[List[str]] = None, ) -> GraphBlock: + # Only a query that fills in $namespace is about that namespace. Node queries aren't, even when the labels have + # one (e.g. the exporter's namespace on a node alert), so they keep the default tenant + if NAMESPACE_PLACEHOLDER.search(promql_query): + prometheus_params = with_namespace_tenant(prometheus_params, labels.get("namespace")) promql_query = __prepare_promql_query(labels, promql_query) - if PROMETHEUS_TENANT_LABEL and not prometheus_params.namespace and labels.get("namespace"): - # Multi-tenant Prometheus: the graph is about the namespace it fills into the query, so that namespace picks - # the tenant. Set on a copy, since an action reuses its params for several graphs - prometheus_params = prometheus_params.copy(update={"namespace": labels["namespace"]}) chart, prom_block = create_chart_from_prometheus_query( prometheus_params, promql_query, diff --git a/tests/test_prometheus_tenant.py b/tests/test_prometheus_tenant.py index 6f964ea8f..8a82f76cf 100644 --- a/tests/test_prometheus_tenant.py +++ b/tests/test_prometheus_tenant.py @@ -52,16 +52,22 @@ def test_feature_off_reads_nothing(tenant_label): tenant_label.assert_not_called() +POD_QUERY = 'container_memory_working_set_bytes{namespace="$namespace", pod=~"$pod"}' +NODE_QUERY = 'node_memory_MemAvailable_bytes{instance=~"$node_internal_ip:.*"}' + + @pytest.mark.parametrize( - "tenant_label_env, params_namespace, labels, expected", + "tenant_label_env, params_namespace, query, labels, expected", [ - ("team", None, {"namespace": "shop", "pod": "web"}, "shop"), # alert graph: namespace from the labels - ("team", "billing", {"namespace": "shop"}, "billing"), # an explicit namespace wins - ("team", None, {"node": "node-1"}, None), # node graph: default tenant - (None, None, {"namespace": "shop"}, None), # feature off: params untouched + ("team", None, POD_QUERY, {"namespace": "shop", "pod": "web"}, "shop"), # pod graph: its namespace + ("team", None, 'up{namespace="${namespace}"}', {"namespace": "shop"}, "shop"), # braced placeholder + ("team", "billing", POD_QUERY, {"namespace": "shop"}, "billing"), # an explicit namespace wins + # node graph on a node alert that carries the exporter's namespace: not about it, default tenant + ("team", None, NODE_QUERY, {"namespace": "monitoring", "node_internal_ip": "10.0.0.1"}, None), + (None, None, POD_QUERY, {"namespace": "shop"}, None), # feature off: params untouched ], ) -def test_graph_enrichment_namespace(tenant_label_env, params_namespace, labels, expected): +def test_graph_enrichment_namespace(tenant_label_env, params_namespace, query, labels, expected): params = PrometheusParams(namespace=params_namespace) with patch.object(prometheus_enrichment_utils, "PROMETHEUS_TENANT_LABEL", tenant_label_env), patch.object( prometheus_enrichment_utils, "create_chart_from_prometheus_query" @@ -69,12 +75,29 @@ def test_graph_enrichment_namespace(tenant_label_env, params_namespace, labels, create_chart.side_effect = Exception("stop") with pytest.raises(Exception, match="stop"): prometheus_enrichment_utils.create_graph_enrichment( - datetime.now(), labels, "up", params, 60, None, None + datetime.now(), labels, query, params, 60, None, None ) assert create_chart.call_args.args[0].namespace == expected assert params.namespace == params_namespace +@pytest.mark.parametrize( + "tenant_label_env, params_namespace, namespace, expected", + [ + ("team", None, "shop", "shop"), # e.g. graph_enricher with the alert's namespace + ("team", "billing", "shop", "billing"), + ("team", None, None, None), + (None, None, "shop", None), + ], +) +def test_with_namespace_tenant(tenant_label_env, params_namespace, namespace, expected): + params = PrometheusParams(namespace=params_namespace) + with patch.object(prometheus_enrichment_utils, "PROMETHEUS_TENANT_LABEL", tenant_label_env): + result = prometheus_enrichment_utils.with_namespace_tenant(params, namespace) + assert result.namespace == expected + assert params.namespace == params_namespace + + def test_configured_headers_are_not_changed(tenant_label): configured = {"X-Scope-OrgID": "default"} _headers(namespace="shop", prometheus_additional_headers=configured) From cfe73d7e6306225587fb0ae1449ef3ed77ed57e6 Mon Sep 17 00:00:00 2001 From: Roi Glinik Date: Thu, 29 Oct 2026 14:54:50 +0200 Subject: [PATCH 6/7] ROB-1291 Test tenant label keys with dots and a DNS prefix Co-Authored-By: Claude Opus 5.5 --- tests/test_prometheus_tenant.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_prometheus_tenant.py b/tests/test_prometheus_tenant.py index 8a82f76cf..58193f0b4 100644 --- a/tests/test_prometheus_tenant.py +++ b/tests/test_prometheus_tenant.py @@ -30,6 +30,15 @@ def test_tenant_from_namespace_label(tenant_label): assert _headers(namespace="shop") == {"X-Scope-OrgID": "corporate"} +@pytest.mark.parametrize("label_key", ["product.productline", "example.com/productline"]) +def test_tenant_label_key_with_dots_or_prefix(label_key): + # The label key is used as written: dots and a DNS prefix are just part of the key + with patch.object(utils, "PROMETHEUS_TENANT_LABEL", label_key), patch.object( + utils, "get_namespace_labels", return_value={label_key: "corporate", "productline": "other"} + ): + assert _headers(namespace="shop") == {"X-Scope-OrgID": "corporate"} + + def test_tenant_replaces_configured_one_and_keeps_other_headers(tenant_label): headers = _headers(namespace="billing", prometheus_additional_headers={"X-Scope-OrgID": "default", "X-Other": "1"}) assert headers == {"X-Scope-OrgID": "retail", "X-Other": "1"} From fb4ccaff19aaa25fee24066d852c356e9d5afe10 Mon Sep 17 00:00:00 2001 From: Roi Glinik Date: Mon, 2 Nov 2026 13:35:07 +0200 Subject: [PATCH 7/7] ROB-1291 Detect $namespace with Template's parser, ignoring escaped $$namespace Co-Authored-By: Claude Opus 5.5 --- src/robusta/core/playbooks/prometheus_enrichment_utils.py | 5 +---- tests/test_prometheus_tenant.py | 1 + 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/robusta/core/playbooks/prometheus_enrichment_utils.py b/src/robusta/core/playbooks/prometheus_enrichment_utils.py index fa0dbc8bd..dad504ba9 100644 --- a/src/robusta/core/playbooks/prometheus_enrichment_utils.py +++ b/src/robusta/core/playbooks/prometheus_enrichment_utils.py @@ -57,9 +57,6 @@ def __init__( self.dasharray = dasharray -NAMESPACE_PLACEHOLDER = re.compile(r"\$(namespace\b|\{namespace\})") - - def __prepare_promql_query(provided_labels: Dict[Any, Any], promql_query_template: str) -> str: labels: Dict[Any, Any] = defaultdict(lambda: "") labels.update(provided_labels) @@ -514,7 +511,7 @@ def create_graph_enrichment( ) -> GraphBlock: # Only a query that fills in $namespace is about that namespace. Node queries aren't, even when the labels have # one (e.g. the exporter's namespace on a node alert), so they keep the default tenant - if NAMESPACE_PLACEHOLDER.search(promql_query): + if "namespace" in Template(promql_query).get_identifiers(): prometheus_params = with_namespace_tenant(prometheus_params, labels.get("namespace")) promql_query = __prepare_promql_query(labels, promql_query) chart, prom_block = create_chart_from_prometheus_query( diff --git a/tests/test_prometheus_tenant.py b/tests/test_prometheus_tenant.py index 58193f0b4..118f2b47e 100644 --- a/tests/test_prometheus_tenant.py +++ b/tests/test_prometheus_tenant.py @@ -70,6 +70,7 @@ def test_feature_off_reads_nothing(tenant_label): [ ("team", None, POD_QUERY, {"namespace": "shop", "pod": "web"}, "shop"), # pod graph: its namespace ("team", None, 'up{namespace="${namespace}"}', {"namespace": "shop"}, "shop"), # braced placeholder + ("team", None, 'up{label="$$namespace"}', {"namespace": "shop"}, None), # escaped: a literal "$namespace" ("team", "billing", POD_QUERY, {"namespace": "shop"}, "billing"), # an explicit namespace wins # node graph on a node alert that carries the exporter's namespace: not about it, default tenant ("team", None, NODE_QUERY, {"namespace": "monitoring", "node_internal_ip": "10.0.0.1"}, None),