diff --git a/src/robusta/core/model/base_params.py b/src/robusta/core/model/base_params.py index 70da1c0f4..4d090b3c9 100644 --- a/src/robusta/core/model/base_params.py +++ b/src/robusta/core/model/base_params.py @@ -4,6 +4,7 @@ from pydantic.v1 import BaseModel, SecretStr, validator, Field +from robusta.core.model.env_vars import PROMETHEUS_TENANT_HEADER, PROMETHEUS_TENANT_LABEL from robusta.integrations import openshift from robusta.utils.documented_pydantic import DocumentedModel @@ -68,6 +69,12 @@ def post_initialization(self): """ pass + def post_event_initialization(self, event) -> None: + """ + Runs right before the action, with the event it runs on + """ + pass + pass @@ -366,6 +373,28 @@ def auto_openshift_token(cls, v: Optional[SecretStr]): return None + def _tenant_namespace(self, event) -> Optional[str]: + return event.get_subject().namespace + + def post_event_initialization(self, event) -> None: + """ + With PROMETHEUS_TENANT_LABEL set, sends the tenant of the action's namespace in PROMETHEUS_TENANT_HEADER. + Every Prometheus call made with these params gets it, since they all build their connection from them. + With no namespace or no label, the configured prometheus_additional_headers are used as they are. + """ + if not PROMETHEUS_TENANT_LABEL: + return + from robusta.integrations.prometheus.tenant import resolve_tenant + + tenant = resolve_tenant(PROMETHEUS_TENANT_LABEL, self._tenant_namespace(event)) + if not tenant: + return + # A new dict, set last: it replaces any tenant header the request itself sent, and never changes a shared one + self.prometheus_additional_headers = { + **(self.prometheus_additional_headers or {}), + PROMETHEUS_TENANT_HEADER: tenant, + } + class PrometheusDuration(BaseModel): """ @@ -401,11 +430,17 @@ class PrometheusQueryParams(PrometheusParams): :var promql_query: the prometheusql query you want to run :var duration: the duration of the query :var step: (str) Query resolution step width in duration format or float number of seconds - i.e 100s, 3d, 2w, 170.3 + :var namespace: the namespace the query is about. With PROMETHEUS_TENANT_LABEL set, its label picks the tenant """ promql_query: str duration: Union[PrometheusDateRange, PrometheusDuration] step: Optional[str] + namespace: Optional[str] = None + + def _tenant_namespace(self, event) -> Optional[str]: + # prometheus_enricher runs on an event with no subject, so the UI names the namespace in the params + return self.namespace or super()._tenant_namespace(event) class TimedPrometheusParams(PrometheusParams): diff --git a/src/robusta/core/model/env_vars.py b/src/robusta/core/model/env_vars.py index 3b07363bc..a8d74ca04 100644 --- a/src/robusta/core/model/env_vars.py +++ b/src/robusta/core/model/env_vars.py @@ -49,6 +49,10 @@ def load_bool(env_var, default: bool): PROMETHEUS_ENABLED = os.environ.get("PROMETHEUS_ENABLED", "false").lower() == "true" MANAGED_CONFIGURATION_ENABLED = os.environ.get("MANAGED_CONFIGURATION_ENABLED", "false").lower() == "true" PROMETHEUS_SSL_ENABLED = os.environ.get("PROMETHEUS_SSL_ENABLED", "false").lower() == "true" +# Multi-tenant Prometheus (e.g. Mimir): label on the action's namespace, sent as the tenant header. +# Env vars, not global_config, so a request from the UI can't override them. +PROMETHEUS_TENANT_LABEL = os.environ.get("PROMETHEUS_TENANT_LABEL") +PROMETHEUS_TENANT_HEADER = os.environ.get("PROMETHEUS_TENANT_HEADER", "X-Scope-OrgID") INCOMING_REQUEST_TIME_WINDOW_SECONDS = int(os.environ.get("INCOMING_REQUEST_TIME_WINDOW_SECONDS", 3600)) diff --git a/src/robusta/core/playbooks/playbooks_event_handler_impl.py b/src/robusta/core/playbooks/playbooks_event_handler_impl.py index 110190b4a..1c0b95e6d 100644 --- a/src/robusta/core/playbooks/playbooks_event_handler_impl.py +++ b/src/robusta/core/playbooks/playbooks_event_handler_impl.py @@ -239,6 +239,7 @@ def __run_playbook_actions( continue try: if action_with_params: + params.post_event_initialization(execution_event) registered_action.func(execution_event, params) else: registered_action.func(execution_event) diff --git a/src/robusta/integrations/prometheus/models.py b/src/robusta/integrations/prometheus/models.py index d169f4a1a..48a4ecd42 100644 --- a/src/robusta/integrations/prometheus/models.py +++ b/src/robusta/integrations/prometheus/models.py @@ -193,6 +193,7 @@ def get_alert_subject(self) -> FindingSubject: elif self.hpa: subject_type = FindingSubjectType.TYPE_HPA name = self.hpa.metadata.name + namespace = self.hpa.metadata.namespace labels = self.hpa.metadata.labels annotations = self.hpa.metadata.annotations diff --git a/src/robusta/integrations/prometheus/tenant.py b/src/robusta/integrations/prometheus/tenant.py new file mode 100644 index 000000000..73872f0af --- /dev/null +++ b/src/robusta/integrations/prometheus/tenant.py @@ -0,0 +1,20 @@ +import logging +from typing import Optional + +from robusta.integrations.kubernetes.api_client_utils import get_namespace_labels + + +def resolve_tenant(key: str, namespace: Optional[str]) -> Optional[str]: + """ + Returns the value of the `key` label of a namespace, read from the cached namespace data (NAMESPACE_DATA_TTL). + Returns None when there is no namespace or no such label, so the caller keeps its configured default tenant. + """ + if not namespace: + return None + try: + tenant = (get_namespace_labels(namespace) or {}).get(key) + except KeyError: # a namespace created after the last reload, or deleted + tenant = None + if not tenant: + logging.warning(f"No '{key}' label on namespace {namespace}, using the default Prometheus tenant") + return tenant diff --git a/tests/test_prometheus_tenant.py b/tests/test_prometheus_tenant.py new file mode 100644 index 000000000..d7598f138 --- /dev/null +++ b/tests/test_prometheus_tenant.py @@ -0,0 +1,125 @@ +from types import SimpleNamespace +from unittest.mock import patch + +from robusta.core.model import base_params +from robusta.core.model.base_params import PrometheusParams, PrometheusQueryParams +from robusta.core.reporting.base import FindingSubject +from robusta.core.reporting.consts import FindingSubjectType +from robusta.integrations.prometheus.tenant import resolve_tenant + +NAMESPACE_LABELS = "robusta.integrations.prometheus.tenant.get_namespace_labels" + + +def _namespaces(labels_by_namespace): + return lambda namespace: labels_by_namespace[namespace] + + +def test_tenant_from_namespace_label(): + with patch(NAMESPACE_LABELS, side_effect=_namespaces({"shop": {"team": "corporate"}})): + assert resolve_tenant("team", "shop") == "corporate" + + +def test_no_namespace_reads_nothing(): + with patch(NAMESPACE_LABELS) as labels: + assert resolve_tenant("team", None) is None + labels.assert_not_called() + + +def test_missing_label_returns_none(): + with patch(NAMESPACE_LABELS, side_effect=_namespaces({"shop": {"app": "web"}})): + assert resolve_tenant("team", "shop") is None + + +def test_namespace_with_no_labels_returns_none(): + with patch(NAMESPACE_LABELS, side_effect=_namespaces({"shop": None})): + assert resolve_tenant("team", "shop") is None + + +def test_unknown_namespace_returns_none(): + with patch(NAMESPACE_LABELS, side_effect=KeyError("shop")): + assert resolve_tenant("team", "shop") is None + + +def _event(subject: FindingSubject): + return SimpleNamespace(get_subject=lambda: subject) + + +# The pod and alert labels carry another value on purpose: only the namespace label counts +POD_SUBJECT = FindingSubject( + name="web", subject_type=FindingSubjectType.TYPE_POD, namespace="shop", labels={"team": "from-pod"} +) +NAMESPACE_ONLY_SUBJECT = FindingSubject(name="Unresolved", subject_type=FindingSubjectType.TYPE_NONE, namespace="shop") +UNRESOLVED_SUBJECT = FindingSubject(name="Unresolved", subject_type=FindingSubjectType.TYPE_NONE) +NAMESPACES = {"shop": {"team": "corporate"}, "billing": {"team": "retail"}} + + +def test_no_tenant_label_configured_changes_nothing(): + params = PrometheusParams(prometheus_additional_headers={"X-Scope-OrgID": "sent"}) + with patch.object(base_params, "PROMETHEUS_TENANT_LABEL", None), patch(NAMESPACE_LABELS) as labels: + params.post_event_initialization(_event(POD_SUBJECT)) + assert params.prometheus_additional_headers == {"X-Scope-OrgID": "sent"} + labels.assert_not_called() + + +def test_tenant_from_subject_namespace_ignores_resource_labels(): + params = PrometheusParams() + with patch.object(base_params, "PROMETHEUS_TENANT_LABEL", "team"), patch( + NAMESPACE_LABELS, side_effect=_namespaces(NAMESPACES) + ): + params.post_event_initialization(_event(POD_SUBJECT)) + assert params.prometheus_additional_headers == {"X-Scope-OrgID": "corporate"} + + +def test_namespace_only_alert_gets_tenant(): + params = PrometheusParams() + with patch.object(base_params, "PROMETHEUS_TENANT_LABEL", "team"), patch( + NAMESPACE_LABELS, side_effect=_namespaces(NAMESPACES) + ): + params.post_event_initialization(_event(NAMESPACE_ONLY_SUBJECT)) + assert params.prometheus_additional_headers == {"X-Scope-OrgID": "corporate"} + + +def test_tenant_header_replaces_sent_one_and_keeps_others(): + params = PrometheusParams(prometheus_additional_headers={"X-Scope-OrgID": "forged", "X-Other": "1"}) + with patch.object(base_params, "PROMETHEUS_TENANT_LABEL", "team"), patch( + NAMESPACE_LABELS, side_effect=_namespaces(NAMESPACES) + ): + params.post_event_initialization(_event(POD_SUBJECT)) + assert params.prometheus_additional_headers == {"X-Scope-OrgID": "corporate", "X-Other": "1"} + + +def test_no_namespace_keeps_default_tenant(): + params = PrometheusParams(prometheus_additional_headers={"X-Scope-OrgID": "default"}) + with patch.object(base_params, "PROMETHEUS_TENANT_LABEL", "team"), patch(NAMESPACE_LABELS) as labels: + params.post_event_initialization(_event(UNRESOLVED_SUBJECT)) + assert params.prometheus_additional_headers == {"X-Scope-OrgID": "default"} + labels.assert_not_called() + + +def test_query_params_namespace_wins_over_event_subject(): + params = PrometheusQueryParams(promql_query="up", duration={"duration_minutes": 5}, namespace="billing") + with patch.object(base_params, "PROMETHEUS_TENANT_LABEL", "team"), patch( + NAMESPACE_LABELS, side_effect=_namespaces(NAMESPACES) + ): + params.post_event_initialization(_event(POD_SUBJECT)) + assert params.prometheus_additional_headers == {"X-Scope-OrgID": "retail"} + + +def test_requests_do_not_share_headers(): + # Two requests built from the same global config, as merge_global_params does, for different tenants + global_config = {"prometheus_additional_headers": {"X-Scope-OrgID": "default"}} + with patch.object(base_params, "PROMETHEUS_TENANT_LABEL", "team"), patch( + NAMESPACE_LABELS, side_effect=_namespaces(NAMESPACES) + ): + first, second = [ + PrometheusQueryParams( + **global_config.copy(), promql_query="up", duration={"duration_minutes": 5}, namespace=namespace + ) + for namespace in ("shop", "billing") + ] + first.post_event_initialization(_event(UNRESOLVED_SUBJECT)) + second.post_event_initialization(_event(UNRESOLVED_SUBJECT)) + + assert first.prometheus_additional_headers == {"X-Scope-OrgID": "corporate"} + assert second.prometheus_additional_headers == {"X-Scope-OrgID": "retail"} + assert global_config == {"prometheus_additional_headers": {"X-Scope-OrgID": "default"}}