From b12d769bba25272ef7cb1373707a54dc41b1df0c Mon Sep 17 00:00:00 2001 From: "NAYANA.R" Date: Thu, 8 Oct 2026 14:02:05 +0530 Subject: [PATCH 01/12] InstanaTracer.start_as_current_span() got an unexpected keyword argument context Signed-off-by: NAYANA.R Signed-off-by: Madhav Kandukuri --- mcpgateway/observability.py | 17 ++++++++- tests/unit/mcpgateway/test_observability.py | 40 +++++++++++++++++++++ 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/mcpgateway/observability.py b/mcpgateway/observability.py index c5ca8258dd1..065bf7d8860 100644 --- a/mcpgateway/observability.py +++ b/mcpgateway/observability.py @@ -1131,7 +1131,22 @@ async def _send_with_span_status(message: Mapping[str, Any]) -> None: span.set_status(Status(StatusCode.OK)) await send(message) - with _TRACER.start_as_current_span(span_name, **start_span_kwargs) as span: + try: + span_context_manager = _TRACER.start_as_current_span(span_name, **start_span_kwargs) + except TypeError: + # Some tracer implementations (e.g. Instana < 3.12) don't accept the + # `context` keyword argument even though it is part of the + # OpenTelemetry Tracer API. Retry without an explicit parent context + # so tracer incompatibilities never break inbound requests. + compatible_kwargs = {key: value for key, value in start_span_kwargs.items() if key != "context"} + span_context_manager = _TRACER.start_as_current_span(span_name, **compatible_kwargs) + logger.debug( + "Tracer %r rejected `context` kwarg; started span %r without explicit parent context", + type(_TRACER).__name__, + span_name, + ) + + with span_context_manager as span: # No valid inbound envelope: publish this new root span's context into # the ASGI headers so downstream raw-header copies (session-task handoff, # affinity envelopes, trusted-internal dispatch) carry the trace and diff --git a/tests/unit/mcpgateway/test_observability.py b/tests/unit/mcpgateway/test_observability.py index 705af2d52c3..eaf30b76833 100644 --- a/tests/unit/mcpgateway/test_observability.py +++ b/tests/unit/mcpgateway/test_observability.py @@ -7,6 +7,7 @@ """ # Standard +from contextlib import nullcontext import inspect import logging import os @@ -832,3 +833,42 @@ async def test_middleware_envelope_publication_failure_is_nonfatal(self): await middleware(scope, receive, send) assert captured["headers"] == original_headers + + @pytest.mark.asyncio + async def test_middleware_tolerates_tracer_without_context_kwarg(self): + """A tracer that rejects `context=` (e.g. Instana < 3.12) must not break the request.""" + captured = {} + app, receive, send = self._make_app(captured) + scope = self._make_scope([(b"content-type", b"application/json")]) + + class ContextRejectingTracer: + """Emulates InstanaTracer < 3.12: start_as_current_span() lacks `context`.""" + + def __init__(self): + self.attempts = [] + + def start_as_current_span(self, _name, **kwargs): + self.attempts.append(sorted(kwargs)) + if "context" in kwargs: + raise TypeError("start_as_current_span() got an unexpected keyword argument 'context'") + return nullcontext(None) + + tracer = ContextRejectingTracer() + mock_trace = MagicMock() + mock_trace.get_current_span.return_value.get_span_context.return_value = MagicMock(is_valid=False, is_remote=False) + + with ( + patch("mcpgateway.observability._TRACER", tracer), + patch("mcpgateway.observability.OTEL_AVAILABLE", True), + patch("mcpgateway.observability.trace", mock_trace), + patch("mcpgateway.observability.otel_extract", return_value=MagicMock()), + patch("mcpgateway.observability.otel_inject", side_effect=self._fake_inject), + ): + middleware = OpenTelemetryRequestMiddleware(app) + await middleware(scope, receive, send) + + # First attempt passed the parent `context`; the compatibility fallback dropped it. + assert ["context", "kind"] in tracer.attempts + assert ["kind"] in tracer.attempts + # The request still completed and the trace envelope was published. + assert (b"traceparent", b"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01") in captured["headers"] From e9b09e57c9f26ffe19da999f34c3f4b42393bd9a Mon Sep 17 00:00:00 2001 From: Madhav Kandukuri Date: Fri, 9 Oct 2026 18:01:01 +0530 Subject: [PATCH 02/12] fix: narrow Instana tracer compatibility fallback Signed-off-by: Madhav Kandukuri --- mcpgateway/observability.py | 10 ++-- tests/unit/mcpgateway/test_observability.py | 58 +++++++++++++++++++++ 2 files changed, 65 insertions(+), 3 deletions(-) diff --git a/mcpgateway/observability.py b/mcpgateway/observability.py index 065bf7d8860..a646d1f3a17 100644 --- a/mcpgateway/observability.py +++ b/mcpgateway/observability.py @@ -1133,11 +1133,15 @@ async def _send_with_span_status(message: Mapping[str, Any]) -> None: try: span_context_manager = _TRACER.start_as_current_span(span_name, **start_span_kwargs) - except TypeError: + except TypeError as exc: # Some tracer implementations (e.g. Instana < 3.12) don't accept the # `context` keyword argument even though it is part of the - # OpenTelemetry Tracer API. Retry without an explicit parent context - # so tracer incompatibilities never break inbound requests. + # OpenTelemetry Tracer API. Retry only for that known incompatibility; + # unrelated TypeErrors must retain their original behavior. + error_message = str(exc) + rejected_context = "context" in start_span_kwargs and "unexpected keyword argument" in error_message and ("'context'" in error_message or '"context"' in error_message) + if not rejected_context: + raise compatible_kwargs = {key: value for key, value in start_span_kwargs.items() if key != "context"} span_context_manager = _TRACER.start_as_current_span(span_name, **compatible_kwargs) logger.debug( diff --git a/tests/unit/mcpgateway/test_observability.py b/tests/unit/mcpgateway/test_observability.py index eaf30b76833..606cf64df4a 100644 --- a/tests/unit/mcpgateway/test_observability.py +++ b/tests/unit/mcpgateway/test_observability.py @@ -872,3 +872,61 @@ def start_as_current_span(self, _name, **kwargs): assert ["kind"] in tracer.attempts # The request still completed and the trace envelope was published. assert (b"traceparent", b"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01") in captured["headers"] + + @pytest.mark.asyncio + async def test_middleware_does_not_retry_type_error_without_context(self): + """A TypeError cannot be a context compatibility error when no context was passed.""" + captured = {} + app, receive, send = self._make_app(captured) + scope = self._make_scope() + + class FailingTracer: + """Track calls before raising an unrelated tracer failure.""" + + def __init__(self): + self.attempts = [] + + def start_as_current_span(self, _name, **kwargs): + self.attempts.append(sorted(kwargs)) + raise TypeError("unrelated tracer failure") + + tracer = FailingTracer() + with ( + patch("mcpgateway.observability._TRACER", tracer), + patch("mcpgateway.observability.otel_extract", return_value=None), + ): + middleware = OpenTelemetryRequestMiddleware(app) + with pytest.raises(TypeError, match="unrelated tracer failure"): + await middleware(scope, receive, send) + + assert tracer.attempts == [["kind"]] + + @pytest.mark.asyncio + async def test_middleware_does_not_swallow_unrelated_type_error_with_context(self): + """Only rejection of the context keyword should activate the compatibility retry.""" + captured = {} + app, receive, send = self._make_app(captured) + scope = self._make_scope() + + class InternallyFailingTracer: + """Accept context but fail internally while creating the span.""" + + def __init__(self): + self.attempts = [] + + def start_as_current_span(self, _name, **kwargs): + self.attempts.append(sorted(kwargs)) + if "context" in kwargs: + raise TypeError("tracer implementation failed internally") + return nullcontext(None) + + tracer = InternallyFailingTracer() + with ( + patch("mcpgateway.observability._TRACER", tracer), + patch("mcpgateway.observability.otel_extract", return_value=MagicMock()), + ): + middleware = OpenTelemetryRequestMiddleware(app) + with pytest.raises(TypeError, match="tracer implementation failed internally"): + await middleware(scope, receive, send) + + assert tracer.attempts == [["context", "kind"]] From 4ed2c0a17ef45f0638cbd56c85fc120e835cf967 Mon Sep 17 00:00:00 2001 From: "NAYANA.R" Date: Tue, 13 Oct 2026 12:02:29 +0530 Subject: [PATCH 03/12] fix the trace-parent defect Signed-off-by: NAYANA.R Signed-off-by: Madhav Kandukuri --- mcpgateway/observability.py | 113 ++++++++++------- tests/unit/mcpgateway/test_observability.py | 132 ++++++++++++++++++++ 2 files changed, 199 insertions(+), 46 deletions(-) diff --git a/mcpgateway/observability.py b/mcpgateway/observability.py index a646d1f3a17..d1798332e83 100644 --- a/mcpgateway/observability.py +++ b/mcpgateway/observability.py @@ -24,6 +24,7 @@ otel_baggage: Any = None # pylint: disable=invalid-name try: # Third-Party + from opentelemetry import context as otel_context from opentelemetry import trace from opentelemetry.propagate import extract as otel_extract from opentelemetry.propagate import inject as otel_inject @@ -42,6 +43,8 @@ OTEL_AVAILABLE = True except ImportError: # OpenTelemetry not installed - set to None for graceful degradation + otel_baggage = None + otel_context = None trace = None otel_extract = None otel_inject = None @@ -1131,6 +1134,7 @@ async def _send_with_span_status(message: Mapping[str, Any]) -> None: span.set_status(Status(StatusCode.OK)) await send(message) + parent_context_token = None try: span_context_manager = _TRACER.start_as_current_span(span_name, **start_span_kwargs) except TypeError as exc: @@ -1142,62 +1146,79 @@ async def _send_with_span_status(message: Mapping[str, Any]) -> None: rejected_context = "context" in start_span_kwargs and "unexpected keyword argument" in error_message and ("'context'" in error_message or '"context"' in error_message) if not rejected_context: raise + # Attach the extracted inbound parent to the current context before + # starting the span, so the fallback span still nests under the inbound + # trace and active spans match raw-header forwarding. Detach the parent + # when the span lifetime ends. compatible_kwargs = {key: value for key, value in start_span_kwargs.items() if key != "context"} + if parent_context is not None and otel_context is not None: + try: + parent_context_token = otel_context.attach(parent_context) + except Exception as attach_exc: # pylint: disable=broad-exception-caught + logger.debug("Failed to attach inbound parent context: %s", attach_exc) + parent_context_token = None span_context_manager = _TRACER.start_as_current_span(span_name, **compatible_kwargs) logger.debug( - "Tracer %r rejected `context` kwarg; started span %r without explicit parent context", + "Tracer %r rejected `context` kwarg; started span %r with attached parent context", type(_TRACER).__name__, span_name, ) - with span_context_manager as span: - # No valid inbound envelope: publish this new root span's context into - # the ASGI headers so downstream raw-header copies (session-task handoff, - # affinity envelopes, trusted-internal dispatch) carry the trace and - # internal spans nest under it instead of rooting detached traces. - if not has_remote_parent and otel_inject is not None: - try: - envelope_carrier: Dict[str, str] = {} - otel_inject(carrier=envelope_carrier) - envelope_traceparent = envelope_carrier.get("traceparent") - if envelope_traceparent: - new_scope_headers = list(scope.get("headers", []) or []) - new_scope_headers.append((b"traceparent", envelope_traceparent.encode("latin-1"))) - envelope_tracestate = envelope_carrier.get("tracestate") - if envelope_tracestate: - new_scope_headers.append((b"tracestate", envelope_tracestate.encode("latin-1"))) - scope["headers"] = new_scope_headers - except Exception as exc: # pylint: disable=broad-exception-caught - logger.debug("Failed to publish trace envelope into ASGI scope: %s", exc) - if span is not None: - for key, value in span_attributes.items(): - if value is not None: - set_span_attribute(span, key, value) + try: + with span_context_manager as span: + # No valid inbound envelope: publish this new root span's context into + # the ASGI headers so downstream raw-header copies (session-task handoff, + # affinity envelopes, trusted-internal dispatch) carry the trace and + # internal spans nest under it instead of rooting detached traces. + if not has_remote_parent and otel_inject is not None: + try: + envelope_carrier: Dict[str, str] = {} + otel_inject(carrier=envelope_carrier) + envelope_traceparent = envelope_carrier.get("traceparent") + if envelope_traceparent: + new_scope_headers = list(scope.get("headers", []) or []) + new_scope_headers.append((b"traceparent", envelope_traceparent.encode("latin-1"))) + envelope_tracestate = envelope_carrier.get("tracestate") + if envelope_tracestate: + new_scope_headers.append((b"tracestate", envelope_tracestate.encode("latin-1"))) + scope["headers"] = new_scope_headers + except Exception as exc: # pylint: disable=broad-exception-caught + logger.debug("Failed to publish trace envelope into ASGI scope: %s", exc) + if span is not None: + for key, value in span_attributes.items(): + if value is not None: + set_span_attribute(span, key, value) + + # Inject baggage as span attributes so request-root spans retain + # the same trace context dimensions carried on child spans. + try: + if OTEL_AVAILABLE and otel_baggage: + baggage_dict = otel_baggage.get_all() + if baggage_dict: + baggage_attrs = _baggage_span_attributes(baggage_dict) + for key, value in baggage_attrs.items(): + set_span_attribute(span, key, value) + logger.debug("Injected %d baggage entries into request span", len(baggage_attrs)) + except Exception as exc: + logger.debug("Failed to inject baggage into request span: %s", exc) - # Inject baggage as span attributes so request-root spans retain - # the same trace context dimensions carried on child spans. try: - if OTEL_AVAILABLE and otel_baggage: - baggage_dict = otel_baggage.get_all() - if baggage_dict: - baggage_attrs = _baggage_span_attributes(baggage_dict) - for key, value in baggage_attrs.items(): - set_span_attribute(span, key, value) - logger.debug("Injected %d baggage entries into request span", len(baggage_attrs)) + await self.app(scope, receive, _send_with_span_status) + if span is not None and "status" not in status_code_holder and OTEL_AVAILABLE and Status and StatusCode: + span.set_status(Status(StatusCode.OK)) except Exception as exc: - logger.debug("Failed to inject baggage into request span: %s", exc) - - try: - await self.app(scope, receive, _send_with_span_status) - if span is not None and "status" not in status_code_holder and OTEL_AVAILABLE and Status and StatusCode: - span.set_status(Status(StatusCode.OK)) - except Exception as exc: - if span is not None: - error_message = _sanitize_span_exception_message(exc) - set_span_error(span, exc, record_exception=True) - if OTEL_AVAILABLE and Status and StatusCode: - span.set_status(Status(StatusCode.ERROR, error_message)) - raise + if span is not None: + error_message = _sanitize_span_exception_message(exc) + set_span_error(span, exc, record_exception=True) + if OTEL_AVAILABLE and Status and StatusCode: + span.set_status(Status(StatusCode.ERROR, error_message)) + raise + finally: + if parent_context_token is not None and otel_context is not None: + try: + otel_context.detach(parent_context_token) + except Exception as exc: # pylint: disable=broad-exception-caught + logger.debug("Failed to detach inbound parent context: %s", exc) def init_telemetry() -> Optional[Any]: diff --git a/tests/unit/mcpgateway/test_observability.py b/tests/unit/mcpgateway/test_observability.py index 606cf64df4a..9ce5c8deb6c 100644 --- a/tests/unit/mcpgateway/test_observability.py +++ b/tests/unit/mcpgateway/test_observability.py @@ -930,3 +930,135 @@ def start_as_current_span(self, _name, **kwargs): await middleware(scope, receive, send) assert tracer.attempts == [["context", "kind"]] + + @pytest.mark.asyncio + async def test_fallback_preserves_inbound_parent_with_context_rejecting_tracer(self, monkeypatch): + """The context-kwarg fallback must nest the request span under the inbound trace. + + A tracer that rejects the ``context`` keyword (e.g. Instana < 3.12) resolves its + parent from the current context. The middleware attaches the extracted inbound + parent for the span lifetime, so the active span and the forwarded ``traceparent`` + stay in the same trace and the outer context is restored afterwards. + """ + trace_sdk = pytest.importorskip("opentelemetry.sdk.trace") + export_sdk = pytest.importorskip("opentelemetry.sdk.trace.export") + memory_export_sdk = pytest.importorskip("opentelemetry.sdk.trace.export.in_memory_span_exporter") + # Third-Party + from opentelemetry import context as otel_context_api + from opentelemetry import trace as otel_trace_api + + exporter = memory_export_sdk.InMemorySpanExporter() + provider = trace_sdk.TracerProvider() + provider.add_span_processor(export_sdk.SimpleSpanProcessor(exporter)) + inner_tracer = provider.get_tracer("instana-compat-parent-test") + + class ContextRejectingTracer: + """Emulates InstanaTracer < 3.12: rejects the ``context`` kwarg.""" + + def __init__(self, inner): + self.inner = inner + self.attempts = [] + + def start_as_current_span(self, name, **kwargs): + self.attempts.append(sorted(kwargs)) + if "context" in kwargs: + raise TypeError("start_as_current_span() got an unexpected keyword argument 'context'") + return self.inner.start_as_current_span(name, **kwargs) + + tracer = ContextRejectingTracer(inner_tracer) + monkeypatch.setattr(observability, "_TRACER", tracer) + + active_trace_id = {} + completed = {} + + async def app(_scope, _receive, send): + active_span = otel_trace_api.get_current_span() + active_trace_id["value"] = f"{active_span.get_span_context().trace_id:032x}" + await send({"type": "http.response.start", "status": 200, "headers": []}) + await send({"type": "http.response.body", "body": b""}) + completed["value"] = True + + async def receive(): + return {"type": "http.request", "body": b"", "more_body": False} + + async def send(_message): + return None + + incoming_trace_id = "0af7651916cd43dd8448eb211c80319c" # pragma: allowlist secret + incoming_span_id = "b7ad6b7169203331" # pragma: allowlist secret + incoming_traceparent = f"00-{incoming_trace_id}-{incoming_span_id}-01" + scope = self._make_scope([(b"traceparent", incoming_traceparent.encode())]) + middleware = OpenTelemetryRequestMiddleware(app) + + context_before = otel_context_api.get_current() + await middleware(scope, receive, send) + context_after = otel_context_api.get_current() + + # The tracer first rejects `context`, then retries without it. + assert ["context", "kind"] in tracer.attempts + assert ["kind"] in tracer.attempts + # The request completed and the outer context was restored after detach. + assert completed.get("value") is True + assert context_after is context_before + # The request span nested under the inbound trace instead of rooting a new one. + (request_span,) = exporter.get_finished_spans() + assert f"{request_span.context.trace_id:032x}" == incoming_trace_id + assert f"{request_span.parent.span_id:016x}" == incoming_span_id + # The active span during request handling shares the forwarded trace. + assert active_trace_id["value"] == incoming_trace_id + assert dict(scope["headers"])[b"traceparent"].decode() == incoming_traceparent + + @pytest.mark.asyncio + async def test_fallback_restores_context_when_app_raises_with_context_rejecting_tracer(self, monkeypatch): + """The fallback detaches the attached parent even when the wrapped app fails.""" + trace_sdk = pytest.importorskip("opentelemetry.sdk.trace") + export_sdk = pytest.importorskip("opentelemetry.sdk.trace.export") + memory_export_sdk = pytest.importorskip("opentelemetry.sdk.trace.export.in_memory_span_exporter") + # Third-Party + from opentelemetry import context as otel_context_api + from opentelemetry.trace import StatusCode + + exporter = memory_export_sdk.InMemorySpanExporter() + provider = trace_sdk.TracerProvider() + provider.add_span_processor(export_sdk.SimpleSpanProcessor(exporter)) + inner_tracer = provider.get_tracer("instana-compat-failure-test") + + class ContextRejectingTracer: + """Emulates InstanaTracer < 3.12: rejects the ``context`` kwarg.""" + + def __init__(self, inner): + self.inner = inner + + def start_as_current_span(self, name, **kwargs): + if "context" in kwargs: + raise TypeError("start_as_current_span() got an unexpected keyword argument 'context'") + return self.inner.start_as_current_span(name, **kwargs) + + monkeypatch.setattr(observability, "_TRACER", ContextRejectingTracer(inner_tracer)) + + async def app(_scope, _receive, _send): + raise RuntimeError("app failure") + + async def receive(): + return {"type": "http.request", "body": b"", "more_body": False} + + async def send(_message): + return None + + incoming_trace_id = "0af7651916cd43dd8448eb211c80319c" # pragma: allowlist secret + incoming_span_id = "b7ad6b7169203331" # pragma: allowlist secret + scope = self._make_scope([(b"traceparent", f"00-{incoming_trace_id}-{incoming_span_id}-01".encode())]) + middleware = OpenTelemetryRequestMiddleware(app) + + context_before = otel_context_api.get_current() + with pytest.raises(RuntimeError, match="app failure"): + await middleware(scope, receive, send) + context_after = otel_context_api.get_current() + + # Detach runs in `finally`, so the outer context survives the app failure. + assert context_after is context_before + # The failed request span still belongs to the inbound trace. + (request_span,) = exporter.get_finished_spans() + assert f"{request_span.context.trace_id:032x}" == incoming_trace_id + assert f"{request_span.parent.span_id:016x}" == incoming_span_id + assert request_span.status.status_code is StatusCode.ERROR From a657c7033c4870b00cbabdb3c9506f436792d2aa Mon Sep 17 00:00:00 2001 From: "NAYANA.R" Date: Tue, 13 Oct 2026 14:30:04 +0530 Subject: [PATCH 04/12] fixing pytest Signed-off-by: NAYANA.R Signed-off-by: Madhav Kandukuri --- tests/unit/mcpgateway/test_observability.py | 148 ++++++++++++++++++++ 1 file changed, 148 insertions(+) diff --git a/tests/unit/mcpgateway/test_observability.py b/tests/unit/mcpgateway/test_observability.py index 9ce5c8deb6c..220a9dab56e 100644 --- a/tests/unit/mcpgateway/test_observability.py +++ b/tests/unit/mcpgateway/test_observability.py @@ -1062,3 +1062,151 @@ async def send(_message): assert f"{request_span.context.trace_id:032x}" == incoming_trace_id assert f"{request_span.parent.span_id:016x}" == incoming_span_id assert request_span.status.status_code is StatusCode.ERROR + + @pytest.mark.asyncio + async def test_middleware_fallback_attaches_parent_injects_baggage_and_detaches(self): + """The context-kwarg fallback attaches the inbound parent, injects baggage, and detaches it. + + Patches the module-level OTEL globals so the assertions hold without the + ``observability`` extra (SDK absent), matching the main pytest CI job. + """ + scope = self._make_scope([(b"traceparent", b"00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01")]) + ran = {} + + async def app(_scope, _receive, _send): + ran["value"] = True + + async def receive(): + return {"type": "http.request", "body": b"", "more_body": False} + + async def send(_message): + return None + + class ContextRejectingTracer: + """Emulates InstanaTracer < 3.12: rejects the ``context`` kwarg.""" + + def start_as_current_span(self, _name, **kwargs): + if "context" in kwargs: + raise TypeError("start_as_current_span() got an unexpected keyword argument 'context'") + return nullcontext(MagicMock()) + + mock_trace = MagicMock() + mock_trace.get_current_span.return_value.get_span_context.return_value = MagicMock(is_valid=False, is_remote=False) + mock_context = MagicMock() + token = object() + mock_context.attach.return_value = token + mock_baggage = MagicMock() + mock_baggage.get_all.return_value = {"tenant": "acme"} + + with ( + patch("mcpgateway.observability._TRACER", ContextRejectingTracer()), + patch("mcpgateway.observability.OTEL_AVAILABLE", True), + patch("mcpgateway.observability.trace", mock_trace), + patch("mcpgateway.observability.otel_extract", return_value=MagicMock()), + patch("mcpgateway.observability.otel_inject", side_effect=self._fake_inject), + patch("mcpgateway.observability.otel_context", mock_context), + patch("mcpgateway.observability.otel_baggage", mock_baggage), + patch("mcpgateway.observability._baggage_span_attributes", return_value={"baggage.tenant": "acme"}), + patch("mcpgateway.observability.Status", MagicMock()), + patch("mcpgateway.observability.StatusCode", MagicMock()), + ): + middleware = OpenTelemetryRequestMiddleware(app) + await middleware(scope, receive, send) + + assert ran["value"] is True + mock_context.attach.assert_called_once() + mock_context.detach.assert_called_once_with(token) + + @pytest.mark.asyncio + async def test_middleware_fallback_swallows_attach_and_baggage_failures_and_marks_span_error(self): + """Attach and baggage failures are logged, and an app failure marks the span as an error.""" + scope = self._make_scope() + + async def app(_scope, _receive, _send): + raise RuntimeError("app boom") + + async def receive(): + return {"type": "http.request", "body": b"", "more_body": False} + + async def send(_message): + return None + + class ContextRejectingTracer: + """Emulates InstanaTracer < 3.12: rejects the ``context`` kwarg.""" + + def start_as_current_span(self, _name, **kwargs): + if "context" in kwargs: + raise TypeError("start_as_current_span() got an unexpected keyword argument 'context'") + return nullcontext(MagicMock()) + + mock_trace = MagicMock() + mock_trace.get_current_span.return_value.get_span_context.return_value = MagicMock(is_valid=False, is_remote=False) + mock_context = MagicMock() + mock_context.attach.side_effect = RuntimeError("attach boom") + mock_baggage = MagicMock() + mock_baggage.get_all.return_value = {"tenant": "acme"} + + with ( + patch("mcpgateway.observability._TRACER", ContextRejectingTracer()), + patch("mcpgateway.observability.OTEL_AVAILABLE", True), + patch("mcpgateway.observability.trace", mock_trace), + patch("mcpgateway.observability.otel_extract", return_value=MagicMock()), + patch("mcpgateway.observability.otel_inject", side_effect=self._fake_inject), + patch("mcpgateway.observability.otel_context", mock_context), + patch("mcpgateway.observability.otel_baggage", mock_baggage), + patch("mcpgateway.observability._baggage_span_attributes", side_effect=RuntimeError("baggage boom")), + patch("mcpgateway.observability.Status", MagicMock()), + patch("mcpgateway.observability.StatusCode", MagicMock()), + ): + middleware = OpenTelemetryRequestMiddleware(app) + with pytest.raises(RuntimeError, match="app boom"): + await middleware(scope, receive, send) + + mock_context.detach.assert_not_called() + + @pytest.mark.asyncio + async def test_middleware_fallback_detach_failure_is_nonfatal(self): + """A detach failure is logged and the request still completes.""" + scope = self._make_scope() + ran = {} + + async def app(_scope, _receive, _send): + ran["value"] = True + + async def receive(): + return {"type": "http.request", "body": b"", "more_body": False} + + async def send(_message): + return None + + class ContextRejectingTracer: + """Emulates InstanaTracer < 3.12: rejects the ``context`` kwarg.""" + + def start_as_current_span(self, _name, **kwargs): + if "context" in kwargs: + raise TypeError("start_as_current_span() got an unexpected keyword argument 'context'") + return nullcontext(MagicMock()) + + mock_trace = MagicMock() + mock_trace.get_current_span.return_value.get_span_context.return_value = MagicMock(is_valid=False, is_remote=False) + mock_context = MagicMock() + token = object() + mock_context.attach.return_value = token + mock_context.detach.side_effect = RuntimeError("detach boom") + + with ( + patch("mcpgateway.observability._TRACER", ContextRejectingTracer()), + patch("mcpgateway.observability.OTEL_AVAILABLE", True), + patch("mcpgateway.observability.trace", mock_trace), + patch("mcpgateway.observability.otel_extract", return_value=MagicMock()), + patch("mcpgateway.observability.otel_inject", side_effect=self._fake_inject), + patch("mcpgateway.observability.otel_context", mock_context), + patch("mcpgateway.observability.otel_baggage", None), + patch("mcpgateway.observability.Status", MagicMock()), + patch("mcpgateway.observability.StatusCode", MagicMock()), + ): + middleware = OpenTelemetryRequestMiddleware(app) + await middleware(scope, receive, send) + + assert ran["value"] is True + mock_context.detach.assert_called_once_with(token) From 315737f1c7d32ba1cc0780a16fe5bc6d0a081d57 Mon Sep 17 00:00:00 2001 From: "NAYANA.R" Date: Thu, 15 Oct 2026 10:56:00 +0530 Subject: [PATCH 05/12] x fallback context cleanup on retry failures and add a live authenticated MCP tools/call regression test or document an explicit waiver Signed-off-by: NAYANA.R Signed-off-by: Madhav Kandukuri --- .github/workflows/pytest.yml | 16 +- mcpgateway/observability.py | 157 ++++++-- .../mcp/test_mcp_trace_context_e2e.py | 340 ++++++++++++++++++ tests/unit/mcpgateway/test_observability.py | 268 +++++++++++++- 4 files changed, 737 insertions(+), 44 deletions(-) create mode 100644 tests/live_gateway/mcp/test_mcp_trace_context_e2e.py diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 402d83e5300..f206ebc9784 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -243,20 +243,8 @@ jobs: tests/live_gateway/mcp/test_request_trace_policy.py \ tests/unit/mcpgateway/services/test_a2a_service.py \ tests/live_gateway/a2a/test_a2a_trace_context_e2e.py \ - --basetemp="${RUNNER_TEMP}/observability-tests" -v --tb=short - uv run --extra plugins --extra observability pytest \ - tests/unit/mcpgateway/services/test_gateway_service.py \ - -k health_completion_metrics -v --tb=short - - - name: Preserve observability failure artifacts - if: failure() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: observability-failure-artifacts - path: | - ${{ runner.temp }}/observability-tests/**/*.log - ${{ runner.temp }}/observability-tests/**/collector.jsonl - retention-days: 7 + tests/live_gateway/mcp/test_mcp_trace_context_e2e.py \ + -v --tb=short # ----------------------------------------------------------- # 4️⃣ Upload coverage artifacts (XML + HTML) diff --git a/mcpgateway/observability.py b/mcpgateway/observability.py index d1798332e83..e4d1c01266d 100644 --- a/mcpgateway/observability.py +++ b/mcpgateway/observability.py @@ -10,7 +10,7 @@ # Standard import base64 -from contextlib import nullcontext +from contextlib import contextmanager, nullcontext from dataclasses import dataclass from importlib import import_module as _im import inspect @@ -1136,35 +1136,35 @@ async def _send_with_span_status(message: Mapping[str, Any]) -> None: parent_context_token = None try: - span_context_manager = _TRACER.start_as_current_span(span_name, **start_span_kwargs) - except TypeError as exc: - # Some tracer implementations (e.g. Instana < 3.12) don't accept the - # `context` keyword argument even though it is part of the - # OpenTelemetry Tracer API. Retry only for that known incompatibility; - # unrelated TypeErrors must retain their original behavior. - error_message = str(exc) - rejected_context = "context" in start_span_kwargs and "unexpected keyword argument" in error_message and ("'context'" in error_message or '"context"' in error_message) - if not rejected_context: - raise - # Attach the extracted inbound parent to the current context before - # starting the span, so the fallback span still nests under the inbound - # trace and active spans match raw-header forwarding. Detach the parent - # when the span lifetime ends. - compatible_kwargs = {key: value for key, value in start_span_kwargs.items() if key != "context"} - if parent_context is not None and otel_context is not None: - try: - parent_context_token = otel_context.attach(parent_context) - except Exception as attach_exc: # pylint: disable=broad-exception-caught - logger.debug("Failed to attach inbound parent context: %s", attach_exc) - parent_context_token = None - span_context_manager = _TRACER.start_as_current_span(span_name, **compatible_kwargs) - logger.debug( - "Tracer %r rejected `context` kwarg; started span %r with attached parent context", - type(_TRACER).__name__, - span_name, - ) + try: + span_context_manager = _TRACER.start_as_current_span(span_name, **start_span_kwargs) + except TypeError as exc: + # Some tracer implementations (e.g. Instana < 3.12) don't accept the + # `context` keyword argument even though it is part of the + # OpenTelemetry Tracer API. Retry only for that known incompatibility; + # unrelated TypeErrors must retain their original behavior. + error_message = str(exc) + rejected_context = "context" in start_span_kwargs and "unexpected keyword argument" in error_message and ("'context'" in error_message or '"context"' in error_message) + if not rejected_context: + raise + # Attach the extracted inbound parent to the current context before + # starting the span, so the fallback span still nests under the inbound + # trace and active spans match raw-header forwarding. The outer finally + # detaches the parent, including when this fallback start raises. + compatible_kwargs = {key: value for key, value in start_span_kwargs.items() if key != "context"} + if parent_context is not None and otel_context is not None: + try: + parent_context_token = otel_context.attach(parent_context) + except Exception as attach_exc: # pylint: disable=broad-exception-caught + logger.debug("Failed to attach inbound parent context: %s", attach_exc) + parent_context_token = None + span_context_manager = _TRACER.start_as_current_span(span_name, **compatible_kwargs) + logger.debug( + "Tracer %r rejected `context` kwarg; started span %r with attached parent context", + type(_TRACER).__name__, + span_name, + ) - try: with span_context_manager as span: # No valid inbound envelope: publish this new root span's context into # the ASGI headers so downstream raw-header copies (session-task handoff, @@ -1713,5 +1713,104 @@ def create_child_span(name: str, attributes: Optional[Dict[str, Any]] = None) -> return create_span(name, attributes) +def _wrap_start_span_with_context(original: Callable) -> Callable: + """Return a ``start_span`` that honors the standard ``context`` argument. + + The wrapped method attaches the supplied parent for the duration of span + creation so a tracer that reads the current OpenTelemetry context nests the + new span under it, then detaches the parent. + + Args: + original: The legacy ``start_span`` method to wrap. + + Returns: + Callable: A wrapper that accepts ``context`` and preserves the parent. + """ + + def _compat(self: Any, name: str, context: Optional[Any] = None, *args: Any, **kwargs: Any) -> Any: + if context is None or otel_context is None: + return original(self, name, *args, **kwargs) + token = otel_context.attach(context) + try: + return original(self, name, *args, **kwargs) + finally: + otel_context.detach(token) + + setattr(_compat, "__cf_context_compat__", True) + return _compat + + +def _wrap_start_as_current_span_with_context(original: Callable) -> Callable: + """Return a ``start_as_current_span`` that honors the standard ``context`` argument. + + The wrapped method attaches the supplied parent for the span's lifetime so a + tracer that reads the current OpenTelemetry context nests the span under it, + then detaches the parent when the span ends. + + Args: + original: The legacy ``start_as_current_span`` method to wrap. + + Returns: + Callable: A context-manager wrapper that accepts ``context`` and preserves the parent. + """ + + @contextmanager + def _compat(self: Any, name: str, context: Optional[Any] = None, *args: Any, **kwargs: Any) -> Any: + if context is None or otel_context is None: + with original(self, name, *args, **kwargs) as span: + yield span + return + token = otel_context.attach(context) + try: + with original(self, name, *args, **kwargs) as span: + yield span + finally: + otel_context.detach(token) + + setattr(_compat, "__cf_context_compat__", True) + return _compat + + +def _install_legacy_tracer_context_compat() -> None: + """Teach a legacy OpenTelemetry tracer to accept the standard ``context`` keyword. + + Instana < 3.12 subclasses the OpenTelemetry ``Tracer`` API but omits the + ``context`` parameter on ``start_span`` and ``start_as_current_span``. Consumers + that follow the published API (the MCP Python SDK middleware, FastAPI's native + ASGI telemetry, and this module's middleware) pass ``context=``, which raises + ``TypeError`` and aborts the request. Patch both methods to accept ``context`` + and honor the supplied parent through the standard OpenTelemetry context API. + + The patch is capability-gated (skipped when ``context`` is already a + parameter), idempotent, preserves unrelated exceptions, and is a no-op when + OpenTelemetry or the legacy tracer is absent. + """ + if not OTEL_AVAILABLE: + return + try: + # Third-Party + from instana.tracer import InstanaTracer + except Exception: # pylint: disable=broad-exception-caught + return + + wrappers = { + "start_span": _wrap_start_span_with_context, + "start_as_current_span": _wrap_start_as_current_span_with_context, + } + for method_name, wrapper in wrappers.items(): + try: + method = getattr(InstanaTracer, method_name) + if "context" in inspect.signature(method).parameters: + continue + if getattr(method, "__cf_context_compat__", False): + continue + setattr(InstanaTracer, method_name, wrapper(method)) + except Exception as exc: # pylint: disable=broad-exception-caught + logger.debug("Skipping %s context compatibility shim: %s", method_name, exc) + + +# Install the compatibility patch before any consumer captures the tracer. +_install_legacy_tracer_context_compat() + # Initialize on module import _TRACER = init_telemetry() diff --git a/tests/live_gateway/mcp/test_mcp_trace_context_e2e.py b/tests/live_gateway/mcp/test_mcp_trace_context_e2e.py new file mode 100644 index 00000000000..f6e3974e85e --- /dev/null +++ b/tests/live_gateway/mcp/test_mcp_trace_context_e2e.py @@ -0,0 +1,340 @@ +# -*- coding: utf-8 -*- +"""Location: ./tests/live_gateway/mcp/test_mcp_trace_context_e2e.py +Copyright contributors to the MCP-CONTEXT-FORGE project +SPDX-License-Identifier: Apache-2.0 + +Black-box W3C trace-context check for MCP ``tools/call`` against an isolated gateway. + +The gateway subprocess runs with a ``sitecustomize`` shim that installs an +``instana.tracer.InstanaTracer`` emulating ``instana < 3.12``: ``get_tracer`` +returns it for every instrumentation name, so the gateway middleware, the MCP +Python SDK (``mcp-python-sdk``), and FastAPI telemetry all see a tracer without +the OpenTelemetry ``context`` parameter. The production compatibility layer must +patch that tracer so authenticated ``tools/call`` still completes. A valid W3C +``traceparent`` arrives with the call; the request-root span must adopt that +inbound parent, so the active span and every span beneath it stay in one trace. +A marker file records that the incompatible path ran. + +Run with the observability extra installed. The gateway uses a private SQLite +DB, real REST tool invocation, real HTTP requests, and the SDK console exporter. +No existing gateway, Redis instance, or credentials are used. +""" + +# Standard +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import json +import os +from pathlib import Path +import secrets +import socket +import subprocess +import sys +import threading +import time +from typing import Any + +# Third-Party +import httpx +import pytest + +# First-Party +from tests.helpers.auth import make_auth_headers, make_test_jwt +from tests.live_gateway.plugins._helpers import create_virtual_server, initialize_session, mcp_headers, result_text + +pytestmark = pytest.mark.e2e +_REPO_ROOT = Path(__file__).resolve().parents[3] + +INBOUND_TRACE_ID = "0af7651916cd43dd8448eb211c80319c" # pragma: allowlist secret +INBOUND_SPAN_ID = "b7ad6b7169203331" # pragma: allowlist secret +TOOL_NAME = "trace-context-probe" + +# Emulate the legacy Instana provider (``instana < 3.12``): a tracer whose +# ``start_span``/``start_as_current_span`` do not accept the OpenTelemetry +# ``context`` keyword. A real legacy Instana AutoTrace replaces the global tracer +# provider, so *every* consumer sees the incompatible tracer -- the gateway's own +# middleware, the MCP Python SDK (``mcp-python-sdk``), and FastAPI telemetry. The +# shim registers the emulated class as ``instana.tracer.InstanaTracer`` so the +# production compatibility layer patches it exactly as it would patch the real one. +_SITECUSTOMIZE = ''' +import os +import sys +import types + +try: + import opentelemetry.trace as _trace +except Exception: + _trace = None + +if _trace is not None and hasattr(_trace, "get_tracer"): + _orig_get_tracer = _trace.get_tracer + _marker = os.environ.get("MCP_TRACE_INCOMPATIBLE_MARKER") + + def _record(name): + if not _marker: + return + try: + with open(_marker, "a", encoding="utf-8") as handle: + handle.write(name + "\\n") + except OSError: + pass + + class InstanaTracer: + """Emulate ``instana.tracer.InstanaTracer`` before 3.12: no ``context`` parameter.""" + + def __init__(self, inner): + self._inner = inner + + def __getattr__(self, name): + return getattr(self._inner, name) + + def start_span(self, name, *args, **kwargs): + if "context" in kwargs: + raise TypeError("start_span() got an unexpected keyword argument 'context'") + return self._inner.start_span(name, *args, **kwargs) + + def start_as_current_span(self, name, *args, **kwargs): + _record(name) + if "context" in kwargs: + raise TypeError("start_as_current_span() got an unexpected keyword argument 'context'") + return self._inner.start_as_current_span(name, *args, **kwargs) + + def _wrapped_get_tracer(*args, **kwargs): + return InstanaTracer(_orig_get_tracer(*args, **kwargs)) + + _trace.get_tracer = _wrapped_get_tracer + + _instana_pkg = types.ModuleType("instana") + _instana_pkg.__path__ = [] + _tracer_mod = types.ModuleType("instana.tracer") + _tracer_mod.InstanaTracer = InstanaTracer + _instana_pkg.tracer = _tracer_mod + sys.modules["instana"] = _instana_pkg + sys.modules["instana.tracer"] = _tracer_mod +''' + + +class _ToolHandler(BaseHTTPRequestHandler): + """Return a fixed JSON body and record inbound request headers.""" + + captured: list[list[tuple[str, str]]] = [] + + def do_GET(self): + """Record headers, then answer 200 with a JSON body.""" + type(self).captured.append(list(self.headers.items())) + body = b'{"ok": true}' + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, format, *args): + """Keep the upstream server quiet.""" + + +def _console_spans(log_path: Path) -> list[dict[str, Any]]: + """Read complete SDK span JSON objects, ignoring gateway log lines.""" + contents = log_path.read_text(encoding="utf-8") + decoder = json.JSONDecoder() + spans = [] + offset = 0 + while (offset := contents.find('{\n "name":', offset)) >= 0: + try: + span, length = decoder.raw_decode(contents[offset:]) + except json.JSONDecodeError: + break + spans.append(span) + offset += length + return spans + + +def _hex_id(value: Any) -> Any: + """Normalize a console-exporter hex identifier by dropping the 0x prefix.""" + return value[2:] if isinstance(value, str) and value.startswith("0x") else value + + +def _wait_for_spans(log_path: Path, predicate, deadline_seconds: float = 15.0) -> list[dict[str, Any]]: + """Poll the gateway log until ``predicate(spans)`` returns truthy or time runs out.""" + deadline = time.monotonic() + deadline_seconds + spans: list[dict[str, Any]] = [] + while time.monotonic() < deadline: + spans = _console_spans(log_path) + if predicate(spans): + return spans + time.sleep(0.1) + return spans + + +def _find_request_span(spans: list[dict[str, Any]], server_id: str) -> dict[str, Any] | None: + """Return the middleware request-root span for ``server_id`` in the inbound trace, if exported.""" + expected_name = f"POST /servers/{server_id}/mcp/" + for span in spans: + if span.get("name") == expected_name and _hex_id(span.get("context", {}).get("trace_id")) == INBOUND_TRACE_ID: + return span + return None + + +def _find_tool_call_span(spans: list[dict[str, Any]], tool_name: str) -> dict[str, Any] | None: + """Return the MCP ``tools/call`` span for ``tool_name`` in the inbound trace, if exported.""" + expected_name = f"tools/call {tool_name}" + for span in spans: + if span.get("name") == expected_name and _hex_id(span.get("context", {}).get("trace_id")) == INBOUND_TRACE_ID: + return span + return None + + +@pytest.fixture +def isolated_incompatible_gateway(tmp_path): + """Start an authenticated gateway whose tracer rejects the ``context`` kwarg. + + Registers a REST tool backed by a local capture server and a virtual server + that exposes it. + + Yields: + A tuple of the authenticated client, bearer token, gateway log path, + incompatibility marker path, virtual server id, and tool name. + """ + trace_sdk = pytest.importorskip("opentelemetry.sdk.trace") + if not hasattr(trace_sdk, "TracerProvider"): + pytest.skip("OpenTelemetry SDK is not installed") + + harness_dir = tmp_path / "harness" + harness_dir.mkdir() + (harness_dir / "sitecustomize.py").write_text(_SITECUSTOMIZE, encoding="utf-8") + marker_path = tmp_path / "incompatible.marker" + + _ToolHandler.captured = [] + upstream = ThreadingHTTPServer(("127.0.0.1", 0), _ToolHandler) + upstream_worker = threading.Thread(target=upstream.serve_forever, daemon=True) + upstream_worker.start() + + with socket.socket() as listener: + listener.bind(("127.0.0.1", 0)) + port = listener.getsockname()[1] + signing_key = secrets.token_urlsafe(48) + env = { + "PATH": os.environ["PATH"], + "PYTHONPATH": os.pathsep.join([str(harness_dir), str(_REPO_ROOT), *[str(Path(path).resolve()) for path in os.getenv("PYTHONPATH", "").split(os.pathsep) if path]]), + "DATABASE_URL": f"sqlite:///{tmp_path / 'gateway.db'}", + "CACHE_TYPE": "memory", + "REDIS_URL": "", + "JWT_SECRET_KEY": signing_key, + "AUTH_ENCRYPTION_SECRET": secrets.token_urlsafe(48), + "PLATFORM_ADMIN_PASSWORD": secrets.token_urlsafe(32), + "DEFAULT_USER_PASSWORD": secrets.token_urlsafe(32), + "PLATFORM_ADMIN_EMAIL": "admin@example.com", + "AUTH_REQUIRED": "true", + "REQUIRE_USER_IN_DB": "false", + "MCPGATEWAY_ADMIN_API_ENABLED": "true", + "MCPGATEWAY_UI_ENABLED": "false", + "MCPGATEWAY_A2A_ENABLED": "false", + "SSRF_ALLOW_LOCALHOST": "true", + "PLUGINS_ENABLED": "false", + "OBSERVABILITY_ENABLED": "false", + "OTEL_ENABLE_OBSERVABILITY": "true", + "OTEL_TRACES_EXPORTER": "console", + "OTEL_EXPORTER_OTLP_ENDPOINT": "", + "MCP_TRACE_INCOMPATIBLE_MARKER": str(marker_path), + "LOG_LEVEL": "ERROR", + "PYTHONUNBUFFERED": "1", + } + token = make_test_jwt("admin@example.com", is_admin=True, teams=None, secret=signing_key, algorithm="HS256") + log_path = tmp_path / "gateway.log" + tool_id = None + server_id = None + with log_path.open("w") as log_file: + process = subprocess.Popen( + [sys.executable, "-m", "uvicorn", "mcpgateway.main:app", "--host", "127.0.0.1", "--port", str(port)], + cwd=tmp_path, + env=env, + stdout=log_file, + stderr=subprocess.STDOUT, + ) + try: + with httpx.Client(base_url=f"http://127.0.0.1:{port}", headers=make_auth_headers(token), timeout=15, trust_env=False) as client: + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + assert process.poll() is None, log_path.read_text(encoding="utf-8")[-5000:] + try: + if client.get("/health").status_code == 200: + break + except httpx.TransportError: + pass + time.sleep(0.1) + else: + pytest.fail(f"Gateway startup timed out: {log_path.read_text(encoding='utf-8')[-5000:]}") + + response = client.post( + "/tools", + json={ + "tool": { + "name": TOOL_NAME, + "description": "Trace-context probe tool", + "integration_type": "REST", + "url": f"http://127.0.0.1:{upstream.server_port}/probe", + "request_type": "GET", + "visibility": "public", + }, + "team_id": None, + }, + ) + assert response.status_code == 200, response.text + tool_id = response.json()["id"] + server_id = create_virtual_server(client, name="trace-context-server", tool_ids=[tool_id]) + + yield client, token, log_path, marker_path, server_id, TOOL_NAME + finally: + process.terminate() + try: + process.wait(timeout=15) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=5) + upstream.shutdown() + upstream.server_close() + upstream_worker.join(timeout=5) + + +def test_mcp_tools_call_preserves_inbound_trace_with_incompatible_tracer(isolated_incompatible_gateway): + """An authenticated ``tools/call`` keeps the inbound trace through an incompatible tracer.""" + client, token, log_path, marker_path, server_id, tool_name = isolated_incompatible_gateway + session_id = initialize_session(client, server_id=server_id, token=token) + _ToolHandler.captured.clear() + + payload = { + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": {"name": tool_name, "arguments": {}}, + } + headers = mcp_headers(token, session_id=session_id) + headers["traceparent"] = f"00-{INBOUND_TRACE_ID}-{INBOUND_SPAN_ID}-01" + response = client.post(f"/servers/{server_id}/mcp/", headers=headers, json=payload) + assert response.status_code == 200, response.text + result = response.json().get("result", {}) + assert result.get("isError") is not True, response.text + assert "ok" in result_text(result) + + # The incompatible tracer path ran: the middleware fell back to the standard + # context API instead of passing the ``context`` keyword. + assert marker_path.exists(), "context-rejecting tracer was never exercised" + assert f"POST /servers/{server_id}/mcp/" in marker_path.read_text(encoding="utf-8") + + spans = _wait_for_spans(log_path, lambda current: _find_request_span(current, server_id) is not None) + request_span = _find_request_span(spans, server_id) + assert request_span is not None, f"No request span was exported; exported spans: {[span['name'] for span in spans]}" + # Parent identity: the request-root span continues the inbound trace. + assert _hex_id(request_span["context"]["trace_id"]) == INBOUND_TRACE_ID + assert _hex_id(request_span["parent_id"]) == INBOUND_SPAN_ID + request_span_id = _hex_id(request_span["context"]["span_id"]) + assert request_span_id not in (None, "") + + # Active-span consistency: the tool invocation runs beneath the request-root + # span, in the same inbound trace, so any downstream span nests under it. + tool_span = _find_tool_call_span(spans, tool_name) + assert tool_span is not None, f"No tools/call span was exported; exported spans: {[span['name'] for span in spans]}" + assert _hex_id(tool_span["parent_id"]) == request_span_id + + # The tool reached the real upstream REST server, proving the request was + # not short-circuited by the incompatible tracer. + assert _ToolHandler.captured, "the tool did not reach the upstream REST server" diff --git a/tests/unit/mcpgateway/test_observability.py b/tests/unit/mcpgateway/test_observability.py index 220a9dab56e..5ecd7e29497 100644 --- a/tests/unit/mcpgateway/test_observability.py +++ b/tests/unit/mcpgateway/test_observability.py @@ -7,10 +7,12 @@ """ # Standard -from contextlib import nullcontext +from contextlib import contextmanager, nullcontext import inspect import logging import os +import sys +import types from unittest.mock import MagicMock, patch # Third-Party @@ -1063,6 +1065,59 @@ async def send(_message): assert f"{request_span.parent.span_id:016x}" == incoming_span_id assert request_span.status.status_code is StatusCode.ERROR + @pytest.mark.asyncio + async def test_fallback_restores_context_when_retry_raises_with_context_rejecting_tracer(self, monkeypatch): + """The fallback restores the outer context when the retry itself raises. + + A context-rejecting tracer that fails again on the retry must not leak the inbound + parent into the task context. The retry failure propagates, but the outer context is + unchanged, so later spans in the task cannot inherit the failed request's parent. + """ + pytest.importorskip("opentelemetry.sdk.trace") + # Third-Party + from opentelemetry import context as otel_context_api + + class RetryFailingTracer: + """Rejects ``context``, then raises an unrelated error on the retry.""" + + def __init__(self): + self.attempts = [] + + def start_as_current_span(self, name, **kwargs): + self.attempts.append(sorted(kwargs)) + if "context" in kwargs: + raise TypeError("start_as_current_span() got an unexpected keyword argument 'context'") + raise RuntimeError("retry failure") + + tracer = RetryFailingTracer() + monkeypatch.setattr(observability, "_TRACER", tracer) + + ran = {} + + async def app(_scope, _receive, _send): + ran["value"] = True + + async def receive(): + return {"type": "http.request", "body": b"", "more_body": False} + + async def send(_message): + return None + + incoming_trace_id = "0af7651916cd43dd8448eb211c80319c" # pragma: allowlist secret + incoming_span_id = "b7ad6b7169203331" # pragma: allowlist secret + scope = self._make_scope([(b"traceparent", f"00-{incoming_trace_id}-{incoming_span_id}-01".encode())]) + middleware = OpenTelemetryRequestMiddleware(app) + + context_before = otel_context_api.get_current() + with pytest.raises(RuntimeError, match="retry failure"): + await middleware(scope, receive, send) + context_after = otel_context_api.get_current() + + # The retry was attempted; the app never ran; the attached parent was detached. + assert tracer.attempts == [["context", "kind"], ["kind"]] + assert ran.get("value") is None + assert context_after is context_before + @pytest.mark.asyncio async def test_middleware_fallback_attaches_parent_injects_baggage_and_detaches(self): """The context-kwarg fallback attaches the inbound parent, injects baggage, and detaches it. @@ -1210,3 +1265,214 @@ def start_as_current_span(self, _name, **kwargs): assert ran["value"] is True mock_context.detach.assert_called_once_with(token) + + @pytest.mark.asyncio + async def test_fallback_detaches_when_retry_raises(self): + """The fallback detaches the attached parent even when the retry itself raises. + + Regression: a tracer that rejects ``context`` and then fails on the retry must not + leave the inbound parent attached to the task context. The request fails, but the + outer context is restored before the exception propagates. + """ + scope = self._make_scope() + ran = {} + + async def app(_scope, _receive, _send): + ran["value"] = True + + async def receive(): + return {"type": "http.request", "body": b"", "more_body": False} + + async def send(_message): + return None + + class RetryFailingTracer: + """Rejects ``context``, then raises an unrelated error on the retry.""" + + def __init__(self): + self.attempts = [] + + def start_as_current_span(self, _name, **kwargs): + self.attempts.append(sorted(kwargs)) + if "context" in kwargs: + raise TypeError("start_as_current_span() got an unexpected keyword argument 'context'") + raise RuntimeError("retry boom") + + mock_trace = MagicMock() + mock_trace.get_current_span.return_value.get_span_context.return_value = MagicMock(is_valid=False, is_remote=False) + mock_context = MagicMock() + token = object() + mock_context.attach.return_value = token + + tracer = RetryFailingTracer() + with ( + patch("mcpgateway.observability._TRACER", tracer), + patch("mcpgateway.observability.OTEL_AVAILABLE", True), + patch("mcpgateway.observability.trace", mock_trace), + patch("mcpgateway.observability.otel_extract", return_value=MagicMock()), + patch("mcpgateway.observability.otel_context", mock_context), + ): + middleware = OpenTelemetryRequestMiddleware(app) + with pytest.raises(RuntimeError, match="retry boom"): + await middleware(scope, receive, send) + + # The retry was attempted; the app never ran; the attached parent was detached. + assert tracer.attempts == [["context", "kind"], ["kind"]] + assert ran.get("value") is None + mock_context.attach.assert_called_once() + mock_context.detach.assert_called_once_with(token) + + +class TestLegacyTracerContextCompat: + """Tests for the Instana < 3.12 ``context`` keyword compatibility shim.""" + + @staticmethod + def _install(monkeypatch, tracer_cls, *, otel_context=None): + """Install a fake ``instana.tracer`` module and run the shim installer.""" + fake_module = types.ModuleType("instana.tracer") + fake_module.InstanaTracer = tracer_cls + monkeypatch.setitem(sys.modules, "instana.tracer", fake_module) + monkeypatch.setattr(observability, "OTEL_AVAILABLE", True) + if otel_context is not None: + monkeypatch.setattr(observability, "otel_context", otel_context) + observability._install_legacy_tracer_context_compat() + + def test_patches_span_starts_that_reject_context(self, monkeypatch): + """Methods lacking ``context`` are wrapped so the kwarg is accepted and the parent attaches.""" + calls = [] + + class LegacyTracer: + """Emulates InstanaTracer < 3.12: no ``context`` parameter.""" + + def start_span(self, name, span_context=None, kind=None): + calls.append(("start_span", name, span_context, kind)) + return "span" + + @contextmanager + def start_as_current_span(self, name, span_context=None, kind=None): + calls.append(("start_as_current_span", name, span_context, kind)) + yield "span" + + mock_context = MagicMock() + self._install(monkeypatch, LegacyTracer, otel_context=mock_context) + + tracer = LegacyTracer() + assert tracer.start_span("child", context=object(), kind="client") == "span" + with tracer.start_as_current_span("child", context=object(), kind="client") as span: + assert span == "span" + + # The rejected `context` never leaks into `span_context`; other kwargs pass through. + assert calls == [ + ("start_span", "child", None, "client"), + ("start_as_current_span", "child", None, "client"), + ] + # The supplied parent is attached for the span and detached afterwards. + assert mock_context.attach.call_count == 2 + assert mock_context.detach.call_count == 2 + + def test_attach_and_detach_wrap_span_lifetime(self, monkeypatch): + """The parent stays attached for the whole span and detaches when the body raises.""" + events = [] + + class LegacyTracer: + def start_span(self, name, span_context=None): + return "span" + + @contextmanager + def start_as_current_span(self, name, span_context=None): + events.append("span-open") + yield "span" + events.append("span-close") + + mock_context = MagicMock() + mock_context.attach.side_effect = lambda ctx: events.append("attach") or "token" + mock_context.detach.side_effect = lambda token: events.append("detach") + self._install(monkeypatch, LegacyTracer, otel_context=mock_context) + + tracer = LegacyTracer() + with pytest.raises(RuntimeError): + with tracer.start_as_current_span("child", context=object()): + events.append("body") + raise RuntimeError("boom") + + # Attach precedes the span; detach follows it even though the body raised. + assert events == ["attach", "span-open", "body", "detach"] + + def test_no_context_does_not_touch_otel_context(self, monkeypatch): + """Without a supplied parent the wrapper only delegates.""" + + class LegacyTracer: + def start_span(self, name, span_context=None): + return "span" + + @contextmanager + def start_as_current_span(self, name, span_context=None): + yield "span" + + mock_context = MagicMock() + self._install(monkeypatch, LegacyTracer, otel_context=mock_context) + + tracer = LegacyTracer() + assert tracer.start_span("child") == "span" + with tracer.start_as_current_span("child"): + pass + + mock_context.attach.assert_not_called() + mock_context.detach.assert_not_called() + + def test_install_is_idempotent(self, monkeypatch): + """A second install does not double-wrap the patched methods.""" + + class LegacyTracer: + def start_span(self, name, span_context=None): + return "span" + + @contextmanager + def start_as_current_span(self, name, span_context=None): + yield "span" + + self._install(monkeypatch, LegacyTracer, otel_context=MagicMock()) + patched_start_span = LegacyTracer.start_span + patched_start_as_current_span = LegacyTracer.start_as_current_span + + observability._install_legacy_tracer_context_compat() + + assert LegacyTracer.start_span is patched_start_span + assert LegacyTracer.start_as_current_span is patched_start_as_current_span + assert getattr(LegacyTracer.start_span, "__cf_context_compat__", False) is True + assert getattr(LegacyTracer.start_as_current_span, "__cf_context_compat__", False) is True + + def test_leaves_tracer_that_accepts_context_untouched(self, monkeypatch): + """A tracer that already declares ``context`` is not patched.""" + + class ModernTracer: + def start_span(self, name, context=None, kind=None): + return "span" + + def start_as_current_span(self, name, context=None, kind=None): + return nullcontext("span") + + self._install(monkeypatch, ModernTracer, otel_context=MagicMock()) + original_start_span = ModernTracer.start_span + original_start_as_current_span = ModernTracer.start_as_current_span + + observability._install_legacy_tracer_context_compat() + + assert ModernTracer.start_span is original_start_span + assert ModernTracer.start_as_current_span is original_start_as_current_span + + def test_missing_instana_is_nonfatal(self, monkeypatch): + """A missing Instana package leaves the shim a no-op.""" + monkeypatch.setitem(sys.modules, "instana.tracer", None) + monkeypatch.setattr(observability, "OTEL_AVAILABLE", True) + + observability._install_legacy_tracer_context_compat() + + def test_skipped_when_otel_unavailable(self, monkeypatch): + """Nothing is patched when OpenTelemetry is not installed.""" + monkeypatch.setattr(observability, "OTEL_AVAILABLE", False) + monkeypatch.delitem(sys.modules, "instana.tracer", raising=False) + + observability._install_legacy_tracer_context_compat() + + assert "instana.tracer" not in sys.modules From 47c3af392ec82364c042a28ed5e1718a74cf913d Mon Sep 17 00:00:00 2001 From: "NAYANA.R" Date: Thu, 15 Oct 2026 11:06:57 +0530 Subject: [PATCH 06/12] pylint fix Signed-off-by: NAYANA.R Signed-off-by: Madhav Kandukuri --- mcpgateway/observability.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mcpgateway/observability.py b/mcpgateway/observability.py index e4d1c01266d..9d8a06818b6 100644 --- a/mcpgateway/observability.py +++ b/mcpgateway/observability.py @@ -1727,7 +1727,7 @@ def _wrap_start_span_with_context(original: Callable) -> Callable: Callable: A wrapper that accepts ``context`` and preserves the parent. """ - def _compat(self: Any, name: str, context: Optional[Any] = None, *args: Any, **kwargs: Any) -> Any: + def _compat(self: Any, name: str, *args: Any, context: Optional[Any] = None, **kwargs: Any) -> Any: if context is None or otel_context is None: return original(self, name, *args, **kwargs) token = otel_context.attach(context) @@ -1755,7 +1755,7 @@ def _wrap_start_as_current_span_with_context(original: Callable) -> Callable: """ @contextmanager - def _compat(self: Any, name: str, context: Optional[Any] = None, *args: Any, **kwargs: Any) -> Any: + def _compat(self: Any, name: str, *args: Any, context: Optional[Any] = None, **kwargs: Any) -> Any: if context is None or otel_context is None: with original(self, name, *args, **kwargs) as span: yield span From 9fecb0ad6645b6efada27198f4bc39b947d0e2c4 Mon Sep 17 00:00:00 2001 From: "NAYANA.R" Date: Thu, 15 Oct 2026 11:33:17 +0530 Subject: [PATCH 07/12] pytest fix Signed-off-by: NAYANA.R Signed-off-by: Madhav Kandukuri --- tests/unit/mcpgateway/test_observability.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/unit/mcpgateway/test_observability.py b/tests/unit/mcpgateway/test_observability.py index 5ecd7e29497..7f9c38848e0 100644 --- a/tests/unit/mcpgateway/test_observability.py +++ b/tests/unit/mcpgateway/test_observability.py @@ -1074,6 +1074,8 @@ async def test_fallback_restores_context_when_retry_raises_with_context_rejectin unchanged, so later spans in the task cannot inherit the failed request's parent. """ pytest.importorskip("opentelemetry.sdk.trace") + pytest.importorskip("opentelemetry.sdk.trace.export") + pytest.importorskip("opentelemetry.sdk.trace.export.in_memory_span_exporter") # Third-Party from opentelemetry import context as otel_context_api From 8bf044976f2e2a3408494e42e61017b579b050af Mon Sep 17 00:00:00 2001 From: Madhav Kandukuri Date: Thu, 15 Oct 2026 14:12:01 +0530 Subject: [PATCH 08/12] fix: centralize legacy Instana context compatibility Signed-off-by: Madhav Kandukuri --- .github/workflows/pytest.yml | 7 +- mcpgateway/observability.py | 372 ++++---- tests/live_gateway/README.md | 1 + .../mcp/test_legacy_instana_e2e.py | 226 +++++ .../mcp/test_mcp_trace_context_e2e.py | 340 -------- tests/unit/mcpgateway/test_observability.py | 799 +++++------------- 6 files changed, 613 insertions(+), 1132 deletions(-) create mode 100644 tests/live_gateway/mcp/test_legacy_instana_e2e.py delete mode 100644 tests/live_gateway/mcp/test_mcp_trace_context_e2e.py diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index f206ebc9784..a7c62ec355e 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -243,7 +243,12 @@ jobs: tests/live_gateway/mcp/test_request_trace_policy.py \ tests/unit/mcpgateway/services/test_a2a_service.py \ tests/live_gateway/a2a/test_a2a_trace_context_e2e.py \ - tests/live_gateway/mcp/test_mcp_trace_context_e2e.py \ + -v --tb=short + + - name: 🧪 Legacy Instana (< 3.12) live-gateway regression + run: | + LEGACY_INSTANA_E2E=1 uv run --extra observability --with "instana==3.11.1" pytest \ + tests/live_gateway/mcp/test_legacy_instana_e2e.py \ -v --tb=short # ----------------------------------------------------------- diff --git a/mcpgateway/observability.py b/mcpgateway/observability.py index 9d8a06818b6..ef54446e41e 100644 --- a/mcpgateway/observability.py +++ b/mcpgateway/observability.py @@ -10,13 +10,14 @@ # Standard import base64 -from contextlib import contextmanager, nullcontext +from contextlib import nullcontext from dataclasses import dataclass from importlib import import_module as _im import inspect import logging import os -from typing import Any, Callable, cast, Dict, Mapping, MutableMapping, Optional +import sys +from typing import Any, Callable, cast, Dict, Mapping, Optional from urllib.parse import urlparse # Third-Party - Try to import OpenTelemetry core components - make them truly optional @@ -24,7 +25,6 @@ otel_baggage: Any = None # pylint: disable=invalid-name try: # Third-Party - from opentelemetry import context as otel_context from opentelemetry import trace from opentelemetry.propagate import extract as otel_extract from opentelemetry.propagate import inject as otel_inject @@ -44,7 +44,6 @@ except ImportError: # OpenTelemetry not installed - set to None for graceful degradation otel_baggage = None - otel_context = None trace = None otel_extract = None otel_inject = None @@ -349,6 +348,139 @@ class BaggageSpanAttributePolicy: _BAGGAGE_SPAN_ATTRIBUTE_POLICY = BaggageSpanAttributePolicy() +_LEGACY_INSTANA_COMPAT_MARKER = "_contextforge_otel_context_compat" + + +def _legacy_instana_span_context(context: Any, span_context_cls: Any) -> Any: + """Translate an OpenTelemetry ``Context`` into a legacy Instana ``SpanContext``. + + Mirrors the parent conversion that Instana >= 3.12 performs for ``context=``. + An explicit context without a valid span yields an invalid Instana span + context, so the new span becomes a root (standard OTel semantics) instead of + silently inheriting the ambient current span. + + Args: + context: OpenTelemetry context supplied by the caller. + span_context_cls: ``instana.span_context.SpanContext``. + + Returns: + An Instana ``SpanContext`` describing the parent. + """ + # Third-Party + from instana.util.ids import hex_id, hex_id_limited # pylint: disable=import-outside-toplevel,import-error + from opentelemetry import trace as otel_trace_api # pylint: disable=import-outside-toplevel + + parent = otel_trace_api.get_current_span(context).get_span_context() + if isinstance(parent, span_context_cls): + return parent + if parent is None or not parent.is_valid: + return span_context_cls(trace_id=otel_trace_api.INVALID_TRACE_ID, span_id=otel_trace_api.INVALID_SPAN_ID, is_remote=False) + hex_trace_id = hex_id(parent.trace_id) + is_long = len(hex_trace_id) > 16 + return span_context_cls( + trace_id=int(hex_id_limited(parent.trace_id), 16) if is_long else parent.trace_id, + span_id=parent.span_id, + is_remote=parent.is_remote, + trace_flags=parent.trace_flags, + trace_state=parent.trace_state, + long_trace_id=hex_trace_id if is_long else None, + ) + + +def _wrap_legacy_instana_method(original: Callable[..., Any], span_context_cls: Any) -> Callable[..., Any]: + """Wrap a legacy Instana tracer method so it accepts the OTel ``context=`` keyword. + + Args: + original: Unbound ``InstanaTracer`` method that only understands ``span_context``. + span_context_cls: ``instana.span_context.SpanContext``. + + Returns: + Wrapper translating ``context`` into ``span_context``; all other arguments, + return values, and exceptions pass through unchanged. + """ + + def wrapper(self: Any, *args: Any, **kwargs: Any) -> Any: + """Translate ``context`` to ``span_context`` and delegate. + + Args: + self: Tracer instance. + *args: Positional arguments forwarded unchanged. + **kwargs: Keyword arguments; ``context`` is translated. + + Returns: + Whatever the original method returns. + """ + if "context" in kwargs: + context = kwargs.pop("context") + if context is not None and kwargs.get("span_context") is None: + kwargs["span_context"] = _legacy_instana_span_context(context, span_context_cls) + return original(self, *args, **kwargs) + + wrapper.__name__ = getattr(original, "__name__", "wrapper") + wrapper.__qualname__ = getattr(original, "__qualname__", wrapper.__name__) + wrapper.__doc__ = getattr(original, "__doc__", None) + wrapper.__wrapped__ = original # type: ignore[attr-defined] + setattr(wrapper, _LEGACY_INSTANA_COMPAT_MARKER, True) + return wrapper + + +def _accepts_kwarg(func: Any, name: str) -> Optional[bool]: + """Return whether ``func`` accepts keyword ``name``; ``None`` if not introspectable. + + Args: + func: Callable to inspect. + name: Keyword argument name. + + Returns: + True/False, or None when the signature cannot be determined. + """ + try: + parameters = inspect.signature(func).parameters + except (TypeError, ValueError): + return None + return name in parameters or any(p.kind == inspect.Parameter.VAR_KEYWORD for p in parameters.values()) + + +def ensure_legacy_instana_context_compat() -> bool: + """Make legacy Instana tracers (< 3.12) accept the standard OTel ``context=`` keyword. + + MCP SDK 2.x, FastAPI native telemetry, and ContextForge call + ``start_span(context=...)`` / ``start_as_current_span(context=...)``. Legacy + ``InstanaTracer`` only accepts ``span_context=`` and raises ``TypeError``. + The adapter patches the ``InstanaTracer`` class once, so tracers cached by + third-party modules before ContextForge initializes are covered too. + + It is a no-op when Instana was not loaded by the process (this function never + imports Instana itself), when the tracer already supports ``context`` + (Instana >= 3.12), or when it was already applied. + + Returns: + True if the legacy tracer is (now) adapted, False otherwise. + """ + tracer_module = sys.modules.get("instana.tracer") + span_context_module = sys.modules.get("instana.span_context") + tracer_cls = getattr(tracer_module, "InstanaTracer", None) + span_context_cls = getattr(span_context_module, "SpanContext", None) + if tracer_cls is None or span_context_cls is None: + return False + + adapted = False + for method_name in ("start_span", "start_as_current_span"): + method = tracer_cls.__dict__.get(method_name) + if method is None: + continue + if getattr(method, _LEGACY_INSTANA_COMPAT_MARKER, False): + adapted = True + continue + # Only adapt the exact legacy capability: explicit `span_context`, no `context`. + if _accepts_kwarg(method, "context") is not False or _accepts_kwarg(method, "span_context") is not True: + continue + setattr(tracer_cls, method_name, _wrap_legacy_instana_method(method, span_context_cls)) + adapted = True + logger.info("Adapted legacy InstanaTracer.%s to accept the OpenTelemetry `context` argument", method_name) + return adapted + + def _supports_exporter_kwarg(exporter_cls: Any, kwarg: str) -> bool: """Return whether an exporter constructor accepts a keyword argument. @@ -1134,91 +1266,54 @@ async def _send_with_span_status(message: Mapping[str, Any]) -> None: span.set_status(Status(StatusCode.OK)) await send(message) - parent_context_token = None - try: - try: - span_context_manager = _TRACER.start_as_current_span(span_name, **start_span_kwargs) - except TypeError as exc: - # Some tracer implementations (e.g. Instana < 3.12) don't accept the - # `context` keyword argument even though it is part of the - # OpenTelemetry Tracer API. Retry only for that known incompatibility; - # unrelated TypeErrors must retain their original behavior. - error_message = str(exc) - rejected_context = "context" in start_span_kwargs and "unexpected keyword argument" in error_message and ("'context'" in error_message or '"context"' in error_message) - if not rejected_context: - raise - # Attach the extracted inbound parent to the current context before - # starting the span, so the fallback span still nests under the inbound - # trace and active spans match raw-header forwarding. The outer finally - # detaches the parent, including when this fallback start raises. - compatible_kwargs = {key: value for key, value in start_span_kwargs.items() if key != "context"} - if parent_context is not None and otel_context is not None: - try: - parent_context_token = otel_context.attach(parent_context) - except Exception as attach_exc: # pylint: disable=broad-exception-caught - logger.debug("Failed to attach inbound parent context: %s", attach_exc) - parent_context_token = None - span_context_manager = _TRACER.start_as_current_span(span_name, **compatible_kwargs) - logger.debug( - "Tracer %r rejected `context` kwarg; started span %r with attached parent context", - type(_TRACER).__name__, - span_name, - ) - - with span_context_manager as span: - # No valid inbound envelope: publish this new root span's context into - # the ASGI headers so downstream raw-header copies (session-task handoff, - # affinity envelopes, trusted-internal dispatch) carry the trace and - # internal spans nest under it instead of rooting detached traces. - if not has_remote_parent and otel_inject is not None: - try: - envelope_carrier: Dict[str, str] = {} - otel_inject(carrier=envelope_carrier) - envelope_traceparent = envelope_carrier.get("traceparent") - if envelope_traceparent: - new_scope_headers = list(scope.get("headers", []) or []) - new_scope_headers.append((b"traceparent", envelope_traceparent.encode("latin-1"))) - envelope_tracestate = envelope_carrier.get("tracestate") - if envelope_tracestate: - new_scope_headers.append((b"tracestate", envelope_tracestate.encode("latin-1"))) - scope["headers"] = new_scope_headers - except Exception as exc: # pylint: disable=broad-exception-caught - logger.debug("Failed to publish trace envelope into ASGI scope: %s", exc) - if span is not None: - for key, value in span_attributes.items(): - if value is not None: - set_span_attribute(span, key, value) - - # Inject baggage as span attributes so request-root spans retain - # the same trace context dimensions carried on child spans. - try: - if OTEL_AVAILABLE and otel_baggage: - baggage_dict = otel_baggage.get_all() - if baggage_dict: - baggage_attrs = _baggage_span_attributes(baggage_dict) - for key, value in baggage_attrs.items(): - set_span_attribute(span, key, value) - logger.debug("Injected %d baggage entries into request span", len(baggage_attrs)) - except Exception as exc: - logger.debug("Failed to inject baggage into request span: %s", exc) + with _TRACER.start_as_current_span(span_name, **start_span_kwargs) as span: + # No valid inbound envelope: publish this new root span's context into + # the ASGI headers so downstream raw-header copies (session-task handoff, + # affinity envelopes, trusted-internal dispatch) carry the trace and + # internal spans nest under it instead of rooting detached traces. + if not has_remote_parent and otel_inject is not None: + try: + envelope_carrier: Dict[str, str] = {} + otel_inject(carrier=envelope_carrier) + envelope_traceparent = envelope_carrier.get("traceparent") + if envelope_traceparent: + new_scope_headers = list(scope.get("headers", []) or []) + new_scope_headers.append((b"traceparent", envelope_traceparent.encode("latin-1"))) + envelope_tracestate = envelope_carrier.get("tracestate") + if envelope_tracestate: + new_scope_headers.append((b"tracestate", envelope_tracestate.encode("latin-1"))) + scope["headers"] = new_scope_headers + except Exception as exc: # pylint: disable=broad-exception-caught + logger.debug("Failed to publish trace envelope into ASGI scope: %s", exc) + if span is not None: + for key, value in span_attributes.items(): + if value is not None: + set_span_attribute(span, key, value) + # Inject baggage as span attributes so request-root spans retain + # the same trace context dimensions carried on child spans. try: - await self.app(scope, receive, _send_with_span_status) - if span is not None and "status" not in status_code_holder and OTEL_AVAILABLE and Status and StatusCode: - span.set_status(Status(StatusCode.OK)) + if OTEL_AVAILABLE and otel_baggage: + baggage_dict = otel_baggage.get_all() + if baggage_dict: + baggage_attrs = _baggage_span_attributes(baggage_dict) + for key, value in baggage_attrs.items(): + set_span_attribute(span, key, value) + logger.debug("Injected %d baggage entries into request span", len(baggage_attrs)) except Exception as exc: - if span is not None: - error_message = _sanitize_span_exception_message(exc) - set_span_error(span, exc, record_exception=True) - if OTEL_AVAILABLE and Status and StatusCode: - span.set_status(Status(StatusCode.ERROR, error_message)) - raise - finally: - if parent_context_token is not None and otel_context is not None: - try: - otel_context.detach(parent_context_token) - except Exception as exc: # pylint: disable=broad-exception-caught - logger.debug("Failed to detach inbound parent context: %s", exc) + logger.debug("Failed to inject baggage into request span: %s", exc) + + try: + await self.app(scope, receive, _send_with_span_status) + if span is not None and "status" not in status_code_holder and OTEL_AVAILABLE and Status and StatusCode: + span.set_status(Status(StatusCode.OK)) + except Exception as exc: + if span is not None: + error_message = _sanitize_span_exception_message(exc) + set_span_error(span, exc, record_exception=True) + if OTEL_AVAILABLE and Status and StatusCode: + span.set_status(Status(StatusCode.ERROR, error_message)) + raise def init_telemetry() -> Optional[Any]: @@ -1236,6 +1331,12 @@ def init_telemetry() -> Optional[Any]: """ # pylint: disable=global-statement global _TRACER + # Process-wide: FastAPI and MCP SDK telemetry use the global (possibly legacy + # Instana) tracer even when ContextForge's own OTEL integration is disabled. + try: + ensure_legacy_instana_context_compat() + except Exception as exc: # pylint: disable=broad-exception-caught + logger.warning("Failed to adapt legacy Instana tracer: %s", exc) cfg = get_settings() # Check if observability is disabled (default: disabled) @@ -1713,104 +1814,5 @@ def create_child_span(name: str, attributes: Optional[Dict[str, Any]] = None) -> return create_span(name, attributes) -def _wrap_start_span_with_context(original: Callable) -> Callable: - """Return a ``start_span`` that honors the standard ``context`` argument. - - The wrapped method attaches the supplied parent for the duration of span - creation so a tracer that reads the current OpenTelemetry context nests the - new span under it, then detaches the parent. - - Args: - original: The legacy ``start_span`` method to wrap. - - Returns: - Callable: A wrapper that accepts ``context`` and preserves the parent. - """ - - def _compat(self: Any, name: str, *args: Any, context: Optional[Any] = None, **kwargs: Any) -> Any: - if context is None or otel_context is None: - return original(self, name, *args, **kwargs) - token = otel_context.attach(context) - try: - return original(self, name, *args, **kwargs) - finally: - otel_context.detach(token) - - setattr(_compat, "__cf_context_compat__", True) - return _compat - - -def _wrap_start_as_current_span_with_context(original: Callable) -> Callable: - """Return a ``start_as_current_span`` that honors the standard ``context`` argument. - - The wrapped method attaches the supplied parent for the span's lifetime so a - tracer that reads the current OpenTelemetry context nests the span under it, - then detaches the parent when the span ends. - - Args: - original: The legacy ``start_as_current_span`` method to wrap. - - Returns: - Callable: A context-manager wrapper that accepts ``context`` and preserves the parent. - """ - - @contextmanager - def _compat(self: Any, name: str, *args: Any, context: Optional[Any] = None, **kwargs: Any) -> Any: - if context is None or otel_context is None: - with original(self, name, *args, **kwargs) as span: - yield span - return - token = otel_context.attach(context) - try: - with original(self, name, *args, **kwargs) as span: - yield span - finally: - otel_context.detach(token) - - setattr(_compat, "__cf_context_compat__", True) - return _compat - - -def _install_legacy_tracer_context_compat() -> None: - """Teach a legacy OpenTelemetry tracer to accept the standard ``context`` keyword. - - Instana < 3.12 subclasses the OpenTelemetry ``Tracer`` API but omits the - ``context`` parameter on ``start_span`` and ``start_as_current_span``. Consumers - that follow the published API (the MCP Python SDK middleware, FastAPI's native - ASGI telemetry, and this module's middleware) pass ``context=``, which raises - ``TypeError`` and aborts the request. Patch both methods to accept ``context`` - and honor the supplied parent through the standard OpenTelemetry context API. - - The patch is capability-gated (skipped when ``context`` is already a - parameter), idempotent, preserves unrelated exceptions, and is a no-op when - OpenTelemetry or the legacy tracer is absent. - """ - if not OTEL_AVAILABLE: - return - try: - # Third-Party - from instana.tracer import InstanaTracer - except Exception: # pylint: disable=broad-exception-caught - return - - wrappers = { - "start_span": _wrap_start_span_with_context, - "start_as_current_span": _wrap_start_as_current_span_with_context, - } - for method_name, wrapper in wrappers.items(): - try: - method = getattr(InstanaTracer, method_name) - if "context" in inspect.signature(method).parameters: - continue - if getattr(method, "__cf_context_compat__", False): - continue - setattr(InstanaTracer, method_name, wrapper(method)) - except Exception as exc: # pylint: disable=broad-exception-caught - logger.debug("Skipping %s context compatibility shim: %s", method_name, exc) - - -# Install the compatibility patch before any consumer captures the tracer. -_install_legacy_tracer_context_compat() - # Initialize on module import _TRACER = init_telemetry() diff --git a/tests/live_gateway/README.md b/tests/live_gateway/README.md index ed1b02e5611..ec4a1b89e7f 100644 --- a/tests/live_gateway/README.md +++ b/tests/live_gateway/README.md @@ -34,6 +34,7 @@ Specific subsuites need additional services on top: | `mcp/` | gateway with MCP transports registered | `make testing-up` (default profile) | | `mcp/test_oauth_status_live.py` | Postgres reachable at `localhost:5433` (the compose default) | `make testing-up` | | `mcp/test_private_key_jwt_e2e.py` | stub AS and upstream reachable via `host.docker.internal` (the compose default) | `make testing-up` | +| `mcp/test_legacy_instana_e2e.py` | self-contained: boots its own gateway under legacy Instana AutoTrace; set `LEGACY_INSTANA_E2E=1` to fail instead of skip | `LEGACY_INSTANA_E2E=1 uv run --extra observability --with "instana==3.11.1" pytest tests/live_gateway/mcp/test_legacy_instana_e2e.py` | | `sso/` | Keycloak (jwks tests) and/or Entra ID (entra tests) | `docker compose --profile sso up -d` for Keycloak; `AZURE_*` env vars for Entra | | `e2e_rust/` | gateway built with the Rust transport (edge or full mode) | `make testing-up` with the Rust profile, or rebuild compose images with Rust enabled | | `a2a/` | self-contained: each test boots its own gateway subprocess; needs the `observability` extra | `uv run --extra plugins --extra observability pytest tests/live_gateway/a2a/` | diff --git a/tests/live_gateway/mcp/test_legacy_instana_e2e.py b/tests/live_gateway/mcp/test_legacy_instana_e2e.py new file mode 100644 index 00000000000..93bd2404dd8 --- /dev/null +++ b/tests/live_gateway/mcp/test_legacy_instana_e2e.py @@ -0,0 +1,226 @@ +# -*- coding: utf-8 -*- +"""Location: ./tests/live_gateway/mcp/test_legacy_instana_e2e.py +Copyright contributors to the MCP-CONTEXT-FORGE project +SPDX-License-Identifier: Apache-2.0 + +Black-box regression for legacy Instana AutoTrace (``instana < 3.12``, issue #7208). + +Legacy ``InstanaTracer`` rejects the OpenTelemetry ``context=`` keyword used by +FastAPI native telemetry (``start_span``), the MCP SDK 2.x (``start_as_current_span``), +and ContextForge. This test boots an isolated gateway subprocess under the real +Instana AutoTrace bootstrap (``AUTOWRAPT_BOOTSTRAP=instana``), registers a real +streamable-HTTP MCP upstream, and verifies ``/health`` plus an authenticated MCP +``tools/call`` succeed with no escaping context-keyword ``TypeError``. + +Run (opt-in; fails rather than skips when ``LEGACY_INSTANA_E2E=1``):: + + LEGACY_INSTANA_E2E=1 uv run --extra observability \\ + --with "instana==3.11.1" \\ + pytest tests/live_gateway/mcp/test_legacy_instana_e2e.py -v + +No Instana backend is needed; the agent host points at an unused loopback port. +The gateway uses a private SQLite DB and per-run ephemeral signing keys. +""" + +# Standard +import asyncio +from importlib.metadata import PackageNotFoundError, version +import os +from pathlib import Path +import re +import secrets +import socket +import subprocess +import sys +import threading +import time + +# Third-Party +import httpx +import pytest + +# First-Party +from tests.helpers.auth import make_auth_headers, make_test_jwt + +pytestmark = pytest.mark.e2e +_REPO_ROOT = Path(__file__).resolve().parents[3] +_OPT_IN = os.getenv("LEGACY_INSTANA_E2E", "").lower() in {"1", "true", "yes"} +_CONTEXT_TYPE_ERROR = re.compile(r"TypeError:.*unexpected keyword argument '?context'?") + + +def _unavailable(reason): + """Fail when explicitly enabled; otherwise skip.""" + if _OPT_IN: + pytest.fail(reason) + pytest.skip(reason) + + +def _free_port(): + """Return an ephemeral loopback port.""" + with socket.socket() as listener: + listener.bind(("127.0.0.1", 0)) + return listener.getsockname()[1] + + +@pytest.fixture(scope="module") +def upstream_mcp_server(): + """Serve a real streamable-HTTP MCP upstream exposing an ``echo`` tool.""" + # Third-Party + from mcp.server.mcpserver import MCPServer + import uvicorn + + app = MCPServer("legacy-instana-upstream") + + @app.tool() + def echo(text: str) -> str: + """Return the supplied text.""" + return f"echo:{text}" + + port = _free_port() + server = uvicorn.Server(uvicorn.Config(app.streamable_http_app(host="127.0.0.1", stateless_http=True, json_response=True), host="127.0.0.1", port=port, log_level="error")) + worker = threading.Thread(target=server.run, daemon=True) + worker.start() + deadline = time.monotonic() + 15 + while not server.started and time.monotonic() < deadline: + time.sleep(0.05) + assert server.started, "upstream MCP server failed to start" + yield f"http://127.0.0.1:{port}/mcp" + server.should_exit = True + worker.join(timeout=5) + + +@pytest.fixture(scope="module") +def legacy_instana_gateway(tmp_path_factory, upstream_mcp_server): + """Boot a gateway under legacy Instana AutoTrace and register the upstream.""" + try: + instana_version = version("instana") + except PackageNotFoundError: + _unavailable("instana is not installed; run with --with 'instana==3.11.1'") + if tuple(int(part) for part in instana_version.split(".")[:2]) >= (3, 12): + _unavailable(f"instana {instana_version} supports `context=`; a legacy (< 3.12) sensor is required") + + tmp_path = tmp_path_factory.mktemp("legacy-instana-gateway") + gateway_port = _free_port() + signing_key = secrets.token_urlsafe(48) + env = { + "PATH": os.environ["PATH"], + "PYTHONPATH": str(_REPO_ROOT), + "AUTOWRAPT_BOOTSTRAP": "instana", + "INSTANA_SERVICE_NAME": "contextforge-legacy-instana-e2e", + "INSTANA_AGENT_HOST": "127.0.0.1", + "INSTANA_AGENT_PORT": str(_free_port()), + # gevent is only present in dev envs (via locust); production images lack it. + # Instana would otherwise gevent.monkey.patch_all(), which breaks anyio. + "INSTANA_GEVENT_MONKEY_OPTIONS": "no-socket,no-dns,no-time,no-select,no-thread,no-os,no-ssl,no-subprocess,no-sys,no-aggressive,no-event,no-builtins,no-signal,no-queue,no-contextvars", + "DATABASE_URL": f"sqlite:///{tmp_path / 'gateway.db'}", + "CACHE_TYPE": "memory", + "REDIS_URL": "", + "JWT_SECRET_KEY": signing_key, + "AUTH_ENCRYPTION_SECRET": secrets.token_urlsafe(48), + "PLATFORM_ADMIN_PASSWORD": secrets.token_urlsafe(32), + "DEFAULT_USER_PASSWORD": secrets.token_urlsafe(32), + "PLATFORM_ADMIN_EMAIL": "admin@example.com", + "AUTH_REQUIRED": "true", + "REQUIRE_USER_IN_DB": "false", + "MCPGATEWAY_ADMIN_API_ENABLED": "true", + "MCPGATEWAY_UI_ENABLED": "false", + "SSRF_ALLOW_LOCALHOST": "true", + "PLUGINS_ENABLED": "false", + "OBSERVABILITY_ENABLED": "false", + # ContextForge's own request-root middleware runs on the Instana global provider. + "OTEL_ENABLE_OBSERVABILITY": "true", + "OTEL_TRACES_EXPORTER": "console", + "LOG_LEVEL": "INFO", + "PYTHONUNBUFFERED": "1", + } + token = make_test_jwt("admin@example.com", is_admin=True, teams=None, secret=signing_key, algorithm="HS256") + log_path = tmp_path / "gateway.log" + with log_path.open("w") as log_file: + process = subprocess.Popen( + [sys.executable, "-m", "uvicorn", "mcpgateway.main:app", "--host", "127.0.0.1", "--port", str(gateway_port)], + cwd=tmp_path, + env=env, + stdout=log_file, + stderr=subprocess.STDOUT, + ) + try: + base_url = f"http://127.0.0.1:{gateway_port}" + with httpx.Client(base_url=base_url, headers=make_auth_headers(token), timeout=15, trust_env=False) as client: + deadline = time.monotonic() + 90 + while time.monotonic() < deadline: + assert process.poll() is None, log_path.read_text()[-5000:] + try: + client.get("/health") + break + except httpx.TransportError: + time.sleep(0.2) + else: + pytest.fail(f"Gateway startup timed out: {log_path.read_text()[-5000:]}") + + response = client.post( + "/gateways", + json={"name": "legacy-instana-upstream", "url": upstream_mcp_server, "transport": "STREAMABLEHTTP", "visibility": "public"}, + ) + assert response.status_code in (200, 201), response.text + gateway_id = response.json()["id"] + + tool_name = None + deadline = time.monotonic() + 30 + while tool_name is None and time.monotonic() < deadline: + tools = client.get("/tools", params={"limit": 0, "gateway_id": gateway_id}) + if tools.status_code == 200: + tool_name = next((tool["name"] for tool in tools.json() if tool.get("name", "").endswith("echo")), None) + if tool_name is None: + time.sleep(0.5) + assert tool_name, f"upstream echo tool was not discovered: {log_path.read_text()[-3000:]}" + + yield {"client": client, "base_url": base_url, "token": token, "tool_name": tool_name, "log_path": log_path} + finally: + process.terminate() + try: + process.wait(timeout=15) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=5) + + +def test_gateway_runs_under_legacy_instana(legacy_instana_gateway): + """The gateway process is actually instrumented by the legacy Instana sensor and adapter.""" + log = legacy_instana_gateway["log_path"].read_text() + assert "Adapted legacy InstanaTracer.start_span" in log, log[-3000:] + assert "Adapted legacy InstanaTracer.start_as_current_span" in log, log[-3000:] + + +def test_health_ok_under_legacy_instana(legacy_instana_gateway): + """FastAPI native telemetry calls ``start_span(context=...)`` on every request.""" + response = legacy_instana_gateway["client"].get("/health") + assert response.status_code == 200, response.text + + +def test_mcp_tools_call_under_legacy_instana(legacy_instana_gateway): + """Authenticated streamable-HTTP MCP ``tools/call`` completes with no context TypeError.""" + # Third-Party + import httpx2 + from mcp import ClientSession + from mcp.client.streamable_http import create_mcp_http_client, streamable_http_client + + async def _call(): + http_client = create_mcp_http_client(headers=make_auth_headers(legacy_instana_gateway["token"]), timeout=httpx2.Timeout(15.0)) + async with streamable_http_client(f"{legacy_instana_gateway['base_url']}/mcp/", http_client=http_client) as (read_stream, write_stream): + async with ClientSession(read_stream, write_stream, read_timeout_seconds=15.0) as session: + await session.initialize() + return await session.call_tool(legacy_instana_gateway["tool_name"], {"text": "instana"}) + + result = asyncio.run(_call()) + assert not result.is_error, result + assert any("echo:instana" in getattr(item, "text", "") for item in result.content), result + + log = legacy_instana_gateway["log_path"].read_text() + assert not _CONTEXT_TYPE_ERROR.search(log), log[-5000:] + + +def test_unauthenticated_mcp_rejected_under_legacy_instana(legacy_instana_gateway): + """Deny path: unauthenticated MCP requests are still rejected.""" + with httpx.Client(base_url=legacy_instana_gateway["base_url"], timeout=15, trust_env=False) as anonymous: + response = anonymous.post("/mcp/", json={"jsonrpc": "2.0", "id": 1, "method": "tools/list"}, headers={"Accept": "application/json, text/event-stream"}) + assert response.status_code == 401, response.text diff --git a/tests/live_gateway/mcp/test_mcp_trace_context_e2e.py b/tests/live_gateway/mcp/test_mcp_trace_context_e2e.py deleted file mode 100644 index f6e3974e85e..00000000000 --- a/tests/live_gateway/mcp/test_mcp_trace_context_e2e.py +++ /dev/null @@ -1,340 +0,0 @@ -# -*- coding: utf-8 -*- -"""Location: ./tests/live_gateway/mcp/test_mcp_trace_context_e2e.py -Copyright contributors to the MCP-CONTEXT-FORGE project -SPDX-License-Identifier: Apache-2.0 - -Black-box W3C trace-context check for MCP ``tools/call`` against an isolated gateway. - -The gateway subprocess runs with a ``sitecustomize`` shim that installs an -``instana.tracer.InstanaTracer`` emulating ``instana < 3.12``: ``get_tracer`` -returns it for every instrumentation name, so the gateway middleware, the MCP -Python SDK (``mcp-python-sdk``), and FastAPI telemetry all see a tracer without -the OpenTelemetry ``context`` parameter. The production compatibility layer must -patch that tracer so authenticated ``tools/call`` still completes. A valid W3C -``traceparent`` arrives with the call; the request-root span must adopt that -inbound parent, so the active span and every span beneath it stay in one trace. -A marker file records that the incompatible path ran. - -Run with the observability extra installed. The gateway uses a private SQLite -DB, real REST tool invocation, real HTTP requests, and the SDK console exporter. -No existing gateway, Redis instance, or credentials are used. -""" - -# Standard -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -import json -import os -from pathlib import Path -import secrets -import socket -import subprocess -import sys -import threading -import time -from typing import Any - -# Third-Party -import httpx -import pytest - -# First-Party -from tests.helpers.auth import make_auth_headers, make_test_jwt -from tests.live_gateway.plugins._helpers import create_virtual_server, initialize_session, mcp_headers, result_text - -pytestmark = pytest.mark.e2e -_REPO_ROOT = Path(__file__).resolve().parents[3] - -INBOUND_TRACE_ID = "0af7651916cd43dd8448eb211c80319c" # pragma: allowlist secret -INBOUND_SPAN_ID = "b7ad6b7169203331" # pragma: allowlist secret -TOOL_NAME = "trace-context-probe" - -# Emulate the legacy Instana provider (``instana < 3.12``): a tracer whose -# ``start_span``/``start_as_current_span`` do not accept the OpenTelemetry -# ``context`` keyword. A real legacy Instana AutoTrace replaces the global tracer -# provider, so *every* consumer sees the incompatible tracer -- the gateway's own -# middleware, the MCP Python SDK (``mcp-python-sdk``), and FastAPI telemetry. The -# shim registers the emulated class as ``instana.tracer.InstanaTracer`` so the -# production compatibility layer patches it exactly as it would patch the real one. -_SITECUSTOMIZE = ''' -import os -import sys -import types - -try: - import opentelemetry.trace as _trace -except Exception: - _trace = None - -if _trace is not None and hasattr(_trace, "get_tracer"): - _orig_get_tracer = _trace.get_tracer - _marker = os.environ.get("MCP_TRACE_INCOMPATIBLE_MARKER") - - def _record(name): - if not _marker: - return - try: - with open(_marker, "a", encoding="utf-8") as handle: - handle.write(name + "\\n") - except OSError: - pass - - class InstanaTracer: - """Emulate ``instana.tracer.InstanaTracer`` before 3.12: no ``context`` parameter.""" - - def __init__(self, inner): - self._inner = inner - - def __getattr__(self, name): - return getattr(self._inner, name) - - def start_span(self, name, *args, **kwargs): - if "context" in kwargs: - raise TypeError("start_span() got an unexpected keyword argument 'context'") - return self._inner.start_span(name, *args, **kwargs) - - def start_as_current_span(self, name, *args, **kwargs): - _record(name) - if "context" in kwargs: - raise TypeError("start_as_current_span() got an unexpected keyword argument 'context'") - return self._inner.start_as_current_span(name, *args, **kwargs) - - def _wrapped_get_tracer(*args, **kwargs): - return InstanaTracer(_orig_get_tracer(*args, **kwargs)) - - _trace.get_tracer = _wrapped_get_tracer - - _instana_pkg = types.ModuleType("instana") - _instana_pkg.__path__ = [] - _tracer_mod = types.ModuleType("instana.tracer") - _tracer_mod.InstanaTracer = InstanaTracer - _instana_pkg.tracer = _tracer_mod - sys.modules["instana"] = _instana_pkg - sys.modules["instana.tracer"] = _tracer_mod -''' - - -class _ToolHandler(BaseHTTPRequestHandler): - """Return a fixed JSON body and record inbound request headers.""" - - captured: list[list[tuple[str, str]]] = [] - - def do_GET(self): - """Record headers, then answer 200 with a JSON body.""" - type(self).captured.append(list(self.headers.items())) - body = b'{"ok": true}' - self.send_response(200) - self.send_header("Content-Type", "application/json") - self.send_header("Content-Length", str(len(body))) - self.end_headers() - self.wfile.write(body) - - def log_message(self, format, *args): - """Keep the upstream server quiet.""" - - -def _console_spans(log_path: Path) -> list[dict[str, Any]]: - """Read complete SDK span JSON objects, ignoring gateway log lines.""" - contents = log_path.read_text(encoding="utf-8") - decoder = json.JSONDecoder() - spans = [] - offset = 0 - while (offset := contents.find('{\n "name":', offset)) >= 0: - try: - span, length = decoder.raw_decode(contents[offset:]) - except json.JSONDecodeError: - break - spans.append(span) - offset += length - return spans - - -def _hex_id(value: Any) -> Any: - """Normalize a console-exporter hex identifier by dropping the 0x prefix.""" - return value[2:] if isinstance(value, str) and value.startswith("0x") else value - - -def _wait_for_spans(log_path: Path, predicate, deadline_seconds: float = 15.0) -> list[dict[str, Any]]: - """Poll the gateway log until ``predicate(spans)`` returns truthy or time runs out.""" - deadline = time.monotonic() + deadline_seconds - spans: list[dict[str, Any]] = [] - while time.monotonic() < deadline: - spans = _console_spans(log_path) - if predicate(spans): - return spans - time.sleep(0.1) - return spans - - -def _find_request_span(spans: list[dict[str, Any]], server_id: str) -> dict[str, Any] | None: - """Return the middleware request-root span for ``server_id`` in the inbound trace, if exported.""" - expected_name = f"POST /servers/{server_id}/mcp/" - for span in spans: - if span.get("name") == expected_name and _hex_id(span.get("context", {}).get("trace_id")) == INBOUND_TRACE_ID: - return span - return None - - -def _find_tool_call_span(spans: list[dict[str, Any]], tool_name: str) -> dict[str, Any] | None: - """Return the MCP ``tools/call`` span for ``tool_name`` in the inbound trace, if exported.""" - expected_name = f"tools/call {tool_name}" - for span in spans: - if span.get("name") == expected_name and _hex_id(span.get("context", {}).get("trace_id")) == INBOUND_TRACE_ID: - return span - return None - - -@pytest.fixture -def isolated_incompatible_gateway(tmp_path): - """Start an authenticated gateway whose tracer rejects the ``context`` kwarg. - - Registers a REST tool backed by a local capture server and a virtual server - that exposes it. - - Yields: - A tuple of the authenticated client, bearer token, gateway log path, - incompatibility marker path, virtual server id, and tool name. - """ - trace_sdk = pytest.importorskip("opentelemetry.sdk.trace") - if not hasattr(trace_sdk, "TracerProvider"): - pytest.skip("OpenTelemetry SDK is not installed") - - harness_dir = tmp_path / "harness" - harness_dir.mkdir() - (harness_dir / "sitecustomize.py").write_text(_SITECUSTOMIZE, encoding="utf-8") - marker_path = tmp_path / "incompatible.marker" - - _ToolHandler.captured = [] - upstream = ThreadingHTTPServer(("127.0.0.1", 0), _ToolHandler) - upstream_worker = threading.Thread(target=upstream.serve_forever, daemon=True) - upstream_worker.start() - - with socket.socket() as listener: - listener.bind(("127.0.0.1", 0)) - port = listener.getsockname()[1] - signing_key = secrets.token_urlsafe(48) - env = { - "PATH": os.environ["PATH"], - "PYTHONPATH": os.pathsep.join([str(harness_dir), str(_REPO_ROOT), *[str(Path(path).resolve()) for path in os.getenv("PYTHONPATH", "").split(os.pathsep) if path]]), - "DATABASE_URL": f"sqlite:///{tmp_path / 'gateway.db'}", - "CACHE_TYPE": "memory", - "REDIS_URL": "", - "JWT_SECRET_KEY": signing_key, - "AUTH_ENCRYPTION_SECRET": secrets.token_urlsafe(48), - "PLATFORM_ADMIN_PASSWORD": secrets.token_urlsafe(32), - "DEFAULT_USER_PASSWORD": secrets.token_urlsafe(32), - "PLATFORM_ADMIN_EMAIL": "admin@example.com", - "AUTH_REQUIRED": "true", - "REQUIRE_USER_IN_DB": "false", - "MCPGATEWAY_ADMIN_API_ENABLED": "true", - "MCPGATEWAY_UI_ENABLED": "false", - "MCPGATEWAY_A2A_ENABLED": "false", - "SSRF_ALLOW_LOCALHOST": "true", - "PLUGINS_ENABLED": "false", - "OBSERVABILITY_ENABLED": "false", - "OTEL_ENABLE_OBSERVABILITY": "true", - "OTEL_TRACES_EXPORTER": "console", - "OTEL_EXPORTER_OTLP_ENDPOINT": "", - "MCP_TRACE_INCOMPATIBLE_MARKER": str(marker_path), - "LOG_LEVEL": "ERROR", - "PYTHONUNBUFFERED": "1", - } - token = make_test_jwt("admin@example.com", is_admin=True, teams=None, secret=signing_key, algorithm="HS256") - log_path = tmp_path / "gateway.log" - tool_id = None - server_id = None - with log_path.open("w") as log_file: - process = subprocess.Popen( - [sys.executable, "-m", "uvicorn", "mcpgateway.main:app", "--host", "127.0.0.1", "--port", str(port)], - cwd=tmp_path, - env=env, - stdout=log_file, - stderr=subprocess.STDOUT, - ) - try: - with httpx.Client(base_url=f"http://127.0.0.1:{port}", headers=make_auth_headers(token), timeout=15, trust_env=False) as client: - deadline = time.monotonic() + 60 - while time.monotonic() < deadline: - assert process.poll() is None, log_path.read_text(encoding="utf-8")[-5000:] - try: - if client.get("/health").status_code == 200: - break - except httpx.TransportError: - pass - time.sleep(0.1) - else: - pytest.fail(f"Gateway startup timed out: {log_path.read_text(encoding='utf-8')[-5000:]}") - - response = client.post( - "/tools", - json={ - "tool": { - "name": TOOL_NAME, - "description": "Trace-context probe tool", - "integration_type": "REST", - "url": f"http://127.0.0.1:{upstream.server_port}/probe", - "request_type": "GET", - "visibility": "public", - }, - "team_id": None, - }, - ) - assert response.status_code == 200, response.text - tool_id = response.json()["id"] - server_id = create_virtual_server(client, name="trace-context-server", tool_ids=[tool_id]) - - yield client, token, log_path, marker_path, server_id, TOOL_NAME - finally: - process.terminate() - try: - process.wait(timeout=15) - except subprocess.TimeoutExpired: - process.kill() - process.wait(timeout=5) - upstream.shutdown() - upstream.server_close() - upstream_worker.join(timeout=5) - - -def test_mcp_tools_call_preserves_inbound_trace_with_incompatible_tracer(isolated_incompatible_gateway): - """An authenticated ``tools/call`` keeps the inbound trace through an incompatible tracer.""" - client, token, log_path, marker_path, server_id, tool_name = isolated_incompatible_gateway - session_id = initialize_session(client, server_id=server_id, token=token) - _ToolHandler.captured.clear() - - payload = { - "jsonrpc": "2.0", - "id": 1, - "method": "tools/call", - "params": {"name": tool_name, "arguments": {}}, - } - headers = mcp_headers(token, session_id=session_id) - headers["traceparent"] = f"00-{INBOUND_TRACE_ID}-{INBOUND_SPAN_ID}-01" - response = client.post(f"/servers/{server_id}/mcp/", headers=headers, json=payload) - assert response.status_code == 200, response.text - result = response.json().get("result", {}) - assert result.get("isError") is not True, response.text - assert "ok" in result_text(result) - - # The incompatible tracer path ran: the middleware fell back to the standard - # context API instead of passing the ``context`` keyword. - assert marker_path.exists(), "context-rejecting tracer was never exercised" - assert f"POST /servers/{server_id}/mcp/" in marker_path.read_text(encoding="utf-8") - - spans = _wait_for_spans(log_path, lambda current: _find_request_span(current, server_id) is not None) - request_span = _find_request_span(spans, server_id) - assert request_span is not None, f"No request span was exported; exported spans: {[span['name'] for span in spans]}" - # Parent identity: the request-root span continues the inbound trace. - assert _hex_id(request_span["context"]["trace_id"]) == INBOUND_TRACE_ID - assert _hex_id(request_span["parent_id"]) == INBOUND_SPAN_ID - request_span_id = _hex_id(request_span["context"]["span_id"]) - assert request_span_id not in (None, "") - - # Active-span consistency: the tool invocation runs beneath the request-root - # span, in the same inbound trace, so any downstream span nests under it. - tool_span = _find_tool_call_span(spans, tool_name) - assert tool_span is not None, f"No tools/call span was exported; exported spans: {[span['name'] for span in spans]}" - assert _hex_id(tool_span["parent_id"]) == request_span_id - - # The tool reached the real upstream REST server, proving the request was - # not short-circuited by the incompatible tracer. - assert _ToolHandler.captured, "the tool did not reach the upstream REST server" diff --git a/tests/unit/mcpgateway/test_observability.py b/tests/unit/mcpgateway/test_observability.py index 7f9c38848e0..63eca0fc1d6 100644 --- a/tests/unit/mcpgateway/test_observability.py +++ b/tests/unit/mcpgateway/test_observability.py @@ -7,12 +7,9 @@ """ # Standard -from contextlib import contextmanager, nullcontext import inspect import logging import os -import sys -import types from unittest.mock import MagicMock, patch # Third-Party @@ -836,645 +833,235 @@ async def test_middleware_envelope_publication_failure_is_nonfatal(self): assert captured["headers"] == original_headers - @pytest.mark.asyncio - async def test_middleware_tolerates_tracer_without_context_kwarg(self): - """A tracer that rejects `context=` (e.g. Instana < 3.12) must not break the request.""" - captured = {} - app, receive, send = self._make_app(captured) - scope = self._make_scope([(b"content-type", b"application/json")]) - class ContextRejectingTracer: - """Emulates InstanaTracer < 3.12: start_as_current_span() lacks `context`.""" +INBOUND_TRACEPARENT = "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01" # pragma: allowlist secret - def __init__(self): - self.attempts = [] - def start_as_current_span(self, _name, **kwargs): - self.attempts.append(sorted(kwargs)) - if "context" in kwargs: - raise TypeError("start_as_current_span() got an unexpected keyword argument 'context'") - return nullcontext(None) +@pytest.fixture +def fake_legacy_instana(monkeypatch): + """Install an ``instana`` stand-in whose tracer mirrors the Instana 3.11 signatures. - tracer = ContextRejectingTracer() - mock_trace = MagicMock() - mock_trace.get_current_span.return_value.get_span_context.return_value = MagicMock(is_valid=False, is_remote=False) + The fake exposes the same ``span_context``-only signatures as Instana < 3.12, + backed by the real OpenTelemetry context API, so no Instana dependency is needed. + """ + pytest.importorskip("opentelemetry.trace") + # Standard + from contextlib import contextmanager + import sys + import types - with ( - patch("mcpgateway.observability._TRACER", tracer), - patch("mcpgateway.observability.OTEL_AVAILABLE", True), - patch("mcpgateway.observability.trace", mock_trace), - patch("mcpgateway.observability.otel_extract", return_value=MagicMock()), - patch("mcpgateway.observability.otel_inject", side_effect=self._fake_inject), - ): - middleware = OpenTelemetryRequestMiddleware(app) - await middleware(scope, receive, send) + # Third-Party + from opentelemetry import trace as otel_trace_api - # First attempt passed the parent `context`; the compatibility fallback dropped it. - assert ["context", "kind"] in tracer.attempts - assert ["kind"] in tracer.attempts - # The request still completed and the trace envelope was published. - assert (b"traceparent", b"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01") in captured["headers"] + class SpanContext(otel_trace_api.SpanContext): + """Legacy Instana span context (extra kwargs accepted and ignored).""" - @pytest.mark.asyncio - async def test_middleware_does_not_retry_type_error_without_context(self): - """A TypeError cannot be a context compatibility error when no context was passed.""" - captured = {} - app, receive, send = self._make_app(captured) - scope = self._make_scope() + def __new__(cls, trace_id, span_id, is_remote, trace_flags=None, trace_state=None, **_kwargs): + return super().__new__(cls, trace_id, span_id, is_remote, trace_flags, trace_state) - class FailingTracer: - """Track calls before raising an unrelated tracer failure.""" + class FakeSpan(otel_trace_api.NonRecordingSpan): + def __init__(self, ctx, parent_id): + super().__init__(ctx) + self.parent_id = parent_id - def __init__(self): - self.attempts = [] + counter = {"id": 0x1000} - def start_as_current_span(self, _name, **kwargs): - self.attempts.append(sorted(kwargs)) - raise TypeError("unrelated tracer failure") + class InstanaTracer: + """Legacy tracer: only ``span_context``; ``context=`` raises TypeError.""" - tracer = FailingTracer() - with ( - patch("mcpgateway.observability._TRACER", tracer), - patch("mcpgateway.observability.otel_extract", return_value=None), - ): - middleware = OpenTelemetryRequestMiddleware(app) - with pytest.raises(TypeError, match="unrelated tracer failure"): - await middleware(scope, receive, send) + def start_span(self, name, span_context=None, kind=None, attributes=None, links=None, start_time=None, record_exception=True, set_status_on_exception=True): + parent = span_context if span_context else otel_trace_api.get_current_span().get_span_context() + counter["id"] += 1 + if parent is not None and parent.is_valid: + ctx = SpanContext(parent.trace_id, counter["id"], False) + return FakeSpan(ctx, parent.span_id) + return FakeSpan(SpanContext(counter["id"], counter["id"], False), None) - assert tracer.attempts == [["kind"]] + @contextmanager + def start_as_current_span(self, name, span_context=None, kind=None, attributes=None, links=None, start_time=None, record_exception=True, set_status_on_exception=True, end_on_exit=True): + span = self.start_span(name, span_context=span_context, kind=kind) + with otel_trace_api.use_span(span, end_on_exit=end_on_exit) as current: + yield current - @pytest.mark.asyncio - async def test_middleware_does_not_swallow_unrelated_type_error_with_context(self): - """Only rejection of the context keyword should activate the compatibility retry.""" - captured = {} - app, receive, send = self._make_app(captured) - scope = self._make_scope() + tracer_mod = types.ModuleType("instana.tracer") + tracer_mod.InstanaTracer = InstanaTracer + span_context_mod = types.ModuleType("instana.span_context") + span_context_mod.SpanContext = SpanContext + ids_mod = types.ModuleType("instana.util.ids") + ids_mod.hex_id = lambda value: format(int(value), "016x") if int(value) < (1 << 64) else format(int(value), "032x") + ids_mod.hex_id_limited = lambda value: format(int(value), "032x")[-16:] + monkeypatch.setitem(sys.modules, "instana.tracer", tracer_mod) + monkeypatch.setitem(sys.modules, "instana.span_context", span_context_mod) + monkeypatch.setitem(sys.modules, "instana.util.ids", ids_mod) + return InstanaTracer - class InternallyFailingTracer: - """Accept context but fail internally while creating the span.""" - def __init__(self): - self.attempts = [] +class TestLegacyInstanaContextCompat: + """Process-wide adapter for legacy Instana tracers rejecting ``context=``.""" - def start_as_current_span(self, _name, **kwargs): - self.attempts.append(sorted(kwargs)) - if "context" in kwargs: - raise TypeError("tracer implementation failed internally") - return nullcontext(None) + @staticmethod + def _inbound_context(): + # Third-Party + from opentelemetry.propagate import extract - tracer = InternallyFailingTracer() - with ( - patch("mcpgateway.observability._TRACER", tracer), - patch("mcpgateway.observability.otel_extract", return_value=MagicMock()), - ): - middleware = OpenTelemetryRequestMiddleware(app) - with pytest.raises(TypeError, match="tracer implementation failed internally"): - await middleware(scope, receive, send) + return extract({"traceparent": INBOUND_TRACEPARENT}) - assert tracer.attempts == [["context", "kind"]] + def test_cached_instance_start_span_accepts_context(self, fake_legacy_instana): + """Instances created before adaptation (e.g. FastAPI/MCP SDK caches) are covered.""" + cached = fake_legacy_instana() + with pytest.raises(TypeError, match="unexpected keyword argument 'context'"): + cached.start_span("pre", context=self._inbound_context()) - @pytest.mark.asyncio - async def test_fallback_preserves_inbound_parent_with_context_rejecting_tracer(self, monkeypatch): - """The context-kwarg fallback must nest the request span under the inbound trace. + assert observability.ensure_legacy_instana_context_compat() is True + span = cached.start_span("GET /health", context=self._inbound_context(), kind=None) + assert span.get_span_context().trace_id == 0x0AF7651916CD43DD8448EB211C80319C & ((1 << 64) - 1) + assert span.parent_id == 0xB7AD6B7169203331 - A tracer that rejects the ``context`` keyword (e.g. Instana < 3.12) resolves its - parent from the current context. The middleware attaches the extracted inbound - parent for the span lifetime, so the active span and the forwarded ``traceparent`` - stay in the same trace and the outer context is restored afterwards. - """ - trace_sdk = pytest.importorskip("opentelemetry.sdk.trace") - export_sdk = pytest.importorskip("opentelemetry.sdk.trace.export") - memory_export_sdk = pytest.importorskip("opentelemetry.sdk.trace.export.in_memory_span_exporter") + def test_start_as_current_span_accepts_context_and_restores(self, fake_legacy_instana): + """``start_as_current_span(context=...)`` parents correctly and restores the outer context.""" # Third-Party from opentelemetry import context as otel_context_api from opentelemetry import trace as otel_trace_api - exporter = memory_export_sdk.InMemorySpanExporter() - provider = trace_sdk.TracerProvider() - provider.add_span_processor(export_sdk.SimpleSpanProcessor(exporter)) - inner_tracer = provider.get_tracer("instana-compat-parent-test") - - class ContextRejectingTracer: - """Emulates InstanaTracer < 3.12: rejects the ``context`` kwarg.""" - - def __init__(self, inner): - self.inner = inner - self.attempts = [] - - def start_as_current_span(self, name, **kwargs): - self.attempts.append(sorted(kwargs)) - if "context" in kwargs: - raise TypeError("start_as_current_span() got an unexpected keyword argument 'context'") - return self.inner.start_as_current_span(name, **kwargs) - - tracer = ContextRejectingTracer(inner_tracer) - monkeypatch.setattr(observability, "_TRACER", tracer) - - active_trace_id = {} - completed = {} - - async def app(_scope, _receive, send): - active_span = otel_trace_api.get_current_span() - active_trace_id["value"] = f"{active_span.get_span_context().trace_id:032x}" - await send({"type": "http.response.start", "status": 200, "headers": []}) - await send({"type": "http.response.body", "body": b""}) - completed["value"] = True - - async def receive(): - return {"type": "http.request", "body": b"", "more_body": False} - - async def send(_message): - return None - - incoming_trace_id = "0af7651916cd43dd8448eb211c80319c" # pragma: allowlist secret - incoming_span_id = "b7ad6b7169203331" # pragma: allowlist secret - incoming_traceparent = f"00-{incoming_trace_id}-{incoming_span_id}-01" - scope = self._make_scope([(b"traceparent", incoming_traceparent.encode())]) - middleware = OpenTelemetryRequestMiddleware(app) - - context_before = otel_context_api.get_current() - await middleware(scope, receive, send) - context_after = otel_context_api.get_current() - - # The tracer first rejects `context`, then retries without it. - assert ["context", "kind"] in tracer.attempts - assert ["kind"] in tracer.attempts - # The request completed and the outer context was restored after detach. - assert completed.get("value") is True - assert context_after is context_before - # The request span nested under the inbound trace instead of rooting a new one. - (request_span,) = exporter.get_finished_spans() - assert f"{request_span.context.trace_id:032x}" == incoming_trace_id - assert f"{request_span.parent.span_id:016x}" == incoming_span_id - # The active span during request handling shares the forwarded trace. - assert active_trace_id["value"] == incoming_trace_id - assert dict(scope["headers"])[b"traceparent"].decode() == incoming_traceparent - - @pytest.mark.asyncio - async def test_fallback_restores_context_when_app_raises_with_context_rejecting_tracer(self, monkeypatch): - """The fallback detaches the attached parent even when the wrapped app fails.""" - trace_sdk = pytest.importorskip("opentelemetry.sdk.trace") - export_sdk = pytest.importorskip("opentelemetry.sdk.trace.export") - memory_export_sdk = pytest.importorskip("opentelemetry.sdk.trace.export.in_memory_span_exporter") + observability.ensure_legacy_instana_context_compat() + tracer = fake_legacy_instana() + before = otel_context_api.get_current() + with tracer.start_as_current_span("mcp", context=self._inbound_context()) as span: + assert span.parent_id == 0xB7AD6B7169203331 + assert otel_trace_api.get_current_span() is span + child = tracer.start_span("child") + assert child.parent_id == span.get_span_context().span_id + assert child.get_span_context().trace_id == span.get_span_context().trace_id + assert otel_context_api.get_current() is before + + def test_context_restored_when_body_raises(self, fake_legacy_instana): + """Application failures inside the span propagate and the context is restored.""" # Third-Party from opentelemetry import context as otel_context_api - from opentelemetry.trace import StatusCode - - exporter = memory_export_sdk.InMemorySpanExporter() - provider = trace_sdk.TracerProvider() - provider.add_span_processor(export_sdk.SimpleSpanProcessor(exporter)) - inner_tracer = provider.get_tracer("instana-compat-failure-test") - - class ContextRejectingTracer: - """Emulates InstanaTracer < 3.12: rejects the ``context`` kwarg.""" - - def __init__(self, inner): - self.inner = inner - - def start_as_current_span(self, name, **kwargs): - if "context" in kwargs: - raise TypeError("start_as_current_span() got an unexpected keyword argument 'context'") - return self.inner.start_as_current_span(name, **kwargs) - - monkeypatch.setattr(observability, "_TRACER", ContextRejectingTracer(inner_tracer)) - - async def app(_scope, _receive, _send): - raise RuntimeError("app failure") - async def receive(): - return {"type": "http.request", "body": b"", "more_body": False} - - async def send(_message): - return None - - incoming_trace_id = "0af7651916cd43dd8448eb211c80319c" # pragma: allowlist secret - incoming_span_id = "b7ad6b7169203331" # pragma: allowlist secret - scope = self._make_scope([(b"traceparent", f"00-{incoming_trace_id}-{incoming_span_id}-01".encode())]) - middleware = OpenTelemetryRequestMiddleware(app) - - context_before = otel_context_api.get_current() - with pytest.raises(RuntimeError, match="app failure"): - await middleware(scope, receive, send) - context_after = otel_context_api.get_current() - - # Detach runs in `finally`, so the outer context survives the app failure. - assert context_after is context_before - # The failed request span still belongs to the inbound trace. - (request_span,) = exporter.get_finished_spans() - assert f"{request_span.context.trace_id:032x}" == incoming_trace_id - assert f"{request_span.parent.span_id:016x}" == incoming_span_id - assert request_span.status.status_code is StatusCode.ERROR + observability.ensure_legacy_instana_context_compat() + before = otel_context_api.get_current() + with pytest.raises(RuntimeError, match="app"): + with fake_legacy_instana().start_as_current_span("x", context=self._inbound_context()): + raise RuntimeError("app") + assert otel_context_api.get_current() is before - @pytest.mark.asyncio - async def test_fallback_restores_context_when_retry_raises_with_context_rejecting_tracer(self, monkeypatch): - """The fallback restores the outer context when the retry itself raises. - - A context-rejecting tracer that fails again on the retry must not leak the inbound - parent into the task context. The retry failure propagates, but the outer context is - unchanged, so later spans in the task cannot inherit the failed request's parent. - """ - pytest.importorskip("opentelemetry.sdk.trace") - pytest.importorskip("opentelemetry.sdk.trace.export") - pytest.importorskip("opentelemetry.sdk.trace.export.in_memory_span_exporter") + def test_explicit_empty_context_creates_root(self, fake_legacy_instana): + """An explicit context without a span roots a new trace rather than inheriting the ambient span.""" # Third-Party - from opentelemetry import context as otel_context_api - - class RetryFailingTracer: - """Rejects ``context``, then raises an unrelated error on the retry.""" - - def __init__(self): - self.attempts = [] - - def start_as_current_span(self, name, **kwargs): - self.attempts.append(sorted(kwargs)) - if "context" in kwargs: - raise TypeError("start_as_current_span() got an unexpected keyword argument 'context'") - raise RuntimeError("retry failure") - - tracer = RetryFailingTracer() - monkeypatch.setattr(observability, "_TRACER", tracer) - - ran = {} - - async def app(_scope, _receive, _send): - ran["value"] = True - - async def receive(): - return {"type": "http.request", "body": b"", "more_body": False} - - async def send(_message): - return None - - incoming_trace_id = "0af7651916cd43dd8448eb211c80319c" # pragma: allowlist secret - incoming_span_id = "b7ad6b7169203331" # pragma: allowlist secret - scope = self._make_scope([(b"traceparent", f"00-{incoming_trace_id}-{incoming_span_id}-01".encode())]) - middleware = OpenTelemetryRequestMiddleware(app) - - context_before = otel_context_api.get_current() - with pytest.raises(RuntimeError, match="retry failure"): - await middleware(scope, receive, send) - context_after = otel_context_api.get_current() - - # The retry was attempted; the app never ran; the attached parent was detached. - assert tracer.attempts == [["context", "kind"], ["kind"]] - assert ran.get("value") is None - assert context_after is context_before - - @pytest.mark.asyncio - async def test_middleware_fallback_attaches_parent_injects_baggage_and_detaches(self): - """The context-kwarg fallback attaches the inbound parent, injects baggage, and detaches it. - - Patches the module-level OTEL globals so the assertions hold without the - ``observability`` extra (SDK absent), matching the main pytest CI job. - """ - scope = self._make_scope([(b"traceparent", b"00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01")]) - ran = {} - - async def app(_scope, _receive, _send): - ran["value"] = True - - async def receive(): - return {"type": "http.request", "body": b"", "more_body": False} - - async def send(_message): - return None - - class ContextRejectingTracer: - """Emulates InstanaTracer < 3.12: rejects the ``context`` kwarg.""" - - def start_as_current_span(self, _name, **kwargs): - if "context" in kwargs: - raise TypeError("start_as_current_span() got an unexpected keyword argument 'context'") - return nullcontext(MagicMock()) - - mock_trace = MagicMock() - mock_trace.get_current_span.return_value.get_span_context.return_value = MagicMock(is_valid=False, is_remote=False) - mock_context = MagicMock() - token = object() - mock_context.attach.return_value = token - mock_baggage = MagicMock() - mock_baggage.get_all.return_value = {"tenant": "acme"} - - with ( - patch("mcpgateway.observability._TRACER", ContextRejectingTracer()), - patch("mcpgateway.observability.OTEL_AVAILABLE", True), - patch("mcpgateway.observability.trace", mock_trace), - patch("mcpgateway.observability.otel_extract", return_value=MagicMock()), - patch("mcpgateway.observability.otel_inject", side_effect=self._fake_inject), - patch("mcpgateway.observability.otel_context", mock_context), - patch("mcpgateway.observability.otel_baggage", mock_baggage), - patch("mcpgateway.observability._baggage_span_attributes", return_value={"baggage.tenant": "acme"}), - patch("mcpgateway.observability.Status", MagicMock()), - patch("mcpgateway.observability.StatusCode", MagicMock()), - ): - middleware = OpenTelemetryRequestMiddleware(app) - await middleware(scope, receive, send) - - assert ran["value"] is True - mock_context.attach.assert_called_once() - mock_context.detach.assert_called_once_with(token) - - @pytest.mark.asyncio - async def test_middleware_fallback_swallows_attach_and_baggage_failures_and_marks_span_error(self): - """Attach and baggage failures are logged, and an app failure marks the span as an error.""" - scope = self._make_scope() - - async def app(_scope, _receive, _send): - raise RuntimeError("app boom") - - async def receive(): - return {"type": "http.request", "body": b"", "more_body": False} - - async def send(_message): - return None - - class ContextRejectingTracer: - """Emulates InstanaTracer < 3.12: rejects the ``context`` kwarg.""" - - def start_as_current_span(self, _name, **kwargs): - if "context" in kwargs: - raise TypeError("start_as_current_span() got an unexpected keyword argument 'context'") - return nullcontext(MagicMock()) - - mock_trace = MagicMock() - mock_trace.get_current_span.return_value.get_span_context.return_value = MagicMock(is_valid=False, is_remote=False) - mock_context = MagicMock() - mock_context.attach.side_effect = RuntimeError("attach boom") - mock_baggage = MagicMock() - mock_baggage.get_all.return_value = {"tenant": "acme"} - - with ( - patch("mcpgateway.observability._TRACER", ContextRejectingTracer()), - patch("mcpgateway.observability.OTEL_AVAILABLE", True), - patch("mcpgateway.observability.trace", mock_trace), - patch("mcpgateway.observability.otel_extract", return_value=MagicMock()), - patch("mcpgateway.observability.otel_inject", side_effect=self._fake_inject), - patch("mcpgateway.observability.otel_context", mock_context), - patch("mcpgateway.observability.otel_baggage", mock_baggage), - patch("mcpgateway.observability._baggage_span_attributes", side_effect=RuntimeError("baggage boom")), - patch("mcpgateway.observability.Status", MagicMock()), - patch("mcpgateway.observability.StatusCode", MagicMock()), - ): - middleware = OpenTelemetryRequestMiddleware(app) - with pytest.raises(RuntimeError, match="app boom"): - await middleware(scope, receive, send) - - mock_context.detach.assert_not_called() - - @pytest.mark.asyncio - async def test_middleware_fallback_detach_failure_is_nonfatal(self): - """A detach failure is logged and the request still completes.""" - scope = self._make_scope() - ran = {} - - async def app(_scope, _receive, _send): - ran["value"] = True - - async def receive(): - return {"type": "http.request", "body": b"", "more_body": False} - - async def send(_message): - return None - - class ContextRejectingTracer: - """Emulates InstanaTracer < 3.12: rejects the ``context`` kwarg.""" - - def start_as_current_span(self, _name, **kwargs): - if "context" in kwargs: - raise TypeError("start_as_current_span() got an unexpected keyword argument 'context'") - return nullcontext(MagicMock()) - - mock_trace = MagicMock() - mock_trace.get_current_span.return_value.get_span_context.return_value = MagicMock(is_valid=False, is_remote=False) - mock_context = MagicMock() - token = object() - mock_context.attach.return_value = token - mock_context.detach.side_effect = RuntimeError("detach boom") - - with ( - patch("mcpgateway.observability._TRACER", ContextRejectingTracer()), - patch("mcpgateway.observability.OTEL_AVAILABLE", True), - patch("mcpgateway.observability.trace", mock_trace), - patch("mcpgateway.observability.otel_extract", return_value=MagicMock()), - patch("mcpgateway.observability.otel_inject", side_effect=self._fake_inject), - patch("mcpgateway.observability.otel_context", mock_context), - patch("mcpgateway.observability.otel_baggage", None), - patch("mcpgateway.observability.Status", MagicMock()), - patch("mcpgateway.observability.StatusCode", MagicMock()), - ): - middleware = OpenTelemetryRequestMiddleware(app) - await middleware(scope, receive, send) - - assert ran["value"] is True - mock_context.detach.assert_called_once_with(token) - - @pytest.mark.asyncio - async def test_fallback_detaches_when_retry_raises(self): - """The fallback detaches the attached parent even when the retry itself raises. - - Regression: a tracer that rejects ``context`` and then fails on the retry must not - leave the inbound parent attached to the task context. The request fails, but the - outer context is restored before the exception propagates. - """ - scope = self._make_scope() - ran = {} - - async def app(_scope, _receive, _send): - ran["value"] = True - - async def receive(): - return {"type": "http.request", "body": b"", "more_body": False} - - async def send(_message): - return None - - class RetryFailingTracer: - """Rejects ``context``, then raises an unrelated error on the retry.""" - - def __init__(self): - self.attempts = [] - - def start_as_current_span(self, _name, **kwargs): - self.attempts.append(sorted(kwargs)) - if "context" in kwargs: - raise TypeError("start_as_current_span() got an unexpected keyword argument 'context'") - raise RuntimeError("retry boom") - - mock_trace = MagicMock() - mock_trace.get_current_span.return_value.get_span_context.return_value = MagicMock(is_valid=False, is_remote=False) - mock_context = MagicMock() - token = object() - mock_context.attach.return_value = token - - tracer = RetryFailingTracer() - with ( - patch("mcpgateway.observability._TRACER", tracer), - patch("mcpgateway.observability.OTEL_AVAILABLE", True), - patch("mcpgateway.observability.trace", mock_trace), - patch("mcpgateway.observability.otel_extract", return_value=MagicMock()), - patch("mcpgateway.observability.otel_context", mock_context), - ): - middleware = OpenTelemetryRequestMiddleware(app) - with pytest.raises(RuntimeError, match="retry boom"): - await middleware(scope, receive, send) - - # The retry was attempted; the app never ran; the attached parent was detached. - assert tracer.attempts == [["context", "kind"], ["kind"]] - assert ran.get("value") is None - mock_context.attach.assert_called_once() - mock_context.detach.assert_called_once_with(token) - - -class TestLegacyTracerContextCompat: - """Tests for the Instana < 3.12 ``context`` keyword compatibility shim.""" - - @staticmethod - def _install(monkeypatch, tracer_cls, *, otel_context=None): - """Install a fake ``instana.tracer`` module and run the shim installer.""" - fake_module = types.ModuleType("instana.tracer") - fake_module.InstanaTracer = tracer_cls - monkeypatch.setitem(sys.modules, "instana.tracer", fake_module) - monkeypatch.setattr(observability, "OTEL_AVAILABLE", True) - if otel_context is not None: - monkeypatch.setattr(observability, "otel_context", otel_context) - observability._install_legacy_tracer_context_compat() - - def test_patches_span_starts_that_reject_context(self, monkeypatch): - """Methods lacking ``context`` are wrapped so the kwarg is accepted and the parent attaches.""" - calls = [] - - class LegacyTracer: - """Emulates InstanaTracer < 3.12: no ``context`` parameter.""" - - def start_span(self, name, span_context=None, kind=None): - calls.append(("start_span", name, span_context, kind)) - return "span" - - @contextmanager - def start_as_current_span(self, name, span_context=None, kind=None): - calls.append(("start_as_current_span", name, span_context, kind)) - yield "span" - - mock_context = MagicMock() - self._install(monkeypatch, LegacyTracer, otel_context=mock_context) - - tracer = LegacyTracer() - assert tracer.start_span("child", context=object(), kind="client") == "span" - with tracer.start_as_current_span("child", context=object(), kind="client") as span: - assert span == "span" - - # The rejected `context` never leaks into `span_context`; other kwargs pass through. - assert calls == [ - ("start_span", "child", None, "client"), - ("start_as_current_span", "child", None, "client"), - ] - # The supplied parent is attached for the span and detached afterwards. - assert mock_context.attach.call_count == 2 - assert mock_context.detach.call_count == 2 - - def test_attach_and_detach_wrap_span_lifetime(self, monkeypatch): - """The parent stays attached for the whole span and detaches when the body raises.""" - events = [] - - class LegacyTracer: - def start_span(self, name, span_context=None): - return "span" - - @contextmanager - def start_as_current_span(self, name, span_context=None): - events.append("span-open") - yield "span" - events.append("span-close") - - mock_context = MagicMock() - mock_context.attach.side_effect = lambda ctx: events.append("attach") or "token" - mock_context.detach.side_effect = lambda token: events.append("detach") - self._install(monkeypatch, LegacyTracer, otel_context=mock_context) - - tracer = LegacyTracer() - with pytest.raises(RuntimeError): - with tracer.start_as_current_span("child", context=object()): - events.append("body") - raise RuntimeError("boom") - - # Attach precedes the span; detach follows it even though the body raised. - assert events == ["attach", "span-open", "body", "detach"] - - def test_no_context_does_not_touch_otel_context(self, monkeypatch): - """Without a supplied parent the wrapper only delegates.""" - - class LegacyTracer: - def start_span(self, name, span_context=None): - return "span" - - @contextmanager - def start_as_current_span(self, name, span_context=None): - yield "span" - - mock_context = MagicMock() - self._install(monkeypatch, LegacyTracer, otel_context=mock_context) - - tracer = LegacyTracer() - assert tracer.start_span("child") == "span" - with tracer.start_as_current_span("child"): - pass - - mock_context.attach.assert_not_called() - mock_context.detach.assert_not_called() - - def test_install_is_idempotent(self, monkeypatch): - """A second install does not double-wrap the patched methods.""" - - class LegacyTracer: - def start_span(self, name, span_context=None): - return "span" - - @contextmanager - def start_as_current_span(self, name, span_context=None): - yield "span" - - self._install(monkeypatch, LegacyTracer, otel_context=MagicMock()) - patched_start_span = LegacyTracer.start_span - patched_start_as_current_span = LegacyTracer.start_as_current_span - - observability._install_legacy_tracer_context_compat() - - assert LegacyTracer.start_span is patched_start_span - assert LegacyTracer.start_as_current_span is patched_start_as_current_span - assert getattr(LegacyTracer.start_span, "__cf_context_compat__", False) is True - assert getattr(LegacyTracer.start_as_current_span, "__cf_context_compat__", False) is True - - def test_leaves_tracer_that_accepts_context_untouched(self, monkeypatch): - """A tracer that already declares ``context`` is not patched.""" - - class ModernTracer: + from opentelemetry.context import Context + + observability.ensure_legacy_instana_context_compat() + tracer = fake_legacy_instana() + with tracer.start_as_current_span("outer") as outer: + span = tracer.start_span("root", context=Context()) + outer_context = outer.get_span_context() + root_context = span.get_span_context() + # Semantic check only: a new trace, not the ambient one (Instana may encode the + # missing parent as None or 0, so the parent_id representation is not asserted). + assert root_context.is_valid + assert root_context.trace_id != outer_context.trace_id + assert span.parent_id != outer_context.span_id + + def test_context_none_uses_current(self, fake_legacy_instana): + """``context=None`` keeps the legacy ambient-parent behavior.""" + observability.ensure_legacy_instana_context_compat() + tracer = fake_legacy_instana() + with tracer.start_as_current_span("outer") as outer: + span = tracer.start_span("inner", context=None) + assert span.parent_id == outer.get_span_context().span_id + + def test_idempotent(self, fake_legacy_instana): + """Repeated initialization wraps each method exactly once.""" + assert observability.ensure_legacy_instana_context_compat() is True + first = (fake_legacy_instana.start_span, fake_legacy_instana.start_as_current_span) + assert observability.ensure_legacy_instana_context_compat() is True + assert (fake_legacy_instana.start_span, fake_legacy_instana.start_as_current_span) == first + assert first[0].__wrapped__.__name__ == "start_span" + assert not hasattr(first[0].__wrapped__, "__wrapped__") + + def test_unrelated_type_error_propagates(self, fake_legacy_instana): + """Unrelated TypeErrors from the tracer are not swallowed or retried.""" + observability.ensure_legacy_instana_context_compat() + with pytest.raises(TypeError): + fake_legacy_instana().start_span("x", context=self._inbound_context(), bogus=True) + + def test_noop_for_modern_instana(self, monkeypatch): + """Instana >= 3.12 (``context`` already supported) is left untouched.""" + # Standard + import sys + import types + + class InstanaTracer: def start_span(self, name, context=None, kind=None): - return "span" + return (name, context) def start_as_current_span(self, name, context=None, kind=None): - return nullcontext("span") - - self._install(monkeypatch, ModernTracer, otel_context=MagicMock()) - original_start_span = ModernTracer.start_span - original_start_as_current_span = ModernTracer.start_as_current_span - - observability._install_legacy_tracer_context_compat() + return (name, context) - assert ModernTracer.start_span is original_start_span - assert ModernTracer.start_as_current_span is original_start_as_current_span + tracer_mod = types.ModuleType("instana.tracer") + tracer_mod.InstanaTracer = InstanaTracer + span_context_mod = types.ModuleType("instana.span_context") + span_context_mod.SpanContext = object + monkeypatch.setitem(sys.modules, "instana.tracer", tracer_mod) + monkeypatch.setitem(sys.modules, "instana.span_context", span_context_mod) + original = (InstanaTracer.__dict__["start_span"], InstanaTracer.__dict__["start_as_current_span"]) - def test_missing_instana_is_nonfatal(self, monkeypatch): - """A missing Instana package leaves the shim a no-op.""" - monkeypatch.setitem(sys.modules, "instana.tracer", None) - monkeypatch.setattr(observability, "OTEL_AVAILABLE", True) + assert observability.ensure_legacy_instana_context_compat() is False + assert (InstanaTracer.__dict__["start_span"], InstanaTracer.__dict__["start_as_current_span"]) == original - observability._install_legacy_tracer_context_compat() + def test_noop_without_instana(self, monkeypatch): + """No Instana loaded (ordinary OTel tracers): no-op and Instana is not imported.""" + # Standard + import sys - def test_skipped_when_otel_unavailable(self, monkeypatch): - """Nothing is patched when OpenTelemetry is not installed.""" - monkeypatch.setattr(observability, "OTEL_AVAILABLE", False) monkeypatch.delitem(sys.modules, "instana.tracer", raising=False) + monkeypatch.delitem(sys.modules, "instana.span_context", raising=False) + assert observability.ensure_legacy_instana_context_compat() is False + assert "instana.tracer" not in sys.modules - observability._install_legacy_tracer_context_compat() + def test_noop_for_sdk_tracer(self): + """Standard OpenTelemetry SDK tracers are never patched.""" + trace_sdk = pytest.importorskip("opentelemetry.sdk.trace") + before = trace_sdk.Tracer.__dict__["start_span"] + observability.ensure_legacy_instana_context_compat() + assert trace_sdk.Tracer.__dict__["start_span"] is before - assert "instana.tracer" not in sys.modules + def test_init_telemetry_applies_adapter_even_when_disabled(self, fake_legacy_instana, monkeypatch): + """Adapter runs process-wide even when ContextForge OTEL is disabled (FastAPI/MCP still trace).""" + monkeypatch.setenv("OTEL_ENABLE_OBSERVABILITY", "false") + get_settings.cache_clear() + try: + assert init_telemetry() is None + finally: + get_settings.cache_clear() + assert getattr(fake_legacy_instana.start_span, observability._LEGACY_INSTANA_COMPAT_MARKER, False) + + def test_init_telemetry_tolerates_adapter_failure(self, monkeypatch): + """Adapter errors are logged, never fatal.""" + monkeypatch.setattr(observability, "ensure_legacy_instana_context_compat", MagicMock(side_effect=RuntimeError("boom"))) + monkeypatch.setenv("OTEL_ENABLE_OBSERVABILITY", "false") + get_settings.cache_clear() + try: + assert init_telemetry() is None + finally: + get_settings.cache_clear() + + def test_uninspectable_method_skipped(self, monkeypatch): + """Methods whose signature cannot be introspected are left alone.""" + # Standard + import sys + import types + + class InstanaTracer: + start_span = print # builtin: no introspectable span_context signature + start_as_current_span = print + + tracer_mod = types.ModuleType("instana.tracer") + tracer_mod.InstanaTracer = InstanaTracer + span_context_mod = types.ModuleType("instana.span_context") + span_context_mod.SpanContext = object + monkeypatch.setitem(sys.modules, "instana.tracer", tracer_mod) + monkeypatch.setitem(sys.modules, "instana.span_context", span_context_mod) + assert observability.ensure_legacy_instana_context_compat() is False + assert InstanaTracer.start_span is print From 75fec16eab75f50ca7b44c3e11c5a38b40ccdd6f Mon Sep 17 00:00:00 2001 From: "NAYANA.R" Date: Wed, 21 Oct 2026 11:42:41 +0530 Subject: [PATCH 09/12] test cases fix Signed-off-by: NAYANA.R Signed-off-by: Madhav Kandukuri --- mcpgateway/observability.py | 1 - 1 file changed, 1 deletion(-) diff --git a/mcpgateway/observability.py b/mcpgateway/observability.py index ef54446e41e..94bb260cae3 100644 --- a/mcpgateway/observability.py +++ b/mcpgateway/observability.py @@ -97,7 +97,6 @@ def create(attrs: Dict[str, Any]) -> Dict[str, Any]: # type: ignore[override] # Provide minimal module shims so tests can patch ConsoleSpanExporter path try: # Standard - import sys import types if ("pytest" in sys.modules) or (os.getenv("MCP_TESTING") == "1"): From ad94dde32978d280eb0ad5747a9839e8054c3771 Mon Sep 17 00:00:00 2001 From: "NAYANA.R" Date: Wed, 21 Oct 2026 21:52:12 +0530 Subject: [PATCH 10/12] fix: import MutableMapping in observability The request middleware annotates the ASGI scope as MutableMapping, but the typing import omitted the name. Importing mcpgateway.observability raised NameError and broke pytest collection, pylint, ruff, and gateway startup. Signed-off-by: NAYANA.R Signed-off-by: Madhav Kandukuri --- mcpgateway/observability.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mcpgateway/observability.py b/mcpgateway/observability.py index 94bb260cae3..d5de217f243 100644 --- a/mcpgateway/observability.py +++ b/mcpgateway/observability.py @@ -17,7 +17,7 @@ import logging import os import sys -from typing import Any, Callable, cast, Dict, Mapping, Optional +from typing import Any, Callable, cast, Dict, Mapping, MutableMapping, Optional from urllib.parse import urlparse # Third-Party - Try to import OpenTelemetry core components - make them truly optional From cd5a060a51b07bf8f4ee5988f5da294eca7d3939 Mon Sep 17 00:00:00 2001 From: "NAYANA.R" Date: Wed, 21 Oct 2026 22:30:06 +0530 Subject: [PATCH 11/12] test: handle OTel SDK Tracer location in no-op check Signed-off-by: NAYANA.R Signed-off-by: Madhav Kandukuri --- tests/unit/mcpgateway/test_observability.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/unit/mcpgateway/test_observability.py b/tests/unit/mcpgateway/test_observability.py index 63eca0fc1d6..98a23573391 100644 --- a/tests/unit/mcpgateway/test_observability.py +++ b/tests/unit/mcpgateway/test_observability.py @@ -1023,9 +1023,16 @@ def test_noop_without_instana(self, monkeypatch): def test_noop_for_sdk_tracer(self): """Standard OpenTelemetry SDK tracers are never patched.""" trace_sdk = pytest.importorskip("opentelemetry.sdk.trace") - before = trace_sdk.Tracer.__dict__["start_span"] + tracer_cls = getattr(trace_sdk, "Tracer", None) + if tracer_cls is None: + tracer_mod = getattr(trace_sdk, "tracer", None) + if tracer_mod is not None: + tracer_cls = getattr(tracer_mod, "Tracer", None) + if tracer_cls is None: + pytest.skip("OpenTelemetry SDK Tracer not exposed in this version") + before = tracer_cls.__dict__["start_span"] observability.ensure_legacy_instana_context_compat() - assert trace_sdk.Tracer.__dict__["start_span"] is before + assert tracer_cls.__dict__["start_span"] is before def test_init_telemetry_applies_adapter_even_when_disabled(self, fake_legacy_instana, monkeypatch): """Adapter runs process-wide even when ContextForge OTEL is disabled (FastAPI/MCP still trace).""" From e4b53d4435b03920c6339b4a287213c54b90f56e Mon Sep 17 00:00:00 2001 From: "NAYANA.R" Date: Tue, 27 Oct 2026 20:47:32 +0530 Subject: [PATCH 12/12] revert to remove out-of-scope CI changes Signed-off-by: NAYANA.R --- .github/workflows/pytest.yml | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index a7c62ec355e..402d83e5300 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -243,13 +243,20 @@ jobs: tests/live_gateway/mcp/test_request_trace_policy.py \ tests/unit/mcpgateway/services/test_a2a_service.py \ tests/live_gateway/a2a/test_a2a_trace_context_e2e.py \ - -v --tb=short + --basetemp="${RUNNER_TEMP}/observability-tests" -v --tb=short + uv run --extra plugins --extra observability pytest \ + tests/unit/mcpgateway/services/test_gateway_service.py \ + -k health_completion_metrics -v --tb=short - - name: 🧪 Legacy Instana (< 3.12) live-gateway regression - run: | - LEGACY_INSTANA_E2E=1 uv run --extra observability --with "instana==3.11.1" pytest \ - tests/live_gateway/mcp/test_legacy_instana_e2e.py \ - -v --tb=short + - name: Preserve observability failure artifacts + if: failure() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: observability-failure-artifacts + path: | + ${{ runner.temp }}/observability-tests/**/*.log + ${{ runner.temp }}/observability-tests/**/collector.jsonl + retention-days: 7 # ----------------------------------------------------------- # 4️⃣ Upload coverage artifacts (XML + HTML)