diff --git a/benchmarks/README.md b/benchmarks/README.md index 5938ac4..8d52800 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -251,6 +251,28 @@ OpenTelemetry rows record the `receive` and `send` spans, which 2.0 leaves out b 4a). Its `sentry_nosess` row now has the same config as `sentry`, and `sentry_noauto` no longer stops request capture, because lean capture needs no auto-enabling integration. +Since 2.0, the Prometheus instrument leaves its HTTP middleware out while OpenTelemetry records HTTP +metrics. Rerun in-process on the M2 host (macOS, CPython 3.14.7, opentelemetry-sdk 1.45.1, +instrumentation-asgi and -fastapi 0.66b1, prometheus-client 0.26.0, +prometheus-fastapi-instrumentator 8.1.0, fastapi 0.162.0, litestar 2.25.0), with +`bench.py bare,otel_0,otel_0_metrics,otel_0_prom,otel_0_metrics_prom_both,otel_0_metrics_prom --repeat 3`, +in µs over each suite's bare app: + +| scenario | FastAPI | Litestar | +|---|---:|---:| +| `otel_0`: OTel at 0% | +47.4 | +38.8 | +| `otel_0_metrics`: + OTel HTTP metrics | +83.3 | +72.8 | +| `otel_0_prom`: + Prometheus | +71.8 | +56.4 | +| `otel_0_metrics_prom_both`: both, `prometheus_http_metrics_enabled=True` (1.x) | +106.4 | +90.2 | +| `otel_0_metrics_prom`: both configured, the 2.0 default | +84.5 | +73.4 | + +Every row's rounds were within 2%. Configuring both now costs what OTel HTTP metrics cost alone, +21.9 µs less on FastAPI and 16.8 µs less on Litestar. On this host the OTel HTTP metrics cost 34 to +36 µs and Prometheus's 18 to 25 µs, the reverse of the Linux table above, where OTel's were the +cheaper set at about 12 µs. The stack suite (`run.py stack async otel,otel_metrics,otel_metrics_prom_both,otel_metrics_prom`, +100% sampling) agrees: +44.4 µs for OTel HTTP metrics, +71.8 for both sets and +48.1 for the 2.0 +default, over `otel` at 109.0 µs per request. + FastAPI real-server runs under uvicorn in the container were unstable (bare repeat medians of 8235 and 6341 RPS, over 100% spread on the full stack), so the Linux real-server numbers below are Litestar on Granian only. @@ -657,6 +679,9 @@ cd benchmarks # the two OpenTelemetry knobs (section 4a) ../.venv/bin/python run.py stack async otel,otel_exclude_spans,otel_sampler,otel_tuned +# HTTP metrics recorded once against both sets (section 4d) +../.venv/bin/python run.py stack async otel,otel_metrics,otel_metrics_prom_both,otel_metrics_prom + # the Sentry ablation, where every familiar knob turns out to be a no-op (section 4b) ../.venv/bin/python run.py sentry async off,errors_only,errors_only_lean,errors_only_no_integrations,errors_only_no_txn diff --git a/benchmarks/frameworks/fastapi_suite.py b/benchmarks/frameworks/fastapi_suite.py index 9d97cea..05597d3 100644 --- a/benchmarks/frameworks/fastapi_suite.py +++ b/benchmarks/frameworks/fastapi_suite.py @@ -46,6 +46,9 @@ "otel_0_metrics": Scenario(config=base_config(**OTEL, **OTEL_METRICS, **ratio(0.0))), "otel_0_prom": Scenario(config=base_config(**OTEL, **ratio(0.0), **PROM)), "otel_0_metrics_prom": Scenario(config=base_config(**OTEL, **OTEL_METRICS, **ratio(0.0), **PROM)), + "otel_0_metrics_prom_both": Scenario( + config=base_config(**OTEL, **OTEL_METRICS, **ratio(0.0), **PROM, prometheus_http_metrics_enabled=True) + ), "sentry": Scenario(config=base_config(**sentry())), "sentry_nosess": Scenario(config=base_config(**sentry(), sentry_auto_session_tracking=False)), "sentry_noauto": Scenario(config=base_config(**sentry(auto_enabling_integrations=False))), diff --git a/benchmarks/frameworks/litestar_suite.py b/benchmarks/frameworks/litestar_suite.py index 527c6f8..365f1c7 100644 --- a/benchmarks/frameworks/litestar_suite.py +++ b/benchmarks/frameworks/litestar_suite.py @@ -61,6 +61,9 @@ "otel_0_metrics": Scenario(config=base_config(**OTEL, **OTEL_METRICS, **ratio(0.0))), "otel_0_prom": Scenario(config=base_config(**OTEL, **ratio(0.0), **PROM)), "otel_0_metrics_prom": Scenario(config=base_config(**OTEL, **OTEL_METRICS, **ratio(0.0), **PROM)), + "otel_0_metrics_prom_both": Scenario( + config=base_config(**OTEL, **OTEL_METRICS, **ratio(0.0), **PROM, prometheus_http_metrics_enabled=True) + ), "sentry": Scenario(config=base_config(**sentry())), "sentry_nosess": Scenario(config=base_config(**sentry(), sentry_auto_session_tracking=False)), "sentry_noauto": Scenario(config=base_config(**sentry(auto_enabling_integrations=False))), diff --git a/benchmarks/stack_scenarios.py b/benchmarks/stack_scenarios.py index e41ff6f..791b8ae 100644 --- a/benchmarks/stack_scenarios.py +++ b/benchmarks/stack_scenarios.py @@ -23,6 +23,7 @@ OTLP_PORT: typing.Final = int(os.environ.get("STUB_OTLP_PORT", "8139")) OTLP_ENDPOINT: typing.Final = f"http://127.0.0.1:{OTLP_PORT}/v1/traces" +OTLP_METRICS_ENDPOINT: typing.Final = f"http://127.0.0.1:{OTLP_PORT}/v1/metrics" SAMPLE_RATIO: typing.Final = 0.01 APPS: typing.Final = ("async", "structlog") @@ -41,7 +42,9 @@ } _OTEL_EXCLUDE_SPANS: typing.Final = {"opentelemetry_exclude_spans": ["receive", "send"]} _OTEL_SAMPLER: typing.Final = {"opentelemetry_sampler": ParentBased(TraceIdRatioBased(SAMPLE_RATIO))} +_OTEL_METRICS: typing.Final = {"opentelemetry_metrics_endpoint": OTLP_METRICS_ENDPOINT} _PROMETHEUS: typing.Final = {"prometheus_metrics_path": "/metrics"} +_PROMETHEUS_HTTP_METRICS: typing.Final = {"prometheus_http_metrics_enabled": True} _LOGGING: typing.Final = {"logging_enabled": True} _SENTRY: typing.Final = { "sentry_dsn": DSN, @@ -86,6 +89,9 @@ def config(*parts: dict[str, typing.Any]) -> dict[str, typing.Any]: "full_no_sentry": lambda: config(_OTEL, _PROMETHEUS, _LOGGING), "full_sentry_tuned": lambda: config(_OTEL, _PROMETHEUS, _LOGGING, _sentry_tuned()), "otel_exclude_spans": lambda: config(_OTEL, _OTEL_EXCLUDE_SPANS), + "otel_metrics": lambda: config(_OTEL, _OTEL_METRICS), + "otel_metrics_prom": lambda: config(_OTEL, _OTEL_METRICS, _PROMETHEUS), + "otel_metrics_prom_both": lambda: config(_OTEL, _OTEL_METRICS, _PROMETHEUS, _PROMETHEUS_HTTP_METRICS), "otel_sampler": lambda: config(_OTEL, _OTEL_SAMPLER), "otel_tuned": lambda: config(_OTEL, _OTEL_EXCLUDE_SPANS, _OTEL_SAMPLER), "full_all_tuned": lambda: config(_OTEL, _OTEL_EXCLUDE_SPANS, _OTEL_SAMPLER, _PROMETHEUS, _LOGGING, _sentry_tuned()), diff --git a/docs/introduction/configuration.md b/docs/introduction/configuration.md index 845822c..3f0910c 100644 --- a/docs/introduction/configuration.md +++ b/docs/introduction/configuration.md @@ -116,6 +116,22 @@ Prometheus is on by default when its extra is installed, serving metrics at `pro Additional parameters: - `prometheus_metrics_include_in_schema`. +- `prometheus_http_metrics_enabled` - FastAPI, Litestar and FastMCP only; see below. + +### Prometheus HTTP metrics + +On FastAPI, Litestar and FastMCP, the Prometheus instrument records every HTTP request, and so does +OpenTelemetry's HTTP instrumentation once `opentelemetry_metrics_endpoint` installs a +`MeterProvider`. Since 2.0 only one of them does. With `prometheus_http_metrics_enabled` left at +`None`, the Prometheus instrument leaves its HTTP middleware out whenever OpenTelemetry records HTTP +metrics: the endpoint is set, and the OpenTelemetry packages and the OTLP metric exporter for +`opentelemetry_exporter_protocol` are installed. The metrics route is still served, for your own +collectors and, on FastMCP, the tool-call metrics. Configuring both then costs 17 to 22 µs less per +request than it did in 1.x. + +- `True` keeps the Prometheus HTTP metrics as well, for a service moving dashboards from one + backend to the other. +- `False` leaves them out even without OpenTelemetry metrics. ### Prometheus Litestar @@ -163,7 +179,8 @@ packages, and serves `prometheus_client.REGISTRY` at `prometheus_metrics_path`. Every ASGI application `application.http_app()` builds counts its requests in `http_requests_total`, `http_request_duration_seconds` and the instrumentator's other default metrics, labelled by route (`handler="/mcp"`); requests to unknown paths are grouped into `handler="none"`, and the metrics -route itself is not counted. Tool calls are counted in `fastmcp_tool_calls_total{tool, status}`, with +route itself is not counted. Those HTTP metrics are left out while OpenTelemetry records its own +([Prometheus HTTP metrics](#prometheus-http-metrics)). Tool calls are counted in `fastmcp_tool_calls_total{tool, status}`, with `status` either `success` or `error`, and timed in `fastmcp_tool_call_duration_seconds{tool}`. Additional parameters for FastMCP integration: @@ -206,10 +223,11 @@ nowhere. The export interval is the SDK's, 60 seconds, overridable with `OTEL_METRIC_EXPORT_INTERVAL`. -If the Prometheus instrument is also configured, request metrics are recorded twice: once for the -scrape endpoint and once for the OTLP pipeline, under different names. That is supported, and it -is what you want when the two feed different backends, but pointing both at one backend -double-counts. +On FastAPI, Litestar and FastMCP, HTTP request metrics are recorded once. Since 2.0, when this +endpoint is set, the Prometheus instrument stops adding its HTTP middleware and OpenTelemetry's set +is the only one; see [Prometheus HTTP metrics](#prometheus-http-metrics). FastStream is the +exception: both of its broker middlewares are opt-in, so with both configured each message is +counted by both. ```python config = FastAPIConfig( diff --git a/docs/introduction/performance.md b/docs/introduction/performance.md index cc4c622..26969f7 100644 --- a/docs/introduction/performance.md +++ b/docs/introduction/performance.md @@ -68,7 +68,7 @@ request, every request then pays the per-record cost measured here: --8<-- "benchmarks/README.md:logging" -lite-bootstrap already applies six savings for you. It turns Sentry Logs off in the logging +lite-bootstrap already applies seven savings for you. It turns Sentry Logs off in the logging integration, because it never enables them and on 2.x the handler formats every record before checking whether they are enabled; that one costs nothing. Since 2.0 it also leaves release-health sessions off and starts breadcrumbs at `WARNING`, so an `INFO` record skips the breadcrumb handler. @@ -80,7 +80,9 @@ sentry-sdk's [auto-enabling integrations](configuration.md#sentry-auto-enabling- which costs you their SQL, Redis and outgoing-HTTP breadcrumbs; set `sentry_auto_enabling_integrations=True` to keep them. On FastAPI, Litestar and FastMCP it also records no span for the ASGI `receive` and `send` events, which costs you those spans; set -`opentelemetry_exclude_spans=[]` to get them back. The numbers on this page were measured before +`opentelemetry_exclude_spans=[]` to get them back. With `opentelemetry_metrics_endpoint` set it +records HTTP metrics only through OpenTelemetry, which costs you the Prometheus HTTP metrics; set +`prometheus_http_metrics_enabled=True` to keep both. The numbers on this page were measured before 2.0, against the old defaults. ## What each saving costs you diff --git a/lite_bootstrap/bootstrappers/fastapi_bootstrapper.py b/lite_bootstrap/bootstrappers/fastapi_bootstrapper.py index cdc1569..f50d1e0 100644 --- a/lite_bootstrap/bootstrappers/fastapi_bootstrapper.py +++ b/lite_bootstrap/bootstrappers/fastapi_bootstrapper.py @@ -17,7 +17,7 @@ ) from lite_bootstrap.instruments.logging_instrument import LoggingConfig, LoggingInstrument from lite_bootstrap.instruments.opentelemetry_instrument import OpenTelemetryAsgiConfig, OpenTelemetryInstrument -from lite_bootstrap.instruments.prometheus_instrument import PrometheusConfig, PrometheusInstrument +from lite_bootstrap.instruments.prometheus_instrument import PrometheusHttpConfig, PrometheusInstrument from lite_bootstrap.instruments.pyroscope_instrument import PyroscopeConfig, PyroscopeInstrument from lite_bootstrap.instruments.sentry_instrument import ( SentryInstrument, @@ -65,7 +65,7 @@ class FastAPIConfig( HealthChecksConfig, LoggingConfig, OpenTelemetryAsgiConfig, - PrometheusConfig, + PrometheusHttpConfig, PyroscopeConfig, SentryLeanCaptureConfig, SwaggerConfig, @@ -248,10 +248,12 @@ def dependencies_installed() -> bool: def bootstrap(self) -> None: config = self.bootstrap_config application = config.app - Instrumentator(**config.prometheus_instrumentator_additional_params).instrument( - application, - **config.prometheus_instrument_additional_params, - ).expose( + instrumentator = Instrumentator(**config.prometheus_instrumentator_additional_params) + if config.is_prometheus_http_metrics_on( + opentelemetry_records_http_metrics=FastAPIOpenTelemetryInstrument.is_recording_metrics(config) + ): + instrumentator.instrument(application, **config.prometheus_instrument_additional_params) + instrumentator.expose( application, endpoint=config.prometheus_metrics_path, include_in_schema=config.prometheus_metrics_include_in_schema, diff --git a/lite_bootstrap/bootstrappers/fastmcp_bootstrapper.py b/lite_bootstrap/bootstrappers/fastmcp_bootstrapper.py index 4b3205c..5e168b5 100644 --- a/lite_bootstrap/bootstrappers/fastmcp_bootstrapper.py +++ b/lite_bootstrap/bootstrappers/fastmcp_bootstrapper.py @@ -12,7 +12,7 @@ from lite_bootstrap.instruments.healthchecks_instrument import HealthChecksConfig, HealthChecksInstrument from lite_bootstrap.instruments.logging_instrument import LoggingConfig, LoggingInstrument from lite_bootstrap.instruments.opentelemetry_instrument import OpenTelemetryAsgiConfig, OpenTelemetryInstrument -from lite_bootstrap.instruments.prometheus_instrument import PrometheusConfig, PrometheusInstrument +from lite_bootstrap.instruments.prometheus_instrument import PrometheusHttpConfig, PrometheusInstrument from lite_bootstrap.instruments.pyroscope_instrument import PyroscopeConfig, PyroscopeInstrument from lite_bootstrap.instruments.sentry_instrument import ( SentryInstrument, @@ -204,7 +204,7 @@ class FastMcpConfig( HealthChecksConfig, LoggingConfig, OpenTelemetryAsgiConfig, - PrometheusConfig, + PrometheusHttpConfig, PyroscopeConfig, SentryLeanCaptureConfig, ): @@ -328,7 +328,10 @@ def _instrument_http_app(self, http_application: "StarletteWithLifespan") -> "St def bootstrap(self) -> None: config = self.bootstrap_config - self._restore_http_app = _postprocess_http_apps(config.application, self._instrument_http_app) + if config.is_prometheus_http_metrics_on( + opentelemetry_records_http_metrics=FastMcpOpenTelemetryInstrument.is_recording_metrics(config) + ): + self._restore_http_app = _postprocess_http_apps(config.application, self._instrument_http_app) if config.fastmcp_prometheus_tool_metrics_enabled: config.application.add_middleware(FastMcpPrometheusMiddleware(*_build_tool_call_metrics())) diff --git a/lite_bootstrap/bootstrappers/litestar_bootstrapper.py b/lite_bootstrap/bootstrappers/litestar_bootstrapper.py index 7d8b7f6..eb1e26f 100644 --- a/lite_bootstrap/bootstrappers/litestar_bootstrapper.py +++ b/lite_bootstrap/bootstrappers/litestar_bootstrapper.py @@ -16,12 +16,7 @@ ) from lite_bootstrap.instruments.logging_instrument import LoggingConfig, LoggingInstrument from lite_bootstrap.instruments.opentelemetry_instrument import OpenTelemetryAsgiConfig, OpenTelemetryInstrument -from lite_bootstrap.instruments.prometheus_instrument import ( - PrometheusConfig as PrometheusBootstrapperConfig, -) -from lite_bootstrap.instruments.prometheus_instrument import ( - PrometheusInstrument, -) +from lite_bootstrap.instruments.prometheus_instrument import PrometheusHttpConfig, PrometheusInstrument from lite_bootstrap.instruments.pyroscope_instrument import PyroscopeConfig, PyroscopeInstrument from lite_bootstrap.instruments.sentry_instrument import ( SENTRY_FAILED_REQUEST_STATUS_CODES, @@ -255,7 +250,7 @@ class LitestarConfig( HealthChecksConfig, LoggingConfig, OpenTelemetryAsgiConfig, - PrometheusBootstrapperConfig, + PrometheusHttpConfig, PyroscopeConfig, SentryLeanCaptureConfig, SwaggerConfig, @@ -432,7 +427,10 @@ class LitestarPrometheusController(PrometheusController): litestar_prometheus_config = PrometheusConfig(app_name=config.service_name, **prometheus_params) config.application_config.route_handlers.append(LitestarPrometheusController) - config.application_config.middleware.append(litestar_prometheus_config.middleware) + if config.is_prometheus_http_metrics_on( + opentelemetry_records_http_metrics=LitestarOpenTelemetryInstrument.is_recording_metrics(config) + ): + config.application_config.middleware.append(litestar_prometheus_config.middleware) @dataclasses.dataclass(kw_only=True, slots=True) diff --git a/lite_bootstrap/instruments/opentelemetry_instrument.py b/lite_bootstrap/instruments/opentelemetry_instrument.py index d1cbd9f..ca8bb5f 100644 --- a/lite_bootstrap/instruments/opentelemetry_instrument.py +++ b/lite_bootstrap/instruments/opentelemetry_instrument.py @@ -194,6 +194,15 @@ def is_configured(cls, bootstrap_config: "OpenTelemetryConfig") -> bool: or bootstrap_config.opentelemetry_log_traces ) + @classmethod + def is_recording_metrics(cls, bootstrap_config: "OpenTelemetryConfig") -> bool: + """Whether bootstrap() installs a MeterProvider, which the framework's HTTP instrumentation records into.""" + if not bootstrap_config.opentelemetry_metrics_endpoint or not cls.dependencies_installed(): + return False + if bootstrap_config.opentelemetry_exporter_protocol == "grpc": + return import_checker.is_otlp_grpc_exporter_installed + return import_checker.is_otlp_http_exporter_installed + @staticmethod def dependencies_installed() -> bool: # The instrument imports from both the api (opentelemetry.trace/.context) and diff --git a/lite_bootstrap/instruments/prometheus_instrument.py b/lite_bootstrap/instruments/prometheus_instrument.py index def18bb..cefa993 100644 --- a/lite_bootstrap/instruments/prometheus_instrument.py +++ b/lite_bootstrap/instruments/prometheus_instrument.py @@ -13,6 +13,17 @@ class PrometheusConfig(BaseConfig): prometheus_metrics_include_in_schema: bool = False +@dataclasses.dataclass(kw_only=True, frozen=True) +class PrometheusHttpConfig(PrometheusConfig): + prometheus_http_metrics_enabled: bool | None = None + + def is_prometheus_http_metrics_on(self, *, opentelemetry_records_http_metrics: bool) -> bool: + """Left None, Prometheus records HTTP metrics unless OpenTelemetry already records them.""" + if self.prometheus_http_metrics_enabled is not None: + return self.prometheus_http_metrics_enabled + return not opentelemetry_records_http_metrics + + PrometheusConfigT = typing.TypeVar("PrometheusConfigT", bound=PrometheusConfig) diff --git a/tests/conftest.py b/tests/conftest.py index 11dcc4b..1c9394c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,9 +8,11 @@ import opentelemetry.metrics._internal import opentelemetry.trace +import prometheus_client import pytest import sentry_sdk from opentelemetry.instrumentation.instrumentor import BaseInstrumentor +from opentelemetry.sdk.metrics.export import InMemoryMetricReader from opentelemetry.util._once import Once from sentry_sdk import integrations as sentry_integrations from sentry_sdk import scope as sentry_scope @@ -23,7 +25,7 @@ from lite_bootstrap import import_checker from lite_bootstrap.exceptions import InstrumentSkippedWarning -from lite_bootstrap.instruments import healthchecks_instrument +from lite_bootstrap.instruments import healthchecks_instrument, opentelemetry_instrument def pytest_configure() -> None: @@ -235,3 +237,35 @@ def _wipe_and_reload() -> None: for key in [k for k in mod_dict if _is_module_state(k)]: del mod_dict[key] mod_dict.update({k: v for k, v in snap.items() if _is_module_state(k)}) + + +@contextlib.contextmanager +def in_memory_opentelemetry_metric_reader() -> Generator[InMemoryMetricReader]: + """Hand the MeterProvider that opentelemetry_metrics_endpoint installs an in-memory reader instead of OTLP.""" + reader: typing.Final = InMemoryMetricReader() + with patch.object(opentelemetry_instrument, "PeriodicExportingMetricReader", return_value=reader): + yield reader + + +def opentelemetry_recorded_http_server_metrics(reader: InMemoryMetricReader) -> bool: + metrics_data: typing.Final = reader.get_metrics_data() + if metrics_data is None: + return False + return any( + metric.name.startswith("http.server.") + for resource_metrics in metrics_data.resource_metrics + for scope_metrics in resource_metrics.scope_metrics + for metric in scope_metrics.metrics + ) + + +def prometheus_recorded_path(path: str) -> bool: + return f'"{path}"' in prometheus_client.generate_latest().decode() + + +def unregister_prometheus_request_metrics() -> None: + """Unregister the instrumentator's request metrics: a second registration is skipped, not reused.""" + registered_collectors: typing.Final = dict(prometheus_client.REGISTRY._collector_to_names) # noqa: SLF001 + for collector, names in registered_collectors.items(): + if any(name.startswith(("http_request", "http_response")) for name in names): + prometheus_client.REGISTRY.unregister(collector) diff --git a/tests/test_fastapi_bootstrap.py b/tests/test_fastapi_bootstrap.py index 003ec04..da47763 100644 --- a/tests/test_fastapi_bootstrap.py +++ b/tests/test_fastapi_bootstrap.py @@ -33,6 +33,10 @@ emulate_package_missing, emulate_package_missing_with_module_reload, failed_probe_names, + in_memory_opentelemetry_metric_reader, + opentelemetry_recorded_http_server_metrics, + prometheus_recorded_path, + unregister_prometheus_request_metrics, warning_source_files, ) @@ -478,6 +482,84 @@ def test_fastapi_otel_hands_the_instrumentor_the_meter_provider(fastapi_config: assert isinstance(get_meter_provider(), SDKMeterProvider) +def _http_metric_sets_one_request_moves(config: FastAPIConfig) -> tuple[bool, bool]: + """Return whether one request reached OpenTelemetry's HTTP metrics and Prometheus's, in that order.""" + path: typing.Final = f"/items-{uuid.uuid4().hex}" + application: typing.Final = config.app + unregister_prometheus_request_metrics() + + @application.get(path) + async def items() -> str: + return "ok" + + with in_memory_opentelemetry_metric_reader() as reader: + bootstrapper = FastAPIBootstrapper(bootstrap_config=config) + bootstrapper.bootstrap() + try: + with TestClient(application) as client: + assert client.get(path).status_code == status.HTTP_200_OK + return opentelemetry_recorded_http_server_metrics(reader), prometheus_recorded_path(path) + finally: + bootstrapper.teardown() + + +@pytest.mark.parametrize( + ("overrides", "expected"), + [ + pytest.param({"opentelemetry_metrics_endpoint": "localhost:4317"}, (True, False), id="both-keeps-otel"), + pytest.param( + { + "opentelemetry_metrics_endpoint": "http://localhost:4318/v1/metrics", + "opentelemetry_exporter_protocol": "http", + }, + (True, False), + id="both-keeps-otel-over-http", + ), + pytest.param( + {"opentelemetry_metrics_endpoint": "localhost:4317", "prometheus_http_metrics_enabled": True}, + (True, True), + id="both-restored", + ), + pytest.param({}, (False, True), id="prometheus-alone"), + pytest.param( + {"opentelemetry_metrics_endpoint": "localhost:4317", "prometheus_metrics_path": ""}, + (True, False), + id="otel-alone", + ), + pytest.param({"prometheus_http_metrics_enabled": False}, (False, False), id="prometheus-http-off"), + ], +) +def test_fastapi_http_metrics_are_recorded_once( + fastapi_config: FastAPIConfig, overrides: dict[str, typing.Any], expected: tuple[bool, bool] +) -> None: + assert _http_metric_sets_one_request_moves(dataclasses.replace(fastapi_config, **overrides)) == expected + + +def test_fastapi_prometheus_keeps_http_metrics_when_the_otlp_metric_exporter_is_missing( + fastapi_config: FastAPIConfig, +) -> None: + config = dataclasses.replace(fastapi_config, opentelemetry_metrics_endpoint="localhost:4317") + with ( + patch.object(import_checker, "is_otlp_grpc_exporter_installed", new=False), + pytest.warns(InstrumentDependencyMissingWarning, match="gRPC OTLP exporter is not installed"), + ): + assert _http_metric_sets_one_request_moves(config) == (False, True) + + +def test_fastapi_prometheus_still_serves_metrics_while_opentelemetry_records_http_metrics( + fastapi_config: FastAPIConfig, +) -> None: + config = dataclasses.replace(fastapi_config, opentelemetry_metrics_endpoint="localhost:4317") + with in_memory_opentelemetry_metric_reader(): + bootstrapper = FastAPIBootstrapper(bootstrap_config=config) + application = bootstrapper.bootstrap() + try: + with TestClient(application) as client: + assert client.get(config.prometheus_metrics_path).status_code == status.HTTP_200_OK + finally: + bootstrapper.teardown() + + def _server_span_names_for(config: FastAPIConfig, paths: typing.Iterable[str]) -> list[str]: application = config.app diff --git a/tests/test_fastmcp_bootstrap.py b/tests/test_fastmcp_bootstrap.py index 089469d..bf43e5e 100644 --- a/tests/test_fastmcp_bootstrap.py +++ b/tests/test_fastmcp_bootstrap.py @@ -37,6 +37,10 @@ emulate_package_missing, emulate_package_missing_with_module_reload, failed_probe_names, + in_memory_opentelemetry_metric_reader, + opentelemetry_recorded_http_server_metrics, + prometheus_recorded_path, + unregister_prometheus_request_metrics, ) @@ -514,11 +518,56 @@ def test_fastmcp_otel_is_skipped_without_asgi_instrumentation() -> None: @pytest.fixture(autouse=True) def _unregister_request_metrics() -> None: - """Give each test unregistered request metrics: a second registration is skipped, not reused.""" - registered_collectors = dict(prometheus_client.REGISTRY._collector_to_names) # noqa: SLF001 - for collector, names in registered_collectors.items(): - if any(name.startswith(("http_request", "http_response")) for name in names): - prometheus_client.REGISTRY.unregister(collector) + unregister_prometheus_request_metrics() + + +def _http_metric_sets_one_request_moves(**overrides: typing.Any) -> tuple[bool, bool]: # noqa: ANN401 + """Return whether one request reached OpenTelemetry's HTTP metrics and Prometheus's, in that order.""" + config: typing.Final = _make_test_config(**overrides) + with in_memory_opentelemetry_metric_reader() as reader: + bootstrapper = FastMcpBootstrapper(bootstrap_config=config) + application = bootstrapper.bootstrap() + try: + with TestClient(application.http_app()) as client: + assert client.get(config.health_checks_path).status_code == status.HTTP_200_OK + return opentelemetry_recorded_http_server_metrics(reader), prometheus_recorded_path(config.health_checks_path) + finally: + bootstrapper.teardown() + + +@pytest.mark.parametrize( + ("overrides", "expected"), + [ + pytest.param({"opentelemetry_metrics_endpoint": "localhost:4317"}, (True, False), id="both-keeps-otel"), + pytest.param( + {"opentelemetry_metrics_endpoint": "localhost:4317", "prometheus_http_metrics_enabled": True}, + (True, True), + id="both-restored", + ), + pytest.param({}, (False, True), id="prometheus-alone"), + pytest.param( + {"opentelemetry_metrics_endpoint": "localhost:4317", "prometheus_metrics_path": ""}, + (True, False), + id="otel-alone", + ), + pytest.param({"prometheus_http_metrics_enabled": False}, (False, False), id="prometheus-http-off"), + ], +) +def test_fastmcp_http_metrics_are_recorded_once(overrides: dict[str, typing.Any], expected: tuple[bool, bool]) -> None: + assert _http_metric_sets_one_request_moves(**overrides) == expected + + +def test_fastmcp_prometheus_keeps_tool_metrics_and_metrics_route_while_opentelemetry_records_http_metrics() -> None: + config: typing.Final = _make_test_config(opentelemetry_metrics_endpoint="localhost:4317") + with in_memory_opentelemetry_metric_reader(): + bootstrapper = FastMcpBootstrapper(bootstrap_config=config) + application = bootstrapper.bootstrap() + try: + assert any(isinstance(middleware, FastMcpPrometheusMiddleware) for middleware in application.middleware) + with TestClient(application.http_app()) as client: + assert client.get(config.prometheus_metrics_path).status_code == status.HTTP_200_OK + finally: + bootstrapper.teardown() def _sample_value(name: str, labels: dict[str, str]) -> float: diff --git a/tests/test_litestar_bootstrap.py b/tests/test_litestar_bootstrap.py index ef3ecad..9e3d7e8 100644 --- a/tests/test_litestar_bootstrap.py +++ b/tests/test_litestar_bootstrap.py @@ -6,6 +6,7 @@ import logging.handlers import sys import typing +import uuid import warnings from collections.abc import Generator @@ -45,6 +46,9 @@ emulate_package_missing, emulate_package_missing_with_module_reload, failed_probe_names, + in_memory_opentelemetry_metric_reader, + opentelemetry_recorded_http_server_metrics, + prometheus_recorded_path, ) @@ -614,6 +618,63 @@ def _server_spans(exporter: InMemorySpanExporter) -> list[ReadableSpan]: return [span for span in exporter.get_finished_spans() if span.kind == SpanKind.SERVER] +def _http_metric_sets_one_request_moves(config: LitestarConfig) -> tuple[bool, bool]: + """Return whether one request reached OpenTelemetry's HTTP metrics and Prometheus's, in that order.""" + path: typing.Final = f"/items-{uuid.uuid4().hex}" + + async def get_items() -> list[int]: + return [1] + + config = dataclasses.replace(config, application_config=AppConfig(route_handlers=[litestar.get(path)(get_items)])) + with in_memory_opentelemetry_metric_reader() as reader: + bootstrapper = LitestarBootstrapper(bootstrap_config=config) + application = bootstrapper.bootstrap() + try: + with TestClient(app=application) as client: + assert client.get(path).status_code == status_codes.HTTP_200_OK + return opentelemetry_recorded_http_server_metrics(reader), prometheus_recorded_path(path) + finally: + bootstrapper.teardown() + + +@pytest.mark.parametrize( + ("overrides", "expected"), + [ + pytest.param({"opentelemetry_metrics_endpoint": "localhost:4317"}, (True, False), id="both-keeps-otel"), + pytest.param( + {"opentelemetry_metrics_endpoint": "localhost:4317", "prometheus_http_metrics_enabled": True}, + (True, True), + id="both-restored", + ), + pytest.param({}, (False, True), id="prometheus-alone"), + pytest.param( + {"opentelemetry_metrics_endpoint": "localhost:4317", "prometheus_metrics_path": ""}, + (True, False), + id="otel-alone", + ), + pytest.param({"prometheus_http_metrics_enabled": False}, (False, False), id="prometheus-http-off"), + ], +) +def test_litestar_http_metrics_are_recorded_once( + litestar_config: LitestarConfig, overrides: dict[str, typing.Any], expected: tuple[bool, bool] +) -> None: + assert _http_metric_sets_one_request_moves(dataclasses.replace(litestar_config, **overrides)) == expected + + +def test_litestar_prometheus_still_serves_metrics_while_opentelemetry_records_http_metrics( + litestar_config: LitestarConfig, +) -> None: + config = dataclasses.replace(litestar_config, opentelemetry_metrics_endpoint="localhost:4317") + with in_memory_opentelemetry_metric_reader(): + bootstrapper = LitestarBootstrapper(bootstrap_config=config) + application = bootstrapper.bootstrap() + try: + with TestClient(app=application) as client: + assert client.get(config.prometheus_metrics_path).status_code == status_codes.HTTP_200_OK + finally: + bootstrapper.teardown() + + def test_litestar_otel_registers_litestars_own_plugin(litestar_config: LitestarConfig) -> None: config = dataclasses.replace(litestar_config, opentelemetry_metrics_endpoint="localhost:4317") bootstrapper = LitestarBootstrapper(bootstrap_config=config)