Record HTTP metrics once when both OpenTelemetry metrics and Prometheus are configured (#365)

Author: lesnik512

benchmarks/README.md

@@ -251,6 +251,28 @@ OpenTelemetry rows record the `receive` and `send` spans, which 2.0 leaves out b
 4a). Its `sentry_nosess` row now has the same config as `sentry`, and `sentry_noauto` no longer stops request
 capture, because lean capture needs no auto-enabling integration.
 
+Since 2.0, the Prometheus instrument leaves its HTTP middleware out while OpenTelemetry records HTTP
+metrics. Rerun in-process on the M2 host (macOS, CPython 3.14.7, opentelemetry-sdk 1.45.1,
+instrumentation-asgi and -fastapi 0.66b1, prometheus-client 0.26.0,
+prometheus-fastapi-instrumentator 8.1.0, fastapi 0.162.0, litestar 2.25.0), with
+`bench.py <suite> bare,otel_0,otel_0_metrics,otel_0_prom,otel_0_metrics_prom_both,otel_0_metrics_prom --repeat 3`,
+in µs over each suite's bare app:
+
+| scenario | FastAPI | Litestar |
+|---|---:|---:|
+| `otel_0`: OTel at 0% | +47.4 | +38.8 |
+| `otel_0_metrics`: + OTel HTTP metrics | +83.3 | +72.8 |
+| `otel_0_prom`: + Prometheus | +71.8 | +56.4 |
+| `otel_0_metrics_prom_both`: both, `prometheus_http_metrics_enabled=True` (1.x) | +106.4 | +90.2 |
+| `otel_0_metrics_prom`: both configured, the 2.0 default | +84.5 | +73.4 |
+
+Every row's rounds were within 2%. Configuring both now costs what OTel HTTP metrics cost alone,
+21.9 µs less on FastAPI and 16.8 µs less on Litestar. On this host the OTel HTTP metrics cost 34 to
+36 µs and Prometheus's 18 to 25 µs, the reverse of the Linux table above, where OTel's were the
+cheaper set at about 12 µs. The stack suite (`run.py stack async otel,otel_metrics,otel_metrics_prom_both,otel_metrics_prom`,
+100% sampling) agrees: +44.4 µs for OTel HTTP metrics, +71.8 for both sets and +48.1 for the 2.0
+default, over `otel` at 109.0 µs per request.
+
 FastAPI real-server runs under uvicorn in the container were unstable (bare repeat medians of 8235
 and 6341 RPS, over 100% spread on the full stack), so the Linux real-server numbers below are
 Litestar on Granian only.
@@ -657,6 +679,9 @@ cd benchmarks
 # the two OpenTelemetry knobs (section 4a)
 ../.venv/bin/python run.py stack async otel,otel_exclude_spans,otel_sampler,otel_tuned
 
+# HTTP metrics recorded once against both sets (section 4d)
+../.venv/bin/python run.py stack async otel,otel_metrics,otel_metrics_prom_both,otel_metrics_prom
+
 # the Sentry ablation, where every familiar knob turns out to be a no-op (section 4b)
 ../.venv/bin/python run.py sentry async off,errors_only,errors_only_lean,errors_only_no_integrations,errors_only_no_txn
 

benchmarks/frameworks/fastapi_suite.py

@@ -46,6 +46,9 @@
     "otel_0_metrics": Scenario(config=base_config(**OTEL, **OTEL_METRICS, **ratio(0.0))),
     "otel_0_prom": Scenario(config=base_config(**OTEL, **ratio(0.0), **PROM)),
     "otel_0_metrics_prom": Scenario(config=base_config(**OTEL, **OTEL_METRICS, **ratio(0.0), **PROM)),
+    "otel_0_metrics_prom_both": Scenario(
+        config=base_config(**OTEL, **OTEL_METRICS, **ratio(0.0), **PROM, prometheus_http_metrics_enabled=True)
+    ),
     "sentry": Scenario(config=base_config(**sentry())),
     "sentry_nosess": Scenario(config=base_config(**sentry(), sentry_auto_session_tracking=False)),
     "sentry_noauto": Scenario(config=base_config(**sentry(auto_enabling_integrations=False))),

benchmarks/frameworks/litestar_suite.py

@@ -61,6 +61,9 @@
     "otel_0_metrics": Scenario(config=base_config(**OTEL, **OTEL_METRICS, **ratio(0.0))),
     "otel_0_prom": Scenario(config=base_config(**OTEL, **ratio(0.0), **PROM)),
     "otel_0_metrics_prom": Scenario(config=base_config(**OTEL, **OTEL_METRICS, **ratio(0.0), **PROM)),
+    "otel_0_metrics_prom_both": Scenario(
+        config=base_config(**OTEL, **OTEL_METRICS, **ratio(0.0), **PROM, prometheus_http_metrics_enabled=True)
+    ),
     "sentry": Scenario(config=base_config(**sentry())),
     "sentry_nosess": Scenario(config=base_config(**sentry(), sentry_auto_session_tracking=False)),
     "sentry_noauto": Scenario(config=base_config(**sentry(auto_enabling_integrations=False))),

benchmarks/stack_scenarios.py

@@ -23,6 +23,7 @@
 
 OTLP_PORT: typing.Final = int(os.environ.get("STUB_OTLP_PORT", "8139"))
 OTLP_ENDPOINT: typing.Final = f"http://127.0.0.1:{OTLP_PORT}/v1/traces"
+OTLP_METRICS_ENDPOINT: typing.Final = f"http://127.0.0.1:{OTLP_PORT}/v1/metrics"
 SAMPLE_RATIO: typing.Final = 0.01
 
 APPS: typing.Final = ("async", "structlog")
@@ -41,7 +42,9 @@
 }
 _OTEL_EXCLUDE_SPANS: typing.Final = {"opentelemetry_exclude_spans": ["receive", "send"]}
 _OTEL_SAMPLER: typing.Final = {"opentelemetry_sampler": ParentBased(TraceIdRatioBased(SAMPLE_RATIO))}
+_OTEL_METRICS: typing.Final = {"opentelemetry_metrics_endpoint": OTLP_METRICS_ENDPOINT}
 _PROMETHEUS: typing.Final = {"prometheus_metrics_path": "/metrics"}
+_PROMETHEUS_HTTP_METRICS: typing.Final = {"prometheus_http_metrics_enabled": True}
 _LOGGING: typing.Final = {"logging_enabled": True}
 _SENTRY: typing.Final = {
     "sentry_dsn": DSN,
@@ -86,6 +89,9 @@ def config(*parts: dict[str, typing.Any]) -> dict[str, typing.Any]:
     "full_no_sentry": lambda: config(_OTEL, _PROMETHEUS, _LOGGING),
     "full_sentry_tuned": lambda: config(_OTEL, _PROMETHEUS, _LOGGING, _sentry_tuned()),
     "otel_exclude_spans": lambda: config(_OTEL, _OTEL_EXCLUDE_SPANS),
+    "otel_metrics": lambda: config(_OTEL, _OTEL_METRICS),
+    "otel_metrics_prom": lambda: config(_OTEL, _OTEL_METRICS, _PROMETHEUS),
+    "otel_metrics_prom_both": lambda: config(_OTEL, _OTEL_METRICS, _PROMETHEUS, _PROMETHEUS_HTTP_METRICS),
     "otel_sampler": lambda: config(_OTEL, _OTEL_SAMPLER),
     "otel_tuned": lambda: config(_OTEL, _OTEL_EXCLUDE_SPANS, _OTEL_SAMPLER),
     "full_all_tuned": lambda: config(_OTEL, _OTEL_EXCLUDE_SPANS, _OTEL_SAMPLER, _PROMETHEUS, _LOGGING, _sentry_tuned()),

docs/introduction/configuration.md

@@ -116,6 +116,22 @@ Prometheus is on by default when its extra is installed, serving metrics at `pro
 Additional parameters:
 
 - `prometheus_metrics_include_in_schema`.
+- `prometheus_http_metrics_enabled` - FastAPI, Litestar and FastMCP only; see below.
+
+### Prometheus HTTP metrics
+
+On FastAPI, Litestar and FastMCP, the Prometheus instrument records every HTTP request, and so does
+OpenTelemetry's HTTP instrumentation once `opentelemetry_metrics_endpoint` installs a
+`MeterProvider`. Since 2.0 only one of them does. With `prometheus_http_metrics_enabled` left at
+`None`, the Prometheus instrument leaves its HTTP middleware out whenever OpenTelemetry records HTTP
+metrics: the endpoint is set, and the OpenTelemetry packages and the OTLP metric exporter for
+`opentelemetry_exporter_protocol` are installed. The metrics route is still served, for your own
+collectors and, on FastMCP, the tool-call metrics. Configuring both then costs 17 to 22 µs less per
+request than it did in 1.x.
+
+- `True` keeps the Prometheus HTTP metrics as well, for a service moving dashboards from one
+  backend to the other.
+- `False` leaves them out even without OpenTelemetry metrics.
 
 ### Prometheus Litestar
 
@@ -163,7 +179,8 @@ packages, and serves `prometheus_client.REGISTRY` at `prometheus_metrics_path`.
 Every ASGI application `application.http_app()` builds counts its requests in `http_requests_total`,
 `http_request_duration_seconds` and the instrumentator's other default metrics, labelled by route
 (`handler="/mcp"`); requests to unknown paths are grouped into `handler="none"`, and the metrics
-route itself is not counted. Tool calls are counted in `fastmcp_tool_calls_total{tool, status}`, with
+route itself is not counted. Those HTTP metrics are left out while OpenTelemetry records its own
+([Prometheus HTTP metrics](#prometheus-http-metrics)). Tool calls are counted in `fastmcp_tool_calls_total{tool, status}`, with
 `status` either `success` or `error`, and timed in `fastmcp_tool_call_duration_seconds{tool}`.
 
 Additional parameters for FastMCP integration:
@@ -206,10 +223,11 @@ nowhere.
 
 The export interval is the SDK's, 60 seconds, overridable with `OTEL_METRIC_EXPORT_INTERVAL`.
 
-If the Prometheus instrument is also configured, request metrics are recorded twice: once for the
-scrape endpoint and once for the OTLP pipeline, under different names. That is supported, and it
-is what you want when the two feed different backends, but pointing both at one backend
-double-counts.
+On FastAPI, Litestar and FastMCP, HTTP request metrics are recorded once. Since 2.0, when this
+endpoint is set, the Prometheus instrument stops adding its HTTP middleware and OpenTelemetry's set
+is the only one; see [Prometheus HTTP metrics](#prometheus-http-metrics). FastStream is the
+exception: both of its broker middlewares are opt-in, so with both configured each message is
+counted by both.
 
 ```python
 config = FastAPIConfig(

docs/introduction/performance.md

@@ -68,7 +68,7 @@ request, every request then pays the per-record cost measured here:
 
 --8<-- "benchmarks/README.md:logging"
 
-lite-bootstrap already applies six savings for you. It turns Sentry Logs off in the logging
+lite-bootstrap already applies seven savings for you. It turns Sentry Logs off in the logging
 integration, because it never enables them and on 2.x the handler formats every record before
 checking whether they are enabled; that one costs nothing. Since 2.0 it also leaves release-health
 sessions off and starts breadcrumbs at `WARNING`, so an `INFO` record skips the breadcrumb handler.
@@ -80,7 +80,9 @@ sentry-sdk's [auto-enabling integrations](configuration.md#sentry-auto-enabling-
 which costs you their SQL, Redis and outgoing-HTTP breadcrumbs; set
 `sentry_auto_enabling_integrations=True` to keep them. On FastAPI, Litestar and FastMCP it also
 records no span for the ASGI `receive` and `send` events, which costs you those spans; set
-`opentelemetry_exclude_spans=[]` to get them back. The numbers on this page were measured before
+`opentelemetry_exclude_spans=[]` to get them back. With `opentelemetry_metrics_endpoint` set it
+records HTTP metrics only through OpenTelemetry, which costs you the Prometheus HTTP metrics; set
+`prometheus_http_metrics_enabled=True` to keep both. The numbers on this page were measured before
 2.0, against the old defaults.
 
 ## What each saving costs you

lite_bootstrap/bootstrappers/fastapi_bootstrapper.py

@@ -17,7 +17,7 @@
 )
 from lite_bootstrap.instruments.logging_instrument import LoggingConfig, LoggingInstrument
 from lite_bootstrap.instruments.opentelemetry_instrument import OpenTelemetryAsgiConfig, OpenTelemetryInstrument
-from lite_bootstrap.instruments.prometheus_instrument import PrometheusConfig, PrometheusInstrument
+from lite_bootstrap.instruments.prometheus_instrument import PrometheusHttpConfig, PrometheusInstrument
 from lite_bootstrap.instruments.pyroscope_instrument import PyroscopeConfig, PyroscopeInstrument
 from lite_bootstrap.instruments.sentry_instrument import (
     SentryInstrument,
@@ -65,7 +65,7 @@ class FastAPIConfig(
     HealthChecksConfig,
     LoggingConfig,
     OpenTelemetryAsgiConfig,
-    PrometheusConfig,
+    PrometheusHttpConfig,
     PyroscopeConfig,
     SentryLeanCaptureConfig,
     SwaggerConfig,
@@ -248,10 +248,12 @@ def dependencies_installed() -> bool:
     def bootstrap(self) -> None:
         config = self.bootstrap_config
         application = config.app
-        Instrumentator(**config.prometheus_instrumentator_additional_params).instrument(
-            application,
-            **config.prometheus_instrument_additional_params,
-        ).expose(
+        instrumentator = Instrumentator(**config.prometheus_instrumentator_additional_params)
+        if config.is_prometheus_http_metrics_on(
+            opentelemetry_records_http_metrics=FastAPIOpenTelemetryInstrument.is_recording_metrics(config)
+        ):
+            instrumentator.instrument(application, **config.prometheus_instrument_additional_params)
+        instrumentator.expose(
             application,
             endpoint=config.prometheus_metrics_path,
             include_in_schema=config.prometheus_metrics_include_in_schema,

lite_bootstrap/bootstrappers/fastmcp_bootstrapper.py

@@ -12,7 +12,7 @@
 from lite_bootstrap.instruments.healthchecks_instrument import HealthChecksConfig, HealthChecksInstrument
 from lite_bootstrap.instruments.logging_instrument import LoggingConfig, LoggingInstrument
 from lite_bootstrap.instruments.opentelemetry_instrument import OpenTelemetryAsgiConfig, OpenTelemetryInstrument
-from lite_bootstrap.instruments.prometheus_instrument import PrometheusConfig, PrometheusInstrument
+from lite_bootstrap.instruments.prometheus_instrument import PrometheusHttpConfig, PrometheusInstrument
 from lite_bootstrap.instruments.pyroscope_instrument import PyroscopeConfig, PyroscopeInstrument
 from lite_bootstrap.instruments.sentry_instrument import (
     SentryInstrument,
@@ -204,7 +204,7 @@ class FastMcpConfig(
     HealthChecksConfig,
     LoggingConfig,
     OpenTelemetryAsgiConfig,
-    PrometheusConfig,
+    PrometheusHttpConfig,
     PyroscopeConfig,
     SentryLeanCaptureConfig,
 ):
@@ -328,7 +328,10 @@ def _instrument_http_app(self, http_application: "StarletteWithLifespan") -> "St
 
     def bootstrap(self) -> None:
         config = self.bootstrap_config
-        self._restore_http_app = _postprocess_http_apps(config.application, self._instrument_http_app)
+        if config.is_prometheus_http_metrics_on(
+            opentelemetry_records_http_metrics=FastMcpOpenTelemetryInstrument.is_recording_metrics(config)
+        ):
+            self._restore_http_app = _postprocess_http_apps(config.application, self._instrument_http_app)
         if config.fastmcp_prometheus_tool_metrics_enabled:
             config.application.add_middleware(FastMcpPrometheusMiddleware(*_build_tool_call_metrics()))
 

lite_bootstrap/bootstrappers/litestar_bootstrapper.py

@@ -16,12 +16,7 @@
 )
 from lite_bootstrap.instruments.logging_instrument import LoggingConfig, LoggingInstrument
 from lite_bootstrap.instruments.opentelemetry_instrument import OpenTelemetryAsgiConfig, OpenTelemetryInstrument
-from lite_bootstrap.instruments.prometheus_instrument import (
-    PrometheusConfig as PrometheusBootstrapperConfig,
-)
-from lite_bootstrap.instruments.prometheus_instrument import (
-    PrometheusInstrument,
-)
+from lite_bootstrap.instruments.prometheus_instrument import PrometheusHttpConfig, PrometheusInstrument
 from lite_bootstrap.instruments.pyroscope_instrument import PyroscopeConfig, PyroscopeInstrument
 from lite_bootstrap.instruments.sentry_instrument import (
     SENTRY_FAILED_REQUEST_STATUS_CODES,
@@ -255,7 +250,7 @@ class LitestarConfig(
     HealthChecksConfig,
     LoggingConfig,
     OpenTelemetryAsgiConfig,
-    PrometheusBootstrapperConfig,
+    PrometheusHttpConfig,
     PyroscopeConfig,
     SentryLeanCaptureConfig,
     SwaggerConfig,
@@ -432,7 +427,10 @@ class LitestarPrometheusController(PrometheusController):
         litestar_prometheus_config = PrometheusConfig(app_name=config.service_name, **prometheus_params)
 
         config.application_config.route_handlers.append(LitestarPrometheusController)
-        config.application_config.middleware.append(litestar_prometheus_config.middleware)
+        if config.is_prometheus_http_metrics_on(
+            opentelemetry_records_http_metrics=LitestarOpenTelemetryInstrument.is_recording_metrics(config)
+        ):
+            config.application_config.middleware.append(litestar_prometheus_config.middleware)
 
 
 @dataclasses.dataclass(kw_only=True, slots=True)

lite_bootstrap/instruments/opentelemetry_instrument.py

@@ -194,6 +194,15 @@ def is_configured(cls, bootstrap_config: "OpenTelemetryConfig") -> bool:
             or bootstrap_config.opentelemetry_log_traces
         )
 
+    @classmethod
+    def is_recording_metrics(cls, bootstrap_config: "OpenTelemetryConfig") -> bool:
+        """Whether bootstrap() installs a MeterProvider, which the framework's HTTP instrumentation records into."""
+        if not bootstrap_config.opentelemetry_metrics_endpoint or not cls.dependencies_installed():
+            return False
+        if bootstrap_config.opentelemetry_exporter_protocol == "grpc":
+            return import_checker.is_otlp_grpc_exporter_installed
+        return import_checker.is_otlp_http_exporter_installed
+
     @staticmethod
     def dependencies_installed() -> bool:
         # The instrument imports from both the api (opentelemetry.trace/.context) and