feat: add an opt-in OpenTelemetry metrics pipeline (#265)

Author: lesnik512

docs/introduction/configuration.md

@@ -99,9 +99,10 @@ Additional parameters for FastAPI integration:
 
 ## Opentelemetry
 
-To bootstrap Opentelemetry, you must provide at least:
+To bootstrap Opentelemetry, you must provide at least one of:
 
-- `opentelemetry_endpoint`.
+- `opentelemetry_endpoint`, for traces.
+- `opentelemetry_metrics_endpoint`, for metrics.
 
 Additional parameters:
 
@@ -116,6 +117,29 @@ Additional parameters:
 - `opentelemetry_sampler` - an `opentelemetry.sdk.trace.sampling.Sampler` deciding which traces are recorded. Unset, the SDK's own default applies: `parentbased_always_on`, which records every trace that is not the child of a non-recording remote parent, unless `OTEL_TRACES_SAMPLER` / `OTEL_TRACES_SAMPLER_ARG` say otherwise.
 - `opentelemetry_generate_health_check_spans` - generate spans for health check handlers if `True`.
 - `opentelemetry_excluded_urls` - extra URLs excluded from tracing; the metrics path and (unless health-check spans are enabled) the health-check path are excluded automatically.
+- `opentelemetry_metrics_endpoint` - will be passed to `OTLPMetricExporter` as endpoint, and turns on the metrics signal. Its own field rather than a flag on `opentelemetry_endpoint`, so upgrading never starts exporting metrics you did not ask for. Under `opentelemetry_exporter_protocol="http"` this is a full URL (e.g. `http://collector:4318/v1/metrics`); `opentelemetry_exporter_protocol` and `opentelemetry_insecure` are shared with traces.
+
+### Metrics
+
+Left unset, no `MeterProvider` is installed and the metrics signal stays off. Set it, and the
+FastAPI, Litestar and FastStream instrumentations start recording their duration histograms
+against it; until then they build those instruments against a no-op provider and the data goes
+nowhere.
+
+The export interval is the SDK's, 60 seconds, overridable with `OTEL_METRIC_EXPORT_INTERVAL`.
+
+If the Prometheus instrument is also configured, request metrics are recorded twice: once for the
+scrape endpoint and once for the OTLP pipeline, under different names. That is supported, and it
+is what you want when the two feed different backends, but pointing both at one backend
+double-counts.
+
+```python
+config = FastAPIConfig(
+    service_name="my-service",
+    opentelemetry_endpoint="localhost:4317",
+    opentelemetry_metrics_endpoint="localhost:4317",
+)
+```
 
 Sampling is the cheapest way to cut what tracing costs: on a benchmark endpoint returning a constant,
 `ParentBased(TraceIdRatioBased(0.01))` saved ~55 µs per request against the always-on default.

lite_bootstrap/bootstrappers/fastapi_bootstrapper.py

@@ -37,6 +37,7 @@
     fastapi_access_logger: typing.Final = structlog.get_logger("http.access")
 
 if import_checker.is_opentelemetry_installed:
+    from opentelemetry.metrics import get_meter_provider
     from opentelemetry.trace import get_tracer_provider
 
 if import_checker.is_fastapi_opentelemetry_installed:
@@ -200,6 +201,7 @@ def bootstrap(self) -> None:
         FastAPIInstrumentor.instrument_app(
             app=self.bootstrap_config.app,
             tracer_provider=get_tracer_provider(),
+            meter_provider=get_meter_provider(),
             excluded_urls=",".join(self._build_excluded_urls()),
             exclude_spans=self.bootstrap_config.opentelemetry_exclude_spans,
         )

lite_bootstrap/bootstrappers/faststream_bootstrapper.py

@@ -26,7 +26,7 @@
 
 if import_checker.is_opentelemetry_installed:
     from opentelemetry import trace
-    from opentelemetry.metrics import Meter, MeterProvider
+    from opentelemetry.metrics import Meter, MeterProvider, get_meter_provider
     from opentelemetry.trace import TracerProvider, get_tracer_provider
 
     tracer: typing.Final = trace.get_tracer(__name__)
@@ -149,7 +149,9 @@ def bootstrap(self) -> None:
         config = self.bootstrap_config
         if config.opentelemetry_middleware_cls and config.application.broker:
             config.application.broker.add_middleware(
-                config.opentelemetry_middleware_cls(tracer_provider=get_tracer_provider())
+                config.opentelemetry_middleware_cls(
+                    tracer_provider=get_tracer_provider(), meter_provider=get_meter_provider()
+                )
             )
 
 

lite_bootstrap/bootstrappers/litestar_bootstrapper.py

@@ -51,10 +51,12 @@
     from litestar.middleware import ASGIMiddleware
     from litestar.types.asgi_types import ASGIApp, Receive, Scope, Send
     from opentelemetry.instrumentation.asgi import OpenTelemetryMiddleware
+    from opentelemetry.metrics import MeterProvider
     from opentelemetry.trace import TracerProvider
     from opentelemetry.util.http import parse_excluded_urls
 
 if import_checker.is_opentelemetry_installed:
+    from opentelemetry.metrics import get_meter_provider
     from opentelemetry.trace import get_tracer_provider
 
 if import_checker.is_structlog_installed:
@@ -101,8 +103,11 @@ def build_litestar_route_details_from_scope(
 if import_checker.is_litestar_opentelemetry_installed:
 
     class LitestarOpenTelemetryInstrumentationMiddleware(ASGIMiddleware):
-        def __init__(self, tracer_provider: "TracerProvider", excluded_urls: set[str]) -> None:
+        def __init__(
+            self, tracer_provider: "TracerProvider", meter_provider: "MeterProvider", excluded_urls: set[str]
+        ) -> None:
             self._tracer_provider = tracer_provider
+            self._meter_provider = meter_provider
             # OpenTelemetryMiddleware only parses a raw string from 0.56b0; the floor is 0.49b0.
             self._excluded_urls = parse_excluded_urls(",".join(excluded_urls))
             # WeakKeyDictionary so wrapper apps are evicted when Litestar drops the
@@ -126,6 +131,7 @@ async def handle(
                     default_span_details=build_litestar_route_details_from_scope,
                     excluded_urls=self._excluded_urls,
                     tracer_provider=self._tracer_provider,
+                    meter_provider=self._meter_provider,
                 )
                 with contextlib.suppress(TypeError):
                     self._otel_apps[next_app] = otel_app  # ty: ignore[invalid-assignment]
@@ -256,6 +262,7 @@ def bootstrap(self) -> None:
         self.bootstrap_config.application_config.middleware.append(
             LitestarOpenTelemetryInstrumentationMiddleware(
                 tracer_provider=get_tracer_provider(),
+                meter_provider=get_meter_provider(),
                 excluded_urls=self._build_excluded_url_patterns(),
             )
         )

lite_bootstrap/instruments/opentelemetry_instrument.py

@@ -16,7 +16,10 @@
 
 if import_checker.is_opentelemetry_sdk_installed:
     from opentelemetry.context import Context
+    from opentelemetry.metrics import get_meter_provider, set_meter_provider
     from opentelemetry.sdk import resources
+    from opentelemetry.sdk.metrics import MeterProvider
+    from opentelemetry.sdk.metrics.export import MetricExporter, PeriodicExportingMetricReader
     from opentelemetry.sdk.trace import ReadableSpan, SpanProcessor, TracerProvider
     from opentelemetry.sdk.trace.export import (
         BatchSpanProcessor,
@@ -30,9 +33,11 @@
     # opentelemetry-api can be present without the grpc otlp exporter package (e.g.
     # lite-bootstrap[fastmcp] pulls bare opentelemetry-api transitively); this must
     # stay a separate guard from is_opentelemetry_installed above.
+    from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter as OTLPGrpcMetricExporter
     from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter as OTLPGrpcSpanExporter
 
 if import_checker.is_otlp_http_exporter_installed:
+    from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter as OTLPHttpMetricExporter
     from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter as OTLPHttpSpanExporter
 
 if import_checker.is_pyroscope_installed:
@@ -64,6 +69,7 @@ class OpenTelemetryConfig(OpenTelemetryServiceFieldsConfig):
         default_factory=lambda: os.environ.get("HOSTNAME") or None
     )
     opentelemetry_endpoint: str | None = None
+    opentelemetry_metrics_endpoint: str | None = None
     opentelemetry_insecure: bool = True
     opentelemetry_exporter_protocol: typing.Literal["grpc", "http"] = "grpc"
     opentelemetry_instrumentors: list[typing.Union[InstrumentorWithParams, "BaseInstrumentor"]] = dataclasses.field(
@@ -77,25 +83,30 @@ class OpenTelemetryConfig(OpenTelemetryServiceFieldsConfig):
     opentelemetry_excluded_urls: list[str] = dataclasses.field(default_factory=list)
 
     def __post_init__(self) -> None:
-        host = self._parse_remote_insecure_host()
-        if host is not None:
+        for host in self._parse_remote_insecure_hosts():
             warn_at_caller(
-                f"OTLP exporter sending traces unencrypted to non-local host {host!r}; "
+                f"OTLP exporter sending telemetry unencrypted to non-local host {host!r}; "
                 "set opentelemetry_insecure=False or use a localhost/unix endpoint."
             )
         super().__post_init__()
 
-    def _parse_remote_insecure_host(self) -> str | None:
+    def _parse_remote_insecure_hosts(self) -> list[str]:
+        """Return each distinct host an insecure, non-local endpoint points at, in declaration order."""
+        endpoints = (self.opentelemetry_endpoint, self.opentelemetry_metrics_endpoint)
+        hosts = (self._parse_remote_insecure_host(endpoint) for endpoint in endpoints)
+        return list(dict.fromkeys(host for host in hosts if host is not None))
+
+    def _parse_remote_insecure_host(self, endpoint: str | None) -> str | None:
         """Return the host name if the endpoint is insecure AND non-local; else None."""
         if self.opentelemetry_exporter_protocol != "grpc":
             return None
-        if not self.opentelemetry_endpoint or not self.opentelemetry_insecure:
+        if not endpoint or not self.opentelemetry_insecure:
             return None
-        if self.opentelemetry_endpoint.startswith("unix://"):
+        if endpoint.startswith("unix://"):
             return None
         # urlparse treats schemeless input as `path`, misparsing `host:port` forms.
         # Prepend `//` so urlparse always sees a network-location-style input.
-        raw = self.opentelemetry_endpoint
+        raw = endpoint
         if "://" not in raw:
             raw = f"//{raw}"
         parsed = urllib.parse.urlparse(raw)
@@ -144,18 +155,27 @@ class OpenTelemetryInstrument(BaseInstrument[OpenTelemetryConfig]):
     ``OpenTelemetryInstrument`` per process; do not bootstrap a second instance.
     """
 
-    not_configured_reason = "opentelemetry_endpoint is empty and opentelemetry_log_traces is False"
+    not_configured_reason = (
+        "opentelemetry_endpoint and opentelemetry_metrics_endpoint are empty and opentelemetry_log_traces is False"
+    )
     missing_dependency_message = "opentelemetry-sdk is not installed"
     _tracer_provider: "TracerProvider | None" = dataclasses.field(
         default_factory=lambda: None, init=False, repr=False, compare=False
     )
+    _meter_provider: "MeterProvider | None" = dataclasses.field(
+        default_factory=lambda: None, init=False, repr=False, compare=False
+    )
     _prior_logger_disabled: dict[str, bool] = dataclasses.field(
         default_factory=dict, init=False, repr=False, compare=False
     )
 
     @classmethod
     def is_configured(cls, bootstrap_config: "OpenTelemetryConfig") -> bool:
-        return bool(bootstrap_config.opentelemetry_endpoint or bootstrap_config.opentelemetry_log_traces)
+        return bool(
+            bootstrap_config.opentelemetry_endpoint
+            or bootstrap_config.opentelemetry_metrics_endpoint
+            or bootstrap_config.opentelemetry_log_traces
+        )
 
     @staticmethod
     def dependencies_installed() -> bool:
@@ -223,6 +243,50 @@ def _build_span_exporter(self) -> "SpanExporter | None":
             return None
         return OTLPHttpSpanExporter(endpoint=config.opentelemetry_endpoint)
 
+    def _build_metric_exporter(self) -> "MetricExporter | None":
+        """Return the OTLP metric exporter for the configured protocol, or None after warning it is missing.
+
+        Only call this once opentelemetry_metrics_endpoint is set: both warnings claim that it is.
+        """
+        config = self.bootstrap_config
+        if config.opentelemetry_exporter_protocol == "grpc":
+            if not import_checker.is_otlp_grpc_exporter_installed:
+                warn_at_caller(
+                    "opentelemetry_metrics_endpoint is set but the gRPC OTLP exporter is not installed; "
+                    "metrics will not be exported. Install lite-bootstrap[otl].",
+                    category=InstrumentDependencyMissingWarning,
+                )
+                return None
+            return OTLPGrpcMetricExporter(
+                endpoint=config.opentelemetry_metrics_endpoint,
+                insecure=config.opentelemetry_insecure,
+            )
+        if not import_checker.is_otlp_http_exporter_installed:
+            warn_at_caller(
+                "opentelemetry_metrics_endpoint is set but the HTTP OTLP exporter is not installed; "
+                "metrics will not be exported. Install lite-bootstrap[otl-http].",
+                category=InstrumentDependencyMissingWarning,
+            )
+            return None
+        return OTLPHttpMetricExporter(endpoint=config.opentelemetry_metrics_endpoint)
+
+    def _bootstrap_metrics(self, resource: "resources.Resource") -> None:
+        config = self.bootstrap_config
+        if not config.opentelemetry_metrics_endpoint:
+            return
+        metric_exporter = self._build_metric_exporter()
+        if metric_exporter is None:
+            return
+        meter_provider = MeterProvider(
+            resource=resource, metric_readers=[PeriodicExportingMetricReader(metric_exporter)]
+        )
+        set_meter_provider(meter_provider)
+        if get_meter_provider() is not meter_provider:
+            warn_at_caller(
+                "a MeterProvider is already installed; the configured metrics exporter and resource will not be used"
+            )
+        self._meter_provider = meter_provider
+
     def _apply_instrumentors(self, tracer_provider: "TracerProvider") -> None:
         for one_instrumentor in self.bootstrap_config.opentelemetry_instrumentors:
             if isinstance(one_instrumentor, InstrumentorWithParams):
@@ -236,7 +300,8 @@ def _apply_instrumentors(self, tracer_provider: "TracerProvider") -> None:
     def bootstrap(self) -> None:
         config = self.bootstrap_config
         self._silence_otel_loggers()
-        tracer_provider = TracerProvider(resource=self._build_resource(), sampler=config.opentelemetry_sampler)
+        resource = self._build_resource()
+        tracer_provider = TracerProvider(resource=resource, sampler=config.opentelemetry_sampler)
         set_tracer_provider(tracer_provider)
         if get_tracer_provider() is not tracer_provider:
             warn_at_caller(
@@ -249,6 +314,7 @@ def bootstrap(self) -> None:
             tracer_provider.add_span_processor(SimpleSpanProcessor(ConsoleSpanExporter(formatter=_format_span)))
         if config.opentelemetry_endpoint and (span_exporter := self._build_span_exporter()):
             tracer_provider.add_span_processor(BatchSpanProcessor(span_exporter))
+        self._bootstrap_metrics(resource)
         self._apply_instrumentors(tracer_provider)
 
     def teardown(self) -> None:
@@ -268,6 +334,12 @@ def teardown(self) -> None:
                         self._tracer_provider.shutdown()
                 finally:
                     self._tracer_provider = None
+            if self._meter_provider is not None:
+                try:
+                    with teardown_errors.capture("MeterProvider"):
+                        self._meter_provider.shutdown()
+                finally:
+                    self._meter_provider = None
 
 
 # Backward-compatible alias preserved for users importing the old (lowercase t) spelling.

scripts/floor_smoke.py

@@ -100,6 +100,7 @@ def _free() -> None:
             sentry_tags={"floor": "smoke"},
             pyroscope_endpoint=PYROSCOPE_ENDPOINT,
             opentelemetry_endpoint=OTLP_ENDPOINT,
+            opentelemetry_metrics_endpoint=OTLP_ENDPOINT,
             opentelemetry_log_traces=True,
         )
     )
@@ -127,6 +128,7 @@ def _fastapi() -> None:
             sentry_additional_params=SENTRY_PARAMS,
             pyroscope_endpoint=PYROSCOPE_ENDPOINT,
             opentelemetry_endpoint=OTLP_ENDPOINT,
+            opentelemetry_metrics_endpoint=OTLP_ENDPOINT,
             opentelemetry_log_traces=True,
         )
     )
@@ -153,6 +155,7 @@ def _litestar() -> None:
             sentry_additional_params=SENTRY_PARAMS,
             pyroscope_endpoint=PYROSCOPE_ENDPOINT,
             opentelemetry_endpoint=OTLP_ENDPOINT,
+            opentelemetry_metrics_endpoint=OTLP_ENDPOINT,
             opentelemetry_log_traces=True,
         )
     )
@@ -174,6 +177,7 @@ def _faststream() -> None:
             sentry_additional_params=SENTRY_PARAMS,
             pyroscope_endpoint=PYROSCOPE_ENDPOINT,
             opentelemetry_endpoint=OTLP_ENDPOINT,
+            opentelemetry_metrics_endpoint=OTLP_ENDPOINT,
             opentelemetry_log_traces=True,
         )
     )

tests/conftest.py

@@ -4,6 +4,7 @@
 import warnings
 from importlib import reload
 
+import opentelemetry.metrics._internal
 import opentelemetry.trace
 import pytest
 import sentry_sdk
@@ -44,16 +45,19 @@ def capture_envelope(self, envelope: Envelope) -> None:
 
 
 @pytest.fixture(autouse=True)
-def _reset_tracer_provider() -> typing.Iterator[None]:
-    """Give each test the process-global TracerProvider unset, as a fresh process has it.
+def _reset_otel_providers() -> typing.Iterator[None]:
+    """Give each test the process-global tracer and meter providers unset, as a fresh process has them.
 
-    `set_tracer_provider` is set-once, so without this the first test to bootstrap owns the global
-    and every later one silently loses the race against it. Restoring the two module-level names
-    the SDK guards it with is the only way back: there is no public reset.
+    Both setters are set-once, so without this the first test to bootstrap owns the global and every
+    later one silently loses the race against it. Restoring the module-level names the SDK guards
+    them with is the only way back: there is no public reset. The meter provider keeps its pair in
+    `opentelemetry.metrics._internal`, one module below the public re-export.
     """
     yield
     opentelemetry.trace._TRACER_PROVIDER = None  # noqa: SLF001
     opentelemetry.trace._TRACER_PROVIDER_SET_ONCE = Once()  # noqa: SLF001
+    opentelemetry.metrics._internal._METER_PROVIDER = None  # noqa: SLF001
+    opentelemetry.metrics._internal._METER_PROVIDER_SET_ONCE = Once()  # noqa: SLF001
 
 
 @pytest.fixture