Default opentelemetry_generate_health_check_spans to False (#366)

Author: lesnik512

benchmarks/README.md

@@ -228,7 +228,7 @@ structlog 26.1.0. In-process, baseline 17.7 µs per request. The handler does no
 | Sentry, `sentry_auto_session_tracking=False` | +51.8 | |
 | Sentry, `auto_enabling_integrations=False` | +0.1 | the Starlette and FastAPI integrations are auto-enabling, so nothing captures request errors |
 | health check `/health/` | +22.6 | noisy, 16% spread |
-| health check with OpenTelemetry, spans on (default) / off | +162.3 / +28.9 | noisy, 32 to 50% spread |
+| health check with OpenTelemetry, spans on (1.x default) / off | +162.3 / +28.9 | noisy, 32 to 50% spread |
 | CORS, request with a matching Origin | +3.8 | |
 | full: OTel + Prometheus + logging + Sentry + health, defaults | +259.6 | |
 <!-- --8<-- [end:fastapilinux] -->
@@ -273,6 +273,18 @@ cheaper set at about 12 µs. The stack suite (`run.py stack async otel,otel_metr
 100% sampling) agrees: +44.4 µs for OTel HTTP metrics, +71.8 for both sets and +48.1 for the 2.0
 default, over `otel` at 109.0 µs per request.
 
+Since 2.0, health checks are untraced by default (`opentelemetry_generate_health_check_spans=False`).
+Rerun on the same host with `bench.py <suite> health,health_otel,health_otel_spans --repeat 3`, at 100%
+sampling, in µs per `/health/` request over the health check without OpenTelemetry:
+
+| scenario | FastAPI | Litestar |
+|---|---:|---:|
+| `health_otel`: the 2.0 default, no span | +5.1 | +1.9 |
+| `health_otel_spans`: `opentelemetry_generate_health_check_spans=True`, the 1.x default | +99.2 | +82.4 |
+
+Litestar's rounds differed by up to 24%. At one probe every few seconds per pod this is far below
+a percent of a core either way. What the default saves is one trace per probe per pod.
+
 FastAPI real-server runs under uvicorn in the container were unstable (bare repeat medians of 8235
 and 6341 RPS, over 100% spread on the full stack), so the Linux real-server numbers below are
 Litestar on Granian only.
@@ -431,7 +443,7 @@ prometheus-client 0.26.0, structlog 26.1.0. In-process, baseline 12.1 µs per re
 | CORS, simple request with a matching Origin | +7.2 | against bare with the same header |
 | Swagger (`/docs`), online or offline | -0.2 | registers routes only |
 | health check `/health/`, against an equivalent `/ping` handler | +7.1 | `asyncio.gather` of one coroutine |
-| health check with OpenTelemetry, spans on (default) | +134.0 | against health without OpenTelemetry |
+| health check with OpenTelemetry, spans on (1.x default) | +134.0 | against health without OpenTelemetry |
 | health check with OpenTelemetry, `opentelemetry_generate_health_check_spans=False` | +9.2 | |
 | full: OTel + Prometheus + logging + Sentry + health, defaults | +297.7 | |
 | full, 3 structlog records per request | +377.8 | |
@@ -741,6 +753,7 @@ cd benchmarks/frameworks
 ../../.venv/bin/python bench.py fastapi bare,log_mw,prom,otel,otel_excl,otel_0,sentry,health,full --repeat 2
 ../../.venv/bin/python bench.py litestar bare,log_log3,log_mw,prom,otel,otel_excl,otel_1_excl,otel_0,sentry,full --repeat 2
 ../../.venv/bin/python bench.py litestar health_ping,health,health_otel,health_otel_nospan --repeat 2
+../../.venv/bin/python bench.py fastapi health,health_otel,health_otel_spans --repeat 3
 
 # Litestar on Granian, over real sockets (section 8c); needs granian in the environment and `ab` on PATH
 ../../.venv/bin/python run_http.py litestar bare,otel,otel_excl,prom,log_mw,sentry,health,full --repeat 2

benchmarks/frameworks/fastapi_suite.py

@@ -64,6 +64,9 @@
     ),
     "health": Scenario(config=base_config(**HEALTH), path="/health/"),
     "health_otel": Scenario(config=base_config(**HEALTH, **OTEL), path="/health/"),
+    "health_otel_spans": Scenario(
+        config=base_config(**HEALTH, **OTEL, opentelemetry_generate_health_check_spans=True), path="/health/"
+    ),
     "health_otel_nospan": Scenario(
         config=base_config(**HEALTH, **OTEL, opentelemetry_generate_health_check_spans=False), path="/health/"
     ),

benchmarks/frameworks/litestar_suite.py

@@ -84,6 +84,9 @@
     "health_ping": Scenario(config=base_config(**HEALTH), kind="healthlike"),
     "health": Scenario(config=base_config(**HEALTH), path="/health/"),
     "health_otel": Scenario(config=base_config(**HEALTH, **OTEL), path="/health/"),
+    "health_otel_spans": Scenario(
+        config=base_config(**HEALTH, **OTEL, opentelemetry_generate_health_check_spans=True), path="/health/"
+    ),
     "health_otel_nospan": Scenario(
         config=base_config(**HEALTH, **OTEL, opentelemetry_generate_health_check_spans=False), path="/health/"
     ),

docs/integrations/fastmcp.md

@@ -104,7 +104,7 @@ wrapped in OpenTelemetry's ASGI middleware. Each request produces a server span
 route, such as `POST /mcp` or `GET /health/`, with the route in `http.route`. A request to an
 unknown path produces a span named after the HTTP method alone, without `http.route`.
 
-The metrics path, the health-check path (unless `opentelemetry_generate_health_check_spans` is on)
+The metrics path, the health-check path (unless `opentelemetry_generate_health_check_spans` is True)
 `opentelemetry_excluded_urls` and `OTEL_PYTHON_STARLETTE_EXCLUDED_URLS` produce no spans. An application already instrumented by
 `StarletteInstrumentor` is left as is, so no request is traced twice.
 

docs/introduction/configuration.md

@@ -210,7 +210,11 @@ Additional parameters:
 - `opentelemetry_instrumentors` - a list of extra instrumentors.
 - `opentelemetry_log_traces` - traces will be logged to stdout.
 - `opentelemetry_sampler` - an `opentelemetry.sdk.trace.sampling.Sampler` deciding which traces are recorded. Unset, the SDK's own default applies: `parentbased_always_on`, which records every trace that is not the child of a non-recording remote parent, unless `OTEL_TRACES_SAMPLER` / `OTEL_TRACES_SAMPLER_ARG` say otherwise.
-- `opentelemetry_generate_health_check_spans` - generate spans for health check handlers if `True`.
+- `opentelemetry_generate_health_check_spans` - trace health-check requests. Defaults to `False`
+  since 2.0, so a probe sends no trace, the same as the access log, which never records the
+  health-check path. While it is `False`, OpenTelemetry's HTTP metrics also leave health-check
+  requests out, because the instrumentation skips an excluded URL entirely. Set it to `True` to
+  trace probes, for example to watch their latency.
 - `opentelemetry_excluded_urls` - extra regexes for URLs excluded from tracing; the metrics path and (unless health-check spans are enabled) the health-check path are excluded automatically. Litestar searches them in the request path, FastAPI and FastMCP in the full URL. The environment variable OpenTelemetry defines for the framework (`OTEL_PYTHON_LITESTAR_EXCLUDED_URLS`, `OTEL_PYTHON_FASTAPI_EXCLUDED_URLS` or `OTEL_PYTHON_STARLETTE_EXCLUDED_URLS` for FastMCP), falling back to `OTEL_PYTHON_EXCLUDED_URLS`, adds to them.
 - `opentelemetry_metrics_endpoint` - will be passed to `OTLPMetricExporter` as endpoint, and turns on the metrics signal. Its own field rather than a flag on `opentelemetry_endpoint`, so upgrading never starts exporting metrics you did not ask for. Under `opentelemetry_exporter_protocol="http"` this is a full URL (e.g. `http://collector:4318/v1/metrics`); `opentelemetry_exporter_protocol` and `opentelemetry_insecure` are shared with traces.
 
@@ -265,7 +269,7 @@ For FastStream there is additionally:
   `faststream.redis.opentelemetry.RedisTelemetryMiddleware`, or a mapping from broker class to
   middleware class (see [multiple brokers](../integrations/faststream.md#multiple-brokers)). You must
   provide it to trace broker messages. Without it the rest of the configuration above still applies: the exporter is built, the
-  health-check span is recorded unless `opentelemetry_generate_health_check_spans` is False, and any
+  health-check span is recorded when `opentelemetry_generate_health_check_spans` is True, and any
   `opentelemetry_instrumentors` are applied.
 
 ### Resource

docs/introduction/performance.md

@@ -68,7 +68,7 @@ request, every request then pays the per-record cost measured here:
 
 --8<-- "benchmarks/README.md:logging"
 
-lite-bootstrap already applies seven savings for you. It turns Sentry Logs off in the logging
+lite-bootstrap already applies eight savings for you. It turns Sentry Logs off in the logging
 integration, because it never enables them and on 2.x the handler formats every record before
 checking whether they are enabled; that one costs nothing. Since 2.0 it also leaves release-health
 sessions off and starts breadcrumbs at `WARNING`, so an `INFO` record skips the breadcrumb handler.
@@ -82,7 +82,9 @@ which costs you their SQL, Redis and outgoing-HTTP breadcrumbs; set
 records no span for the ASGI `receive` and `send` events, which costs you those spans; set
 `opentelemetry_exclude_spans=[]` to get them back. With `opentelemetry_metrics_endpoint` set it
 records HTTP metrics only through OpenTelemetry, which costs you the Prometheus HTTP metrics; set
-`prometheus_http_metrics_enabled=True` to keep both. The numbers on this page were measured before
+`prometheus_http_metrics_enabled=True` to keep both. It also leaves health checks untraced, which
+costs you probe traces; set `opentelemetry_generate_health_check_spans=True` to get them back. The
+numbers on this page were measured before
 2.0, against the old defaults.
 
 ## What each saving costs you

lite_bootstrap/instruments/opentelemetry_instrument.py

@@ -83,7 +83,7 @@ class OpenTelemetryConfig(OpenTelemetryServiceFieldsConfig):
     # Left None, the SDK picks its own default, which reads OTEL_TRACES_SAMPLER; see ADR-0008
     # (docs/adr/0008-otel-sampling-is-a-sampler-object.md).
     opentelemetry_sampler: "Sampler | None" = None
-    opentelemetry_generate_health_check_spans: bool = True
+    opentelemetry_generate_health_check_spans: bool = False
     opentelemetry_excluded_urls: list[str] = dataclasses.field(default_factory=list)
 
     def __post_init__(self) -> None:

tests/test_fastapi_bootstrap.py

@@ -602,6 +602,17 @@ def test_fastapi_otel_excludes_derived_paths_only_where_they_match_whole_segment
     assert sorted(span_names) == ["GET /api/custom-metrics/report", "GET /custom-healthy"]
 
 
+def test_fastapi_otel_leaves_the_health_check_untraced_by_default() -> None:
+    config = FastAPIConfig(
+        service_name="microservice",
+        health_checks_path="/custom-health/",
+        logging_buffer_capacity=0,
+        opentelemetry_log_traces=True,
+    )
+
+    assert _server_span_names_for(config, ["/custom-health/", "/custom-healthy"]) == ["GET /custom-healthy"]
+
+
 @pytest.mark.parametrize("environment_variable", ["OTEL_PYTHON_FASTAPI_EXCLUDED_URLS", "OTEL_PYTHON_EXCLUDED_URLS"])
 def test_fastapi_otel_honors_excluded_urls_from_environment(
     fastapi_config: FastAPIConfig, monkeypatch: pytest.MonkeyPatch, environment_variable: str

tests/test_fastmcp_bootstrap.py

@@ -398,7 +398,11 @@ def _server_spans(exporter: InMemorySpanExporter) -> list[ReadableSpan]:
 def _bootstrapped_with_span_exporter(
     **overrides: typing.Any,  # noqa: ANN401
 ) -> Generator[tuple[FastMcpBootstrapper, FastMCP, InMemorySpanExporter]]:
-    bootstrapper = FastMcpBootstrapper(bootstrap_config=_make_test_config(opentelemetry_log_traces=True, **overrides))
+    """Trace with health-check spans on, since the health route is the cheapest request these tests send."""
+    config_overrides: typing.Final = {"opentelemetry_generate_health_check_spans": True, **overrides}
+    bootstrapper = FastMcpBootstrapper(
+        bootstrap_config=_make_test_config(opentelemetry_log_traces=True, **config_overrides)
+    )
     application = bootstrapper.bootstrap()
     tracer_provider = get_tracer_provider()
     assert isinstance(tracer_provider, SDKTracerProvider)
@@ -522,8 +526,8 @@ def _unregister_request_metrics() -> None:
 
 
 def _http_metric_sets_one_request_moves(**overrides: typing.Any) -> tuple[bool, bool]:  # noqa: ANN401
-    """Return whether one request reached OpenTelemetry's HTTP metrics and Prometheus's, in that order."""
-    config: typing.Final = _make_test_config(**overrides)
+    """Return whether one health-check request reached OpenTelemetry's HTTP metrics and Prometheus's, in that order."""
+    config: typing.Final = _make_test_config(**{"opentelemetry_generate_health_check_spans": True, **overrides})
     with in_memory_opentelemetry_metric_reader() as reader:
         bootstrapper = FastMcpBootstrapper(bootstrap_config=config)
         application = bootstrapper.bootstrap()

tests/test_faststream_bootstrap.py

@@ -20,11 +20,15 @@
 from opentelemetry.metrics import MeterProvider, get_meter_provider
 from opentelemetry.sdk import resources
 from opentelemetry.sdk.metrics import MeterProvider as SDKMeterProvider
-from opentelemetry.trace import TracerProvider
+from opentelemetry.sdk.trace import TracerProvider as SDKTracerProvider
+from opentelemetry.sdk.trace.export import SimpleSpanProcessor
+from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
+from opentelemetry.trace import TracerProvider, get_tracer, get_tracer_provider
 from starlette import status
 from starlette.testclient import TestClient
 
 from lite_bootstrap import FastStreamBootstrapper, FastStreamConfig
+from lite_bootstrap.bootstrappers import faststream_bootstrapper
 from lite_bootstrap.bootstrappers.faststream_bootstrapper import (
     FastStreamLoggingInstrument,
     FastStreamOpenTelemetryInstrument,
@@ -213,11 +217,39 @@ def test_faststream_bootstrap_without_prometheus_client(broker: RedisBroker) ->
         bootstrapper.bootstrap()
 
 
+def _span_names_for_one_health_check(config: FastStreamConfig) -> list[str]:
+    bootstrapper = FastStreamBootstrapper(bootstrap_config=config)
+    # The module's tracer stays bound to the first provider a test installed; this one resolves to this test's.
+    with patch.object(faststream_bootstrapper, "tracer", get_tracer(__name__)):
+        application = bootstrapper.bootstrap()
+    tracer_provider = get_tracer_provider()
+    assert isinstance(tracer_provider, SDKTracerProvider)
+    exporter = InMemorySpanExporter()
+    tracer_provider.add_span_processor(SimpleSpanProcessor(exporter))
+    try:
+        TestClient(app=application).get(config.health_checks_path)
+    finally:
+        bootstrapper.teardown()
+    return [span.name for span in exporter.get_finished_spans()]
+
+
+def test_faststream_health_check_records_no_span_by_default() -> None:
+    assert _span_names_for_one_health_check(build_faststream_config()) == []
+
+
+def test_faststream_health_check_span_is_recorded_when_turned_on() -> None:
+    config = dataclasses.replace(build_faststream_config(), opentelemetry_generate_health_check_spans=True)
+
+    assert _span_names_for_one_health_check(config) == [f"GET {config.health_checks_path}"]
+
+
 def test_faststream_bootstrap_without_opentelemetry(broker: RedisBroker) -> None:
     # Regression: issue #87 bug 2 — FastStreamHealthChecksInstrument.bootstrap
     # referenced unbound `tracer` when opentelemetry was absent and
     # opentelemetry_generate_health_check_spans defaulted to True.
-    bootstrap_config = build_faststream_config(broker=broker)
+    bootstrap_config = dataclasses.replace(
+        build_faststream_config(broker=broker), opentelemetry_generate_health_check_spans=True
+    )
     with emulate_package_missing_with_module_reload(
         "opentelemetry",
         ["lite_bootstrap.bootstrappers.faststream_bootstrapper"],
@@ -326,7 +358,9 @@ def test_faststream_build_excluded_urls_covers_prometheus_and_health_paths(broke
     assert config_no_health_spans.prometheus_metrics_path in excluded  # always excluded
     assert config_no_health_spans.health_checks_path in excluded  # excluded when health spans are off
 
-    config_with_health_spans = build_faststream_config(broker=broker)  # generate_health_check_spans defaults True
+    config_with_health_spans = dataclasses.replace(
+        build_faststream_config(broker=broker), opentelemetry_generate_health_check_spans=True
+    )
     excluded_with = FastStreamOpenTelemetryInstrument(bootstrap_config=config_with_health_spans)._build_excluded_urls()  # noqa: SLF001
     assert config_with_health_spans.prometheus_metrics_path in excluded_with
     assert config_with_health_spans.health_checks_path not in excluded_with  # kept when health spans are on