From d4dbdc1e613d2fc96258166f29c5e2184e1632da Mon Sep 17 00:00:00 2001 From: Artur Shiriev Date: Thu, 29 Oct 2026 15:53:01 +0300 Subject: [PATCH] Default opentelemetry_generate_health_check_spans to False --- benchmarks/README.md | 17 +++++++- benchmarks/frameworks/fastapi_suite.py | 3 ++ benchmarks/frameworks/litestar_suite.py | 3 ++ docs/integrations/fastmcp.md | 2 +- docs/introduction/configuration.md | 8 +++- docs/introduction/performance.md | 6 ++- .../instruments/opentelemetry_instrument.py | 2 +- tests/test_fastapi_bootstrap.py | 11 +++++ tests/test_fastmcp_bootstrap.py | 10 +++-- tests/test_faststream_bootstrap.py | 40 +++++++++++++++++-- 10 files changed, 88 insertions(+), 14 deletions(-) diff --git a/benchmarks/README.md b/benchmarks/README.md index 8d52800..297d562 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -228,7 +228,7 @@ structlog 26.1.0. In-process, baseline 17.7 µs per request. The handler does no | Sentry, `sentry_auto_session_tracking=False` | +51.8 | | | Sentry, `auto_enabling_integrations=False` | +0.1 | the Starlette and FastAPI integrations are auto-enabling, so nothing captures request errors | | health check `/health/` | +22.6 | noisy, 16% spread | -| health check with OpenTelemetry, spans on (default) / off | +162.3 / +28.9 | noisy, 32 to 50% spread | +| health check with OpenTelemetry, spans on (1.x default) / off | +162.3 / +28.9 | noisy, 32 to 50% spread | | CORS, request with a matching Origin | +3.8 | | | full: OTel + Prometheus + logging + Sentry + health, defaults | +259.6 | | @@ -273,6 +273,18 @@ cheaper set at about 12 µs. The stack suite (`run.py stack async otel,otel_metr 100% sampling) agrees: +44.4 µs for OTel HTTP metrics, +71.8 for both sets and +48.1 for the 2.0 default, over `otel` at 109.0 µs per request. +Since 2.0, health checks are untraced by default (`opentelemetry_generate_health_check_spans=False`). +Rerun on the same host with `bench.py health,health_otel,health_otel_spans --repeat 3`, at 100% +sampling, in µs per `/health/` request over the health check without OpenTelemetry: + +| scenario | FastAPI | Litestar | +|---|---:|---:| +| `health_otel`: the 2.0 default, no span | +5.1 | +1.9 | +| `health_otel_spans`: `opentelemetry_generate_health_check_spans=True`, the 1.x default | +99.2 | +82.4 | + +Litestar's rounds differed by up to 24%. At one probe every few seconds per pod this is far below +a percent of a core either way. What the default saves is one trace per probe per pod. + FastAPI real-server runs under uvicorn in the container were unstable (bare repeat medians of 8235 and 6341 RPS, over 100% spread on the full stack), so the Linux real-server numbers below are Litestar on Granian only. @@ -431,7 +443,7 @@ prometheus-client 0.26.0, structlog 26.1.0. In-process, baseline 12.1 µs per re | CORS, simple request with a matching Origin | +7.2 | against bare with the same header | | Swagger (`/docs`), online or offline | -0.2 | registers routes only | | health check `/health/`, against an equivalent `/ping` handler | +7.1 | `asyncio.gather` of one coroutine | -| health check with OpenTelemetry, spans on (default) | +134.0 | against health without OpenTelemetry | +| health check with OpenTelemetry, spans on (1.x default) | +134.0 | against health without OpenTelemetry | | health check with OpenTelemetry, `opentelemetry_generate_health_check_spans=False` | +9.2 | | | full: OTel + Prometheus + logging + Sentry + health, defaults | +297.7 | | | full, 3 structlog records per request | +377.8 | | @@ -741,6 +753,7 @@ cd benchmarks/frameworks ../../.venv/bin/python bench.py fastapi bare,log_mw,prom,otel,otel_excl,otel_0,sentry,health,full --repeat 2 ../../.venv/bin/python bench.py litestar bare,log_log3,log_mw,prom,otel,otel_excl,otel_1_excl,otel_0,sentry,full --repeat 2 ../../.venv/bin/python bench.py litestar health_ping,health,health_otel,health_otel_nospan --repeat 2 +../../.venv/bin/python bench.py fastapi health,health_otel,health_otel_spans --repeat 3 # Litestar on Granian, over real sockets (section 8c); needs granian in the environment and `ab` on PATH ../../.venv/bin/python run_http.py litestar bare,otel,otel_excl,prom,log_mw,sentry,health,full --repeat 2 diff --git a/benchmarks/frameworks/fastapi_suite.py b/benchmarks/frameworks/fastapi_suite.py index 05597d3..b85ee37 100644 --- a/benchmarks/frameworks/fastapi_suite.py +++ b/benchmarks/frameworks/fastapi_suite.py @@ -64,6 +64,9 @@ ), "health": Scenario(config=base_config(**HEALTH), path="/health/"), "health_otel": Scenario(config=base_config(**HEALTH, **OTEL), path="/health/"), + "health_otel_spans": Scenario( + config=base_config(**HEALTH, **OTEL, opentelemetry_generate_health_check_spans=True), path="/health/" + ), "health_otel_nospan": Scenario( config=base_config(**HEALTH, **OTEL, opentelemetry_generate_health_check_spans=False), path="/health/" ), diff --git a/benchmarks/frameworks/litestar_suite.py b/benchmarks/frameworks/litestar_suite.py index 365f1c7..2e08277 100644 --- a/benchmarks/frameworks/litestar_suite.py +++ b/benchmarks/frameworks/litestar_suite.py @@ -84,6 +84,9 @@ "health_ping": Scenario(config=base_config(**HEALTH), kind="healthlike"), "health": Scenario(config=base_config(**HEALTH), path="/health/"), "health_otel": Scenario(config=base_config(**HEALTH, **OTEL), path="/health/"), + "health_otel_spans": Scenario( + config=base_config(**HEALTH, **OTEL, opentelemetry_generate_health_check_spans=True), path="/health/" + ), "health_otel_nospan": Scenario( config=base_config(**HEALTH, **OTEL, opentelemetry_generate_health_check_spans=False), path="/health/" ), diff --git a/docs/integrations/fastmcp.md b/docs/integrations/fastmcp.md index c2ba6a6..b446de0 100644 --- a/docs/integrations/fastmcp.md +++ b/docs/integrations/fastmcp.md @@ -104,7 +104,7 @@ wrapped in OpenTelemetry's ASGI middleware. Each request produces a server span route, such as `POST /mcp` or `GET /health/`, with the route in `http.route`. A request to an unknown path produces a span named after the HTTP method alone, without `http.route`. -The metrics path, the health-check path (unless `opentelemetry_generate_health_check_spans` is on) +The metrics path, the health-check path (unless `opentelemetry_generate_health_check_spans` is True) `opentelemetry_excluded_urls` and `OTEL_PYTHON_STARLETTE_EXCLUDED_URLS` produce no spans. An application already instrumented by `StarletteInstrumentor` is left as is, so no request is traced twice. diff --git a/docs/introduction/configuration.md b/docs/introduction/configuration.md index 3f0910c..8e9dc59 100644 --- a/docs/introduction/configuration.md +++ b/docs/introduction/configuration.md @@ -210,7 +210,11 @@ Additional parameters: - `opentelemetry_instrumentors` - a list of extra instrumentors. - `opentelemetry_log_traces` - traces will be logged to stdout. - `opentelemetry_sampler` - an `opentelemetry.sdk.trace.sampling.Sampler` deciding which traces are recorded. Unset, the SDK's own default applies: `parentbased_always_on`, which records every trace that is not the child of a non-recording remote parent, unless `OTEL_TRACES_SAMPLER` / `OTEL_TRACES_SAMPLER_ARG` say otherwise. -- `opentelemetry_generate_health_check_spans` - generate spans for health check handlers if `True`. +- `opentelemetry_generate_health_check_spans` - trace health-check requests. Defaults to `False` + since 2.0, so a probe sends no trace, the same as the access log, which never records the + health-check path. While it is `False`, OpenTelemetry's HTTP metrics also leave health-check + requests out, because the instrumentation skips an excluded URL entirely. Set it to `True` to + trace probes, for example to watch their latency. - `opentelemetry_excluded_urls` - extra regexes for URLs excluded from tracing; the metrics path and (unless health-check spans are enabled) the health-check path are excluded automatically. Litestar searches them in the request path, FastAPI and FastMCP in the full URL. The environment variable OpenTelemetry defines for the framework (`OTEL_PYTHON_LITESTAR_EXCLUDED_URLS`, `OTEL_PYTHON_FASTAPI_EXCLUDED_URLS` or `OTEL_PYTHON_STARLETTE_EXCLUDED_URLS` for FastMCP), falling back to `OTEL_PYTHON_EXCLUDED_URLS`, adds to them. - `opentelemetry_metrics_endpoint` - will be passed to `OTLPMetricExporter` as endpoint, and turns on the metrics signal. Its own field rather than a flag on `opentelemetry_endpoint`, so upgrading never starts exporting metrics you did not ask for. Under `opentelemetry_exporter_protocol="http"` this is a full URL (e.g. `http://collector:4318/v1/metrics`); `opentelemetry_exporter_protocol` and `opentelemetry_insecure` are shared with traces. @@ -265,7 +269,7 @@ For FastStream there is additionally: `faststream.redis.opentelemetry.RedisTelemetryMiddleware`, or a mapping from broker class to middleware class (see [multiple brokers](../integrations/faststream.md#multiple-brokers)). You must provide it to trace broker messages. Without it the rest of the configuration above still applies: the exporter is built, the - health-check span is recorded unless `opentelemetry_generate_health_check_spans` is False, and any + health-check span is recorded when `opentelemetry_generate_health_check_spans` is True, and any `opentelemetry_instrumentors` are applied. ### Resource diff --git a/docs/introduction/performance.md b/docs/introduction/performance.md index 26969f7..4293c2b 100644 --- a/docs/introduction/performance.md +++ b/docs/introduction/performance.md @@ -68,7 +68,7 @@ request, every request then pays the per-record cost measured here: --8<-- "benchmarks/README.md:logging" -lite-bootstrap already applies seven savings for you. It turns Sentry Logs off in the logging +lite-bootstrap already applies eight savings for you. It turns Sentry Logs off in the logging integration, because it never enables them and on 2.x the handler formats every record before checking whether they are enabled; that one costs nothing. Since 2.0 it also leaves release-health sessions off and starts breadcrumbs at `WARNING`, so an `INFO` record skips the breadcrumb handler. @@ -82,7 +82,9 @@ which costs you their SQL, Redis and outgoing-HTTP breadcrumbs; set records no span for the ASGI `receive` and `send` events, which costs you those spans; set `opentelemetry_exclude_spans=[]` to get them back. With `opentelemetry_metrics_endpoint` set it records HTTP metrics only through OpenTelemetry, which costs you the Prometheus HTTP metrics; set -`prometheus_http_metrics_enabled=True` to keep both. The numbers on this page were measured before +`prometheus_http_metrics_enabled=True` to keep both. It also leaves health checks untraced, which +costs you probe traces; set `opentelemetry_generate_health_check_spans=True` to get them back. The +numbers on this page were measured before 2.0, against the old defaults. ## What each saving costs you diff --git a/lite_bootstrap/instruments/opentelemetry_instrument.py b/lite_bootstrap/instruments/opentelemetry_instrument.py index ca8bb5f..82567ca 100644 --- a/lite_bootstrap/instruments/opentelemetry_instrument.py +++ b/lite_bootstrap/instruments/opentelemetry_instrument.py @@ -83,7 +83,7 @@ class OpenTelemetryConfig(OpenTelemetryServiceFieldsConfig): # Left None, the SDK picks its own default, which reads OTEL_TRACES_SAMPLER; see ADR-0008 # (docs/adr/0008-otel-sampling-is-a-sampler-object.md). opentelemetry_sampler: "Sampler | None" = None - opentelemetry_generate_health_check_spans: bool = True + opentelemetry_generate_health_check_spans: bool = False opentelemetry_excluded_urls: list[str] = dataclasses.field(default_factory=list) def __post_init__(self) -> None: diff --git a/tests/test_fastapi_bootstrap.py b/tests/test_fastapi_bootstrap.py index da47763..36f47ea 100644 --- a/tests/test_fastapi_bootstrap.py +++ b/tests/test_fastapi_bootstrap.py @@ -602,6 +602,17 @@ def test_fastapi_otel_excludes_derived_paths_only_where_they_match_whole_segment assert sorted(span_names) == ["GET /api/custom-metrics/report", "GET /custom-healthy"] +def test_fastapi_otel_leaves_the_health_check_untraced_by_default() -> None: + config = FastAPIConfig( + service_name="microservice", + health_checks_path="/custom-health/", + logging_buffer_capacity=0, + opentelemetry_log_traces=True, + ) + + assert _server_span_names_for(config, ["/custom-health/", "/custom-healthy"]) == ["GET /custom-healthy"] + + @pytest.mark.parametrize("environment_variable", ["OTEL_PYTHON_FASTAPI_EXCLUDED_URLS", "OTEL_PYTHON_EXCLUDED_URLS"]) def test_fastapi_otel_honors_excluded_urls_from_environment( fastapi_config: FastAPIConfig, monkeypatch: pytest.MonkeyPatch, environment_variable: str diff --git a/tests/test_fastmcp_bootstrap.py b/tests/test_fastmcp_bootstrap.py index bf43e5e..8a2301f 100644 --- a/tests/test_fastmcp_bootstrap.py +++ b/tests/test_fastmcp_bootstrap.py @@ -398,7 +398,11 @@ def _server_spans(exporter: InMemorySpanExporter) -> list[ReadableSpan]: def _bootstrapped_with_span_exporter( **overrides: typing.Any, # noqa: ANN401 ) -> Generator[tuple[FastMcpBootstrapper, FastMCP, InMemorySpanExporter]]: - bootstrapper = FastMcpBootstrapper(bootstrap_config=_make_test_config(opentelemetry_log_traces=True, **overrides)) + """Trace with health-check spans on, since the health route is the cheapest request these tests send.""" + config_overrides: typing.Final = {"opentelemetry_generate_health_check_spans": True, **overrides} + bootstrapper = FastMcpBootstrapper( + bootstrap_config=_make_test_config(opentelemetry_log_traces=True, **config_overrides) + ) application = bootstrapper.bootstrap() tracer_provider = get_tracer_provider() assert isinstance(tracer_provider, SDKTracerProvider) @@ -522,8 +526,8 @@ def _unregister_request_metrics() -> None: def _http_metric_sets_one_request_moves(**overrides: typing.Any) -> tuple[bool, bool]: # noqa: ANN401 - """Return whether one request reached OpenTelemetry's HTTP metrics and Prometheus's, in that order.""" - config: typing.Final = _make_test_config(**overrides) + """Return whether one health-check request reached OpenTelemetry's HTTP metrics and Prometheus's, in that order.""" + config: typing.Final = _make_test_config(**{"opentelemetry_generate_health_check_spans": True, **overrides}) with in_memory_opentelemetry_metric_reader() as reader: bootstrapper = FastMcpBootstrapper(bootstrap_config=config) application = bootstrapper.bootstrap() diff --git a/tests/test_faststream_bootstrap.py b/tests/test_faststream_bootstrap.py index b1b02ef..b3e41d4 100644 --- a/tests/test_faststream_bootstrap.py +++ b/tests/test_faststream_bootstrap.py @@ -20,11 +20,15 @@ from opentelemetry.metrics import MeterProvider, get_meter_provider from opentelemetry.sdk import resources from opentelemetry.sdk.metrics import MeterProvider as SDKMeterProvider -from opentelemetry.trace import TracerProvider +from opentelemetry.sdk.trace import TracerProvider as SDKTracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import TracerProvider, get_tracer, get_tracer_provider from starlette import status from starlette.testclient import TestClient from lite_bootstrap import FastStreamBootstrapper, FastStreamConfig +from lite_bootstrap.bootstrappers import faststream_bootstrapper from lite_bootstrap.bootstrappers.faststream_bootstrapper import ( FastStreamLoggingInstrument, FastStreamOpenTelemetryInstrument, @@ -213,11 +217,39 @@ def test_faststream_bootstrap_without_prometheus_client(broker: RedisBroker) -> bootstrapper.bootstrap() +def _span_names_for_one_health_check(config: FastStreamConfig) -> list[str]: + bootstrapper = FastStreamBootstrapper(bootstrap_config=config) + # The module's tracer stays bound to the first provider a test installed; this one resolves to this test's. + with patch.object(faststream_bootstrapper, "tracer", get_tracer(__name__)): + application = bootstrapper.bootstrap() + tracer_provider = get_tracer_provider() + assert isinstance(tracer_provider, SDKTracerProvider) + exporter = InMemorySpanExporter() + tracer_provider.add_span_processor(SimpleSpanProcessor(exporter)) + try: + TestClient(app=application).get(config.health_checks_path) + finally: + bootstrapper.teardown() + return [span.name for span in exporter.get_finished_spans()] + + +def test_faststream_health_check_records_no_span_by_default() -> None: + assert _span_names_for_one_health_check(build_faststream_config()) == [] + + +def test_faststream_health_check_span_is_recorded_when_turned_on() -> None: + config = dataclasses.replace(build_faststream_config(), opentelemetry_generate_health_check_spans=True) + + assert _span_names_for_one_health_check(config) == [f"GET {config.health_checks_path}"] + + def test_faststream_bootstrap_without_opentelemetry(broker: RedisBroker) -> None: # Regression: issue #87 bug 2 — FastStreamHealthChecksInstrument.bootstrap # referenced unbound `tracer` when opentelemetry was absent and # opentelemetry_generate_health_check_spans defaulted to True. - bootstrap_config = build_faststream_config(broker=broker) + bootstrap_config = dataclasses.replace( + build_faststream_config(broker=broker), opentelemetry_generate_health_check_spans=True + ) with emulate_package_missing_with_module_reload( "opentelemetry", ["lite_bootstrap.bootstrappers.faststream_bootstrapper"], @@ -326,7 +358,9 @@ def test_faststream_build_excluded_urls_covers_prometheus_and_health_paths(broke assert config_no_health_spans.prometheus_metrics_path in excluded # always excluded assert config_no_health_spans.health_checks_path in excluded # excluded when health spans are off - config_with_health_spans = build_faststream_config(broker=broker) # generate_health_check_spans defaults True + config_with_health_spans = dataclasses.replace( + build_faststream_config(broker=broker), opentelemetry_generate_health_check_spans=True + ) excluded_with = FastStreamOpenTelemetryInstrument(bootstrap_config=config_with_health_spans)._build_excluded_urls() # noqa: SLF001 assert config_with_health_spans.prometheus_metrics_path in excluded_with assert config_with_health_spans.health_checks_path not in excluded_with # kept when health spans are on