Skip to content

Commit 822d8c5

Browse files
authored
test: verify HTTP metric counts and simplify probe coverage (#446)
1 parent c069e70 commit 822d8c5

7 files changed

Lines changed: 128 additions & 143 deletions

‎tests/conftest.py‎

Lines changed: 36 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -4,15 +4,15 @@
44
import warnings
55
from collections.abc import Generator
66
from importlib import reload
7-
from unittest.mock import MagicMock, patch
7+
from unittest.mock import patch
88

99
import opentelemetry.metrics._internal
1010
import opentelemetry.trace
1111
import prometheus_client
1212
import pytest
1313
import sentry_sdk
1414
from opentelemetry.instrumentation.instrumentor import BaseInstrumentor
15-
from opentelemetry.sdk.metrics.export import InMemoryMetricReader
15+
from opentelemetry.sdk.metrics.export import Histogram, InMemoryMetricReader
1616
from opentelemetry.util._once import Once
1717
from sentry_sdk import integrations as sentry_integrations
1818
from sentry_sdk import scope as sentry_scope
@@ -25,7 +25,7 @@
2525

2626
from lite_bootstrap import import_checker
2727
from lite_bootstrap.exceptions import InstrumentDependencyMissingWarning
28-
from lite_bootstrap.instruments import healthchecks_instrument, opentelemetry_instrument
28+
from lite_bootstrap.instruments import opentelemetry_instrument
2929

3030

3131
def pytest_configure() -> None:
@@ -64,22 +64,11 @@ async def raising_probe() -> bool:
6464

6565

6666
HEALTH_PROBE_CASES: typing.Final = [
67-
pytest.param({"db": passing_probe, "cache": passing_probe}, 200, True, [], id="all-pass"),
68-
pytest.param({"db": passing_probe, "cache": failing_probe}, 503, False, ["cache"], id="one-returns-false"),
69-
pytest.param({"db": passing_probe, "cache": raising_probe}, 503, False, ["cache"], id="one-raises"),
67+
pytest.param(passing_probe, 200, True, id="passes"),
68+
pytest.param(failing_probe, 503, False, id="fails"),
7069
]
7170

7271

73-
@contextlib.contextmanager
74-
def capture_probe_warnings() -> Generator[MagicMock]:
75-
with patch.object(healthchecks_instrument, "logger") as mock_logger:
76-
yield mock_logger.warning
77-
78-
79-
def failed_probe_names(warning_mock: MagicMock) -> list[str]:
80-
return [call.args[1] for call in warning_mock.call_args_list]
81-
82-
8372
class SentryTestTransport(sentry_sdk.Transport):
8473
def __init__(self, *args: typing.Any, **kwargs: typing.Any) -> None: # noqa: ANN401
8574
super().__init__(*args, **kwargs)
@@ -247,20 +236,39 @@ def in_memory_opentelemetry_metric_reader() -> Generator[InMemoryMetricReader]:
247236
yield reader
248237

249238

250-
def opentelemetry_recorded_http_server_metrics(reader: InMemoryMetricReader) -> bool:
251-
metrics_data: typing.Final = reader.get_metrics_data()
252-
if metrics_data is None:
253-
return False
254-
return any(
255-
metric.name.startswith("http.server.")
256-
for resource_metrics in metrics_data.resource_metrics
257-
for scope_metrics in resource_metrics.scope_metrics
258-
for metric in scope_metrics.metrics
259-
)
239+
HTTP_METRIC_CASES: typing.Final = [
240+
pytest.param({"opentelemetry_metrics_endpoint": "localhost:4317"}, (1, 0), id="both-keeps-otel"),
241+
pytest.param(
242+
{"opentelemetry_metrics_endpoint": "localhost:4317", "prometheus_http_metrics_enabled": True},
243+
(1, 1),
244+
id="both-restored",
245+
),
246+
pytest.param({}, (0, 1), id="prometheus-alone"),
247+
pytest.param(
248+
{"opentelemetry_metrics_endpoint": "localhost:4317", "prometheus_metrics_path": ""},
249+
(1, 0),
250+
id="otel-alone",
251+
),
252+
pytest.param({"prometheus_http_metrics_enabled": False}, (0, 0), id="prometheus-http-off"),
253+
]
260254

261255

262-
def prometheus_recorded_path(path: str) -> bool:
263-
return f'"{path}"' in prometheus_client.generate_latest().decode()
256+
def opentelemetry_http_request_count(reader: InMemoryMetricReader) -> int:
257+
"""Count requests without adding the legacy and stable series when both are emitted."""
258+
metrics_data = reader.get_metrics_data()
259+
if metrics_data is None:
260+
return 0
261+
counts: dict[str, int] = {}
262+
for resource_metrics in metrics_data.resource_metrics:
263+
for scope_metrics in resource_metrics.scope_metrics:
264+
for metric in scope_metrics.metrics:
265+
if metric.name in {"http.server.duration", "http.server.request.duration"} and isinstance(
266+
metric.data, Histogram
267+
):
268+
counts[metric.name] = counts.get(metric.name, 0) + sum(
269+
point.count for point in metric.data.data_points
270+
)
271+
return max(counts.values(), default=0)
264272

265273

266274
def prometheus_request_count(metric_name: str, path_label: str, path: str) -> float:

‎tests/instruments/test_healthchecks_instrument.py‎

Lines changed: 42 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
11
import asyncio
2-
import time
32
from unittest.mock import MagicMock
43

54
import pytest
65

76
from lite_bootstrap.instruments import healthchecks_instrument
8-
from lite_bootstrap.instruments.healthchecks_instrument import HealthChecksConfig, HealthChecksInstrument
7+
from lite_bootstrap.instruments.healthchecks_instrument import HealthChecksConfig, HealthChecksInstrument, HealthProbe
8+
from tests.conftest import ProbeError, failing_probe, passing_probe, raising_probe
99

1010

1111
def test_healthchecks_instrument_configured_by_default() -> None:
@@ -52,18 +52,51 @@ def test_healthchecks_config_defaults() -> None:
5252

5353

5454
async def test_healthchecks_probes_run_concurrently() -> None:
55-
interval = 0.2
55+
first_started = asyncio.Event()
56+
second_started = asyncio.Event()
5657

57-
async def slow_probe() -> bool:
58-
await asyncio.sleep(interval)
58+
async def first() -> bool:
59+
first_started.set()
60+
await second_started.wait()
61+
return True
62+
63+
async def second() -> bool:
64+
await first_started.wait()
65+
second_started.set()
5966
return True
6067

6168
instrument = HealthChecksInstrument(
62-
bootstrap_config=HealthChecksConfig(health_checks_probes={"first": slow_probe, "second": slow_probe}),
69+
bootstrap_config=HealthChecksConfig(health_checks_probes={"first": first, "second": second}),
6370
)
64-
started = time.perf_counter()
65-
assert await instrument.define_health_status()
66-
assert time.perf_counter() - started < interval * 1.5
71+
assert await asyncio.wait_for(instrument.define_health_status(), timeout=5)
72+
73+
74+
@pytest.mark.parametrize(
75+
("probe", "expected_message", "expected_exception"),
76+
[
77+
pytest.param(failing_probe, "Health probe cache failed", None, id="returns-false"),
78+
pytest.param(raising_probe, "Health probe cache raised", ProbeError, id="raises"),
79+
],
80+
)
81+
async def test_failed_probe_makes_health_unhealthy_and_warns(
82+
probe: HealthProbe,
83+
expected_message: str,
84+
expected_exception: type[Exception] | None,
85+
caplog: pytest.LogCaptureFixture,
86+
) -> None:
87+
instrument = HealthChecksInstrument(
88+
bootstrap_config=HealthChecksConfig(health_checks_probes={"db": passing_probe, "cache": probe}),
89+
)
90+
91+
assert not await instrument.define_health_status()
92+
93+
[record] = [record for record in caplog.records if record.name == healthchecks_instrument.__name__]
94+
assert record.getMessage() == expected_message
95+
if expected_exception is None:
96+
assert record.exc_info is None
97+
else:
98+
assert record.exc_info
99+
assert isinstance(record.exc_info[1], expected_exception)
67100

68101

69102
async def test_healthchecks_status_is_healthy_without_probes() -> None:

‎tests/test_fastapi_bootstrap.py‎

Lines changed: 16 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -29,15 +29,13 @@
2929
from lite_bootstrap.types import UNSET
3030
from tests.conftest import (
3131
HEALTH_PROBE_CASES,
32+
HTTP_METRIC_CASES,
3233
CustomInstrumentor,
3334
SentryTestTransport,
34-
capture_probe_warnings,
3535
emulate_package_missing,
3636
emulate_package_missing_with_module_reload,
37-
failed_probe_names,
3837
in_memory_opentelemetry_metric_reader,
39-
opentelemetry_recorded_http_server_metrics,
40-
prometheus_recorded_path,
38+
opentelemetry_http_request_count,
4139
prometheus_request_count,
4240
unregister_prometheus_request_metrics,
4341
warning_source_files,
@@ -614,8 +612,8 @@ async def items() -> list[int]:
614612
assert prometheus_request_count("http_requests_total", "handler", normal_path) == 1
615613

616614

617-
def _http_metric_sets_one_request_moves(config: FastAPIConfig) -> tuple[bool, bool]:
618-
"""Return whether one request reached OpenTelemetry's HTTP metrics and Prometheus's, in that order."""
615+
def _http_request_counts(config: FastAPIConfig) -> tuple[int, float]:
616+
"""Return the OpenTelemetry and Prometheus request counts after one request."""
619617
path: typing.Final = f"/items-{uuid.uuid4().hex}"
620618
application: typing.Final = config.app
621619
unregister_prometheus_request_metrics()
@@ -630,41 +628,31 @@ async def items() -> str:
630628
try:
631629
with TestClient(application) as client:
632630
assert client.get(path).status_code == status.HTTP_200_OK
633-
return opentelemetry_recorded_http_server_metrics(reader), prometheus_recorded_path(path)
631+
return opentelemetry_http_request_count(reader), prometheus_request_count(
632+
"http_requests_total", "handler", path
633+
)
634634
finally:
635635
bootstrapper.teardown()
636636

637637

638638
@pytest.mark.parametrize(
639639
("overrides", "expected"),
640640
[
641-
pytest.param({"opentelemetry_metrics_endpoint": "localhost:4317"}, (True, False), id="both-keeps-otel"),
641+
*HTTP_METRIC_CASES,
642642
pytest.param(
643643
{
644644
"opentelemetry_metrics_endpoint": "http://localhost:4318/v1/metrics",
645645
"opentelemetry_exporter_protocol": "http",
646646
},
647-
(True, False),
647+
(1, 0),
648648
id="both-keeps-otel-over-http",
649649
),
650-
pytest.param(
651-
{"opentelemetry_metrics_endpoint": "localhost:4317", "prometheus_http_metrics_enabled": True},
652-
(True, True),
653-
id="both-restored",
654-
),
655-
pytest.param({}, (False, True), id="prometheus-alone"),
656-
pytest.param(
657-
{"opentelemetry_metrics_endpoint": "localhost:4317", "prometheus_metrics_path": ""},
658-
(True, False),
659-
id="otel-alone",
660-
),
661-
pytest.param({"prometheus_http_metrics_enabled": False}, (False, False), id="prometheus-http-off"),
662650
],
663651
)
664652
def test_fastapi_http_metrics_are_recorded_once(
665-
fastapi_config: FastAPIConfig, overrides: dict[str, typing.Any], expected: tuple[bool, bool]
653+
fastapi_config: FastAPIConfig, overrides: dict[str, typing.Any], expected: tuple[int, int]
666654
) -> None:
667-
assert _http_metric_sets_one_request_moves(dataclasses.replace(fastapi_config, **overrides)) == expected
655+
assert _http_request_counts(dataclasses.replace(fastapi_config, **overrides)) == expected
668656

669657

670658
def test_fastapi_prometheus_keeps_http_metrics_when_the_otlp_metric_exporter_is_missing(
@@ -675,7 +663,7 @@ def test_fastapi_prometheus_keeps_http_metrics_when_the_otlp_metric_exporter_is_
675663
patch.object(import_checker, "is_otlp_grpc_exporter_installed", new=False),
676664
pytest.warns(InstrumentDependencyMissingWarning, match="gRPC OTLP exporter is not installed"),
677665
):
678-
assert _http_metric_sets_one_request_moves(config) == (False, True)
666+
assert _http_request_counts(config) == (0, 1)
679667

680668

681669
def test_fastapi_prometheus_still_serves_metrics_while_opentelemetry_records_http_metrics(
@@ -771,18 +759,17 @@ def test_fastapi_otel_honors_excluded_urls_from_environment(
771759
assert span_names == ["GET /custom-healthy"]
772760

773761

774-
@pytest.mark.parametrize(("probes", "expected_status", "expected_health_status", "expected_failed"), HEALTH_PROBE_CASES)
762+
@pytest.mark.parametrize(("probe", "expected_status", "expected_health_status"), HEALTH_PROBE_CASES)
775763
def test_fastapi_health_check_runs_probes(
776764
fastapi_config: FastAPIConfig,
777-
probes: dict[str, HealthProbe],
765+
probe: HealthProbe,
778766
expected_status: int,
779767
expected_health_status: bool,
780-
expected_failed: list[str],
781768
) -> None:
782-
config = dataclasses.replace(fastapi_config, health_checks_probes=probes)
769+
config = dataclasses.replace(fastapi_config, health_checks_probes={"cache": probe})
783770
bootstrapper = FastAPIBootstrapper(bootstrap_config=config)
784771
application = bootstrapper.bootstrap()
785-
with capture_probe_warnings() as probe_warnings, TestClient(application) as test_client:
772+
with TestClient(application) as test_client:
786773
response = test_client.get(config.health_checks_path)
787774

788775
assert response.status_code == expected_status
@@ -791,7 +778,6 @@ def test_fastapi_health_check_runs_probes(
791778
"service_name": "microservice",
792779
"service_version": "2.0.0",
793780
}
794-
assert failed_probe_names(probe_warnings) == expected_failed
795781

796782

797783
def test_built_app_title_and_version_fall_back_when_unset() -> None:

‎tests/test_fastmcp_bootstrap.py‎

Lines changed: 15 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -40,13 +40,12 @@
4040
from lite_bootstrap.instruments.healthchecks_instrument import HealthProbe
4141
from tests.conftest import (
4242
HEALTH_PROBE_CASES,
43-
capture_probe_warnings,
43+
HTTP_METRIC_CASES,
4444
emulate_package_missing,
4545
emulate_package_missing_with_module_reload,
46-
failed_probe_names,
4746
in_memory_opentelemetry_metric_reader,
48-
opentelemetry_recorded_http_server_metrics,
49-
prometheus_recorded_path,
47+
opentelemetry_http_request_count,
48+
prometheus_request_count,
5049
unregister_prometheus_request_metrics,
5150
)
5251

@@ -573,8 +572,8 @@ def _unregister_request_metrics() -> None:
573572
unregister_prometheus_request_metrics()
574573

575574

576-
def _http_metric_sets_one_request_moves(**overrides: typing.Any) -> tuple[bool, bool]: # noqa: ANN401
577-
"""Return whether one application request reached OpenTelemetry and Prometheus HTTP metrics, in that order."""
575+
def _http_request_counts(**overrides: typing.Any) -> tuple[int, float]: # noqa: ANN401
576+
"""Return the OpenTelemetry and Prometheus request counts after one request."""
578577
config = _make_test_config(**overrides)
579578
path = f"/items-{uuid.uuid4().hex}"
580579

@@ -588,31 +587,16 @@ async def items(_request: Request) -> Response:
588587
try:
589588
with TestClient(application.http_app()) as client:
590589
assert client.get(path).status_code == status.HTTP_200_OK
591-
return opentelemetry_recorded_http_server_metrics(reader), prometheus_recorded_path(path)
590+
return opentelemetry_http_request_count(reader), prometheus_request_count(
591+
"http_requests_total", "handler", path
592+
)
592593
finally:
593594
bootstrapper.teardown()
594595

595596

596-
@pytest.mark.parametrize(
597-
("overrides", "expected"),
598-
[
599-
pytest.param({"opentelemetry_metrics_endpoint": "localhost:4317"}, (True, False), id="both-keeps-otel"),
600-
pytest.param(
601-
{"opentelemetry_metrics_endpoint": "localhost:4317", "prometheus_http_metrics_enabled": True},
602-
(True, True),
603-
id="both-restored",
604-
),
605-
pytest.param({}, (False, True), id="prometheus-alone"),
606-
pytest.param(
607-
{"opentelemetry_metrics_endpoint": "localhost:4317", "prometheus_metrics_path": ""},
608-
(True, False),
609-
id="otel-alone",
610-
),
611-
pytest.param({"prometheus_http_metrics_enabled": False}, (False, False), id="prometheus-http-off"),
612-
],
613-
)
614-
def test_fastmcp_http_metrics_are_recorded_once(overrides: dict[str, typing.Any], expected: tuple[bool, bool]) -> None:
615-
assert _http_metric_sets_one_request_moves(**overrides) == expected
597+
@pytest.mark.parametrize(("overrides", "expected"), HTTP_METRIC_CASES)
598+
def test_fastmcp_http_metrics_are_recorded_once(overrides: dict[str, typing.Any], expected: tuple[int, int]) -> None:
599+
assert _http_request_counts(**overrides) == expected
616600

617601

618602
def test_fastmcp_prometheus_keeps_tool_metrics_and_metrics_route_while_opentelemetry_records_http_metrics() -> None:
@@ -876,18 +860,17 @@ def test_fastmcp_otel_honors_excluded_urls_from_environment(monkeypatch: pytest.
876860
]
877861

878862

879-
@pytest.mark.parametrize(("probes", "expected_status", "expected_health_status", "expected_failed"), HEALTH_PROBE_CASES)
863+
@pytest.mark.parametrize(("probe", "expected_status", "expected_health_status"), HEALTH_PROBE_CASES)
880864
def test_fastmcp_health_check_runs_probes(
881-
probes: dict[str, HealthProbe],
865+
probe: HealthProbe,
882866
expected_status: int,
883867
expected_health_status: bool,
884-
expected_failed: list[str],
885868
) -> None:
886-
config = _make_test_config(health_checks_probes=probes)
869+
config = _make_test_config(health_checks_probes={"cache": probe})
887870
bootstrapper = FastMcpBootstrapper(bootstrap_config=config)
888871
application = bootstrapper.bootstrap()
889872
try:
890-
with capture_probe_warnings() as probe_warnings, TestClient(application.http_app()) as test_client:
873+
with TestClient(application.http_app()) as test_client:
891874
response = test_client.get(config.health_checks_path)
892875
finally:
893876
bootstrapper.teardown()
@@ -898,4 +881,3 @@ def test_fastmcp_health_check_runs_probes(
898881
"service_name": "test-mcp",
899882
"service_version": "1.2.3",
900883
}
901-
assert failed_probe_names(probe_warnings) == expected_failed

0 commit comments

Comments
 (0)