Skip to content

Commit 2af92e0

Browse files
authored
Bound FastMCP metrics to available tools (#433) (#442)
1 parent e0b2065 commit 2af92e0

4 files changed

Lines changed: 117 additions & 4 deletions

File tree

‎docs/introduction/configuration.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -196,6 +196,10 @@ Every ASGI application `application.http_app()` builds counts its requests in `h
196196
route itself is not counted. Those HTTP metrics are left out while OpenTelemetry records its own
197197
([Prometheus HTTP metrics](#prometheus-http-metrics)). Tool calls are counted in `fastmcp_tool_calls_total{tool, status}`, with
198198
`status` either `success` or `error`, and timed in `fastmcp_tool_call_duration_seconds{tool}`.
199+
Calls whose names do not resolve to an available tool are omitted from both tool metrics,
200+
preventing arbitrary names from creating unbounded series. Registered, mounted and transformed
201+
tools retain their client-visible names as labels, including when execution fails.
202+
Hashed app-tool calls use `<hash>_<resolved tool name>` so aliases share one series.
199203

200204
Additional parameters for FastMCP integration:
201205

‎docs/migration/to-2.x.md‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -325,6 +325,15 @@ headers, including `Content-Type`. Add `Authorization` explicitly when needed
325325

326326
## Prometheus
327327

328+
### Unknown FastMCP tools
329+
330+
Calls to unresolved tool names no longer create `fastmcp_tool_calls_total` or
331+
`fastmcp_tool_call_duration_seconds` series. These calls still return errors, but
332+
are omitted from tool metrics to bound retained series. Registered, mounted and
333+
transformed tools keep their labels and success/failure counts
334+
([#433](https://github.com/modern-python/lite-bootstrap/issues/433)).
335+
Hashed app-tool labels use `<hash>_<resolved tool name>`; aliases share that label.
336+
328337
### HTTP metrics alongside OpenTelemetry
329338

330339
On FastAPI, Litestar, and FastMCP, configuring OpenTelemetry HTTP metrics suppresses

‎lite_bootstrap/bootstrappers/fastmcp_bootstrapper.py‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
from fastmcp import FastMCP
2626
from fastmcp.server.http import StarletteWithLifespan
2727
from fastmcp.server.middleware import Middleware, MiddlewareContext
28+
from fastmcp.server.providers.addressing import parse_hashed_backend_name
2829
from starlette import status
2930
from starlette.requests import Request
3031
from starlette.responses import JSONResponse, Response
@@ -139,7 +140,19 @@ async def on_call_tool(
139140
context: "MiddlewareContext[typing.Any]",
140141
call_next: "typing.Callable[[MiddlewareContext[typing.Any]], typing.Awaitable[typing.Any]]",
141142
) -> typing.Any: # noqa: ANN401
142-
tool_name: typing.Final = context.message.name
143+
tool_name = context.message.name
144+
fastmcp_context: typing.Final = context.fastmcp_context
145+
tool = None
146+
if fastmcp_context is not None:
147+
tool = await fastmcp_context.fastmcp.get_tool(tool_name)
148+
if tool is None:
149+
hashed_name = parse_hashed_backend_name(tool_name)
150+
if hashed_name is not None:
151+
tool = await fastmcp_context.fastmcp.get_tool_by_hash(*hashed_name)
152+
if tool is not None:
153+
tool_name = f"{hashed_name[0]}_{tool.name}"
154+
if tool is None:
155+
return await call_next(context)
143156
start_time: typing.Final = time.perf_counter()
144157
try:
145158
result = await call_next(context)

‎tests/test_fastmcp_bootstrap.py‎

Lines changed: 90 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,13 @@
99
import pytest
1010
from fastmcp import Client, FastMCP
1111
from fastmcp.exceptions import ToolError
12+
from fastmcp.server import create_proxy
1213
from fastmcp.server.context import Context
1314
from fastmcp.server.http import StarletteWithLifespan
1415
from fastmcp.server.middleware import MiddlewareContext
16+
from fastmcp.server.providers.addressing import hash_tool, hashed_backend_name
17+
from fastmcp.server.transforms.tool_transform import ToolTransform
18+
from fastmcp.tools.tool_transform import ToolTransformConfig
1519
from opentelemetry.sdk.trace import ReadableSpan
1620
from opentelemetry.sdk.trace import TracerProvider as SDKTracerProvider
1721
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
@@ -702,7 +706,63 @@ def test_fastmcp_prometheus_teardown_restores_http_app() -> None:
702706
assert "http_app" not in vars(application)
703707

704708

705-
async def test_fastmcp_prometheus_counts_tool_calls() -> None:
709+
def _retained_tool_metric_samples() -> int:
710+
return sum(
711+
len(metric.samples)
712+
for metric in prometheus_client.REGISTRY.collect()
713+
if metric.name.startswith("fastmcp_tool_call")
714+
)
715+
716+
717+
@pytest.mark.parametrize("name_prefix", ["missing", "000000000000_missing"])
718+
async def test_fastmcp_prometheus_unknown_tools_do_not_retain_metric_samples(name_prefix: str) -> None:
719+
bootstrapper = FastMcpBootstrapper(bootstrap_config=_make_test_config(logging_enabled=False))
720+
application = bootstrapper.bootstrap()
721+
samples_before = _retained_tool_metric_samples()
722+
missing_prefix = f"{name_prefix}_{uuid.uuid4().hex}"
723+
try:
724+
async with Client(application) as client:
725+
for index in range(100):
726+
with pytest.raises(ToolError, match="Unknown tool"):
727+
await client.call_tool(f"{missing_prefix}_{index}")
728+
finally:
729+
bootstrapper.teardown()
730+
731+
assert _retained_tool_metric_samples() == samples_before
732+
733+
734+
async def test_fastmcp_prometheus_hashed_proxy_aliases_share_metric_samples() -> None:
735+
remote = FastMCP()
736+
tool_name = f"echo_{uuid.uuid4().hex}"
737+
tool_hash = hash_tool("metrics", tool_name)
738+
739+
def echo() -> str:
740+
return "ok"
741+
742+
remote.tool(echo, name=tool_name, meta={"fastmcp": {"tool_hash": tool_hash}, "ui": {"visibility": ["app"]}})
743+
bootstrapper = FastMcpBootstrapper(
744+
bootstrap_config=_make_test_config(application=create_proxy(remote), logging_enabled=False)
745+
)
746+
application = bootstrapper.bootstrap()
747+
metric_label = hashed_backend_name("metrics", tool_name)
748+
try:
749+
async with Client(application) as client:
750+
await client.call_tool(metric_label)
751+
samples_after_first_call = _retained_tool_metric_samples()
752+
for index in range(100):
753+
result = await client.call_tool(f"{tool_hash}_alias_{index}")
754+
assert result.data == "ok"
755+
assert _retained_tool_metric_samples() == samples_after_first_call
756+
finally:
757+
bootstrapper.teardown()
758+
759+
expected_calls = 101
760+
assert _sample_value("fastmcp_tool_calls_total", {"tool": metric_label, "status": "success"}) == expected_calls
761+
assert _sample_value("fastmcp_tool_call_duration_seconds_count", {"tool": metric_label}) == expected_calls
762+
763+
764+
@pytest.mark.parametrize("tool_source", ["registered", "mounted", "transformed", "hashed"])
765+
async def test_fastmcp_prometheus_counts_tool_calls(tool_source: str) -> None:
706766
bootstrapper = FastMcpBootstrapper(bootstrap_config=_make_test_config())
707767
application = bootstrapper.bootstrap()
708768
echo_name = f"echo_{uuid.uuid4().hex}"
@@ -715,8 +775,32 @@ def failing() -> str:
715775
message = "boom"
716776
raise ValueError(message)
717777

718-
application.tool(echo, name=echo_name)
719-
application.tool(failing, name=failing_name)
778+
tool_server = FastMCP() if tool_source == "mounted" else application
779+
for tool_function, tool_name in ((echo, echo_name), (failing, failing_name)):
780+
meta = (
781+
{"fastmcp": {"tool_hash": hash_tool("metrics", tool_name)}, "ui": {"visibility": ["app"]}}
782+
if tool_source == "hashed"
783+
else None
784+
)
785+
tool_server.tool(tool_function, name=tool_name, meta=meta)
786+
if tool_source == "mounted":
787+
application.mount(tool_server, namespace="mounted")
788+
echo_name = f"mounted_{echo_name}"
789+
failing_name = f"mounted_{failing_name}"
790+
elif tool_source == "transformed":
791+
application.add_transform(
792+
ToolTransform(
793+
{
794+
echo_name: ToolTransformConfig(name=f"renamed_{echo_name}"),
795+
failing_name: ToolTransformConfig(name=f"renamed_{failing_name}"),
796+
}
797+
)
798+
)
799+
echo_name = f"renamed_{echo_name}"
800+
failing_name = f"renamed_{failing_name}"
801+
elif tool_source == "hashed":
802+
echo_name = hashed_backend_name("metrics", echo_name)
803+
failing_name = hashed_backend_name("metrics", failing_name)
720804
try:
721805
async with Client(application) as client:
722806
await client.call_tool(echo_name, {"text": "hi"})
@@ -729,7 +813,10 @@ def failing() -> str:
729813
expected_successes = 2
730814
assert _sample_value("fastmcp_tool_calls_total", {"tool": echo_name, "status": "success"}) == expected_successes
731815
assert _sample_value("fastmcp_tool_calls_total", {"tool": failing_name, "status": "error"}) == 1
816+
assert _sample_value("fastmcp_tool_calls_total", {"tool": echo_name, "status": "error"}) == 0
817+
assert _sample_value("fastmcp_tool_calls_total", {"tool": failing_name, "status": "success"}) == 0
732818
assert _sample_value("fastmcp_tool_call_duration_seconds_count", {"tool": echo_name}) == expected_successes
819+
assert _sample_value("fastmcp_tool_call_duration_seconds_count", {"tool": failing_name}) == 1
733820

734821

735822
def test_fastmcp_prometheus_tool_metrics_can_be_disabled() -> None:

0 commit comments

Comments
 (0)