Skip to content

Commit dcc9ce6

Browse files
authored
Fix Common AI durable retries not replaying tools from a capability without an id (#74314)
* Fix Common AI durable retries not replaying tools from a capability without an id pydantic-ai 2.40+ gives a capability without an explicit id a random one per run and stamps it on each of its tools. The durable fingerprint hashed the whole request parameters, so a retry never matched its cached model response and every step re-ran live, including tools with side effects. The run-local capability ids are now left out of the fingerprint. * Sort revealed tool names in the durable fingerprint and tighten its tests revealed_tool_names is a set and dumped in iteration order, which differs between processes, so a durable retry with two or more revealed tools also missed the cache.
1 parent 77b21f3 commit dcc9ce6

3 files changed

Lines changed: 171 additions & 9 deletions

File tree

‎providers/common/ai/src/airflow/providers/common/ai/durable/fingerprint.py‎

Lines changed: 27 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -32,11 +32,11 @@
3232
longer match.
3333
3434
Fields that pydantic-ai regenerates on every attempt (message-level
35-
``timestamp``/``run_id``/``conversation_id`` and part-level ``timestamp``)
36-
are excluded from the fingerprint. Requests that cannot be serialized to
37-
JSON fingerprint as ``None``, which degrades that step to unverified
38-
positional replay (the pre-fingerprint behavior) rather than disabling
39-
caching.
35+
``timestamp``/``run_id``/``conversation_id``, part-level ``timestamp``) and
36+
capability ids are excluded from the fingerprint, and set-valued request
37+
parameters are sorted. Requests that cannot be serialized to JSON
38+
fingerprint as ``None``, which degrades that step to unverified positional
39+
replay (the pre-fingerprint behavior) rather than disabling caching.
4040
"""
4141

4242
from __future__ import annotations
@@ -102,6 +102,24 @@ def _strip_volatile(messages_dump: list[dict[str, Any]]) -> list[dict[str, Any]]
102102
return stripped
103103

104104

105+
def _normalize_params(params_dump: dict[str, Any]) -> dict[str, Any]:
106+
"""
107+
Drop capability ids and sort set-valued fields from dumped request parameters.
108+
109+
A capability without an explicit ``id`` gets a random one per run (``<toolset:d0d75e>``),
110+
stamped on its tools' ``capability_id``, so hashing it would make every retry miss the
111+
cache. What the model sees of capabilities (the deferred-capability catalog in the
112+
instructions, tool visibility, revealed tool names) is hashed through other fields, so
113+
``deferred_capability_ids`` is dropped too. A set dumps in iteration order, which differs
114+
between processes (``PYTHONHASHSEED``), and a retry runs in a new process.
115+
"""
116+
cleaned = {k: v for k, v in params_dump.items() if k != "deferred_capability_ids"}
117+
cleaned["revealed_tool_names"] = sorted(cleaned["revealed_tool_names"])
118+
for key in ("function_tools", "output_tools"):
119+
cleaned[key] = [{k: v for k, v in tool.items() if k != "capability_id"} for tool in cleaned[key]]
120+
return cleaned
121+
122+
105123
def _digest(payload: Any) -> str:
106124
# No ``default=`` fallback: a non-JSON-serializable value must raise so the
107125
# callers degrade to an unverifiable (None) fingerprint instead of hashing
@@ -119,9 +137,9 @@ def fingerprint_model_request(
119137
"""
120138
Fingerprint a model request: model identity, message history, settings, and request parameters.
121139
122-
The full ``ModelRequestParameters`` object is hashed (tool definitions,
123-
output mode and schema, native tools, ...) so any change to what is sent
124-
to the model invalidates the cached response.
140+
The ``ModelRequestParameters`` object is hashed (tool definitions, output
141+
mode and schema, native tools, ...) so any change to what is sent to the
142+
model invalidates the cached response; only capability ids are left out.
125143
126144
Returns ``None`` when the request cannot be serialized; ``None`` compares
127145
equal to ``None``, so requests that cannot be fingerprinted degrade to
@@ -135,7 +153,7 @@ def fingerprint_model_request(
135153
"model": model_identifier,
136154
"messages": _strip_volatile(dumped),
137155
"settings": _content_settings(model_settings),
138-
"params": params,
156+
"params": _normalize_params(params),
139157
}
140158
)
141159
except (TypeError, ValueError):

‎providers/common/ai/tests/unit/common/ai/durable/test_fingerprint.py‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,17 +19,20 @@
1919
import datetime
2020

2121
import httpx
22+
import pytest
2223
from pydantic_ai.messages import (
2324
ModelRequest,
2425
ModelResponse,
2526
SystemPromptPart,
27+
TextPart,
2628
ToolCallPart,
2729
UserPromptPart,
2830
)
2931
from pydantic_ai.models import ModelRequestParameters
3032
from pydantic_ai.tools import ToolDefinition
3133

3234
from airflow.providers.common.ai.durable.fingerprint import (
35+
_normalize_params,
3336
fingerprint_model_request,
3437
fingerprint_tool_call,
3538
)
@@ -118,6 +121,90 @@ def test_changes_with_tool_definition_fields(self):
118121

119122
assert fp1 != fp2
120123

124+
def test_stable_across_capability_ids(self):
125+
"""A capability without an ``id`` gets a random one per run; it never reaches the model."""
126+
127+
def params(capability_id):
128+
tool = ToolDefinition(
129+
name="t", parameters_json_schema={"type": "object"}, capability_id=capability_id
130+
)
131+
return ModelRequestParameters(function_tools=[tool], output_tools=[tool])
132+
133+
fp1 = fingerprint_model_request("m", make_messages(), None, params("<toolset:d0d75e>"))
134+
fp2 = fingerprint_model_request("m", make_messages(), None, params("<toolset:78ba70>"))
135+
136+
assert fp1 is not None
137+
assert fp1 == fp2
138+
139+
@pytest.mark.parametrize("tools_param", ["function_tools", "output_tools"])
140+
@pytest.mark.parametrize(
141+
"change",
142+
[
143+
pytest.param({"name": "other"}, id="name"),
144+
pytest.param({"description": "other"}, id="description"),
145+
pytest.param(
146+
{"parameters_json_schema": {"type": "object", "properties": {"q": {"type": "string"}}}},
147+
id="schema",
148+
),
149+
],
150+
)
151+
def test_changes_with_tool_content_next_to_the_capability_id(self, tools_param, change):
152+
"""Only ``capability_id`` is dropped from a tool definition; what the model sees still counts."""
153+
base = {"name": "t", "parameters_json_schema": {"type": "object"}, "capability_id": "lookup"}
154+
fp1 = fingerprint_model_request(
155+
"m", make_messages(), None, ModelRequestParameters(**{tools_param: [ToolDefinition(**base)]})
156+
)
157+
fp2 = fingerprint_model_request(
158+
"m",
159+
make_messages(),
160+
None,
161+
ModelRequestParameters(**{tools_param: [ToolDefinition(**{**base, **change})]}),
162+
)
163+
164+
assert fp1 != fp2
165+
166+
def test_changes_with_revealed_tool_names(self):
167+
fp1 = fingerprint_model_request(
168+
"m", make_messages(), None, ModelRequestParameters(revealed_tool_names={"search"})
169+
)
170+
fp2 = fingerprint_model_request(
171+
"m", make_messages(), None, ModelRequestParameters(revealed_tool_names={"search", "fetch"})
172+
)
173+
174+
assert fp1 != fp2
175+
176+
def test_revealed_tool_names_hash_in_any_order(self):
177+
"""A set dumps in iteration order, which differs between processes, and a retry is a new one."""
178+
names = ["search", "fetch", "summarize", "rank"]
179+
dumps = [
180+
{"revealed_tool_names": order, "function_tools": [], "output_tools": []}
181+
for order in (names, list(reversed(names)))
182+
]
183+
184+
assert _normalize_params(dumps[0]) == _normalize_params(dumps[1])
185+
186+
def test_changes_with_message_metadata(self):
187+
"""
188+
Message ``metadata`` is not sent to the model, but pydantic-ai keeps routing state in it
189+
(``FallbackModel``'s continuation pin under ``__pydantic_ai__``), so it stays in the hash.
190+
"""
191+
192+
def messages(pinned_model):
193+
return [
194+
ModelRequest(parts=[UserPromptPart(content="q")]),
195+
ModelResponse(
196+
parts=[TextPart(content="partial")],
197+
metadata={"__pydantic_ai__": {"fallback_model_id": pinned_model}},
198+
),
199+
]
200+
201+
fp1 = fingerprint_model_request("m", messages("openai:gpt-5"), None, ModelRequestParameters())
202+
fp2 = fingerprint_model_request(
203+
"m", messages("anthropic:claude-sonnet-4-5"), None, ModelRequestParameters()
204+
)
205+
206+
assert fp1 != fp2
207+
121208
def test_volatile_keys_inside_user_data_are_not_stripped(self):
122209
"""Only pydantic-ai's own message/part fields are volatile; a tool argument
123210
legitimately named run_id must still affect the fingerprint."""

‎providers/common/ai/tests/unit/common/ai/operators/test_agent.py‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1639,6 +1639,63 @@ def model_fn(messages, info):
16391639

16401640
assert calls["n"] == 1
16411641

1642+
@pytest.mark.parametrize(
1643+
"capability",
1644+
[
1645+
pytest.param(lambda tool: Toolset(FunctionToolset([tool])), id="anonymous"),
1646+
pytest.param(lambda tool: Toolset(FunctionToolset([tool]), id="lookup"), id="with-id"),
1647+
],
1648+
)
1649+
def test_retry_replays_steps_of_a_toolset_capability(self, capability):
1650+
"""
1651+
pydantic-ai gives a capability without an ``id`` a random one per run and stamps it
1652+
on its tools. A retry is a new run, so the model request differs only in that id;
1653+
it must still replay; ``with-id`` is the control. The model issues fresh tool call ids,
1654+
as a real provider does.
1655+
"""
1656+
storage = _InMemoryDurableStorage()
1657+
live = {"model": 0, "tool": 0}
1658+
fail_after_tool = [True]
1659+
1660+
def my_tool() -> str:
1661+
live["tool"] += 1
1662+
return "tool-result"
1663+
1664+
def model_fn(messages, info):
1665+
live["model"] += 1
1666+
if any(isinstance(p, ToolReturnPart) for m in messages for p in m.parts):
1667+
if fail_after_tool[0]:
1668+
fail_after_tool[0] = False
1669+
raise RuntimeError("transient model failure")
1670+
return ModelResponse(parts=[TextPart(content="done")])
1671+
return ModelResponse(parts=[ToolCallPart(tool_name="my_tool", args={})])
1672+
1673+
for try_number in (1, 2):
1674+
live.update(model=0, tool=0)
1675+
op = AgentOperator(
1676+
task_id="t",
1677+
prompt="hi",
1678+
llm_conn_id="c",
1679+
durable=True,
1680+
enable_tool_logging=False,
1681+
capabilities=[capability(my_tool)],
1682+
)
1683+
op.llm_hook = MagicMock(spec=["create_agent"])
1684+
op.llm_hook.create_agent.side_effect = lambda **kw: Agent(FunctionModel(model_fn), **kw)
1685+
context = _make_context(ti=_make_ti(id=f"ti-{try_number}", try_number=try_number))
1686+
with (
1687+
patch.object(AgentOperator, "_build_durable_storage", autospec=True, return_value=storage),
1688+
pytest.raises(RuntimeError, match="transient") if try_number == 1 else nullcontext(),
1689+
):
1690+
op.execute(context=context)
1691+
if try_number == 1:
1692+
# Verified replay, not positional replay of unfingerprintable (None) steps.
1693+
assert storage.models
1694+
assert all(fingerprint is not None for _, fingerprint in storage.models.values())
1695+
1696+
# Attempt 2 replays model step 0 and the tool call; only the step that failed runs live.
1697+
assert live == {"model": 1, "tool": 0}
1698+
16421699
def test_tool_result_refused_by_storage_is_counted_skipped_and_reruns(self):
16431700
"""A tool result the backend refuses to store is not counted as cached, and a
16441701
retry runs the tool again instead of replaying it."""

0 commit comments

Comments
 (0)