Skip to content

Commit ad85806

Browse files
fix(server): record executor failures through the event consumer (a2aproject#1366)
# Description When an executor enqueues its `Task` and raises before that Task is saved, `_run_producer` and the consumer both insert the same row through the shared `TaskManager`. With `DatabaseTaskStore` one insert fails, and the request either returns the `SUBMITTED` task as a success or raises `IntegrityError` instead of the executor's error. This moves the failure write to `EventConsumer`, as suggested on the issue: - `_run_producer` no longer saves the failure. It only enqueues the exception, as before. - The consumer records the failure when it dequeues the exception, so FAILED is written after every event the executor enqueued before raising. - `_write_failed_status` creates the fallback task through `ensure_task_id`, with the context id from `_task_manager.context_id`, only when no task exists, which keeps the case a2aproject#1106 fixed. FAILED is stamped with the failure time, so `last_updated` is set. The write goes through `_handle_task_modification_event` rather than `_handle_task_event`, so it still lands when the agent answered with a Message. Visible differences from the old producer write: - FAILED carries the time of the failure. The old write kept the previous status time, or none, which left `last_updated` NULL. - The agent's last status message moves from `status.message` into `history`, as on any status update. - When a resumed task fails, the follow-up user message is now recorded in `history`. The old write dropped it. Three tests are added. One holds the store's insert of the agent's Task until the executor has raised, which is the window a database round trip leaves open. It fails on `main` with `UNIQUE constraint failed: tasks.id` and passes with this change. The other two check that a task which fails after reporting progress is stamped with the failure time, and that a follow-up answered with a Message still leaves the task FAILED. Checks on current `main` (79a6f45), Python 3.10: the full suite passes with Postgres 18 behind the database tests (2396 passed, 92 skipped; the 4 cross-version tests timed out starting their servers in that run and pass on their own). I did not run the MySQL tests. `ruff check`, `ruff format --check` and `ty check` show nothing new. - [x] Follow the [`CONTRIBUTING` Guide](https://github.com/a2aproject/a2a-python/blob/main/CONTRIBUTING.md). - [x] Make your Pull Request title in the <https://www.conventionalcommits.org/> specification. - [x] Ensure the tests and linter pass - [x] Appropriate docs were updated (if necessary) Fixes a2aproject#1351 --------- Co-authored-by: Mateusz Iwaniuk <45974414+Iwaniukooo11@users.noreply.github.com>
1 parent 30b6d50 commit ad85806

3 files changed

Lines changed: 234 additions & 19 deletions

File tree

‎src/a2a/server/agent_execution/active_task.py‎

Lines changed: 43 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
import logging
4141
import uuid
4242

43+
from datetime import datetime, timezone
4344
from typing import TYPE_CHECKING, Any, cast
4445

4546
from a2a.server.agent_execution.context import RequestContext
@@ -115,6 +116,9 @@ def __init__(self, active_task: ActiveTask) -> None:
115116
self.active_task = active_task
116117
self.task_mode: bool | None = None
117118
self.message_to_save: Message | None = None
119+
# Set when the producer failed, so a request that failed before any
120+
# task was created still leaves a FAILED task behind.
121+
self.producer_failed = False
118122

119123
async def run(self) -> None:
120124
"""Consumes events from the agent and updates system state."""
@@ -160,16 +164,45 @@ async def run(self) -> None:
160164
)
161165

162166
async def _write_failed_status(self) -> Task | None:
163-
task = await self.active_task._task_manager.get_task()
167+
"""Records FAILED once a failure ends the event stream.
168+
169+
After a producer failure this runs once every event the executor
170+
enqueued before raising has been processed, so the write lands in
171+
order. There are three cases:
172+
173+
- The request failed before a task was created: create one as FAILED.
174+
- No task exists and the failure is the consumer's own, or the task
175+
already reached a terminal state: nothing is written.
176+
- Otherwise: the existing task is marked FAILED.
177+
"""
178+
task_manager = self.active_task._task_manager
179+
task = await task_manager.get_task()
180+
if task is None and self.producer_failed:
181+
task = await task_manager.ensure_task_id(
182+
self.active_task._task_id, task_manager.context_id or ''
183+
)
184+
if task.status.state not in TERMINAL_TASK_STATES:
185+
task.status.state = TaskState.TASK_STATE_FAILED
186+
task.status.timestamp.FromDatetime(datetime.now(timezone.utc))
187+
await task_manager.save_task_event(task)
188+
self.active_task._task_created.set()
189+
return task
164190
if not task or task.status.state in TERMINAL_TASK_STATES:
165191
return None
192+
# Stamped with the failure time, which DatabaseTaskStore also uses for
193+
# last_updated.
194+
status = TaskStatus(state=TaskState.TASK_STATE_FAILED)
195+
status.timestamp.FromDatetime(datetime.now(timezone.utc))
166196
handled_event = TaskStatusUpdateEvent(
167197
task_id=task.id,
168198
context_id=task.context_id,
169-
status=TaskStatus(state=TaskState.TASK_STATE_FAILED),
199+
status=status,
170200
)
171201
try:
172-
return await self._handle_task_event(handled_event)
202+
# Not _handle_task_event: that validates what the agent sent, and
203+
# this write must land even after the agent replied with a Message.
204+
await self._handle_task_modification_event(handled_event)
205+
return await task_manager.get_task()
173206
except ConcurrentTaskModificationError:
174207
# Another writer advanced the task; keep its state.
175208
self.active_task._task_manager.invalidate()
@@ -219,6 +252,8 @@ async def _process_event_inner(self, event: Event) -> None:
219252
)
220253
self.message_to_save = event.request_context.message
221254
elif isinstance(event, BaseException):
255+
# Only the producer enqueues exceptions, when a request fails.
256+
self.producer_failed = True
222257
raise event
223258
elif isinstance(event, Message):
224259
self._handle_message_event(event)
@@ -599,7 +634,6 @@ async def _run_producer(self) -> None:
599634
Runs as a detached asyncio.Task. Safe to cancel.
600635
"""
601636
logger.debug('Producer[%s]: Started', self._task_id)
602-
request_context = None
603637
try:
604638
while True:
605639
(
@@ -667,17 +701,8 @@ async def _run_producer(self) -> None:
667701
'Producer[%s]: Execution failed',
668702
self._task_id,
669703
)
670-
# Persist the failure directly instead of relying on the closing
671-
# event queue to carry a final status update.
672-
if request_context:
673-
task = await self._task_manager.ensure_task_id(
674-
self._task_id,
675-
request_context.context_id or '',
676-
)
677-
if task.status.state not in TERMINAL_TASK_STATES:
678-
task.status.state = TaskState.TASK_STATE_FAILED
679-
await self._task_manager.save_task_event(task)
680-
self._task_created.set()
704+
# The consumer writes FAILED: writing here would race its save of
705+
# an event the executor enqueued just before raising.
681706
await self._event_queue_agent.enqueue_event(cast('Event', e))
682707

683708
finally:
@@ -907,10 +932,9 @@ async def cancel(self, call_context: ServerCallContext) -> Task:
907932
# returns the shared _task_manager._current_task, which may already
908933
# have been yielded to a subscriber. Copying keeps THIS terminal
909934
# write off the yielded reference. It is not a file-wide guarantee:
910-
# ordinary status writes (save_task_event -> ensure_task -> CopyFrom,
911-
# and the producer-failure write above) still update the shared
912-
# object in place. Making every write copy-on-yield is the general
913-
# property tracked in #1175 and #1191.
935+
# ordinary status writes (save_task_event -> ensure_task -> CopyFrom)
936+
# still update the shared object in place. Making every write
937+
# copy-on-yield is the general property tracked in #1175 and #1191.
914938
updated = Task()
915939
updated.CopyFrom(task)
916940
updated.status.state = TaskState.TASK_STATE_CANCELED

‎tests/server/agent_execution/test_active_task.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ def task_manager(self) -> Mock:
4242
tm.get_task = AsyncMock(return_value=None)
4343
tm.context_id = 'test-context-id'
4444
tm._init_task_obj = Mock(return_value=Task(id='test-task-id'))
45+
tm.ensure_task_id = AsyncMock(return_value=Task(id='test-task-id'))
4546
tm.save_task_event = AsyncMock()
4647
return tm
4748

‎tests/server/request_handlers/test_default_request_handler_v2.py‎

Lines changed: 190 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import uuid
66
import warnings
77

8+
from datetime import datetime, timezone
89
from unittest.mock import AsyncMock, MagicMock, patch
910

1011
import httpx
@@ -390,6 +391,69 @@ async def cancel(self, context: RequestContext, event_queue: EventQueue):
390391
pass
391392

392393

394+
class FailsAfterTaskAgentExecutor(AgentExecutor):
395+
"""Enqueues a Task with its own metadata, then raises when allowed to."""
396+
397+
def __init__(self) -> None:
398+
self.may_raise = asyncio.Event()
399+
self.task_id: str | None = None
400+
401+
async def execute(self, context: RequestContext, event_queue: EventQueue):
402+
assert context.message is not None
403+
task = new_task_from_user_message(context.message)
404+
task.metadata.update({'agent_key': 'kept'})
405+
self.task_id = task.id
406+
await event_queue.enqueue_event(task)
407+
# Bounded, so the test fails rather than hangs if the hook never fires.
408+
await asyncio.wait_for(self.may_raise.wait(), timeout=5)
409+
raise RuntimeError('model call failed')
410+
411+
async def cancel(self, context: RequestContext, event_queue: EventQueue):
412+
pass
413+
414+
415+
class WorksThenFailsAgentExecutor(AgentExecutor):
416+
"""Reports WORKING with a status message, then raises."""
417+
418+
def __init__(self) -> None:
419+
self.task_id: str | None = None
420+
self.raised_at: datetime | None = None
421+
422+
async def execute(self, context: RequestContext, event_queue: EventQueue):
423+
assert context.message is not None
424+
task = new_task_from_user_message(context.message)
425+
self.task_id = task.id
426+
await event_queue.enqueue_event(task)
427+
task_updater = TaskUpdater(event_queue, task.id, task.context_id)
428+
await task_updater.update_status(
429+
TaskState.TASK_STATE_WORKING,
430+
message=task_updater.new_agent_message([Part(text='halfway')]),
431+
)
432+
# Fail only after the WORKING status, with its timestamp, is stored.
433+
await asyncio.sleep(0.05)
434+
self.raised_at = datetime.now(timezone.utc)
435+
raise RuntimeError('model call failed')
436+
437+
async def cancel(self, context: RequestContext, event_queue: EventQueue):
438+
pass
439+
440+
441+
class RepliesThenFailsAgentExecutor(AgentExecutor):
442+
"""Answers a follow-up with a Message, then raises."""
443+
444+
async def execute(self, context: RequestContext, event_queue: EventQueue):
445+
task_updater = TaskUpdater(
446+
event_queue, context.task_id, context.context_id
447+
)
448+
await event_queue.enqueue_event(
449+
task_updater.new_agent_message([Part(text='on it')])
450+
)
451+
raise RuntimeError('model call failed')
452+
453+
async def cancel(self, context: RequestContext, event_queue: EventQueue):
454+
pass
455+
456+
393457
class FailedStatusAgentExecutor(AgentExecutor):
394458
async def execute(
395459
self, context: RequestContext, event_queue: EventQueue
@@ -1599,6 +1663,132 @@ async def save_task_and_signal_terminal_state(self, task):
15991663
assert stored_task.status.state == terminal_state
16001664

16011665

1666+
@pytest.mark.asyncio
1667+
async def test_on_message_send_failure_right_after_task_is_recorded_on_that_task(
1668+
tmp_path,
1669+
):
1670+
"""The failure lands on the agent's Task, even while that Task is saving.
1671+
1672+
The executor raises while the store's save of its Task has looked the
1673+
task up but not yet inserted it: the window a database round trip
1674+
leaves open.
1675+
"""
1676+
pytest.importorskip('sqlalchemy')
1677+
pytest.importorskip('aiosqlite')
1678+
from a2a.server.tasks.database_task_store import DatabaseTaskStore
1679+
from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine
1680+
1681+
engine = create_async_engine(f'sqlite+aiosqlite:///{tmp_path / "tasks.db"}')
1682+
task_store = DatabaseTaskStore(engine)
1683+
agent_executor = FailsAfterTaskAgentExecutor()
1684+
request_handler = DefaultRequestHandlerV2(
1685+
agent_executor=agent_executor,
1686+
task_store=task_store,
1687+
agent_card=create_default_agent_card(),
1688+
)
1689+
params = SendMessageRequest(
1690+
message=Message(
1691+
role=Role.ROLE_USER,
1692+
message_id='msg_failure_right_after_task',
1693+
parts=[Part(text='Hi')],
1694+
)
1695+
)
1696+
context = create_server_call_context()
1697+
original_merge = AsyncSession.merge
1698+
held = False
1699+
1700+
async def hold_agent_task_insert(self, instance, **kwargs):
1701+
nonlocal held
1702+
merged = await original_merge(self, instance, **kwargs)
1703+
if not held and 'agent_key' in (
1704+
getattr(instance, 'task_metadata', None) or {}
1705+
):
1706+
held = True
1707+
agent_executor.may_raise.set()
1708+
# The failure path runs before this insert is flushed.
1709+
await asyncio.sleep(0.2)
1710+
return merged
1711+
1712+
try:
1713+
with patch.object(AsyncSession, 'merge', hold_agent_task_insert):
1714+
with pytest.raises(RuntimeError, match='model call failed'):
1715+
await request_handler.on_message_send(params, context)
1716+
assert agent_executor.task_id is not None
1717+
stored_task = await task_store.get(agent_executor.task_id, context)
1718+
finally:
1719+
await engine.dispose()
1720+
1721+
assert held
1722+
assert stored_task is not None
1723+
assert stored_task.status.state == TaskState.TASK_STATE_FAILED
1724+
assert stored_task.metadata['agent_key'] == 'kept'
1725+
1726+
1727+
@pytest.mark.asyncio
1728+
async def test_on_message_send_failure_after_work_stamps_failure_time():
1729+
"""A task failing after it reported progress is stamped when it failed."""
1730+
task_store = InMemoryTaskStore()
1731+
agent_executor = WorksThenFailsAgentExecutor()
1732+
request_handler = DefaultRequestHandlerV2(
1733+
agent_executor=agent_executor,
1734+
task_store=task_store,
1735+
agent_card=create_default_agent_card(),
1736+
)
1737+
params = SendMessageRequest(
1738+
message=Message(
1739+
role=Role.ROLE_USER,
1740+
message_id='msg_failure_after_work',
1741+
parts=[Part(text='Hi')],
1742+
)
1743+
)
1744+
context = create_server_call_context()
1745+
1746+
with pytest.raises(RuntimeError, match='model call failed'):
1747+
await request_handler.on_message_send(params, context)
1748+
1749+
assert agent_executor.task_id is not None
1750+
stored_task = await task_store.get(agent_executor.task_id, context)
1751+
assert stored_task is not None
1752+
assert stored_task.status.state == TaskState.TASK_STATE_FAILED
1753+
assert agent_executor.raised_at is not None
1754+
failed_at = stored_task.status.timestamp.ToDatetime(tzinfo=timezone.utc)
1755+
assert failed_at >= agent_executor.raised_at
1756+
1757+
1758+
@pytest.mark.asyncio
1759+
async def test_on_message_send_failure_after_message_reply_marks_task_failed():
1760+
"""A follow-up answered with a Message still fails the task on a raise."""
1761+
task_store = InMemoryTaskStore()
1762+
context = create_server_call_context()
1763+
task = Task(
1764+
id='task_awaiting_input',
1765+
context_id='ctx_awaiting_input',
1766+
status=TaskStatus(state=TaskState.TASK_STATE_INPUT_REQUIRED),
1767+
)
1768+
await task_store.save(task, context)
1769+
request_handler = DefaultRequestHandlerV2(
1770+
agent_executor=RepliesThenFailsAgentExecutor(),
1771+
task_store=task_store,
1772+
agent_card=create_default_agent_card(),
1773+
)
1774+
params = SendMessageRequest(
1775+
message=Message(
1776+
role=Role.ROLE_USER,
1777+
message_id='msg_follow_up',
1778+
parts=[Part(text='large')],
1779+
task_id=task.id,
1780+
context_id=task.context_id,
1781+
)
1782+
)
1783+
1784+
with pytest.raises(RuntimeError, match='model call failed'):
1785+
await request_handler.on_message_send(params, context)
1786+
1787+
stored_task = await task_store.get(task.id, context)
1788+
assert stored_task is not None
1789+
assert stored_task.status.state == TaskState.TASK_STATE_FAILED
1790+
1791+
16021792
@pytest.mark.asyncio
16031793
async def test_on_message_send_failed_task_does_not_hide_producer_exception() -> (
16041794
None

0 commit comments

Comments
 (0)