From 69f48e104ec4e560aacd6a5631a8a64133ae23d2 Mon Sep 17 00:00:00 2001 From: andy Date: Fri, 16 Oct 2026 22:02:34 +0800 Subject: [PATCH 1/2] fix(server): always stamp status.timestamp on terminal task states Framework failure paths could persist a FAILED task without status.timestamp: the consumer failure path built a TaskStatusUpdateEvent with an unstamped TaskStatus (which TaskManager.save_task_event then CopyFrom'd over the task, clearing any prior timestamp), and the producer failure path mutated task.status.state directly before saving. DatabaseTaskStore derives last_updated from status.timestamp, so the task was persisted with last_updated = NULL, breaking ListTasks ordering and time-based filters (issue #1331). Per the approach endorsed on the issue, this lands both layers: - failure sites in ActiveTask (_write_failed_status, _mark_task_as_failed, and the producer failure path) stamp the current time explicitly; - TaskManager.save_task_event gains a defensive backstop that auto-populates the timestamp whenever a terminal state arrives unstamped, covering raw Task and status-event paths alike. Non-terminal states are left untouched, and explicitly provided timestamps are never overwritten. Signed-off-by: andy --- src/a2a/server/agent_execution/active_task.py | 4 + .../default_request_handler_v2.py | 4 + src/a2a/server/tasks/task_manager.py | 17 +++ .../test_default_request_handler_v2.py | 49 +++++++++ tests/server/tasks/test_task_manager.py | 103 ++++++++++++++++++ 5 files changed, 177 insertions(+) diff --git a/src/a2a/server/agent_execution/active_task.py b/src/a2a/server/agent_execution/active_task.py index 7016cd169..46819118c 100644 --- a/src/a2a/server/agent_execution/active_task.py +++ b/src/a2a/server/agent_execution/active_task.py @@ -979,6 +979,9 @@ async def cancel( updated = Task() updated.CopyFrom(task) updated.status.state = TaskState.TASK_STATE_CANCELED + # CopyFrom carried over the previous state's timestamp; a terminal + # transition has to be stamped with its own time. + updated.status.timestamp.FromDatetime(datetime.now(timezone.utc)) await self._task_manager.save_task_event(updated) task = updated return task @@ -1113,6 +1116,7 @@ async def _mark_task_as_failed(self, exception: Exception) -> Task | None: context_id=task.context_id, status=TaskStatus( state=TaskState.TASK_STATE_FAILED, + timestamp=datetime.now(timezone.utc), ), ) ) diff --git a/src/a2a/server/request_handlers/default_request_handler_v2.py b/src/a2a/server/request_handlers/default_request_handler_v2.py index 621d8c976..7c7d84aa9 100644 --- a/src/a2a/server/request_handlers/default_request_handler_v2.py +++ b/src/a2a/server/request_handlers/default_request_handler_v2.py @@ -5,6 +5,7 @@ import warnings from contextlib import aclosing +from datetime import datetime, timezone from typing import TYPE_CHECKING, Any, cast from a2a.plugins.manager import get_global @@ -354,6 +355,9 @@ async def _write_cancel( cancelled = Task() cancelled.CopyFrom(current) cancelled.status.state = TaskState.TASK_STATE_CANCELED + # CopyFrom carried over the previous state's timestamp; stamping here + # also covers the event built from this status below. + cancelled.status.timestamp.FromDatetime(datetime.now(timezone.utc)) event = TaskStatusUpdateEvent( task_id=task_id, context_id=current.context_id, diff --git a/src/a2a/server/tasks/task_manager.py b/src/a2a/server/tasks/task_manager.py index 78722e570..7ca83c2de 100644 --- a/src/a2a/server/tasks/task_manager.py +++ b/src/a2a/server/tasks/task_manager.py @@ -2,6 +2,7 @@ import logging +from datetime import datetime, timezone from typing import TYPE_CHECKING from a2a.plugins.types import TaskEventInfo @@ -14,6 +15,7 @@ TaskStatus, TaskStatusUpdateEvent, ) +from a2a.utils.constants import TERMINAL_TASK_STATES from a2a.utils.errors import InvalidAgentResponseError, InvalidParamsError from a2a.utils.telemetry import trace_function @@ -220,6 +222,16 @@ async def get_task(self) -> Task | None: logger.debug('Task %s not found.', self.task_id) return self._current_task + def _stamp_terminal_timestamp( + self, holder: Task | TaskStatusUpdateEvent + ) -> None: + """Auto-stamps a terminal status that arrives without a timestamp.""" + if ( + holder.status.state in TERMINAL_TASK_STATES + and not holder.status.HasField('timestamp') + ): + holder.status.timestamp.FromDatetime(datetime.now(timezone.utc)) + async def save_task_event( self, event: Task | TaskStatusUpdateEvent | TaskArtifactUpdateEvent ) -> Task | None: @@ -260,6 +272,7 @@ async def save_task_event( task_id_from_event, ) if isinstance(event, Task): + self._stamp_terminal_timestamp(event) self._pending_event = event try: await self._save_task(event) @@ -278,6 +291,10 @@ async def save_task_event( logger.debug( 'Updating task %s status to: %s', task.id, event.status.state ) + # Stamped on the event, not the copy below: the same event object + # is published to subscribers, push notifications and the event + # table, so it has to carry the timestamp too. + self._stamp_terminal_timestamp(event) if task.status.HasField('message'): task.history.append(task.status.message) if event.metadata: diff --git a/tests/server/request_handlers/test_default_request_handler_v2.py b/tests/server/request_handlers/test_default_request_handler_v2.py index 9ac52bc4b..54029f829 100644 --- a/tests/server/request_handlers/test_default_request_handler_v2.py +++ b/tests/server/request_handlers/test_default_request_handler_v2.py @@ -3669,6 +3669,21 @@ async def cancel(self, context: RequestContext, event_queue: EventQueue): pass +class ExplodingExecutor(AgentExecutor): + """Creates the task, then crashes mid-flight (issue #1331).""" + + task_id = None + + async def execute(self, context: RequestContext, event_queue: EventQueue): + task = new_task_from_user_message(context.message) + self.task_id = task.id + await event_queue.enqueue_event(task) + raise RuntimeError('agent crashed mid-flight') + + async def cancel(self, context: RequestContext, event_queue: EventQueue): + pass + + @pytest.mark.asyncio async def test_on_message_send_raises_when_agent_produces_no_events(): """A silent executor must fail fast instead of hanging forever (#1330).""" @@ -3718,3 +3733,37 @@ async def test_on_message_send_stream_raises_when_agent_produces_no_events(): params, create_server_call_context() ): pass + + +@pytest.mark.asyncio +async def test_framework_failure_persists_timestamp_on_failed_task(): + """A framework-written FAILED status carries a timestamp (#1331).""" + task_store = InMemoryTaskStore() + executor = ExplodingExecutor() + request_handler = DefaultRequestHandlerV2( + agent_executor=executor, + task_store=task_store, + agent_card=create_default_agent_card(), + ) + params = SendMessageRequest( + message=Message( + role=Role.ROLE_USER, + message_id='msg_explode', + parts=[Part(text='Hi')], + ), + configuration=SendMessageConfiguration( + accepted_output_modes=['text/plain'] + ), + ) + + with pytest.raises(RuntimeError, match='agent crashed mid-flight'): + await request_handler.on_message_send( + params, create_server_call_context() + ) + + stored = await task_store.get( + executor.task_id, create_server_call_context() + ) + assert stored is not None + assert stored.status.state == TaskState.TASK_STATE_FAILED + assert stored.status.HasField('timestamp') diff --git a/tests/server/tasks/test_task_manager.py b/tests/server/tasks/test_task_manager.py index dc8306d06..e06fa490b 100644 --- a/tests/server/tasks/test_task_manager.py +++ b/tests/server/tasks/test_task_manager.py @@ -1,3 +1,4 @@ +from datetime import datetime, timezone from typing import Any from unittest.mock import AsyncMock @@ -20,6 +21,7 @@ TaskStatusUpdateEvent, ) from a2a.utils.errors import InvalidAgentResponseError, InvalidParamsError +from google.protobuf.timestamp_pb2 import Timestamp class SampleUser(User): @@ -509,3 +511,104 @@ def test_append_artifact_to_task(): match='append=True for nonexistent artifact_id', ): append_artifact_to_task(task, append_event_5) + + +@pytest.mark.asyncio +async def test_save_task_event_stamps_terminal_status_without_timestamp( + task_manager: TaskManager, mock_task_store: AsyncMock +) -> None: + """Terminal states arriving unstamped are auto-stamped (#1331).""" + initial_task = create_minimal_task() + mock_task_store.get.return_value = initial_task + event = TaskStatusUpdateEvent( + task_id=MINIMAL_TASK_ID, + context_id=MINIMAL_CONTEXT_ID, + status=TaskStatus(state=TaskState.TASK_STATE_FAILED), + ) + + await task_manager.save_task_event(event) + + saved_task = mock_task_store.save.call_args[0][0] + assert saved_task.status.state == TaskState.TASK_STATE_FAILED + assert saved_task.status.HasField('timestamp') + + +@pytest.mark.asyncio +async def test_save_task_event_preserves_provided_terminal_timestamp( + task_manager: TaskManager, mock_task_store: AsyncMock +) -> None: + """An explicitly provided timestamp is never overwritten (#1331).""" + initial_task = create_minimal_task() + mock_task_store.get.return_value = initial_task + provided = Timestamp() + provided.FromDatetime(datetime(2026, 1, 1, tzinfo=timezone.utc)) + event = TaskStatusUpdateEvent( + task_id=MINIMAL_TASK_ID, + context_id=MINIMAL_CONTEXT_ID, + status=TaskStatus( + state=TaskState.TASK_STATE_FAILED, timestamp=provided + ), + ) + + await task_manager.save_task_event(event) + + saved_task = mock_task_store.save.call_args[0][0] + assert saved_task.status.timestamp.ToDatetime( + tzinfo=timezone.utc + ) == datetime(2026, 1, 1, tzinfo=timezone.utc) + + +@pytest.mark.asyncio +async def test_save_task_event_does_not_stamp_non_terminal_status( + task_manager: TaskManager, mock_task_store: AsyncMock +) -> None: + """Non-terminal states are left alone by the backstop (#1331).""" + initial_task = create_minimal_task() + mock_task_store.get.return_value = initial_task + event = TaskStatusUpdateEvent( + task_id=MINIMAL_TASK_ID, + context_id=MINIMAL_CONTEXT_ID, + status=TaskStatus(state=TaskState.TASK_STATE_WORKING), + ) + + await task_manager.save_task_event(event) + + saved_task = mock_task_store.save.call_args[0][0] + assert saved_task.status.state == TaskState.TASK_STATE_WORKING + assert not saved_task.status.HasField('timestamp') + + +@pytest.mark.asyncio +async def test_save_task_event_stamps_raw_failed_task_without_timestamp( + task_manager: TaskManager, mock_task_store: AsyncMock +) -> None: + """The raw-Task path (direct mutation failure sites) is guarded too + (#1331).""" + failing_task = create_minimal_task() + failing_task.status.state = TaskState.TASK_STATE_FAILED + + returned = await task_manager.save_task_event(failing_task) + + assert returned.status.state == TaskState.TASK_STATE_FAILED + assert returned.status.HasField('timestamp') + + +@pytest.mark.asyncio +async def test_save_task_event_stamps_the_published_event( + task_manager: TaskManager, mock_task_store: AsyncMock +) -> None: + """The caller's event object is stamped too, since the same instance is + published to subscribers, push notifications and the event table.""" + initial_task = create_minimal_task() + mock_task_store.get.return_value = initial_task + event = TaskStatusUpdateEvent( + task_id=MINIMAL_TASK_ID, + context_id=MINIMAL_CONTEXT_ID, + status=TaskStatus(state=TaskState.TASK_STATE_COMPLETED), + ) + + await task_manager.save_task_event(event) + + assert event.status.HasField('timestamp') + saved_task = mock_task_store.save.call_args[0][0] + assert saved_task.status.timestamp == event.status.timestamp From d01ba16685fa9b6c402b00bd2daf6fd62f0cf61b Mon Sep 17 00:00:00 2001 From: andy Date: Sat, 14 Nov 2026 11:02:50 +0800 Subject: [PATCH 2/2] fix(server): stamp missing task status timestamps --- src/a2a/server/tasks/task_manager.py | 18 +-- .../test_default_request_handler_v2.py | 1 + .../server/tasks/test_database_task_store.py | 36 ++++++ tests/server/tasks/test_task_manager.py | 105 +++++++++++++++--- 4 files changed, 133 insertions(+), 27 deletions(-) diff --git a/src/a2a/server/tasks/task_manager.py b/src/a2a/server/tasks/task_manager.py index 7ca83c2de..9dd8547ac 100644 --- a/src/a2a/server/tasks/task_manager.py +++ b/src/a2a/server/tasks/task_manager.py @@ -15,7 +15,6 @@ TaskStatus, TaskStatusUpdateEvent, ) -from a2a.utils.constants import TERMINAL_TASK_STATES from a2a.utils.errors import InvalidAgentResponseError, InvalidParamsError from a2a.utils.telemetry import trace_function @@ -222,13 +221,12 @@ async def get_task(self) -> Task | None: logger.debug('Task %s not found.', self.task_id) return self._current_task - def _stamp_terminal_timestamp( + def _stamp_status_timestamp( self, holder: Task | TaskStatusUpdateEvent ) -> None: - """Auto-stamps a terminal status that arrives without a timestamp.""" - if ( - holder.status.state in TERMINAL_TASK_STATES - and not holder.status.HasField('timestamp') + """Supply a timestamp for a status when the agent did not provide one.""" + if holder.HasField('status') and not holder.status.HasField( + 'timestamp' ): holder.status.timestamp.FromDatetime(datetime.now(timezone.utc)) @@ -272,7 +270,7 @@ async def save_task_event( task_id_from_event, ) if isinstance(event, Task): - self._stamp_terminal_timestamp(event) + self._stamp_status_timestamp(event) self._pending_event = event try: await self._save_task(event) @@ -294,7 +292,7 @@ async def save_task_event( # Stamped on the event, not the copy below: the same event object # is published to subscribers, push notifications and the event # table, so it has to carry the timestamp too. - self._stamp_terminal_timestamp(event) + self._stamp_status_timestamp(event) if task.status.HasField('message'): task.history.append(task.status.message) if event.metadata: @@ -400,12 +398,14 @@ def _init_task_obj(self, task_id: str, context_id: str) -> Task: context_id, ) history = [self._initial_message] if self._initial_message else [] - return Task( + task = Task( id=task_id, context_id=context_id, status=TaskStatus(state=TaskState.TASK_STATE_SUBMITTED), history=history, ) + self._stamp_status_timestamp(task) + return task async def _save_task(self, task: Task) -> None: """Saves the task and updates the snapshot. diff --git a/tests/server/request_handlers/test_default_request_handler_v2.py b/tests/server/request_handlers/test_default_request_handler_v2.py index 54029f829..ef8f7543d 100644 --- a/tests/server/request_handlers/test_default_request_handler_v2.py +++ b/tests/server/request_handlers/test_default_request_handler_v2.py @@ -2080,6 +2080,7 @@ async def test_on_message_send_non_blocking(): assert result is not None assert isinstance(result, Task) assert result.status.state == TaskState.TASK_STATE_SUBMITTED + assert result.status.HasField('timestamp') @pytest.mark.asyncio diff --git a/tests/server/tasks/test_database_task_store.py b/tests/server/tasks/test_database_task_store.py index d0e1b3dc6..cb56be946 100644 --- a/tests/server/tasks/test_database_task_store.py +++ b/tests/server/tasks/test_database_task_store.py @@ -24,6 +24,7 @@ from a2a.compat.v0_3.model_conversions import core_to_compat_task_model from a2a.server.context import ServerCallContext from a2a.server.models import Base, TaskModel # Important: To get Base.metadata +from a2a.server.tasks import TaskManager from a2a.server.tasks.database_task_store import DatabaseTaskStore from a2a.types.a2a_pb2 import ( Artifact, @@ -33,6 +34,7 @@ Task, TaskState, TaskStatus, + TaskStatusUpdateEvent, ) from a2a.utils.constants import DEFAULT_LIST_TASKS_PAGE_SIZE from a2a.utils.errors import InvalidParamsError @@ -227,6 +229,40 @@ async def test_get_task(db_store_parameterized: DatabaseTaskStore) -> None: ) # Cleanup +@pytest.mark.asyncio +async def test_manager_timestamps_support_database_status_filter( + db_store_parameterized: DatabaseTaskStore, +) -> None: + manager = TaskManager( + task_id='timestamp-filter-task', + context_id='timestamp-filter-context', + task_store=db_store_parameterized, + initial_message=None, + context=TEST_CONTEXT, + ) + event = TaskStatusUpdateEvent( + task_id='timestamp-filter-task', + context_id='timestamp-filter-context', + status=TaskStatus(state=TaskState.TASK_STATE_WORKING), + ) + await manager.save_task_event(event) + assert event.status.HasField('timestamp') + + timestamp = event.status.timestamp.ToDatetime(tzinfo=timezone.utc) + request = ListTasksRequest(status=TaskState.TASK_STATE_WORKING) + request.status_timestamp_after.FromDatetime( + timestamp - timedelta(seconds=1) + ) + listed = await db_store_parameterized.list(request, TEST_CONTEXT) + assert [task.id for task in listed.tasks] == ['timestamp-filter-task'] + + request.status_timestamp_after.FromDatetime( + timestamp + timedelta(seconds=1) + ) + listed = await db_store_parameterized.list(request, TEST_CONTEXT) + assert not listed.tasks + + @pytest.mark.asyncio @pytest.mark.parametrize( 'params, expected_ids, total_count, next_cursor', diff --git a/tests/server/tasks/test_task_manager.py b/tests/server/tasks/test_task_manager.py index e06fa490b..ef1e11463 100644 --- a/tests/server/tasks/test_task_manager.py +++ b/tests/server/tasks/test_task_manager.py @@ -534,8 +534,11 @@ async def test_save_task_event_stamps_terminal_status_without_timestamp( @pytest.mark.asyncio -async def test_save_task_event_preserves_provided_terminal_timestamp( - task_manager: TaskManager, mock_task_store: AsyncMock +@pytest.mark.parametrize( + 'state', [TaskState.TASK_STATE_FAILED, TaskState.TASK_STATE_WORKING] +) +async def test_save_task_event_preserves_provided_timestamp( + task_manager: TaskManager, mock_task_store: AsyncMock, state: int ) -> None: """An explicitly provided timestamp is never overwritten (#1331).""" initial_task = create_minimal_task() @@ -545,9 +548,7 @@ async def test_save_task_event_preserves_provided_terminal_timestamp( event = TaskStatusUpdateEvent( task_id=MINIMAL_TASK_ID, context_id=MINIMAL_CONTEXT_ID, - status=TaskStatus( - state=TaskState.TASK_STATE_FAILED, timestamp=provided - ), + status=TaskStatus(state=state, timestamp=provided), ) await task_manager.save_task_event(event) @@ -559,37 +560,105 @@ async def test_save_task_event_preserves_provided_terminal_timestamp( @pytest.mark.asyncio -async def test_save_task_event_does_not_stamp_non_terminal_status( - task_manager: TaskManager, mock_task_store: AsyncMock +@pytest.mark.parametrize( + 'state', + [ + TaskState.TASK_STATE_WORKING, + TaskState.TASK_STATE_INPUT_REQUIRED, + TaskState.TASK_STATE_AUTH_REQUIRED, + ], +) +async def test_save_task_event_stamps_non_terminal_status( + task_manager: TaskManager, mock_task_store: AsyncMock, state: int ) -> None: - """Non-terminal states are left alone by the backstop (#1331).""" + """Non-terminal status updates receive timestamps too.""" initial_task = create_minimal_task() mock_task_store.get.return_value = initial_task event = TaskStatusUpdateEvent( task_id=MINIMAL_TASK_ID, context_id=MINIMAL_CONTEXT_ID, - status=TaskStatus(state=TaskState.TASK_STATE_WORKING), + status=TaskStatus(state=state), ) await task_manager.save_task_event(event) saved_task = mock_task_store.save.call_args[0][0] - assert saved_task.status.state == TaskState.TASK_STATE_WORKING - assert not saved_task.status.HasField('timestamp') + assert saved_task.status.state == state + assert saved_task.status.HasField('timestamp') + assert event.status.timestamp == saved_task.status.timestamp + + +@pytest.mark.asyncio +async def test_ensure_task_id_stamps_initial_submitted_status( + task_manager: TaskManager, mock_task_store: AsyncMock +) -> None: + mock_task_store.get.return_value = None + + task = await task_manager.ensure_task_id( + MINIMAL_TASK_ID, MINIMAL_CONTEXT_ID + ) + + assert task.status.state == TaskState.TASK_STATE_SUBMITTED + assert task.status.HasField('timestamp') + saved_task = mock_task_store.save.call_args[0][0] + assert saved_task.status.timestamp == task.status.timestamp @pytest.mark.asyncio -async def test_save_task_event_stamps_raw_failed_task_without_timestamp( +async def test_artifact_first_stamps_initial_status( task_manager: TaskManager, mock_task_store: AsyncMock ) -> None: - """The raw-Task path (direct mutation failure sites) is guarded too - (#1331).""" - failing_task = create_minimal_task() - failing_task.status.state = TaskState.TASK_STATE_FAILED + mock_task_store.get.return_value = None + event = TaskArtifactUpdateEvent( + task_id=MINIMAL_TASK_ID, + context_id=MINIMAL_CONTEXT_ID, + artifact=Artifact( + artifact_id='artifact-1', parts=[Part(text='result')] + ), + ) + + task = await task_manager.save_task_event(event) + + assert task is not None + assert task.status.state == TaskState.TASK_STATE_SUBMITTED + assert task.status.HasField('timestamp') + assert ( + mock_task_store.save.call_args[0][0].status.timestamp + == task.status.timestamp + ) + + +@pytest.mark.asyncio +async def test_raw_working_task_preserves_provided_timestamp( + task_manager: TaskManager, +) -> None: + task = create_minimal_task() + task.status.state = TaskState.TASK_STATE_WORKING + task.status.timestamp.FromDatetime( + datetime(2026, 1, 1, tzinfo=timezone.utc) + ) + expected = Timestamp() + expected.CopyFrom(task.status.timestamp) + + await task_manager.save_task_event(task) + + assert task.status.timestamp == expected + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + 'state', [TaskState.TASK_STATE_FAILED, TaskState.TASK_STATE_WORKING] +) +async def test_save_task_event_stamps_raw_task_without_timestamp( + task_manager: TaskManager, mock_task_store: AsyncMock, state: int +) -> None: + """Raw task status updates receive timestamps for any state.""" + task = create_minimal_task() + task.status.state = state - returned = await task_manager.save_task_event(failing_task) + returned = await task_manager.save_task_event(task) - assert returned.status.state == TaskState.TASK_STATE_FAILED + assert returned.status.state == state assert returned.status.HasField('timestamp')