Skip to content

Commit eb0323d

Browse files
authored
refactor: adopt asyncio.TaskGroup for structured concurrency (#909)
Adopts `asyncio.TaskGroup` (Python 3.11+) in four places: - **`ApifyRequestList._fetch_requests_from_url`** — structured task scope for concurrent remote-list fetches; errors aggregate as `ExceptionGroup`. - **`Actor.reboot` pre-reboot dispatch** — `asyncio.timeout(...)` + `TaskGroup`, each listener wrapped in `safe_dispatch` so one failure no longer aborts siblings (preserves `test_reboot_runs_all_listeners_even_when_one_fails`). - **`Actor._save_actor_state`** — per-store persists fan out instead of running sequentially; per-task try/except keeps cleanup best-effort. - **`ApifyFileSystemKeyValueStoreClient.purge`** — `unlink` calls fan out instead of running sequentially. Closes #765
1 parent f4304e8 commit eb0323d

4 files changed

Lines changed: 35 additions & 25 deletions

File tree

‎docs/04_upgrading/upgrading_to_v4.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,4 +8,4 @@ This guide lists the breaking changes between Apify Python SDK v3.x and v4.0.
88

99
## Python 3.11+ required
1010

11-
Support for Python 3.10 has been dropped. The Apify Python SDK v4.x now requires Python 3.11 or later. Make sure your environment is on a compatible version before upgrading.
11+
Support for Python 3.10 has been dropped. The Apify Python SDK v4.x now requires Python 3.11 or later — make sure your environment is on a compatible version before upgrading.

‎src/apify/_actor.py‎

Lines changed: 22 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1231,22 +1231,21 @@ async def reboot(
12311231
(self.event_manager._listeners_to_wrappers[Event.MIGRATING] or {}).values() # noqa: SLF001
12321232
)
12331233

1234+
async def safe_dispatch(listener: Any, data: Any) -> None:
1235+
try:
1236+
await listener(data)
1237+
except Exception:
1238+
self.log.exception('A pre-reboot event listener failed')
1239+
1240+
timeout = event_listeners_timeout.total_seconds() if event_listeners_timeout else None
12341241
try:
1235-
results = await asyncio.wait_for(
1236-
asyncio.gather(
1237-
*[listener(EventPersistStateData(is_migrating=True)) for listener in persist_state_listeners],
1238-
*[listener(EventMigratingData()) for listener in migrating_listeners],
1239-
return_exceptions=True,
1240-
),
1241-
timeout=event_listeners_timeout.total_seconds() if event_listeners_timeout else None,
1242-
)
1242+
async with asyncio.timeout(timeout), asyncio.TaskGroup() as tg:
1243+
for listener in persist_state_listeners:
1244+
tg.create_task(safe_dispatch(listener, EventPersistStateData(is_migrating=True)))
1245+
for listener in migrating_listeners:
1246+
tg.create_task(safe_dispatch(listener, EventMigratingData()))
12431247
except TimeoutError:
12441248
self.log.warning('Pre-reboot event listeners did not finish within timeout; proceeding with reboot')
1245-
results = []
1246-
1247-
for result in results:
1248-
if isinstance(result, Exception):
1249-
self.log.exception('A pre-reboot event listener failed', exc_info=result)
12501249

12511250
if not self.configuration.actor_run_id:
12521251
raise RuntimeError('actor_run_id cannot be None when running on the Apify platform.')
@@ -1419,9 +1418,16 @@ async def use_state(
14191418
return await kvs.get_auto_saved_value(key or self._ACTOR_STATE_KEY, default_value)
14201419

14211420
async def _save_actor_state(self) -> None:
1422-
for kvs_name in self._use_state_stores:
1423-
store = await self.open_key_value_store(name=kvs_name)
1424-
await store.persist_autosaved_values()
1421+
async def safe_persist(kvs_name: str | None) -> None:
1422+
try:
1423+
store = await self.open_key_value_store(name=kvs_name)
1424+
await store.persist_autosaved_values()
1425+
except Exception:
1426+
self.log.exception('Failed to persist auto-saved values', extra={'kvs_name': kvs_name})
1427+
1428+
async with asyncio.TaskGroup() as tg:
1429+
for kvs_name in self._use_state_stores:
1430+
tg.create_task(safe_persist(kvs_name))
14251431

14261432
def _get_default_exit_process(self) -> bool:
14271433
"""Return False for IPython and Scrapy environments, True otherwise."""

‎src/apify/request_loaders/_apify_request_list.py‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -117,9 +117,12 @@ async def _fetch_requests_from_url(
117117
are extracted from the response body and turned into `Request` objects, inheriting `method`, `payload`,
118118
`headers`, and `user_data` from the source entry.
119119
"""
120-
tasks = [cls._process_remote_url(request_input, http_client) for request_input in remote_url_requests_inputs]
121-
results = await asyncio.gather(*tasks)
122-
return list(chain.from_iterable(results))
120+
async with asyncio.TaskGroup() as tg:
121+
tasks = [
122+
tg.create_task(cls._process_remote_url(request_input, http_client))
123+
for request_input in remote_url_requests_inputs
124+
]
125+
return list(chain.from_iterable(task.result() for task in tasks))
123126

124127
@staticmethod
125128
def _create_requests_from_input(simple_url_inputs: list[_SimpleUrlInput]) -> list[Request]:

‎src/apify/storage_clients/_file_system/_key_value_store_client.py‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -75,11 +75,12 @@ async def purge(self) -> None:
7575
files_to_keep = {self._input_key_filename, f'{self._input_key_filename}.{METADATA_FILENAME}'}
7676
files_to_keep.add(METADATA_FILENAME)
7777

78-
for file_path in self.path_to_kvs.glob('*'):
79-
if file_path.name in files_to_keep:
80-
continue
81-
if file_path.is_file():
82-
await asyncio.to_thread(file_path.unlink, missing_ok=True)
78+
async with asyncio.TaskGroup() as tg:
79+
for file_path in self.path_to_kvs.glob('*'):
80+
if file_path.name in files_to_keep:
81+
continue
82+
if file_path.is_file():
83+
tg.create_task(asyncio.to_thread(file_path.unlink, missing_ok=True))
8384

8485
await self._update_metadata(
8586
update_accessed_at=True,

0 commit comments

Comments
 (0)