Skip to content

Commit f0e905c

Browse files
Mantisusvdusek
authored andcommitted
feat(autoscaling): Let callers wake AutoscaledPool immediately with notify() (#2386)
### Description `AutoscaledPool.notify()` makes a running pool look for a ready task at once, instead of waiting for its next periodic check every 0.5 s. On a pool that isn't running it does nothing. Nothing in Crawlee calls it yet. A follow-up PR uses it to resume a paced crawl as soon as the request manager reports that work is ready again. *✍️ Drafted by Claude Code*
1 parent 96ecd7d commit f0e905c

2 files changed

Lines changed: 61 additions & 5 deletions

File tree

‎src/crawlee/autoscaling/_autoscaled_pool.py‎

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import asyncio
44
from contextlib import suppress
5+
from datetime import timedelta
56
from functools import partial
67
from logging import getLogger
78
from typing import TYPE_CHECKING
@@ -10,7 +11,6 @@
1011

1112
if TYPE_CHECKING:
1213
from collections.abc import Awaitable, Callable
13-
from datetime import timedelta
1414

1515
from ._types import ConcurrencyConsumer, ConcurrencySystemInterface
1616

@@ -26,7 +26,9 @@ def __init__(self) -> None:
2626
self.worker_tasks = list[asyncio.Task]()
2727
"""A list of worker tasks currently in progress"""
2828

29-
self.worker_tasks_updated = asyncio.Event()
29+
self.wake = asyncio.Event()
30+
"""Set when a worker task finishes or `notify` is called, so that the orchestrator looks for work at once."""
31+
3032
self.cleanup_done = asyncio.Event()
3133
self.result: asyncio.Future = asyncio.Future()
3234

@@ -44,6 +46,9 @@ class AutoscaledPool:
4446
_TASK_TIMEOUT: timedelta | None = None
4547
"""Timeout within which the `run_task_function` must complete."""
4648

49+
_POLL_INTERVAL = timedelta(seconds=0.5)
50+
"""How often the pool looks for a ready task when nothing wakes it up earlier."""
51+
4752
def __init__(
4853
self,
4954
*,
@@ -173,6 +178,12 @@ async def pause(self, *, timeout: timedelta | None = None) -> None:
173178
def resume(self) -> None:
174179
"""Resume a paused autoscaled pool so that it continues starting new tasks."""
175180
self._is_paused = False
181+
self.notify()
182+
183+
def notify(self) -> None:
184+
"""Look for a ready task at once, instead of waiting for the next periodic check."""
185+
if self._current_run is not None:
186+
self._current_run.wake.set()
176187

177188
@property
178189
def concurrency_system(self) -> ConcurrencySystemInterface:
@@ -210,7 +221,7 @@ async def _worker_task_orchestrator(self, run: _AutoscaledPoolRun) -> None:
210221

211222
try:
212223
while not run.result.done():
213-
run.worker_tasks_updated.clear()
224+
run.wake.clear()
214225

215226
if self._is_paused:
216227
logger.debug('Not scheduling new tasks - the autoscaled pool is paused')
@@ -246,7 +257,7 @@ async def _worker_task_orchestrator(self, run: _AutoscaledPoolRun) -> None:
246257
logger.debug('Not scheduling new task - the task was not accepted by the concurrency system')
247258

248259
with suppress(TimeoutError):
249-
await asyncio.wait_for(run.worker_tasks_updated.wait(), timeout=0.5)
260+
await asyncio.wait_for(run.wake.wait(), timeout=self._POLL_INTERVAL.total_seconds())
250261
except Exception as exc:
251262
# Surface the error through `run.result` only once the cleanup below has awaited the worker tasks,
252263
# so that the caller does not observe the failure while tasks are still in flight.
@@ -293,7 +304,7 @@ def _reap_worker_task(self, task: asyncio.Task, *, run: _AutoscaledPoolRun, task
293304
- Notify the orchestrator about the task completion.
294305
- Return the task's slot to the concurrency system.
295306
"""
296-
run.worker_tasks_updated.set()
307+
run.wake.set()
297308
run.worker_tasks.remove(task)
298309

299310
if not task.cancelled() and (exception := task.exception()) and not run.result.done():

‎tests/unit/autoscaling/test_autoscaled_pool.py‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -450,6 +450,51 @@ async def answer_after_pause() -> bool:
450450
assert is_finished.await_count == 2
451451

452452

453+
async def test_notify_wakes_pool(
454+
make_concurrency_system: type[ConcurrencySystem], monkeypatch: pytest.MonkeyPatch
455+
) -> None:
456+
"""`notify` makes the pool look for a ready task at once, not at its next periodic check."""
457+
# Longer than the test's timeout, so only `notify` can wake the pool in time.
458+
monkeypatch.setattr(AutoscaledPool, '_POLL_INTERVAL', timedelta(hours=1))
459+
run = AsyncMock()
460+
is_task_ready = AsyncMock(return_value=False)
461+
462+
async with make_concurrency_system() as concurrency_system:
463+
pool = AutoscaledPool(
464+
concurrency_system=concurrency_system,
465+
consumer=CONSUMER,
466+
run_task_function=run,
467+
is_task_ready_function=is_task_ready,
468+
is_finished_function=lambda: future(run.await_count == 1),
469+
)
470+
471+
pool_run_task = asyncio.create_task(pool.run(), name='pool run task')
472+
try:
473+
assert await poll_until_condition(lambda: is_task_ready.await_count == 1)
474+
475+
is_task_ready.side_effect = chain([True], repeat(False))
476+
pool.notify()
477+
await asyncio.wait_for(pool_run_task, timeout=5)
478+
finally:
479+
pool_run_task.cancel()
480+
await asyncio.gather(pool_run_task, return_exceptions=True)
481+
482+
run.assert_awaited_once()
483+
484+
485+
async def test_notify_without_run() -> None:
486+
"""`notify` on a pool that isn't running has nothing to wake, so it does nothing."""
487+
pool = AutoscaledPool(
488+
concurrency_system=Mock(spec=ConcurrencySystemInterface),
489+
consumer=CONSUMER,
490+
run_task_function=AsyncMock(),
491+
is_task_ready_function=AsyncMock(),
492+
is_finished_function=AsyncMock(),
493+
)
494+
495+
pool.notify()
496+
497+
453498
@pytest.mark.run_alone
454499
async def test_runs_concurrently(make_concurrency_system: type[ConcurrencySystem]) -> None:
455500
"""The pool fills the whole budget the system allows, rather than running the tasks one after another."""

0 commit comments

Comments
 (0)