Skip to content

Commit dbec1f5

Browse files
authored
fix: Pause the crawl on Ctrl+C on Windows instead of crashing (#2277)
### Description - On Windows, Ctrl+C during `crawler.run()` now pauses the crawl the same way as on Linux and macOS. The crawler logs the pause message, lets the requests in progress finish, starts no new ones, and logs how to resume the crawl. ### Issues - Closes: #228
1 parent 8598dd6 commit dbec1f5

2 files changed

Lines changed: 43 additions & 6 deletions

File tree

‎src/crawlee/crawlers/_basic/_basic_crawler.py‎

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
import traceback
1212
from asyncio import CancelledError
1313
from collections.abc import AsyncGenerator, Awaitable, Callable, Iterable, Sequence
14-
from contextlib import AsyncExitStack, suppress
14+
from contextlib import AsyncExitStack
1515
from datetime import timedelta
1616
from functools import partial
1717
from http import HTTPStatus
@@ -763,19 +763,29 @@ def sigint_handler() -> None:
763763

764764
run_task = asyncio.create_task(self._run_crawler(), name='run_crawler_task')
765765

766+
loop = asyncio.get_running_loop()
767+
remove_sigint_handler: Callable[[], object] | None = None
768+
766769
# `add_signal_handler` works only in the main thread
767770
if threading.current_thread() is threading.main_thread():
768-
with suppress(NotImplementedError): # event loop signal handlers are not supported on Windows
769-
asyncio.get_running_loop().add_signal_handler(signal.SIGINT, sigint_handler)
771+
try:
772+
loop.add_signal_handler(signal.SIGINT, sigint_handler)
773+
remove_sigint_handler = partial(loop.remove_signal_handler, signal.SIGINT)
774+
except NotImplementedError:
775+
# Windows event loops, such as the default `ProactorEventLoop`, don't support `add_signal_handler`.
776+
# Like `asyncio.Runner`, set the handler with `signal.signal` and pass the call to the event loop.
777+
previous_sigint_handler = signal.signal(
778+
signal.SIGINT, lambda *_: loop.call_soon_threadsafe(sigint_handler)
779+
)
780+
remove_sigint_handler = partial(signal.signal, signal.SIGINT, previous_sigint_handler)
770781

771782
try:
772783
await run_task
773784
except CancelledError:
774785
pass
775786
finally:
776-
if threading.current_thread() is threading.main_thread():
777-
with suppress(NotImplementedError):
778-
asyncio.get_running_loop().remove_signal_handler(signal.SIGINT)
787+
if remove_sigint_handler is not None:
788+
remove_sigint_handler()
779789

780790
if self._statistics.error_tracker.total > 0:
781791
self._logger.info(

‎tests/unit/crawlers/_basic/test_basic_crawler.py‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
import logging
77
import os
88
import re
9+
import signal
910
import sys
1011
import time
1112
from asyncio import Future
@@ -2981,3 +2982,29 @@ async def handler(context: BasicCrawlingContext) -> None:
29812982

29822983
assert outcomes == []
29832984
assert stats.retry_histogram == expected_histogram
2985+
2986+
2987+
async def test_sigint_without_loop_signal_handlers(caplog: pytest.LogCaptureFixture) -> None:
2988+
"""SIGINT pauses the crawl on event loops without `add_signal_handler`, as on Windows."""
2989+
crawler = BasicCrawler(
2990+
configure_logging=False,
2991+
concurrency_settings=ConcurrencySettings(desired_concurrency=1, max_concurrency=1),
2992+
)
2993+
sigint_handler_before_run = signal.getsignal(signal.SIGINT)
2994+
2995+
@crawler.router.default_handler
2996+
async def handler(context: BasicCrawlingContext) -> None:
2997+
signal.raise_signal(signal.SIGINT)
2998+
# The pause lets the running request finish, so keep it running until the pause takes effect.
2999+
await asyncio.sleep(0.5)
3000+
3001+
with (
3002+
patch.object(asyncio.get_running_loop(), 'add_signal_handler', side_effect=NotImplementedError),
3003+
caplog.at_level(logging.INFO, logger='crawlee'),
3004+
):
3005+
stats = await crawler.run(['https://a.placeholder.com', 'https://b.placeholder.com'])
3006+
3007+
assert stats.requests_finished == 1
3008+
assert 'Pausing...' in caplog.text
3009+
assert 'The crawl was interrupted' in caplog.text
3010+
assert signal.getsignal(signal.SIGINT) is sigint_handler_before_run

0 commit comments

Comments
 (0)