pause the crawl on Ctrl+C on Windows

Author: Mantisus

src/crawlee/crawlers/_basic/_basic_crawler.py

@@ -11,7 +11,7 @@
 import traceback
 from asyncio import CancelledError
 from collections.abc import AsyncGenerator, Awaitable, Callable, Iterable, Sequence
-from contextlib import AsyncExitStack, suppress
+from contextlib import AsyncExitStack
 from datetime import timedelta
 from functools import partial
 from http import HTTPStatus
@@ -763,19 +763,29 @@ def sigint_handler() -> None:
 
             run_task = asyncio.create_task(self._run_crawler(), name='run_crawler_task')
 
+            loop = asyncio.get_running_loop()
+            remove_sigint_handler: Callable[[], object] | None = None
+
             # `add_signal_handler` works only in the main thread
             if threading.current_thread() is threading.main_thread():
-                with suppress(NotImplementedError):  # event loop signal handlers are not supported on Windows
-                    asyncio.get_running_loop().add_signal_handler(signal.SIGINT, sigint_handler)
+                try:
+                    loop.add_signal_handler(signal.SIGINT, sigint_handler)
+                    remove_sigint_handler = partial(loop.remove_signal_handler, signal.SIGINT)
+                except NotImplementedError:
+                    # Windows event loops, such as the default `ProactorEventLoop`, don't support `add_signal_handler`.
+                    # Like `asyncio.Runner`, set the handler with `signal.signal` and pass the call to the event loop.
+                    previous_sigint_handler = signal.signal(
+                        signal.SIGINT, lambda *_: loop.call_soon_threadsafe(sigint_handler)
+                    )
+                    remove_sigint_handler = partial(signal.signal, signal.SIGINT, previous_sigint_handler)
 
             try:
                 await run_task
             except CancelledError:
                 pass
             finally:
-                if threading.current_thread() is threading.main_thread():
-                    with suppress(NotImplementedError):
-                        asyncio.get_running_loop().remove_signal_handler(signal.SIGINT)
+                if remove_sigint_handler is not None:
+                    remove_sigint_handler()
 
             if self._statistics.error_tracker.total > 0:
                 self._logger.info(

tests/unit/crawlers/_basic/test_basic_crawler.py

@@ -6,6 +6,7 @@
 import logging
 import os
 import re
+import signal
 import sys
 import time
 from asyncio import Future
@@ -2981,3 +2982,29 @@ async def handler(context: BasicCrawlingContext) -> None:
 
     assert outcomes == []
     assert stats.retry_histogram == expected_histogram
+
+
+async def test_sigint_without_loop_signal_handlers(caplog: pytest.LogCaptureFixture) -> None:
+    """SIGINT pauses the crawl on event loops without `add_signal_handler`, as on Windows."""
+    crawler = BasicCrawler(
+        configure_logging=False,
+        concurrency_settings=ConcurrencySettings(desired_concurrency=1, max_concurrency=1),
+    )
+    sigint_handler_before_run = signal.getsignal(signal.SIGINT)
+
+    @crawler.router.default_handler
+    async def handler(context: BasicCrawlingContext) -> None:
+        signal.raise_signal(signal.SIGINT)
+        # The pause lets the running request finish, so keep it running until the pause takes effect.
+        await asyncio.sleep(0.5)
+
+    with (
+        patch.object(asyncio.get_running_loop(), 'add_signal_handler', side_effect=NotImplementedError),
+        caplog.at_level(logging.INFO, logger='crawlee'),
+    ):
+        stats = await crawler.run(['https://a.placeholder.com', 'https://b.placeholder.com'])
+
+    assert stats.requests_finished == 1
+    assert 'Pausing...' in caplog.text
+    assert 'The crawl was interrupted' in caplog.text
+    assert signal.getsignal(signal.SIGINT) is sigint_handler_before_run