From 067152acf285a3a4f67b621e6d29694772cb8187 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 13 Oct 2026 13:55:24 +0200 Subject: [PATCH 1/2] refactor!: make `RequestQueueClient.is_finished` abstract BREAKING CHANGE: Custom `RequestQueueClient` subclasses must implement `is_finished`; the `is_empty` fallback is removed. --- docs/upgrading/upgrading_to_v2.md | 4 ++++ .../storage_clients/_base/_request_queue_client.py | 10 +--------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/docs/upgrading/upgrading_to_v2.md b/docs/upgrading/upgrading_to_v2.md index 960fea7142..4823e139db 100644 --- a/docs/upgrading/upgrading_to_v2.md +++ b/docs/upgrading/upgrading_to_v2.md @@ -16,3 +16,7 @@ We drop support for Python 3.10, which reached its end of life in October 2026. ## `StatisticsState.crawler_runtime` is read-only `StatisticsState.crawler_runtime` is computed from the crawler's start and finish timestamps. In v1, assigning to it only emitted a `DeprecationWarning` and the value was ignored. In v2, the property has no setter, so an assignment raises `AttributeError`. Remove any such assignments. + +## `RequestQueueClient.is_finished` is abstract + +`RequestQueueClient.is_finished` is now an abstract method. In v1, it fell back to `is_empty`, which ignores requests still being processed and could report the queue as finished too early. If you have a custom request queue client, implement `is_finished` so that it returns `True` only when the queue is empty and no requests are in progress. The built-in clients already implement it. diff --git a/src/crawlee/storage_clients/_base/_request_queue_client.py b/src/crawlee/storage_clients/_base/_request_queue_client.py index 4a09c81f21..768733f2da 100644 --- a/src/crawlee/storage_clients/_base/_request_queue_client.py +++ b/src/crawlee/storage_clients/_base/_request_queue_client.py @@ -125,20 +125,12 @@ async def is_empty(self) -> bool: True if the request queue is empty, False otherwise. """ + @abstractmethod async def is_finished(self) -> bool: """Check if the request queue is finished. A finished queue is empty and has no requests currently being processed. - Warning: - This default only checks `is_empty`, which reports whether requests are available to fetch, not - whether requests are still being processed. It can therefore return `True` while requests are in - progress. Subclasses that track in-progress requests should override this method; all built-in - clients already do. - Returns: True if the request queue is finished, False otherwise. """ - # TODO: Make this method abstract. - # https://github.com/apify/crawlee-python/issues/1985 - return await self.is_empty() From a60e1c1e724ca2700f25d0740412a2abd630a644 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 13 Oct 2026 14:05:28 +0200 Subject: [PATCH 2/2] docs: note the TypeError for clients without `is_finished` --- docs/upgrading/upgrading_to_v2.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/upgrading/upgrading_to_v2.md b/docs/upgrading/upgrading_to_v2.md index 4823e139db..f796a24e5b 100644 --- a/docs/upgrading/upgrading_to_v2.md +++ b/docs/upgrading/upgrading_to_v2.md @@ -19,4 +19,4 @@ We drop support for Python 3.10, which reached its end of life in October 2026. ## `RequestQueueClient.is_finished` is abstract -`RequestQueueClient.is_finished` is now an abstract method. In v1, it fell back to `is_empty`, which ignores requests still being processed and could report the queue as finished too early. If you have a custom request queue client, implement `is_finished` so that it returns `True` only when the queue is empty and no requests are in progress. The built-in clients already implement it. +`RequestQueueClient.is_finished` is now an abstract method. In v1, it fell back to `is_empty`, which ignores requests still being processed and could report the queue as finished too early. A subclass that doesn't implement it raises `TypeError` on instantiation. If you have a custom request queue client, implement `is_finished` so that it returns `True` only when the queue is empty and no requests are in progress. The built-in clients already implement it.