diff --git a/docs/upgrading/upgrading_to_v2.md b/docs/upgrading/upgrading_to_v2.md index 960fea7142..f796a24e5b 100644 --- a/docs/upgrading/upgrading_to_v2.md +++ b/docs/upgrading/upgrading_to_v2.md @@ -16,3 +16,7 @@ We drop support for Python 3.10, which reached its end of life in October 2026. ## `StatisticsState.crawler_runtime` is read-only `StatisticsState.crawler_runtime` is computed from the crawler's start and finish timestamps. In v1, assigning to it only emitted a `DeprecationWarning` and the value was ignored. In v2, the property has no setter, so an assignment raises `AttributeError`. Remove any such assignments. + +## `RequestQueueClient.is_finished` is abstract + +`RequestQueueClient.is_finished` is now an abstract method. In v1, it fell back to `is_empty`, which ignores requests still being processed and could report the queue as finished too early. A subclass that doesn't implement it raises `TypeError` on instantiation. If you have a custom request queue client, implement `is_finished` so that it returns `True` only when the queue is empty and no requests are in progress. The built-in clients already implement it. diff --git a/src/crawlee/storage_clients/_base/_request_queue_client.py b/src/crawlee/storage_clients/_base/_request_queue_client.py index 4a09c81f21..768733f2da 100644 --- a/src/crawlee/storage_clients/_base/_request_queue_client.py +++ b/src/crawlee/storage_clients/_base/_request_queue_client.py @@ -125,20 +125,12 @@ async def is_empty(self) -> bool: True if the request queue is empty, False otherwise. """ + @abstractmethod async def is_finished(self) -> bool: """Check if the request queue is finished. A finished queue is empty and has no requests currently being processed. - Warning: - This default only checks `is_empty`, which reports whether requests are available to fetch, not - whether requests are still being processed. It can therefore return `True` while requests are in - progress. Subclasses that track in-progress requests should override this method; all built-in - clients already do. - Returns: True if the request queue is finished, False otherwise. """ - # TODO: Make this method abstract. - # https://github.com/apify/crawlee-python/issues/1985 - return await self.is_empty()