From fb60f4ac88868d4e95477b2ff4785641d9df42fc Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 13 Oct 2026 14:15:43 +0200 Subject: [PATCH 1/3] refactor!: mark secondary arguments as keyword-only BREAKING CHANGE: Secondary arguments of the public API are keyword-only. The v2 upgrade guide lists the affected signatures. --- .../adaptive_playwright_crawler.py | 2 +- .../playwright_crawler_adaptive/handler.py | 2 +- docs/upgrading/upgrading_to_v2.md | 34 +++++++++++++ src/crawlee/_types.py | 8 +++ src/crawlee/_utils/recoverable_state.py | 2 +- src/crawlee/browsers/_browser_controller.py | 1 + src/crawlee/browsers/_playwright_browser.py | 1 + .../_playwright_browser_controller.py | 1 + .../browsers/_playwright_browser_plugin.py | 4 +- .../browsers/_stagehand_browser_controller.py | 1 + .../_abstract_http/_abstract_http_crawler.py | 2 +- .../_abstract_http/_abstract_http_parser.py | 2 +- .../_adaptive_playwright_crawler.py | 6 ++- .../_adaptive_playwright_crawling_context.py | 10 ++-- src/crawlee/crawlers/_basic/_basic_crawler.py | 5 +- .../_beautifulsoup/_beautifulsoup_parser.py | 2 +- src/crawlee/crawlers/_http/_http_parser.py | 2 +- .../crawlers/_parsel/_parsel_parser.py | 2 +- src/crawlee/crawlers/_playwright/_types.py | 2 +- src/crawlee/crawlers/_playwright/_utils.py | 2 +- .../fingerprint_suite/_header_generator.py | 2 +- src/crawlee/proxy_configuration.py | 6 +-- src/crawlee/request_loaders/_request_list.py | 1 + src/crawlee/storages/_dataset.py | 10 ++-- src/crawlee/storages/_key_value_store.py | 5 ++ .../test_playwright_browser_controller.py | 4 +- .../test_adaptive_playwright_crawler.py | 8 +-- .../test_new_proxy_info.py | 50 +++++++++---------- tests/unit/proxy_configuration/test_tiers.py | 42 ++++++++-------- 29 files changed, 140 insertions(+), 79 deletions(-) diff --git a/docs/examples/code_examples/adaptive_playwright_crawler.py b/docs/examples/code_examples/adaptive_playwright_crawler.py index 904a000379..4485a553d6 100644 --- a/docs/examples/code_examples/adaptive_playwright_crawler.py +++ b/docs/examples/code_examples/adaptive_playwright_crawler.py @@ -26,7 +26,7 @@ async def request_handler_for_label( context.log.info(context.parsed_content.title) # Locate element h2 within 5 seconds - h2 = await context.query_selector_one('h2', timedelta(milliseconds=5000)) + h2 = await context.query_selector_one('h2', timeout=timedelta(milliseconds=5000)) # Do stuff with element found by the selector context.log.info(h2) diff --git a/docs/guides/code_examples/playwright_crawler_adaptive/handler.py b/docs/guides/code_examples/playwright_crawler_adaptive/handler.py index 629b49449e..5d8f426efa 100644 --- a/docs/guides/code_examples/playwright_crawler_adaptive/handler.py +++ b/docs/guides/code_examples/playwright_crawler_adaptive/handler.py @@ -10,7 +10,7 @@ async def main() -> None: @crawler.router.default_handler async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None: # Locate element h2 within 5 seconds - h2 = await context.query_selector_one('h2', timedelta(milliseconds=5000)) + h2 = await context.query_selector_one('h2', timeout=timedelta(milliseconds=5000)) # Do stuff with element found by the selector context.log.info(h2) diff --git a/docs/upgrading/upgrading_to_v2.md b/docs/upgrading/upgrading_to_v2.md index f796a24e5b..70474ca148 100644 --- a/docs/upgrading/upgrading_to_v2.md +++ b/docs/upgrading/upgrading_to_v2.md @@ -20,3 +20,37 @@ We drop support for Python 3.10, which reached its end of life in October 2026. ## `RequestQueueClient.is_finished` is abstract `RequestQueueClient.is_finished` is now an abstract method. In v1, it fell back to `is_empty`, which ignores requests still being processed and could report the queue as finished too early. A subclass that doesn't implement it raises `TypeError` on instantiation. If you have a custom request queue client, implement `is_finished` so that it returns `True` only when the queue is empty and no requests are in progress. The built-in clients already implement it. + +## Keyword-only arguments + +Secondary arguments of the public API are now keyword-only. The main argument of each call, such as a key, a value, a selector or a path, stays positional. A call that passes any of the following arguments positionally raises `TypeError`, so pass them by name: + +- `context.push_data(data, *, dataset_id, dataset_name, dataset_alias)`, and the same for `RequestHandlerRunResult.push_data`. +- `context.add_requests(requests, *, rq_id, rq_name, rq_alias)`, and the same for `RequestHandlerRunResult.add_requests`. +- `context.block_requests(*, url_patterns, extra_url_patterns)`, and `block_requests(page, *, url_patterns, extra_url_patterns)`. +- `BasicCrawlingContext.create_modified_copy(*, push_data, add_requests, get_key_value_store)`. +- `BasicCrawler.get_data(*, dataset_id, dataset_name, dataset_alias, ...)` and `BasicCrawler.export_data(path, *, dataset_id, dataset_name, dataset_alias, ...)`. +- `KeyValueStore.set_value(key, value, *, content_type)`, also on the key-value store returned by `context.get_key_value_store()`. +- `KeyValueStore.iterate_keys`, `iterate_values`, `iterate_entries` and `list_keys`, which take `exclusive_start_key` and `limit` by keyword only. +- `Dataset.export_to(key, *, content_type, to_kvs_id, ...)`. +- `ConcurrencySettings(*, min_concurrency, max_concurrency, max_tasks_per_minute, desired_concurrency)`. +- `RequestList(requests, *, name, persist_state_key, persist_requests_key)`. +- `ProxyConfiguration.new_url(*, session_id, request, proxy_tier)` and `ProxyConfiguration.new_proxy_info(*, session_id, request, proxy_tier)`. All three arguments of `new_proxy_info` now default to `None`. +- `AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser(*, ...)` and `with_parsel_static_parser(*, ...)`. +- `AdaptivePlaywrightCrawlingContext.wait_for_selector`, `query_selector_one`, `query_selector_all` and `parse_with_static_parser`, which take `timeout` by keyword only. +- `HeaderGenerator.get_specific_headers(header_names, *, browser_type)`. +- `BrowserController.new_page(*, browser_new_context_options, proxy_info)`, including the Playwright and Stagehand controllers. +- `PlaywrightPersistentBrowser(browser_type, user_data_dir, *, browser_launch_options)`. +- `AbstractHttpParser.find_links(parsed_content, selector, *, attribute)`, including the built-in parsers. Custom parsers must use the same signature. + +For example: + +```python +# v1 +await kvs.set_value('report', data, 'application/json') +h2 = await context.query_selector_one('h2', timedelta(seconds=5)) + +# v2 +await kvs.set_value('report', data, content_type='application/json') +h2 = await context.query_selector_one('h2', timeout=timedelta(seconds=5)) +``` diff --git a/src/crawlee/_types.py b/src/crawlee/_types.py index 1a33653c31..7df51120af 100644 --- a/src/crawlee/_types.py +++ b/src/crawlee/_types.py @@ -106,6 +106,7 @@ class ConcurrencySettings: def __init__( self, + *, min_concurrency: int = 1, max_concurrency: int = 100, max_tasks_per_minute: float = float('inf'), @@ -223,6 +224,7 @@ async def set_value( self, key: str, value: Any, + *, content_type: str | None = None, ) -> None: ... @@ -242,6 +244,7 @@ async def set_value( self, key: str, value: Any, + *, content_type: str | None = None, ) -> None: self.updates[key] = KeyValueStoreValue(value, content_type) @@ -278,6 +281,7 @@ def __init__( async def add_requests( self, requests: Sequence[str | Request], + *, rq_id: str | None = None, rq_name: str | None = None, rq_alias: str | None = None, @@ -294,6 +298,7 @@ async def add_requests( async def push_data( self, data: Sequence[Mapping[str, JsonSerializable]] | Mapping[str, JsonSerializable], + *, dataset_id: str | None = None, dataset_name: str | None = None, dataset_alias: str | None = None, @@ -336,6 +341,7 @@ class AddRequestsFunction(Protocol): def __call__( self, requests: Sequence[str | Request], + *, rq_id: str | None = None, rq_name: str | None = None, rq_alias: str | None = None, @@ -538,6 +544,7 @@ class PushDataFunction(Protocol): def __call__( self, data: Sequence[Mapping[str, JsonSerializable]] | Mapping[str, JsonSerializable], + *, dataset_id: str | None = None, dataset_name: str | None = None, dataset_alias: str | None = None, @@ -673,6 +680,7 @@ def __hash__(self) -> int: def create_modified_copy( self, + *, push_data: PushDataFunction | None = None, add_requests: AddRequestsFunction | None = None, get_key_value_store: GetKeyValueStoreFromRequestHandlerFunction | None = None, diff --git a/src/crawlee/_utils/recoverable_state.py b/src/crawlee/_utils/recoverable_state.py index 9916083a72..5d33ccc1b5 100644 --- a/src/crawlee/_utils/recoverable_state.py +++ b/src/crawlee/_utils/recoverable_state.py @@ -192,7 +192,7 @@ async def persist_state(self, event_data: EventPersistStateData | None = None) - await self._key_value_store.set_value( self._persist_state_key, self._state.model_dump(mode='json', by_alias=True), - 'application/json', + content_type='application/json', ) else: self._log.debug('Persistence is not enabled - not doing anything') diff --git a/src/crawlee/browsers/_browser_controller.py b/src/crawlee/browsers/_browser_controller.py index 7489dc25fb..49515b2fec 100644 --- a/src/crawlee/browsers/_browser_controller.py +++ b/src/crawlee/browsers/_browser_controller.py @@ -72,6 +72,7 @@ def browser_type(self) -> BrowserType: @abstractmethod async def new_page( self, + *, browser_new_context_options: Mapping[str, Any] | None = None, proxy_info: ProxyInfo | None = None, ) -> Page: diff --git a/src/crawlee/browsers/_playwright_browser.py b/src/crawlee/browsers/_playwright_browser.py index 2c42f1f0f7..3c71a5a246 100644 --- a/src/crawlee/browsers/_playwright_browser.py +++ b/src/crawlee/browsers/_playwright_browser.py @@ -40,6 +40,7 @@ def __init__( self, browser_type: BrowserType, user_data_dir: str | Path | None, + *, browser_launch_options: dict[str, Any], ) -> None: self._browser_type = browser_type diff --git a/src/crawlee/browsers/_playwright_browser_controller.py b/src/crawlee/browsers/_playwright_browser_controller.py index be72f8f90e..45b5df7113 100644 --- a/src/crawlee/browsers/_playwright_browser_controller.py +++ b/src/crawlee/browsers/_playwright_browser_controller.py @@ -157,6 +157,7 @@ def browser_type(self) -> BrowserType: @override async def new_page( self, + *, browser_new_context_options: Mapping[str, Any] | None = None, proxy_info: ProxyInfo | None = None, ) -> Page: diff --git a/src/crawlee/browsers/_playwright_browser_plugin.py b/src/crawlee/browsers/_playwright_browser_plugin.py index 755aec68e1..04349e5a75 100644 --- a/src/crawlee/browsers/_playwright_browser_plugin.py +++ b/src/crawlee/browsers/_playwright_browser_plugin.py @@ -190,7 +190,9 @@ async def new_browser(self) -> PlaywrightBrowserController: if self._use_incognito_pages: browser: Browser | PlaywrightPersistentBrowser = await browser_type.launch(**self._browser_launch_options) else: - browser = PlaywrightPersistentBrowser(browser_type, self._user_data_dir, self._browser_launch_options) + browser = PlaywrightPersistentBrowser( + browser_type, self._user_data_dir, browser_launch_options=self._browser_launch_options + ) return PlaywrightBrowserController( browser, diff --git a/src/crawlee/browsers/_stagehand_browser_controller.py b/src/crawlee/browsers/_stagehand_browser_controller.py index 96eae12f4e..170c525229 100644 --- a/src/crawlee/browsers/_stagehand_browser_controller.py +++ b/src/crawlee/browsers/_stagehand_browser_controller.py @@ -124,6 +124,7 @@ def browser_type(self) -> BrowserType: @override async def new_page( self, + *, browser_new_context_options: Mapping[str, Any] | None = None, proxy_info: ProxyInfo | None = None, ) -> StagehandPage: diff --git a/src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py b/src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py index c7d1276ff0..db0dd29e40 100644 --- a/src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py +++ b/src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py @@ -221,7 +221,7 @@ async def extract_links( # A relative `` is resolved against the page URL, and one that does not resolve to an absolute # URL (malformed, `mailto:`, ...) is ignored. base_url = context.request.loaded_url or context.request.url - extracted_base_urls = list(self._parser.find_links(parsed_content, 'base[href]', 'href')) + extracted_base_urls = list(self._parser.find_links(parsed_content, 'base[href]', attribute='href')) if extracted_base_urls: try: resolved_base_url = convert_to_absolute_url(base_url, extracted_base_urls[0]) diff --git a/src/crawlee/crawlers/_abstract_http/_abstract_http_parser.py b/src/crawlee/crawlers/_abstract_http/_abstract_http_parser.py index 5f64565e76..cb13e83eea 100644 --- a/src/crawlee/crawlers/_abstract_http/_abstract_http_parser.py +++ b/src/crawlee/crawlers/_abstract_http/_abstract_http_parser.py @@ -93,7 +93,7 @@ def is_matching_selector(self, parsed_content: TParseResult, selector: str) -> b """ @abstractmethod - def find_links(self, parsed_content: TParseResult, selector: str, attribute: str) -> Iterable[str]: + def find_links(self, parsed_content: TParseResult, selector: str, *, attribute: str) -> Iterable[str]: """Find all links in result using selector. Args: diff --git a/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py b/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py index e1189dbdf9..3c47839fa6 100644 --- a/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py +++ b/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py @@ -105,7 +105,7 @@ async def request_handler_for_label(context: AdaptivePlaywrightCrawlingContext) context.log.info(context.parsed_content.title) # Locate element h2 within 5 seconds - h2 = await context.query_selector_one('h2', timedelta(milliseconds=5000)) + h2 = await context.query_selector_one('h2', timeout=timedelta(milliseconds=5000)) # Do stuff with element found by the selector context.log.info(h2) @@ -234,8 +234,9 @@ async def adaptive_post_navigation_hook_pw(context: PlaywrightPostNavCrawlingCon self._static_parser = static_parser @classmethod - def with_beautifulsoup_static_parser( # noqa: PLR0917 + def with_beautifulsoup_static_parser( cls, + *, rendering_type_predictor: RenderingTypePredictor | None = None, result_checker: Callable[[RequestHandlerRunResult], bool] | None = None, result_comparator: Callable[[RequestHandlerRunResult, RequestHandlerRunResult], bool] | None = None, @@ -262,6 +263,7 @@ def with_beautifulsoup_static_parser( # noqa: PLR0917 @classmethod def with_parsel_static_parser( cls, + *, rendering_type_predictor: RenderingTypePredictor | None = None, result_checker: Callable[[RequestHandlerRunResult], bool] | None = None, result_comparator: Callable[[RequestHandlerRunResult, RequestHandlerRunResult], bool] | None = None, diff --git a/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawling_context.py b/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawling_context.py index 65ac612f34..f5023962e3 100644 --- a/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawling_context.py +++ b/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawling_context.py @@ -77,7 +77,7 @@ def response(self) -> Response: raise AdaptiveContextError('Page was not crawled with PlaywrightCrawler.') return self._response - async def wait_for_selector(self, selector: str, timeout: timedelta = timedelta(seconds=5)) -> None: + async def wait_for_selector(self, selector: str, *, timeout: timedelta = timedelta(seconds=5)) -> None: """Locate element by css selector and return `None` once it is found. If element is not found within timeout, `TimeoutError` is raised. @@ -91,7 +91,7 @@ async def wait_for_selector(self, selector: str, timeout: timedelta = timedelta( await self.page.locator(selector).wait_for(timeout=timeout.total_seconds() * 1000) async def query_selector_one( - self, selector: str, timeout: timedelta = timedelta(seconds=5) + self, selector: str, *, timeout: timedelta = timedelta(seconds=5) ) -> TStaticSelectResult | None: """Locate element by css selector and return first element found. @@ -109,7 +109,7 @@ async def query_selector_one( return None async def query_selector_all( - self, selector: str, timeout: timedelta = timedelta(seconds=5) + self, selector: str, *, timeout: timedelta = timedelta(seconds=5) ) -> Sequence[TStaticSelectResult]: """Locate element by css selector and return all elements found. @@ -147,7 +147,7 @@ async def query_selector_all( ) async def parse_with_static_parser( - self, selector: str | None = None, timeout: timedelta = timedelta(seconds=5) + self, selector: str | None = None, *, timeout: timedelta = timedelta(seconds=5) ) -> TStaticParseResult: """Parse whole page with static parser. If `selector` argument is used, wait for selector first. @@ -161,7 +161,7 @@ async def parse_with_static_parser( Result of used static parser `parse_text` method. """ if selector: - await self.wait_for_selector(selector, timeout) + await self.wait_for_selector(selector, timeout=timeout) if self._page: return await self._static_parser.parse_text(await self.page.content()) return self.parsed_content diff --git a/src/crawlee/crawlers/_basic/_basic_crawler.py b/src/crawlee/crawlers/_basic/_basic_crawler.py index fdf43eadfa..7bd4dfae9e 100644 --- a/src/crawlee/crawlers/_basic/_basic_crawler.py +++ b/src/crawlee/crawlers/_basic/_basic_crawler.py @@ -913,6 +913,7 @@ async def _save_crawler_state(self) -> None: async def get_data( self, + *, dataset_id: str | None = None, dataset_name: str | None = None, dataset_alias: str | None = None, @@ -944,10 +945,10 @@ async def get_data( async def export_data( self, path: str | Path, + *, dataset_id: str | None = None, dataset_name: str | None = None, dataset_alias: str | None = None, - *, collect_all_keys: bool = False, **additional_kwargs: Unpack[ExportDataKwargs], ) -> None: @@ -1436,7 +1437,7 @@ async def _commit_key_value_store_changes( for (id, name, alias), changes in result.key_value_store_changes.items(): store = await get_kvs(id=id, name=name, alias=alias) for key, value in changes.updates.items(): - await store.set_value(key, value.content, value.content_type) + await store.set_value(key, value.content, content_type=value.content_type) async def __is_finished_function(self) -> bool: self._stop_if_max_requests_count_exceeded() diff --git a/src/crawlee/crawlers/_beautifulsoup/_beautifulsoup_parser.py b/src/crawlee/crawlers/_beautifulsoup/_beautifulsoup_parser.py index 99e88d7eda..7bd67b0d9b 100644 --- a/src/crawlee/crawlers/_beautifulsoup/_beautifulsoup_parser.py +++ b/src/crawlee/crawlers/_beautifulsoup/_beautifulsoup_parser.py @@ -42,7 +42,7 @@ async def select(self, parsed_content: Tag, selector: str) -> Sequence[Tag]: return tuple(match for match in parsed_content.select(selector)) @override - def find_links(self, parsed_content: Tag, selector: str, attribute: str) -> Iterable[str]: + def find_links(self, parsed_content: Tag, selector: str, *, attribute: str) -> Iterable[str]: link: Tag urls: list[str] = [] for link in parsed_content.select(selector): diff --git a/src/crawlee/crawlers/_http/_http_parser.py b/src/crawlee/crawlers/_http/_http_parser.py index adfcc52aec..fc3d138c60 100644 --- a/src/crawlee/crawlers/_http/_http_parser.py +++ b/src/crawlee/crawlers/_http/_http_parser.py @@ -44,6 +44,6 @@ def is_matching_selector(self, parsed_content: bytes, selector: str) -> bool: # @override def find_links( - self, parsed_content: bytes, selector: str, attribute: str + self, parsed_content: bytes, selector: str, *, attribute: str ) -> Iterable[str]: # Intentional unused argument. return [] diff --git a/src/crawlee/crawlers/_parsel/_parsel_parser.py b/src/crawlee/crawlers/_parsel/_parsel_parser.py index acba550b3d..142b0671dc 100644 --- a/src/crawlee/crawlers/_parsel/_parsel_parser.py +++ b/src/crawlee/crawlers/_parsel/_parsel_parser.py @@ -39,7 +39,7 @@ def is_matching_selector(self, parsed_content: Selector, selector: str) -> bool: return parsed_content.type in ('html', 'xml') and parsed_content.css(selector).get() is not None @override - def find_links(self, parsed_content: Selector, selector: str, attribute: str) -> Iterable[str]: + def find_links(self, parsed_content: Selector, selector: str, *, attribute: str) -> Iterable[str]: link: Selector urls: list[str] = [] for link in parsed_content.css(selector): diff --git a/src/crawlee/crawlers/_playwright/_types.py b/src/crawlee/crawlers/_playwright/_types.py index 7422f0f1d6..afc01a36dd 100644 --- a/src/crawlee/crawlers/_playwright/_types.py +++ b/src/crawlee/crawlers/_playwright/_types.py @@ -24,7 +24,7 @@ class BlockRequestsFunction(Protocol): """ async def __call__( - self, url_patterns: list[str] | None = None, extra_url_patterns: list[str] | None = None + self, *, url_patterns: list[str] | None = None, extra_url_patterns: list[str] | None = None ) -> None: """Call dunder method. diff --git a/src/crawlee/crawlers/_playwright/_utils.py b/src/crawlee/crawlers/_playwright/_utils.py index 06e91ee60f..5ce1f736b1 100644 --- a/src/crawlee/crawlers/_playwright/_utils.py +++ b/src/crawlee/crawlers/_playwright/_utils.py @@ -126,7 +126,7 @@ async def check_finished() -> None: async def block_requests( - page: Page, url_patterns: list[str] | None = None, extra_url_patterns: list[str] | None = None + page: Page, *, url_patterns: list[str] | None = None, extra_url_patterns: list[str] | None = None ) -> None: """Blocks network requests matching specified URL patterns. diff --git a/src/crawlee/fingerprint_suite/_header_generator.py b/src/crawlee/fingerprint_suite/_header_generator.py index 8d41d6c955..9e0ad61947 100644 --- a/src/crawlee/fingerprint_suite/_header_generator.py +++ b/src/crawlee/fingerprint_suite/_header_generator.py @@ -34,7 +34,7 @@ def _select_specific_headers(self, all_headers: dict[str, str], header_names: se return HttpHeaders({key: value for key, value in all_headers.items() if key in header_names}) def get_specific_headers( - self, header_names: set[str] | None = None, browser_type: SupportedBrowserType = 'chrome' + self, header_names: set[str] | None = None, *, browser_type: SupportedBrowserType = 'chrome' ) -> HttpHeaders: """Return subset of headers based on the selected `header_names`. diff --git a/src/crawlee/proxy_configuration.py b/src/crawlee/proxy_configuration.py index 2047931d06..fd118d7034 100644 --- a/src/crawlee/proxy_configuration.py +++ b/src/crawlee/proxy_configuration.py @@ -112,7 +112,7 @@ def _create_url(self, url: str | None) -> URL | None: return URL(url) async def new_proxy_info( - self, session_id: str | None, request: Request | None, proxy_tier: int | None + self, *, session_id: str | None = None, request: Request | None = None, proxy_tier: int | None = None ) -> ProxyInfo | None: """Return a new ProxyInfo object based on the configured proxy rotation strategy. @@ -157,7 +157,7 @@ async def new_proxy_info( return info async def new_url( - self, session_id: str | None = None, request: Request | None = None, proxy_tier: int | None = None + self, *, session_id: str | None = None, request: Request | None = None, proxy_tier: int | None = None ) -> str | None: """Return a proxy URL string based on the configured proxy rotation strategy. @@ -170,7 +170,7 @@ async def new_url( proxy_tier: Specific proxy tier to use. If not provided, will be automatically selected based on configuration. """ - proxy_info = await self.new_proxy_info(session_id, request, proxy_tier) + proxy_info = await self.new_proxy_info(session_id=session_id, request=request, proxy_tier=proxy_tier) return proxy_info.url if proxy_info else None async def _pick_url( diff --git a/src/crawlee/request_loaders/_request_list.py b/src/crawlee/request_loaders/_request_list.py index 0303ea06d1..302a4ec55b 100644 --- a/src/crawlee/request_loaders/_request_list.py +++ b/src/crawlee/request_loaders/_request_list.py @@ -35,6 +35,7 @@ class RequestList(RequestLoader): def __init__( self, requests: Iterable[str | Request] | AsyncIterable[str | Request] | None = None, + *, name: str | None = None, persist_state_key: str | None = None, persist_requests_key: str | None = None, diff --git a/src/crawlee/storages/_dataset.py b/src/crawlee/storages/_dataset.py index 0b4aece7ac..0c8ccc8cf0 100644 --- a/src/crawlee/storages/_dataset.py +++ b/src/crawlee/storages/_dataset.py @@ -315,6 +315,7 @@ async def list_items( async def export_to( self, key: str, + *, content_type: Literal['json'], to_kvs_id: str | None = None, to_kvs_name: str | None = None, @@ -327,19 +328,20 @@ async def export_to( async def export_to( self, key: str, + *, content_type: Literal['csv'], to_kvs_id: str | None = None, to_kvs_name: str | None = None, to_kvs_storage_client: StorageClient | None = None, to_kvs_configuration: Configuration | None = None, - *, collect_all_keys: bool = False, **kwargs: Unpack[ExportDataCsvKwargs], ) -> None: ... - async def export_to( # noqa: PLR0917 + async def export_to( self, key: str, + *, content_type: Literal['json', 'csv'] = 'json', to_kvs_id: str | None = None, to_kvs_name: str | None = None, @@ -378,9 +380,9 @@ async def export_to( # noqa: PLR0917 if content_type == 'csv': await export_csv_to_stream(self.iterate_items(), dst, **kwargs) - await kvs.set_value(key, dst.getvalue(), 'text/csv') + await kvs.set_value(key, dst.getvalue(), content_type='text/csv') elif content_type == 'json': await export_json_to_stream(self.iterate_items(), dst, **kwargs) - await kvs.set_value(key, dst.getvalue(), 'application/json') + await kvs.set_value(key, dst.getvalue(), content_type='application/json') else: raise ValueError('Unsupported content type, expecting CSV or JSON') diff --git a/src/crawlee/storages/_key_value_store.py b/src/crawlee/storages/_key_value_store.py index 1eb82087e1..bb06642f67 100644 --- a/src/crawlee/storages/_key_value_store.py +++ b/src/crawlee/storages/_key_value_store.py @@ -174,6 +174,7 @@ async def set_value( self, key: str, value: Any, + *, content_type: str | None = None, ) -> None: """Set a value in the KVS. @@ -195,6 +196,7 @@ async def delete_value(self, key: str) -> None: async def iterate_keys( self, + *, exclusive_start_key: str | None = None, limit: int | None = None, ) -> AsyncIterator[KeyValueStoreRecordMetadata]: @@ -215,6 +217,7 @@ async def iterate_keys( async def iterate_values( self, + *, exclusive_start_key: str | None = None, limit: int | None = None, ) -> AsyncIterator[Any]: @@ -236,6 +239,7 @@ async def iterate_values( async def iterate_entries( self, + *, exclusive_start_key: str | None = None, limit: int | None = None, ) -> AsyncIterator[tuple[str, Any]]: @@ -274,6 +278,7 @@ def __aiter__(self) -> AsyncIterator[str]: async def list_keys( self, + *, exclusive_start_key: str | None = None, limit: int = 1000, ) -> list[KeyValueStoreRecordMetadata]: diff --git a/tests/unit/browsers/test_playwright_browser_controller.py b/tests/unit/browsers/test_playwright_browser_controller.py index dad50ee642..ed2ac466f5 100644 --- a/tests/unit/browsers/test_playwright_browser_controller.py +++ b/tests/unit/browsers/test_playwright_browser_controller.py @@ -131,7 +131,9 @@ async def delayed_launch_persistent_context(*args: Any, **kwargs: Any) -> Any: # Create minimal instance of PlaywrightBrowserController with mocked browser controller = PlaywrightBrowserController( - PlaywrightPersistentBrowser(mocked_browser, None, {}), header_generator=None, fingerprint_generator=None + PlaywrightPersistentBrowser(mocked_browser, None, browser_launch_options={}), + header_generator=None, + fingerprint_generator=None, ) # Both calls will try to create browser context at the same time, but only one context should be created. diff --git a/tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py b/tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py index 67f15fd487..a86fdc6b05 100644 --- a/tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py +++ b/tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py @@ -680,9 +680,9 @@ async def test_adaptive_context_query_selector_beautiful_soup(test_urls: list[st @crawler.router.default_handler async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None: - h1 = await context.query_selector_one('h1', _ELEMENT_WAIT_TIMEOUT) + h1 = await context.query_selector_one('h1', timeout=_ELEMENT_WAIT_TIMEOUT) mocked_h1_handler(h1) - h2 = await context.query_selector_one('h2', _ELEMENT_WAIT_TIMEOUT) + h2 = await context.query_selector_one('h2', timeout=_ELEMENT_WAIT_TIMEOUT) mocked_h2_handler(h2) await crawler.run(test_urls[:1]) @@ -724,9 +724,9 @@ async def test_adaptive_context_query_selector_parsel(test_urls: list[str]) -> N @crawler.router.default_handler async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None: - if h1 := await context.query_selector_one('h1', _ELEMENT_WAIT_TIMEOUT): + if h1 := await context.query_selector_one('h1', timeout=_ELEMENT_WAIT_TIMEOUT): mocked_h1_handler(type(h1), h1.get()) - if h2 := await context.query_selector_one('h2', _ELEMENT_WAIT_TIMEOUT): + if h2 := await context.query_selector_one('h2', timeout=_ELEMENT_WAIT_TIMEOUT): mocked_h2_handler(type(h2), h2.get()) await crawler.run(test_urls[:1]) diff --git a/tests/unit/proxy_configuration/test_new_proxy_info.py b/tests/unit/proxy_configuration/test_new_proxy_info.py index 1a8efe0289..a2e38f5046 100644 --- a/tests/unit/proxy_configuration/test_new_proxy_info.py +++ b/tests/unit/proxy_configuration/test_new_proxy_info.py @@ -12,10 +12,10 @@ async def test_returns_proxy_info() -> None: """Test that proxy_urls can return contain both string and None.""" config = ProxyConfiguration(proxy_urls=[None, 'http://proxy.com:1111']) - proxy_info = await config.new_proxy_info(None, None, None) + proxy_info = await config.new_proxy_info() assert proxy_info is None - proxy_info = await config.new_proxy_info(None, None, None) + proxy_info = await config.new_proxy_info() assert proxy_info is not None assert proxy_info.url == 'http://proxy.com:1111' assert proxy_info.hostname == 'proxy.com' @@ -30,7 +30,7 @@ async def test_throws_on_invalid_new_url_function() -> None: ) with pytest.raises(ValueError): # noqa: PT011 - await config.new_proxy_info(None, None, None) + await config.new_proxy_info() async def test_returns_proxy_info_with_new_url_function() -> None: @@ -39,10 +39,10 @@ async def test_returns_proxy_info_with_new_url_function() -> None: config = ProxyConfiguration(new_url_function=lambda session_id=None, request=None: next(proxy_iterator)) # noqa: ARG005 - proxy_info = await config.new_proxy_info(None, None, None) + proxy_info = await config.new_proxy_info() assert proxy_info is None - proxy_info = await config.new_proxy_info(None, None, None) + proxy_info = await config.new_proxy_info() assert proxy_info is not None assert proxy_info.url == 'http://proxy.com:1111' assert proxy_info.hostname == 'proxy.com' @@ -57,7 +57,7 @@ async def new_url(session_id: str | None = None, request: Request | None = None) config = ProxyConfiguration(new_url_function=new_url) - proxy_info = await config.new_proxy_info(None, None, None) + proxy_info = await config.new_proxy_info() assert proxy_info is not None assert proxy_info.url == 'http://proxy.com:1111' @@ -71,15 +71,15 @@ async def test_rotates_proxies() -> None: proxy_urls: list[str | None] = ['http://proxy:1111', 'http://proxy:2222', 'http://proxy:3333'] config = ProxyConfiguration(proxy_urls=proxy_urls) - info = await config.new_proxy_info(None, None, None) + info = await config.new_proxy_info() assert info is not None assert info.url == proxy_urls[0] - info = await config.new_proxy_info(None, None, None) + info = await config.new_proxy_info() assert info is not None assert info.url == proxy_urls[1] - info = await config.new_proxy_info(None, None, None) + info = await config.new_proxy_info() assert info is not None assert info.url == proxy_urls[2] @@ -92,65 +92,65 @@ async def test_rotates_proxies_with_sessions() -> None: config = ProxyConfiguration(proxy_urls=proxy_urls) # A single session should always receive the same proxy - info = await config.new_proxy_info(sessions[0], None, None) + info = await config.new_proxy_info(session_id=sessions[0]) assert info is not None assert info.url == proxy_urls[0] - info = await config.new_proxy_info(sessions[0], None, None) + info = await config.new_proxy_info(session_id=sessions[0]) assert info is not None assert info.url == proxy_urls[0] - info = await config.new_proxy_info(sessions[0], None, None) + info = await config.new_proxy_info(session_id=sessions[0]) assert info is not None assert info.url == proxy_urls[0] - info = await config.new_proxy_info(sessions[0], request, None) + info = await config.new_proxy_info(session_id=sessions[0], request=request) assert info is not None assert info.url == proxy_urls[0] - info = await config.new_proxy_info(sessions[0], request, None) + info = await config.new_proxy_info(session_id=sessions[0], request=request) assert info is not None assert info.url == proxy_urls[0] # Different sessions should get rotated proxies - info = await config.new_proxy_info(sessions[1], None, None) + info = await config.new_proxy_info(session_id=sessions[1]) assert info is not None assert info.url == proxy_urls[1] - info = await config.new_proxy_info(sessions[2], request, None) + info = await config.new_proxy_info(session_id=sessions[2], request=request) assert info is not None assert info.url == proxy_urls[2] - info = await config.new_proxy_info(sessions[3], None, None) + info = await config.new_proxy_info(session_id=sessions[3]) assert info is not None assert info.url == proxy_urls[0] - info = await config.new_proxy_info(sessions[4], None, None) + info = await config.new_proxy_info(session_id=sessions[4]) assert info is not None assert info.url == proxy_urls[1] - info = await config.new_proxy_info(sessions[5], request, None) + info = await config.new_proxy_info(session_id=sessions[5], request=request) assert info is not None assert info.url == proxy_urls[2] # Without sessions should get rotated proxies - info = await config.new_proxy_info(None, None, None) + info = await config.new_proxy_info() assert info is not None assert info.url == proxy_urls[0] - info = await config.new_proxy_info(None, request, None) + info = await config.new_proxy_info(request=request) assert info is not None assert info.url == proxy_urls[1] - info = await config.new_proxy_info(None, None, None) + info = await config.new_proxy_info() assert info is not None assert info.url == proxy_urls[2] - info = await config.new_proxy_info(None, None, None) + info = await config.new_proxy_info() assert info is not None assert info.url == proxy_urls[0] - info = await config.new_proxy_info(None, request, None) + info = await config.new_proxy_info(request=request) assert info is not None assert info.url == proxy_urls[1] @@ -173,6 +173,6 @@ async def test_sets_port(url: str, expected_port: int) -> None: """ config = ProxyConfiguration(proxy_urls=[url]) - info = await config.new_proxy_info(None, None, None) + info = await config.new_proxy_info() assert info is not None assert info.port == expected_port diff --git a/tests/unit/proxy_configuration/test_tiers.py b/tests/unit/proxy_configuration/test_tiers.py index 59db9a43d7..1f179beb3a 100644 --- a/tests/unit/proxy_configuration/test_tiers.py +++ b/tests/unit/proxy_configuration/test_tiers.py @@ -12,23 +12,23 @@ async def test_rotates_proxies_uniformly_with_no_request() -> None: config = ProxyConfiguration(tiered_proxy_urls=tiered_proxy_urls) - info = await config.new_proxy_info(None, None, None) + info = await config.new_proxy_info() assert info is not None assert info.url == tiered_proxy_urls[0][0] - info = await config.new_proxy_info(None, None, None) + info = await config.new_proxy_info() assert info is not None assert info.url == tiered_proxy_urls[0][1] - info = await config.new_proxy_info(None, None, None) + info = await config.new_proxy_info() assert info is not None assert info.url == tiered_proxy_urls[1][0] - info = await config.new_proxy_info(None, None, None) + info = await config.new_proxy_info() assert info is not None assert info.url == tiered_proxy_urls[1][1] - info = await config.new_proxy_info(None, None, None) + info = await config.new_proxy_info() assert info is not None assert info.url == tiered_proxy_urls[0][0] @@ -46,22 +46,22 @@ async def test_retrying_request_makes_tier_go_up() -> None: # Calling `new_proxy_info` with the same request most probably means it's being retried request_1 = Request(url='http://some.domain/abc', unique_key='1') - info = await config.new_proxy_info(None, request_1, None) + info = await config.new_proxy_info(request=request_1) assert info is not None assert info.url == tiered_proxy_urls[0][0] - info = await config.new_proxy_info(None, request_1, None) + info = await config.new_proxy_info(request=request_1) assert info is not None assert info.url == tiered_proxy_urls[1][0] - info = await config.new_proxy_info(None, request_1, None) + info = await config.new_proxy_info(request=request_1) assert info is not None assert info.url == tiered_proxy_urls[2][0] # Subsequent requests with the same domain should use the same tier request_2 = Request(url='http://some.domain/xyz', unique_key='2') - info = await config.new_proxy_info(None, request_2, None) + info = await config.new_proxy_info(request=request_2) assert info is not None assert info.url == tiered_proxy_urls[2][0] @@ -80,24 +80,24 @@ async def test_retrying_request_makes_tier_go_up_with_sessions() -> None: # Calling `new_proxy_info` with the same request likely means that it is being retried. # However, a single session should always receive the same proxy - info = await config.new_proxy_info('session_id', request, None) + info = await config.new_proxy_info(session_id='session_id', request=request) assert info is not None assert info.url == tiered_proxy_urls[0][0] - info = await config.new_proxy_info('session_id', request, None) + info = await config.new_proxy_info(session_id='session_id', request=request) assert info is not None assert info.url == tiered_proxy_urls[0][0] - info = await config.new_proxy_info('session_id', request, None) + info = await config.new_proxy_info(session_id='session_id', request=request) assert info is not None assert info.url == tiered_proxy_urls[0][0] # For a new session, we will get a proxy from the corresponding tier - info = await config.new_proxy_info('session_id2', request, None) + info = await config.new_proxy_info(session_id='session_id2', request=request) assert info is not None assert info.url == tiered_proxy_urls[3][0] - info = await config.new_proxy_info('session_id2', request, None) + info = await config.new_proxy_info(session_id='session_id2', request=request) assert info is not None assert info.url == tiered_proxy_urls[3][0] @@ -120,13 +120,13 @@ async def test_successful_request_makes_tier_go_down() -> None: info = None for tier in tiered_proxy_urls: - info = await config.new_proxy_info(None, request_1, None) + info = await config.new_proxy_info(request=request_1) assert info is not None assert info.url == tier[0] for i in range(100): new_request = Request(url=f'http://some.domain/{i}', unique_key=str(i)) - info = await config.new_proxy_info(None, new_request, None) + info = await config.new_proxy_info(request=new_request) assert info is not None assert info.url == tiered_proxy_urls[0][0] @@ -144,11 +144,11 @@ async def test_none_proxy_retrying_request_makes_tier_go_up() -> None: request_1 = Request(url='http://some.domain/abc', unique_key='1') # No proxy used. - info = await config.new_proxy_info(None, request_1, None) + info = await config.new_proxy_info(request=request_1) assert info is None, 'First entry in tired_proxy_urls is None. config.new_proxy_info is expected to generate None.' # Proxy should go up one tier for same request that was already sent before. - info = await config.new_proxy_info(None, request_1, None) + info = await config.new_proxy_info(request=request_1) assert info is not None, ( 'config.new_proxy_info is expected to generate non-none proxy info from non-none tiered_proxy_urls.' ) @@ -163,16 +163,16 @@ async def test_none_proxy_rotates_proxies_uniformly_with_no_request() -> None: config = ProxyConfiguration(tiered_proxy_urls=tiered_proxy_urls) # No proxy used. - info = await config.new_proxy_info(None, None, None) + info = await config.new_proxy_info() assert info is None, 'First entry in tired_proxy_urls is None. config.new_proxy_info is expected to generate None.' # Proxy should be rotated on the same proxy tier for a new request. - info = await config.new_proxy_info(None, None, None) + info = await config.new_proxy_info() assert info is not None, ( 'config.new_proxy_info is expected to generate non-none proxy info from non-none tiered_proxy_urls.' ) assert info.url == tiered_proxy_urls[0][1] # Proxy rotation starts from the beginning of the proxy list after last proxy in tier was used. No proxy used again. - info = await config.new_proxy_info(None, None, None) + info = await config.new_proxy_info() assert info is None, 'First entry in tired_proxy_urls is None. config.new_proxy_info is expected to generate None.' From cb871fa19e575c4f4e0937ec9867fe9b60b28559 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 13 Oct 2026 14:24:01 +0200 Subject: [PATCH 2/3] refactor: make error-handler push_data and add_requests keyword-only --- src/crawlee/crawlers/_basic/_basic_crawler.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/crawlee/crawlers/_basic/_basic_crawler.py b/src/crawlee/crawlers/_basic/_basic_crawler.py index 7bd4dfae9e..e24a484d08 100644 --- a/src/crawlee/crawlers/_basic/_basic_crawler.py +++ b/src/crawlee/crawlers/_basic/_basic_crawler.py @@ -994,6 +994,7 @@ async def export_data( async def _push_data( self, data: Sequence[Mapping[str, JsonSerializable]] | Mapping[str, JsonSerializable], + *, dataset_id: str | None = None, dataset_name: str | None = None, dataset_alias: str | None = None, @@ -1387,6 +1388,7 @@ async def _add_requests( self, context: BasicCrawlingContext, requests: Sequence[str | Request], + *, rq_id: str | None = None, rq_name: str | None = None, rq_alias: str | None = None, From be20cf2d31cf5555c197dfa791b56ad860c24798 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 13 Oct 2026 14:24:03 +0200 Subject: [PATCH 3/3] docs: correct find_links note and Selectolax example --- docs/guides/code_examples/http_crawlers/selectolax_parser.py | 2 +- docs/upgrading/upgrading_to_v2.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/guides/code_examples/http_crawlers/selectolax_parser.py b/docs/guides/code_examples/http_crawlers/selectolax_parser.py index b8fca8b38c..a43983b305 100644 --- a/docs/guides/code_examples/http_crawlers/selectolax_parser.py +++ b/docs/guides/code_examples/http_crawlers/selectolax_parser.py @@ -45,7 +45,7 @@ def is_matching_selector( @override def find_links( - self, parsed_content: LexborHTMLParser, selector: str, attribute: str + self, parsed_content: LexborHTMLParser, selector: str, *, attribute: str ) -> Iterable[str]: """Extract href attributes from elements matching the selector. diff --git a/docs/upgrading/upgrading_to_v2.md b/docs/upgrading/upgrading_to_v2.md index 70474ca148..10b085a11f 100644 --- a/docs/upgrading/upgrading_to_v2.md +++ b/docs/upgrading/upgrading_to_v2.md @@ -41,7 +41,7 @@ Secondary arguments of the public API are now keyword-only. The main argument of - `HeaderGenerator.get_specific_headers(header_names, *, browser_type)`. - `BrowserController.new_page(*, browser_new_context_options, proxy_info)`, including the Playwright and Stagehand controllers. - `PlaywrightPersistentBrowser(browser_type, user_data_dir, *, browser_launch_options)`. -- `AbstractHttpParser.find_links(parsed_content, selector, *, attribute)`, including the built-in parsers. Custom parsers must use the same signature. +- `AbstractHttpParser.find_links(parsed_content, selector, *, attribute)`, including the built-in parsers. Custom parsers keep working, since the crawler passes `attribute` by keyword. For example: