refactor!: mark secondary arguments as keyword-only

docs/examples/code_examples/adaptive_playwright_crawler.py

@@ -26,7 +26,7 @@ async def request_handler_for_label(
         context.log.info(context.parsed_content.title)
 
         # Locate element h2 within 5 seconds
-        h2 = await context.query_selector_one('h2', timedelta(milliseconds=5000))
+        h2 = await context.query_selector_one('h2', timeout=timedelta(milliseconds=5000))
         # Do stuff with element found by the selector
         context.log.info(h2)
 

docs/guides/code_examples/http_crawlers/selectolax_parser.py

@@ -45,7 +45,7 @@ def is_matching_selector(
 
     @override
     def find_links(
-        self, parsed_content: LexborHTMLParser, selector: str, attribute: str
+        self, parsed_content: LexborHTMLParser, selector: str, *, attribute: str
     ) -> Iterable[str]:
         """Extract href attributes from elements matching the selector.
 

docs/guides/code_examples/playwright_crawler_adaptive/handler.py

@@ -10,7 +10,7 @@ async def main() -> None:
     @crawler.router.default_handler
     async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None:
         # Locate element h2 within 5 seconds
-        h2 = await context.query_selector_one('h2', timedelta(milliseconds=5000))
+        h2 = await context.query_selector_one('h2', timeout=timedelta(milliseconds=5000))
         # Do stuff with element found by the selector
         context.log.info(h2)
 

docs/upgrading/upgrading_to_v2.md

@@ -20,3 +20,37 @@ We drop support for Python 3.10, which reached its end of life in October 2026.
 ## `RequestQueueClient.is_finished` is abstract
 
 `RequestQueueClient.is_finished` is now an abstract method. In v1, it fell back to `is_empty`, which ignores requests still being processed and could report the queue as finished too early. A subclass that doesn't implement it raises `TypeError` on instantiation. If you have a custom request queue client, implement `is_finished` so that it returns `True` only when the queue is empty and no requests are in progress. The built-in clients already implement it.
+
+## Keyword-only arguments
+
+Secondary arguments of the public API are now keyword-only. The main argument of each call, such as a key, a value, a selector or a path, stays positional. A call that passes any of the following arguments positionally raises `TypeError`, so pass them by name:
+
+- `context.push_data(data, *, dataset_id, dataset_name, dataset_alias)`, and the same for `RequestHandlerRunResult.push_data`.
+- `context.add_requests(requests, *, rq_id, rq_name, rq_alias)`, and the same for `RequestHandlerRunResult.add_requests`.
+- `context.block_requests(*, url_patterns, extra_url_patterns)`, and `block_requests(page, *, url_patterns, extra_url_patterns)`.
+- `BasicCrawlingContext.create_modified_copy(*, push_data, add_requests, get_key_value_store)`.
+- `BasicCrawler.get_data(*, dataset_id, dataset_name, dataset_alias, ...)` and `BasicCrawler.export_data(path, *, dataset_id, dataset_name, dataset_alias, ...)`.
+- `KeyValueStore.set_value(key, value, *, content_type)`, also on the key-value store returned by `context.get_key_value_store()`.
+- `KeyValueStore.iterate_keys`, `iterate_values`, `iterate_entries` and `list_keys`, which take `exclusive_start_key` and `limit` by keyword only.
+- `Dataset.export_to(key, *, content_type, to_kvs_id, ...)`.
+- `ConcurrencySettings(*, min_concurrency, max_concurrency, max_tasks_per_minute, desired_concurrency)`.
+- `RequestList(requests, *, name, persist_state_key, persist_requests_key)`.
+- `ProxyConfiguration.new_url(*, session_id, request, proxy_tier)` and `ProxyConfiguration.new_proxy_info(*, session_id, request, proxy_tier)`. All three arguments of `new_proxy_info` now default to `None`.
+- `AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser(*, ...)` and `with_parsel_static_parser(*, ...)`.
+- `AdaptivePlaywrightCrawlingContext.wait_for_selector`, `query_selector_one`, `query_selector_all` and `parse_with_static_parser`, which take `timeout` by keyword only.
+- `HeaderGenerator.get_specific_headers(header_names, *, browser_type)`.
+- `BrowserController.new_page(*, browser_new_context_options, proxy_info)`, including the Playwright and Stagehand controllers.
+- `PlaywrightPersistentBrowser(browser_type, user_data_dir, *, browser_launch_options)`.
+- `AbstractHttpParser.find_links(parsed_content, selector, *, attribute)`, including the built-in parsers. Custom parsers keep working, since the crawler passes `attribute` by keyword.
+
+For example:
+
+```python
+# v1
+await kvs.set_value('report', data, 'application/json')
+h2 = await context.query_selector_one('h2', timedelta(seconds=5))
+
+# v2
+await kvs.set_value('report', data, content_type='application/json')
+h2 = await context.query_selector_one('h2', timeout=timedelta(seconds=5))
+```

src/crawlee/_types.py

@@ -106,6 +106,7 @@ class ConcurrencySettings:
 
     def __init__(
         self,
+        *,
         min_concurrency: int = 1,
         max_concurrency: int = 100,
         max_tasks_per_minute: float = float('inf'),
@@ -223,6 +224,7 @@ async def set_value(
         self,
         key: str,
         value: Any,
+        *,
         content_type: str | None = None,
     ) -> None: ...
 
@@ -242,6 +244,7 @@ async def set_value(
         self,
         key: str,
         value: Any,
+        *,
         content_type: str | None = None,
     ) -> None:
         self.updates[key] = KeyValueStoreValue(value, content_type)
@@ -278,6 +281,7 @@ def __init__(
     async def add_requests(
         self,
         requests: Sequence[str | Request],
+        *,
         rq_id: str | None = None,
         rq_name: str | None = None,
         rq_alias: str | None = None,
@@ -294,6 +298,7 @@ async def add_requests(
     async def push_data(
         self,
         data: Sequence[Mapping[str, JsonSerializable]] | Mapping[str, JsonSerializable],
+        *,
         dataset_id: str | None = None,
         dataset_name: str | None = None,
         dataset_alias: str | None = None,
@@ -336,6 +341,7 @@ class AddRequestsFunction(Protocol):
     def __call__(
         self,
         requests: Sequence[str | Request],
+        *,
         rq_id: str | None = None,
         rq_name: str | None = None,
         rq_alias: str | None = None,
@@ -538,6 +544,7 @@ class PushDataFunction(Protocol):
     def __call__(
         self,
         data: Sequence[Mapping[str, JsonSerializable]] | Mapping[str, JsonSerializable],
+        *,
         dataset_id: str | None = None,
         dataset_name: str | None = None,
         dataset_alias: str | None = None,
@@ -673,6 +680,7 @@ def __hash__(self) -> int:
 
     def create_modified_copy(
         self,
+        *,
         push_data: PushDataFunction | None = None,
         add_requests: AddRequestsFunction | None = None,
         get_key_value_store: GetKeyValueStoreFromRequestHandlerFunction | None = None,

src/crawlee/_utils/recoverable_state.py

@@ -192,7 +192,7 @@ async def persist_state(self, event_data: EventPersistStateData | None = None) -
             await self._key_value_store.set_value(
                 self._persist_state_key,
                 self._state.model_dump(mode='json', by_alias=True),
-                'application/json',
+                content_type='application/json',
             )
         else:
             self._log.debug('Persistence is not enabled - not doing anything')

src/crawlee/browsers/_browser_controller.py

@@ -72,6 +72,7 @@ def browser_type(self) -> BrowserType:
     @abstractmethod
     async def new_page(
         self,
+        *,
         browser_new_context_options: Mapping[str, Any] | None = None,
         proxy_info: ProxyInfo | None = None,
     ) -> Page:

src/crawlee/browsers/_playwright_browser.py

@@ -40,6 +40,7 @@ def __init__(
         self,
         browser_type: BrowserType,
         user_data_dir: str | Path | None,
+        *,
         browser_launch_options: dict[str, Any],
     ) -> None:
         self._browser_type = browser_type

src/crawlee/browsers/_playwright_browser_controller.py

@@ -157,6 +157,7 @@ def browser_type(self) -> BrowserType:
     @override
     async def new_page(
         self,
+        *,
         browser_new_context_options: Mapping[str, Any] | None = None,
         proxy_info: ProxyInfo | None = None,
     ) -> Page:

src/crawlee/browsers/_playwright_browser_plugin.py

@@ -190,7 +190,9 @@ async def new_browser(self) -> PlaywrightBrowserController:
         if self._use_incognito_pages:
             browser: Browser | PlaywrightPersistentBrowser = await browser_type.launch(**self._browser_launch_options)
         else:
-            browser = PlaywrightPersistentBrowser(browser_type, self._user_data_dir, self._browser_launch_options)
+            browser = PlaywrightPersistentBrowser(
+                browser_type, self._user_data_dir, browser_launch_options=self._browser_launch_options
+            )
 
         return PlaywrightBrowserController(
             browser,

src/crawlee/browsers/_stagehand_browser_controller.py

@@ -124,6 +124,7 @@ def browser_type(self) -> BrowserType:
     @override
     async def new_page(
         self,
+        *,
         browser_new_context_options: Mapping[str, Any] | None = None,
         proxy_info: ProxyInfo | None = None,
     ) -> StagehandPage:

src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py

@@ -221,7 +221,7 @@ async def extract_links(
             # A relative `<base href>` is resolved against the page URL, and one that does not resolve to an absolute
             # URL (malformed, `mailto:`, ...) is ignored.
             base_url = context.request.loaded_url or context.request.url
-            extracted_base_urls = list(self._parser.find_links(parsed_content, 'base[href]', 'href'))
+            extracted_base_urls = list(self._parser.find_links(parsed_content, 'base[href]', attribute='href'))
             if extracted_base_urls:
                 try:
                     resolved_base_url = convert_to_absolute_url(base_url, extracted_base_urls[0])

src/crawlee/crawlers/_abstract_http/_abstract_http_parser.py

@@ -93,7 +93,7 @@ def is_matching_selector(self, parsed_content: TParseResult, selector: str) -> b
         """
 
     @abstractmethod
-    def find_links(self, parsed_content: TParseResult, selector: str, attribute: str) -> Iterable[str]:
+    def find_links(self, parsed_content: TParseResult, selector: str, *, attribute: str) -> Iterable[str]:
         """Find all links in result using selector.
 
         Args:

src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py

@@ -105,7 +105,7 @@ async def request_handler_for_label(context: AdaptivePlaywrightCrawlingContext)
         context.log.info(context.parsed_content.title)
 
         # Locate element h2 within 5 seconds
-        h2 = await context.query_selector_one('h2', timedelta(milliseconds=5000))
+        h2 = await context.query_selector_one('h2', timeout=timedelta(milliseconds=5000))
         # Do stuff with element found by the selector
         context.log.info(h2)
 
@@ -234,8 +234,9 @@ async def adaptive_post_navigation_hook_pw(context: PlaywrightPostNavCrawlingCon
         self._static_parser = static_parser
 
     @classmethod
-    def with_beautifulsoup_static_parser(  # noqa: PLR0917
+    def with_beautifulsoup_static_parser(
         cls,
+        *,
         rendering_type_predictor: RenderingTypePredictor | None = None,
         result_checker: Callable[[RequestHandlerRunResult], bool] | None = None,
         result_comparator: Callable[[RequestHandlerRunResult, RequestHandlerRunResult], bool] | None = None,
@@ -262,6 +263,7 @@ def with_beautifulsoup_static_parser(  # noqa: PLR0917
     @classmethod
     def with_parsel_static_parser(
         cls,
+        *,
         rendering_type_predictor: RenderingTypePredictor | None = None,
         result_checker: Callable[[RequestHandlerRunResult], bool] | None = None,
         result_comparator: Callable[[RequestHandlerRunResult, RequestHandlerRunResult], bool] | None = None,

src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawling_context.py

@@ -77,7 +77,7 @@ def response(self) -> Response:
             raise AdaptiveContextError('Page was not crawled with PlaywrightCrawler.')
         return self._response
 
-    async def wait_for_selector(self, selector: str, timeout: timedelta = timedelta(seconds=5)) -> None:
+    async def wait_for_selector(self, selector: str, *, timeout: timedelta = timedelta(seconds=5)) -> None:
         """Locate element by css selector and return `None` once it is found.
 
         If element is not found within timeout, `TimeoutError` is raised.
@@ -91,7 +91,7 @@ async def wait_for_selector(self, selector: str, timeout: timedelta = timedelta(
         await self.page.locator(selector).wait_for(timeout=timeout.total_seconds() * 1000)
 
     async def query_selector_one(
-        self, selector: str, timeout: timedelta = timedelta(seconds=5)
+        self, selector: str, *, timeout: timedelta = timedelta(seconds=5)
     ) -> TStaticSelectResult | None:
         """Locate element by css selector and return first element found.
 
@@ -109,7 +109,7 @@ async def query_selector_one(
         return None
 
     async def query_selector_all(
-        self, selector: str, timeout: timedelta = timedelta(seconds=5)
+        self, selector: str, *, timeout: timedelta = timedelta(seconds=5)
     ) -> Sequence[TStaticSelectResult]:
         """Locate element by css selector and return all elements found.
 
@@ -147,7 +147,7 @@ async def query_selector_all(
         )
 
     async def parse_with_static_parser(
-        self, selector: str | None = None, timeout: timedelta = timedelta(seconds=5)
+        self, selector: str | None = None, *, timeout: timedelta = timedelta(seconds=5)
     ) -> TStaticParseResult:
         """Parse whole page with static parser. If `selector` argument is used, wait for selector first.
 
@@ -161,7 +161,7 @@ async def parse_with_static_parser(
             Result of used static parser `parse_text` method.
         """
         if selector:
-            await self.wait_for_selector(selector, timeout)
+            await self.wait_for_selector(selector, timeout=timeout)
         if self._page:
             return await self._static_parser.parse_text(await self.page.content())
         return self.parsed_content

src/crawlee/crawlers/_basic/_basic_crawler.py

@@ -913,6 +913,7 @@ async def _save_crawler_state(self) -> None:
 
     async def get_data(
         self,
+        *,
         dataset_id: str | None = None,
         dataset_name: str | None = None,
         dataset_alias: str | None = None,
@@ -944,10 +945,10 @@ async def get_data(
     async def export_data(
         self,
         path: str | Path,
+        *,
         dataset_id: str | None = None,
         dataset_name: str | None = None,
         dataset_alias: str | None = None,
-        *,
         collect_all_keys: bool = False,
         **additional_kwargs: Unpack[ExportDataKwargs],
     ) -> None:
@@ -993,6 +994,7 @@ async def export_data(
     async def _push_data(
         self,
         data: Sequence[Mapping[str, JsonSerializable]] | Mapping[str, JsonSerializable],
+        *,
         dataset_id: str | None = None,
         dataset_name: str | None = None,
         dataset_alias: str | None = None,
@@ -1386,6 +1388,7 @@ async def _add_requests(
         self,
         context: BasicCrawlingContext,
         requests: Sequence[str | Request],
+        *,
         rq_id: str | None = None,
         rq_name: str | None = None,
         rq_alias: str | None = None,
@@ -1436,7 +1439,7 @@ async def _commit_key_value_store_changes(
         for (id, name, alias), changes in result.key_value_store_changes.items():
             store = await get_kvs(id=id, name=name, alias=alias)
             for key, value in changes.updates.items():
-                await store.set_value(key, value.content, value.content_type)
+                await store.set_value(key, value.content, content_type=value.content_type)
 
     async def __is_finished_function(self) -> bool:
         self._stop_if_max_requests_count_exceeded()

src/crawlee/crawlers/_beautifulsoup/_beautifulsoup_parser.py

@@ -42,7 +42,7 @@ async def select(self, parsed_content: Tag, selector: str) -> Sequence[Tag]:
         return tuple(match for match in parsed_content.select(selector))
 
     @override
-    def find_links(self, parsed_content: Tag, selector: str, attribute: str) -> Iterable[str]:
+    def find_links(self, parsed_content: Tag, selector: str, *, attribute: str) -> Iterable[str]:
         link: Tag
         urls: list[str] = []
         for link in parsed_content.select(selector):

src/crawlee/crawlers/_http/_http_parser.py

@@ -44,6 +44,6 @@ def is_matching_selector(self, parsed_content: bytes, selector: str) -> bool:  #
 
     @override
     def find_links(
-        self, parsed_content: bytes, selector: str, attribute: str
+        self, parsed_content: bytes, selector: str, *, attribute: str
     ) -> Iterable[str]:  # Intentional unused argument.
         return []

src/crawlee/crawlers/_parsel/_parsel_parser.py

@@ -39,7 +39,7 @@ def is_matching_selector(self, parsed_content: Selector, selector: str) -> bool:
         return parsed_content.type in ('html', 'xml') and parsed_content.css(selector).get() is not None
 
     @override
-    def find_links(self, parsed_content: Selector, selector: str, attribute: str) -> Iterable[str]:
+    def find_links(self, parsed_content: Selector, selector: str, *, attribute: str) -> Iterable[str]:
         link: Selector
         urls: list[str] = []
         for link in parsed_content.css(selector):

src/crawlee/crawlers/_playwright/_types.py

@@ -24,7 +24,7 @@ class BlockRequestsFunction(Protocol):
     """
 
     async def __call__(
-        self, url_patterns: list[str] | None = None, extra_url_patterns: list[str] | None = None
+        self, *, url_patterns: list[str] | None = None, extra_url_patterns: list[str] | None = None
     ) -> None:
         """Call dunder method.
 

src/crawlee/crawlers/_playwright/_utils.py

@@ -126,7 +126,7 @@ async def check_finished() -> None:
 
 
 async def block_requests(
-    page: Page, url_patterns: list[str] | None = None, extra_url_patterns: list[str] | None = None
+    page: Page, *, url_patterns: list[str] | None = None, extra_url_patterns: list[str] | None = None
 ) -> None:
     """Blocks network requests matching specified URL patterns.
 

src/crawlee/fingerprint_suite/_header_generator.py

@@ -34,7 +34,7 @@ def _select_specific_headers(self, all_headers: dict[str, str], header_names: se
         return HttpHeaders({key: value for key, value in all_headers.items() if key in header_names})
 
     def get_specific_headers(
-        self, header_names: set[str] | None = None, browser_type: SupportedBrowserType = 'chrome'
+        self, header_names: set[str] | None = None, *, browser_type: SupportedBrowserType = 'chrome'
     ) -> HttpHeaders:
         """Return subset of headers based on the selected `header_names`.
 

src/crawlee/proxy_configuration.py

@@ -112,7 +112,7 @@ def _create_url(self, url: str | None) -> URL | None:
         return URL(url)
 
     async def new_proxy_info(
-        self, session_id: str | None, request: Request | None, proxy_tier: int | None
+        self, *, session_id: str | None = None, request: Request | None = None, proxy_tier: int | None = None
     ) -> ProxyInfo | None:
         """Return a new ProxyInfo object based on the configured proxy rotation strategy.
 
@@ -157,7 +157,7 @@ async def new_proxy_info(
         return info
 
     async def new_url(
-        self, session_id: str | None = None, request: Request | None = None, proxy_tier: int | None = None
+        self, *, session_id: str | None = None, request: Request | None = None, proxy_tier: int | None = None
     ) -> str | None:
         """Return a proxy URL string based on the configured proxy rotation strategy.
 
@@ -170,7 +170,7 @@ async def new_url(
             proxy_tier: Specific proxy tier to use. If not provided, will be automatically
                 selected based on configuration.
         """
-        proxy_info = await self.new_proxy_info(session_id, request, proxy_tier)
+        proxy_info = await self.new_proxy_info(session_id=session_id, request=request, proxy_tier=proxy_tier)
         return proxy_info.url if proxy_info else None
 
     async def _pick_url(