From d87317fb2a73c0a90f13b898025959c90f425170 Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Tue, 13 Oct 2026 11:50:48 +0000 Subject: [PATCH 1/2] polish `typing` imports --- .../code_examples/http_crawlers/selectolax_crawler.py | 3 +-- src/crawlee/_types.py | 4 +--- src/crawlee/_utils/file.py | 4 +--- src/crawlee/_utils/html.py | 3 +-- src/crawlee/browsers/_stagehand_types.py | 3 ++- .../crawlers/_abstract_http/_abstract_http_crawler.py | 3 +-- .../_adaptive_playwright/_adaptive_playwright_crawler.py | 3 +-- src/crawlee/crawlers/_basic/_basic_crawler.py | 4 ++-- .../crawlers/_beautifulsoup/_beautifulsoup_crawler.py | 3 +-- .../_beautifulsoup/_beautifulsoup_crawling_context.py | 4 +--- .../crawlers/_file_download/_file_download_crawler.py | 3 +-- src/crawlee/crawlers/_http/_http_crawler.py | 2 +- src/crawlee/crawlers/_parsel/_parsel_crawler.py | 3 +-- src/crawlee/crawlers/_parsel/_parsel_crawling_context.py | 4 +--- src/crawlee/crawlers/_playwright/_playwright_crawler.py | 6 +++--- src/crawlee/crawlers/_pydantic_ai/_pydantic_ai_crawler.py | 2 +- src/crawlee/crawlers/_pydantic_ai/_types.py | 3 +-- src/crawlee/crawlers/_stagehand/_stagehand_crawler.py | 3 +-- src/crawlee/events/_local_event_manager.py | 4 +--- src/crawlee/proxy_configuration.py | 3 +-- src/crawlee/storages/_dataset.py | 4 +--- 21 files changed, 25 insertions(+), 46 deletions(-) diff --git a/docs/guides/code_examples/http_crawlers/selectolax_crawler.py b/docs/guides/code_examples/http_crawlers/selectolax_crawler.py index 677a6a3b00..64615a4d88 100644 --- a/docs/guides/code_examples/http_crawlers/selectolax_crawler.py +++ b/docs/guides/code_examples/http_crawlers/selectolax_crawler.py @@ -11,8 +11,7 @@ if TYPE_CHECKING: from collections.abc import AsyncGenerator - - from typing_extensions import Unpack + from typing import Unpack from crawlee.crawlers._abstract_http import ParsedHttpCrawlingContext diff --git a/src/crawlee/_types.py b/src/crawlee/_types.py index 1a33653c31..5480f5d0e9 100644 --- a/src/crawlee/_types.py +++ b/src/crawlee/_types.py @@ -15,9 +15,7 @@ import re from collections.abc import Awaitable, Coroutine, MutableMapping, Sequence from datetime import timedelta - from typing import NotRequired, Required, Self - - from typing_extensions import Unpack + from typing import NotRequired, Required, Self, Unpack from crawlee import Glob, Request from crawlee._request import RequestOptions diff --git a/src/crawlee/_utils/file.py b/src/crawlee/_utils/file.py index 167c297aef..6c1843fc56 100644 --- a/src/crawlee/_utils/file.py +++ b/src/crawlee/_utils/file.py @@ -12,9 +12,7 @@ if TYPE_CHECKING: from collections.abc import AsyncIterator, Mapping - from typing import Any, TextIO - - from typing_extensions import Unpack + from typing import Any, TextIO, Unpack from crawlee._types import ExportDataCsvKwargs, ExportDataJsonKwargs, JsonSerializable diff --git a/src/crawlee/_utils/html.py b/src/crawlee/_utils/html.py index b731ce2dfa..746e3b23ca 100644 --- a/src/crawlee/_utils/html.py +++ b/src/crawlee/_utils/html.py @@ -17,10 +17,9 @@ if TYPE_CHECKING: from collections.abc import Mapping, Sequence - from typing import NotRequired + from typing import NotRequired, Unpack from lxml.html import HtmlElement - from typing_extensions import Unpack from crawlee._types import JsonSerializable diff --git a/src/crawlee/browsers/_stagehand_types.py b/src/crawlee/browsers/_stagehand_types.py index 9feee93d61..8a62ee1c72 100644 --- a/src/crawlee/browsers/_stagehand_types.py +++ b/src/crawlee/browsers/_stagehand_types.py @@ -8,6 +8,8 @@ from crawlee._utils.docs import docs_group if TYPE_CHECKING: + from typing import Unpack + from stagehand import AsyncSession from stagehand.types.session_act_params import SessionActParamsNonStreaming from stagehand.types.session_act_response import SessionActResponse @@ -17,7 +19,6 @@ from stagehand.types.session_extract_response import SessionExtractResponse from stagehand.types.session_observe_params import SessionObserveParamsNonStreaming from stagehand.types.session_observe_response import SessionObserveResponse - from typing_extensions import Unpack @dataclass diff --git a/src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py b/src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py index c7d1276ff0..7fdcf1c489 100644 --- a/src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py +++ b/src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py @@ -22,8 +22,7 @@ if TYPE_CHECKING: from collections.abc import AsyncGenerator, Awaitable, Callable, Iterator, Mapping - - from typing_extensions import Unpack + from typing import Unpack from crawlee import RequestTransformAction from crawlee._types import BasicCrawlingContext, EnqueueLinksKwargs, ExtractLinksFunction, JsonSerializable diff --git a/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py b/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py index e1189dbdf9..0d43d8f2b8 100644 --- a/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py +++ b/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py @@ -45,8 +45,7 @@ if TYPE_CHECKING: from collections.abc import MutableMapping from types import TracebackType - - from typing_extensions import Unpack + from typing import Unpack from crawlee.crawlers._basic._basic_crawler import _BasicCrawlerOptions diff --git a/src/crawlee/crawlers/_basic/_basic_crawler.py b/src/crawlee/crawlers/_basic/_basic_crawler.py index fdf43eadfa..6ca2a285ff 100644 --- a/src/crawlee/crawlers/_basic/_basic_crawler.py +++ b/src/crawlee/crawlers/_basic/_basic_crawler.py @@ -17,11 +17,11 @@ from http import HTTPStatus from io import StringIO from pathlib import Path -from typing import TYPE_CHECKING, Any, Generic, Literal, NotRequired, ParamSpec, cast +from typing import TYPE_CHECKING, Any, Generic, Literal, NotRequired, ParamSpec, TypedDict, Unpack, cast from weakref import WeakKeyDictionary from cachetools import LRUCache -from typing_extensions import TypedDict, TypeVar, Unpack +from typing_extensions import TypeVar from yarl import URL from crawlee import Glob, RequestTransformAction, service_locator diff --git a/src/crawlee/crawlers/_beautifulsoup/_beautifulsoup_crawler.py b/src/crawlee/crawlers/_beautifulsoup/_beautifulsoup_crawler.py index 919f26221e..35a2410f2c 100644 --- a/src/crawlee/crawlers/_beautifulsoup/_beautifulsoup_crawler.py +++ b/src/crawlee/crawlers/_beautifulsoup/_beautifulsoup_crawler.py @@ -12,8 +12,7 @@ if TYPE_CHECKING: from collections.abc import AsyncGenerator - - from typing_extensions import Unpack + from typing import Unpack from crawlee.crawlers._abstract_http import ParsedHttpCrawlingContext diff --git a/src/crawlee/crawlers/_beautifulsoup/_beautifulsoup_crawling_context.py b/src/crawlee/crawlers/_beautifulsoup/_beautifulsoup_crawling_context.py index e240de9204..bac39bd0e5 100644 --- a/src/crawlee/crawlers/_beautifulsoup/_beautifulsoup_crawling_context.py +++ b/src/crawlee/crawlers/_beautifulsoup/_beautifulsoup_crawling_context.py @@ -27,9 +27,7 @@ if TYPE_CHECKING: from collections.abc import Mapping, Sequence - from typing import Self - - from typing_extensions import Unpack + from typing import Self, Unpack from crawlee._request import Request from crawlee._utils.html import FormRequestOptions diff --git a/src/crawlee/crawlers/_file_download/_file_download_crawler.py b/src/crawlee/crawlers/_file_download/_file_download_crawler.py index 0d05c6eba9..4df3cd365f 100644 --- a/src/crawlee/crawlers/_file_download/_file_download_crawler.py +++ b/src/crawlee/crawlers/_file_download/_file_download_crawler.py @@ -14,8 +14,7 @@ if TYPE_CHECKING: from collections.abc import AsyncGenerator - - from typing_extensions import Unpack + from typing import Unpack from crawlee._types import BasicCrawlingContext from crawlee.crawlers._abstract_http import HttpCrawlerOptions diff --git a/src/crawlee/crawlers/_http/_http_crawler.py b/src/crawlee/crawlers/_http/_http_crawler.py index 2c098ecbc6..d2fd82f172 100644 --- a/src/crawlee/crawlers/_http/_http_crawler.py +++ b/src/crawlee/crawlers/_http/_http_crawler.py @@ -8,7 +8,7 @@ from ._http_parser import NoParser if TYPE_CHECKING: - from typing_extensions import Unpack + from typing import Unpack from crawlee.crawlers import BasicCrawlerOptions diff --git a/src/crawlee/crawlers/_parsel/_parsel_crawler.py b/src/crawlee/crawlers/_parsel/_parsel_crawler.py index ac8e9c9f09..1198328531 100644 --- a/src/crawlee/crawlers/_parsel/_parsel_crawler.py +++ b/src/crawlee/crawlers/_parsel/_parsel_crawler.py @@ -12,8 +12,7 @@ if TYPE_CHECKING: from collections.abc import AsyncGenerator - - from typing_extensions import Unpack + from typing import Unpack from crawlee.crawlers._abstract_http import ParsedHttpCrawlingContext diff --git a/src/crawlee/crawlers/_parsel/_parsel_crawling_context.py b/src/crawlee/crawlers/_parsel/_parsel_crawling_context.py index ad7363a1af..766fe1fd0d 100644 --- a/src/crawlee/crawlers/_parsel/_parsel_crawling_context.py +++ b/src/crawlee/crawlers/_parsel/_parsel_crawling_context.py @@ -16,9 +16,7 @@ if TYPE_CHECKING: from collections.abc import Mapping, Sequence - from typing import Self - - from typing_extensions import Unpack + from typing import Self, Unpack from crawlee._request import Request from crawlee._utils.html import FormRequestOptions diff --git a/src/crawlee/crawlers/_playwright/_playwright_crawler.py b/src/crawlee/crawlers/_playwright/_playwright_crawler.py index 6877f62ea9..497342607a 100644 --- a/src/crawlee/crawlers/_playwright/_playwright_crawler.py +++ b/src/crawlee/crawlers/_playwright/_playwright_crawler.py @@ -5,12 +5,12 @@ import warnings from datetime import timedelta from functools import partial -from typing import TYPE_CHECKING, Any, Generic, Literal, NotRequired, cast +from typing import TYPE_CHECKING, Any, Generic, Literal, NotRequired, TypedDict, cast import playwright.async_api from more_itertools import partition from pydantic import ValidationError -from typing_extensions import TypedDict, TypeVar +from typing_extensions import TypeVar from crawlee._request import Request, RequestOptions, RequestState from crawlee._types import BasicCrawlingContext, ConcurrencySettings @@ -38,9 +38,9 @@ if TYPE_CHECKING: from collections.abc import AsyncGenerator, Awaitable, Callable, Iterator, Mapping from pathlib import Path + from typing import Unpack from playwright.async_api import Page, Response - from typing_extensions import Unpack from crawlee import RequestTransformAction from crawlee._types import ( diff --git a/src/crawlee/crawlers/_pydantic_ai/_pydantic_ai_crawler.py b/src/crawlee/crawlers/_pydantic_ai/_pydantic_ai_crawler.py index 39318a400d..75ec9bb8b4 100644 --- a/src/crawlee/crawlers/_pydantic_ai/_pydantic_ai_crawler.py +++ b/src/crawlee/crawlers/_pydantic_ai/_pydantic_ai_crawler.py @@ -17,9 +17,9 @@ if TYPE_CHECKING: from collections.abc import AsyncGenerator + from typing import Unpack from pydantic_ai.models import Model - from typing_extensions import Unpack from crawlee import Request from crawlee.crawlers._abstract_http import ParsedHttpCrawlingContext diff --git a/src/crawlee/crawlers/_pydantic_ai/_types.py b/src/crawlee/crawlers/_pydantic_ai/_types.py index 7a0ce6528c..4ddad30eda 100644 --- a/src/crawlee/crawlers/_pydantic_ai/_types.py +++ b/src/crawlee/crawlers/_pydantic_ai/_types.py @@ -1,10 +1,9 @@ from __future__ import annotations from dataclasses import dataclass -from typing import TYPE_CHECKING, Protocol +from typing import TYPE_CHECKING, Protocol, TypeVar from pydantic import BaseModel -from typing_extensions import TypeVar from crawlee._utils.docs import docs_group diff --git a/src/crawlee/crawlers/_stagehand/_stagehand_crawler.py b/src/crawlee/crawlers/_stagehand/_stagehand_crawler.py index 45180176ad..c4f3cf9bd0 100644 --- a/src/crawlee/crawlers/_stagehand/_stagehand_crawler.py +++ b/src/crawlee/crawlers/_stagehand/_stagehand_crawler.py @@ -17,8 +17,7 @@ if TYPE_CHECKING: from datetime import timedelta from pathlib import Path - - from typing_extensions import Unpack + from typing import Unpack from crawlee.browsers import StagehandOptions from crawlee.crawlers._basic import BasicCrawlerOptions diff --git a/src/crawlee/events/_local_event_manager.py b/src/crawlee/events/_local_event_manager.py index c66483d2ed..e35494a756 100644 --- a/src/crawlee/events/_local_event_manager.py +++ b/src/crawlee/events/_local_event_manager.py @@ -16,9 +16,7 @@ if TYPE_CHECKING: from types import TracebackType - from typing import Self - - from typing_extensions import Unpack + from typing import Self, Unpack logger = getLogger(__name__) diff --git a/src/crawlee/proxy_configuration.py b/src/crawlee/proxy_configuration.py index 2047931d06..3a15963901 100644 --- a/src/crawlee/proxy_configuration.py +++ b/src/crawlee/proxy_configuration.py @@ -3,11 +3,10 @@ import inspect from collections import defaultdict from dataclasses import dataclass -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Protocol from more_itertools import flatten from pydantic import AnyHttpUrl, TypeAdapter -from typing_extensions import Protocol from yarl import URL from crawlee._utils.crypto import crypto_random_object_id diff --git a/src/crawlee/storages/_dataset.py b/src/crawlee/storages/_dataset.py index 0b4aece7ac..8b0fb89aac 100644 --- a/src/crawlee/storages/_dataset.py +++ b/src/crawlee/storages/_dataset.py @@ -16,9 +16,7 @@ if TYPE_CHECKING: from collections.abc import AsyncIterator, Mapping, Sequence - from typing import Any, Literal - - from typing_extensions import Unpack + from typing import Any, Literal, Unpack from crawlee._types import ExportDataCsvKwargs, ExportDataJsonKwargs, JsonSerializable from crawlee.configuration import Configuration From ed8666dd4769ab4e7fb8b6a05c1e6b3081c75874 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 13 Oct 2026 14:29:48 +0200 Subject: [PATCH 2/2] chore: import remaining plain `TypeVar`s from `typing` --- src/crawlee/crawlers/_abstract_http/_http_crawling_context.py | 4 +--- .../_adaptive_playwright/_adaptive_playwright_crawler.py | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/crawlee/crawlers/_abstract_http/_http_crawling_context.py b/src/crawlee/crawlers/_abstract_http/_http_crawling_context.py index 6538fd61fe..e687e16a68 100644 --- a/src/crawlee/crawlers/_abstract_http/_http_crawling_context.py +++ b/src/crawlee/crawlers/_abstract_http/_http_crawling_context.py @@ -1,9 +1,7 @@ from __future__ import annotations from dataclasses import dataclass, fields -from typing import Generic, Self - -from typing_extensions import TypeVar +from typing import Generic, Self, TypeVar from crawlee._types import BasicCrawlingContext, EnqueueLinksFunction, ExtractLinksFunction, PageSnapshot from crawlee._utils.docs import docs_group diff --git a/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py b/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py index 0d43d8f2b8..5bddea0490 100644 --- a/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py +++ b/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py @@ -6,11 +6,11 @@ from dataclasses import dataclass from logging import getLogger from random import random -from typing import TYPE_CHECKING, Any, Generic, Self, get_args +from typing import TYPE_CHECKING, Any, Generic, Self, TypeVar, get_args from bs4 import BeautifulSoup, Tag from parsel import Selector -from typing_extensions import TypeVar, override +from typing_extensions import override from crawlee._types import BasicCrawlingContext, ConcurrencySettings, JsonSerializable, RequestHandlerRunResult from crawlee._utils.docs import docs_group