From def2793c3ceb9981197098daf1797fecd716d2cf Mon Sep 17 00:00:00 2001 From: CheViana <656043+CheViana@users.noreply.github.com> Date: Tue, 22 Sep 2026 12:22:14 -0400 Subject: [PATCH 01/34] add docstring to types that are mentioned in public interface; remove empty json file; clean up ignore list --- src/trio/_core/_io_epoll.py | 11 +++ src/trio/_core/_io_kqueue.py | 11 +++ src/trio/_core/_io_windows.py | 13 +++ src/trio/_core/_local.py | 8 +- src/trio/_core/_run.py | 8 ++ src/trio/_file_io.py | 9 +- src/trio/_socket.py | 4 + src/trio/_subprocess.py | 7 +- src/trio/_sync.py | 20 +++- src/trio/_tests/_check_type_completeness.json | 24 ----- src/trio/_tests/check_type_completeness.py | 94 ++++++------------- src/trio/_unix_pipes.py | 12 +++ 12 files changed, 127 insertions(+), 94 deletions(-) delete mode 100644 src/trio/_tests/_check_type_completeness.json diff --git a/src/trio/_core/_io_epoll.py b/src/trio/_core/_io_epoll.py index d7064dc718..7bf1a0da07 100644 --- a/src/trio/_core/_io_epoll.py +++ b/src/trio/_core/_io_epoll.py @@ -33,6 +33,17 @@ class EpollWaiters: @attrs.frozen(eq=False) class _EpollStatistics: + """ + DTO class that holds information on current epoll status: + - amount of tasks waiting on read + - amount of tasks waiting on write + - backend = "epoll" + + This class is used with platform "Linux" or "Darwin" + (or any other platform that supports epoll functionality). + It has a sibling :class:`~trio._core._io_windows._WindowsStatistics` + that is used with platform "Windows". + """ tasks_waiting_read: int tasks_waiting_write: int backend: Literal["epoll"] = attrs.field(init=False, default="epoll") diff --git a/src/trio/_core/_io_kqueue.py b/src/trio/_core/_io_kqueue.py index 80d0ce6296..ad029d0509 100644 --- a/src/trio/_core/_io_kqueue.py +++ b/src/trio/_core/_io_kqueue.py @@ -26,6 +26,17 @@ @attrs.frozen(eq=False) class _KqueueStatistics: + """ + DTO class that holds information on current kqueue status: + - amount of tasks waiting + - amount of monitors + - backend = "kqueue" + + This class is used with platform "Linux" or "Darwin" + (or any other platform that supports epoll functionality). + It has a sibling :class:`~trio._core._io_windows._WindowsStatistics` + that is used with platform "Windows". + """ tasks_waiting: int monitors: int backend: Literal["kqueue"] = attrs.field(init=False, default="kqueue") diff --git a/src/trio/_core/_io_windows.py b/src/trio/_core/_io_windows.py index 7b789c3dec..1f5a1d94a3 100644 --- a/src/trio/_core/_io_windows.py +++ b/src/trio/_core/_io_windows.py @@ -293,6 +293,19 @@ class AFDGroup: @attrs.frozen(eq=False) class _WindowsStatistics: + """ + DTO class that holds information on current status + that pertains to IO waits: + - amount of tasks waiting on read + - amount of tasks waiting on write + - amount of overlapped tasks + - amount of completion monitors + - backend = "windows" + + This class is used with platform "Windows". + It has a sibling :class:`~trio._core._io_epoll._EpollStatistics` + that is used with platform "Linux" or "Darwin". + """ tasks_waiting_read: int tasks_waiting_write: int tasks_waiting_overlapped: int diff --git a/src/trio/_core/_local.py b/src/trio/_core/_local.py index 21bb913cd9..0fbda3fb1a 100644 --- a/src/trio/_core/_local.py +++ b/src/trio/_core/_local.py @@ -12,7 +12,13 @@ @final -class _NoValue: ... +class _NoValue: + """ + Indicates `RunVar` variable without a value + (e.g. when variable is not initialized with value yet). + Like `undefined` in JS (but better). + """ + ... @final diff --git a/src/trio/_core/_run.py b/src/trio/_core/_run.py index ddd4ea7ee2..b30fcfb604 100644 --- a/src/trio/_core/_run.py +++ b/src/trio/_core/_run.py @@ -1486,6 +1486,14 @@ def __del__(self) -> None: @final @attrs.define(eq=False, repr=False) class Task(metaclass=NoPublicConstructor): # type: ignore[explicit-any] + """ + Class for Trio's `Task` primitive. + Wraps coroutine, adding more functionality for stopping and resuming tasks, + for scheduling and canceling tasks, for assosiating a `Task` with `Runner` and + context (`contextvars.Context`), among other things. + `Task` can belong to `trio._core._run.Nursery` and can spawn child nurseries. + See `trio._core._run.Nursery` docs for more information. + """ _parent_nursery: Nursery | None coro: types.CoroutineType[Any, Outcome[object], Any] # type: ignore[explicit-any] _runner: Runner diff --git a/src/trio/_file_io.py b/src/trio/_file_io.py index d9305ef4ff..fe7319d774 100644 --- a/src/trio/_file_io.py +++ b/src/trio/_file_io.py @@ -114,7 +114,14 @@ class _HasErrors(Protocol): def errors(self) -> str | None: ... class _HasFileNo(Protocol): - def fileno(self) -> int: ... + """Represents any file-like object that has a file descriptor.""" + + def fileno(self) -> int: + """ + Integer that corresponds to file descriptor, as in + `sys.stdout.fileno()` + """ + ... class _HasIsATTY(Protocol): def isatty(self) -> bool: ... diff --git a/src/trio/_socket.py b/src/trio/_socket.py index 504874553a..578882ee1a 100644 --- a/src/trio/_socket.py +++ b/src/trio/_socket.py @@ -544,6 +544,10 @@ async def _resolve_address_nocp( class SocketType: + """ + Trio's version of standard library's :class:`socket.socket`. + Encompasses some of Trio-specific platform handling logic. + """ def __init__(self) -> None: # make sure this __init__ works with multiple inheritance super().__init__() diff --git a/src/trio/_subprocess.py b/src/trio/_subprocess.py index d73ba3dc23..76e9d54d77 100644 --- a/src/trio/_subprocess.py +++ b/src/trio/_subprocess.py @@ -98,7 +98,12 @@ def pidfd_open(fd: int, flags: int) -> int: class HasFileno(Protocol): """Represents any file-like object that has a file descriptor.""" - def fileno(self) -> int: ... + def fileno(self) -> int: + """ + Integer that corresponds to file descriptor, as in + `sys.stdout.fileno()` + """ + ... @final diff --git a/src/trio/_sync.py b/src/trio/_sync.py index 75d3be721a..f1d466d0fa 100644 --- a/src/trio/_sync.py +++ b/src/trio/_sync.py @@ -122,12 +122,28 @@ def __bool__(self) -> Literal[True]: class _HasAcquireRelease(Protocol): """Only classes with acquire() and release() can use the mixin's implementations.""" - async def acquire(self) -> object: ... + async def acquire(self) -> object: + """ + Method for resource locking. + Protocol implies this is similar to `threading.Semaphore.acquire`. + """ + ... - def release(self) -> object: ... + def release(self) -> object: + """ + Method for resource releasing. + Protocol implies this is similar to `threading.Semaphore.release`. + """ + ... class AsyncContextManagerMixin: + """ + An async context manager base class. + Should be used with :class:`~trio._sync._HasAcquireRelease` Protocol. + Calls `await self.acquire()` on entry and `self.release` on exit, + adding KeyboardInterrupt protection support. + """ @enable_ki_protection async def __aenter__(self: _HasAcquireRelease) -> None: await self.acquire() diff --git a/src/trio/_tests/_check_type_completeness.json b/src/trio/_tests/_check_type_completeness.json deleted file mode 100644 index d8a2948cd9..0000000000 --- a/src/trio/_tests/_check_type_completeness.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "Darwin": [ - "No docstring found for function \"trio._unix_pipes.FdStream.close\"", - "No docstring found for function \"trio._unix_pipes.FdStream.fileno\"" - ], - "Linux": [ - "No docstring found for class \"trio._core._io_epoll._EpollStatistics\"", - "No docstring found for function \"trio._unix_pipes.FdStream.close\"", - "No docstring found for function \"trio._unix_pipes.FdStream.fileno\"" - ], - "Windows": [], - "all": [ - "No docstring found for class \"trio._core._run.Task\"", - "No docstring found for class \"trio._socket.SocketType\"", - "No docstring found for function \"trio._subprocess.HasFileno.fileno\"", - "No docstring found for class \"trio._sync.AsyncContextManagerMixin\"", - "No docstring found for function \"trio._sync._HasAcquireRelease.acquire\"", - "No docstring found for function \"trio._sync._HasAcquireRelease.release\"", - "No docstring found for class \"trio._sync._LockImpl\"", - "No docstring found for class \"trio._core._local._NoValue\"", - "No docstring found for class \"trio.lowlevel.Task\"", - "No docstring found for class \"trio.socket.SocketType\"" - ] -} diff --git a/src/trio/_tests/check_type_completeness.py b/src/trio/_tests/check_type_completeness.py index bbf38080d3..284afee58e 100755 --- a/src/trio/_tests/check_type_completeness.py +++ b/src/trio/_tests/check_type_completeness.py @@ -69,37 +69,41 @@ def has_docstring_at_runtime(name: str) -> bool: if "AsyncIOWrapper" in str(exc) or name in ( # Symbols not existing on all platforms, so we can't dynamically inspect them. # Manually confirmed to have docstrings but pyright doesn't see them due to - # export shenanigans. TODO: actually manually confirm that. + # export shenanigans. # In theory we could verify these at runtime, probably by running the script separately # on separate platforms. It might also be a decent idea to work the other way around, # a la test_static_tool_sees_class_members # darwin - "trio.lowlevel.current_kqueue", - "trio.lowlevel.monitor_kevent", - "trio.lowlevel.wait_kevent", - "trio._core._io_kqueue._KqueueStatistics", + "trio._core._io_kqueue._KqueueStatistics", # confirmed to have docstring # windows - "trio._socket.SocketType.share", - "trio._core._io_windows._WindowsStatistics", - "trio._core._windows_cffi.Handle", - "trio.lowlevel.current_iocp", - "trio.lowlevel.monitor_completion_key", - "trio.lowlevel.readinto_overlapped", - "trio.lowlevel.register_with_iocp", - "trio.lowlevel.wait_overlapped", - "trio.lowlevel.write_overlapped", - "trio.lowlevel.WaitForSingleObject", - "trio.socket.fromshare", + "trio._socket.SocketType.share", # doesn't have docstring, like other + # SocketType methods. Shows up here bc method def is wrapped in `if not TYPE_CHECKING` + "trio._core._io_windows._WindowsStatistics", # confirmed to have docstring + "trio._core._windows_cffi.Handle", # doesn't have docstring, it's basically type alias # linux # this test will fail on linux, but I don't develop on linux. So the next # person to do so is very welcome to open a pull request and populate with # objects # TODO: these are erroring on all platforms, why? - "trio._highlevel_generic.StapledStream.send_stream", - "trio._highlevel_generic.StapledStream.receive_stream", - "trio._ssl.SSLStream.transport_stream", + # It could be due to the fact that this file runs with + # `typing.TYPE_CHECKING=False` flag, whereas `pyright` executes + # with flag `typing.TYPE_CHECKING=True`. + # `AsyncIOWrapper` has conditional method stubs, only for type checking, + # and docstring is in real method. + # In other places, private types are imported (or publicly reexported) + # when `not TYPE_CHECKING`. + # This could be why this the whole of this exception handler is like this. + # If one puts `typing.TYPE_CHECKING=True` before trio imports on top + # of this file, there's circular import error. + # + # _io_poll.py and _io_windows.py and _io_kqueue.py: + # ``` + # if TYPE_CHECKING: + # from .._file_io import _HasFileNo + # ``` "trio._file_io._HasFileNo", "trio._file_io._HasFileNo.fileno", + # verified that `HasFileNo` has docstrings ): return True @@ -116,7 +120,6 @@ def has_docstring_at_runtime(name: str) -> bool: def check_type( platform: str, full_diagnostics_file: Path | None, - expected_errors: list[object], ) -> list[object]: # convince isort we use the trio import assert trio is not None @@ -161,7 +164,7 @@ def check_type( # Missing docstring messages include the name of the object. # Other errors don't, so we add it. message = f"{name}: {message}" - if message not in expected_errors and message not in printed_diagnostics: + if message not in printed_diagnostics: print(f"new error: {message}", file=sys.stderr) errors.append(message) printed_diagnostics.add(message) @@ -178,65 +181,26 @@ def main(args: argparse.Namespace) -> int: else: full_diagnostics_file = None - errors_by_platform_file = Path(__file__).parent / "_check_type_completeness.json" - if errors_by_platform_file.exists(): - with open(errors_by_platform_file) as f: - errors_by_platform = json.load(f) - else: - errors_by_platform = {"Linux": [], "Windows": [], "Darwin": [], "all": []} - changed = False + has_errors = False for platform in "Linux", "Windows", "Darwin": - platform_errors = errors_by_platform[platform] + errors_by_platform["all"] print("*" * 20, f"\nChecking {platform}...") - errors = check_type(platform, full_diagnostics_file, platform_errors) - - new_errors = [e for e in errors if e not in platform_errors] - missing_errors = [e for e in platform_errors if e not in errors] + errors = check_type(platform, full_diagnostics_file) - if new_errors: + if errors: print( f"New errors introduced in `pyright --verifytypes`. Fix them, or ignore them by modifying {errors_by_platform_file}, either manually or with '--overwrite-file'.", file=sys.stderr, ) - changed = True - if missing_errors: - print( - f"Congratulations, you have resolved existing errors! Please remove them from {errors_by_platform_file}, either manually or with '--overwrite-file'.", - file=sys.stderr, - ) - changed = True - print(missing_errors, file=sys.stderr) + has_errors = True - errors_by_platform[platform] = errors print("*" * 20) - # cut down the size of the json file by a lot, and make it easier to parse for - # humans, by moving errors that appear on all platforms to a separate category - errors_by_platform["all"] = [] - for e in errors_by_platform["Linux"].copy(): - if e in errors_by_platform["Darwin"] and e in errors_by_platform["Windows"]: - for platform in "Linux", "Windows", "Darwin": - errors_by_platform[platform].remove(e) - errors_by_platform["all"].append(e) - - if changed and args.overwrite_file: - with open(errors_by_platform_file, "w") as f: - json.dump(errors_by_platform, f, indent=4, sort_keys=True) - # newline at end of file - f.write("\n") - # True -> 1 -> non-zero exit value -> error - return changed + return has_errors parser = argparse.ArgumentParser() -parser.add_argument( - "--overwrite-file", - action="store_true", - default=False, - help="Use this flag to overwrite the current stored results. Either in CI together with a diff check, or to avoid having to manually correct it.", -) parser.add_argument( "--full-diagnostics-file", type=Path, diff --git a/src/trio/_unix_pipes.py b/src/trio/_unix_pipes.py index 598feb61e2..eeee5be8f4 100644 --- a/src/trio/_unix_pipes.py +++ b/src/trio/_unix_pipes.py @@ -187,11 +187,23 @@ async def receive_some(self, max_bytes: int | None = None) -> bytes: return data def close(self) -> None: + """ + Close the stream and close file descriptor it's wrapping. + See class docstring for details. + """ self._fd_holder.close() async def aclose(self) -> None: + """ + Close (syncronously) the stream and close file descriptor + it's wrapping. See class docstring for details. + """ self.close() await trio.lowlevel.checkpoint() def fileno(self) -> int: + """ + Returns integer representing the file descriptor this `FdStream` class + instance is wrapping. See class docstring for details. + """ return self._fd_holder.fd From e699edfc0cf1186da95e430ae4c99b349e3c1ff8 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 16:27:26 +0000 Subject: [PATCH 02/34] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/trio/_core/_io_epoll.py | 1 + src/trio/_core/_io_kqueue.py | 1 + src/trio/_core/_io_windows.py | 1 + src/trio/_core/_local.py | 1 - src/trio/_core/_run.py | 3 ++- src/trio/_socket.py | 1 + src/trio/_sync.py | 1 + src/trio/_tests/check_type_completeness.py | 1 - src/trio/_unix_pipes.py | 2 +- 9 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/trio/_core/_io_epoll.py b/src/trio/_core/_io_epoll.py index 7bf1a0da07..9b13da428c 100644 --- a/src/trio/_core/_io_epoll.py +++ b/src/trio/_core/_io_epoll.py @@ -44,6 +44,7 @@ class _EpollStatistics: It has a sibling :class:`~trio._core._io_windows._WindowsStatistics` that is used with platform "Windows". """ + tasks_waiting_read: int tasks_waiting_write: int backend: Literal["epoll"] = attrs.field(init=False, default="epoll") diff --git a/src/trio/_core/_io_kqueue.py b/src/trio/_core/_io_kqueue.py index ad029d0509..e1979a6a4f 100644 --- a/src/trio/_core/_io_kqueue.py +++ b/src/trio/_core/_io_kqueue.py @@ -37,6 +37,7 @@ class _KqueueStatistics: It has a sibling :class:`~trio._core._io_windows._WindowsStatistics` that is used with platform "Windows". """ + tasks_waiting: int monitors: int backend: Literal["kqueue"] = attrs.field(init=False, default="kqueue") diff --git a/src/trio/_core/_io_windows.py b/src/trio/_core/_io_windows.py index 1f5a1d94a3..c30d4e101b 100644 --- a/src/trio/_core/_io_windows.py +++ b/src/trio/_core/_io_windows.py @@ -306,6 +306,7 @@ class _WindowsStatistics: It has a sibling :class:`~trio._core._io_epoll._EpollStatistics` that is used with platform "Linux" or "Darwin". """ + tasks_waiting_read: int tasks_waiting_write: int tasks_waiting_overlapped: int diff --git a/src/trio/_core/_local.py b/src/trio/_core/_local.py index 0fbda3fb1a..8f8751a5d7 100644 --- a/src/trio/_core/_local.py +++ b/src/trio/_core/_local.py @@ -18,7 +18,6 @@ class _NoValue: (e.g. when variable is not initialized with value yet). Like `undefined` in JS (but better). """ - ... @final diff --git a/src/trio/_core/_run.py b/src/trio/_core/_run.py index b30fcfb604..e9b3d05a99 100644 --- a/src/trio/_core/_run.py +++ b/src/trio/_core/_run.py @@ -1489,11 +1489,12 @@ class Task(metaclass=NoPublicConstructor): # type: ignore[explicit-any] """ Class for Trio's `Task` primitive. Wraps coroutine, adding more functionality for stopping and resuming tasks, - for scheduling and canceling tasks, for assosiating a `Task` with `Runner` and + for scheduling and canceling tasks, for associating a `Task` with `Runner` and context (`contextvars.Context`), among other things. `Task` can belong to `trio._core._run.Nursery` and can spawn child nurseries. See `trio._core._run.Nursery` docs for more information. """ + _parent_nursery: Nursery | None coro: types.CoroutineType[Any, Outcome[object], Any] # type: ignore[explicit-any] _runner: Runner diff --git a/src/trio/_socket.py b/src/trio/_socket.py index 578882ee1a..4ba9d2cbb1 100644 --- a/src/trio/_socket.py +++ b/src/trio/_socket.py @@ -548,6 +548,7 @@ class SocketType: Trio's version of standard library's :class:`socket.socket`. Encompasses some of Trio-specific platform handling logic. """ + def __init__(self) -> None: # make sure this __init__ works with multiple inheritance super().__init__() diff --git a/src/trio/_sync.py b/src/trio/_sync.py index f1d466d0fa..d0364a2b99 100644 --- a/src/trio/_sync.py +++ b/src/trio/_sync.py @@ -144,6 +144,7 @@ class AsyncContextManagerMixin: Calls `await self.acquire()` on entry and `self.release` on exit, adding KeyboardInterrupt protection support. """ + @enable_ki_protection async def __aenter__(self: _HasAcquireRelease) -> None: await self.acquire() diff --git a/src/trio/_tests/check_type_completeness.py b/src/trio/_tests/check_type_completeness.py index 284afee58e..460a4be345 100755 --- a/src/trio/_tests/check_type_completeness.py +++ b/src/trio/_tests/check_type_completeness.py @@ -181,7 +181,6 @@ def main(args: argparse.Namespace) -> int: else: full_diagnostics_file = None - has_errors = False for platform in "Linux", "Windows", "Darwin": print("*" * 20, f"\nChecking {platform}...") diff --git a/src/trio/_unix_pipes.py b/src/trio/_unix_pipes.py index eeee5be8f4..6c60e069cb 100644 --- a/src/trio/_unix_pipes.py +++ b/src/trio/_unix_pipes.py @@ -195,7 +195,7 @@ def close(self) -> None: async def aclose(self) -> None: """ - Close (syncronously) the stream and close file descriptor + Close (synchronously) the stream and close file descriptor it's wrapping. See class docstring for details. """ self.close() From 77431bfafae02e10616965572392beed349fc55b Mon Sep 17 00:00:00 2001 From: CheViana <656043+CheViana@users.noreply.github.com> Date: Tue, 22 Sep 2026 12:32:56 -0400 Subject: [PATCH 03/34] fix sphinx docs, add newsfragment --- docs/source/conf.py | 1 - newsfragments/3226.bugfix.rst | 1 + src/trio/_subprocess.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 newsfragments/3226.bugfix.rst diff --git a/docs/source/conf.py b/docs/source/conf.py index eb278bf65f..46a3f6bae3 100755 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -173,7 +173,6 @@ def autodoc_process_signature( # currently undocumented things logger = getLogger("trio") UNDOCUMENTED = { - "trio._subprocess.HasFileno.fileno", "trio.lowlevel.ParkingLot.broken_by", } diff --git a/newsfragments/3226.bugfix.rst b/newsfragments/3226.bugfix.rst new file mode 100644 index 0000000000..e706d0b90e --- /dev/null +++ b/newsfragments/3226.bugfix.rst @@ -0,0 +1 @@ +Add docstrings for the classes/functions that pyright --verifytypes complains about. \ No newline at end of file diff --git a/src/trio/_subprocess.py b/src/trio/_subprocess.py index 76e9d54d77..897a102eca 100644 --- a/src/trio/_subprocess.py +++ b/src/trio/_subprocess.py @@ -101,7 +101,7 @@ class HasFileno(Protocol): def fileno(self) -> int: """ Integer that corresponds to file descriptor, as in - `sys.stdout.fileno()` + :func:`~sys.stdout.fileno()` """ ... From 9f7cf26d177d666123098c53cb2480ebf9c7eacc Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 16:33:15 +0000 Subject: [PATCH 04/34] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- newsfragments/3226.bugfix.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newsfragments/3226.bugfix.rst b/newsfragments/3226.bugfix.rst index e706d0b90e..bef67aeef8 100644 --- a/newsfragments/3226.bugfix.rst +++ b/newsfragments/3226.bugfix.rst @@ -1 +1 @@ -Add docstrings for the classes/functions that pyright --verifytypes complains about. \ No newline at end of file +Add docstrings for the classes/functions that pyright --verifytypes complains about. From 7b5ff58b37e3583fdd99c3e78a512a2b074dee5d Mon Sep 17 00:00:00 2001 From: CheViana <656043+CheViana@users.noreply.github.com> Date: Tue, 22 Sep 2026 12:35:29 -0400 Subject: [PATCH 05/34] shorter docstring --- src/trio/_subprocess.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/trio/_subprocess.py b/src/trio/_subprocess.py index 897a102eca..d4966b8d91 100644 --- a/src/trio/_subprocess.py +++ b/src/trio/_subprocess.py @@ -100,8 +100,7 @@ class HasFileno(Protocol): def fileno(self) -> int: """ - Integer that corresponds to file descriptor, as in - :func:`~sys.stdout.fileno()` + Integer that corresponds to file descriptor. """ ... From c412c9378abfe6b04fadd34c007b02951eb41c93 Mon Sep 17 00:00:00 2001 From: CheViana <656043+CheViana@users.noreply.github.com> Date: Tue, 22 Sep 2026 12:40:28 -0400 Subject: [PATCH 06/34] fix print with missing var --- src/trio/_tests/check_type_completeness.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/trio/_tests/check_type_completeness.py b/src/trio/_tests/check_type_completeness.py index 460a4be345..d43d80fc04 100755 --- a/src/trio/_tests/check_type_completeness.py +++ b/src/trio/_tests/check_type_completeness.py @@ -188,7 +188,7 @@ def main(args: argparse.Namespace) -> int: if errors: print( - f"New errors introduced in `pyright --verifytypes`. Fix them, or ignore them by modifying {errors_by_platform_file}, either manually or with '--overwrite-file'.", + "New errors introduced in `pyright --verifytypes`. Fix them.", file=sys.stderr, ) has_errors = True From 077db9a6b51cc903a9351172f2c02250b968075c Mon Sep 17 00:00:00 2001 From: Ievgeniia Radetska <656043+CheViana@users.noreply.github.com> Date: Thu, 24 Sep 2026 09:54:40 -0400 Subject: [PATCH 07/34] Update 3226.bugfix.rst --- newsfragments/3226.bugfix.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newsfragments/3226.bugfix.rst b/newsfragments/3226.bugfix.rst index bef67aeef8..6aa4e8c525 100644 --- a/newsfragments/3226.bugfix.rst +++ b/newsfragments/3226.bugfix.rst @@ -1 +1 @@ -Add docstrings for the classes/functions that pyright --verifytypes complains about. +Add docstrings for the classes and functions that ``pyright --verifytypes`` complains about. From e12f14e5c8b56838fae575fd799a5fd53c13e4f4 Mon Sep 17 00:00:00 2001 From: Ievgeniia Radetska <656043+CheViana@users.noreply.github.com> Date: Thu, 24 Sep 2026 10:02:27 -0400 Subject: [PATCH 08/34] print missing_names for debug --- src/trio/_tests/test_exports.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/trio/_tests/test_exports.py b/src/trio/_tests/test_exports.py index 1c2b46ca47..f7bff8d210 100644 --- a/src/trio/_tests/test_exports.py +++ b/src/trio/_tests/test_exports.py @@ -239,6 +239,9 @@ def no_underscores(symbols: Iterable[str]) -> set[str]: # static analysis (e.g. in trio.socket or trio.lowlevel) # So we check that the runtime names are a subset of the static names. missing_names = runtime_names - static_names + print(f"missing_names=}) + print(f"runtime_names=}) + print(f"static_names=}) # ignore warnings about deprecated module tests missing_names -= {"tests"} From d6a1c2357529e173f8297e9c3f54b161038ac0f6 Mon Sep 17 00:00:00 2001 From: Jane R Date: Fri, 25 Sep 2026 10:12:46 -0400 Subject: [PATCH 09/34] fix prints --- src/trio/_tests/test_exports.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/trio/_tests/test_exports.py b/src/trio/_tests/test_exports.py index f7bff8d210..684dba9d3e 100644 --- a/src/trio/_tests/test_exports.py +++ b/src/trio/_tests/test_exports.py @@ -239,9 +239,9 @@ def no_underscores(symbols: Iterable[str]) -> set[str]: # static analysis (e.g. in trio.socket or trio.lowlevel) # So we check that the runtime names are a subset of the static names. missing_names = runtime_names - static_names - print(f"missing_names=}) - print(f"runtime_names=}) - print(f"static_names=}) + print(f"{missing_names=}") + print(f"{runtime_names=}") + print(f"{static_names=}") # ignore warnings about deprecated module tests missing_names -= {"tests"} From 75b504835a11342bdf7b64bd474b5c1288cd95ce Mon Sep 17 00:00:00 2001 From: Ievgeniia Radetska <656043+CheViana@users.noreply.github.com> Date: Sat, 26 Sep 2026 09:13:51 -0400 Subject: [PATCH 10/34] debug test_exports.py where ALG_SET_PUB_KEY is from? should be called ALG_SET_PUBKEY https://github.com/python/cpython/blob/8122ff4818f418e8ae984cf6964215689b00808d/Modules/socketmodule.h#L211 --- src/trio/_tests/test_exports.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/trio/_tests/test_exports.py b/src/trio/_tests/test_exports.py index 684dba9d3e..a5d77944c8 100644 --- a/src/trio/_tests/test_exports.py +++ b/src/trio/_tests/test_exports.py @@ -132,14 +132,29 @@ def iter_modules( sys.version_info.releaselevel == "alpha", reason="skip static introspection tools on Python dev/alpha releases", ) -@pytest.mark.parametrize("modname", PUBLIC_MODULE_NAMES) -@pytest.mark.parametrize("tool", ["pylint", "jedi", "mypy", "pyright_verifytypes"]) +@pytest.mark.parametrize( + "modname", + # PUBLIC_MODULE_NAMES + ["trio.socket"] +) +@pytest.mark.parametrize( + "tool", + [ + "pylint", + # "jedi", + # "mypy", + # "pyright_verifytypes" + ] +) @pytest.mark.filterwarnings( # https://github.com/pypa/setuptools/issues/3274 "ignore:module 'sre_constants' is deprecated:DeprecationWarning", ) def test_static_tool_sees_all_symbols(tool: str, modname: str, tmp_path: Path) -> None: + print(sys.version_info) module = importlib.import_module(modname) + print(dir(module)) + assert False def no_underscores(symbols: Iterable[str]) -> set[str]: return {symbol for symbol in symbols if not symbol.startswith("_")} From 61f6fe1116d32549e4794a67d4ee1230cb3215b6 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 26 Sep 2026 13:14:09 +0000 Subject: [PATCH 11/34] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/trio/_tests/test_exports.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/trio/_tests/test_exports.py b/src/trio/_tests/test_exports.py index a5d77944c8..333ae698ad 100644 --- a/src/trio/_tests/test_exports.py +++ b/src/trio/_tests/test_exports.py @@ -135,7 +135,7 @@ def iter_modules( @pytest.mark.parametrize( "modname", # PUBLIC_MODULE_NAMES - ["trio.socket"] + ["trio.socket"], ) @pytest.mark.parametrize( "tool", @@ -144,7 +144,7 @@ def iter_modules( # "jedi", # "mypy", # "pyright_verifytypes" - ] + ], ) @pytest.mark.filterwarnings( # https://github.com/pypa/setuptools/issues/3274 From ff19be6a5436b01c95134787ece5a15c776454c8 Mon Sep 17 00:00:00 2001 From: Jane R Date: Sat, 26 Sep 2026 09:33:19 -0400 Subject: [PATCH 12/34] remove debugs from test_Exports.py --- src/trio/_tests/test_exports.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/src/trio/_tests/test_exports.py b/src/trio/_tests/test_exports.py index 333ae698ad..8ab833175a 100644 --- a/src/trio/_tests/test_exports.py +++ b/src/trio/_tests/test_exports.py @@ -134,16 +134,15 @@ def iter_modules( ) @pytest.mark.parametrize( "modname", - # PUBLIC_MODULE_NAMES - ["trio.socket"], + PUBLIC_MODULE_NAMES ) @pytest.mark.parametrize( "tool", [ "pylint", - # "jedi", - # "mypy", - # "pyright_verifytypes" + "jedi", + "mypy", + "pyright_verifytypes" ], ) @pytest.mark.filterwarnings( @@ -151,10 +150,7 @@ def iter_modules( "ignore:module 'sre_constants' is deprecated:DeprecationWarning", ) def test_static_tool_sees_all_symbols(tool: str, modname: str, tmp_path: Path) -> None: - print(sys.version_info) module = importlib.import_module(modname) - print(dir(module)) - assert False def no_underscores(symbols: Iterable[str]) -> set[str]: return {symbol for symbol in symbols if not symbol.startswith("_")} @@ -254,9 +250,6 @@ def no_underscores(symbols: Iterable[str]) -> set[str]: # static analysis (e.g. in trio.socket or trio.lowlevel) # So we check that the runtime names are a subset of the static names. missing_names = runtime_names - static_names - print(f"{missing_names=}") - print(f"{runtime_names=}") - print(f"{static_names=}") # ignore warnings about deprecated module tests missing_names -= {"tests"} From a9c50d58123060df4c5fe477fc0d8e4309fea3ff Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 26 Sep 2026 13:33:51 +0000 Subject: [PATCH 13/34] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/trio/_tests/test_exports.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/src/trio/_tests/test_exports.py b/src/trio/_tests/test_exports.py index 8ab833175a..550a1979cd 100644 --- a/src/trio/_tests/test_exports.py +++ b/src/trio/_tests/test_exports.py @@ -132,18 +132,10 @@ def iter_modules( sys.version_info.releaselevel == "alpha", reason="skip static introspection tools on Python dev/alpha releases", ) -@pytest.mark.parametrize( - "modname", - PUBLIC_MODULE_NAMES -) +@pytest.mark.parametrize("modname", PUBLIC_MODULE_NAMES) @pytest.mark.parametrize( "tool", - [ - "pylint", - "jedi", - "mypy", - "pyright_verifytypes" - ], + ["pylint", "jedi", "mypy", "pyright_verifytypes"], ) @pytest.mark.filterwarnings( # https://github.com/pypa/setuptools/issues/3274 From 18ab53186694cee4689bcdf9599560727d2f660b Mon Sep 17 00:00:00 2001 From: Jane R Date: Sat, 26 Sep 2026 09:35:45 -0400 Subject: [PATCH 14/34] exclude missing name with typo per bug in pypy --- src/trio/_tests/test_exports.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/trio/_tests/test_exports.py b/src/trio/_tests/test_exports.py index 8ab833175a..1f76ed52be 100644 --- a/src/trio/_tests/test_exports.py +++ b/src/trio/_tests/test_exports.py @@ -253,6 +253,7 @@ def no_underscores(symbols: Iterable[str]) -> set[str]: # ignore warnings about deprecated module tests missing_names -= {"tests"} + missing_names -= {"ALG_SET_PUB_KEY"} # https://github.com/pypy/pypy/issues/5594 if missing_names: # pragma: no cover print(f"{tool} can't see the following names in {modname}:") From 5ff554f41ed39e7cb7d2af96da8f6382920c8801 Mon Sep 17 00:00:00 2001 From: Jane R Date: Sat, 26 Sep 2026 13:18:56 -0400 Subject: [PATCH 15/34] pin pypy to 7.3 for windows --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ef26a0b2bb..f567e7131d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -184,7 +184,7 @@ jobs: lsp: 'https://www.proxifier.com/download/legacy/ProxifierSetup342.exe' lsp_extract_file: '' extra_name: ', with IFS LSP' - - python: 'pypy-3.11' + - python: 'pypy-3.11-v7.3.23' # pin to 7.3 bc of bugs in pypy 8.0 for win arch: 'x64' lsp: '' lsp_extract_file: '' From a20ef57fd40fa5c88c835d10d433906b1dbef88e Mon Sep 17 00:00:00 2001 From: Jane R Date: Wed, 30 Sep 2026 12:19:40 -0400 Subject: [PATCH 16/34] pip install verbose --- .github/workflows/ci.yml | 2 +- ci.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f567e7131d..ef26a0b2bb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -184,7 +184,7 @@ jobs: lsp: 'https://www.proxifier.com/download/legacy/ProxifierSetup342.exe' lsp_extract_file: '' extra_name: ', with IFS LSP' - - python: 'pypy-3.11-v7.3.23' # pin to 7.3 bc of bugs in pypy 8.0 for win + - python: 'pypy-3.11' arch: 'x64' lsp: '' lsp_extract_file: '' diff --git a/ci.sh b/ci.sh index 9362dbc727..303da4de0d 100755 --- a/ci.sh +++ b/ci.sh @@ -54,7 +54,7 @@ if [ "${NO_TEST_REQUIREMENTS-0}" == 1 ]; then python -m uv pip install pytest coverage -c test-requirements.txt flags="--skip-optional-imports" else - python -m uv pip install -r test-requirements.txt --no-deps + python -m uv pip install -r test-requirements.txt --no-deps -vvv flags="" fi From 715c75cbd7aa6b4cfe132d48ee1cad904aeef575 Mon Sep 17 00:00:00 2001 From: CheViana <656043+CheViana@users.noreply.github.com> Date: Wed, 30 Sep 2026 13:21:22 -0400 Subject: [PATCH 17/34] verbose pip install --- ci.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci.sh b/ci.sh index d8d1094699..1ea2c49219 100755 --- a/ci.sh +++ b/ci.sh @@ -60,7 +60,7 @@ else PIP_ARGS="--exclude excluded_deps" flags="--skip-ssl-imports" fi - python -m uv pip install -r test-requirements.txt --no-deps $PIP_ARGS + python -m uv pip install -r test-requirements.txt -vvv --no-deps $PIP_ARGS fi # If we're testing with a LSP installed, then it might break network From fdc413bbc9b3badcfb5204dace50ec02f3522c85 Mon Sep 17 00:00:00 2001 From: Jane R Date: Wed, 30 Sep 2026 14:23:25 -0400 Subject: [PATCH 18/34] merge, bump crypto --- .github/workflows/ci.yml | 1016 +++--- ci.sh | 314 +- src/trio/_tests/pytest_plugin.py | 120 +- src/trio/_tests/test_dtls.py | 1904 ++++++------ src/trio/_tests/test_exports.py | 1276 ++++---- src/trio/_tests/test_ssl.py | 2744 ++++++++--------- src/trio/_tests/tools/test_gen_exports.py | 358 +-- .../_tests/tools/test_sync_requirements.py | 158 +- test-requirements.txt | 420 +-- 9 files changed, 4155 insertions(+), 4155 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5e2e758ecd..6efe8bc63b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,508 +1,508 @@ -name: CI - -permissions: {} - -on: - push: - branches-ignore: - # these branches always have another event associated - - gh-readonly-queue/** # GitHub's merge queue uses `merge_group` - - autodeps/** # autodeps always makes a PR - - pre-commit-ci-update-config # pre-commit.ci's updates always have a PR - pull_request: - merge_group: - -concurrency: - group: ${{ github.ref }}-${{ github.workflow }}-${{ github.event_name }}${{ github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && format('-{0}', github.sha) || '' }} - cancel-in-progress: true - -env: - dists-artifact-name: python-package-distributions - dist-name: trio - -jobs: - build: - name: 👷 dists - - runs-on: ubuntu-latest - - outputs: - dist-version: ${{ steps.dist-version.outputs.version }} - sdist-artifact-name: ${{ steps.artifact-name.outputs.sdist }} - wheel-artifact-name: ${{ steps.artifact-name.outputs.wheel }} - - steps: - - name: Switch to using Python 3.11 - uses: actions/setup-python@v6 - with: - python-version: 3.11 - - - name: Grab the source from Git - uses: actions/checkout@v7 - with: - persist-credentials: false - - - name: Get the dist version - id: dist-version - run: >- - echo "version=$( - grep ^__version__ src/trio/_version.py - | sed 's#__version__ = "\([^"]\+\)"#\1#' - )" - >> "${GITHUB_OUTPUT}" - - - name: Set the expected dist artifact names - id: artifact-name - run: | - echo "sdist=${DIST_NAME}-${VERSION}.tar.gz" >> "${GITHUB_OUTPUT}" - echo "wheel=${DIST_NAME}-${VERSION}-py3-none-any.whl" >> "${GITHUB_OUTPUT}" - env: - DIST_NAME: ${{ env.dist-name }} - VERSION: ${{ steps.dist-version.outputs.version }} - - - name: Install build - run: python -Im pip install build - - - name: Build dists - run: python -Im build - - name: Verify that the artifacts with expected names got created - run: >- - ls -1 - dist/${SDIST} - dist/${WHEEL} - env: - SDIST: ${{ steps.artifact-name.outputs.sdist }} - WHEEL: ${{ steps.artifact-name.outputs.wheel }} - - name: Store the distribution packages - uses: actions/upload-artifact@v7 - with: - name: ${{ env.dists-artifact-name }} - # NOTE: Exact expected file names are specified here - # NOTE: as a safety measure — if anything weird ends - # NOTE: up being in this dir or not all dists will be - # NOTE: produced, this will fail the workflow. - path: | - dist/${{ steps.artifact-name.outputs.sdist }} - dist/${{ steps.artifact-name.outputs.wheel }} - retention-days: 5 - - smoke-tests: - name: Smoke tests - needs: - - build - - runs-on: ubuntu-latest - - steps: - - name: Switch to using Python 3.11 - uses: actions/setup-python@v6 - with: - python-version: 3.11 - - - name: >- - Smoke-test: - retrieve the project source from an sdist inside the GHA artifact - uses: re-actors/checkout-python-sdist@release/v2 - with: - source-tarball-name: ${{ needs.build.outputs.sdist-artifact-name }} - workflow-artifact-name: ${{ env.dists-artifact-name }} - - - name: >- - Smoke-test: move the sdist-retrieved dir into sdist-src - run: | - mv -v '${{ github.workspace }}' '${{ runner.temp }}/sdist-src' - mkdir -pv '${{ github.workspace }}' - mv -v '${{ runner.temp }}/sdist-src' '${{ github.workspace }}/sdist-src' - shell: bash -eEuo pipefail {0} - - - name: >- - Smoke-test: grab the source from Git into git-src - uses: actions/checkout@v7 - with: - path: git-src - persist-credentials: false - - - name: >- - Smoke-test: install test requirements from the Git repo - run: >- - python -Im - pip install -c test-requirements.txt -r test-requirements.txt - shell: bash -eEuo pipefail {0} - working-directory: git-src - - - name: >- - Smoke-test: collect tests from the Git repo - env: - PYTHONPATH: src/ - run: >- - pytest --collect-only -qq . - | sort - | tee collected-tests - shell: bash -eEuo pipefail {0} - working-directory: git-src - - - name: >- - Smoke-test: collect tests from the sdist tarball - env: - PYTHONPATH: src/ - run: >- - pytest --collect-only -qq . - | sort - | tee collected-tests - shell: bash -eEuo pipefail {0} - working-directory: sdist-src - - - name: >- - Smoke-test: - verify that all the tests from Git are included in the sdist - run: diff --unified sdist-src/collected-tests git-src/collected-tests - shell: bash -eEuo pipefail {0} - - Windows: - name: 'Windows (${{ matrix.python }}, ${{ matrix.arch }}${{ matrix.extra_name }})' - needs: - - build - - timeout-minutes: 20 - runs-on: 'windows-latest' - strategy: - fail-fast: false - matrix: - python: ['3.10', '3.11', '3.12', '3.13', '3.14', '3.14t', '3.15-dev'] - arch: ['x86', 'x64'] - lsp: [''] - lsp_extract_file: [''] - extra_name: [''] - include: - - python: '3.10' - arch: 'x64' - lsp: 'https://raw.githubusercontent.com/python-trio/trio-ci-assets/master/komodia-based-vpn-setup.zip' - lsp_extract_file: 'komodia-based-vpn-setup.exe' - extra_name: ', with Komodia LSP' - - python: '3.10' - arch: 'x64' - lsp: 'https://www.proxifier.com/download/legacy/ProxifierSetup342.exe' - lsp_extract_file: '' - extra_name: ', with IFS LSP' - - python: 'pypy-3.11' - arch: 'x64' - lsp: '' - lsp_extract_file: '' - extra_name: '' - #- python: '3.10' - # arch: 'x64' - # lsp: 'http://download.pctools.com/mirror/updates/9.0.0.2308-SDavfree-lite_en.exe' - # lsp_extract_file: '' - # extra_name: ', with non-IFS LSP' - - continue-on-error: >- - ${{ - ( - endsWith(matrix.python, '-dev') - || endsWith(matrix.python, '-nightly') - ) - && true - || false - }} - steps: - - name: Retrieve the project source from an sdist inside the GHA artifact - uses: re-actors/checkout-python-sdist@release/v2 - with: - source-tarball-name: ${{ needs.build.outputs.sdist-artifact-name }} - workflow-artifact-name: ${{ env.dists-artifact-name }} - - name: Setup Python - uses: actions/setup-python@v6 - with: - python-version: '${{ matrix.python }}' - architecture: '${{ matrix.arch }}' - allow-prereleases: true - check-latest: true - - name: Run tests - run: ./ci.sh - shell: bash - env: - LSP: '${{ matrix.lsp }}' - LSP_EXTRACT_FILE: '${{ matrix.lsp_extract_file }}' - SKIP_SSL_INSTALLATION: "${{ matrix.arch == 'x86' && 1 || 0 }}" - - if: always() - uses: codecov/codecov-action@v7 - with: - name: Windows (${{ matrix.python }}, ${{ matrix.arch }}${{ matrix.extra_name }}) - # multiple flags is marked as an error in codecov UI, but is actually fine - # https://github.com/codecov/feedback/issues/567 - flags: Windows,${{ matrix.python }} - fail_ci_if_error: true - - Ubuntu: - name: 'Ubuntu (${{ matrix.python }}${{ matrix.extra_name }})' - needs: - - build - - timeout-minutes: 10 - runs-on: 'ubuntu-latest' - strategy: - fail-fast: false - matrix: - python: ['pypy-3.11', '3.10', '3.11', '3.12', '3.13', '3.14', '3.14t', '3.15-dev'] - check_formatting: ['0'] - no_test_requirements: ['0'] - extra_name: [''] - include: - - python: '3.13' - check_formatting: '1' - extra_name: ', check formatting' - # separate test run that doesn't install test-requirements.txt - - python: '3.10' - no_test_requirements: '1' - extra_name: ', no test-requirements' - continue-on-error: >- - ${{ - ( - endsWith(matrix.python, '-dev') - || endsWith(matrix.python, '-nightly') - ) - && true - || false - }} - steps: - - name: Retrieve the project source from an sdist inside the GHA artifact - if: matrix.check_formatting != '1' - uses: re-actors/checkout-python-sdist@release/v2 - with: - source-tarball-name: ${{ needs.build.outputs.sdist-artifact-name }} - workflow-artifact-name: ${{ env.dists-artifact-name }} - - name: Grab the source from Git - if: matrix.check_formatting == '1' - uses: actions/checkout@v7 - with: - persist-credentials: false - - name: Setup Python - uses: actions/setup-python@v6 - with: - python-version: '${{ matrix.python }}' - cache: pip - cache-dependency-path: test-requirements.txt - allow-prereleases: true - check-latest: true - - name: Setup minimum Python version - if: matrix.check_formatting == '1' - uses: actions/setup-python@v6 - with: - python-version: '3.10' - cache: pip - cache-dependency-path: test-requirements.txt - allow-prereleases: true - - name: Check Formatting - if: matrix.check_formatting == '1' - run: - python -m pip install tox && - tox -m check - - name: Run tests - if: matrix.check_formatting == '0' - run: ./ci.sh - env: - NO_TEST_REQUIREMENTS: '${{ matrix.no_test_requirements }}' - - if: >- - always() - && matrix.check_formatting != '1' - uses: codecov/codecov-action@v7 - with: - name: Ubuntu (${{ matrix.python }}${{ matrix.extra_name }}) - flags: Ubuntu,${{ matrix.python }} - fail_ci_if_error: true - - macOS: - name: 'macOS (${{ matrix.python }})' - needs: - - build - - timeout-minutes: 15 - runs-on: 'macos-latest' - strategy: - fail-fast: false - matrix: - python: ['pypy-3.11', '3.10', '3.11', '3.12', '3.13', '3.14', '3.14t', '3.15-dev'] - continue-on-error: >- - ${{ - ( - endsWith(matrix.python, '-dev') - || endsWith(matrix.python, '-nightly') - ) - && true - || false - }} - steps: - - name: Retrieve the project source from an sdist inside the GHA artifact - uses: re-actors/checkout-python-sdist@release/v2 - with: - source-tarball-name: ${{ needs.build.outputs.sdist-artifact-name }} - workflow-artifact-name: ${{ env.dists-artifact-name }} - - name: Setup Python - uses: actions/setup-python@v6 - with: - python-version: '${{ matrix.python }}' - cache: pip - cache-dependency-path: test-requirements.txt - allow-prereleases: true - check-latest: true - - name: Run tests - run: ./ci.sh - - if: always() - uses: codecov/codecov-action@v7 - with: - name: macOS (${{ matrix.python }}) - flags: macOS,${{ matrix.python }} - fail_ci_if_error: true - - # run CI on a musl linux - Alpine: - name: "Alpine" - needs: - - build - - runs-on: ubuntu-latest - container: alpine:3 - steps: - - name: Install necessary packages - # can't use setup-python because that python doesn't seem to work; - # `python3-dev` (rather than `python:alpine`) for some ctypes reason, - # `curl`, `gpg`, `git` for codecov-action v4/v5 to work (https://github.com/codecov/codecov-action/issues/1320). - # `nodejs` for pyright (`node-env` pulls in nodejs but that takes a while and can time out the test). - # `perl` for a platform independent `sed -i` alternative - run: apk update && apk add python3-dev bash curl gpg git nodejs perl - - - name: Retrieve the project source from an sdist inside the GHA artifact - # must be after `apk add` because it relies on `bash` existing - uses: re-actors/checkout-python-sdist@release/v2 - with: - source-tarball-name: ${{ needs.build.outputs.sdist-artifact-name }} - workflow-artifact-name: ${{ env.dists-artifact-name }} - - - name: Enter virtual environment - run: python -m venv .venv - - - name: Run tests - run: source .venv/bin/activate && ./ci.sh - - - name: Get Python version for codecov flag - id: get-version - shell: python - run: | - import sys, os - with open(os.environ["GITHUB_OUTPUT"], "a") as f: - f.write("version=" + ".".join(map(str, sys.version_info[:2]))) - f.write("\n") - - - if: always() - uses: codecov/codecov-action@v7 - with: - name: Alpine - flags: Alpine,${{ steps.get-version.outputs.version }} - fail_ci_if_error: true - - Cython: - name: "Cython" - needs: - - build - - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - include: - # Cython 2 supports 3.10 and 3.11 and Cython 3 supports all versions we do, - # so test both the lowest and higher version for both - - python: '3.10' - cython: '<3' - - python: '3.11' - cython: '<3' - # TODO: technically we should pin cython versions - - python: '3.10' - cython: '>=3' - - python: '3.14' - cython: '>=3' - steps: - - name: Retrieve the project source from an sdist inside the GHA artifact - uses: re-actors/checkout-python-sdist@release/v2 - with: - source-tarball-name: ${{ needs.build.outputs.sdist-artifact-name }} - workflow-artifact-name: ${{ env.dists-artifact-name }} - - name: Setup Python - uses: actions/setup-python@v6 - with: - python-version: '${{ matrix.python }}' - cache: pip - # setuptools is needed to get distutils on 3.12, which cythonize requires - - name: install trio and setuptools - run: python -m pip install --upgrade pip . setuptools 'coverage[toml]' - - - name: add cython plugin to the coveragepy config - run: >- - sed -i 's#plugins\s=\s\[\]#plugins = ["Cython.Coverage"]#' - pyproject.toml - - - name: install cython & compile pyx file - env: - CFLAGS: ${{ env.CFLAGS }} -DCYTHON_TRACE_NOGIL=1 - run: | - python -m pip install "cython${{ matrix.cython }}" - cythonize --inplace -X linetrace=True tests/cython/test_cython.pyx - - - name: import & run module - run: coverage run -m tests.cython.run_test_cython - - - name: Get Python version for codecov flag - id: get-version - shell: python - run: | - import sys, os - with open(os.environ["GITHUB_OUTPUT"], "a") as f: - f.write("version=" + ".".join(map(str, sys.version_info[:2]))) - f.write("\n") - - - run: | - coverage combine - coverage report - - - if: always() - uses: codecov/codecov-action@v7 - with: - name: Cython - flags: Cython,${{ steps.get-version.outputs.version }} - fail_ci_if_error: true - - zizmor: - name: Run zizmor 🌈 - runs-on: ubuntu-latest - permissions: - security-events: write # Required for upload-sarif (used by zizmor-action) to upload SARIF files. - steps: - - name: Checkout repository - uses: actions/checkout@v7 - with: - persist-credentials: false - - - name: Run zizmor 🌈 - uses: zizmorcore/zizmor-action@v0.6.1 - - # https://github.com/marketplace/actions/alls-green#why - check: # This job does nothing and is only used for the branch protection - - if: always() - - needs: - - smoke-tests - - Windows - - Ubuntu - - macOS - - Alpine - - Cython - - zizmor - - runs-on: ubuntu-latest - - steps: - - name: Decide whether the needed jobs succeeded or failed - uses: re-actors/alls-green@release/v1 - with: - jobs: ${{ toJSON(needs) }} +name: CI + +permissions: {} + +on: + push: + branches-ignore: + # these branches always have another event associated + - gh-readonly-queue/** # GitHub's merge queue uses `merge_group` + - autodeps/** # autodeps always makes a PR + - pre-commit-ci-update-config # pre-commit.ci's updates always have a PR + pull_request: + merge_group: + +concurrency: + group: ${{ github.ref }}-${{ github.workflow }}-${{ github.event_name }}${{ github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && format('-{0}', github.sha) || '' }} + cancel-in-progress: true + +env: + dists-artifact-name: python-package-distributions + dist-name: trio + +jobs: + build: + name: 👷 dists + + runs-on: ubuntu-latest + + outputs: + dist-version: ${{ steps.dist-version.outputs.version }} + sdist-artifact-name: ${{ steps.artifact-name.outputs.sdist }} + wheel-artifact-name: ${{ steps.artifact-name.outputs.wheel }} + + steps: + - name: Switch to using Python 3.11 + uses: actions/setup-python@v6 + with: + python-version: 3.11 + + - name: Grab the source from Git + uses: actions/checkout@v7 + with: + persist-credentials: false + + - name: Get the dist version + id: dist-version + run: >- + echo "version=$( + grep ^__version__ src/trio/_version.py + | sed 's#__version__ = "\([^"]\+\)"#\1#' + )" + >> "${GITHUB_OUTPUT}" + + - name: Set the expected dist artifact names + id: artifact-name + run: | + echo "sdist=${DIST_NAME}-${VERSION}.tar.gz" >> "${GITHUB_OUTPUT}" + echo "wheel=${DIST_NAME}-${VERSION}-py3-none-any.whl" >> "${GITHUB_OUTPUT}" + env: + DIST_NAME: ${{ env.dist-name }} + VERSION: ${{ steps.dist-version.outputs.version }} + + - name: Install build + run: python -Im pip install build + + - name: Build dists + run: python -Im build + - name: Verify that the artifacts with expected names got created + run: >- + ls -1 + dist/${SDIST} + dist/${WHEEL} + env: + SDIST: ${{ steps.artifact-name.outputs.sdist }} + WHEEL: ${{ steps.artifact-name.outputs.wheel }} + - name: Store the distribution packages + uses: actions/upload-artifact@v7 + with: + name: ${{ env.dists-artifact-name }} + # NOTE: Exact expected file names are specified here + # NOTE: as a safety measure — if anything weird ends + # NOTE: up being in this dir or not all dists will be + # NOTE: produced, this will fail the workflow. + path: | + dist/${{ steps.artifact-name.outputs.sdist }} + dist/${{ steps.artifact-name.outputs.wheel }} + retention-days: 5 + + smoke-tests: + name: Smoke tests + needs: + - build + + runs-on: ubuntu-latest + + steps: + - name: Switch to using Python 3.11 + uses: actions/setup-python@v6 + with: + python-version: 3.11 + + - name: >- + Smoke-test: + retrieve the project source from an sdist inside the GHA artifact + uses: re-actors/checkout-python-sdist@release/v2 + with: + source-tarball-name: ${{ needs.build.outputs.sdist-artifact-name }} + workflow-artifact-name: ${{ env.dists-artifact-name }} + + - name: >- + Smoke-test: move the sdist-retrieved dir into sdist-src + run: | + mv -v '${{ github.workspace }}' '${{ runner.temp }}/sdist-src' + mkdir -pv '${{ github.workspace }}' + mv -v '${{ runner.temp }}/sdist-src' '${{ github.workspace }}/sdist-src' + shell: bash -eEuo pipefail {0} + + - name: >- + Smoke-test: grab the source from Git into git-src + uses: actions/checkout@v7 + with: + path: git-src + persist-credentials: false + + - name: >- + Smoke-test: install test requirements from the Git repo + run: >- + python -Im + pip install -c test-requirements.txt -r test-requirements.txt + shell: bash -eEuo pipefail {0} + working-directory: git-src + + - name: >- + Smoke-test: collect tests from the Git repo + env: + PYTHONPATH: src/ + run: >- + pytest --collect-only -qq . + | sort + | tee collected-tests + shell: bash -eEuo pipefail {0} + working-directory: git-src + + - name: >- + Smoke-test: collect tests from the sdist tarball + env: + PYTHONPATH: src/ + run: >- + pytest --collect-only -qq . + | sort + | tee collected-tests + shell: bash -eEuo pipefail {0} + working-directory: sdist-src + + - name: >- + Smoke-test: + verify that all the tests from Git are included in the sdist + run: diff --unified sdist-src/collected-tests git-src/collected-tests + shell: bash -eEuo pipefail {0} + + Windows: + name: 'Windows (${{ matrix.python }}, ${{ matrix.arch }}${{ matrix.extra_name }})' + needs: + - build + + timeout-minutes: 20 + runs-on: 'windows-latest' + strategy: + fail-fast: false + matrix: + python: ['3.10', '3.11', '3.12', '3.13', '3.14', '3.14t', '3.15-dev'] + arch: ['x86', 'x64'] + lsp: [''] + lsp_extract_file: [''] + extra_name: [''] + include: + - python: '3.10' + arch: 'x64' + lsp: 'https://raw.githubusercontent.com/python-trio/trio-ci-assets/master/komodia-based-vpn-setup.zip' + lsp_extract_file: 'komodia-based-vpn-setup.exe' + extra_name: ', with Komodia LSP' + - python: '3.10' + arch: 'x64' + lsp: 'https://www.proxifier.com/download/legacy/ProxifierSetup342.exe' + lsp_extract_file: '' + extra_name: ', with IFS LSP' + - python: 'pypy-3.11' + arch: 'x64' + lsp: '' + lsp_extract_file: '' + extra_name: '' + #- python: '3.10' + # arch: 'x64' + # lsp: 'http://download.pctools.com/mirror/updates/9.0.0.2308-SDavfree-lite_en.exe' + # lsp_extract_file: '' + # extra_name: ', with non-IFS LSP' + + continue-on-error: >- + ${{ + ( + endsWith(matrix.python, '-dev') + || endsWith(matrix.python, '-nightly') + ) + && true + || false + }} + steps: + - name: Retrieve the project source from an sdist inside the GHA artifact + uses: re-actors/checkout-python-sdist@release/v2 + with: + source-tarball-name: ${{ needs.build.outputs.sdist-artifact-name }} + workflow-artifact-name: ${{ env.dists-artifact-name }} + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: '${{ matrix.python }}' + architecture: '${{ matrix.arch }}' + allow-prereleases: true + check-latest: true + - name: Run tests + run: ./ci.sh + shell: bash + env: + LSP: '${{ matrix.lsp }}' + LSP_EXTRACT_FILE: '${{ matrix.lsp_extract_file }}' + SKIP_SSL_INSTALLATION: "${{ matrix.arch == 'x86' && 1 || 0 }}" + - if: always() + uses: codecov/codecov-action@v7 + with: + name: Windows (${{ matrix.python }}, ${{ matrix.arch }}${{ matrix.extra_name }}) + # multiple flags is marked as an error in codecov UI, but is actually fine + # https://github.com/codecov/feedback/issues/567 + flags: Windows,${{ matrix.python }} + fail_ci_if_error: true + + Ubuntu: + name: 'Ubuntu (${{ matrix.python }}${{ matrix.extra_name }})' + needs: + - build + + timeout-minutes: 10 + runs-on: 'ubuntu-latest' + strategy: + fail-fast: false + matrix: + python: ['pypy-3.11', '3.10', '3.11', '3.12', '3.13', '3.14', '3.14t', '3.15-dev'] + check_formatting: ['0'] + no_test_requirements: ['0'] + extra_name: [''] + include: + - python: '3.13' + check_formatting: '1' + extra_name: ', check formatting' + # separate test run that doesn't install test-requirements.txt + - python: '3.10' + no_test_requirements: '1' + extra_name: ', no test-requirements' + continue-on-error: >- + ${{ + ( + endsWith(matrix.python, '-dev') + || endsWith(matrix.python, '-nightly') + ) + && true + || false + }} + steps: + - name: Retrieve the project source from an sdist inside the GHA artifact + if: matrix.check_formatting != '1' + uses: re-actors/checkout-python-sdist@release/v2 + with: + source-tarball-name: ${{ needs.build.outputs.sdist-artifact-name }} + workflow-artifact-name: ${{ env.dists-artifact-name }} + - name: Grab the source from Git + if: matrix.check_formatting == '1' + uses: actions/checkout@v7 + with: + persist-credentials: false + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: '${{ matrix.python }}' + cache: pip + cache-dependency-path: test-requirements.txt + allow-prereleases: true + check-latest: true + - name: Setup minimum Python version + if: matrix.check_formatting == '1' + uses: actions/setup-python@v6 + with: + python-version: '3.10' + cache: pip + cache-dependency-path: test-requirements.txt + allow-prereleases: true + - name: Check Formatting + if: matrix.check_formatting == '1' + run: + python -m pip install tox && + tox -m check + - name: Run tests + if: matrix.check_formatting == '0' + run: ./ci.sh + env: + NO_TEST_REQUIREMENTS: '${{ matrix.no_test_requirements }}' + - if: >- + always() + && matrix.check_formatting != '1' + uses: codecov/codecov-action@v7 + with: + name: Ubuntu (${{ matrix.python }}${{ matrix.extra_name }}) + flags: Ubuntu,${{ matrix.python }} + fail_ci_if_error: true + + macOS: + name: 'macOS (${{ matrix.python }})' + needs: + - build + + timeout-minutes: 15 + runs-on: 'macos-latest' + strategy: + fail-fast: false + matrix: + python: ['pypy-3.11', '3.10', '3.11', '3.12', '3.13', '3.14', '3.14t', '3.15-dev'] + continue-on-error: >- + ${{ + ( + endsWith(matrix.python, '-dev') + || endsWith(matrix.python, '-nightly') + ) + && true + || false + }} + steps: + - name: Retrieve the project source from an sdist inside the GHA artifact + uses: re-actors/checkout-python-sdist@release/v2 + with: + source-tarball-name: ${{ needs.build.outputs.sdist-artifact-name }} + workflow-artifact-name: ${{ env.dists-artifact-name }} + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: '${{ matrix.python }}' + cache: pip + cache-dependency-path: test-requirements.txt + allow-prereleases: true + check-latest: true + - name: Run tests + run: ./ci.sh + - if: always() + uses: codecov/codecov-action@v7 + with: + name: macOS (${{ matrix.python }}) + flags: macOS,${{ matrix.python }} + fail_ci_if_error: true + + # run CI on a musl linux + Alpine: + name: "Alpine" + needs: + - build + + runs-on: ubuntu-latest + container: alpine:3 + steps: + - name: Install necessary packages + # can't use setup-python because that python doesn't seem to work; + # `python3-dev` (rather than `python:alpine`) for some ctypes reason, + # `curl`, `gpg`, `git` for codecov-action v4/v5 to work (https://github.com/codecov/codecov-action/issues/1320). + # `nodejs` for pyright (`node-env` pulls in nodejs but that takes a while and can time out the test). + # `perl` for a platform independent `sed -i` alternative + run: apk update && apk add python3-dev bash curl gpg git nodejs perl + + - name: Retrieve the project source from an sdist inside the GHA artifact + # must be after `apk add` because it relies on `bash` existing + uses: re-actors/checkout-python-sdist@release/v2 + with: + source-tarball-name: ${{ needs.build.outputs.sdist-artifact-name }} + workflow-artifact-name: ${{ env.dists-artifact-name }} + + - name: Enter virtual environment + run: python -m venv .venv + + - name: Run tests + run: source .venv/bin/activate && ./ci.sh + + - name: Get Python version for codecov flag + id: get-version + shell: python + run: | + import sys, os + with open(os.environ["GITHUB_OUTPUT"], "a") as f: + f.write("version=" + ".".join(map(str, sys.version_info[:2]))) + f.write("\n") + + - if: always() + uses: codecov/codecov-action@v7 + with: + name: Alpine + flags: Alpine,${{ steps.get-version.outputs.version }} + fail_ci_if_error: true + + Cython: + name: "Cython" + needs: + - build + + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + # Cython 2 supports 3.10 and 3.11 and Cython 3 supports all versions we do, + # so test both the lowest and higher version for both + - python: '3.10' + cython: '<3' + - python: '3.11' + cython: '<3' + # TODO: technically we should pin cython versions + - python: '3.10' + cython: '>=3' + - python: '3.14' + cython: '>=3' + steps: + - name: Retrieve the project source from an sdist inside the GHA artifact + uses: re-actors/checkout-python-sdist@release/v2 + with: + source-tarball-name: ${{ needs.build.outputs.sdist-artifact-name }} + workflow-artifact-name: ${{ env.dists-artifact-name }} + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: '${{ matrix.python }}' + cache: pip + # setuptools is needed to get distutils on 3.12, which cythonize requires + - name: install trio and setuptools + run: python -m pip install --upgrade pip . setuptools 'coverage[toml]' + + - name: add cython plugin to the coveragepy config + run: >- + sed -i 's#plugins\s=\s\[\]#plugins = ["Cython.Coverage"]#' + pyproject.toml + + - name: install cython & compile pyx file + env: + CFLAGS: ${{ env.CFLAGS }} -DCYTHON_TRACE_NOGIL=1 + run: | + python -m pip install "cython${{ matrix.cython }}" + cythonize --inplace -X linetrace=True tests/cython/test_cython.pyx + + - name: import & run module + run: coverage run -m tests.cython.run_test_cython + + - name: Get Python version for codecov flag + id: get-version + shell: python + run: | + import sys, os + with open(os.environ["GITHUB_OUTPUT"], "a") as f: + f.write("version=" + ".".join(map(str, sys.version_info[:2]))) + f.write("\n") + + - run: | + coverage combine + coverage report + + - if: always() + uses: codecov/codecov-action@v7 + with: + name: Cython + flags: Cython,${{ steps.get-version.outputs.version }} + fail_ci_if_error: true + + zizmor: + name: Run zizmor 🌈 + runs-on: ubuntu-latest + permissions: + security-events: write # Required for upload-sarif (used by zizmor-action) to upload SARIF files. + steps: + - name: Checkout repository + uses: actions/checkout@v7 + with: + persist-credentials: false + + - name: Run zizmor 🌈 + uses: zizmorcore/zizmor-action@v0.6.1 + + # https://github.com/marketplace/actions/alls-green#why + check: # This job does nothing and is only used for the branch protection + + if: always() + + needs: + - smoke-tests + - Windows + - Ubuntu + - macOS + - Alpine + - Cython + - zizmor + + runs-on: ubuntu-latest + + steps: + - name: Decide whether the needed jobs succeeded or failed + uses: re-actors/alls-green@release/v1 + with: + jobs: ${{ toJSON(needs) }} diff --git a/ci.sh b/ci.sh index 1ea2c49219..bd86bf5454 100755 --- a/ci.sh +++ b/ci.sh @@ -1,157 +1,157 @@ -#!/bin/bash - -set -ex -o pipefail - -# disable warnings about pyright being out of date -# used in test_exports and in check.sh -export PYRIGHT_PYTHON_IGNORE_WARNINGS=1 - -# Log some general info about the environment -echo "::group::Environment" -uname -a -env | sort -echo "::endgroup::" - -# Curl's built-in retry system is not very robust; it gives up on lots of -# network errors that we want to retry on. Wget might work better, but it's -# not installed on azure pipelines's windows boxes. So... let's try some good -# old-fashioned brute force. (This is also a convenient place to put options -# we always want, like -f to tell curl to give an error if the server sends an -# error response, and -L to follow redirects.) -function curl-harder() { - for BACKOFF in 0 1 2 4 8 15 15 15 15; do - sleep $BACKOFF - if curl -fL --connect-timeout 5 "$@"; then - return 0 - fi - done - return 1 -} - -################################################################ -# We have a Python environment! -################################################################ - -echo "::group::Versions" -python -c "import sys, struct, ssl; print('python:', sys.version); print('version_info:', sys.version_info); print('bits:', struct.calcsize('P') * 8); print('openssl:', ssl.OPENSSL_VERSION, ssl.OPENSSL_VERSION_INFO)" -echo "::endgroup::" - -echo "::group::Install dependencies" -python -m pip install -U pip uv -c test-requirements.txt -python -m pip --version -python -m uv --version - -python -m uv pip install build - -python -m build -wheel_package=$(ls dist/*.whl) -python -m uv pip install "trio @ $wheel_package" -c test-requirements.txt - -# Actual tests -# expands to 0 != 1 if NO_TEST_REQUIREMENTS is not set, if set the `-0` has no effect -# https://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html#tag_18_06_02 -if [ "${NO_TEST_REQUIREMENTS-0}" == 1 ]; then - python -m uv pip install pytest coverage -c test-requirements.txt - flags="--skip-optional-imports" -else - flags="" - if [ "${SKIP_SSL_INSTALLATION-0}" == 1 ]; then - echo cryptography >excluded_deps - PIP_ARGS="--exclude excluded_deps" - flags="--skip-ssl-imports" - fi - python -m uv pip install -r test-requirements.txt -vvv --no-deps $PIP_ARGS -fi - -# If we're testing with a LSP installed, then it might break network -# stuff, so wait until after we've finished setting everything else -# up. -if [ "$LSP" != "" ]; then - echo "Installing LSP from ${LSP}" - # We use --insecure because one of the LSP's has been observed to give - # cert verification errors: - # - # https://github.com/python-trio/trio/issues/1478 - # - # *Normally*, you should never ever use --insecure, especially when - # fetching an executable! But *in this case*, we're intentionally - # installing some untrustworthy quasi-malware onto into a sandboxed - # machine for testing. So MITM attacks are really the least of our - # worries. - if [ "$LSP_EXTRACT_FILE" != "" ]; then - # We host the Astrill VPN installer ourselves, and encrypt it - # so as to decrease the chances of becoming an inadvertent - # public redistributor. - curl-harder -o lsp-installer.zip "$LSP" - unzip -P "not very secret trio ci key" lsp-installer.zip "$LSP_EXTRACT_FILE" - mv "$LSP_EXTRACT_FILE" lsp-installer.exe - else - curl-harder --insecure -o lsp-installer.exe "$LSP" - fi - # This is only needed for the Astrill LSP, but there's no harm in - # doing it all the time. The cert was manually extracted by installing - # the package in a VM, clicking "Always trust from this publisher" - # when installing, and then running 'certmgr.msc' and exporting the - # certificate. See: - # http://www.migee.com/2010/09/24/solution-for-unattendedsilent-installs-and-would-you-like-to-install-this-device-software/ - certutil -addstore "TrustedPublisher" src/trio/_tests/astrill-codesigning-cert.cer - # Double-slashes are how you tell windows-bash that you want a single - # slash, and don't treat this as a unix-style filename that needs to - # be replaced by a windows-style filename. - # http://www.mingw.org/wiki/Posix_path_conversion - ./lsp-installer.exe //silent //norestart - echo "Waiting for LSP to appear in Winsock catalog" - while ! netsh winsock show catalog | grep "Layered Chain Entry"; do - sleep 1 - done - netsh winsock show catalog -fi -echo "::endgroup::" - -echo "::group::Setup for tests" - -# We run the tests from inside an empty directory, to make sure Python -# doesn't pick up any .py files from our working dir. Might have already -# been created by a previous run. -mkdir empty || true -cd empty - -INSTALLDIR=$(python -c "import os, trio; print(os.path.dirname(trio.__file__))") -cp ../pyproject.toml "$INSTALLDIR" # TODO: remove this - -# get mypy tests a nice cache -MYPYPATH=".." mypy --config-file= --cache-dir=./.mypy_cache -c "import trio" >/dev/null 2>/dev/null || true - -# support subprocess spawning with coverage.py -echo "import coverage; coverage.process_startup()" | tee -a "$INSTALLDIR/../sitecustomize.py" - -perl -i -pe 's/-p trio\._tests\.pytest_plugin//' "$INSTALLDIR/pyproject.toml" - -echo "::endgroup::" -echo "::group:: Run Tests" -if PYTHONPATH=../tests COVERAGE_PROCESS_START=$(pwd)/../pyproject.toml \ - coverage run --rcfile=../pyproject.toml -m \ - pytest -ra --junitxml=../test-results.xml \ - -p _trio_check_attrs_aliases --verbose --durations=10 \ - -p trio._tests.pytest_plugin --run-slow $flags "${INSTALLDIR}"; then - PASSED=true -else - PASSED=false -fi -echo "::endgroup::" -echo "::group::Coverage" - -coverage combine --rcfile ../pyproject.toml -cd .. # coverage needs to be in the folder containing src/trio -cp empty/.coverage . -coverage report -m --rcfile ./pyproject.toml -coverage xml --rcfile ./pyproject.toml - -# Remove the LSP again; again we want to do this ASAP to avoid -# accidentally breaking other stuff. -if [ "$LSP" != "" ]; then - netsh winsock reset -fi - -echo "::endgroup::" -$PASSED +#!/bin/bash + +set -ex -o pipefail + +# disable warnings about pyright being out of date +# used in test_exports and in check.sh +export PYRIGHT_PYTHON_IGNORE_WARNINGS=1 + +# Log some general info about the environment +echo "::group::Environment" +uname -a +env | sort +echo "::endgroup::" + +# Curl's built-in retry system is not very robust; it gives up on lots of +# network errors that we want to retry on. Wget might work better, but it's +# not installed on azure pipelines's windows boxes. So... let's try some good +# old-fashioned brute force. (This is also a convenient place to put options +# we always want, like -f to tell curl to give an error if the server sends an +# error response, and -L to follow redirects.) +function curl-harder() { + for BACKOFF in 0 1 2 4 8 15 15 15 15; do + sleep $BACKOFF + if curl -fL --connect-timeout 5 "$@"; then + return 0 + fi + done + return 1 +} + +################################################################ +# We have a Python environment! +################################################################ + +echo "::group::Versions" +python -c "import sys, struct, ssl; print('python:', sys.version); print('version_info:', sys.version_info); print('bits:', struct.calcsize('P') * 8); print('openssl:', ssl.OPENSSL_VERSION, ssl.OPENSSL_VERSION_INFO)" +echo "::endgroup::" + +echo "::group::Install dependencies" +python -m pip install -U pip uv -c test-requirements.txt +python -m pip --version +python -m uv --version + +python -m uv pip install build + +python -m build +wheel_package=$(ls dist/*.whl) +python -m uv pip install "trio @ $wheel_package" -c test-requirements.txt + +# Actual tests +# expands to 0 != 1 if NO_TEST_REQUIREMENTS is not set, if set the `-0` has no effect +# https://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html#tag_18_06_02 +if [ "${NO_TEST_REQUIREMENTS-0}" == 1 ]; then + python -m uv pip install pytest coverage -c test-requirements.txt + flags="--skip-optional-imports" +else + flags="" + if [ "${SKIP_SSL_INSTALLATION-0}" == 1 ]; then + echo cryptography >excluded_deps + PIP_ARGS="--exclude excluded_deps" + flags="--skip-ssl-imports" + fi + python -m uv pip install -r test-requirements.txt -v --no-deps $PIP_ARGS +fi + +# If we're testing with a LSP installed, then it might break network +# stuff, so wait until after we've finished setting everything else +# up. +if [ "$LSP" != "" ]; then + echo "Installing LSP from ${LSP}" + # We use --insecure because one of the LSP's has been observed to give + # cert verification errors: + # + # https://github.com/python-trio/trio/issues/1478 + # + # *Normally*, you should never ever use --insecure, especially when + # fetching an executable! But *in this case*, we're intentionally + # installing some untrustworthy quasi-malware onto into a sandboxed + # machine for testing. So MITM attacks are really the least of our + # worries. + if [ "$LSP_EXTRACT_FILE" != "" ]; then + # We host the Astrill VPN installer ourselves, and encrypt it + # so as to decrease the chances of becoming an inadvertent + # public redistributor. + curl-harder -o lsp-installer.zip "$LSP" + unzip -P "not very secret trio ci key" lsp-installer.zip "$LSP_EXTRACT_FILE" + mv "$LSP_EXTRACT_FILE" lsp-installer.exe + else + curl-harder --insecure -o lsp-installer.exe "$LSP" + fi + # This is only needed for the Astrill LSP, but there's no harm in + # doing it all the time. The cert was manually extracted by installing + # the package in a VM, clicking "Always trust from this publisher" + # when installing, and then running 'certmgr.msc' and exporting the + # certificate. See: + # http://www.migee.com/2010/09/24/solution-for-unattendedsilent-installs-and-would-you-like-to-install-this-device-software/ + certutil -addstore "TrustedPublisher" src/trio/_tests/astrill-codesigning-cert.cer + # Double-slashes are how you tell windows-bash that you want a single + # slash, and don't treat this as a unix-style filename that needs to + # be replaced by a windows-style filename. + # http://www.mingw.org/wiki/Posix_path_conversion + ./lsp-installer.exe //silent //norestart + echo "Waiting for LSP to appear in Winsock catalog" + while ! netsh winsock show catalog | grep "Layered Chain Entry"; do + sleep 1 + done + netsh winsock show catalog +fi +echo "::endgroup::" + +echo "::group::Setup for tests" + +# We run the tests from inside an empty directory, to make sure Python +# doesn't pick up any .py files from our working dir. Might have already +# been created by a previous run. +mkdir empty || true +cd empty + +INSTALLDIR=$(python -c "import os, trio; print(os.path.dirname(trio.__file__))") +cp ../pyproject.toml "$INSTALLDIR" # TODO: remove this + +# get mypy tests a nice cache +MYPYPATH=".." mypy --config-file= --cache-dir=./.mypy_cache -c "import trio" >/dev/null 2>/dev/null || true + +# support subprocess spawning with coverage.py +echo "import coverage; coverage.process_startup()" | tee -a "$INSTALLDIR/../sitecustomize.py" + +perl -i -pe 's/-p trio\._tests\.pytest_plugin//' "$INSTALLDIR/pyproject.toml" + +echo "::endgroup::" +echo "::group:: Run Tests" +if PYTHONPATH=../tests COVERAGE_PROCESS_START=$(pwd)/../pyproject.toml \ + coverage run --rcfile=../pyproject.toml -m \ + pytest -ra --junitxml=../test-results.xml \ + -p _trio_check_attrs_aliases --verbose --durations=10 \ + -p trio._tests.pytest_plugin --run-slow $flags "${INSTALLDIR}"; then + PASSED=true +else + PASSED=false +fi +echo "::endgroup::" +echo "::group::Coverage" + +coverage combine --rcfile ../pyproject.toml +cd .. # coverage needs to be in the folder containing src/trio +cp empty/.coverage . +coverage report -m --rcfile ./pyproject.toml +coverage xml --rcfile ./pyproject.toml + +# Remove the LSP again; again we want to do this ASAP to avoid +# accidentally breaking other stuff. +if [ "$LSP" != "" ]; then + netsh winsock reset +fi + +echo "::endgroup::" +$PASSED diff --git a/src/trio/_tests/pytest_plugin.py b/src/trio/_tests/pytest_plugin.py index b929ae3a73..a67b8a85a6 100644 --- a/src/trio/_tests/pytest_plugin.py +++ b/src/trio/_tests/pytest_plugin.py @@ -1,60 +1,60 @@ -from __future__ import annotations - -import inspect -from typing import NoReturn - -import pytest - -from ..testing import MockClock, trio_test - -RUN_SLOW = True -SKIP_OPTIONAL_IMPORTS = False -SKIP_SSL_IMPORTS = False - - -def pytest_addoption(parser: pytest.Parser) -> None: - parser.addoption("--run-slow", action="store_true", help="run slow tests") - parser.addoption( - "--skip-optional-imports", - action="store_true", - help="skip tests that rely on libraries not required by trio itself", - ) - parser.addoption( - "--skip-ssl-imports", - action="store_true", - help="skip tests that rely on cryptography", - ) - - -def pytest_configure(config: pytest.Config) -> None: - global RUN_SLOW, SKIP_OPTIONAL_IMPORTS, SKIP_SSL_IMPORTS - RUN_SLOW = config.getoption("--run-slow", default=True) - SKIP_OPTIONAL_IMPORTS = config.getoption("--skip-optional-imports", default=False) - SKIP_SSL_IMPORTS = config.getoption("--skip-ssl-imports", default=False) - - -@pytest.fixture -def mock_clock() -> MockClock: - return MockClock() - - -@pytest.fixture -def autojump_clock() -> MockClock: - return MockClock(autojump_threshold=0) - - -# FIXME: split off into a package (or just make part of Trio's public -# interface?), with config file to enable? and I guess a mark option too; I -# guess it's useful with the class- and file-level marking machinery (where -# the raw @trio_test decorator isn't enough). -@pytest.hookimpl(tryfirst=True) -def pytest_pyfunc_call(pyfuncitem: pytest.Function) -> None: - if inspect.iscoroutinefunction(pyfuncitem.obj): - pyfuncitem.obj = trio_test(pyfuncitem.obj) - - -def maybe_ignore_import_error(error: ImportError) -> NoReturn: - if SKIP_OPTIONAL_IMPORTS or (SKIP_SSL_IMPORTS and error.name == "cryptography"): - pytest.skip(error.msg, allow_module_level=True) - else: # pragma: no cover - raise error +from __future__ import annotations + +import inspect +from typing import NoReturn + +import pytest + +from ..testing import MockClock, trio_test + +RUN_SLOW = True +SKIP_OPTIONAL_IMPORTS = False +SKIP_SSL_IMPORTS = False + + +def pytest_addoption(parser: pytest.Parser) -> None: + parser.addoption("--run-slow", action="store_true", help="run slow tests") + parser.addoption( + "--skip-optional-imports", + action="store_true", + help="skip tests that rely on libraries not required by trio itself", + ) + parser.addoption( + "--skip-ssl-imports", + action="store_true", + help="skip tests that rely on cryptography", + ) + + +def pytest_configure(config: pytest.Config) -> None: + global RUN_SLOW, SKIP_OPTIONAL_IMPORTS, SKIP_SSL_IMPORTS + RUN_SLOW = config.getoption("--run-slow", default=True) + SKIP_OPTIONAL_IMPORTS = config.getoption("--skip-optional-imports", default=False) + SKIP_SSL_IMPORTS = config.getoption("--skip-ssl-imports", default=False) + + +@pytest.fixture +def mock_clock() -> MockClock: + return MockClock() + + +@pytest.fixture +def autojump_clock() -> MockClock: + return MockClock(autojump_threshold=0) + + +# FIXME: split off into a package (or just make part of Trio's public +# interface?), with config file to enable? and I guess a mark option too; I +# guess it's useful with the class- and file-level marking machinery (where +# the raw @trio_test decorator isn't enough). +@pytest.hookimpl(tryfirst=True) +def pytest_pyfunc_call(pyfuncitem: pytest.Function) -> None: + if inspect.iscoroutinefunction(pyfuncitem.obj): + pyfuncitem.obj = trio_test(pyfuncitem.obj) + + +def maybe_ignore_import_error(error: ImportError) -> NoReturn: + if SKIP_OPTIONAL_IMPORTS or (SKIP_SSL_IMPORTS and error.name == "cryptography"): + pytest.skip(error.msg, allow_module_level=True) + else: # pragma: no cover + raise error diff --git a/src/trio/_tests/test_dtls.py b/src/trio/_tests/test_dtls.py index 5a34d8c811..7b3bc205c2 100644 --- a/src/trio/_tests/test_dtls.py +++ b/src/trio/_tests/test_dtls.py @@ -1,952 +1,952 @@ -from __future__ import annotations - -import random -from contextlib import asynccontextmanager -from itertools import count -from typing import TYPE_CHECKING, NoReturn - -import attrs -import pytest - -from trio._tests.pytest_plugin import maybe_ignore_import_error - -try: - import trustme - from OpenSSL import SSL -except ImportError as error: - maybe_ignore_import_error(error) - - -import trio -import trio.testing -from trio import DTLSChannel, DTLSEndpoint -from trio.testing._fake_net import FakeNet, UDPPacket - -from .._core._tests.tutil import binds_ipv6, gc_collect_harder, slow - -if TYPE_CHECKING: - from collections.abc import AsyncGenerator - -ca = trustme.CA() -server_cert = ca.issue_cert("example.com") - - -@pytest.fixture -def server_ctx() -> SSL.Context: - ctx = SSL.Context(SSL.DTLS_METHOD) - server_cert.configure_cert(ctx) - return ctx - - -def client_ctx_fn() -> SSL.Context: - ctx = SSL.Context(SSL.DTLS_METHOD) - ca.configure_trust(ctx) - return ctx - - -@pytest.fixture -def client_ctx() -> SSL.Context: - return client_ctx_fn() - - -parametrize_ipv6 = pytest.mark.parametrize( - "ipv6", - [False, pytest.param(True, marks=binds_ipv6)], - ids=["ipv4", "ipv6"], -) - - -def endpoint(**kwargs: int | bool) -> DTLSEndpoint: - ipv6 = kwargs.pop("ipv6", False) - family = trio.socket.AF_INET6 if ipv6 else trio.socket.AF_INET - sock = trio.socket.socket(type=trio.socket.SOCK_DGRAM, family=family) - return DTLSEndpoint(sock, **kwargs) - - -@asynccontextmanager -async def dtls_echo_server( - *, - server_ctx: SSL.Context, - autocancel: bool = True, - mtu: int | None = None, - ipv6: bool = False, -) -> AsyncGenerator[tuple[DTLSEndpoint, tuple[str, int]], None]: - with endpoint(ipv6=ipv6) as server: - localhost = "::1" if ipv6 else "127.0.0.1" - await server.socket.bind((localhost, 0)) - async with trio.open_nursery() as nursery: - - async def echo_handler(dtls_channel: DTLSChannel) -> None: - print( - "echo handler started: " - f"server {dtls_channel.endpoint.socket.getsockname()!r} " - f"client {dtls_channel.peer_address!r}", - ) - if mtu is not None: - dtls_channel.set_ciphertext_mtu(mtu) - try: - print("server starting do_handshake") - await dtls_channel.do_handshake() - print("server finished do_handshake") - # no branch for leaving this for loop because we only leave - # a channel by cancellation. - async for packet in dtls_channel: # pragma: no branch - print(f"echoing {packet!r} -> {dtls_channel.peer_address!r}") - await dtls_channel.send(packet) - except trio.BrokenResourceError: # pragma: no cover - print("echo handler channel broken") - - await nursery.start(server.serve, server_ctx, echo_handler) - - yield server, server.socket.getsockname() # noqa: RUF075 - - if autocancel: - nursery.cancel_scope.cancel() - - -@parametrize_ipv6 -async def test_smoke( - ipv6: bool, server_ctx: SSL.Context, client_ctx: SSL.Context -) -> None: - async with dtls_echo_server(ipv6=ipv6, server_ctx=server_ctx) as ( - _server_endpoint, - address, - ): - with endpoint(ipv6=ipv6) as client_endpoint: - client_channel = client_endpoint.connect(address, client_ctx) - with pytest.raises(trio.NeedHandshakeError): - client_channel.get_cleartext_mtu() - - await client_channel.do_handshake() - await client_channel.send(b"hello") - assert await client_channel.receive() == b"hello" - await client_channel.send(b"goodbye") - assert await client_channel.receive() == b"goodbye" - - with pytest.raises( - ValueError, - match=r"^openssl doesn't support sending empty DTLS packets$", - ): - await client_channel.send(b"") - - client_channel.set_ciphertext_mtu(1234) - cleartext_mtu_1234 = client_channel.get_cleartext_mtu() - client_channel.set_ciphertext_mtu(4321) - assert client_channel.get_cleartext_mtu() > cleartext_mtu_1234 - client_channel.set_ciphertext_mtu(1234) - assert client_channel.get_cleartext_mtu() == cleartext_mtu_1234 - - -@slow -async def test_handshake_over_terrible_network( - autojump_clock: trio.testing.MockClock, - server_ctx: SSL.Context, -) -> None: - HANDSHAKES = 100 - r = random.Random(0) - fn = FakeNet() - fn.enable() - # avoid spurious timeouts on slow machines - autojump_clock.autojump_threshold = 0.001 - - async with dtls_echo_server(server_ctx=server_ctx) as (_, address): - async with trio.open_nursery() as nursery: - - async def route_packet(packet: UDPPacket) -> None: - while True: - op = r.choices( - ["deliver", "drop", "dupe", "delay"], - weights=[0.7, 0.1, 0.1, 0.1], - )[0] - print(f"{packet.source} -> {packet.destination}: {op}") - if op == "drop": - return - elif op == "dupe": - fn.send_packet(packet) - elif op == "delay": - await trio.sleep(r.random() * 3) - # I wanted to test random packet corruption too, but it turns out - # openssl has a bug in the following scenario: - # - # - client sends ClientHello - # - server sends HelloVerifyRequest with cookie -- but cookie is - # invalid b/c either the ClientHello or HelloVerifyRequest was - # corrupted - # - client re-sends ClientHello with invalid cookie - # - server replies with new HelloVerifyRequest and correct cookie - # - # At this point, the client *should* switch to the new, valid - # cookie. But OpenSSL doesn't; it stubbornly insists on re-sending - # the original, invalid cookie over and over. In theory we could - # work around this by detecting cookie changes and starting over - # with a whole new SSL object, but (a) it doesn't seem worth it, (b) - # when I tried then I ran into another issue where OpenSSL got stuck - # in an infinite loop sending alerts over and over, which I didn't - # dig into because see (a). - # - # elif op == "distort": - # payload = bytearray(packet.payload) - # payload[r.randrange(len(payload))] ^= 1 << r.randrange(8) - # packet = attrs.evolve(packet, payload=payload) - else: - assert op == "deliver" - print( - f"{packet.source} -> {packet.destination}: delivered" - f" {packet.payload.hex()}", - ) - fn.deliver_packet(packet) - break - - def route_packet_wrapper(packet: UDPPacket) -> None: - try: # noqa: SIM105 # suppressible-exception - nursery.start_soon(route_packet, packet) - except RuntimeError: # pragma: no cover - # We're exiting the nursery, so any remaining packets can just get - # dropped - pass - - fn.route_packet = route_packet_wrapper # type: ignore[assignment] # TODO: Fix FakeNet typing - - for i in range(HANDSHAKES): - print("#" * 80) - print("#" * 80) - print("#" * 80) - with endpoint() as client_endpoint: - client = client_endpoint.connect(address, client_ctx_fn()) - print("client starting do_handshake") - await client.do_handshake() - print("client finished do_handshake") - msg = str(i).encode() - # Make multiple attempts to send data, because the network might - # drop it - while True: - with trio.move_on_after(10) as cscope: - await client.send(msg) - assert await client.receive() == msg - if not cscope.cancelled_caught: - break - - -async def test_implicit_handshake( - server_ctx: SSL.Context, client_ctx: SSL.Context -) -> None: - async with dtls_echo_server(server_ctx=server_ctx) as (_, address): - with endpoint() as client_endpoint: - client = client_endpoint.connect(address, client_ctx) - - # Implicit handshake - await client.send(b"xyz") - assert await client.receive() == b"xyz" - - -async def test_full_duplex(server_ctx: SSL.Context, client_ctx: SSL.Context) -> None: - # Tests simultaneous send/receive, and also multiple methods implicitly invoking - # do_handshake simultaneously. - with endpoint() as server_endpoint, endpoint() as client_endpoint: - await server_endpoint.socket.bind(("127.0.0.1", 0)) - async with trio.open_nursery() as server_nursery: - - async def handler(channel: DTLSChannel) -> None: - async with trio.open_nursery() as nursery: - nursery.start_soon(channel.send, b"from server") - nursery.start_soon(channel.receive) - - await server_nursery.start(server_endpoint.serve, server_ctx, handler) - - client = client_endpoint.connect( - server_endpoint.socket.getsockname(), - client_ctx, - ) - async with trio.open_nursery() as nursery: - nursery.start_soon(client.send, b"from client") - nursery.start_soon(client.receive) - - server_nursery.cancel_scope.cancel() - - -async def test_channel_closing( - server_ctx: SSL.Context, client_ctx: SSL.Context -) -> None: - async with dtls_echo_server(server_ctx=server_ctx) as (_, address): - with endpoint() as client_endpoint: - client = client_endpoint.connect(address, client_ctx) - await client.do_handshake() - client.close() - - with pytest.raises(trio.ClosedResourceError): - await client.send(b"abc") - with pytest.raises(trio.ClosedResourceError): - await client.receive() - - # close is idempotent - client.close() - # can also aclose - await client.aclose() - - -async def test_serve_exits_cleanly_on_close(server_ctx: SSL.Context) -> None: - async with dtls_echo_server(autocancel=False, server_ctx=server_ctx) as ( - server_endpoint, - _address, - ): - server_endpoint.close() - # Testing that the nursery exits even without being cancelled - # close is idempotent - server_endpoint.close() - - -async def test_client_multiplex(server_ctx: SSL.Context) -> None: - async with ( - dtls_echo_server(server_ctx=server_ctx) as (_, address1), - dtls_echo_server(server_ctx=server_ctx) as (_, address2), - ): - with endpoint() as client_endpoint: - client1 = client_endpoint.connect(address1, client_ctx_fn()) - client2 = client_endpoint.connect(address2, client_ctx_fn()) - - await client1.send(b"abc") - await client2.send(b"xyz") - assert await client2.receive() == b"xyz" - assert await client1.receive() == b"abc" - - client_endpoint.close() - - with pytest.raises(trio.ClosedResourceError): - await client1.send(b"xxx") - with pytest.raises(trio.ClosedResourceError): - await client2.receive() - with pytest.raises(trio.ClosedResourceError): - client_endpoint.connect(address1, client_ctx_fn()) - - async def null_handler(_: object) -> None: # pragma: no cover - pass - - async with trio.open_nursery() as nursery: - with pytest.raises(trio.ClosedResourceError): - await nursery.start(client_endpoint.serve, server_ctx, null_handler) - - -async def test_dtls_over_dgram_only() -> None: - with trio.socket.socket() as s: - with pytest.raises(ValueError, match=r"^DTLS requires a SOCK_DGRAM socket$"): - DTLSEndpoint(s) - - -async def test_double_serve(server_ctx: SSL.Context) -> None: - async def null_handler(_: object) -> None: # pragma: no cover - pass - - with endpoint() as server_endpoint: - await server_endpoint.socket.bind(("127.0.0.1", 0)) - async with trio.open_nursery() as nursery: - await nursery.start(server_endpoint.serve, server_ctx, null_handler) - with pytest.raises(trio.BusyResourceError): - await nursery.start(server_endpoint.serve, server_ctx, null_handler) - - nursery.cancel_scope.cancel() - - async with trio.open_nursery() as nursery: - await nursery.start(server_endpoint.serve, server_ctx, null_handler) - nursery.cancel_scope.cancel() - - -async def test_connect_to_non_server( - autojump_clock: trio.abc.Clock, client_ctx: SSL.Context -) -> None: - fn = FakeNet() - fn.enable() - with endpoint() as client1, endpoint() as client2: - await client1.socket.bind(("127.0.0.1", 0)) - # This should just time out - with trio.move_on_after(100) as cscope: - channel = client2.connect(client1.socket.getsockname(), client_ctx) - await channel.do_handshake() - assert cscope.cancelled_caught - - -@pytest.mark.parametrize("buffer_size", [10, 20]) -async def test_incoming_buffer_overflow( - autojump_clock: trio.abc.Clock, - server_ctx: SSL.Context, - client_ctx: SSL.Context, - buffer_size: int, -) -> None: - fn = FakeNet() - fn.enable() - async with dtls_echo_server(server_ctx=server_ctx) as (_, address): - with endpoint(incoming_packets_buffer=buffer_size) as client_endpoint: - assert client_endpoint.incoming_packets_buffer == buffer_size - client = client_endpoint.connect(address, client_ctx) - for i in range(buffer_size + 15): - await client.send(str(i).encode()) - await trio.sleep(1) - stats = client.statistics() - assert stats.incoming_packets_dropped_in_trio == 15 - for i in range(buffer_size): - assert await client.receive() == str(i).encode() - await client.send(b"buffer clear now") - assert await client.receive() == b"buffer clear now" - - -async def test_server_socket_doesnt_crash_on_garbage( - autojump_clock: trio.abc.Clock, server_ctx: SSL.Context -) -> None: - fn = FakeNet() - fn.enable() - - from trio._dtls import ( - ContentType, - HandshakeFragment, - HandshakeType, - ProtocolVersion, - Record, - encode_handshake_fragment, - encode_record, - ) - - client_hello = encode_record( - Record( - content_type=ContentType.handshake, - version=ProtocolVersion.DTLS10, - epoch_seqno=0, - payload=encode_handshake_fragment( - HandshakeFragment( - msg_type=HandshakeType.client_hello, - msg_len=10, - msg_seq=0, - frag_offset=0, - frag_len=10, - frag=bytes(10), - ), - ), - ), - ) - - client_hello_extended = client_hello + b"\x00" - client_hello_short = client_hello[:-1] - # cuts off in middle of handshake message header - client_hello_really_short = client_hello[:14] - client_hello_corrupt_record_len = bytearray(client_hello) - client_hello_corrupt_record_len[11] = 0xFF - - client_hello_fragmented = encode_record( - Record( - content_type=ContentType.handshake, - version=ProtocolVersion.DTLS10, - epoch_seqno=0, - payload=encode_handshake_fragment( - HandshakeFragment( - msg_type=HandshakeType.client_hello, - msg_len=20, - msg_seq=0, - frag_offset=0, - frag_len=10, - frag=bytes(10), - ), - ), - ), - ) - - client_hello_trailing_data_in_record = encode_record( - Record( - content_type=ContentType.handshake, - version=ProtocolVersion.DTLS10, - epoch_seqno=0, - payload=encode_handshake_fragment( - HandshakeFragment( - msg_type=HandshakeType.client_hello, - msg_len=20, - msg_seq=0, - frag_offset=0, - frag_len=10, - frag=bytes(10), - ), - ) - + b"\x00", - ), - ) - - handshake_empty = encode_record( - Record( - content_type=ContentType.handshake, - version=ProtocolVersion.DTLS10, - epoch_seqno=0, - payload=b"", - ), - ) - - client_hello_truncated_in_cookie = encode_record( - Record( - content_type=ContentType.handshake, - version=ProtocolVersion.DTLS10, - epoch_seqno=0, - payload=bytes(2 + 32 + 1) + b"\xff", - ), - ) - - async with dtls_echo_server(server_ctx=server_ctx) as (_, address): - with trio.socket.socket(type=trio.socket.SOCK_DGRAM) as sock: - bad_packets: list[bytes | bytearray | memoryview[int]] = [ - b"", - b"xyz", - client_hello_extended, - client_hello_short, - client_hello_really_short, - client_hello_corrupt_record_len, - client_hello_fragmented, - client_hello_trailing_data_in_record, - handshake_empty, - client_hello_truncated_in_cookie, - ] - - for bad_packet in bad_packets: - await sock.sendto(bad_packet, address) - await trio.sleep(1) - - -async def test_invalid_cookie_rejected( - autojump_clock: trio.abc.Clock, server_ctx: SSL.Context, client_ctx: SSL.Context -) -> None: - fn = FakeNet() - fn.enable() - - from trio._dtls import BadPacket, decode_client_hello_untrusted - - with trio.CancelScope() as cscope: - # the first 11 bytes of ClientHello aren't protected by the cookie, so only test - # corrupting bytes after that. - offset_to_corrupt = count(11) - - def route_packet(packet: UDPPacket) -> None: - try: - _, cookie, _ = decode_client_hello_untrusted(packet.payload) - except BadPacket: - pass - else: - if len(cookie) != 0: - # this is a challenge response packet - # let's corrupt the next offset so the handshake should fail - payload = bytearray(packet.payload) - offset = next(offset_to_corrupt) - if offset >= len(payload): - # We've tried all offsets. Clamp offset to the end of the - # payload, and terminate the test. - offset = len(payload) - 1 - cscope.cancel() - payload[offset] ^= 0x01 - packet = attrs.evolve(packet, payload=bytes(payload)) - - fn.deliver_packet(packet) - - fn.route_packet = route_packet # type: ignore[assignment] # TODO: Fix FakeNet typing - - async with dtls_echo_server(server_ctx=server_ctx) as (_, address): - while True: - with endpoint() as client: - channel = client.connect(address, client_ctx) - await channel.do_handshake() - assert cscope.cancelled_caught - - -async def test_client_cancels_handshake_and_starts_new_one( - autojump_clock: trio.abc.Clock, server_ctx: SSL.Context -) -> None: - # if a client disappears during the handshake, and then starts a new handshake from - # scratch, then the first handler's channel should fail, and a new handler get - # started - fn = FakeNet() - fn.enable() - - with endpoint() as server, endpoint() as client: - await server.socket.bind(("127.0.0.1", 0)) - async with trio.open_nursery() as nursery: - first_time = True - - async def handler(channel: DTLSChannel) -> None: - nonlocal first_time - if first_time: - first_time = False - print("handler: first time, cancelling connect") - connect_cscope.cancel() - await trio.sleep(0.5) - print("handler: handshake should fail now") - with pytest.raises(trio.BrokenResourceError): - await channel.do_handshake() - else: - print("handler: not first time, sending hello") - await channel.send(b"hello") - - await nursery.start(server.serve, server_ctx, handler) - - print("client: starting first connect") - with trio.CancelScope() as connect_cscope: - channel = client.connect(server.socket.getsockname(), client_ctx_fn()) - await channel.do_handshake() - assert connect_cscope.cancelled_caught - - print("client: starting second connect") - channel = client.connect(server.socket.getsockname(), client_ctx_fn()) - assert await channel.receive() == b"hello" - - # Give handlers a chance to finish - await trio.sleep(10) - nursery.cancel_scope.cancel() - - -async def test_swap_client_server(server_ctx: SSL.Context) -> None: - with endpoint() as a, endpoint() as b: - await a.socket.bind(("127.0.0.1", 0)) - await b.socket.bind(("127.0.0.1", 0)) - - async def echo_handler(channel: DTLSChannel) -> None: - async for packet in channel: - await channel.send(packet) - - async def crashing_echo_handler(channel: DTLSChannel) -> None: - with pytest.raises(trio.BrokenResourceError): - await echo_handler(channel) - - async with trio.open_nursery() as nursery: - await nursery.start(a.serve, server_ctx, crashing_echo_handler) - await nursery.start(b.serve, server_ctx, echo_handler) - - b_to_a = b.connect(a.socket.getsockname(), client_ctx_fn()) - await b_to_a.send(b"b as client") - assert await b_to_a.receive() == b"b as client" - - a_to_b = a.connect(b.socket.getsockname(), client_ctx_fn()) - await a_to_b.do_handshake() - with pytest.raises(trio.BrokenResourceError): - await b_to_a.send(b"association broken") - await a_to_b.send(b"a as client") - assert await a_to_b.receive() == b"a as client" - - nursery.cancel_scope.cancel() - - -@slow -async def test_openssl_retransmit_doesnt_break_stuff( - server_ctx: SSL.Context, client_ctx: SSL.Context -) -> None: - # can't use autojump_clock here, because the point of the test is to wait for - # openssl's built-in retransmit timer to expire, which is hard-coded to use - # wall-clock time. - fn = FakeNet() - fn.enable() - - blackholed = True - - def route_packet(packet: UDPPacket) -> None: - if blackholed: - print("dropped packet", packet) - return - print("delivered packet", packet) - # packets.append( - # scapy.all.IP( - # src=packet.source.ip.compressed, dst=packet.destination.ip.compressed - # ) - # / scapy.all.UDP(sport=packet.source.port, dport=packet.destination.port) - # / packet.payload - # ) - fn.deliver_packet(packet) - - fn.route_packet = route_packet # type: ignore[assignment] # TODO add type annotations for FakeNet - - async with dtls_echo_server(server_ctx=server_ctx) as (server_endpoint, address): - with endpoint() as client_endpoint: - async with trio.open_nursery() as nursery: - - async def connecter() -> None: - client = client_endpoint.connect(address, client_ctx) - await client.do_handshake(initial_retransmit_timeout=1.5) - await client.send(b"hi") - assert await client.receive() == b"hi" - - nursery.start_soon(connecter) - - # openssl's default timeout is 1 second, so this ensures that it thinks - # the timeout has expired - await trio.sleep(1.1) - # disable blackholing and send a garbage packet to wake up openssl so it - # notices the timeout has expired - blackholed = False - await server_endpoint.socket.sendto( - b"xxx", - client_endpoint.socket.getsockname(), - ) - # now the client task should finish connecting and exit cleanly - - # scapy.all.wrpcap("/tmp/trace.pcap", packets) - - -async def test_initial_retransmit_timeout_configuration( - autojump_clock: trio.abc.Clock, server_ctx: SSL.Context -) -> None: - fn = FakeNet() - fn.enable() - - blackholed = True - - def route_packet(packet: UDPPacket) -> None: - nonlocal blackholed - if blackholed: - blackholed = False - else: - fn.deliver_packet(packet) - - fn.route_packet = route_packet # type: ignore[assignment] # TODO: add type annotations for FakeNet - - async with dtls_echo_server(server_ctx=server_ctx) as (_, address): - for t in [1, 2, 4]: - with endpoint() as client: - before = trio.current_time() - blackholed = True - channel = client.connect(address, client_ctx_fn()) - await channel.do_handshake(initial_retransmit_timeout=t) - after = trio.current_time() - assert after - before == t - - -async def test_explicit_tiny_mtu_is_respected( - server_ctx: SSL.Context, client_ctx: SSL.Context -) -> None: - # ClientHello is ~240 bytes, and it can't be fragmented, so our mtu has to - # be larger than that. (300 is still smaller than any real network though.) - MTU = 300 - - fn = FakeNet() - fn.enable() - - def route_packet(packet: UDPPacket) -> None: - print(f"delivering {packet}") - print(f"payload size: {len(packet.payload)}") - assert len(packet.payload) <= MTU - fn.deliver_packet(packet) - - fn.route_packet = route_packet # type: ignore[assignment] # TODO add type annotations for FakeNet - - async with dtls_echo_server(mtu=MTU, server_ctx=server_ctx) as (_server, address): - with endpoint() as client: - channel = client.connect(address, client_ctx) - channel.set_ciphertext_mtu(MTU) - await channel.do_handshake() - await channel.send(b"hi") - assert await channel.receive() == b"hi" - - -@parametrize_ipv6 -async def test_handshake_handles_minimum_network_mtu( - ipv6: bool, - autojump_clock: trio.abc.Clock, - server_ctx: SSL.Context, - client_ctx: SSL.Context, -) -> None: - # Fake network that has the minimum allowable MTU for whatever protocol we're using. - fn = FakeNet() - fn.enable() - - mtu = 1280 - 48 if ipv6 else 576 - 28 - - def route_packet(packet: UDPPacket) -> None: - if len(packet.payload) > mtu: - print(f"dropping {packet}") - else: - print(f"delivering {packet}") - fn.deliver_packet(packet) - - fn.route_packet = route_packet # type: ignore[assignment] # TODO: add type annotations for FakeNet - - # See if we can successfully do a handshake -- some of the volleys will get dropped, - # and the retransmit logic should detect this and back off the MTU to something - # smaller until it succeeds. - async with dtls_echo_server(ipv6=ipv6, server_ctx=server_ctx) as (_, address): - with endpoint(ipv6=ipv6) as client_endpoint: - client = client_endpoint.connect(address, client_ctx) - # the handshake mtu backoff shouldn't affect the return value from - # get_cleartext_mtu, b/c that's under the user's control via - # set_ciphertext_mtu - client.set_ciphertext_mtu(9999) - await client.send(b"xyz") - assert await client.receive() == b"xyz" - assert client.get_cleartext_mtu() > 9000 # as vegeta said - - -@pytest.mark.filterwarnings("always:unclosed DTLS:ResourceWarning") -async def test_system_task_cleaned_up_on_gc(client_ctx: SSL.Context) -> None: - before_tasks = trio.lowlevel.current_statistics().tasks_living - - # We put this into a sub-function so that everything automatically becomes garbage - # when the frame exits. For some reason just doing 'del e' wasn't enough on pypy - # with coverage enabled -- I think we were hitting this bug: - # https://foss.heptapod.net/pypy/pypy/-/issues/3656 - async def start_and_forget_endpoint() -> int: - e = endpoint() - - # This connection/handshake attempt can't succeed. The only purpose is to force - # the endpoint to set up a receive loop. - with trio.socket.socket(type=trio.socket.SOCK_DGRAM) as s: - await s.bind(("127.0.0.1", 0)) - c = e.connect(s.getsockname(), client_ctx) - async with trio.open_nursery() as nursery: - nursery.start_soon(c.do_handshake) - await trio.testing.wait_all_tasks_blocked() - nursery.cancel_scope.cancel() - - during_tasks = trio.lowlevel.current_statistics().tasks_living - return during_tasks - - with pytest.warns(ResourceWarning): # noqa: PT031 - during_tasks = await start_and_forget_endpoint() - await trio.testing.wait_all_tasks_blocked() - gc_collect_harder() - - await trio.testing.wait_all_tasks_blocked() - - after_tasks = trio.lowlevel.current_statistics().tasks_living - assert before_tasks < during_tasks - assert before_tasks == after_tasks - - -@pytest.mark.filterwarnings("always:unclosed DTLS:ResourceWarning") -async def test_gc_before_system_task_starts() -> None: - e = endpoint() - - with pytest.warns(ResourceWarning): # noqa: PT031 - del e - gc_collect_harder() - - await trio.testing.wait_all_tasks_blocked() - - -@pytest.mark.filterwarnings("always:unclosed DTLS:ResourceWarning") -async def test_gc_as_packet_received() -> None: - fn = FakeNet() - fn.enable() - - e = endpoint() - await e.socket.bind(("127.0.0.1", 0)) - e._ensure_receive_loop() - - await trio.testing.wait_all_tasks_blocked() - - with trio.socket.socket(type=trio.socket.SOCK_DGRAM) as s: - await s.sendto(b"xxx", e.socket.getsockname()) - # At this point, the endpoint's receive loop has been marked runnable because it - # just received a packet; closing the endpoint socket won't interrupt that. But by - # the time it wakes up to process the packet, the endpoint will be gone. - with pytest.warns(ResourceWarning): # noqa: PT031 - del e - gc_collect_harder() - - -@pytest.mark.filterwarnings("always:unclosed DTLS:ResourceWarning") -def test_gc_after_trio_exits() -> None: - async def main() -> DTLSEndpoint: - # We use fakenet just to make sure no real sockets can leak out of the test - # case - on pypy somehow the socket was outliving the gc_collect_harder call - # below. Since the test is just making sure DTLSEndpoint.__del__ doesn't explode - # when called after trio exits, it doesn't need a real socket. - fn = FakeNet() - fn.enable() - return endpoint() - - e = trio.run(main) - with pytest.warns(ResourceWarning): # noqa: PT031 - del e - gc_collect_harder() - - -async def test_already_closed_socket_doesnt_crash() -> None: - with endpoint() as e: - # We close the socket before checkpointing, so the socket will already be closed - # when the system task starts up - e.socket.close() - # Now give it a chance to start up, and hopefully not crash - await trio.testing.wait_all_tasks_blocked() - - -async def test_socket_closed_while_processing_clienthello( - autojump_clock: trio.abc.Clock, server_ctx: SSL.Context, client_ctx: SSL.Context -) -> None: - fn = FakeNet() - fn.enable() - - # Check what happens if the socket is discovered to be closed when sending a - # HelloVerifyRequest, since that has its own sending logic - async with dtls_echo_server(server_ctx=server_ctx) as (server, address): - - def route_packet(packet: UDPPacket) -> None: - fn.deliver_packet(packet) - server.socket.close() - - fn.route_packet = route_packet # type: ignore[assignment] # TODO add type annotations for FakeNet - - with endpoint() as client_endpoint: - with trio.move_on_after(10): - client = client_endpoint.connect(address, client_ctx) - await client.do_handshake() - - -async def test_association_replaced_while_handshake_running( - autojump_clock: trio.abc.Clock, server_ctx: SSL.Context -) -> None: - fn = FakeNet() - fn.enable() - - def route_packet(packet: UDPPacket) -> None: - pass - - fn.route_packet = route_packet # type: ignore[assignment] # TODO: add type annotations for FakeNet - - async with dtls_echo_server(server_ctx=server_ctx) as (_, address): - with endpoint() as client_endpoint: - # TODO: should this have the same exact client_ctx? - c1 = client_endpoint.connect(address, client_ctx_fn()) - async with trio.open_nursery() as nursery: - - async def doomed_handshake() -> None: - with pytest.raises(trio.BrokenResourceError): - await c1.do_handshake() - - nursery.start_soon(doomed_handshake) - - await trio.sleep(10) - - client_endpoint.connect(address, client_ctx_fn()) - - -async def test_association_replaced_before_handshake_starts( - server_ctx: SSL.Context, -) -> None: - fn = FakeNet() - fn.enable() - - # This test shouldn't send any packets - def route_packet(packet: UDPPacket) -> NoReturn: # pragma: no cover - raise AssertionError() - - fn.route_packet = route_packet # type: ignore[assignment] # TODO add type annotations for FakeNet - - async with dtls_echo_server(server_ctx=server_ctx) as (_, address): - with endpoint() as client_endpoint: - # TODO: should this use the same client_ctx? - c1 = client_endpoint.connect(address, client_ctx_fn()) - client_endpoint.connect(address, client_ctx_fn()) - with pytest.raises(trio.BrokenResourceError): - await c1.do_handshake() - - -async def test_send_to_closed_local_port(server_ctx: SSL.Context) -> None: - # On Windows, sending a UDP packet to a closed local port can cause a weird - # ECONNRESET error later, inside the receive task. Make sure we're handling it - # properly. - async with dtls_echo_server(server_ctx=server_ctx) as (_, address): - with endpoint() as client_endpoint: - async with trio.open_nursery() as nursery: - for i in range(1, 10): - channel = client_endpoint.connect(("127.0.0.1", i), client_ctx_fn()) - nursery.start_soon(channel.do_handshake) - channel = client_endpoint.connect(address, client_ctx_fn()) - await channel.send(b"xxx") - assert await channel.receive() == b"xxx" - nursery.cancel_scope.cancel() +from __future__ import annotations + +import random +from contextlib import asynccontextmanager +from itertools import count +from typing import TYPE_CHECKING, NoReturn + +import attrs +import pytest + +from trio._tests.pytest_plugin import maybe_ignore_import_error + +try: + import trustme + from OpenSSL import SSL +except ImportError as error: + maybe_ignore_import_error(error) + + +import trio +import trio.testing +from trio import DTLSChannel, DTLSEndpoint +from trio.testing._fake_net import FakeNet, UDPPacket + +from .._core._tests.tutil import binds_ipv6, gc_collect_harder, slow + +if TYPE_CHECKING: + from collections.abc import AsyncGenerator + +ca = trustme.CA() +server_cert = ca.issue_cert("example.com") + + +@pytest.fixture +def server_ctx() -> SSL.Context: + ctx = SSL.Context(SSL.DTLS_METHOD) + server_cert.configure_cert(ctx) + return ctx + + +def client_ctx_fn() -> SSL.Context: + ctx = SSL.Context(SSL.DTLS_METHOD) + ca.configure_trust(ctx) + return ctx + + +@pytest.fixture +def client_ctx() -> SSL.Context: + return client_ctx_fn() + + +parametrize_ipv6 = pytest.mark.parametrize( + "ipv6", + [False, pytest.param(True, marks=binds_ipv6)], + ids=["ipv4", "ipv6"], +) + + +def endpoint(**kwargs: int | bool) -> DTLSEndpoint: + ipv6 = kwargs.pop("ipv6", False) + family = trio.socket.AF_INET6 if ipv6 else trio.socket.AF_INET + sock = trio.socket.socket(type=trio.socket.SOCK_DGRAM, family=family) + return DTLSEndpoint(sock, **kwargs) + + +@asynccontextmanager +async def dtls_echo_server( + *, + server_ctx: SSL.Context, + autocancel: bool = True, + mtu: int | None = None, + ipv6: bool = False, +) -> AsyncGenerator[tuple[DTLSEndpoint, tuple[str, int]], None]: + with endpoint(ipv6=ipv6) as server: + localhost = "::1" if ipv6 else "127.0.0.1" + await server.socket.bind((localhost, 0)) + async with trio.open_nursery() as nursery: + + async def echo_handler(dtls_channel: DTLSChannel) -> None: + print( + "echo handler started: " + f"server {dtls_channel.endpoint.socket.getsockname()!r} " + f"client {dtls_channel.peer_address!r}", + ) + if mtu is not None: + dtls_channel.set_ciphertext_mtu(mtu) + try: + print("server starting do_handshake") + await dtls_channel.do_handshake() + print("server finished do_handshake") + # no branch for leaving this for loop because we only leave + # a channel by cancellation. + async for packet in dtls_channel: # pragma: no branch + print(f"echoing {packet!r} -> {dtls_channel.peer_address!r}") + await dtls_channel.send(packet) + except trio.BrokenResourceError: # pragma: no cover + print("echo handler channel broken") + + await nursery.start(server.serve, server_ctx, echo_handler) + + yield server, server.socket.getsockname() # noqa: RUF075 + + if autocancel: + nursery.cancel_scope.cancel() + + +@parametrize_ipv6 +async def test_smoke( + ipv6: bool, server_ctx: SSL.Context, client_ctx: SSL.Context +) -> None: + async with dtls_echo_server(ipv6=ipv6, server_ctx=server_ctx) as ( + _server_endpoint, + address, + ): + with endpoint(ipv6=ipv6) as client_endpoint: + client_channel = client_endpoint.connect(address, client_ctx) + with pytest.raises(trio.NeedHandshakeError): + client_channel.get_cleartext_mtu() + + await client_channel.do_handshake() + await client_channel.send(b"hello") + assert await client_channel.receive() == b"hello" + await client_channel.send(b"goodbye") + assert await client_channel.receive() == b"goodbye" + + with pytest.raises( + ValueError, + match=r"^openssl doesn't support sending empty DTLS packets$", + ): + await client_channel.send(b"") + + client_channel.set_ciphertext_mtu(1234) + cleartext_mtu_1234 = client_channel.get_cleartext_mtu() + client_channel.set_ciphertext_mtu(4321) + assert client_channel.get_cleartext_mtu() > cleartext_mtu_1234 + client_channel.set_ciphertext_mtu(1234) + assert client_channel.get_cleartext_mtu() == cleartext_mtu_1234 + + +@slow +async def test_handshake_over_terrible_network( + autojump_clock: trio.testing.MockClock, + server_ctx: SSL.Context, +) -> None: + HANDSHAKES = 100 + r = random.Random(0) + fn = FakeNet() + fn.enable() + # avoid spurious timeouts on slow machines + autojump_clock.autojump_threshold = 0.001 + + async with dtls_echo_server(server_ctx=server_ctx) as (_, address): + async with trio.open_nursery() as nursery: + + async def route_packet(packet: UDPPacket) -> None: + while True: + op = r.choices( + ["deliver", "drop", "dupe", "delay"], + weights=[0.7, 0.1, 0.1, 0.1], + )[0] + print(f"{packet.source} -> {packet.destination}: {op}") + if op == "drop": + return + elif op == "dupe": + fn.send_packet(packet) + elif op == "delay": + await trio.sleep(r.random() * 3) + # I wanted to test random packet corruption too, but it turns out + # openssl has a bug in the following scenario: + # + # - client sends ClientHello + # - server sends HelloVerifyRequest with cookie -- but cookie is + # invalid b/c either the ClientHello or HelloVerifyRequest was + # corrupted + # - client re-sends ClientHello with invalid cookie + # - server replies with new HelloVerifyRequest and correct cookie + # + # At this point, the client *should* switch to the new, valid + # cookie. But OpenSSL doesn't; it stubbornly insists on re-sending + # the original, invalid cookie over and over. In theory we could + # work around this by detecting cookie changes and starting over + # with a whole new SSL object, but (a) it doesn't seem worth it, (b) + # when I tried then I ran into another issue where OpenSSL got stuck + # in an infinite loop sending alerts over and over, which I didn't + # dig into because see (a). + # + # elif op == "distort": + # payload = bytearray(packet.payload) + # payload[r.randrange(len(payload))] ^= 1 << r.randrange(8) + # packet = attrs.evolve(packet, payload=payload) + else: + assert op == "deliver" + print( + f"{packet.source} -> {packet.destination}: delivered" + f" {packet.payload.hex()}", + ) + fn.deliver_packet(packet) + break + + def route_packet_wrapper(packet: UDPPacket) -> None: + try: # noqa: SIM105 # suppressible-exception + nursery.start_soon(route_packet, packet) + except RuntimeError: # pragma: no cover + # We're exiting the nursery, so any remaining packets can just get + # dropped + pass + + fn.route_packet = route_packet_wrapper # type: ignore[assignment] # TODO: Fix FakeNet typing + + for i in range(HANDSHAKES): + print("#" * 80) + print("#" * 80) + print("#" * 80) + with endpoint() as client_endpoint: + client = client_endpoint.connect(address, client_ctx_fn()) + print("client starting do_handshake") + await client.do_handshake() + print("client finished do_handshake") + msg = str(i).encode() + # Make multiple attempts to send data, because the network might + # drop it + while True: + with trio.move_on_after(10) as cscope: + await client.send(msg) + assert await client.receive() == msg + if not cscope.cancelled_caught: + break + + +async def test_implicit_handshake( + server_ctx: SSL.Context, client_ctx: SSL.Context +) -> None: + async with dtls_echo_server(server_ctx=server_ctx) as (_, address): + with endpoint() as client_endpoint: + client = client_endpoint.connect(address, client_ctx) + + # Implicit handshake + await client.send(b"xyz") + assert await client.receive() == b"xyz" + + +async def test_full_duplex(server_ctx: SSL.Context, client_ctx: SSL.Context) -> None: + # Tests simultaneous send/receive, and also multiple methods implicitly invoking + # do_handshake simultaneously. + with endpoint() as server_endpoint, endpoint() as client_endpoint: + await server_endpoint.socket.bind(("127.0.0.1", 0)) + async with trio.open_nursery() as server_nursery: + + async def handler(channel: DTLSChannel) -> None: + async with trio.open_nursery() as nursery: + nursery.start_soon(channel.send, b"from server") + nursery.start_soon(channel.receive) + + await server_nursery.start(server_endpoint.serve, server_ctx, handler) + + client = client_endpoint.connect( + server_endpoint.socket.getsockname(), + client_ctx, + ) + async with trio.open_nursery() as nursery: + nursery.start_soon(client.send, b"from client") + nursery.start_soon(client.receive) + + server_nursery.cancel_scope.cancel() + + +async def test_channel_closing( + server_ctx: SSL.Context, client_ctx: SSL.Context +) -> None: + async with dtls_echo_server(server_ctx=server_ctx) as (_, address): + with endpoint() as client_endpoint: + client = client_endpoint.connect(address, client_ctx) + await client.do_handshake() + client.close() + + with pytest.raises(trio.ClosedResourceError): + await client.send(b"abc") + with pytest.raises(trio.ClosedResourceError): + await client.receive() + + # close is idempotent + client.close() + # can also aclose + await client.aclose() + + +async def test_serve_exits_cleanly_on_close(server_ctx: SSL.Context) -> None: + async with dtls_echo_server(autocancel=False, server_ctx=server_ctx) as ( + server_endpoint, + _address, + ): + server_endpoint.close() + # Testing that the nursery exits even without being cancelled + # close is idempotent + server_endpoint.close() + + +async def test_client_multiplex(server_ctx: SSL.Context) -> None: + async with ( + dtls_echo_server(server_ctx=server_ctx) as (_, address1), + dtls_echo_server(server_ctx=server_ctx) as (_, address2), + ): + with endpoint() as client_endpoint: + client1 = client_endpoint.connect(address1, client_ctx_fn()) + client2 = client_endpoint.connect(address2, client_ctx_fn()) + + await client1.send(b"abc") + await client2.send(b"xyz") + assert await client2.receive() == b"xyz" + assert await client1.receive() == b"abc" + + client_endpoint.close() + + with pytest.raises(trio.ClosedResourceError): + await client1.send(b"xxx") + with pytest.raises(trio.ClosedResourceError): + await client2.receive() + with pytest.raises(trio.ClosedResourceError): + client_endpoint.connect(address1, client_ctx_fn()) + + async def null_handler(_: object) -> None: # pragma: no cover + pass + + async with trio.open_nursery() as nursery: + with pytest.raises(trio.ClosedResourceError): + await nursery.start(client_endpoint.serve, server_ctx, null_handler) + + +async def test_dtls_over_dgram_only() -> None: + with trio.socket.socket() as s: + with pytest.raises(ValueError, match=r"^DTLS requires a SOCK_DGRAM socket$"): + DTLSEndpoint(s) + + +async def test_double_serve(server_ctx: SSL.Context) -> None: + async def null_handler(_: object) -> None: # pragma: no cover + pass + + with endpoint() as server_endpoint: + await server_endpoint.socket.bind(("127.0.0.1", 0)) + async with trio.open_nursery() as nursery: + await nursery.start(server_endpoint.serve, server_ctx, null_handler) + with pytest.raises(trio.BusyResourceError): + await nursery.start(server_endpoint.serve, server_ctx, null_handler) + + nursery.cancel_scope.cancel() + + async with trio.open_nursery() as nursery: + await nursery.start(server_endpoint.serve, server_ctx, null_handler) + nursery.cancel_scope.cancel() + + +async def test_connect_to_non_server( + autojump_clock: trio.abc.Clock, client_ctx: SSL.Context +) -> None: + fn = FakeNet() + fn.enable() + with endpoint() as client1, endpoint() as client2: + await client1.socket.bind(("127.0.0.1", 0)) + # This should just time out + with trio.move_on_after(100) as cscope: + channel = client2.connect(client1.socket.getsockname(), client_ctx) + await channel.do_handshake() + assert cscope.cancelled_caught + + +@pytest.mark.parametrize("buffer_size", [10, 20]) +async def test_incoming_buffer_overflow( + autojump_clock: trio.abc.Clock, + server_ctx: SSL.Context, + client_ctx: SSL.Context, + buffer_size: int, +) -> None: + fn = FakeNet() + fn.enable() + async with dtls_echo_server(server_ctx=server_ctx) as (_, address): + with endpoint(incoming_packets_buffer=buffer_size) as client_endpoint: + assert client_endpoint.incoming_packets_buffer == buffer_size + client = client_endpoint.connect(address, client_ctx) + for i in range(buffer_size + 15): + await client.send(str(i).encode()) + await trio.sleep(1) + stats = client.statistics() + assert stats.incoming_packets_dropped_in_trio == 15 + for i in range(buffer_size): + assert await client.receive() == str(i).encode() + await client.send(b"buffer clear now") + assert await client.receive() == b"buffer clear now" + + +async def test_server_socket_doesnt_crash_on_garbage( + autojump_clock: trio.abc.Clock, server_ctx: SSL.Context +) -> None: + fn = FakeNet() + fn.enable() + + from trio._dtls import ( + ContentType, + HandshakeFragment, + HandshakeType, + ProtocolVersion, + Record, + encode_handshake_fragment, + encode_record, + ) + + client_hello = encode_record( + Record( + content_type=ContentType.handshake, + version=ProtocolVersion.DTLS10, + epoch_seqno=0, + payload=encode_handshake_fragment( + HandshakeFragment( + msg_type=HandshakeType.client_hello, + msg_len=10, + msg_seq=0, + frag_offset=0, + frag_len=10, + frag=bytes(10), + ), + ), + ), + ) + + client_hello_extended = client_hello + b"\x00" + client_hello_short = client_hello[:-1] + # cuts off in middle of handshake message header + client_hello_really_short = client_hello[:14] + client_hello_corrupt_record_len = bytearray(client_hello) + client_hello_corrupt_record_len[11] = 0xFF + + client_hello_fragmented = encode_record( + Record( + content_type=ContentType.handshake, + version=ProtocolVersion.DTLS10, + epoch_seqno=0, + payload=encode_handshake_fragment( + HandshakeFragment( + msg_type=HandshakeType.client_hello, + msg_len=20, + msg_seq=0, + frag_offset=0, + frag_len=10, + frag=bytes(10), + ), + ), + ), + ) + + client_hello_trailing_data_in_record = encode_record( + Record( + content_type=ContentType.handshake, + version=ProtocolVersion.DTLS10, + epoch_seqno=0, + payload=encode_handshake_fragment( + HandshakeFragment( + msg_type=HandshakeType.client_hello, + msg_len=20, + msg_seq=0, + frag_offset=0, + frag_len=10, + frag=bytes(10), + ), + ) + + b"\x00", + ), + ) + + handshake_empty = encode_record( + Record( + content_type=ContentType.handshake, + version=ProtocolVersion.DTLS10, + epoch_seqno=0, + payload=b"", + ), + ) + + client_hello_truncated_in_cookie = encode_record( + Record( + content_type=ContentType.handshake, + version=ProtocolVersion.DTLS10, + epoch_seqno=0, + payload=bytes(2 + 32 + 1) + b"\xff", + ), + ) + + async with dtls_echo_server(server_ctx=server_ctx) as (_, address): + with trio.socket.socket(type=trio.socket.SOCK_DGRAM) as sock: + bad_packets: list[bytes | bytearray | memoryview[int]] = [ + b"", + b"xyz", + client_hello_extended, + client_hello_short, + client_hello_really_short, + client_hello_corrupt_record_len, + client_hello_fragmented, + client_hello_trailing_data_in_record, + handshake_empty, + client_hello_truncated_in_cookie, + ] + + for bad_packet in bad_packets: + await sock.sendto(bad_packet, address) + await trio.sleep(1) + + +async def test_invalid_cookie_rejected( + autojump_clock: trio.abc.Clock, server_ctx: SSL.Context, client_ctx: SSL.Context +) -> None: + fn = FakeNet() + fn.enable() + + from trio._dtls import BadPacket, decode_client_hello_untrusted + + with trio.CancelScope() as cscope: + # the first 11 bytes of ClientHello aren't protected by the cookie, so only test + # corrupting bytes after that. + offset_to_corrupt = count(11) + + def route_packet(packet: UDPPacket) -> None: + try: + _, cookie, _ = decode_client_hello_untrusted(packet.payload) + except BadPacket: + pass + else: + if len(cookie) != 0: + # this is a challenge response packet + # let's corrupt the next offset so the handshake should fail + payload = bytearray(packet.payload) + offset = next(offset_to_corrupt) + if offset >= len(payload): + # We've tried all offsets. Clamp offset to the end of the + # payload, and terminate the test. + offset = len(payload) - 1 + cscope.cancel() + payload[offset] ^= 0x01 + packet = attrs.evolve(packet, payload=bytes(payload)) + + fn.deliver_packet(packet) + + fn.route_packet = route_packet # type: ignore[assignment] # TODO: Fix FakeNet typing + + async with dtls_echo_server(server_ctx=server_ctx) as (_, address): + while True: + with endpoint() as client: + channel = client.connect(address, client_ctx) + await channel.do_handshake() + assert cscope.cancelled_caught + + +async def test_client_cancels_handshake_and_starts_new_one( + autojump_clock: trio.abc.Clock, server_ctx: SSL.Context +) -> None: + # if a client disappears during the handshake, and then starts a new handshake from + # scratch, then the first handler's channel should fail, and a new handler get + # started + fn = FakeNet() + fn.enable() + + with endpoint() as server, endpoint() as client: + await server.socket.bind(("127.0.0.1", 0)) + async with trio.open_nursery() as nursery: + first_time = True + + async def handler(channel: DTLSChannel) -> None: + nonlocal first_time + if first_time: + first_time = False + print("handler: first time, cancelling connect") + connect_cscope.cancel() + await trio.sleep(0.5) + print("handler: handshake should fail now") + with pytest.raises(trio.BrokenResourceError): + await channel.do_handshake() + else: + print("handler: not first time, sending hello") + await channel.send(b"hello") + + await nursery.start(server.serve, server_ctx, handler) + + print("client: starting first connect") + with trio.CancelScope() as connect_cscope: + channel = client.connect(server.socket.getsockname(), client_ctx_fn()) + await channel.do_handshake() + assert connect_cscope.cancelled_caught + + print("client: starting second connect") + channel = client.connect(server.socket.getsockname(), client_ctx_fn()) + assert await channel.receive() == b"hello" + + # Give handlers a chance to finish + await trio.sleep(10) + nursery.cancel_scope.cancel() + + +async def test_swap_client_server(server_ctx: SSL.Context) -> None: + with endpoint() as a, endpoint() as b: + await a.socket.bind(("127.0.0.1", 0)) + await b.socket.bind(("127.0.0.1", 0)) + + async def echo_handler(channel: DTLSChannel) -> None: + async for packet in channel: + await channel.send(packet) + + async def crashing_echo_handler(channel: DTLSChannel) -> None: + with pytest.raises(trio.BrokenResourceError): + await echo_handler(channel) + + async with trio.open_nursery() as nursery: + await nursery.start(a.serve, server_ctx, crashing_echo_handler) + await nursery.start(b.serve, server_ctx, echo_handler) + + b_to_a = b.connect(a.socket.getsockname(), client_ctx_fn()) + await b_to_a.send(b"b as client") + assert await b_to_a.receive() == b"b as client" + + a_to_b = a.connect(b.socket.getsockname(), client_ctx_fn()) + await a_to_b.do_handshake() + with pytest.raises(trio.BrokenResourceError): + await b_to_a.send(b"association broken") + await a_to_b.send(b"a as client") + assert await a_to_b.receive() == b"a as client" + + nursery.cancel_scope.cancel() + + +@slow +async def test_openssl_retransmit_doesnt_break_stuff( + server_ctx: SSL.Context, client_ctx: SSL.Context +) -> None: + # can't use autojump_clock here, because the point of the test is to wait for + # openssl's built-in retransmit timer to expire, which is hard-coded to use + # wall-clock time. + fn = FakeNet() + fn.enable() + + blackholed = True + + def route_packet(packet: UDPPacket) -> None: + if blackholed: + print("dropped packet", packet) + return + print("delivered packet", packet) + # packets.append( + # scapy.all.IP( + # src=packet.source.ip.compressed, dst=packet.destination.ip.compressed + # ) + # / scapy.all.UDP(sport=packet.source.port, dport=packet.destination.port) + # / packet.payload + # ) + fn.deliver_packet(packet) + + fn.route_packet = route_packet # type: ignore[assignment] # TODO add type annotations for FakeNet + + async with dtls_echo_server(server_ctx=server_ctx) as (server_endpoint, address): + with endpoint() as client_endpoint: + async with trio.open_nursery() as nursery: + + async def connecter() -> None: + client = client_endpoint.connect(address, client_ctx) + await client.do_handshake(initial_retransmit_timeout=1.5) + await client.send(b"hi") + assert await client.receive() == b"hi" + + nursery.start_soon(connecter) + + # openssl's default timeout is 1 second, so this ensures that it thinks + # the timeout has expired + await trio.sleep(1.1) + # disable blackholing and send a garbage packet to wake up openssl so it + # notices the timeout has expired + blackholed = False + await server_endpoint.socket.sendto( + b"xxx", + client_endpoint.socket.getsockname(), + ) + # now the client task should finish connecting and exit cleanly + + # scapy.all.wrpcap("/tmp/trace.pcap", packets) + + +async def test_initial_retransmit_timeout_configuration( + autojump_clock: trio.abc.Clock, server_ctx: SSL.Context +) -> None: + fn = FakeNet() + fn.enable() + + blackholed = True + + def route_packet(packet: UDPPacket) -> None: + nonlocal blackholed + if blackholed: + blackholed = False + else: + fn.deliver_packet(packet) + + fn.route_packet = route_packet # type: ignore[assignment] # TODO: add type annotations for FakeNet + + async with dtls_echo_server(server_ctx=server_ctx) as (_, address): + for t in [1, 2, 4]: + with endpoint() as client: + before = trio.current_time() + blackholed = True + channel = client.connect(address, client_ctx_fn()) + await channel.do_handshake(initial_retransmit_timeout=t) + after = trio.current_time() + assert after - before == t + + +async def test_explicit_tiny_mtu_is_respected( + server_ctx: SSL.Context, client_ctx: SSL.Context +) -> None: + # ClientHello is ~240 bytes, and it can't be fragmented, so our mtu has to + # be larger than that. (300 is still smaller than any real network though.) + MTU = 300 + + fn = FakeNet() + fn.enable() + + def route_packet(packet: UDPPacket) -> None: + print(f"delivering {packet}") + print(f"payload size: {len(packet.payload)}") + assert len(packet.payload) <= MTU + fn.deliver_packet(packet) + + fn.route_packet = route_packet # type: ignore[assignment] # TODO add type annotations for FakeNet + + async with dtls_echo_server(mtu=MTU, server_ctx=server_ctx) as (_server, address): + with endpoint() as client: + channel = client.connect(address, client_ctx) + channel.set_ciphertext_mtu(MTU) + await channel.do_handshake() + await channel.send(b"hi") + assert await channel.receive() == b"hi" + + +@parametrize_ipv6 +async def test_handshake_handles_minimum_network_mtu( + ipv6: bool, + autojump_clock: trio.abc.Clock, + server_ctx: SSL.Context, + client_ctx: SSL.Context, +) -> None: + # Fake network that has the minimum allowable MTU for whatever protocol we're using. + fn = FakeNet() + fn.enable() + + mtu = 1280 - 48 if ipv6 else 576 - 28 + + def route_packet(packet: UDPPacket) -> None: + if len(packet.payload) > mtu: + print(f"dropping {packet}") + else: + print(f"delivering {packet}") + fn.deliver_packet(packet) + + fn.route_packet = route_packet # type: ignore[assignment] # TODO: add type annotations for FakeNet + + # See if we can successfully do a handshake -- some of the volleys will get dropped, + # and the retransmit logic should detect this and back off the MTU to something + # smaller until it succeeds. + async with dtls_echo_server(ipv6=ipv6, server_ctx=server_ctx) as (_, address): + with endpoint(ipv6=ipv6) as client_endpoint: + client = client_endpoint.connect(address, client_ctx) + # the handshake mtu backoff shouldn't affect the return value from + # get_cleartext_mtu, b/c that's under the user's control via + # set_ciphertext_mtu + client.set_ciphertext_mtu(9999) + await client.send(b"xyz") + assert await client.receive() == b"xyz" + assert client.get_cleartext_mtu() > 9000 # as vegeta said + + +@pytest.mark.filterwarnings("always:unclosed DTLS:ResourceWarning") +async def test_system_task_cleaned_up_on_gc(client_ctx: SSL.Context) -> None: + before_tasks = trio.lowlevel.current_statistics().tasks_living + + # We put this into a sub-function so that everything automatically becomes garbage + # when the frame exits. For some reason just doing 'del e' wasn't enough on pypy + # with coverage enabled -- I think we were hitting this bug: + # https://foss.heptapod.net/pypy/pypy/-/issues/3656 + async def start_and_forget_endpoint() -> int: + e = endpoint() + + # This connection/handshake attempt can't succeed. The only purpose is to force + # the endpoint to set up a receive loop. + with trio.socket.socket(type=trio.socket.SOCK_DGRAM) as s: + await s.bind(("127.0.0.1", 0)) + c = e.connect(s.getsockname(), client_ctx) + async with trio.open_nursery() as nursery: + nursery.start_soon(c.do_handshake) + await trio.testing.wait_all_tasks_blocked() + nursery.cancel_scope.cancel() + + during_tasks = trio.lowlevel.current_statistics().tasks_living + return during_tasks + + with pytest.warns(ResourceWarning): # noqa: PT031 + during_tasks = await start_and_forget_endpoint() + await trio.testing.wait_all_tasks_blocked() + gc_collect_harder() + + await trio.testing.wait_all_tasks_blocked() + + after_tasks = trio.lowlevel.current_statistics().tasks_living + assert before_tasks < during_tasks + assert before_tasks == after_tasks + + +@pytest.mark.filterwarnings("always:unclosed DTLS:ResourceWarning") +async def test_gc_before_system_task_starts() -> None: + e = endpoint() + + with pytest.warns(ResourceWarning): # noqa: PT031 + del e + gc_collect_harder() + + await trio.testing.wait_all_tasks_blocked() + + +@pytest.mark.filterwarnings("always:unclosed DTLS:ResourceWarning") +async def test_gc_as_packet_received() -> None: + fn = FakeNet() + fn.enable() + + e = endpoint() + await e.socket.bind(("127.0.0.1", 0)) + e._ensure_receive_loop() + + await trio.testing.wait_all_tasks_blocked() + + with trio.socket.socket(type=trio.socket.SOCK_DGRAM) as s: + await s.sendto(b"xxx", e.socket.getsockname()) + # At this point, the endpoint's receive loop has been marked runnable because it + # just received a packet; closing the endpoint socket won't interrupt that. But by + # the time it wakes up to process the packet, the endpoint will be gone. + with pytest.warns(ResourceWarning): # noqa: PT031 + del e + gc_collect_harder() + + +@pytest.mark.filterwarnings("always:unclosed DTLS:ResourceWarning") +def test_gc_after_trio_exits() -> None: + async def main() -> DTLSEndpoint: + # We use fakenet just to make sure no real sockets can leak out of the test + # case - on pypy somehow the socket was outliving the gc_collect_harder call + # below. Since the test is just making sure DTLSEndpoint.__del__ doesn't explode + # when called after trio exits, it doesn't need a real socket. + fn = FakeNet() + fn.enable() + return endpoint() + + e = trio.run(main) + with pytest.warns(ResourceWarning): # noqa: PT031 + del e + gc_collect_harder() + + +async def test_already_closed_socket_doesnt_crash() -> None: + with endpoint() as e: + # We close the socket before checkpointing, so the socket will already be closed + # when the system task starts up + e.socket.close() + # Now give it a chance to start up, and hopefully not crash + await trio.testing.wait_all_tasks_blocked() + + +async def test_socket_closed_while_processing_clienthello( + autojump_clock: trio.abc.Clock, server_ctx: SSL.Context, client_ctx: SSL.Context +) -> None: + fn = FakeNet() + fn.enable() + + # Check what happens if the socket is discovered to be closed when sending a + # HelloVerifyRequest, since that has its own sending logic + async with dtls_echo_server(server_ctx=server_ctx) as (server, address): + + def route_packet(packet: UDPPacket) -> None: + fn.deliver_packet(packet) + server.socket.close() + + fn.route_packet = route_packet # type: ignore[assignment] # TODO add type annotations for FakeNet + + with endpoint() as client_endpoint: + with trio.move_on_after(10): + client = client_endpoint.connect(address, client_ctx) + await client.do_handshake() + + +async def test_association_replaced_while_handshake_running( + autojump_clock: trio.abc.Clock, server_ctx: SSL.Context +) -> None: + fn = FakeNet() + fn.enable() + + def route_packet(packet: UDPPacket) -> None: + pass + + fn.route_packet = route_packet # type: ignore[assignment] # TODO: add type annotations for FakeNet + + async with dtls_echo_server(server_ctx=server_ctx) as (_, address): + with endpoint() as client_endpoint: + # TODO: should this have the same exact client_ctx? + c1 = client_endpoint.connect(address, client_ctx_fn()) + async with trio.open_nursery() as nursery: + + async def doomed_handshake() -> None: + with pytest.raises(trio.BrokenResourceError): + await c1.do_handshake() + + nursery.start_soon(doomed_handshake) + + await trio.sleep(10) + + client_endpoint.connect(address, client_ctx_fn()) + + +async def test_association_replaced_before_handshake_starts( + server_ctx: SSL.Context, +) -> None: + fn = FakeNet() + fn.enable() + + # This test shouldn't send any packets + def route_packet(packet: UDPPacket) -> NoReturn: # pragma: no cover + raise AssertionError() + + fn.route_packet = route_packet # type: ignore[assignment] # TODO add type annotations for FakeNet + + async with dtls_echo_server(server_ctx=server_ctx) as (_, address): + with endpoint() as client_endpoint: + # TODO: should this use the same client_ctx? + c1 = client_endpoint.connect(address, client_ctx_fn()) + client_endpoint.connect(address, client_ctx_fn()) + with pytest.raises(trio.BrokenResourceError): + await c1.do_handshake() + + +async def test_send_to_closed_local_port(server_ctx: SSL.Context) -> None: + # On Windows, sending a UDP packet to a closed local port can cause a weird + # ECONNRESET error later, inside the receive task. Make sure we're handling it + # properly. + async with dtls_echo_server(server_ctx=server_ctx) as (_, address): + with endpoint() as client_endpoint: + async with trio.open_nursery() as nursery: + for i in range(1, 10): + channel = client_endpoint.connect(("127.0.0.1", i), client_ctx_fn()) + nursery.start_soon(channel.do_handshake) + channel = client_endpoint.connect(address, client_ctx_fn()) + await channel.send(b"xxx") + assert await channel.receive() == b"xxx" + nursery.cancel_scope.cancel() diff --git a/src/trio/_tests/test_exports.py b/src/trio/_tests/test_exports.py index 1ecb7b13d5..59eac99041 100644 --- a/src/trio/_tests/test_exports.py +++ b/src/trio/_tests/test_exports.py @@ -1,638 +1,638 @@ -from __future__ import annotations # isort: split - -import __future__ # Regular import, not special! - -import enum -import functools -import importlib -import inspect -import json -import socket as stdlib_socket -import sys -import types -from pathlib import Path, PurePath -from types import ModuleType -from typing import TYPE_CHECKING, Any, Protocol - -import attrs -import pytest - -import trio -import trio.testing -from trio._tests.pytest_plugin import RUN_SLOW, maybe_ignore_import_error - -from .. import _core, _util -from .._core._tests.tutil import slow - -if TYPE_CHECKING: - from collections.abc import Iterable, Iterator - -mypy_cache_updated = False - - -try: # If installed, check both versions of this class. - from typing_extensions import Protocol as Protocol_ext -except ImportError: # pragma: no cover - Protocol_ext = Protocol - - -def _ensure_mypy_cache_updated() -> None: - # This pollutes the `empty` dir. Should this be changed? - try: - from mypy.api import run - except ImportError as error: - maybe_ignore_import_error(error) - - global mypy_cache_updated - if not mypy_cache_updated: - # mypy cache was *probably* already updated by the other tests, - # but `pytest -k ...` might run just this test on its own - result = run( - [ - "--config-file=", - "--cache-dir=./.mypy_cache", - "--no-error-summary", - # TODO: update our tests to use the exposed APIs - # instead of reading JSON... or use dmypy? - "--no-fixed-format-cache", - "--no-sqlite-cache", - "-c", - "import trio", - ], - ) - assert not result[1] # stderr - assert not result[0] # stdout - mypy_cache_updated = True - - -def test_core_is_properly_reexported() -> None: - # Each export from _core should be re-exported by exactly one of these - # three modules: - sources = [trio, trio.lowlevel, trio.testing] - for symbol in dir(_core): - if symbol.startswith("_"): - continue - found = 0 - for source in sources: - if symbol in dir(source) and getattr(source, symbol) is getattr( - _core, - symbol, - ): - found += 1 - print(symbol, found) - assert found == 1 - - -def class_is_final(cls: type) -> bool: - """Check if a class cannot be subclassed.""" - try: - # new_class() handles metaclasses properly, type(...) does not. - types.new_class("SubclassTester", (cls,)) - except TypeError: - return True - else: - return False - - -def iter_modules( - module: types.ModuleType, - only_public: bool, -) -> Iterator[types.ModuleType]: - yield module - for name, class_ in module.__dict__.items(): - if name.startswith("_") and only_public: - continue - if not isinstance(class_, ModuleType): - continue - if not class_.__name__.startswith(module.__name__): # pragma: no cover - continue - if class_ is module: # pragma: no cover - continue - yield from iter_modules(class_, only_public) - - -PUBLIC_MODULES = list(iter_modules(trio, only_public=True)) -ALL_MODULES = list(iter_modules(trio, only_public=False)) -PUBLIC_MODULE_NAMES = [m.__name__ for m in PUBLIC_MODULES] - - -# It doesn't make sense for downstream redistributors to run this test, since -# they might be using a newer version of Python with additional symbols which -# won't be reflected in trio.socket, and this shouldn't cause downstream test -# runs to start failing. -@pytest.mark.redistributors_should_skip -@pytest.mark.skipif( - sys.version_info[:4] == (3, 14, 0, "beta"), - # 12 pass, 16 fail - reason="several tools don't support 3.14", -) -# Static analysis tools often have trouble with alpha releases, where Python's -# internals are in flux, grammar may not have settled down, etc. -@pytest.mark.skipif( - sys.version_info.releaselevel == "alpha", - reason="skip static introspection tools on Python dev/alpha releases", -) -@pytest.mark.parametrize("modname", PUBLIC_MODULE_NAMES) -@pytest.mark.parametrize( - "tool", - ["pylint", "jedi", "mypy", "pyright_verifytypes"], -) -@pytest.mark.filterwarnings( - # https://github.com/pypa/setuptools/issues/3274 - "ignore:module 'sre_constants' is deprecated:DeprecationWarning", -) -def test_static_tool_sees_all_symbols(tool: str, modname: str, tmp_path: Path) -> None: - module = importlib.import_module(modname) - - def no_underscores(symbols: Iterable[str]) -> set[str]: - return {symbol for symbol in symbols if not symbol.startswith("_")} - - runtime_names = no_underscores(dir(module)) - - # ignore deprecated module `tests` being invisible - if modname == "trio": - runtime_names.discard("tests") - - # Ignore any __future__ feature objects, if imported under that name. - for name in __future__.all_feature_names: - if getattr(module, name, None) is getattr(__future__, name): - runtime_names.remove(name) - - if tool == "pylint": - try: - from pylint.lint import PyLinter - except ImportError as error: - maybe_ignore_import_error(error) - - linter = PyLinter() - assert module.__file__ is not None - ast = linter.get_ast(module.__file__, modname) - static_names = no_underscores(ast) # type: ignore[arg-type] - elif tool == "jedi": - if sys.implementation.name != "cpython": - pytest.skip("jedi does not support pypy") - - try: - import jedi - except ImportError as error: - maybe_ignore_import_error(error) - - # Simulate typing "import trio; trio." - script = jedi.Script(f"import {modname}; {modname}.") - completions = script.complete() - static_names = no_underscores(c.name for c in completions) - elif tool == "mypy": - if sys.implementation.name != "cpython": - # https://github.com/python/mypy/issues/20329 - pytest.skip("mypy does not support pypy") - - if not RUN_SLOW: # pragma: no cover - pytest.skip("use --run-slow to check against mypy") - - cache = Path.cwd() / ".mypy_cache" - - _ensure_mypy_cache_updated() - - trio_cache = next(cache.glob("*/trio")) - _, modname = (modname + ".").split(".", 1) - modname = modname[:-1] - mod_cache = trio_cache / modname if modname else trio_cache - if mod_cache.is_dir(): # pragma: no coverage - mod_cache = mod_cache / "__init__.data.json" - else: - mod_cache = trio_cache / (modname + ".data.json") - - assert mod_cache.exists() - assert mod_cache.is_file() - with mod_cache.open() as cache_file: - cache_json = json.loads(cache_file.read()) - static_names = no_underscores( - key - for key, value in cache_json["names"].items() - if not key.startswith(".") and value["kind"] == "Gdef" - ) - elif tool == "pyright_verifytypes": - if not RUN_SLOW: # pragma: no cover - pytest.skip("use --run-slow to check against pyright") - - try: - import pyright # noqa: F401 - except ImportError as error: - maybe_ignore_import_error(error) - import subprocess - - res = subprocess.run( - ["pyright", f"--verifytypes={modname}", "--outputjson"], - capture_output=True, - ) - current_result = json.loads(res.stdout) - - static_names = { - x["name"][len(modname) + 1 :] - for x in current_result["typeCompleteness"]["symbols"] - if x["name"].startswith(modname) - } - else: # pragma: no cover - raise AssertionError() - - # It's expected that the static set will contain more names than the - # runtime set: - # - static tools are sometimes sloppy and include deleted names - # - some symbols are platform-specific at runtime, but always show up in - # static analysis (e.g. in trio.socket or trio.lowlevel) - # So we check that the runtime names are a subset of the static names. - missing_names = runtime_names - static_names - - # ignore warnings about deprecated module tests - missing_names -= {"tests"} - missing_names -= {"ALG_SET_PUB_KEY"} # https://github.com/pypy/pypy/issues/5594 - - if missing_names: # pragma: no cover - print(f"{tool} can't see the following names in {modname}:") - print() - for name in sorted(missing_names): - print(f" {name}") - raise AssertionError() - - -@slow -# see comment on test_static_tool_sees_all_symbols -@pytest.mark.redistributors_should_skip -# Static analysis tools often have trouble with alpha releases, where Python's -# internals are in flux, grammar may not have settled down, etc. -@pytest.mark.skipif( - sys.version_info.releaselevel == "alpha", - reason="skip static introspection tools on Python dev/alpha releases", -) -@pytest.mark.parametrize("module_name", PUBLIC_MODULE_NAMES) -@pytest.mark.parametrize("tool", ["jedi", "mypy"]) -def test_static_tool_sees_class_members( - tool: str, - module_name: str, - tmp_path: Path, -) -> None: - module = PUBLIC_MODULES[PUBLIC_MODULE_NAMES.index(module_name)] - - # ignore hidden, but not dunder, symbols - def no_hidden(symbols: Iterable[str]) -> set[str]: - return { - symbol - for symbol in symbols - if (not symbol.startswith("_")) or symbol.startswith("__") - } - - if tool == "jedi" and sys.implementation.name != "cpython": - pytest.skip("jedi does not support pypy") - - if tool == "mypy" and sys.implementation.name != "cpython": - # https://github.com/python/mypy/issues/20329 - pytest.skip("mypy does not support pypy") - - if tool == "mypy": - cache = Path.cwd() / ".mypy_cache" - - _ensure_mypy_cache_updated() - - trio_cache = next(cache.glob("*/trio")) - modname = module_name - _, modname = (modname + ".").split(".", 1) - modname = modname[:-1] - mod_cache = trio_cache / modname if modname else trio_cache - if mod_cache.is_dir(): - mod_cache = mod_cache / "__init__.data.json" - else: - mod_cache = trio_cache / (modname + ".data.json") - - assert mod_cache.exists() - assert mod_cache.is_file() - with mod_cache.open() as cache_file: - cache_json = json.loads(cache_file.read()) - - # skip a bunch of file-system activity (probably can un-memoize?) - @functools.lru_cache - def lookup_symbol(symbol: str) -> dict[str, Any]: # type: ignore[misc, explicit-any] - topname, *modname, name = symbol.split(".") - version = next(cache.glob("3.*/")) - mod_cache = version / topname - if not mod_cache.is_dir(): - mod_cache = version / (topname + ".data.json") - - if modname: - for piece in modname[:-1]: - mod_cache /= piece - next_cache = mod_cache / modname[-1] - if next_cache.is_dir(): # pragma: no coverage - mod_cache = next_cache / "__init__.data.json" - else: - mod_cache = mod_cache / (modname[-1] + ".data.json") - elif mod_cache.is_dir(): - mod_cache /= "__init__.data.json" - with mod_cache.open() as f: - return json.loads(f.read())["names"][name] # type: ignore[no-any-return] - - errors: dict[str, object] = {} - for class_name, class_ in module.__dict__.items(): - if not isinstance(class_, type): - continue - if module_name == "trio.socket" and class_name in dir(stdlib_socket): - continue - - # Ignore classes that don't use attrs, they only define their members once - # __init__ is called (and reason they don't use attrs is because they're going - # to be reimplemented in pytest). - # Not 100% that's the case, and it works locally, so whatever /shrug - if module_name == "trio.testing" and class_name in ("_RaisesGroup", "_Matcher"): - continue - - # dir() and inspect.getmembers doesn't display properties from the metaclass - # also ignore some dunder methods that tend to differ but are of no consequence - ignore_names = set(dir(type(class_))) | { - "__annotations__", - "__attrs_attrs__", - "__attrs_own_setattr__", - "__callable_proto_members_only__", - "__class_getitem__", - "__final__", - "__getstate__", - "__match_args__", - "__order__", - "__orig_bases__", - "__parameters__", - "__protocol_attrs__", - "__setstate__", - "__slots__", - "__weakref__", - # ignore errors about dunders inherited from stdlib that tools might - # not see - "__copy__", - "__deepcopy__", - } - - if type(class_) is type: - # C extension classes don't have these dunders, but Python classes do - ignore_names.add("__firstlineno__") - ignore_names.add("__static_attributes__") - - # inspect.getmembers sees `name` and `value` in Enums, otherwise - # it behaves the same way as `dir` - # runtime_names = no_underscores(dir(class_)) - runtime_names = ( - no_hidden(x[0] for x in inspect.getmembers(class_)) - ignore_names - ) - - if tool == "jedi": - try: - import jedi - except ImportError as error: - maybe_ignore_import_error(error) - - script = jedi.Script( - f"from {module_name} import {class_name}; {class_name}.", - ) - completions = script.complete() - static_names = no_hidden(c.name for c in completions) - ignore_names - - elif tool == "mypy": - # load the cached type information - cached_type_info = cache_json["names"][class_name] - assert ( - "node" not in cached_type_info - ), "previously this was an 'if' but it seems it's no longer possible for this cache to contain 'node', if this assert raises for you please let us know!" - cached_type_info = lookup_symbol(cached_type_info["cross_ref"]) - - assert "node" in cached_type_info - node = cached_type_info["node"] - static_names = no_hidden( - k for k in node.get("names", ()) if not k.startswith(".") - ) - for symbol in node["mro"][1:]: - node = lookup_symbol(symbol)["node"] - static_names |= no_hidden( - k for k in node.get("names", ()) if not k.startswith(".") - ) - static_names -= ignore_names - - else: # pragma: no cover - raise AssertionError("unknown tool") - - missing = runtime_names - static_names - extra = static_names - runtime_names - - # using .remove() instead of .delete() to get an error in case they start not - # being missing - - if BaseException in class_.__mro__ and sys.version_info >= (3, 11): - extra.remove("__notes__") - - if tool == "mypy" and attrs.has(class_): - # e.g. __trio__core__run_CancelScope_AttrsAttributes__ - before = len(extra) - extra = {e for e in extra if not e.endswith("AttrsAttributes__")} - assert len(extra) == before - 1 - - if ( - tool == "jedi" - and issubclass(class_, trio.Path) - and sys.version_info >= (3, 15) - ): - # needs a typeshed update for https://github.com/python/typeshed/pull/15737 - extra.remove("is_reserved") - - if attrs.has(class_): - # dynamically created attribute by attrs? - missing.remove("__attrs_props__") - - # dir does not see `__signature__` on enums until 3.14 - if ( - enum.Enum in class_.__mro__ - and sys.version_info >= (3, 12) - and sys.version_info < (3, 14) - ): - extra.remove("__signature__") - - # TODO: this *should* be visible via `dir`!! - if tool == "mypy" and class_ == trio.Nursery: - extra.remove("cancel_scope") - - # These are (mostly? solely?) *runtime* attributes, often set in - # __init__, which doesn't show up with dir() or inspect.getmembers, - # but we get them in the way we query mypy & jedi - EXTRAS = { - trio.DTLSChannel: {"peer_address", "endpoint"}, - trio.DTLSEndpoint: {"socket", "incoming_packets_buffer"}, - trio.Process: {"args", "pid", "stderr", "stdin", "stdio", "stdout"}, - trio.SSLListener: {"transport_listener"}, - trio.SSLStream: {"transport_stream"}, - trio.SocketListener: {"socket"}, - trio.SocketStream: {"socket"}, - trio.testing.MemoryReceiveStream: {"close_hook", "receive_some_hook"}, - trio.testing.MemorySendStream: { - "close_hook", - "send_all_hook", - "wait_send_all_might_not_block_hook", - }, - } - if tool == "mypy" and class_ in EXTRAS: - before = len(extra) - extra -= EXTRAS[class_] - assert len(extra) == before - len(EXTRAS[class_]) - - # TODO: why is this? Is it a problem? - # see https://github.com/python-trio/trio/pull/2631#discussion_r1185615916 - if class_ == trio.StapledStream: - extra.remove("receive_stream") - extra.remove("send_stream") - - # I have not researched why these are missing, should maybe create an issue - # upstream with jedi - if tool == "jedi" and sys.version_info >= (3, 11): - if class_ in ( - trio.DTLSChannel, - trio.MemoryReceiveChannel, - trio.MemorySendChannel, - trio.SSLListener, - trio.SocketListener, - ): - missing.remove("__aenter__") - missing.remove("__aexit__") - if class_ in (trio.DTLSChannel, trio.MemoryReceiveChannel): - missing.remove("__aiter__") - missing.remove("__anext__") - - if class_ in (trio.Path, trio.WindowsPath, trio.PosixPath): - # These are from inherited subclasses. - missing -= PurePath.__dict__.keys() - # These are unix-only. - if tool == "mypy" and sys.platform == "win32": - missing -= {"owner", "is_mount", "group"} - if tool == "jedi" and sys.platform == "win32": - extra -= {"owner", "is_mount", "group"} - - # tuple subclasses are weird - if tool == "mypy" and issubclass(class_, tuple): - extra.remove("__reversed__") - missing.remove("__getnewargs__") - - if sys.version_info >= (3, 13) and attrs.has(class_): - missing.remove("__replace__") - - if sys.version_info >= (3, 14): - # these depend on whether a class has processed deferred annotations. - # (which might or might not happen and we don't know) - missing.discard("__annotate_func__") - missing.discard("__annotations_cache__") - - if tool == "jedi" and class_ == trio.open_memory_channel: - # something is seriously wrong with jedi's understanding of open_memory_channel... - for attrib in ( - "__add__", - "__contains__", - "__getitem__", - "__getnewargs__", - "__iter__", - "__len__", - "__mul__", - "__rmul__", - "count", - "index", - ): - missing.remove(attrib) - - if missing or extra: # pragma: no cover - errors[f"{module_name}.{class_name}"] = { - "missing": missing, - "extra": extra, - } - - # `assert not errors` will not print the full content of errors, even with - # `--verbose`, so we manually print it - if errors: # pragma: no cover - from pprint import pprint - - print(f"\n{tool} can't see the following symbols in {module_name}:") - pprint(errors) - assert not errors - - -def test_nopublic_is_final() -> None: - """Check all NoPublicConstructor classes are also @final.""" - assert class_is_final(_util.NoPublicConstructor) # This is itself final. - - for module in ALL_MODULES: - for class_ in module.__dict__.values(): - if isinstance(class_, _util.NoPublicConstructor): - assert class_is_final(class_) - - -def test_classes_are_final() -> None: - # Sanity checks. - assert not class_is_final(object) - assert class_is_final(bool) - - for module in PUBLIC_MODULES: - for name, class_ in module.__dict__.items(): - if not isinstance(class_, type): - continue - # Deprecated classes are exported with a leading underscore - if name.startswith("_"): # pragma: no cover - continue - - # Abstract classes can be subclassed, because that's the whole - # point of ABCs - if inspect.isabstract(class_): - continue - # Same with protocols, but only direct children. - if Protocol in class_.__bases__ or Protocol_ext in class_.__bases__: - continue - # Exceptions are allowed to be subclassed, because exception - # subclassing isn't used to inherit behavior. - if issubclass(class_, BaseException): - continue - # These are classes that are conceptually abstract, but - # inspect.isabstract returns False for boring reasons. - if class_ is trio.abc.Instrument or class_ is trio.socket.SocketType: - continue - # ... insert other special cases here ... - - # The `Path` class needs to support inheritance to allow `WindowsPath` and `PosixPath`. - if class_ is trio.Path: - continue - # don't care about the *Statistics classes - if name.endswith("Statistics"): - continue - - assert class_is_final(class_) - - -# Plugin might not be running, especially if running from an installed version. -@pytest.mark.skipif( - not hasattr(attrs.field, "trio_modded"), - reason="Pytest plugin not installed.", -) -def test_pyright_recognizes_init_attributes() -> None: - """Check whether we provide `alias` for all underscore prefixed attributes. - - Attrs always sets the `alias` attribute on fields, so a pytest plugin is used - to monkeypatch `field()` to record whether an alias was defined in the metadata. - See `_trio_check_attrs_aliases`. - """ - for module in PUBLIC_MODULES: - for class_ in module.__dict__.values(): - if not attrs.has(class_): - continue - if isinstance(class_, _util.NoPublicConstructor): - continue - - attributes = [ - attr - for attr in attrs.fields(class_) - if attr.init - if attr.alias - not in ( - attr.name, - # trio_original_args may not be present in autoattribs - attr.metadata.get("trio_original_args", {}).get("alias"), - ) - ] - - assert attributes == [], class_ +from __future__ import annotations # isort: split + +import __future__ # Regular import, not special! + +import enum +import functools +import importlib +import inspect +import json +import socket as stdlib_socket +import sys +import types +from pathlib import Path, PurePath +from types import ModuleType +from typing import TYPE_CHECKING, Any, Protocol + +import attrs +import pytest + +import trio +import trio.testing +from trio._tests.pytest_plugin import RUN_SLOW, maybe_ignore_import_error + +from .. import _core, _util +from .._core._tests.tutil import slow + +if TYPE_CHECKING: + from collections.abc import Iterable, Iterator + +mypy_cache_updated = False + + +try: # If installed, check both versions of this class. + from typing_extensions import Protocol as Protocol_ext +except ImportError: # pragma: no cover + Protocol_ext = Protocol + + +def _ensure_mypy_cache_updated() -> None: + # This pollutes the `empty` dir. Should this be changed? + try: + from mypy.api import run + except ImportError as error: + maybe_ignore_import_error(error) + + global mypy_cache_updated + if not mypy_cache_updated: + # mypy cache was *probably* already updated by the other tests, + # but `pytest -k ...` might run just this test on its own + result = run( + [ + "--config-file=", + "--cache-dir=./.mypy_cache", + "--no-error-summary", + # TODO: update our tests to use the exposed APIs + # instead of reading JSON... or use dmypy? + "--no-fixed-format-cache", + "--no-sqlite-cache", + "-c", + "import trio", + ], + ) + assert not result[1] # stderr + assert not result[0] # stdout + mypy_cache_updated = True + + +def test_core_is_properly_reexported() -> None: + # Each export from _core should be re-exported by exactly one of these + # three modules: + sources = [trio, trio.lowlevel, trio.testing] + for symbol in dir(_core): + if symbol.startswith("_"): + continue + found = 0 + for source in sources: + if symbol in dir(source) and getattr(source, symbol) is getattr( + _core, + symbol, + ): + found += 1 + print(symbol, found) + assert found == 1 + + +def class_is_final(cls: type) -> bool: + """Check if a class cannot be subclassed.""" + try: + # new_class() handles metaclasses properly, type(...) does not. + types.new_class("SubclassTester", (cls,)) + except TypeError: + return True + else: + return False + + +def iter_modules( + module: types.ModuleType, + only_public: bool, +) -> Iterator[types.ModuleType]: + yield module + for name, class_ in module.__dict__.items(): + if name.startswith("_") and only_public: + continue + if not isinstance(class_, ModuleType): + continue + if not class_.__name__.startswith(module.__name__): # pragma: no cover + continue + if class_ is module: # pragma: no cover + continue + yield from iter_modules(class_, only_public) + + +PUBLIC_MODULES = list(iter_modules(trio, only_public=True)) +ALL_MODULES = list(iter_modules(trio, only_public=False)) +PUBLIC_MODULE_NAMES = [m.__name__ for m in PUBLIC_MODULES] + + +# It doesn't make sense for downstream redistributors to run this test, since +# they might be using a newer version of Python with additional symbols which +# won't be reflected in trio.socket, and this shouldn't cause downstream test +# runs to start failing. +@pytest.mark.redistributors_should_skip +@pytest.mark.skipif( + sys.version_info[:4] == (3, 14, 0, "beta"), + # 12 pass, 16 fail + reason="several tools don't support 3.14", +) +# Static analysis tools often have trouble with alpha releases, where Python's +# internals are in flux, grammar may not have settled down, etc. +@pytest.mark.skipif( + sys.version_info.releaselevel == "alpha", + reason="skip static introspection tools on Python dev/alpha releases", +) +@pytest.mark.parametrize("modname", PUBLIC_MODULE_NAMES) +@pytest.mark.parametrize( + "tool", + ["pylint", "jedi", "mypy", "pyright_verifytypes"], +) +@pytest.mark.filterwarnings( + # https://github.com/pypa/setuptools/issues/3274 + "ignore:module 'sre_constants' is deprecated:DeprecationWarning", +) +def test_static_tool_sees_all_symbols(tool: str, modname: str, tmp_path: Path) -> None: + module = importlib.import_module(modname) + + def no_underscores(symbols: Iterable[str]) -> set[str]: + return {symbol for symbol in symbols if not symbol.startswith("_")} + + runtime_names = no_underscores(dir(module)) + + # ignore deprecated module `tests` being invisible + if modname == "trio": + runtime_names.discard("tests") + + # Ignore any __future__ feature objects, if imported under that name. + for name in __future__.all_feature_names: + if getattr(module, name, None) is getattr(__future__, name): + runtime_names.remove(name) + + if tool == "pylint": + try: + from pylint.lint import PyLinter + except ImportError as error: + maybe_ignore_import_error(error) + + linter = PyLinter() + assert module.__file__ is not None + ast = linter.get_ast(module.__file__, modname) + static_names = no_underscores(ast) # type: ignore[arg-type] + elif tool == "jedi": + if sys.implementation.name != "cpython": + pytest.skip("jedi does not support pypy") + + try: + import jedi + except ImportError as error: + maybe_ignore_import_error(error) + + # Simulate typing "import trio; trio." + script = jedi.Script(f"import {modname}; {modname}.") + completions = script.complete() + static_names = no_underscores(c.name for c in completions) + elif tool == "mypy": + if sys.implementation.name != "cpython": + # https://github.com/python/mypy/issues/20329 + pytest.skip("mypy does not support pypy") + + if not RUN_SLOW: # pragma: no cover + pytest.skip("use --run-slow to check against mypy") + + cache = Path.cwd() / ".mypy_cache" + + _ensure_mypy_cache_updated() + + trio_cache = next(cache.glob("*/trio")) + _, modname = (modname + ".").split(".", 1) + modname = modname[:-1] + mod_cache = trio_cache / modname if modname else trio_cache + if mod_cache.is_dir(): # pragma: no coverage + mod_cache = mod_cache / "__init__.data.json" + else: + mod_cache = trio_cache / (modname + ".data.json") + + assert mod_cache.exists() + assert mod_cache.is_file() + with mod_cache.open() as cache_file: + cache_json = json.loads(cache_file.read()) + static_names = no_underscores( + key + for key, value in cache_json["names"].items() + if not key.startswith(".") and value["kind"] == "Gdef" + ) + elif tool == "pyright_verifytypes": + if not RUN_SLOW: # pragma: no cover + pytest.skip("use --run-slow to check against pyright") + + try: + import pyright # noqa: F401 + except ImportError as error: + maybe_ignore_import_error(error) + import subprocess + + res = subprocess.run( + ["pyright", f"--verifytypes={modname}", "--outputjson"], + capture_output=True, + ) + current_result = json.loads(res.stdout) + + static_names = { + x["name"][len(modname) + 1 :] + for x in current_result["typeCompleteness"]["symbols"] + if x["name"].startswith(modname) + } + else: # pragma: no cover + raise AssertionError() + + # It's expected that the static set will contain more names than the + # runtime set: + # - static tools are sometimes sloppy and include deleted names + # - some symbols are platform-specific at runtime, but always show up in + # static analysis (e.g. in trio.socket or trio.lowlevel) + # So we check that the runtime names are a subset of the static names. + missing_names = runtime_names - static_names + + # ignore warnings about deprecated module tests + missing_names -= {"tests"} + missing_names -= {"ALG_SET_PUB_KEY"} # https://github.com/pypy/pypy/issues/5594 + + if missing_names: # pragma: no cover + print(f"{tool} can't see the following names in {modname}:") + print() + for name in sorted(missing_names): + print(f" {name}") + raise AssertionError() + + +@slow +# see comment on test_static_tool_sees_all_symbols +@pytest.mark.redistributors_should_skip +# Static analysis tools often have trouble with alpha releases, where Python's +# internals are in flux, grammar may not have settled down, etc. +@pytest.mark.skipif( + sys.version_info.releaselevel == "alpha", + reason="skip static introspection tools on Python dev/alpha releases", +) +@pytest.mark.parametrize("module_name", PUBLIC_MODULE_NAMES) +@pytest.mark.parametrize("tool", ["jedi", "mypy"]) +def test_static_tool_sees_class_members( + tool: str, + module_name: str, + tmp_path: Path, +) -> None: + module = PUBLIC_MODULES[PUBLIC_MODULE_NAMES.index(module_name)] + + # ignore hidden, but not dunder, symbols + def no_hidden(symbols: Iterable[str]) -> set[str]: + return { + symbol + for symbol in symbols + if (not symbol.startswith("_")) or symbol.startswith("__") + } + + if tool == "jedi" and sys.implementation.name != "cpython": + pytest.skip("jedi does not support pypy") + + if tool == "mypy" and sys.implementation.name != "cpython": + # https://github.com/python/mypy/issues/20329 + pytest.skip("mypy does not support pypy") + + if tool == "mypy": + cache = Path.cwd() / ".mypy_cache" + + _ensure_mypy_cache_updated() + + trio_cache = next(cache.glob("*/trio")) + modname = module_name + _, modname = (modname + ".").split(".", 1) + modname = modname[:-1] + mod_cache = trio_cache / modname if modname else trio_cache + if mod_cache.is_dir(): + mod_cache = mod_cache / "__init__.data.json" + else: + mod_cache = trio_cache / (modname + ".data.json") + + assert mod_cache.exists() + assert mod_cache.is_file() + with mod_cache.open() as cache_file: + cache_json = json.loads(cache_file.read()) + + # skip a bunch of file-system activity (probably can un-memoize?) + @functools.lru_cache + def lookup_symbol(symbol: str) -> dict[str, Any]: # type: ignore[misc, explicit-any] + topname, *modname, name = symbol.split(".") + version = next(cache.glob("3.*/")) + mod_cache = version / topname + if not mod_cache.is_dir(): + mod_cache = version / (topname + ".data.json") + + if modname: + for piece in modname[:-1]: + mod_cache /= piece + next_cache = mod_cache / modname[-1] + if next_cache.is_dir(): # pragma: no coverage + mod_cache = next_cache / "__init__.data.json" + else: + mod_cache = mod_cache / (modname[-1] + ".data.json") + elif mod_cache.is_dir(): + mod_cache /= "__init__.data.json" + with mod_cache.open() as f: + return json.loads(f.read())["names"][name] # type: ignore[no-any-return] + + errors: dict[str, object] = {} + for class_name, class_ in module.__dict__.items(): + if not isinstance(class_, type): + continue + if module_name == "trio.socket" and class_name in dir(stdlib_socket): + continue + + # Ignore classes that don't use attrs, they only define their members once + # __init__ is called (and reason they don't use attrs is because they're going + # to be reimplemented in pytest). + # Not 100% that's the case, and it works locally, so whatever /shrug + if module_name == "trio.testing" and class_name in ("_RaisesGroup", "_Matcher"): + continue + + # dir() and inspect.getmembers doesn't display properties from the metaclass + # also ignore some dunder methods that tend to differ but are of no consequence + ignore_names = set(dir(type(class_))) | { + "__annotations__", + "__attrs_attrs__", + "__attrs_own_setattr__", + "__callable_proto_members_only__", + "__class_getitem__", + "__final__", + "__getstate__", + "__match_args__", + "__order__", + "__orig_bases__", + "__parameters__", + "__protocol_attrs__", + "__setstate__", + "__slots__", + "__weakref__", + # ignore errors about dunders inherited from stdlib that tools might + # not see + "__copy__", + "__deepcopy__", + } + + if type(class_) is type: + # C extension classes don't have these dunders, but Python classes do + ignore_names.add("__firstlineno__") + ignore_names.add("__static_attributes__") + + # inspect.getmembers sees `name` and `value` in Enums, otherwise + # it behaves the same way as `dir` + # runtime_names = no_underscores(dir(class_)) + runtime_names = ( + no_hidden(x[0] for x in inspect.getmembers(class_)) - ignore_names + ) + + if tool == "jedi": + try: + import jedi + except ImportError as error: + maybe_ignore_import_error(error) + + script = jedi.Script( + f"from {module_name} import {class_name}; {class_name}.", + ) + completions = script.complete() + static_names = no_hidden(c.name for c in completions) - ignore_names + + elif tool == "mypy": + # load the cached type information + cached_type_info = cache_json["names"][class_name] + assert ( + "node" not in cached_type_info + ), "previously this was an 'if' but it seems it's no longer possible for this cache to contain 'node', if this assert raises for you please let us know!" + cached_type_info = lookup_symbol(cached_type_info["cross_ref"]) + + assert "node" in cached_type_info + node = cached_type_info["node"] + static_names = no_hidden( + k for k in node.get("names", ()) if not k.startswith(".") + ) + for symbol in node["mro"][1:]: + node = lookup_symbol(symbol)["node"] + static_names |= no_hidden( + k for k in node.get("names", ()) if not k.startswith(".") + ) + static_names -= ignore_names + + else: # pragma: no cover + raise AssertionError("unknown tool") + + missing = runtime_names - static_names + extra = static_names - runtime_names + + # using .remove() instead of .delete() to get an error in case they start not + # being missing + + if BaseException in class_.__mro__ and sys.version_info >= (3, 11): + extra.remove("__notes__") + + if tool == "mypy" and attrs.has(class_): + # e.g. __trio__core__run_CancelScope_AttrsAttributes__ + before = len(extra) + extra = {e for e in extra if not e.endswith("AttrsAttributes__")} + assert len(extra) == before - 1 + + if ( + tool == "jedi" + and issubclass(class_, trio.Path) + and sys.version_info >= (3, 15) + ): + # needs a typeshed update for https://github.com/python/typeshed/pull/15737 + extra.remove("is_reserved") + + if attrs.has(class_): + # dynamically created attribute by attrs? + missing.remove("__attrs_props__") + + # dir does not see `__signature__` on enums until 3.14 + if ( + enum.Enum in class_.__mro__ + and sys.version_info >= (3, 12) + and sys.version_info < (3, 14) + ): + extra.remove("__signature__") + + # TODO: this *should* be visible via `dir`!! + if tool == "mypy" and class_ == trio.Nursery: + extra.remove("cancel_scope") + + # These are (mostly? solely?) *runtime* attributes, often set in + # __init__, which doesn't show up with dir() or inspect.getmembers, + # but we get them in the way we query mypy & jedi + EXTRAS = { + trio.DTLSChannel: {"peer_address", "endpoint"}, + trio.DTLSEndpoint: {"socket", "incoming_packets_buffer"}, + trio.Process: {"args", "pid", "stderr", "stdin", "stdio", "stdout"}, + trio.SSLListener: {"transport_listener"}, + trio.SSLStream: {"transport_stream"}, + trio.SocketListener: {"socket"}, + trio.SocketStream: {"socket"}, + trio.testing.MemoryReceiveStream: {"close_hook", "receive_some_hook"}, + trio.testing.MemorySendStream: { + "close_hook", + "send_all_hook", + "wait_send_all_might_not_block_hook", + }, + } + if tool == "mypy" and class_ in EXTRAS: + before = len(extra) + extra -= EXTRAS[class_] + assert len(extra) == before - len(EXTRAS[class_]) + + # TODO: why is this? Is it a problem? + # see https://github.com/python-trio/trio/pull/2631#discussion_r1185615916 + if class_ == trio.StapledStream: + extra.remove("receive_stream") + extra.remove("send_stream") + + # I have not researched why these are missing, should maybe create an issue + # upstream with jedi + if tool == "jedi" and sys.version_info >= (3, 11): + if class_ in ( + trio.DTLSChannel, + trio.MemoryReceiveChannel, + trio.MemorySendChannel, + trio.SSLListener, + trio.SocketListener, + ): + missing.remove("__aenter__") + missing.remove("__aexit__") + if class_ in (trio.DTLSChannel, trio.MemoryReceiveChannel): + missing.remove("__aiter__") + missing.remove("__anext__") + + if class_ in (trio.Path, trio.WindowsPath, trio.PosixPath): + # These are from inherited subclasses. + missing -= PurePath.__dict__.keys() + # These are unix-only. + if tool == "mypy" and sys.platform == "win32": + missing -= {"owner", "is_mount", "group"} + if tool == "jedi" and sys.platform == "win32": + extra -= {"owner", "is_mount", "group"} + + # tuple subclasses are weird + if tool == "mypy" and issubclass(class_, tuple): + extra.remove("__reversed__") + missing.remove("__getnewargs__") + + if sys.version_info >= (3, 13) and attrs.has(class_): + missing.remove("__replace__") + + if sys.version_info >= (3, 14): + # these depend on whether a class has processed deferred annotations. + # (which might or might not happen and we don't know) + missing.discard("__annotate_func__") + missing.discard("__annotations_cache__") + + if tool == "jedi" and class_ == trio.open_memory_channel: + # something is seriously wrong with jedi's understanding of open_memory_channel... + for attrib in ( + "__add__", + "__contains__", + "__getitem__", + "__getnewargs__", + "__iter__", + "__len__", + "__mul__", + "__rmul__", + "count", + "index", + ): + missing.remove(attrib) + + if missing or extra: # pragma: no cover + errors[f"{module_name}.{class_name}"] = { + "missing": missing, + "extra": extra, + } + + # `assert not errors` will not print the full content of errors, even with + # `--verbose`, so we manually print it + if errors: # pragma: no cover + from pprint import pprint + + print(f"\n{tool} can't see the following symbols in {module_name}:") + pprint(errors) + assert not errors + + +def test_nopublic_is_final() -> None: + """Check all NoPublicConstructor classes are also @final.""" + assert class_is_final(_util.NoPublicConstructor) # This is itself final. + + for module in ALL_MODULES: + for class_ in module.__dict__.values(): + if isinstance(class_, _util.NoPublicConstructor): + assert class_is_final(class_) + + +def test_classes_are_final() -> None: + # Sanity checks. + assert not class_is_final(object) + assert class_is_final(bool) + + for module in PUBLIC_MODULES: + for name, class_ in module.__dict__.items(): + if not isinstance(class_, type): + continue + # Deprecated classes are exported with a leading underscore + if name.startswith("_"): # pragma: no cover + continue + + # Abstract classes can be subclassed, because that's the whole + # point of ABCs + if inspect.isabstract(class_): + continue + # Same with protocols, but only direct children. + if Protocol in class_.__bases__ or Protocol_ext in class_.__bases__: + continue + # Exceptions are allowed to be subclassed, because exception + # subclassing isn't used to inherit behavior. + if issubclass(class_, BaseException): + continue + # These are classes that are conceptually abstract, but + # inspect.isabstract returns False for boring reasons. + if class_ is trio.abc.Instrument or class_ is trio.socket.SocketType: + continue + # ... insert other special cases here ... + + # The `Path` class needs to support inheritance to allow `WindowsPath` and `PosixPath`. + if class_ is trio.Path: + continue + # don't care about the *Statistics classes + if name.endswith("Statistics"): + continue + + assert class_is_final(class_) + + +# Plugin might not be running, especially if running from an installed version. +@pytest.mark.skipif( + not hasattr(attrs.field, "trio_modded"), + reason="Pytest plugin not installed.", +) +def test_pyright_recognizes_init_attributes() -> None: + """Check whether we provide `alias` for all underscore prefixed attributes. + + Attrs always sets the `alias` attribute on fields, so a pytest plugin is used + to monkeypatch `field()` to record whether an alias was defined in the metadata. + See `_trio_check_attrs_aliases`. + """ + for module in PUBLIC_MODULES: + for class_ in module.__dict__.values(): + if not attrs.has(class_): + continue + if isinstance(class_, _util.NoPublicConstructor): + continue + + attributes = [ + attr + for attr in attrs.fields(class_) + if attr.init + if attr.alias + not in ( + attr.name, + # trio_original_args may not be present in autoattribs + attr.metadata.get("trio_original_args", {}).get("alias"), + ) + ] + + assert attributes == [], class_ diff --git a/src/trio/_tests/test_ssl.py b/src/trio/_tests/test_ssl.py index f0dd884c64..0e30eccfd0 100644 --- a/src/trio/_tests/test_ssl.py +++ b/src/trio/_tests/test_ssl.py @@ -1,1372 +1,1372 @@ -from __future__ import annotations - -import os -import socket as stdlib_socket -import ssl -import sys -import threading -from contextlib import asynccontextmanager, contextmanager, suppress -from functools import partial -from ssl import SSLContext -from typing import TYPE_CHECKING, Any, NoReturn, TypeAlias - -import pytest - -from trio import StapledStream -from trio._tests.pytest_plugin import maybe_ignore_import_error -from trio.abc import ReceiveStream, SendStream -from trio.testing import MemoryReceiveStream, MemorySendStream - -try: - import trustme - from OpenSSL import SSL -except ImportError as error: - maybe_ignore_import_error(error) - -import trio - -from .. import _core, socket as tsocket -from .._abc import Stream -from .._core import BrokenResourceError, ClosedResourceError -from .._core._tests.tutil import slow -from .._highlevel_generic import aclose_forcefully -from .._highlevel_open_tcp_stream import open_tcp_stream -from .._highlevel_socket import SocketListener, SocketStream -from .._ssl import NeedHandshakeError, SSLListener, SSLStream, _is_eof -from .._util import ConflictDetector -from ..testing import ( - Sequencer, - assert_checkpoints, - check_two_way_stream, - lockstep_stream_pair, - memory_stream_pair, -) - -if TYPE_CHECKING: - from collections.abc import AsyncIterator, Awaitable, Callable, Iterator - - from trio._core import MockClock - from trio._ssl import T_Stream - - from .._core._run import CancelScope - -# We have two different kinds of echo server fixtures we use for testing. The -# first is a real server written using the stdlib ssl module and blocking -# sockets. It runs in a thread and we talk to it over a real socketpair(), to -# validate interoperability in a semi-realistic setting. -# -# The second is a very weird virtual echo server that lives inside a custom -# Stream class. It lives entirely inside the Python object space; there are no -# operating system calls in it at all. No threads, no I/O, nothing. It's -# 'send_all' call takes encrypted data from a client and feeds it directly into -# the server-side TLS state engine to decrypt, then takes that data, feeds it -# back through to get the encrypted response, and returns it from 'receive_some'. This -# gives us full control and reproducibility. This server is written using -# PyOpenSSL, so that we can trigger renegotiations on demand. It also allows -# us to insert random (virtual) delays, to really exercise all the weird paths -# in SSLStream's state engine. -# -# Both present a certificate for "trio-test-1.example.org". - -TRIO_TEST_CA = trustme.CA() -TRIO_TEST_1_CERT = TRIO_TEST_CA.issue_server_cert("trio-test-1.example.org") - -SERVER_CTX = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) -if hasattr(ssl, "OP_IGNORE_UNEXPECTED_EOF"): - SERVER_CTX.options &= ~ssl.OP_IGNORE_UNEXPECTED_EOF - -TRIO_TEST_1_CERT.configure_cert(SERVER_CTX) - - -# TLS 1.3 has a lot of changes from previous versions. So we want to run tests -# with both TLS 1.3, and TLS 1.2. -# "tls13" means that we're willing to negotiate TLS 1.3. Usually that's -# what will happen, but the renegotiation tests explicitly force a -# downgrade on the server side. "tls12" means we refuse to negotiate TLS -# 1.3, so we'll almost certainly use TLS 1.2. -@pytest.fixture(scope="module", params=["tls13", "tls12"]) -def client_ctx(request: pytest.FixtureRequest) -> ssl.SSLContext: - ctx = ssl.create_default_context() - - if hasattr(ssl, "OP_IGNORE_UNEXPECTED_EOF"): - ctx.options &= ~ssl.OP_IGNORE_UNEXPECTED_EOF - - TRIO_TEST_CA.configure_trust(ctx) - if request.param in ["default", "tls13"]: - return ctx - elif request.param == "tls12": - ctx.maximum_version = ssl.TLSVersion.TLSv1_2 - return ctx - else: # pragma: no cover - raise AssertionError() - - -# The blocking socket server. -def ssl_echo_serve_sync( - sock: stdlib_socket.socket, - *, - expect_fail: bool = False, -) -> None: - try: - wrapped = SERVER_CTX.wrap_socket( - sock, - server_side=True, - suppress_ragged_eofs=False, - ) - with wrapped: - wrapped.do_handshake() - while True: - data = wrapped.recv(4096) - if not data: - # other side has initiated a graceful shutdown; we try to - # respond in kind but it's legal for them to have already - # gone away. - with suppress(BrokenPipeError, ssl.SSLZeroReturnError): - wrapped.unwrap() - return - wrapped.sendall(data) - # This is an obscure workaround for an openssl bug. In server mode, in - # some versions, openssl sends some extra data at the end of do_handshake - # that it shouldn't send. Normally this is harmless, but, if the other - # side shuts down the connection before it reads that data, it might cause - # the OS to report a ECONNREST or even ECONNABORTED (which is just wrong, - # since ECONNABORTED is supposed to mean that connect() failed, but what - # can you do). In this case the other side did nothing wrong, but there's - # no way to recover, so we let it pass, and just cross our fingers its not - # hiding any (other) real bugs. For more details see: - # - # https://github.com/python-trio/trio/issues/1293 - # - # Also, this happens frequently but non-deterministically, so we have to - # 'no cover' it to avoid coverage flapping. - except (ConnectionResetError, ConnectionAbortedError): # pragma: no cover - return - except Exception as exc: - if expect_fail: - print("ssl_echo_serve_sync got error as expected:", exc) - else: # pragma: no cover - print("ssl_echo_serve_sync got unexpected error:", exc) - raise - else: - if expect_fail: # pragma: no cover - raise RuntimeError("failed to fail?") - finally: - sock.close() - - -# Fixture that gives a raw socket connected to a trio-test-1 echo server -# (running in a thread). Useful for testing making connections with different -# SSLContexts. -@asynccontextmanager -async def ssl_echo_server_raw(expect_fail: bool = False) -> AsyncIterator[SocketStream]: - a, b = stdlib_socket.socketpair() - async with trio.open_nursery() as nursery: - # Exiting the 'with a, b' context manager closes the sockets, which - # causes the thread to exit (possibly with an error), which allows the - # nursery context manager to exit too. - with a, b: - nursery.start_soon( - trio.to_thread.run_sync, - partial(ssl_echo_serve_sync, b, expect_fail=expect_fail), - ) - - yield SocketStream(tsocket.from_stdlib_socket(a)) - - -# Fixture that gives a properly set up SSLStream connected to a trio-test-1 -# echo server (running in a thread) -@asynccontextmanager -async def ssl_echo_server( - client_ctx: SSLContext, - expect_fail: bool = False, -) -> AsyncIterator[SSLStream[Stream]]: - async with ssl_echo_server_raw(expect_fail=expect_fail) as sock: - yield SSLStream(sock, client_ctx, server_hostname="trio-test-1.example.org") - - -# The weird in-memory server ... thing. -# Doesn't inherit from Stream because I left out the methods that we don't -# actually need. -# jakkdl: it seems to implement all the abstract methods (now), so I made it inherit -# from Stream for the sake of typechecking. -class PyOpenSSLEchoStream(Stream): - def __init__( - self, - sleeper: Callable[[str], Awaitable[None]] | None = None, - ) -> None: - ctx = SSL.Context(SSL.SSLv23_METHOD) - # TLS 1.3 removes renegotiation support. Which is great for them, but - # we still have to support versions before that, and that means we - # need to test renegotiation support, which means we need to force this - # to use a lower version where this test server can trigger - # renegotiations. - from cryptography.hazmat.bindings.openssl.binding import Binding - - b = Binding() - ctx.set_options(b.lib.SSL_OP_NO_TLSv1_3) - - # Unfortunately there's currently no way to say "use 1.3 or worse", we - # can only disable specific versions. And if the two sides start - # negotiating 1.4 at some point in the future, it *might* mean that - # our tests silently stop working properly. So the next line is a - # tripwire to remind us we need to revisit this stuff in 5 years or - # whatever when the next TLS version is released: - assert not hasattr(SSL, "OP_NO_TLSv1_4") - TRIO_TEST_1_CERT.configure_cert(ctx) - self._conn = SSL.Connection(ctx, None) - self._conn.set_accept_state() - self._lot = _core.ParkingLot() - self._pending_cleartext = bytearray() - - self._send_all_conflict_detector = ConflictDetector( - "simultaneous calls to PyOpenSSLEchoStream.send_all", - ) - self._receive_some_conflict_detector = ConflictDetector( - "simultaneous calls to PyOpenSSLEchoStream.receive_some", - ) - - self.sleeper: Callable[[str], Awaitable[None]] - if sleeper is None: - - async def no_op_sleeper(_: object) -> None: - return - - self.sleeper = no_op_sleeper - else: - self.sleeper = sleeper - - async def aclose(self) -> None: - self._conn.bio_shutdown() - - def renegotiate_pending(self) -> bool: - return self._conn.renegotiate_pending() - - def renegotiate(self) -> None: - # Returns false if a renegotiation is already in progress, meaning - # nothing happens. - assert self._conn.renegotiate() - - async def wait_send_all_might_not_block(self) -> None: - with self._send_all_conflict_detector: - await _core.checkpoint() - await _core.checkpoint() - await self.sleeper("wait_send_all_might_not_block") - - async def send_all(self, data: bytes | bytearray | memoryview[int]) -> None: - print(" --> transport_stream.send_all") - data_ = bytes(data) - with self._send_all_conflict_detector: - await _core.checkpoint() - await _core.checkpoint() - await self.sleeper("send_all") - self._conn.bio_write(data_) - while True: - await self.sleeper("send_all") - try: - data_ = self._conn.recv(1) - except SSL.ZeroReturnError: - self._conn.shutdown() - print("renegotiations:", self._conn.total_renegotiations()) - break - except SSL.WantReadError: - break - else: - self._pending_cleartext += data_ - self._lot.unpark_all() - await self.sleeper("send_all") - print(" <-- transport_stream.send_all finished") - - async def receive_some(self, nbytes: int | None = None) -> bytes: - print(" --> transport_stream.receive_some") - if nbytes is None: - nbytes = 65536 # arbitrary - with self._receive_some_conflict_detector: - try: - await _core.checkpoint() - await _core.checkpoint() - while True: - await self.sleeper("receive_some") - try: - return self._conn.bio_read(nbytes) - except SSL.WantReadError: - # No data in our ciphertext buffer; try to generate - # some. - if self._pending_cleartext: - # We have some cleartext; maybe we can encrypt it - # and then return it. - print(" trying", self._pending_cleartext) - try: - # PyOpenSSL bug: doesn't accept bytearray - # https://github.com/pyca/pyopenssl/issues/621 - next_byte = self._pending_cleartext[0:1] - self._conn.send(bytes(next_byte)) - # Apparently this next bit never gets hit in the - # test suite, but it's not an interesting omission - # so let's pragma it. - except SSL.WantReadError: # pragma: no cover - # We didn't manage to send the cleartext (and - # in particular we better leave it there to - # try again, due to openssl's retry - # semantics), but it's possible we pushed a - # renegotiation forward and *now* we have data - # to send. - try: - return self._conn.bio_read(nbytes) - except SSL.WantReadError: - # Nope. We're just going to have to wait - # for someone to call send_all() to give - # use more data. - print("parking (a)") - await self._lot.park() - else: - # We successfully sent that byte, so we don't - # have to again. - del self._pending_cleartext[0:1] - else: - # no pending cleartext; nothing to do but wait for - # someone to call send_all - print("parking (b)") - await self._lot.park() - finally: - await self.sleeper("receive_some") - print(" <-- transport_stream.receive_some finished") - - -async def test_PyOpenSSLEchoStream_gives_resource_busy_errors() -> None: - # Make sure that PyOpenSSLEchoStream complains if two tasks call send_all - # at the same time, or ditto for receive_some. The tricky cases where SSLStream - # might accidentally do this are during renegotiation, which we test using - # PyOpenSSLEchoStream, so this makes sure that if we do have a bug then - # PyOpenSSLEchoStream will notice and complain. - - async def do_test( - func1: str, - args1: tuple[object, ...], - func2: str, - args2: tuple[object, ...], - ) -> None: - s = PyOpenSSLEchoStream() - with pytest.RaisesGroup( - pytest.RaisesExc(_core.BusyResourceError, match="simultaneous") - ): - async with _core.open_nursery() as nursery: - nursery.start_soon(getattr(s, func1), *args1) - nursery.start_soon(getattr(s, func2), *args2) - - await do_test("send_all", (b"x",), "send_all", (b"x",)) - await do_test("send_all", (b"x",), "wait_send_all_might_not_block", ()) - await do_test( - "wait_send_all_might_not_block", - (), - "wait_send_all_might_not_block", - (), - ) - await do_test("receive_some", (1,), "receive_some", (1,)) - - -@contextmanager -def virtual_ssl_echo_server( - client_ctx: SSLContext, - sleeper: Callable[[str], Awaitable[None]] | None = None, -) -> Iterator[SSLStream[PyOpenSSLEchoStream]]: - fakesock = PyOpenSSLEchoStream(sleeper=sleeper) - yield SSLStream(fakesock, client_ctx, server_hostname="trio-test-1.example.org") - - -def ssl_wrap_pair( # type: ignore[explicit-any] - client_ctx: SSLContext, - client_transport: T_Stream, - server_transport: T_Stream, - *, - client_kwargs: dict[str, Any] | None = None, - server_kwargs: dict[str, Any] | None = None, -) -> tuple[SSLStream[T_Stream], SSLStream[T_Stream]]: - if server_kwargs is None: - server_kwargs = {} - if client_kwargs is None: - client_kwargs = {} - client_ssl = SSLStream( - client_transport, - client_ctx, - server_hostname="trio-test-1.example.org", - **client_kwargs, - ) - server_ssl = SSLStream( - server_transport, - SERVER_CTX, - server_side=True, - **server_kwargs, - ) - return client_ssl, server_ssl - - -MemoryStapledStream: TypeAlias = StapledStream[MemorySendStream, MemoryReceiveStream] - - -def ssl_memory_stream_pair( - client_ctx: SSLContext, - client_kwargs: dict[str, str | bytes | bool | None] | None = None, - server_kwargs: dict[str, str | bytes | bool | None] | None = None, -) -> tuple[ - SSLStream[MemoryStapledStream], - SSLStream[MemoryStapledStream], -]: - client_transport, server_transport = memory_stream_pair() - return ssl_wrap_pair( - client_ctx, - client_transport, - server_transport, - client_kwargs=client_kwargs, - server_kwargs=server_kwargs, - ) - - -MyStapledStream: TypeAlias = StapledStream[SendStream, ReceiveStream] - - -def ssl_lockstep_stream_pair( - client_ctx: SSLContext, - client_kwargs: dict[str, str | bytes | bool | None] | None = None, - server_kwargs: dict[str, str | bytes | bool | None] | None = None, -) -> tuple[ - SSLStream[MyStapledStream], - SSLStream[MyStapledStream], -]: - client_transport, server_transport = lockstep_stream_pair() - return ssl_wrap_pair( - client_ctx, - client_transport, - server_transport, - client_kwargs=client_kwargs, - server_kwargs=server_kwargs, - ) - - -# Simple smoke test for handshake/send/receive/shutdown talking to a -# synchronous server, plus make sure that we do the bare minimum of -# certificate checking (even though this is really Python's responsibility) -async def test_ssl_client_basics(client_ctx: SSLContext) -> None: - # Everything OK - async with ssl_echo_server(client_ctx) as s: - assert not s.server_side - await s.send_all(b"x") - assert await s.receive_some(1) == b"x" - await s.aclose() - - # Didn't configure the CA file, should fail - async with ssl_echo_server_raw(expect_fail=True) as sock: - bad_client_ctx = ssl.create_default_context() - s = SSLStream(sock, bad_client_ctx, server_hostname="trio-test-1.example.org") - assert not s.server_side - with pytest.raises(BrokenResourceError) as excinfo: - await s.send_all(b"x") - assert isinstance(excinfo.value.__cause__, ssl.SSLError) - - # Trusted CA, but wrong host name - async with ssl_echo_server_raw(expect_fail=True) as sock: - s = SSLStream(sock, client_ctx, server_hostname="trio-test-2.example.org") - assert not s.server_side - with pytest.raises(BrokenResourceError) as excinfo: - await s.send_all(b"x") - assert isinstance(excinfo.value.__cause__, ssl.CertificateError) - - -async def test_ssl_server_basics(client_ctx: SSLContext) -> None: - a, b = stdlib_socket.socketpair() - with a, b: - server_sock = tsocket.from_stdlib_socket(b) - server_transport = SSLStream( - SocketStream(server_sock), - SERVER_CTX, - server_side=True, - ) - assert server_transport.server_side - - def client() -> None: - with client_ctx.wrap_socket( - a, - server_hostname="trio-test-1.example.org", - ) as client_sock: - client_sock.sendall(b"x") - assert client_sock.recv(1) == b"y" - client_sock.sendall(b"z") - client_sock.unwrap() - - t = threading.Thread(target=client) - t.start() - - assert await server_transport.receive_some(1) == b"x" - await server_transport.send_all(b"y") - assert await server_transport.receive_some(1) == b"z" - assert await server_transport.receive_some(1) == b"" - await server_transport.aclose() - - t.join() - - -async def test_attributes(client_ctx: SSLContext) -> None: - async with ssl_echo_server_raw(expect_fail=True) as sock: - good_ctx = client_ctx - bad_ctx = ssl.create_default_context() - s = SSLStream(sock, good_ctx, server_hostname="trio-test-1.example.org") - - assert s.transport_stream is sock - - # Forwarded attribute getting - assert s.context is good_ctx - assert s.server_side == False # noqa - assert s.server_hostname == "trio-test-1.example.org" - with pytest.raises(AttributeError): - s.asfdasdfsa # noqa: B018 # "useless expression" - - # __dir__ - assert "transport_stream" in dir(s) - assert "context" in dir(s) - - # Setting the attribute goes through to the underlying object - - # most attributes on SSLObject are read-only - with pytest.raises(AttributeError): - s.server_side = True - with pytest.raises(AttributeError): - s.server_hostname = "asdf" - - # but .context is *not*. Check that we forward attribute setting by - # making sure that after we set the bad context our handshake indeed - # fails: - s.context = bad_ctx - assert s.context is bad_ctx - with pytest.raises(BrokenResourceError) as excinfo: - await s.do_handshake() - assert isinstance(excinfo.value.__cause__, ssl.SSLError) - - -# Note: this test fails horribly if we force TLS 1.2 and trigger a -# renegotiation at the beginning (e.g. by switching to the pyopenssl -# server). Usually the client crashes in SSLObject.write with "UNEXPECTED -# RECORD"; sometimes we get something more exotic like a SyscallError. This is -# odd because openssl isn't doing any syscalls, but so it goes. After lots of -# websearching I'm pretty sure this is due to a bug in OpenSSL, where it just -# can't reliably handle full-duplex communication combined with -# renegotiation. Nice, eh? -# -# https://rt.openssl.org/Ticket/Display.html?id=3712 -# https://rt.openssl.org/Ticket/Display.html?id=2481 -# http://openssl.6102.n7.nabble.com/TLS-renegotiation-failure-on-receiving-application-data-during-handshake-td48127.html -# https://stackoverflow.com/questions/18728355/ssl-renegotiation-with-full-duplex-socket-communication -# -# In some variants of this test (maybe only against the java server?) I've -# also seen cases where our send_all blocks waiting to write, and then our receive_some -# also blocks waiting to write, and they never wake up again. It looks like -# some kind of deadlock. I suspect there may be an issue where we've filled up -# the send buffers, and the remote side is trying to handle the renegotiation -# from inside a write() call, so it has a problem: there's all this application -# data clogging up the pipe, but it can't process and return it to the -# application because it's in write(), and it doesn't want to buffer infinite -# amounts of data, and... actually I guess those are the only two choices. -# -# NSS even documents that you shouldn't try to do a renegotiation except when -# the connection is idle: -# -# https://developer.mozilla.org/en-US/docs/Mozilla/Projects/NSS/SSL_functions/sslfnc.html#1061582 -# -# I begin to see why HTTP/2 forbids renegotiation and TLS 1.3 removes it... - - -async def test_full_duplex_basics(client_ctx: SSLContext) -> None: - CHUNKS = 30 - CHUNK_SIZE = 32768 - EXPECTED = CHUNKS * CHUNK_SIZE - - sent = bytearray() - received = bytearray() - - async def sender(s: Stream) -> None: - nonlocal sent - for i in range(CHUNKS): - print(i) - chunk = bytes([i] * CHUNK_SIZE) - sent += chunk - await s.send_all(chunk) - - async def receiver(s: Stream) -> None: - nonlocal received - while len(received) < EXPECTED: - chunk = await s.receive_some(CHUNK_SIZE // 2) - received += chunk - - async with ssl_echo_server(client_ctx) as s: - async with _core.open_nursery() as nursery: - nursery.start_soon(sender, s) - nursery.start_soon(receiver, s) - # And let's have some doing handshakes too, everyone - # simultaneously - nursery.start_soon(s.do_handshake) - nursery.start_soon(s.do_handshake) - - await s.aclose() - - assert len(sent) == len(received) == EXPECTED - assert sent == received - - -async def test_renegotiation_simple(client_ctx: SSLContext) -> None: - with virtual_ssl_echo_server(client_ctx) as s: - await s.do_handshake() - s.transport_stream.renegotiate() - await s.send_all(b"a") - assert await s.receive_some(1) == b"a" - - # Have to send some more data back and forth to make sure the - # renegotiation is finished before shutting down the - # connection... otherwise openssl raises an error. I think this is a - # bug in openssl but what can ya do. - await s.send_all(b"b") - assert await s.receive_some(1) == b"b" - - await s.aclose() - - -@slow -async def test_renegotiation_randomized( - mock_clock: MockClock, - client_ctx: SSLContext, -) -> None: - # The only blocking things in this function are our random sleeps, so 0 is - # a good threshold. - mock_clock.autojump_threshold = 0 - - import random - - r = random.Random(0) - - async def sleeper(_: object) -> None: - await trio.sleep(r.uniform(0, 10)) - - async def clear() -> None: - while s.transport_stream.renegotiate_pending(): - with assert_checkpoints(): - await send(b"-") - with assert_checkpoints(): - await expect(b"-") - print("-- clear --") - - async def send(byte: bytes) -> None: - await s.transport_stream.sleeper("outer send") - print("calling SSLStream.send_all", byte) - with assert_checkpoints(): - await s.send_all(byte) - - async def expect(expected: bytes) -> None: - await s.transport_stream.sleeper("expect") - print("calling SSLStream.receive_some, expecting", expected) - assert len(expected) == 1 - with assert_checkpoints(): - assert await s.receive_some(1) == expected - - with virtual_ssl_echo_server(client_ctx, sleeper=sleeper) as s: - await s.do_handshake() - - await send(b"a") - s.transport_stream.renegotiate() - await expect(b"a") - - await clear() - - for i in range(100): - b1 = bytes([i % 0xFF]) - b2 = bytes([(2 * i) % 0xFF]) - s.transport_stream.renegotiate() - async with _core.open_nursery() as nursery: - nursery.start_soon(send, b1) - nursery.start_soon(expect, b1) - async with _core.open_nursery() as nursery: - nursery.start_soon(expect, b2) - nursery.start_soon(send, b2) - await clear() - - for i in range(100): - b1 = bytes([i % 0xFF]) - b2 = bytes([(2 * i) % 0xFF]) - await send(b1) - s.transport_stream.renegotiate() - await expect(b1) - async with _core.open_nursery() as nursery: - nursery.start_soon(expect, b2) - nursery.start_soon(send, b2) - await clear() - - # Checking that wait_send_all_might_not_block and receive_some don't - # conflict: - - # 1) Set up a situation where expect (receive_some) is blocked sending, - # and wait_send_all_might_not_block comes in. - - # Our receive_some() call will get stuck when it hits send_all - async def sleeper_with_slow_send_all(method: str) -> None: - if method == "send_all": - # ignore ASYNC116, not sleep_forever, trying to test a large but finite sleep - await trio.sleep(100000) # noqa: ASYNC116 - - # And our wait_send_all_might_not_block call will give it time to get - # stuck, and then start - async def sleep_then_wait_writable() -> None: - await trio.sleep(1000) - await s.wait_send_all_might_not_block() - - with virtual_ssl_echo_server(client_ctx, sleeper=sleeper_with_slow_send_all) as s: - await send(b"x") - s.transport_stream.renegotiate() - async with _core.open_nursery() as nursery: - nursery.start_soon(expect, b"x") - nursery.start_soon(sleep_then_wait_writable) - - await clear() - - await s.aclose() - - # 2) Same, but now wait_send_all_might_not_block is stuck when - # receive_some tries to send. - - async def sleeper_with_slow_wait_writable_and_expect(method: str) -> None: - if method == "wait_send_all_might_not_block": - # ignore ASYNC116, not sleep_forever, trying to test a large but finite sleep - await trio.sleep(100000) # noqa: ASYNC116 - elif method == "expect": - await trio.sleep(1000) - - with virtual_ssl_echo_server( - client_ctx, - sleeper=sleeper_with_slow_wait_writable_and_expect, - ) as s: - await send(b"x") - s.transport_stream.renegotiate() - async with _core.open_nursery() as nursery: - nursery.start_soon(expect, b"x") - nursery.start_soon(s.wait_send_all_might_not_block) - - await clear() - - await s.aclose() - - -async def test_resource_busy_errors(client_ctx: SSLContext) -> None: - S: TypeAlias = trio.SSLStream[ - trio.StapledStream[trio.abc.SendStream, trio.abc.ReceiveStream] - ] - - async def do_send_all(s: S) -> None: - with assert_checkpoints(): - await s.send_all(b"x") - - async def do_receive_some(s: S) -> None: - with assert_checkpoints(): - await s.receive_some(1) - - async def do_wait_send_all_might_not_block(s: S) -> None: - with assert_checkpoints(): - await s.wait_send_all_might_not_block() - - async def do_test( - func1: Callable[[S], Awaitable[None]], - func2: Callable[[S], Awaitable[None]], - ) -> None: - s, _ = ssl_lockstep_stream_pair(client_ctx) - with pytest.RaisesGroup( - pytest.RaisesExc(_core.BusyResourceError, match="another task") - ): - async with _core.open_nursery() as nursery: - nursery.start_soon(func1, s) - nursery.start_soon(func2, s) - - await do_test(do_send_all, do_send_all) - await do_test(do_receive_some, do_receive_some) - await do_test(do_send_all, do_wait_send_all_might_not_block) - await do_test(do_wait_send_all_might_not_block, do_wait_send_all_might_not_block) - - -async def test_wait_writable_calls_underlying_wait_writable() -> None: - record = [] - - class NotAStream(Stream): - async def wait_send_all_might_not_block(self) -> None: - record.append("ok") - - # define methods that are abstract in Stream - async def aclose(self) -> None: - raise AssertionError("Should not get called") # pragma: no cover - - async def receive_some(self, max_bytes: int | None = None) -> bytes | bytearray: - raise AssertionError("Should not get called") # pragma: no cover - - async def send_all(self, data: bytes | bytearray | memoryview) -> None: - raise AssertionError("Should not get called") # pragma: no cover - - ctx = ssl.create_default_context() - s = SSLStream(NotAStream(), ctx, server_hostname="x") - await s.wait_send_all_might_not_block() - assert record == ["ok"] - - -@pytest.mark.skipif( - os.name == "nt" and sys.version_info >= (3, 10), - reason="frequently fails on Windows + Python 3.10", -) -async def test_checkpoints(client_ctx: SSLContext) -> None: - async with ssl_echo_server(client_ctx) as s: - with assert_checkpoints(): - await s.do_handshake() - with assert_checkpoints(): - await s.do_handshake() - with assert_checkpoints(): - await s.wait_send_all_might_not_block() - with assert_checkpoints(): - await s.send_all(b"xxx") - with assert_checkpoints(): - await s.receive_some(1) - # These receive_some's in theory could return immediately, because the - # "xxx" was sent in a single record and after the first - # receive_some(1) the rest are sitting inside the SSLObject's internal - # buffers. - with assert_checkpoints(): - await s.receive_some(1) - with assert_checkpoints(): - await s.receive_some(1) - with assert_checkpoints(): - await s.unwrap() - - async with ssl_echo_server(client_ctx) as s: - await s.do_handshake() - with assert_checkpoints(): - await s.aclose() - - -async def test_send_all_empty_string(client_ctx: SSLContext) -> None: - async with ssl_echo_server(client_ctx) as s: - await s.do_handshake() - - # underlying SSLObject interprets writing b"" as indicating an EOF, - # for some reason. Make sure we don't inherit this. - with assert_checkpoints(): - await s.send_all(b"") - with assert_checkpoints(): - await s.send_all(b"") - await s.send_all(b"x") - assert await s.receive_some(1) == b"x" - - await s.aclose() - - -@pytest.mark.parametrize("https_compatible", [False, True]) -async def test_SSLStream_generic( - client_ctx: SSLContext, - https_compatible: bool, -) -> None: - async def stream_maker() -> tuple[ - SSLStream[MemoryStapledStream], - SSLStream[MemoryStapledStream], - ]: - return ssl_memory_stream_pair( - client_ctx, - client_kwargs={"https_compatible": https_compatible}, - server_kwargs={"https_compatible": https_compatible}, - ) - - async def clogged_stream_maker() -> tuple[ - SSLStream[MyStapledStream], - SSLStream[MyStapledStream], - ]: - client, server = ssl_lockstep_stream_pair(client_ctx) - # If we don't do handshakes up front, then we run into a problem in - # the following situation: - # - server does wait_send_all_might_not_block - # - client does receive_some to unclog it - # Then the client's receive_some will actually send some data to start - # the handshake, and itself get stuck. - async with _core.open_nursery() as nursery: - nursery.start_soon(client.do_handshake) - nursery.start_soon(server.do_handshake) - return client, server - - await check_two_way_stream(stream_maker, clogged_stream_maker) - - -async def test_unwrap(client_ctx: SSLContext) -> None: - client_ssl, server_ssl = ssl_memory_stream_pair(client_ctx) - client_transport = client_ssl.transport_stream - server_transport = server_ssl.transport_stream - - seq = Sequencer() - - async def client() -> None: - await client_ssl.do_handshake() - await client_ssl.send_all(b"x") - assert await client_ssl.receive_some(1) == b"y" - await client_ssl.send_all(b"z") - - # After sending that, disable outgoing data from our end, to make - # sure the server doesn't see our EOF until after we've sent some - # trailing data - async with seq(0): - send_all_hook = client_transport.send_stream.send_all_hook - client_transport.send_stream.send_all_hook = None - - assert await client_ssl.receive_some(1) == b"" - assert client_ssl.transport_stream is client_transport - # We just received EOF. Unwrap the connection and send some more. - raw, trailing = await client_ssl.unwrap() - assert raw is client_transport - assert trailing == b"" - assert client_ssl.transport_stream is None - await raw.send_all(b"trailing") - - # Reconnect the streams. Now the server will receive both our shutdown - # acknowledgement + the trailing data in a single lump. - client_transport.send_stream.send_all_hook = send_all_hook - await client_transport.send_stream.send_all_hook() - - async def server() -> None: - await server_ssl.do_handshake() - assert await server_ssl.receive_some(1) == b"x" - await server_ssl.send_all(b"y") - assert await server_ssl.receive_some(1) == b"z" - # Now client is blocked waiting for us to send something, but - # instead we close the TLS connection (with sequencer to make sure - # that the client won't see and automatically respond before we've had - # a chance to disable the client->server transport) - async with seq(1): - raw, trailing = await server_ssl.unwrap() - assert raw is server_transport - assert trailing == b"trailing" - assert server_ssl.transport_stream is None - - async with _core.open_nursery() as nursery: - nursery.start_soon(client) - nursery.start_soon(server) - - -async def test_closing_nice_case(client_ctx: SSLContext) -> None: - # the nice case: graceful closes all around - - client_ssl, server_ssl = ssl_memory_stream_pair(client_ctx) - client_transport = client_ssl.transport_stream - - # Both the handshake and the close require back-and-forth discussion, so - # we need to run them concurrently - async def client_closer() -> None: - with assert_checkpoints(): - await client_ssl.aclose() - - async def server_closer() -> None: - assert await server_ssl.receive_some(10) == b"" - assert await server_ssl.receive_some(10) == b"" - with assert_checkpoints(): - await server_ssl.aclose() - - async with _core.open_nursery() as nursery: - nursery.start_soon(client_closer) - nursery.start_soon(server_closer) - - # closing the SSLStream also closes its transport - with pytest.raises(ClosedResourceError): - await client_transport.send_all(b"123") - - # once closed, it's OK to close again - with assert_checkpoints(): - await client_ssl.aclose() - with assert_checkpoints(): - await client_ssl.aclose() - - # Trying to send more data does not work - with pytest.raises(ClosedResourceError): - await server_ssl.send_all(b"123") - - # And once the connection is has been closed *locally*, then instead of - # getting empty bytestrings we get a proper error - with pytest.raises(ClosedResourceError): - assert await client_ssl.receive_some(10) == b"" - - with pytest.raises(ClosedResourceError): - await client_ssl.unwrap() - - with pytest.raises(ClosedResourceError): - await client_ssl.do_handshake() - - # Check that a graceful close *before* handshaking gives a clean EOF on - # the other side - client_ssl, server_ssl = ssl_memory_stream_pair(client_ctx) - - async def expect_eof_server() -> None: - with assert_checkpoints(): - assert await server_ssl.receive_some(10) == b"" - with assert_checkpoints(): - await server_ssl.aclose() - - async with _core.open_nursery() as nursery: - nursery.start_soon(client_ssl.aclose) - nursery.start_soon(expect_eof_server) - - -async def test_send_all_fails_in_the_middle(client_ctx: SSLContext) -> None: - client, server = ssl_memory_stream_pair(client_ctx) - - async with _core.open_nursery() as nursery: - nursery.start_soon(client.do_handshake) - nursery.start_soon(server.do_handshake) - - async def bad_hook() -> NoReturn: - raise KeyError - - client.transport_stream.send_stream.send_all_hook = bad_hook - - with pytest.raises(KeyError): - await client.send_all(b"x") - - with pytest.raises(BrokenResourceError): - await client.wait_send_all_might_not_block() - - closed = 0 - - def close_hook() -> None: - nonlocal closed - closed += 1 - - client.transport_stream.send_stream.close_hook = close_hook - client.transport_stream.receive_stream.close_hook = close_hook - await client.aclose() - - assert closed == 2 - - -async def test_ssl_over_ssl(client_ctx: SSLContext) -> None: - client_0, server_0 = memory_stream_pair() - - client_1 = SSLStream( - client_0, - client_ctx, - server_hostname="trio-test-1.example.org", - ) - server_1 = SSLStream(server_0, SERVER_CTX, server_side=True) - - client_2 = SSLStream( - client_1, - client_ctx, - server_hostname="trio-test-1.example.org", - ) - server_2 = SSLStream(server_1, SERVER_CTX, server_side=True) - - async def client() -> None: - await client_2.send_all(b"hi") - assert await client_2.receive_some(10) == b"bye" - - async def server() -> None: - assert await server_2.receive_some(10) == b"hi" - await server_2.send_all(b"bye") - - async with _core.open_nursery() as nursery: - nursery.start_soon(client) - nursery.start_soon(server) - - -async def test_ssl_bad_shutdown(client_ctx: SSLContext) -> None: - client, server = ssl_memory_stream_pair(client_ctx) - - async with _core.open_nursery() as nursery: - nursery.start_soon(client.do_handshake) - nursery.start_soon(server.do_handshake) - - await trio.aclose_forcefully(client) - # now the server sees a broken stream - with pytest.raises(BrokenResourceError): - await server.receive_some(10) - with pytest.raises(BrokenResourceError): - await server.send_all(b"x" * 10) - - await server.aclose() - - -async def test_ssl_bad_shutdown_but_its_ok(client_ctx: SSLContext) -> None: - client, server = ssl_memory_stream_pair( - client_ctx, - server_kwargs={"https_compatible": True}, - client_kwargs={"https_compatible": True}, - ) - - async with _core.open_nursery() as nursery: - nursery.start_soon(client.do_handshake) - nursery.start_soon(server.do_handshake) - - await trio.aclose_forcefully(client) - # the server sees that as a clean shutdown - assert await server.receive_some(10) == b"" - with pytest.raises(BrokenResourceError): - await server.send_all(b"x" * 10) - - await server.aclose() - - -async def test_ssl_handshake_failure_during_aclose() -> None: - # Weird scenario: aclose() triggers an automatic handshake, and this - # fails. This also exercises a bit of code in aclose() that was otherwise - # uncovered, for re-raising exceptions after calling aclose_forcefully on - # the underlying transport. - async with ssl_echo_server_raw(expect_fail=True) as sock: - # Don't configure trust correctly - client_ctx = ssl.create_default_context() - s = SSLStream(sock, client_ctx, server_hostname="trio-test-1.example.org") - # It's a little unclear here whether aclose should swallow the error - # or let it escape. We *do* swallow the error if it arrives when we're - # sending close_notify, because both sides closing the connection - # simultaneously is allowed. But I guess when https_compatible=False - # then it's bad if we can get through a whole connection with a peer - # that has no valid certificate, and never raise an error. - with pytest.raises(BrokenResourceError): - await s.aclose() - - -async def test_ssl_only_closes_stream_once(client_ctx: SSLContext) -> None: - # We used to have a bug where if transport_stream.aclose() raised an - # error, we would call it again. This checks that that's fixed. - client, server = ssl_memory_stream_pair(client_ctx) - - async with _core.open_nursery() as nursery: - nursery.start_soon(client.do_handshake) - nursery.start_soon(server.do_handshake) - - client_orig_close_hook = client.transport_stream.send_stream.close_hook - transport_close_count = 0 - - def close_hook() -> NoReturn: - nonlocal transport_close_count - assert client_orig_close_hook is not None - client_orig_close_hook() - transport_close_count += 1 - raise KeyError - - client.transport_stream.send_stream.close_hook = close_hook - - with pytest.raises(KeyError): - await client.aclose() - assert transport_close_count == 1 - - -async def test_ssl_https_compatibility_disagreement(client_ctx: SSLContext) -> None: - client, server = ssl_memory_stream_pair( - client_ctx, - server_kwargs={"https_compatible": False}, - client_kwargs={"https_compatible": True}, - ) - - async with _core.open_nursery() as nursery: - nursery.start_soon(client.do_handshake) - nursery.start_soon(server.do_handshake) - - # client is in HTTPS-mode, server is not - # so client doing graceful_shutdown causes an error on server - async def receive_and_expect_error() -> None: - with pytest.raises(BrokenResourceError) as excinfo: - await server.receive_some(10) - - assert _is_eof(excinfo.value.__cause__) - - async with _core.open_nursery() as nursery: - nursery.start_soon(client.aclose) - nursery.start_soon(receive_and_expect_error) - - -async def test_https_mode_eof_before_handshake(client_ctx: SSLContext) -> None: - client, server = ssl_memory_stream_pair( - client_ctx, - server_kwargs={"https_compatible": True}, - client_kwargs={"https_compatible": True}, - ) - - async def server_expect_clean_eof() -> None: - assert await server.receive_some(10) == b"" - - async with _core.open_nursery() as nursery: - nursery.start_soon(client.aclose) - nursery.start_soon(server_expect_clean_eof) - - -async def test_send_error_during_handshake(client_ctx: SSLContext) -> None: - client, _server = ssl_memory_stream_pair(client_ctx) - - async def bad_hook() -> NoReturn: - raise KeyError - - client.transport_stream.send_stream.send_all_hook = bad_hook - - with pytest.raises(KeyError): - with assert_checkpoints(): - await client.do_handshake() - - with pytest.raises(BrokenResourceError): - with assert_checkpoints(): - await client.do_handshake() - - -async def test_receive_error_during_handshake(client_ctx: SSLContext) -> None: - client, server = ssl_memory_stream_pair(client_ctx) - - async def bad_hook() -> NoReturn: - raise KeyError - - client.transport_stream.receive_stream.receive_some_hook = bad_hook - - async def client_side(cancel_scope: CancelScope) -> None: - with pytest.raises(KeyError): - with assert_checkpoints(): - await client.do_handshake() - cancel_scope.cancel() - - async with _core.open_nursery() as nursery: - nursery.start_soon(client_side, nursery.cancel_scope) - nursery.start_soon(server.do_handshake) - - with pytest.raises(BrokenResourceError): - with assert_checkpoints(): - await client.do_handshake() - - -def test_selected_alpn_protocol_before_handshake(client_ctx: SSLContext) -> None: - client, server = ssl_memory_stream_pair(client_ctx) - - with pytest.raises(NeedHandshakeError): - client.selected_alpn_protocol() - - with pytest.raises(NeedHandshakeError): - server.selected_alpn_protocol() - - -async def test_selected_alpn_protocol_when_not_set(client_ctx: SSLContext) -> None: - # ALPN protocol still returns None when it's not set, - # instead of raising an exception - client, server = ssl_memory_stream_pair(client_ctx) - - async with _core.open_nursery() as nursery: - nursery.start_soon(client.do_handshake) - nursery.start_soon(server.do_handshake) - - assert client.selected_alpn_protocol() is None - assert server.selected_alpn_protocol() is None - - assert client.selected_alpn_protocol() == server.selected_alpn_protocol() - - -def test_selected_npn_protocol_before_handshake(client_ctx: SSLContext) -> None: - client, server = ssl_memory_stream_pair(client_ctx) - - with pytest.raises(NeedHandshakeError): - client.selected_npn_protocol() - - with pytest.raises(NeedHandshakeError): - server.selected_npn_protocol() - - -@pytest.mark.filterwarnings( - r"ignore: ssl module. NPN is deprecated, use ALPN instead:UserWarning", - r"ignore:ssl NPN is deprecated, use ALPN instead:DeprecationWarning", -) -async def test_selected_npn_protocol_when_not_set(client_ctx: SSLContext) -> None: - # NPN protocol still returns None when it's not set, - # instead of raising an exception - client, server = ssl_memory_stream_pair(client_ctx) - - async with _core.open_nursery() as nursery: - nursery.start_soon(client.do_handshake) - nursery.start_soon(server.do_handshake) - - assert client.selected_npn_protocol() is None - assert server.selected_npn_protocol() is None - - assert client.selected_npn_protocol() == server.selected_npn_protocol() - - -def test_get_channel_binding_before_handshake(client_ctx: SSLContext) -> None: - client, server = ssl_memory_stream_pair(client_ctx) - - with pytest.raises(NeedHandshakeError): - client.get_channel_binding() - - with pytest.raises(NeedHandshakeError): - server.get_channel_binding() - - -async def test_get_channel_binding_after_handshake(client_ctx: SSLContext) -> None: - client, server = ssl_memory_stream_pair(client_ctx) - - async with _core.open_nursery() as nursery: - nursery.start_soon(client.do_handshake) - nursery.start_soon(server.do_handshake) - - assert client.get_channel_binding() is not None - assert server.get_channel_binding() is not None - - assert client.get_channel_binding() == server.get_channel_binding() - - -async def test_getpeercert(client_ctx: SSLContext) -> None: - # Make sure we're not affected by https://bugs.python.org/issue29334 - client, server = ssl_memory_stream_pair(client_ctx) - - async with _core.open_nursery() as nursery: - nursery.start_soon(client.do_handshake) - nursery.start_soon(server.do_handshake) - - assert server.getpeercert() is None - print(client.getpeercert()) - assert ("DNS", "trio-test-1.example.org") in client.getpeercert()["subjectAltName"] - - -async def test_SSLListener(client_ctx: SSLContext) -> None: - async def setup( - https_compatible: bool = False, - ) -> tuple[tsocket.SocketType, SSLListener[SocketStream], SSLStream[SocketStream]]: - listen_sock = tsocket.socket() - await listen_sock.bind(("127.0.0.1", 0)) - listen_sock.listen(1) - socket_listener = SocketListener(listen_sock) - ssl_listener = SSLListener( - socket_listener, - SERVER_CTX, - https_compatible=https_compatible, - ) - - transport_client = await open_tcp_stream(*listen_sock.getsockname()) - ssl_client = SSLStream( - transport_client, - client_ctx, - server_hostname="trio-test-1.example.org", - ) - return listen_sock, ssl_listener, ssl_client - - listen_sock, ssl_listener, ssl_client = await setup() - - async with ssl_client: - ssl_server = await ssl_listener.accept() - - async with ssl_server: - assert not ssl_server._https_compatible - - # Make sure the connection works - async with _core.open_nursery() as nursery: - nursery.start_soon(ssl_client.do_handshake) - nursery.start_soon(ssl_server.do_handshake) - - # Test SSLListener.aclose - await ssl_listener.aclose() - assert listen_sock.fileno() == -1 - - ################ - - # Test https_compatible - _, ssl_listener, ssl_client = await setup(https_compatible=True) - - ssl_server = await ssl_listener.accept() - - assert ssl_server._https_compatible - - await aclose_forcefully(ssl_listener) - await aclose_forcefully(ssl_client) - await aclose_forcefully(ssl_server) +from __future__ import annotations + +import os +import socket as stdlib_socket +import ssl +import sys +import threading +from contextlib import asynccontextmanager, contextmanager, suppress +from functools import partial +from ssl import SSLContext +from typing import TYPE_CHECKING, Any, NoReturn, TypeAlias + +import pytest + +from trio import StapledStream +from trio._tests.pytest_plugin import maybe_ignore_import_error +from trio.abc import ReceiveStream, SendStream +from trio.testing import MemoryReceiveStream, MemorySendStream + +try: + import trustme + from OpenSSL import SSL +except ImportError as error: + maybe_ignore_import_error(error) + +import trio + +from .. import _core, socket as tsocket +from .._abc import Stream +from .._core import BrokenResourceError, ClosedResourceError +from .._core._tests.tutil import slow +from .._highlevel_generic import aclose_forcefully +from .._highlevel_open_tcp_stream import open_tcp_stream +from .._highlevel_socket import SocketListener, SocketStream +from .._ssl import NeedHandshakeError, SSLListener, SSLStream, _is_eof +from .._util import ConflictDetector +from ..testing import ( + Sequencer, + assert_checkpoints, + check_two_way_stream, + lockstep_stream_pair, + memory_stream_pair, +) + +if TYPE_CHECKING: + from collections.abc import AsyncIterator, Awaitable, Callable, Iterator + + from trio._core import MockClock + from trio._ssl import T_Stream + + from .._core._run import CancelScope + +# We have two different kinds of echo server fixtures we use for testing. The +# first is a real server written using the stdlib ssl module and blocking +# sockets. It runs in a thread and we talk to it over a real socketpair(), to +# validate interoperability in a semi-realistic setting. +# +# The second is a very weird virtual echo server that lives inside a custom +# Stream class. It lives entirely inside the Python object space; there are no +# operating system calls in it at all. No threads, no I/O, nothing. It's +# 'send_all' call takes encrypted data from a client and feeds it directly into +# the server-side TLS state engine to decrypt, then takes that data, feeds it +# back through to get the encrypted response, and returns it from 'receive_some'. This +# gives us full control and reproducibility. This server is written using +# PyOpenSSL, so that we can trigger renegotiations on demand. It also allows +# us to insert random (virtual) delays, to really exercise all the weird paths +# in SSLStream's state engine. +# +# Both present a certificate for "trio-test-1.example.org". + +TRIO_TEST_CA = trustme.CA() +TRIO_TEST_1_CERT = TRIO_TEST_CA.issue_server_cert("trio-test-1.example.org") + +SERVER_CTX = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) +if hasattr(ssl, "OP_IGNORE_UNEXPECTED_EOF"): + SERVER_CTX.options &= ~ssl.OP_IGNORE_UNEXPECTED_EOF + +TRIO_TEST_1_CERT.configure_cert(SERVER_CTX) + + +# TLS 1.3 has a lot of changes from previous versions. So we want to run tests +# with both TLS 1.3, and TLS 1.2. +# "tls13" means that we're willing to negotiate TLS 1.3. Usually that's +# what will happen, but the renegotiation tests explicitly force a +# downgrade on the server side. "tls12" means we refuse to negotiate TLS +# 1.3, so we'll almost certainly use TLS 1.2. +@pytest.fixture(scope="module", params=["tls13", "tls12"]) +def client_ctx(request: pytest.FixtureRequest) -> ssl.SSLContext: + ctx = ssl.create_default_context() + + if hasattr(ssl, "OP_IGNORE_UNEXPECTED_EOF"): + ctx.options &= ~ssl.OP_IGNORE_UNEXPECTED_EOF + + TRIO_TEST_CA.configure_trust(ctx) + if request.param in ["default", "tls13"]: + return ctx + elif request.param == "tls12": + ctx.maximum_version = ssl.TLSVersion.TLSv1_2 + return ctx + else: # pragma: no cover + raise AssertionError() + + +# The blocking socket server. +def ssl_echo_serve_sync( + sock: stdlib_socket.socket, + *, + expect_fail: bool = False, +) -> None: + try: + wrapped = SERVER_CTX.wrap_socket( + sock, + server_side=True, + suppress_ragged_eofs=False, + ) + with wrapped: + wrapped.do_handshake() + while True: + data = wrapped.recv(4096) + if not data: + # other side has initiated a graceful shutdown; we try to + # respond in kind but it's legal for them to have already + # gone away. + with suppress(BrokenPipeError, ssl.SSLZeroReturnError): + wrapped.unwrap() + return + wrapped.sendall(data) + # This is an obscure workaround for an openssl bug. In server mode, in + # some versions, openssl sends some extra data at the end of do_handshake + # that it shouldn't send. Normally this is harmless, but, if the other + # side shuts down the connection before it reads that data, it might cause + # the OS to report a ECONNREST or even ECONNABORTED (which is just wrong, + # since ECONNABORTED is supposed to mean that connect() failed, but what + # can you do). In this case the other side did nothing wrong, but there's + # no way to recover, so we let it pass, and just cross our fingers its not + # hiding any (other) real bugs. For more details see: + # + # https://github.com/python-trio/trio/issues/1293 + # + # Also, this happens frequently but non-deterministically, so we have to + # 'no cover' it to avoid coverage flapping. + except (ConnectionResetError, ConnectionAbortedError): # pragma: no cover + return + except Exception as exc: + if expect_fail: + print("ssl_echo_serve_sync got error as expected:", exc) + else: # pragma: no cover + print("ssl_echo_serve_sync got unexpected error:", exc) + raise + else: + if expect_fail: # pragma: no cover + raise RuntimeError("failed to fail?") + finally: + sock.close() + + +# Fixture that gives a raw socket connected to a trio-test-1 echo server +# (running in a thread). Useful for testing making connections with different +# SSLContexts. +@asynccontextmanager +async def ssl_echo_server_raw(expect_fail: bool = False) -> AsyncIterator[SocketStream]: + a, b = stdlib_socket.socketpair() + async with trio.open_nursery() as nursery: + # Exiting the 'with a, b' context manager closes the sockets, which + # causes the thread to exit (possibly with an error), which allows the + # nursery context manager to exit too. + with a, b: + nursery.start_soon( + trio.to_thread.run_sync, + partial(ssl_echo_serve_sync, b, expect_fail=expect_fail), + ) + + yield SocketStream(tsocket.from_stdlib_socket(a)) + + +# Fixture that gives a properly set up SSLStream connected to a trio-test-1 +# echo server (running in a thread) +@asynccontextmanager +async def ssl_echo_server( + client_ctx: SSLContext, + expect_fail: bool = False, +) -> AsyncIterator[SSLStream[Stream]]: + async with ssl_echo_server_raw(expect_fail=expect_fail) as sock: + yield SSLStream(sock, client_ctx, server_hostname="trio-test-1.example.org") + + +# The weird in-memory server ... thing. +# Doesn't inherit from Stream because I left out the methods that we don't +# actually need. +# jakkdl: it seems to implement all the abstract methods (now), so I made it inherit +# from Stream for the sake of typechecking. +class PyOpenSSLEchoStream(Stream): + def __init__( + self, + sleeper: Callable[[str], Awaitable[None]] | None = None, + ) -> None: + ctx = SSL.Context(SSL.SSLv23_METHOD) + # TLS 1.3 removes renegotiation support. Which is great for them, but + # we still have to support versions before that, and that means we + # need to test renegotiation support, which means we need to force this + # to use a lower version where this test server can trigger + # renegotiations. + from cryptography.hazmat.bindings.openssl.binding import Binding + + b = Binding() + ctx.set_options(b.lib.SSL_OP_NO_TLSv1_3) + + # Unfortunately there's currently no way to say "use 1.3 or worse", we + # can only disable specific versions. And if the two sides start + # negotiating 1.4 at some point in the future, it *might* mean that + # our tests silently stop working properly. So the next line is a + # tripwire to remind us we need to revisit this stuff in 5 years or + # whatever when the next TLS version is released: + assert not hasattr(SSL, "OP_NO_TLSv1_4") + TRIO_TEST_1_CERT.configure_cert(ctx) + self._conn = SSL.Connection(ctx, None) + self._conn.set_accept_state() + self._lot = _core.ParkingLot() + self._pending_cleartext = bytearray() + + self._send_all_conflict_detector = ConflictDetector( + "simultaneous calls to PyOpenSSLEchoStream.send_all", + ) + self._receive_some_conflict_detector = ConflictDetector( + "simultaneous calls to PyOpenSSLEchoStream.receive_some", + ) + + self.sleeper: Callable[[str], Awaitable[None]] + if sleeper is None: + + async def no_op_sleeper(_: object) -> None: + return + + self.sleeper = no_op_sleeper + else: + self.sleeper = sleeper + + async def aclose(self) -> None: + self._conn.bio_shutdown() + + def renegotiate_pending(self) -> bool: + return self._conn.renegotiate_pending() + + def renegotiate(self) -> None: + # Returns false if a renegotiation is already in progress, meaning + # nothing happens. + assert self._conn.renegotiate() + + async def wait_send_all_might_not_block(self) -> None: + with self._send_all_conflict_detector: + await _core.checkpoint() + await _core.checkpoint() + await self.sleeper("wait_send_all_might_not_block") + + async def send_all(self, data: bytes | bytearray | memoryview[int]) -> None: + print(" --> transport_stream.send_all") + data_ = bytes(data) + with self._send_all_conflict_detector: + await _core.checkpoint() + await _core.checkpoint() + await self.sleeper("send_all") + self._conn.bio_write(data_) + while True: + await self.sleeper("send_all") + try: + data_ = self._conn.recv(1) + except SSL.ZeroReturnError: + self._conn.shutdown() + print("renegotiations:", self._conn.total_renegotiations()) + break + except SSL.WantReadError: + break + else: + self._pending_cleartext += data_ + self._lot.unpark_all() + await self.sleeper("send_all") + print(" <-- transport_stream.send_all finished") + + async def receive_some(self, nbytes: int | None = None) -> bytes: + print(" --> transport_stream.receive_some") + if nbytes is None: + nbytes = 65536 # arbitrary + with self._receive_some_conflict_detector: + try: + await _core.checkpoint() + await _core.checkpoint() + while True: + await self.sleeper("receive_some") + try: + return self._conn.bio_read(nbytes) + except SSL.WantReadError: + # No data in our ciphertext buffer; try to generate + # some. + if self._pending_cleartext: + # We have some cleartext; maybe we can encrypt it + # and then return it. + print(" trying", self._pending_cleartext) + try: + # PyOpenSSL bug: doesn't accept bytearray + # https://github.com/pyca/pyopenssl/issues/621 + next_byte = self._pending_cleartext[0:1] + self._conn.send(bytes(next_byte)) + # Apparently this next bit never gets hit in the + # test suite, but it's not an interesting omission + # so let's pragma it. + except SSL.WantReadError: # pragma: no cover + # We didn't manage to send the cleartext (and + # in particular we better leave it there to + # try again, due to openssl's retry + # semantics), but it's possible we pushed a + # renegotiation forward and *now* we have data + # to send. + try: + return self._conn.bio_read(nbytes) + except SSL.WantReadError: + # Nope. We're just going to have to wait + # for someone to call send_all() to give + # use more data. + print("parking (a)") + await self._lot.park() + else: + # We successfully sent that byte, so we don't + # have to again. + del self._pending_cleartext[0:1] + else: + # no pending cleartext; nothing to do but wait for + # someone to call send_all + print("parking (b)") + await self._lot.park() + finally: + await self.sleeper("receive_some") + print(" <-- transport_stream.receive_some finished") + + +async def test_PyOpenSSLEchoStream_gives_resource_busy_errors() -> None: + # Make sure that PyOpenSSLEchoStream complains if two tasks call send_all + # at the same time, or ditto for receive_some. The tricky cases where SSLStream + # might accidentally do this are during renegotiation, which we test using + # PyOpenSSLEchoStream, so this makes sure that if we do have a bug then + # PyOpenSSLEchoStream will notice and complain. + + async def do_test( + func1: str, + args1: tuple[object, ...], + func2: str, + args2: tuple[object, ...], + ) -> None: + s = PyOpenSSLEchoStream() + with pytest.RaisesGroup( + pytest.RaisesExc(_core.BusyResourceError, match="simultaneous") + ): + async with _core.open_nursery() as nursery: + nursery.start_soon(getattr(s, func1), *args1) + nursery.start_soon(getattr(s, func2), *args2) + + await do_test("send_all", (b"x",), "send_all", (b"x",)) + await do_test("send_all", (b"x",), "wait_send_all_might_not_block", ()) + await do_test( + "wait_send_all_might_not_block", + (), + "wait_send_all_might_not_block", + (), + ) + await do_test("receive_some", (1,), "receive_some", (1,)) + + +@contextmanager +def virtual_ssl_echo_server( + client_ctx: SSLContext, + sleeper: Callable[[str], Awaitable[None]] | None = None, +) -> Iterator[SSLStream[PyOpenSSLEchoStream]]: + fakesock = PyOpenSSLEchoStream(sleeper=sleeper) + yield SSLStream(fakesock, client_ctx, server_hostname="trio-test-1.example.org") + + +def ssl_wrap_pair( # type: ignore[explicit-any] + client_ctx: SSLContext, + client_transport: T_Stream, + server_transport: T_Stream, + *, + client_kwargs: dict[str, Any] | None = None, + server_kwargs: dict[str, Any] | None = None, +) -> tuple[SSLStream[T_Stream], SSLStream[T_Stream]]: + if server_kwargs is None: + server_kwargs = {} + if client_kwargs is None: + client_kwargs = {} + client_ssl = SSLStream( + client_transport, + client_ctx, + server_hostname="trio-test-1.example.org", + **client_kwargs, + ) + server_ssl = SSLStream( + server_transport, + SERVER_CTX, + server_side=True, + **server_kwargs, + ) + return client_ssl, server_ssl + + +MemoryStapledStream: TypeAlias = StapledStream[MemorySendStream, MemoryReceiveStream] + + +def ssl_memory_stream_pair( + client_ctx: SSLContext, + client_kwargs: dict[str, str | bytes | bool | None] | None = None, + server_kwargs: dict[str, str | bytes | bool | None] | None = None, +) -> tuple[ + SSLStream[MemoryStapledStream], + SSLStream[MemoryStapledStream], +]: + client_transport, server_transport = memory_stream_pair() + return ssl_wrap_pair( + client_ctx, + client_transport, + server_transport, + client_kwargs=client_kwargs, + server_kwargs=server_kwargs, + ) + + +MyStapledStream: TypeAlias = StapledStream[SendStream, ReceiveStream] + + +def ssl_lockstep_stream_pair( + client_ctx: SSLContext, + client_kwargs: dict[str, str | bytes | bool | None] | None = None, + server_kwargs: dict[str, str | bytes | bool | None] | None = None, +) -> tuple[ + SSLStream[MyStapledStream], + SSLStream[MyStapledStream], +]: + client_transport, server_transport = lockstep_stream_pair() + return ssl_wrap_pair( + client_ctx, + client_transport, + server_transport, + client_kwargs=client_kwargs, + server_kwargs=server_kwargs, + ) + + +# Simple smoke test for handshake/send/receive/shutdown talking to a +# synchronous server, plus make sure that we do the bare minimum of +# certificate checking (even though this is really Python's responsibility) +async def test_ssl_client_basics(client_ctx: SSLContext) -> None: + # Everything OK + async with ssl_echo_server(client_ctx) as s: + assert not s.server_side + await s.send_all(b"x") + assert await s.receive_some(1) == b"x" + await s.aclose() + + # Didn't configure the CA file, should fail + async with ssl_echo_server_raw(expect_fail=True) as sock: + bad_client_ctx = ssl.create_default_context() + s = SSLStream(sock, bad_client_ctx, server_hostname="trio-test-1.example.org") + assert not s.server_side + with pytest.raises(BrokenResourceError) as excinfo: + await s.send_all(b"x") + assert isinstance(excinfo.value.__cause__, ssl.SSLError) + + # Trusted CA, but wrong host name + async with ssl_echo_server_raw(expect_fail=True) as sock: + s = SSLStream(sock, client_ctx, server_hostname="trio-test-2.example.org") + assert not s.server_side + with pytest.raises(BrokenResourceError) as excinfo: + await s.send_all(b"x") + assert isinstance(excinfo.value.__cause__, ssl.CertificateError) + + +async def test_ssl_server_basics(client_ctx: SSLContext) -> None: + a, b = stdlib_socket.socketpair() + with a, b: + server_sock = tsocket.from_stdlib_socket(b) + server_transport = SSLStream( + SocketStream(server_sock), + SERVER_CTX, + server_side=True, + ) + assert server_transport.server_side + + def client() -> None: + with client_ctx.wrap_socket( + a, + server_hostname="trio-test-1.example.org", + ) as client_sock: + client_sock.sendall(b"x") + assert client_sock.recv(1) == b"y" + client_sock.sendall(b"z") + client_sock.unwrap() + + t = threading.Thread(target=client) + t.start() + + assert await server_transport.receive_some(1) == b"x" + await server_transport.send_all(b"y") + assert await server_transport.receive_some(1) == b"z" + assert await server_transport.receive_some(1) == b"" + await server_transport.aclose() + + t.join() + + +async def test_attributes(client_ctx: SSLContext) -> None: + async with ssl_echo_server_raw(expect_fail=True) as sock: + good_ctx = client_ctx + bad_ctx = ssl.create_default_context() + s = SSLStream(sock, good_ctx, server_hostname="trio-test-1.example.org") + + assert s.transport_stream is sock + + # Forwarded attribute getting + assert s.context is good_ctx + assert s.server_side == False # noqa + assert s.server_hostname == "trio-test-1.example.org" + with pytest.raises(AttributeError): + s.asfdasdfsa # noqa: B018 # "useless expression" + + # __dir__ + assert "transport_stream" in dir(s) + assert "context" in dir(s) + + # Setting the attribute goes through to the underlying object + + # most attributes on SSLObject are read-only + with pytest.raises(AttributeError): + s.server_side = True + with pytest.raises(AttributeError): + s.server_hostname = "asdf" + + # but .context is *not*. Check that we forward attribute setting by + # making sure that after we set the bad context our handshake indeed + # fails: + s.context = bad_ctx + assert s.context is bad_ctx + with pytest.raises(BrokenResourceError) as excinfo: + await s.do_handshake() + assert isinstance(excinfo.value.__cause__, ssl.SSLError) + + +# Note: this test fails horribly if we force TLS 1.2 and trigger a +# renegotiation at the beginning (e.g. by switching to the pyopenssl +# server). Usually the client crashes in SSLObject.write with "UNEXPECTED +# RECORD"; sometimes we get something more exotic like a SyscallError. This is +# odd because openssl isn't doing any syscalls, but so it goes. After lots of +# websearching I'm pretty sure this is due to a bug in OpenSSL, where it just +# can't reliably handle full-duplex communication combined with +# renegotiation. Nice, eh? +# +# https://rt.openssl.org/Ticket/Display.html?id=3712 +# https://rt.openssl.org/Ticket/Display.html?id=2481 +# http://openssl.6102.n7.nabble.com/TLS-renegotiation-failure-on-receiving-application-data-during-handshake-td48127.html +# https://stackoverflow.com/questions/18728355/ssl-renegotiation-with-full-duplex-socket-communication +# +# In some variants of this test (maybe only against the java server?) I've +# also seen cases where our send_all blocks waiting to write, and then our receive_some +# also blocks waiting to write, and they never wake up again. It looks like +# some kind of deadlock. I suspect there may be an issue where we've filled up +# the send buffers, and the remote side is trying to handle the renegotiation +# from inside a write() call, so it has a problem: there's all this application +# data clogging up the pipe, but it can't process and return it to the +# application because it's in write(), and it doesn't want to buffer infinite +# amounts of data, and... actually I guess those are the only two choices. +# +# NSS even documents that you shouldn't try to do a renegotiation except when +# the connection is idle: +# +# https://developer.mozilla.org/en-US/docs/Mozilla/Projects/NSS/SSL_functions/sslfnc.html#1061582 +# +# I begin to see why HTTP/2 forbids renegotiation and TLS 1.3 removes it... + + +async def test_full_duplex_basics(client_ctx: SSLContext) -> None: + CHUNKS = 30 + CHUNK_SIZE = 32768 + EXPECTED = CHUNKS * CHUNK_SIZE + + sent = bytearray() + received = bytearray() + + async def sender(s: Stream) -> None: + nonlocal sent + for i in range(CHUNKS): + print(i) + chunk = bytes([i] * CHUNK_SIZE) + sent += chunk + await s.send_all(chunk) + + async def receiver(s: Stream) -> None: + nonlocal received + while len(received) < EXPECTED: + chunk = await s.receive_some(CHUNK_SIZE // 2) + received += chunk + + async with ssl_echo_server(client_ctx) as s: + async with _core.open_nursery() as nursery: + nursery.start_soon(sender, s) + nursery.start_soon(receiver, s) + # And let's have some doing handshakes too, everyone + # simultaneously + nursery.start_soon(s.do_handshake) + nursery.start_soon(s.do_handshake) + + await s.aclose() + + assert len(sent) == len(received) == EXPECTED + assert sent == received + + +async def test_renegotiation_simple(client_ctx: SSLContext) -> None: + with virtual_ssl_echo_server(client_ctx) as s: + await s.do_handshake() + s.transport_stream.renegotiate() + await s.send_all(b"a") + assert await s.receive_some(1) == b"a" + + # Have to send some more data back and forth to make sure the + # renegotiation is finished before shutting down the + # connection... otherwise openssl raises an error. I think this is a + # bug in openssl but what can ya do. + await s.send_all(b"b") + assert await s.receive_some(1) == b"b" + + await s.aclose() + + +@slow +async def test_renegotiation_randomized( + mock_clock: MockClock, + client_ctx: SSLContext, +) -> None: + # The only blocking things in this function are our random sleeps, so 0 is + # a good threshold. + mock_clock.autojump_threshold = 0 + + import random + + r = random.Random(0) + + async def sleeper(_: object) -> None: + await trio.sleep(r.uniform(0, 10)) + + async def clear() -> None: + while s.transport_stream.renegotiate_pending(): + with assert_checkpoints(): + await send(b"-") + with assert_checkpoints(): + await expect(b"-") + print("-- clear --") + + async def send(byte: bytes) -> None: + await s.transport_stream.sleeper("outer send") + print("calling SSLStream.send_all", byte) + with assert_checkpoints(): + await s.send_all(byte) + + async def expect(expected: bytes) -> None: + await s.transport_stream.sleeper("expect") + print("calling SSLStream.receive_some, expecting", expected) + assert len(expected) == 1 + with assert_checkpoints(): + assert await s.receive_some(1) == expected + + with virtual_ssl_echo_server(client_ctx, sleeper=sleeper) as s: + await s.do_handshake() + + await send(b"a") + s.transport_stream.renegotiate() + await expect(b"a") + + await clear() + + for i in range(100): + b1 = bytes([i % 0xFF]) + b2 = bytes([(2 * i) % 0xFF]) + s.transport_stream.renegotiate() + async with _core.open_nursery() as nursery: + nursery.start_soon(send, b1) + nursery.start_soon(expect, b1) + async with _core.open_nursery() as nursery: + nursery.start_soon(expect, b2) + nursery.start_soon(send, b2) + await clear() + + for i in range(100): + b1 = bytes([i % 0xFF]) + b2 = bytes([(2 * i) % 0xFF]) + await send(b1) + s.transport_stream.renegotiate() + await expect(b1) + async with _core.open_nursery() as nursery: + nursery.start_soon(expect, b2) + nursery.start_soon(send, b2) + await clear() + + # Checking that wait_send_all_might_not_block and receive_some don't + # conflict: + + # 1) Set up a situation where expect (receive_some) is blocked sending, + # and wait_send_all_might_not_block comes in. + + # Our receive_some() call will get stuck when it hits send_all + async def sleeper_with_slow_send_all(method: str) -> None: + if method == "send_all": + # ignore ASYNC116, not sleep_forever, trying to test a large but finite sleep + await trio.sleep(100000) # noqa: ASYNC116 + + # And our wait_send_all_might_not_block call will give it time to get + # stuck, and then start + async def sleep_then_wait_writable() -> None: + await trio.sleep(1000) + await s.wait_send_all_might_not_block() + + with virtual_ssl_echo_server(client_ctx, sleeper=sleeper_with_slow_send_all) as s: + await send(b"x") + s.transport_stream.renegotiate() + async with _core.open_nursery() as nursery: + nursery.start_soon(expect, b"x") + nursery.start_soon(sleep_then_wait_writable) + + await clear() + + await s.aclose() + + # 2) Same, but now wait_send_all_might_not_block is stuck when + # receive_some tries to send. + + async def sleeper_with_slow_wait_writable_and_expect(method: str) -> None: + if method == "wait_send_all_might_not_block": + # ignore ASYNC116, not sleep_forever, trying to test a large but finite sleep + await trio.sleep(100000) # noqa: ASYNC116 + elif method == "expect": + await trio.sleep(1000) + + with virtual_ssl_echo_server( + client_ctx, + sleeper=sleeper_with_slow_wait_writable_and_expect, + ) as s: + await send(b"x") + s.transport_stream.renegotiate() + async with _core.open_nursery() as nursery: + nursery.start_soon(expect, b"x") + nursery.start_soon(s.wait_send_all_might_not_block) + + await clear() + + await s.aclose() + + +async def test_resource_busy_errors(client_ctx: SSLContext) -> None: + S: TypeAlias = trio.SSLStream[ + trio.StapledStream[trio.abc.SendStream, trio.abc.ReceiveStream] + ] + + async def do_send_all(s: S) -> None: + with assert_checkpoints(): + await s.send_all(b"x") + + async def do_receive_some(s: S) -> None: + with assert_checkpoints(): + await s.receive_some(1) + + async def do_wait_send_all_might_not_block(s: S) -> None: + with assert_checkpoints(): + await s.wait_send_all_might_not_block() + + async def do_test( + func1: Callable[[S], Awaitable[None]], + func2: Callable[[S], Awaitable[None]], + ) -> None: + s, _ = ssl_lockstep_stream_pair(client_ctx) + with pytest.RaisesGroup( + pytest.RaisesExc(_core.BusyResourceError, match="another task") + ): + async with _core.open_nursery() as nursery: + nursery.start_soon(func1, s) + nursery.start_soon(func2, s) + + await do_test(do_send_all, do_send_all) + await do_test(do_receive_some, do_receive_some) + await do_test(do_send_all, do_wait_send_all_might_not_block) + await do_test(do_wait_send_all_might_not_block, do_wait_send_all_might_not_block) + + +async def test_wait_writable_calls_underlying_wait_writable() -> None: + record = [] + + class NotAStream(Stream): + async def wait_send_all_might_not_block(self) -> None: + record.append("ok") + + # define methods that are abstract in Stream + async def aclose(self) -> None: + raise AssertionError("Should not get called") # pragma: no cover + + async def receive_some(self, max_bytes: int | None = None) -> bytes | bytearray: + raise AssertionError("Should not get called") # pragma: no cover + + async def send_all(self, data: bytes | bytearray | memoryview) -> None: + raise AssertionError("Should not get called") # pragma: no cover + + ctx = ssl.create_default_context() + s = SSLStream(NotAStream(), ctx, server_hostname="x") + await s.wait_send_all_might_not_block() + assert record == ["ok"] + + +@pytest.mark.skipif( + os.name == "nt" and sys.version_info >= (3, 10), + reason="frequently fails on Windows + Python 3.10", +) +async def test_checkpoints(client_ctx: SSLContext) -> None: + async with ssl_echo_server(client_ctx) as s: + with assert_checkpoints(): + await s.do_handshake() + with assert_checkpoints(): + await s.do_handshake() + with assert_checkpoints(): + await s.wait_send_all_might_not_block() + with assert_checkpoints(): + await s.send_all(b"xxx") + with assert_checkpoints(): + await s.receive_some(1) + # These receive_some's in theory could return immediately, because the + # "xxx" was sent in a single record and after the first + # receive_some(1) the rest are sitting inside the SSLObject's internal + # buffers. + with assert_checkpoints(): + await s.receive_some(1) + with assert_checkpoints(): + await s.receive_some(1) + with assert_checkpoints(): + await s.unwrap() + + async with ssl_echo_server(client_ctx) as s: + await s.do_handshake() + with assert_checkpoints(): + await s.aclose() + + +async def test_send_all_empty_string(client_ctx: SSLContext) -> None: + async with ssl_echo_server(client_ctx) as s: + await s.do_handshake() + + # underlying SSLObject interprets writing b"" as indicating an EOF, + # for some reason. Make sure we don't inherit this. + with assert_checkpoints(): + await s.send_all(b"") + with assert_checkpoints(): + await s.send_all(b"") + await s.send_all(b"x") + assert await s.receive_some(1) == b"x" + + await s.aclose() + + +@pytest.mark.parametrize("https_compatible", [False, True]) +async def test_SSLStream_generic( + client_ctx: SSLContext, + https_compatible: bool, +) -> None: + async def stream_maker() -> tuple[ + SSLStream[MemoryStapledStream], + SSLStream[MemoryStapledStream], + ]: + return ssl_memory_stream_pair( + client_ctx, + client_kwargs={"https_compatible": https_compatible}, + server_kwargs={"https_compatible": https_compatible}, + ) + + async def clogged_stream_maker() -> tuple[ + SSLStream[MyStapledStream], + SSLStream[MyStapledStream], + ]: + client, server = ssl_lockstep_stream_pair(client_ctx) + # If we don't do handshakes up front, then we run into a problem in + # the following situation: + # - server does wait_send_all_might_not_block + # - client does receive_some to unclog it + # Then the client's receive_some will actually send some data to start + # the handshake, and itself get stuck. + async with _core.open_nursery() as nursery: + nursery.start_soon(client.do_handshake) + nursery.start_soon(server.do_handshake) + return client, server + + await check_two_way_stream(stream_maker, clogged_stream_maker) + + +async def test_unwrap(client_ctx: SSLContext) -> None: + client_ssl, server_ssl = ssl_memory_stream_pair(client_ctx) + client_transport = client_ssl.transport_stream + server_transport = server_ssl.transport_stream + + seq = Sequencer() + + async def client() -> None: + await client_ssl.do_handshake() + await client_ssl.send_all(b"x") + assert await client_ssl.receive_some(1) == b"y" + await client_ssl.send_all(b"z") + + # After sending that, disable outgoing data from our end, to make + # sure the server doesn't see our EOF until after we've sent some + # trailing data + async with seq(0): + send_all_hook = client_transport.send_stream.send_all_hook + client_transport.send_stream.send_all_hook = None + + assert await client_ssl.receive_some(1) == b"" + assert client_ssl.transport_stream is client_transport + # We just received EOF. Unwrap the connection and send some more. + raw, trailing = await client_ssl.unwrap() + assert raw is client_transport + assert trailing == b"" + assert client_ssl.transport_stream is None + await raw.send_all(b"trailing") + + # Reconnect the streams. Now the server will receive both our shutdown + # acknowledgement + the trailing data in a single lump. + client_transport.send_stream.send_all_hook = send_all_hook + await client_transport.send_stream.send_all_hook() + + async def server() -> None: + await server_ssl.do_handshake() + assert await server_ssl.receive_some(1) == b"x" + await server_ssl.send_all(b"y") + assert await server_ssl.receive_some(1) == b"z" + # Now client is blocked waiting for us to send something, but + # instead we close the TLS connection (with sequencer to make sure + # that the client won't see and automatically respond before we've had + # a chance to disable the client->server transport) + async with seq(1): + raw, trailing = await server_ssl.unwrap() + assert raw is server_transport + assert trailing == b"trailing" + assert server_ssl.transport_stream is None + + async with _core.open_nursery() as nursery: + nursery.start_soon(client) + nursery.start_soon(server) + + +async def test_closing_nice_case(client_ctx: SSLContext) -> None: + # the nice case: graceful closes all around + + client_ssl, server_ssl = ssl_memory_stream_pair(client_ctx) + client_transport = client_ssl.transport_stream + + # Both the handshake and the close require back-and-forth discussion, so + # we need to run them concurrently + async def client_closer() -> None: + with assert_checkpoints(): + await client_ssl.aclose() + + async def server_closer() -> None: + assert await server_ssl.receive_some(10) == b"" + assert await server_ssl.receive_some(10) == b"" + with assert_checkpoints(): + await server_ssl.aclose() + + async with _core.open_nursery() as nursery: + nursery.start_soon(client_closer) + nursery.start_soon(server_closer) + + # closing the SSLStream also closes its transport + with pytest.raises(ClosedResourceError): + await client_transport.send_all(b"123") + + # once closed, it's OK to close again + with assert_checkpoints(): + await client_ssl.aclose() + with assert_checkpoints(): + await client_ssl.aclose() + + # Trying to send more data does not work + with pytest.raises(ClosedResourceError): + await server_ssl.send_all(b"123") + + # And once the connection is has been closed *locally*, then instead of + # getting empty bytestrings we get a proper error + with pytest.raises(ClosedResourceError): + assert await client_ssl.receive_some(10) == b"" + + with pytest.raises(ClosedResourceError): + await client_ssl.unwrap() + + with pytest.raises(ClosedResourceError): + await client_ssl.do_handshake() + + # Check that a graceful close *before* handshaking gives a clean EOF on + # the other side + client_ssl, server_ssl = ssl_memory_stream_pair(client_ctx) + + async def expect_eof_server() -> None: + with assert_checkpoints(): + assert await server_ssl.receive_some(10) == b"" + with assert_checkpoints(): + await server_ssl.aclose() + + async with _core.open_nursery() as nursery: + nursery.start_soon(client_ssl.aclose) + nursery.start_soon(expect_eof_server) + + +async def test_send_all_fails_in_the_middle(client_ctx: SSLContext) -> None: + client, server = ssl_memory_stream_pair(client_ctx) + + async with _core.open_nursery() as nursery: + nursery.start_soon(client.do_handshake) + nursery.start_soon(server.do_handshake) + + async def bad_hook() -> NoReturn: + raise KeyError + + client.transport_stream.send_stream.send_all_hook = bad_hook + + with pytest.raises(KeyError): + await client.send_all(b"x") + + with pytest.raises(BrokenResourceError): + await client.wait_send_all_might_not_block() + + closed = 0 + + def close_hook() -> None: + nonlocal closed + closed += 1 + + client.transport_stream.send_stream.close_hook = close_hook + client.transport_stream.receive_stream.close_hook = close_hook + await client.aclose() + + assert closed == 2 + + +async def test_ssl_over_ssl(client_ctx: SSLContext) -> None: + client_0, server_0 = memory_stream_pair() + + client_1 = SSLStream( + client_0, + client_ctx, + server_hostname="trio-test-1.example.org", + ) + server_1 = SSLStream(server_0, SERVER_CTX, server_side=True) + + client_2 = SSLStream( + client_1, + client_ctx, + server_hostname="trio-test-1.example.org", + ) + server_2 = SSLStream(server_1, SERVER_CTX, server_side=True) + + async def client() -> None: + await client_2.send_all(b"hi") + assert await client_2.receive_some(10) == b"bye" + + async def server() -> None: + assert await server_2.receive_some(10) == b"hi" + await server_2.send_all(b"bye") + + async with _core.open_nursery() as nursery: + nursery.start_soon(client) + nursery.start_soon(server) + + +async def test_ssl_bad_shutdown(client_ctx: SSLContext) -> None: + client, server = ssl_memory_stream_pair(client_ctx) + + async with _core.open_nursery() as nursery: + nursery.start_soon(client.do_handshake) + nursery.start_soon(server.do_handshake) + + await trio.aclose_forcefully(client) + # now the server sees a broken stream + with pytest.raises(BrokenResourceError): + await server.receive_some(10) + with pytest.raises(BrokenResourceError): + await server.send_all(b"x" * 10) + + await server.aclose() + + +async def test_ssl_bad_shutdown_but_its_ok(client_ctx: SSLContext) -> None: + client, server = ssl_memory_stream_pair( + client_ctx, + server_kwargs={"https_compatible": True}, + client_kwargs={"https_compatible": True}, + ) + + async with _core.open_nursery() as nursery: + nursery.start_soon(client.do_handshake) + nursery.start_soon(server.do_handshake) + + await trio.aclose_forcefully(client) + # the server sees that as a clean shutdown + assert await server.receive_some(10) == b"" + with pytest.raises(BrokenResourceError): + await server.send_all(b"x" * 10) + + await server.aclose() + + +async def test_ssl_handshake_failure_during_aclose() -> None: + # Weird scenario: aclose() triggers an automatic handshake, and this + # fails. This also exercises a bit of code in aclose() that was otherwise + # uncovered, for re-raising exceptions after calling aclose_forcefully on + # the underlying transport. + async with ssl_echo_server_raw(expect_fail=True) as sock: + # Don't configure trust correctly + client_ctx = ssl.create_default_context() + s = SSLStream(sock, client_ctx, server_hostname="trio-test-1.example.org") + # It's a little unclear here whether aclose should swallow the error + # or let it escape. We *do* swallow the error if it arrives when we're + # sending close_notify, because both sides closing the connection + # simultaneously is allowed. But I guess when https_compatible=False + # then it's bad if we can get through a whole connection with a peer + # that has no valid certificate, and never raise an error. + with pytest.raises(BrokenResourceError): + await s.aclose() + + +async def test_ssl_only_closes_stream_once(client_ctx: SSLContext) -> None: + # We used to have a bug where if transport_stream.aclose() raised an + # error, we would call it again. This checks that that's fixed. + client, server = ssl_memory_stream_pair(client_ctx) + + async with _core.open_nursery() as nursery: + nursery.start_soon(client.do_handshake) + nursery.start_soon(server.do_handshake) + + client_orig_close_hook = client.transport_stream.send_stream.close_hook + transport_close_count = 0 + + def close_hook() -> NoReturn: + nonlocal transport_close_count + assert client_orig_close_hook is not None + client_orig_close_hook() + transport_close_count += 1 + raise KeyError + + client.transport_stream.send_stream.close_hook = close_hook + + with pytest.raises(KeyError): + await client.aclose() + assert transport_close_count == 1 + + +async def test_ssl_https_compatibility_disagreement(client_ctx: SSLContext) -> None: + client, server = ssl_memory_stream_pair( + client_ctx, + server_kwargs={"https_compatible": False}, + client_kwargs={"https_compatible": True}, + ) + + async with _core.open_nursery() as nursery: + nursery.start_soon(client.do_handshake) + nursery.start_soon(server.do_handshake) + + # client is in HTTPS-mode, server is not + # so client doing graceful_shutdown causes an error on server + async def receive_and_expect_error() -> None: + with pytest.raises(BrokenResourceError) as excinfo: + await server.receive_some(10) + + assert _is_eof(excinfo.value.__cause__) + + async with _core.open_nursery() as nursery: + nursery.start_soon(client.aclose) + nursery.start_soon(receive_and_expect_error) + + +async def test_https_mode_eof_before_handshake(client_ctx: SSLContext) -> None: + client, server = ssl_memory_stream_pair( + client_ctx, + server_kwargs={"https_compatible": True}, + client_kwargs={"https_compatible": True}, + ) + + async def server_expect_clean_eof() -> None: + assert await server.receive_some(10) == b"" + + async with _core.open_nursery() as nursery: + nursery.start_soon(client.aclose) + nursery.start_soon(server_expect_clean_eof) + + +async def test_send_error_during_handshake(client_ctx: SSLContext) -> None: + client, _server = ssl_memory_stream_pair(client_ctx) + + async def bad_hook() -> NoReturn: + raise KeyError + + client.transport_stream.send_stream.send_all_hook = bad_hook + + with pytest.raises(KeyError): + with assert_checkpoints(): + await client.do_handshake() + + with pytest.raises(BrokenResourceError): + with assert_checkpoints(): + await client.do_handshake() + + +async def test_receive_error_during_handshake(client_ctx: SSLContext) -> None: + client, server = ssl_memory_stream_pair(client_ctx) + + async def bad_hook() -> NoReturn: + raise KeyError + + client.transport_stream.receive_stream.receive_some_hook = bad_hook + + async def client_side(cancel_scope: CancelScope) -> None: + with pytest.raises(KeyError): + with assert_checkpoints(): + await client.do_handshake() + cancel_scope.cancel() + + async with _core.open_nursery() as nursery: + nursery.start_soon(client_side, nursery.cancel_scope) + nursery.start_soon(server.do_handshake) + + with pytest.raises(BrokenResourceError): + with assert_checkpoints(): + await client.do_handshake() + + +def test_selected_alpn_protocol_before_handshake(client_ctx: SSLContext) -> None: + client, server = ssl_memory_stream_pair(client_ctx) + + with pytest.raises(NeedHandshakeError): + client.selected_alpn_protocol() + + with pytest.raises(NeedHandshakeError): + server.selected_alpn_protocol() + + +async def test_selected_alpn_protocol_when_not_set(client_ctx: SSLContext) -> None: + # ALPN protocol still returns None when it's not set, + # instead of raising an exception + client, server = ssl_memory_stream_pair(client_ctx) + + async with _core.open_nursery() as nursery: + nursery.start_soon(client.do_handshake) + nursery.start_soon(server.do_handshake) + + assert client.selected_alpn_protocol() is None + assert server.selected_alpn_protocol() is None + + assert client.selected_alpn_protocol() == server.selected_alpn_protocol() + + +def test_selected_npn_protocol_before_handshake(client_ctx: SSLContext) -> None: + client, server = ssl_memory_stream_pair(client_ctx) + + with pytest.raises(NeedHandshakeError): + client.selected_npn_protocol() + + with pytest.raises(NeedHandshakeError): + server.selected_npn_protocol() + + +@pytest.mark.filterwarnings( + r"ignore: ssl module. NPN is deprecated, use ALPN instead:UserWarning", + r"ignore:ssl NPN is deprecated, use ALPN instead:DeprecationWarning", +) +async def test_selected_npn_protocol_when_not_set(client_ctx: SSLContext) -> None: + # NPN protocol still returns None when it's not set, + # instead of raising an exception + client, server = ssl_memory_stream_pair(client_ctx) + + async with _core.open_nursery() as nursery: + nursery.start_soon(client.do_handshake) + nursery.start_soon(server.do_handshake) + + assert client.selected_npn_protocol() is None + assert server.selected_npn_protocol() is None + + assert client.selected_npn_protocol() == server.selected_npn_protocol() + + +def test_get_channel_binding_before_handshake(client_ctx: SSLContext) -> None: + client, server = ssl_memory_stream_pair(client_ctx) + + with pytest.raises(NeedHandshakeError): + client.get_channel_binding() + + with pytest.raises(NeedHandshakeError): + server.get_channel_binding() + + +async def test_get_channel_binding_after_handshake(client_ctx: SSLContext) -> None: + client, server = ssl_memory_stream_pair(client_ctx) + + async with _core.open_nursery() as nursery: + nursery.start_soon(client.do_handshake) + nursery.start_soon(server.do_handshake) + + assert client.get_channel_binding() is not None + assert server.get_channel_binding() is not None + + assert client.get_channel_binding() == server.get_channel_binding() + + +async def test_getpeercert(client_ctx: SSLContext) -> None: + # Make sure we're not affected by https://bugs.python.org/issue29334 + client, server = ssl_memory_stream_pair(client_ctx) + + async with _core.open_nursery() as nursery: + nursery.start_soon(client.do_handshake) + nursery.start_soon(server.do_handshake) + + assert server.getpeercert() is None + print(client.getpeercert()) + assert ("DNS", "trio-test-1.example.org") in client.getpeercert()["subjectAltName"] + + +async def test_SSLListener(client_ctx: SSLContext) -> None: + async def setup( + https_compatible: bool = False, + ) -> tuple[tsocket.SocketType, SSLListener[SocketStream], SSLStream[SocketStream]]: + listen_sock = tsocket.socket() + await listen_sock.bind(("127.0.0.1", 0)) + listen_sock.listen(1) + socket_listener = SocketListener(listen_sock) + ssl_listener = SSLListener( + socket_listener, + SERVER_CTX, + https_compatible=https_compatible, + ) + + transport_client = await open_tcp_stream(*listen_sock.getsockname()) + ssl_client = SSLStream( + transport_client, + client_ctx, + server_hostname="trio-test-1.example.org", + ) + return listen_sock, ssl_listener, ssl_client + + listen_sock, ssl_listener, ssl_client = await setup() + + async with ssl_client: + ssl_server = await ssl_listener.accept() + + async with ssl_server: + assert not ssl_server._https_compatible + + # Make sure the connection works + async with _core.open_nursery() as nursery: + nursery.start_soon(ssl_client.do_handshake) + nursery.start_soon(ssl_server.do_handshake) + + # Test SSLListener.aclose + await ssl_listener.aclose() + assert listen_sock.fileno() == -1 + + ################ + + # Test https_compatible + _, ssl_listener, ssl_client = await setup(https_compatible=True) + + ssl_server = await ssl_listener.accept() + + assert ssl_server._https_compatible + + await aclose_forcefully(ssl_listener) + await aclose_forcefully(ssl_client) + await aclose_forcefully(ssl_server) diff --git a/src/trio/_tests/tools/test_gen_exports.py b/src/trio/_tests/tools/test_gen_exports.py index 3b923ed83d..2c31ac1ec9 100644 --- a/src/trio/_tests/tools/test_gen_exports.py +++ b/src/trio/_tests/tools/test_gen_exports.py @@ -1,179 +1,179 @@ -import ast -import sys -from pathlib import Path - -import pytest - -from trio._tests.pytest_plugin import maybe_ignore_import_error - -# imports in gen_exports that are not in `install_requires` in setup.py -try: - import astor # noqa: F401 - import isort # noqa: F401 -except ImportError as error: - maybe_ignore_import_error(error) - - -from trio._tools.gen_exports import ( - File, - create_passthrough_args, - get_public_methods, - process, - run_linters, - run_ruff, -) - -from ..._core._tests.tutil import slow - -SOURCE = '''from _run import _public -from collections import Counter - -class Test: - @_public - def public_func(self): - """With doc string""" - - @ignore_this - @_public - @another_decorator - async def public_async_func(self) -> Counter: - pass # no doc string - - def not_public(self): - pass - - async def not_public_async(self): - pass -''' - -IMPORT_1 = """\ -from collections import Counter -""" - -IMPORT_2 = """\ -from collections import Counter -import os -""" - -IMPORT_3 = """\ -from typing import TYPE_CHECKING -if TYPE_CHECKING: - from collections import Counter -""" - - -def test_get_public_methods() -> None: - methods = list(get_public_methods(ast.parse(SOURCE))) - assert {m.name for m in methods} == {"public_func", "public_async_func"} - - -def test_create_pass_through_args() -> None: - testcases = [ - ("def f()", "()"), - ("def f(one)", "(one)"), - ("def f(one, two)", "(one, two)"), - ("def f(one, *args)", "(one, *args)"), - ( - "def f(one, *args, kw1, kw2=None, **kwargs)", - "(one, *args, kw1=kw1, kw2=kw2, **kwargs)", - ), - ] - - for funcdef, expected in testcases: - func_node = ast.parse(funcdef + ":\n pass").body[0] - assert isinstance(func_node, ast.FunctionDef) - assert create_passthrough_args(func_node) == expected - - -skip_lints = pytest.mark.skipif( - sys.implementation.name != "cpython", - reason="gen_exports is internal, black/isort only runs on CPython", -) - - -@slow -@skip_lints -@pytest.mark.parametrize("imports", [IMPORT_1, IMPORT_2, IMPORT_3]) -def test_process( - tmp_path: Path, - imports: str, - capsys: pytest.CaptureFixture[str], -) -> None: - try: - import black # noqa: F401 - # there's no dedicated CI run that has astor+isort, but lacks black. - except ImportError as error: # pragma: no cover - maybe_ignore_import_error(error) - - modpath = tmp_path / "_module.py" - genpath = tmp_path / "_generated_module.py" - modpath.write_text(SOURCE, encoding="utf-8") - file = File(modpath, "runner", platform="linux", imports=imports) - assert not genpath.exists() - with pytest.raises(SystemExit) as excinfo: - process([file], do_test=True) - assert excinfo.value.code == 1 - captured = capsys.readouterr() - assert "Generated sources are outdated. Please regenerate." in captured.out - with pytest.raises(SystemExit) as excinfo: - process([file], do_test=False) - assert excinfo.value.code == 1 - captured = capsys.readouterr() - assert "Regenerated sources successfully." in captured.out - assert genpath.exists() - process([file], do_test=True) - # But if we change the lookup path it notices - with pytest.raises(SystemExit) as excinfo: - process( - [File(modpath, "runner.io_manager", platform="linux", imports=imports)], - do_test=True, - ) - assert excinfo.value.code == 1 - # Also if the platform is changed. - with pytest.raises(SystemExit) as excinfo: - process([File(modpath, "runner", imports=imports)], do_test=True) - assert excinfo.value.code == 1 - - -@skip_lints -def test_run_ruff(tmp_path: Path) -> None: - """Test that processing properly fails if ruff does.""" - try: - import ruff # noqa: F401 - except ImportError as error: # pragma: no cover - maybe_ignore_import_error(error) - - file = File(tmp_path / "module.py", "module") - - success, _ = run_ruff(file, "class not valid code ><") - assert not success - - test_function = '''def combine_and(data: list[str]) -> str: - """Join values of text, and have 'and' with the last one properly.""" - if len(data) >= 2: - data[-1] = 'and ' + data[-1] - if len(data) > 2: - return ', '.join(data) - return ' '.join(data)''' - - success, response = run_ruff(file, test_function) - assert success - assert response == test_function - - -@skip_lints -def test_lint_failure(tmp_path: Path) -> None: - """Test that processing properly fails if black or ruff does.""" - try: - import black # noqa: F401 - import ruff # noqa: F401 - except ImportError as error: # pragma: no cover - maybe_ignore_import_error(error) - - file = File(tmp_path / "module.py", "module") - - with pytest.raises(SystemExit): - run_linters(file, "class not valid code ><") - - with pytest.raises(SystemExit): - run_linters(file, "import waffle\n;import trio") +import ast +import sys +from pathlib import Path + +import pytest + +from trio._tests.pytest_plugin import maybe_ignore_import_error + +# imports in gen_exports that are not in `install_requires` in setup.py +try: + import astor # noqa: F401 + import isort # noqa: F401 +except ImportError as error: + maybe_ignore_import_error(error) + + +from trio._tools.gen_exports import ( + File, + create_passthrough_args, + get_public_methods, + process, + run_linters, + run_ruff, +) + +from ..._core._tests.tutil import slow + +SOURCE = '''from _run import _public +from collections import Counter + +class Test: + @_public + def public_func(self): + """With doc string""" + + @ignore_this + @_public + @another_decorator + async def public_async_func(self) -> Counter: + pass # no doc string + + def not_public(self): + pass + + async def not_public_async(self): + pass +''' + +IMPORT_1 = """\ +from collections import Counter +""" + +IMPORT_2 = """\ +from collections import Counter +import os +""" + +IMPORT_3 = """\ +from typing import TYPE_CHECKING +if TYPE_CHECKING: + from collections import Counter +""" + + +def test_get_public_methods() -> None: + methods = list(get_public_methods(ast.parse(SOURCE))) + assert {m.name for m in methods} == {"public_func", "public_async_func"} + + +def test_create_pass_through_args() -> None: + testcases = [ + ("def f()", "()"), + ("def f(one)", "(one)"), + ("def f(one, two)", "(one, two)"), + ("def f(one, *args)", "(one, *args)"), + ( + "def f(one, *args, kw1, kw2=None, **kwargs)", + "(one, *args, kw1=kw1, kw2=kw2, **kwargs)", + ), + ] + + for funcdef, expected in testcases: + func_node = ast.parse(funcdef + ":\n pass").body[0] + assert isinstance(func_node, ast.FunctionDef) + assert create_passthrough_args(func_node) == expected + + +skip_lints = pytest.mark.skipif( + sys.implementation.name != "cpython", + reason="gen_exports is internal, black/isort only runs on CPython", +) + + +@slow +@skip_lints +@pytest.mark.parametrize("imports", [IMPORT_1, IMPORT_2, IMPORT_3]) +def test_process( + tmp_path: Path, + imports: str, + capsys: pytest.CaptureFixture[str], +) -> None: + try: + import black # noqa: F401 + # there's no dedicated CI run that has astor+isort, but lacks black. + except ImportError as error: # pragma: no cover + maybe_ignore_import_error(error) + + modpath = tmp_path / "_module.py" + genpath = tmp_path / "_generated_module.py" + modpath.write_text(SOURCE, encoding="utf-8") + file = File(modpath, "runner", platform="linux", imports=imports) + assert not genpath.exists() + with pytest.raises(SystemExit) as excinfo: + process([file], do_test=True) + assert excinfo.value.code == 1 + captured = capsys.readouterr() + assert "Generated sources are outdated. Please regenerate." in captured.out + with pytest.raises(SystemExit) as excinfo: + process([file], do_test=False) + assert excinfo.value.code == 1 + captured = capsys.readouterr() + assert "Regenerated sources successfully." in captured.out + assert genpath.exists() + process([file], do_test=True) + # But if we change the lookup path it notices + with pytest.raises(SystemExit) as excinfo: + process( + [File(modpath, "runner.io_manager", platform="linux", imports=imports)], + do_test=True, + ) + assert excinfo.value.code == 1 + # Also if the platform is changed. + with pytest.raises(SystemExit) as excinfo: + process([File(modpath, "runner", imports=imports)], do_test=True) + assert excinfo.value.code == 1 + + +@skip_lints +def test_run_ruff(tmp_path: Path) -> None: + """Test that processing properly fails if ruff does.""" + try: + import ruff # noqa: F401 + except ImportError as error: # pragma: no cover + maybe_ignore_import_error(error) + + file = File(tmp_path / "module.py", "module") + + success, _ = run_ruff(file, "class not valid code ><") + assert not success + + test_function = '''def combine_and(data: list[str]) -> str: + """Join values of text, and have 'and' with the last one properly.""" + if len(data) >= 2: + data[-1] = 'and ' + data[-1] + if len(data) > 2: + return ', '.join(data) + return ' '.join(data)''' + + success, response = run_ruff(file, test_function) + assert success + assert response == test_function + + +@skip_lints +def test_lint_failure(tmp_path: Path) -> None: + """Test that processing properly fails if black or ruff does.""" + try: + import black # noqa: F401 + import ruff # noqa: F401 + except ImportError as error: # pragma: no cover + maybe_ignore_import_error(error) + + file = File(tmp_path / "module.py", "module") + + with pytest.raises(SystemExit): + run_linters(file, "class not valid code ><") + + with pytest.raises(SystemExit): + run_linters(file, "import waffle\n;import trio") diff --git a/src/trio/_tests/tools/test_sync_requirements.py b/src/trio/_tests/tools/test_sync_requirements.py index 5beb707558..544b79bd0f 100644 --- a/src/trio/_tests/tools/test_sync_requirements.py +++ b/src/trio/_tests/tools/test_sync_requirements.py @@ -1,79 +1,79 @@ -from __future__ import annotations - -from typing import TYPE_CHECKING - -from trio._tests.pytest_plugin import maybe_ignore_import_error - -# imports in gen_exports that are not in `install_requires` in requirements -try: - import yaml # noqa: F401 -except ImportError as error: - maybe_ignore_import_error(error) - -from trio._tools.sync_requirements import ( - update_requirements, - yield_pre_commit_version_data, -) - -if TYPE_CHECKING: - from pathlib import Path - - -def test_yield_pre_commit_version_data() -> None: - text = """ -repos: - - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.11.0 - - repo: https://github.com/psf/black-pre-commit-mirror - rev: 25.1.0 - - bad: data -""" - results = tuple(yield_pre_commit_version_data(text)) - assert results == ( - ("ruff-pre-commit", "0.11.0"), - ("black-pre-commit-mirror", "25.1.0"), - ) - - -def test_update_requirements( - tmp_path: Path, -) -> None: - requirements_file = tmp_path / "requirements.txt" - assert not requirements_file.exists() - requirements_file.write_text( - """# comment - # also comment but spaces line start -waffles are delicious no equals -black==3.1.4 ; specific version thingy -mypy==1.15.0 -ruff==1.2.5 -# required by soupy cat""", - encoding="utf-8", - ) - assert update_requirements(requirements_file, {"black": "3.1.5", "ruff": "1.2.7"}) - assert requirements_file.read_text(encoding="utf-8") == """# comment - # also comment but spaces line start -waffles are delicious no equals -black==3.1.5 ; specific version thingy -mypy==1.15.0 -ruff==1.2.7 -# required by soupy cat""" - - -def test_update_requirements_no_changes( - tmp_path: Path, -) -> None: - requirements_file = tmp_path / "requirements.txt" - assert not requirements_file.exists() - original = """# comment - # also comment but spaces line start -waffles are delicious no equals -black==3.1.4 ; specific version thingy -mypy==1.15.0 -ruff==1.2.5 -# required by soupy cat""" - requirements_file.write_text(original, encoding="utf-8") - assert not update_requirements( - requirements_file, {"black": "3.1.4", "ruff": "1.2.5"} - ) - assert requirements_file.read_text(encoding="utf-8") == original +from __future__ import annotations + +from typing import TYPE_CHECKING + +from trio._tests.pytest_plugin import maybe_ignore_import_error + +# imports in gen_exports that are not in `install_requires` in requirements +try: + import yaml # noqa: F401 +except ImportError as error: + maybe_ignore_import_error(error) + +from trio._tools.sync_requirements import ( + update_requirements, + yield_pre_commit_version_data, +) + +if TYPE_CHECKING: + from pathlib import Path + + +def test_yield_pre_commit_version_data() -> None: + text = """ +repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.11.0 + - repo: https://github.com/psf/black-pre-commit-mirror + rev: 25.1.0 + - bad: data +""" + results = tuple(yield_pre_commit_version_data(text)) + assert results == ( + ("ruff-pre-commit", "0.11.0"), + ("black-pre-commit-mirror", "25.1.0"), + ) + + +def test_update_requirements( + tmp_path: Path, +) -> None: + requirements_file = tmp_path / "requirements.txt" + assert not requirements_file.exists() + requirements_file.write_text( + """# comment + # also comment but spaces line start +waffles are delicious no equals +black==3.1.4 ; specific version thingy +mypy==1.15.0 +ruff==1.2.5 +# required by soupy cat""", + encoding="utf-8", + ) + assert update_requirements(requirements_file, {"black": "3.1.5", "ruff": "1.2.7"}) + assert requirements_file.read_text(encoding="utf-8") == """# comment + # also comment but spaces line start +waffles are delicious no equals +black==3.1.5 ; specific version thingy +mypy==1.15.0 +ruff==1.2.7 +# required by soupy cat""" + + +def test_update_requirements_no_changes( + tmp_path: Path, +) -> None: + requirements_file = tmp_path / "requirements.txt" + assert not requirements_file.exists() + original = """# comment + # also comment but spaces line start +waffles are delicious no equals +black==3.1.4 ; specific version thingy +mypy==1.15.0 +ruff==1.2.5 +# required by soupy cat""" + requirements_file.write_text(original, encoding="utf-8") + assert not update_requirements( + requirements_file, {"black": "3.1.4", "ruff": "1.2.5"} + ) + assert requirements_file.read_text(encoding="utf-8") == original diff --git a/test-requirements.txt b/test-requirements.txt index 80c2bf538b..a342bc0b11 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,210 +1,210 @@ -# This file was autogenerated by uv via the following command: -# uv pip compile --universal --python-version=3.10 test-requirements.in -o test-requirements.txt -alabaster==1.0.0 - # via sphinx -ast-serialize==0.8.0 ; implementation_name == 'cpython' - # via mypy -astor==0.8.1 - # via -r test-requirements.in -astroid==4.0.4 - # via pylint -async-generator==1.10 - # via -r test-requirements.in -attrs==26.1.0 - # via - # -r test-requirements.in - # outcome -babel==2.18.0 - # via sphinx -black==26.5.1 ; implementation_name == 'cpython' - # via -r test-requirements.in -certifi==2026.7.22 - # via requests -cffi==2.1.1 ; os_name == 'nt' or platform_python_implementation != 'PyPy' - # via - # -r test-requirements.in - # cryptography -cfgv==3.5.0 - # via pre-commit -charset-normalizer==3.5.1 - # via requests -click==8.5.0 ; implementation_name == 'cpython' - # via black -codespell==2.4.3 - # via -r test-requirements.in -colorama==0.4.6 ; sys_platform == 'win32' - # via - # pylint - # pytest - # sphinx -coverage==7.16.0 - # via -r test-requirements.in -cryptography==50.0.1 - # via - # -r test-requirements.in - # pyopenssl - # trustme - # types-pyopenssl -dill==0.4.1 - # via pylint -distlib==0.4.3 - # via virtualenv -docutils==0.21.2 ; python_full_version < '3.11' - # via sphinx -docutils==0.22.4 ; python_full_version >= '3.11' - # via sphinx -exceptiongroup==1.3.1 ; python_full_version < '3.11' - # via - # -r test-requirements.in - # pytest -filelock==3.32.5 - # via - # python-discovery - # virtualenv -identify==2.6.19 - # via pre-commit -idna==3.19 - # via - # -r test-requirements.in - # requests - # trustme -imagesize==2.0.1 - # via sphinx -iniconfig==2.3.0 - # via pytest -isort==9.0.1 - # via pylint -jedi==0.20.0 ; implementation_name == 'cpython' - # via -r test-requirements.in -jinja2==3.1.6 - # via sphinx -librt==0.15.0 ; implementation_name == 'cpython' and platform_python_implementation != 'PyPy' - # via mypy -markupsafe==3.0.3 - # via jinja2 -mccabe==0.7.0 - # via pylint -mypy==2.3.1 ; implementation_name == 'cpython' - # via -r test-requirements.in -mypy-extensions==1.1.0 - # via - # -r test-requirements.in - # black - # isort - # mypy -nodeenv==1.10.0 - # via - # pre-commit - # pyright -outcome==1.3.0.post0 - # via -r test-requirements.in -packaging==26.3 - # via - # black - # pytest - # sphinx -parso==0.8.7 ; implementation_name == 'cpython' - # via jedi -pathspec==1.1.1 ; implementation_name == 'cpython' - # via - # black - # mypy -platformdirs==4.11.5 - # via - # black - # pylint - # virtualenv -pluggy==1.6.0 - # via pytest -pre-commit==4.6.2 - # via -r test-requirements.in -pycparser==3.0 ; (implementation_name != 'PyPy' and os_name == 'nt') or (implementation_name != 'PyPy' and platform_python_implementation != 'PyPy') - # via cffi -pygments==2.21.0 - # via - # pytest - # sphinx -pylint==4.0.8 - # via -r test-requirements.in -pyopenssl==26.4.0 - # via -r test-requirements.in -pyright==1.1.411 - # via -r test-requirements.in -pytest==9.1.1 - # via -r test-requirements.in -python-discovery==1.6.0 - # via virtualenv -pytokens==0.4.1 ; implementation_name == 'cpython' - # via black -pyyaml==6.0.3 - # via pre-commit -requests==2.34.2 - # via sphinx -roman-numerals==4.1.0 ; python_full_version >= '3.11' - # via sphinx -ruff==0.16.7 - # via -r test-requirements.in -sniffio==1.3.1 - # via -r test-requirements.in -snowballstemmer==3.1.1 - # via sphinx -sortedcontainers==2.4.0 - # via -r test-requirements.in -sphinx==8.1.3 ; python_full_version < '3.11' - # via -r test-requirements.in -sphinx==9.0.4 ; python_full_version == '3.11.*' - # via -r test-requirements.in -sphinx==9.1.0 ; python_full_version >= '3.12' - # via -r test-requirements.in -sphinxcontrib-applehelp==2.0.0 - # via sphinx -sphinxcontrib-devhelp==2.0.0 - # via sphinx -sphinxcontrib-htmlhelp==2.1.0 - # via sphinx -sphinxcontrib-jsmath==1.0.1 - # via sphinx -sphinxcontrib-qthelp==2.0.0 - # via sphinx -sphinxcontrib-serializinghtml==2.0.0 - # via sphinx -tomli==2.4.1 ; python_full_version < '3.11' - # via - # black - # mypy - # pylint - # pytest - # sphinx -tomlkit==0.15.1 - # via pylint -trustme==1.2.1 - # via -r test-requirements.in -types-cffi==2.1.0.20260827 - # via - # -r test-requirements.in - # types-pyopenssl -types-docutils==0.23.0.20260827 - # via -r test-requirements.in -types-pyopenssl==24.1.0.20240722 - # via -r test-requirements.in -types-pyyaml==6.0.12.20260815 - # via -r test-requirements.in -types-setuptools==84.0.0.20260812 - # via types-cffi -typing-extensions==4.16.0 - # via - # -r test-requirements.in - # astroid - # black - # cryptography - # exceptiongroup - # mypy - # pyopenssl - # pyright - # virtualenv -urllib3==2.7.0 - # via requests -uv==0.12.13 - # via -r test-requirements.in -virtualenv==21.7.7 - # via pre-commit +# This file was autogenerated by uv via the following command: +# uv pip compile --universal --python-version=3.10 test-requirements.in -o test-requirements.txt +alabaster==1.0.0 + # via sphinx +ast-serialize==0.8.0 ; implementation_name == 'cpython' + # via mypy +astor==0.8.1 + # via -r test-requirements.in +astroid==4.0.4 + # via pylint +async-generator==1.10 + # via -r test-requirements.in +attrs==26.1.0 + # via + # -r test-requirements.in + # outcome +babel==2.18.0 + # via sphinx +black==26.5.1 ; implementation_name == 'cpython' + # via -r test-requirements.in +certifi==2026.7.22 + # via requests +cffi==2.1.1 ; os_name == 'nt' or platform_python_implementation != 'PyPy' + # via + # -r test-requirements.in + # cryptography +cfgv==3.5.0 + # via pre-commit +charset-normalizer==3.5.1 + # via requests +click==8.5.0 ; implementation_name == 'cpython' + # via black +codespell==2.4.3 + # via -r test-requirements.in +colorama==0.4.6 ; sys_platform == 'win32' + # via + # pylint + # pytest + # sphinx +coverage==7.16.0 + # via -r test-requirements.in +cryptography==50.0.2 + # via + # -r test-requirements.in + # pyopenssl + # trustme + # types-pyopenssl +dill==0.4.1 + # via pylint +distlib==0.4.3 + # via virtualenv +docutils==0.21.2 ; python_full_version < '3.11' + # via sphinx +docutils==0.22.4 ; python_full_version >= '3.11' + # via sphinx +exceptiongroup==1.3.1 ; python_full_version < '3.11' + # via + # -r test-requirements.in + # pytest +filelock==3.32.5 + # via + # python-discovery + # virtualenv +identify==2.6.19 + # via pre-commit +idna==3.19 + # via + # -r test-requirements.in + # requests + # trustme +imagesize==2.0.1 + # via sphinx +iniconfig==2.3.0 + # via pytest +isort==9.0.1 + # via pylint +jedi==0.20.0 ; implementation_name == 'cpython' + # via -r test-requirements.in +jinja2==3.1.6 + # via sphinx +librt==0.15.0 ; implementation_name == 'cpython' and platform_python_implementation != 'PyPy' + # via mypy +markupsafe==3.0.3 + # via jinja2 +mccabe==0.7.0 + # via pylint +mypy==2.3.1 ; implementation_name == 'cpython' + # via -r test-requirements.in +mypy-extensions==1.1.0 + # via + # -r test-requirements.in + # black + # isort + # mypy +nodeenv==1.10.0 + # via + # pre-commit + # pyright +outcome==1.3.0.post0 + # via -r test-requirements.in +packaging==26.3 + # via + # black + # pytest + # sphinx +parso==0.8.7 ; implementation_name == 'cpython' + # via jedi +pathspec==1.1.1 ; implementation_name == 'cpython' + # via + # black + # mypy +platformdirs==4.11.5 + # via + # black + # pylint + # virtualenv +pluggy==1.6.0 + # via pytest +pre-commit==4.6.2 + # via -r test-requirements.in +pycparser==3.0 ; (implementation_name != 'PyPy' and os_name == 'nt') or (implementation_name != 'PyPy' and platform_python_implementation != 'PyPy') + # via cffi +pygments==2.21.0 + # via + # pytest + # sphinx +pylint==4.0.8 + # via -r test-requirements.in +pyopenssl==26.4.0 + # via -r test-requirements.in +pyright==1.1.411 + # via -r test-requirements.in +pytest==9.1.1 + # via -r test-requirements.in +python-discovery==1.6.0 + # via virtualenv +pytokens==0.4.1 ; implementation_name == 'cpython' + # via black +pyyaml==6.0.3 + # via pre-commit +requests==2.34.2 + # via sphinx +roman-numerals==4.1.0 ; python_full_version >= '3.11' + # via sphinx +ruff==0.16.7 + # via -r test-requirements.in +sniffio==1.3.1 + # via -r test-requirements.in +snowballstemmer==3.1.1 + # via sphinx +sortedcontainers==2.4.0 + # via -r test-requirements.in +sphinx==8.1.3 ; python_full_version < '3.11' + # via -r test-requirements.in +sphinx==9.0.4 ; python_full_version == '3.11.*' + # via -r test-requirements.in +sphinx==9.1.0 ; python_full_version >= '3.12' + # via -r test-requirements.in +sphinxcontrib-applehelp==2.0.0 + # via sphinx +sphinxcontrib-devhelp==2.0.0 + # via sphinx +sphinxcontrib-htmlhelp==2.1.0 + # via sphinx +sphinxcontrib-jsmath==1.0.1 + # via sphinx +sphinxcontrib-qthelp==2.0.0 + # via sphinx +sphinxcontrib-serializinghtml==2.0.0 + # via sphinx +tomli==2.4.1 ; python_full_version < '3.11' + # via + # black + # mypy + # pylint + # pytest + # sphinx +tomlkit==0.15.1 + # via pylint +trustme==1.2.1 + # via -r test-requirements.in +types-cffi==2.1.0.20260827 + # via + # -r test-requirements.in + # types-pyopenssl +types-docutils==0.23.0.20260827 + # via -r test-requirements.in +types-pyopenssl==24.1.0.20240722 + # via -r test-requirements.in +types-pyyaml==6.0.12.20260815 + # via -r test-requirements.in +types-setuptools==84.0.0.20260812 + # via types-cffi +typing-extensions==4.16.0 + # via + # -r test-requirements.in + # astroid + # black + # cryptography + # exceptiongroup + # mypy + # pyopenssl + # pyright + # virtualenv +urllib3==2.7.0 + # via requests +uv==0.12.13 + # via -r test-requirements.in +virtualenv==21.7.7 + # via pre-commit From b042f696f898f9eef402cd1136bf3c29b637c64c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 30 Sep 2026 18:23:55 +0000 Subject: [PATCH 19/34] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- test-requirements.txt | 420 +++++++++++++++++++++--------------------- 1 file changed, 210 insertions(+), 210 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index a342bc0b11..3314ac8663 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,210 +1,210 @@ -# This file was autogenerated by uv via the following command: -# uv pip compile --universal --python-version=3.10 test-requirements.in -o test-requirements.txt -alabaster==1.0.0 - # via sphinx -ast-serialize==0.8.0 ; implementation_name == 'cpython' - # via mypy -astor==0.8.1 - # via -r test-requirements.in -astroid==4.0.4 - # via pylint -async-generator==1.10 - # via -r test-requirements.in -attrs==26.1.0 - # via - # -r test-requirements.in - # outcome -babel==2.18.0 - # via sphinx -black==26.5.1 ; implementation_name == 'cpython' - # via -r test-requirements.in -certifi==2026.7.22 - # via requests -cffi==2.1.1 ; os_name == 'nt' or platform_python_implementation != 'PyPy' - # via - # -r test-requirements.in - # cryptography -cfgv==3.5.0 - # via pre-commit -charset-normalizer==3.5.1 - # via requests -click==8.5.0 ; implementation_name == 'cpython' - # via black -codespell==2.4.3 - # via -r test-requirements.in -colorama==0.4.6 ; sys_platform == 'win32' - # via - # pylint - # pytest - # sphinx -coverage==7.16.0 - # via -r test-requirements.in -cryptography==50.0.2 - # via - # -r test-requirements.in - # pyopenssl - # trustme - # types-pyopenssl -dill==0.4.1 - # via pylint -distlib==0.4.3 - # via virtualenv -docutils==0.21.2 ; python_full_version < '3.11' - # via sphinx -docutils==0.22.4 ; python_full_version >= '3.11' - # via sphinx -exceptiongroup==1.3.1 ; python_full_version < '3.11' - # via - # -r test-requirements.in - # pytest -filelock==3.32.5 - # via - # python-discovery - # virtualenv -identify==2.6.19 - # via pre-commit -idna==3.19 - # via - # -r test-requirements.in - # requests - # trustme -imagesize==2.0.1 - # via sphinx -iniconfig==2.3.0 - # via pytest -isort==9.0.1 - # via pylint -jedi==0.20.0 ; implementation_name == 'cpython' - # via -r test-requirements.in -jinja2==3.1.6 - # via sphinx -librt==0.15.0 ; implementation_name == 'cpython' and platform_python_implementation != 'PyPy' - # via mypy -markupsafe==3.0.3 - # via jinja2 -mccabe==0.7.0 - # via pylint -mypy==2.3.1 ; implementation_name == 'cpython' - # via -r test-requirements.in -mypy-extensions==1.1.0 - # via - # -r test-requirements.in - # black - # isort - # mypy -nodeenv==1.10.0 - # via - # pre-commit - # pyright -outcome==1.3.0.post0 - # via -r test-requirements.in -packaging==26.3 - # via - # black - # pytest - # sphinx -parso==0.8.7 ; implementation_name == 'cpython' - # via jedi -pathspec==1.1.1 ; implementation_name == 'cpython' - # via - # black - # mypy -platformdirs==4.11.5 - # via - # black - # pylint - # virtualenv -pluggy==1.6.0 - # via pytest -pre-commit==4.6.2 - # via -r test-requirements.in -pycparser==3.0 ; (implementation_name != 'PyPy' and os_name == 'nt') or (implementation_name != 'PyPy' and platform_python_implementation != 'PyPy') - # via cffi -pygments==2.21.0 - # via - # pytest - # sphinx -pylint==4.0.8 - # via -r test-requirements.in -pyopenssl==26.4.0 - # via -r test-requirements.in -pyright==1.1.411 - # via -r test-requirements.in -pytest==9.1.1 - # via -r test-requirements.in -python-discovery==1.6.0 - # via virtualenv -pytokens==0.4.1 ; implementation_name == 'cpython' - # via black -pyyaml==6.0.3 - # via pre-commit -requests==2.34.2 - # via sphinx -roman-numerals==4.1.0 ; python_full_version >= '3.11' - # via sphinx -ruff==0.16.7 - # via -r test-requirements.in -sniffio==1.3.1 - # via -r test-requirements.in -snowballstemmer==3.1.1 - # via sphinx -sortedcontainers==2.4.0 - # via -r test-requirements.in -sphinx==8.1.3 ; python_full_version < '3.11' - # via -r test-requirements.in -sphinx==9.0.4 ; python_full_version == '3.11.*' - # via -r test-requirements.in -sphinx==9.1.0 ; python_full_version >= '3.12' - # via -r test-requirements.in -sphinxcontrib-applehelp==2.0.0 - # via sphinx -sphinxcontrib-devhelp==2.0.0 - # via sphinx -sphinxcontrib-htmlhelp==2.1.0 - # via sphinx -sphinxcontrib-jsmath==1.0.1 - # via sphinx -sphinxcontrib-qthelp==2.0.0 - # via sphinx -sphinxcontrib-serializinghtml==2.0.0 - # via sphinx -tomli==2.4.1 ; python_full_version < '3.11' - # via - # black - # mypy - # pylint - # pytest - # sphinx -tomlkit==0.15.1 - # via pylint -trustme==1.2.1 - # via -r test-requirements.in -types-cffi==2.1.0.20260827 - # via - # -r test-requirements.in - # types-pyopenssl -types-docutils==0.23.0.20260827 - # via -r test-requirements.in -types-pyopenssl==24.1.0.20240722 - # via -r test-requirements.in -types-pyyaml==6.0.12.20260815 - # via -r test-requirements.in -types-setuptools==84.0.0.20260812 - # via types-cffi -typing-extensions==4.16.0 - # via - # -r test-requirements.in - # astroid - # black - # cryptography - # exceptiongroup - # mypy - # pyopenssl - # pyright - # virtualenv -urllib3==2.7.0 - # via requests -uv==0.12.13 - # via -r test-requirements.in -virtualenv==21.7.7 - # via pre-commit +# This file was autogenerated by uv via the following command: +# uv pip compile --universal --python-version=3.10 test-requirements.in -o test-requirements.txt +alabaster==1.0.0 + # via sphinx +ast-serialize==0.8.0 ; implementation_name == 'cpython' + # via mypy +astor==0.8.1 + # via -r test-requirements.in +astroid==4.0.4 + # via pylint +async-generator==1.10 + # via -r test-requirements.in +attrs==26.1.0 + # via + # -r test-requirements.in + # outcome +babel==2.18.0 + # via sphinx +black==26.5.1 ; implementation_name == 'cpython' + # via -r test-requirements.in +certifi==2026.7.22 + # via requests +cffi==2.1.1 ; os_name == 'nt' or platform_python_implementation != 'PyPy' + # via + # -r test-requirements.in + # cryptography +cfgv==3.5.0 + # via pre-commit +charset-normalizer==3.5.1 + # via requests +click==8.5.0 ; implementation_name == 'cpython' + # via black +codespell==2.4.3 + # via -r test-requirements.in +colorama==0.4.6 ; sys_platform == 'win32' + # via + # pylint + # pytest + # sphinx +coverage==7.16.0 + # via -r test-requirements.in +cryptography==50.0.2 + # via + # -r test-requirements.in + # pyopenssl + # trustme + # types-pyopenssl +dill==0.4.1 + # via pylint +distlib==0.4.3 + # via virtualenv +docutils==0.21.2 ; python_full_version < '3.11' + # via sphinx +docutils==0.22.4 ; python_full_version >= '3.11' + # via sphinx +exceptiongroup==1.3.1 ; python_full_version < '3.11' + # via + # -r test-requirements.in + # pytest +filelock==3.32.5 + # via + # python-discovery + # virtualenv +identify==2.6.19 + # via pre-commit +idna==3.19 + # via + # -r test-requirements.in + # requests + # trustme +imagesize==2.0.1 + # via sphinx +iniconfig==2.3.0 + # via pytest +isort==9.0.1 + # via pylint +jedi==0.20.0 ; implementation_name == 'cpython' + # via -r test-requirements.in +jinja2==3.1.6 + # via sphinx +librt==0.15.0 ; implementation_name == 'cpython' and platform_python_implementation != 'PyPy' + # via mypy +markupsafe==3.0.3 + # via jinja2 +mccabe==0.7.0 + # via pylint +mypy==2.3.1 ; implementation_name == 'cpython' + # via -r test-requirements.in +mypy-extensions==1.1.0 + # via + # -r test-requirements.in + # black + # isort + # mypy +nodeenv==1.10.0 + # via + # pre-commit + # pyright +outcome==1.3.0.post0 + # via -r test-requirements.in +packaging==26.3 + # via + # black + # pytest + # sphinx +parso==0.8.7 ; implementation_name == 'cpython' + # via jedi +pathspec==1.1.1 ; implementation_name == 'cpython' + # via + # black + # mypy +platformdirs==4.11.5 + # via + # black + # pylint + # virtualenv +pluggy==1.6.0 + # via pytest +pre-commit==4.6.2 + # via -r test-requirements.in +pycparser==3.0 ; (implementation_name != 'PyPy' and os_name == 'nt') or (implementation_name != 'PyPy' and platform_python_implementation != 'PyPy') + # via cffi +pygments==2.21.0 + # via + # pytest + # sphinx +pylint==4.0.8 + # via -r test-requirements.in +pyopenssl==26.4.0 + # via -r test-requirements.in +pyright==1.1.411 + # via -r test-requirements.in +pytest==9.1.1 + # via -r test-requirements.in +python-discovery==1.6.0 + # via virtualenv +pytokens==0.4.1 ; implementation_name == 'cpython' + # via black +pyyaml==6.0.3 + # via pre-commit +requests==2.34.2 + # via sphinx +roman-numerals==4.1.0 ; python_full_version >= '3.11' + # via sphinx +ruff==0.16.7 + # via -r test-requirements.in +sniffio==1.3.1 + # via -r test-requirements.in +snowballstemmer==3.1.1 + # via sphinx +sortedcontainers==2.4.0 + # via -r test-requirements.in +sphinx==8.1.3 ; python_full_version < '3.11' + # via -r test-requirements.in +sphinx==9.0.4 ; python_full_version == '3.11.*' + # via -r test-requirements.in +sphinx==9.1.0 ; python_full_version >= '3.12' + # via -r test-requirements.in +sphinxcontrib-applehelp==2.0.0 + # via sphinx +sphinxcontrib-devhelp==2.0.0 + # via sphinx +sphinxcontrib-htmlhelp==2.1.0 + # via sphinx +sphinxcontrib-jsmath==1.0.1 + # via sphinx +sphinxcontrib-qthelp==2.0.0 + # via sphinx +sphinxcontrib-serializinghtml==2.0.0 + # via sphinx +tomli==2.4.1 ; python_full_version < '3.11' + # via + # black + # mypy + # pylint + # pytest + # sphinx +tomlkit==0.15.1 + # via pylint +trustme==1.2.1 + # via -r test-requirements.in +types-cffi==2.1.0.20260827 + # via + # -r test-requirements.in + # types-pyopenssl +types-docutils==0.23.0.20260827 + # via -r test-requirements.in +types-pyopenssl==24.1.0.20240722 + # via -r test-requirements.in +types-pyyaml==6.0.12.20260815 + # via -r test-requirements.in +types-setuptools==84.0.0.20260812 + # via types-cffi +typing-extensions==4.16.0 + # via + # -r test-requirements.in + # astroid + # black + # cryptography + # exceptiongroup + # mypy + # pyopenssl + # pyright + # virtualenv +urllib3==2.7.0 + # via requests +uv==0.12.13 + # via -r test-requirements.in +virtualenv==21.7.7 + # via pre-commit From e71c0dd4540c7a5f0a43ea96e3f53c133375544d Mon Sep 17 00:00:00 2001 From: Jane R Date: Wed, 30 Sep 2026 14:33:36 -0400 Subject: [PATCH 20/34] exclude one more name --- src/trio/_tests/test_exports.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/trio/_tests/test_exports.py b/src/trio/_tests/test_exports.py index 59eac99041..77aead2795 100644 --- a/src/trio/_tests/test_exports.py +++ b/src/trio/_tests/test_exports.py @@ -245,7 +245,7 @@ def no_underscores(symbols: Iterable[str]) -> set[str]: # ignore warnings about deprecated module tests missing_names -= {"tests"} - missing_names -= {"ALG_SET_PUB_KEY"} # https://github.com/pypy/pypy/issues/5594 + missing_names -= {"ALG_SET_PUB_KEY", "WSA_FLAG_NO_HANDLE_INHERIT"} # https://github.com/pypy/pypy/issues/5594 if missing_names: # pragma: no cover print(f"{tool} can't see the following names in {modname}:") From 32b12b036bf751ea2f143bfcd95dc6d00b4a9a00 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 30 Sep 2026 18:34:51 +0000 Subject: [PATCH 21/34] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/trio/_tests/test_exports.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/trio/_tests/test_exports.py b/src/trio/_tests/test_exports.py index 77aead2795..a16556fe93 100644 --- a/src/trio/_tests/test_exports.py +++ b/src/trio/_tests/test_exports.py @@ -245,7 +245,10 @@ def no_underscores(symbols: Iterable[str]) -> set[str]: # ignore warnings about deprecated module tests missing_names -= {"tests"} - missing_names -= {"ALG_SET_PUB_KEY", "WSA_FLAG_NO_HANDLE_INHERIT"} # https://github.com/pypy/pypy/issues/5594 + missing_names -= { + "ALG_SET_PUB_KEY", + "WSA_FLAG_NO_HANDLE_INHERIT", + } # https://github.com/pypy/pypy/issues/5594 if missing_names: # pragma: no cover print(f"{tool} can't see the following names in {modname}:") From e01363ac1562126d132b889c973cbe202ac14949 Mon Sep 17 00:00:00 2001 From: CheViana <656043+CheViana@users.noreply.github.com> Date: Thu, 1 Oct 2026 08:48:13 -0400 Subject: [PATCH 22/34] Revert "merge, bump crypto" This reverts commit fdc413bbc9b3badcfb5204dace50ec02f3522c85. --- .github/workflows/ci.yml | 1016 +++--- ci.sh | 314 +- src/trio/_tests/pytest_plugin.py | 120 +- src/trio/_tests/test_dtls.py | 1904 ++++++------ src/trio/_tests/test_exports.py | 646 +++- src/trio/_tests/test_ssl.py | 2744 ++++++++--------- src/trio/_tests/tools/test_gen_exports.py | 358 +-- .../_tests/tools/test_sync_requirements.py | 158 +- test-requirements.txt | 2 +- 9 files changed, 3950 insertions(+), 3312 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6efe8bc63b..5e2e758ecd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,508 +1,508 @@ -name: CI - -permissions: {} - -on: - push: - branches-ignore: - # these branches always have another event associated - - gh-readonly-queue/** # GitHub's merge queue uses `merge_group` - - autodeps/** # autodeps always makes a PR - - pre-commit-ci-update-config # pre-commit.ci's updates always have a PR - pull_request: - merge_group: - -concurrency: - group: ${{ github.ref }}-${{ github.workflow }}-${{ github.event_name }}${{ github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && format('-{0}', github.sha) || '' }} - cancel-in-progress: true - -env: - dists-artifact-name: python-package-distributions - dist-name: trio - -jobs: - build: - name: 👷 dists - - runs-on: ubuntu-latest - - outputs: - dist-version: ${{ steps.dist-version.outputs.version }} - sdist-artifact-name: ${{ steps.artifact-name.outputs.sdist }} - wheel-artifact-name: ${{ steps.artifact-name.outputs.wheel }} - - steps: - - name: Switch to using Python 3.11 - uses: actions/setup-python@v6 - with: - python-version: 3.11 - - - name: Grab the source from Git - uses: actions/checkout@v7 - with: - persist-credentials: false - - - name: Get the dist version - id: dist-version - run: >- - echo "version=$( - grep ^__version__ src/trio/_version.py - | sed 's#__version__ = "\([^"]\+\)"#\1#' - )" - >> "${GITHUB_OUTPUT}" - - - name: Set the expected dist artifact names - id: artifact-name - run: | - echo "sdist=${DIST_NAME}-${VERSION}.tar.gz" >> "${GITHUB_OUTPUT}" - echo "wheel=${DIST_NAME}-${VERSION}-py3-none-any.whl" >> "${GITHUB_OUTPUT}" - env: - DIST_NAME: ${{ env.dist-name }} - VERSION: ${{ steps.dist-version.outputs.version }} - - - name: Install build - run: python -Im pip install build - - - name: Build dists - run: python -Im build - - name: Verify that the artifacts with expected names got created - run: >- - ls -1 - dist/${SDIST} - dist/${WHEEL} - env: - SDIST: ${{ steps.artifact-name.outputs.sdist }} - WHEEL: ${{ steps.artifact-name.outputs.wheel }} - - name: Store the distribution packages - uses: actions/upload-artifact@v7 - with: - name: ${{ env.dists-artifact-name }} - # NOTE: Exact expected file names are specified here - # NOTE: as a safety measure — if anything weird ends - # NOTE: up being in this dir or not all dists will be - # NOTE: produced, this will fail the workflow. - path: | - dist/${{ steps.artifact-name.outputs.sdist }} - dist/${{ steps.artifact-name.outputs.wheel }} - retention-days: 5 - - smoke-tests: - name: Smoke tests - needs: - - build - - runs-on: ubuntu-latest - - steps: - - name: Switch to using Python 3.11 - uses: actions/setup-python@v6 - with: - python-version: 3.11 - - - name: >- - Smoke-test: - retrieve the project source from an sdist inside the GHA artifact - uses: re-actors/checkout-python-sdist@release/v2 - with: - source-tarball-name: ${{ needs.build.outputs.sdist-artifact-name }} - workflow-artifact-name: ${{ env.dists-artifact-name }} - - - name: >- - Smoke-test: move the sdist-retrieved dir into sdist-src - run: | - mv -v '${{ github.workspace }}' '${{ runner.temp }}/sdist-src' - mkdir -pv '${{ github.workspace }}' - mv -v '${{ runner.temp }}/sdist-src' '${{ github.workspace }}/sdist-src' - shell: bash -eEuo pipefail {0} - - - name: >- - Smoke-test: grab the source from Git into git-src - uses: actions/checkout@v7 - with: - path: git-src - persist-credentials: false - - - name: >- - Smoke-test: install test requirements from the Git repo - run: >- - python -Im - pip install -c test-requirements.txt -r test-requirements.txt - shell: bash -eEuo pipefail {0} - working-directory: git-src - - - name: >- - Smoke-test: collect tests from the Git repo - env: - PYTHONPATH: src/ - run: >- - pytest --collect-only -qq . - | sort - | tee collected-tests - shell: bash -eEuo pipefail {0} - working-directory: git-src - - - name: >- - Smoke-test: collect tests from the sdist tarball - env: - PYTHONPATH: src/ - run: >- - pytest --collect-only -qq . - | sort - | tee collected-tests - shell: bash -eEuo pipefail {0} - working-directory: sdist-src - - - name: >- - Smoke-test: - verify that all the tests from Git are included in the sdist - run: diff --unified sdist-src/collected-tests git-src/collected-tests - shell: bash -eEuo pipefail {0} - - Windows: - name: 'Windows (${{ matrix.python }}, ${{ matrix.arch }}${{ matrix.extra_name }})' - needs: - - build - - timeout-minutes: 20 - runs-on: 'windows-latest' - strategy: - fail-fast: false - matrix: - python: ['3.10', '3.11', '3.12', '3.13', '3.14', '3.14t', '3.15-dev'] - arch: ['x86', 'x64'] - lsp: [''] - lsp_extract_file: [''] - extra_name: [''] - include: - - python: '3.10' - arch: 'x64' - lsp: 'https://raw.githubusercontent.com/python-trio/trio-ci-assets/master/komodia-based-vpn-setup.zip' - lsp_extract_file: 'komodia-based-vpn-setup.exe' - extra_name: ', with Komodia LSP' - - python: '3.10' - arch: 'x64' - lsp: 'https://www.proxifier.com/download/legacy/ProxifierSetup342.exe' - lsp_extract_file: '' - extra_name: ', with IFS LSP' - - python: 'pypy-3.11' - arch: 'x64' - lsp: '' - lsp_extract_file: '' - extra_name: '' - #- python: '3.10' - # arch: 'x64' - # lsp: 'http://download.pctools.com/mirror/updates/9.0.0.2308-SDavfree-lite_en.exe' - # lsp_extract_file: '' - # extra_name: ', with non-IFS LSP' - - continue-on-error: >- - ${{ - ( - endsWith(matrix.python, '-dev') - || endsWith(matrix.python, '-nightly') - ) - && true - || false - }} - steps: - - name: Retrieve the project source from an sdist inside the GHA artifact - uses: re-actors/checkout-python-sdist@release/v2 - with: - source-tarball-name: ${{ needs.build.outputs.sdist-artifact-name }} - workflow-artifact-name: ${{ env.dists-artifact-name }} - - name: Setup Python - uses: actions/setup-python@v6 - with: - python-version: '${{ matrix.python }}' - architecture: '${{ matrix.arch }}' - allow-prereleases: true - check-latest: true - - name: Run tests - run: ./ci.sh - shell: bash - env: - LSP: '${{ matrix.lsp }}' - LSP_EXTRACT_FILE: '${{ matrix.lsp_extract_file }}' - SKIP_SSL_INSTALLATION: "${{ matrix.arch == 'x86' && 1 || 0 }}" - - if: always() - uses: codecov/codecov-action@v7 - with: - name: Windows (${{ matrix.python }}, ${{ matrix.arch }}${{ matrix.extra_name }}) - # multiple flags is marked as an error in codecov UI, but is actually fine - # https://github.com/codecov/feedback/issues/567 - flags: Windows,${{ matrix.python }} - fail_ci_if_error: true - - Ubuntu: - name: 'Ubuntu (${{ matrix.python }}${{ matrix.extra_name }})' - needs: - - build - - timeout-minutes: 10 - runs-on: 'ubuntu-latest' - strategy: - fail-fast: false - matrix: - python: ['pypy-3.11', '3.10', '3.11', '3.12', '3.13', '3.14', '3.14t', '3.15-dev'] - check_formatting: ['0'] - no_test_requirements: ['0'] - extra_name: [''] - include: - - python: '3.13' - check_formatting: '1' - extra_name: ', check formatting' - # separate test run that doesn't install test-requirements.txt - - python: '3.10' - no_test_requirements: '1' - extra_name: ', no test-requirements' - continue-on-error: >- - ${{ - ( - endsWith(matrix.python, '-dev') - || endsWith(matrix.python, '-nightly') - ) - && true - || false - }} - steps: - - name: Retrieve the project source from an sdist inside the GHA artifact - if: matrix.check_formatting != '1' - uses: re-actors/checkout-python-sdist@release/v2 - with: - source-tarball-name: ${{ needs.build.outputs.sdist-artifact-name }} - workflow-artifact-name: ${{ env.dists-artifact-name }} - - name: Grab the source from Git - if: matrix.check_formatting == '1' - uses: actions/checkout@v7 - with: - persist-credentials: false - - name: Setup Python - uses: actions/setup-python@v6 - with: - python-version: '${{ matrix.python }}' - cache: pip - cache-dependency-path: test-requirements.txt - allow-prereleases: true - check-latest: true - - name: Setup minimum Python version - if: matrix.check_formatting == '1' - uses: actions/setup-python@v6 - with: - python-version: '3.10' - cache: pip - cache-dependency-path: test-requirements.txt - allow-prereleases: true - - name: Check Formatting - if: matrix.check_formatting == '1' - run: - python -m pip install tox && - tox -m check - - name: Run tests - if: matrix.check_formatting == '0' - run: ./ci.sh - env: - NO_TEST_REQUIREMENTS: '${{ matrix.no_test_requirements }}' - - if: >- - always() - && matrix.check_formatting != '1' - uses: codecov/codecov-action@v7 - with: - name: Ubuntu (${{ matrix.python }}${{ matrix.extra_name }}) - flags: Ubuntu,${{ matrix.python }} - fail_ci_if_error: true - - macOS: - name: 'macOS (${{ matrix.python }})' - needs: - - build - - timeout-minutes: 15 - runs-on: 'macos-latest' - strategy: - fail-fast: false - matrix: - python: ['pypy-3.11', '3.10', '3.11', '3.12', '3.13', '3.14', '3.14t', '3.15-dev'] - continue-on-error: >- - ${{ - ( - endsWith(matrix.python, '-dev') - || endsWith(matrix.python, '-nightly') - ) - && true - || false - }} - steps: - - name: Retrieve the project source from an sdist inside the GHA artifact - uses: re-actors/checkout-python-sdist@release/v2 - with: - source-tarball-name: ${{ needs.build.outputs.sdist-artifact-name }} - workflow-artifact-name: ${{ env.dists-artifact-name }} - - name: Setup Python - uses: actions/setup-python@v6 - with: - python-version: '${{ matrix.python }}' - cache: pip - cache-dependency-path: test-requirements.txt - allow-prereleases: true - check-latest: true - - name: Run tests - run: ./ci.sh - - if: always() - uses: codecov/codecov-action@v7 - with: - name: macOS (${{ matrix.python }}) - flags: macOS,${{ matrix.python }} - fail_ci_if_error: true - - # run CI on a musl linux - Alpine: - name: "Alpine" - needs: - - build - - runs-on: ubuntu-latest - container: alpine:3 - steps: - - name: Install necessary packages - # can't use setup-python because that python doesn't seem to work; - # `python3-dev` (rather than `python:alpine`) for some ctypes reason, - # `curl`, `gpg`, `git` for codecov-action v4/v5 to work (https://github.com/codecov/codecov-action/issues/1320). - # `nodejs` for pyright (`node-env` pulls in nodejs but that takes a while and can time out the test). - # `perl` for a platform independent `sed -i` alternative - run: apk update && apk add python3-dev bash curl gpg git nodejs perl - - - name: Retrieve the project source from an sdist inside the GHA artifact - # must be after `apk add` because it relies on `bash` existing - uses: re-actors/checkout-python-sdist@release/v2 - with: - source-tarball-name: ${{ needs.build.outputs.sdist-artifact-name }} - workflow-artifact-name: ${{ env.dists-artifact-name }} - - - name: Enter virtual environment - run: python -m venv .venv - - - name: Run tests - run: source .venv/bin/activate && ./ci.sh - - - name: Get Python version for codecov flag - id: get-version - shell: python - run: | - import sys, os - with open(os.environ["GITHUB_OUTPUT"], "a") as f: - f.write("version=" + ".".join(map(str, sys.version_info[:2]))) - f.write("\n") - - - if: always() - uses: codecov/codecov-action@v7 - with: - name: Alpine - flags: Alpine,${{ steps.get-version.outputs.version }} - fail_ci_if_error: true - - Cython: - name: "Cython" - needs: - - build - - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - include: - # Cython 2 supports 3.10 and 3.11 and Cython 3 supports all versions we do, - # so test both the lowest and higher version for both - - python: '3.10' - cython: '<3' - - python: '3.11' - cython: '<3' - # TODO: technically we should pin cython versions - - python: '3.10' - cython: '>=3' - - python: '3.14' - cython: '>=3' - steps: - - name: Retrieve the project source from an sdist inside the GHA artifact - uses: re-actors/checkout-python-sdist@release/v2 - with: - source-tarball-name: ${{ needs.build.outputs.sdist-artifact-name }} - workflow-artifact-name: ${{ env.dists-artifact-name }} - - name: Setup Python - uses: actions/setup-python@v6 - with: - python-version: '${{ matrix.python }}' - cache: pip - # setuptools is needed to get distutils on 3.12, which cythonize requires - - name: install trio and setuptools - run: python -m pip install --upgrade pip . setuptools 'coverage[toml]' - - - name: add cython plugin to the coveragepy config - run: >- - sed -i 's#plugins\s=\s\[\]#plugins = ["Cython.Coverage"]#' - pyproject.toml - - - name: install cython & compile pyx file - env: - CFLAGS: ${{ env.CFLAGS }} -DCYTHON_TRACE_NOGIL=1 - run: | - python -m pip install "cython${{ matrix.cython }}" - cythonize --inplace -X linetrace=True tests/cython/test_cython.pyx - - - name: import & run module - run: coverage run -m tests.cython.run_test_cython - - - name: Get Python version for codecov flag - id: get-version - shell: python - run: | - import sys, os - with open(os.environ["GITHUB_OUTPUT"], "a") as f: - f.write("version=" + ".".join(map(str, sys.version_info[:2]))) - f.write("\n") - - - run: | - coverage combine - coverage report - - - if: always() - uses: codecov/codecov-action@v7 - with: - name: Cython - flags: Cython,${{ steps.get-version.outputs.version }} - fail_ci_if_error: true - - zizmor: - name: Run zizmor 🌈 - runs-on: ubuntu-latest - permissions: - security-events: write # Required for upload-sarif (used by zizmor-action) to upload SARIF files. - steps: - - name: Checkout repository - uses: actions/checkout@v7 - with: - persist-credentials: false - - - name: Run zizmor 🌈 - uses: zizmorcore/zizmor-action@v0.6.1 - - # https://github.com/marketplace/actions/alls-green#why - check: # This job does nothing and is only used for the branch protection - - if: always() - - needs: - - smoke-tests - - Windows - - Ubuntu - - macOS - - Alpine - - Cython - - zizmor - - runs-on: ubuntu-latest - - steps: - - name: Decide whether the needed jobs succeeded or failed - uses: re-actors/alls-green@release/v1 - with: - jobs: ${{ toJSON(needs) }} +name: CI + +permissions: {} + +on: + push: + branches-ignore: + # these branches always have another event associated + - gh-readonly-queue/** # GitHub's merge queue uses `merge_group` + - autodeps/** # autodeps always makes a PR + - pre-commit-ci-update-config # pre-commit.ci's updates always have a PR + pull_request: + merge_group: + +concurrency: + group: ${{ github.ref }}-${{ github.workflow }}-${{ github.event_name }}${{ github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && format('-{0}', github.sha) || '' }} + cancel-in-progress: true + +env: + dists-artifact-name: python-package-distributions + dist-name: trio + +jobs: + build: + name: 👷 dists + + runs-on: ubuntu-latest + + outputs: + dist-version: ${{ steps.dist-version.outputs.version }} + sdist-artifact-name: ${{ steps.artifact-name.outputs.sdist }} + wheel-artifact-name: ${{ steps.artifact-name.outputs.wheel }} + + steps: + - name: Switch to using Python 3.11 + uses: actions/setup-python@v6 + with: + python-version: 3.11 + + - name: Grab the source from Git + uses: actions/checkout@v7 + with: + persist-credentials: false + + - name: Get the dist version + id: dist-version + run: >- + echo "version=$( + grep ^__version__ src/trio/_version.py + | sed 's#__version__ = "\([^"]\+\)"#\1#' + )" + >> "${GITHUB_OUTPUT}" + + - name: Set the expected dist artifact names + id: artifact-name + run: | + echo "sdist=${DIST_NAME}-${VERSION}.tar.gz" >> "${GITHUB_OUTPUT}" + echo "wheel=${DIST_NAME}-${VERSION}-py3-none-any.whl" >> "${GITHUB_OUTPUT}" + env: + DIST_NAME: ${{ env.dist-name }} + VERSION: ${{ steps.dist-version.outputs.version }} + + - name: Install build + run: python -Im pip install build + + - name: Build dists + run: python -Im build + - name: Verify that the artifacts with expected names got created + run: >- + ls -1 + dist/${SDIST} + dist/${WHEEL} + env: + SDIST: ${{ steps.artifact-name.outputs.sdist }} + WHEEL: ${{ steps.artifact-name.outputs.wheel }} + - name: Store the distribution packages + uses: actions/upload-artifact@v7 + with: + name: ${{ env.dists-artifact-name }} + # NOTE: Exact expected file names are specified here + # NOTE: as a safety measure — if anything weird ends + # NOTE: up being in this dir or not all dists will be + # NOTE: produced, this will fail the workflow. + path: | + dist/${{ steps.artifact-name.outputs.sdist }} + dist/${{ steps.artifact-name.outputs.wheel }} + retention-days: 5 + + smoke-tests: + name: Smoke tests + needs: + - build + + runs-on: ubuntu-latest + + steps: + - name: Switch to using Python 3.11 + uses: actions/setup-python@v6 + with: + python-version: 3.11 + + - name: >- + Smoke-test: + retrieve the project source from an sdist inside the GHA artifact + uses: re-actors/checkout-python-sdist@release/v2 + with: + source-tarball-name: ${{ needs.build.outputs.sdist-artifact-name }} + workflow-artifact-name: ${{ env.dists-artifact-name }} + + - name: >- + Smoke-test: move the sdist-retrieved dir into sdist-src + run: | + mv -v '${{ github.workspace }}' '${{ runner.temp }}/sdist-src' + mkdir -pv '${{ github.workspace }}' + mv -v '${{ runner.temp }}/sdist-src' '${{ github.workspace }}/sdist-src' + shell: bash -eEuo pipefail {0} + + - name: >- + Smoke-test: grab the source from Git into git-src + uses: actions/checkout@v7 + with: + path: git-src + persist-credentials: false + + - name: >- + Smoke-test: install test requirements from the Git repo + run: >- + python -Im + pip install -c test-requirements.txt -r test-requirements.txt + shell: bash -eEuo pipefail {0} + working-directory: git-src + + - name: >- + Smoke-test: collect tests from the Git repo + env: + PYTHONPATH: src/ + run: >- + pytest --collect-only -qq . + | sort + | tee collected-tests + shell: bash -eEuo pipefail {0} + working-directory: git-src + + - name: >- + Smoke-test: collect tests from the sdist tarball + env: + PYTHONPATH: src/ + run: >- + pytest --collect-only -qq . + | sort + | tee collected-tests + shell: bash -eEuo pipefail {0} + working-directory: sdist-src + + - name: >- + Smoke-test: + verify that all the tests from Git are included in the sdist + run: diff --unified sdist-src/collected-tests git-src/collected-tests + shell: bash -eEuo pipefail {0} + + Windows: + name: 'Windows (${{ matrix.python }}, ${{ matrix.arch }}${{ matrix.extra_name }})' + needs: + - build + + timeout-minutes: 20 + runs-on: 'windows-latest' + strategy: + fail-fast: false + matrix: + python: ['3.10', '3.11', '3.12', '3.13', '3.14', '3.14t', '3.15-dev'] + arch: ['x86', 'x64'] + lsp: [''] + lsp_extract_file: [''] + extra_name: [''] + include: + - python: '3.10' + arch: 'x64' + lsp: 'https://raw.githubusercontent.com/python-trio/trio-ci-assets/master/komodia-based-vpn-setup.zip' + lsp_extract_file: 'komodia-based-vpn-setup.exe' + extra_name: ', with Komodia LSP' + - python: '3.10' + arch: 'x64' + lsp: 'https://www.proxifier.com/download/legacy/ProxifierSetup342.exe' + lsp_extract_file: '' + extra_name: ', with IFS LSP' + - python: 'pypy-3.11' + arch: 'x64' + lsp: '' + lsp_extract_file: '' + extra_name: '' + #- python: '3.10' + # arch: 'x64' + # lsp: 'http://download.pctools.com/mirror/updates/9.0.0.2308-SDavfree-lite_en.exe' + # lsp_extract_file: '' + # extra_name: ', with non-IFS LSP' + + continue-on-error: >- + ${{ + ( + endsWith(matrix.python, '-dev') + || endsWith(matrix.python, '-nightly') + ) + && true + || false + }} + steps: + - name: Retrieve the project source from an sdist inside the GHA artifact + uses: re-actors/checkout-python-sdist@release/v2 + with: + source-tarball-name: ${{ needs.build.outputs.sdist-artifact-name }} + workflow-artifact-name: ${{ env.dists-artifact-name }} + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: '${{ matrix.python }}' + architecture: '${{ matrix.arch }}' + allow-prereleases: true + check-latest: true + - name: Run tests + run: ./ci.sh + shell: bash + env: + LSP: '${{ matrix.lsp }}' + LSP_EXTRACT_FILE: '${{ matrix.lsp_extract_file }}' + SKIP_SSL_INSTALLATION: "${{ matrix.arch == 'x86' && 1 || 0 }}" + - if: always() + uses: codecov/codecov-action@v7 + with: + name: Windows (${{ matrix.python }}, ${{ matrix.arch }}${{ matrix.extra_name }}) + # multiple flags is marked as an error in codecov UI, but is actually fine + # https://github.com/codecov/feedback/issues/567 + flags: Windows,${{ matrix.python }} + fail_ci_if_error: true + + Ubuntu: + name: 'Ubuntu (${{ matrix.python }}${{ matrix.extra_name }})' + needs: + - build + + timeout-minutes: 10 + runs-on: 'ubuntu-latest' + strategy: + fail-fast: false + matrix: + python: ['pypy-3.11', '3.10', '3.11', '3.12', '3.13', '3.14', '3.14t', '3.15-dev'] + check_formatting: ['0'] + no_test_requirements: ['0'] + extra_name: [''] + include: + - python: '3.13' + check_formatting: '1' + extra_name: ', check formatting' + # separate test run that doesn't install test-requirements.txt + - python: '3.10' + no_test_requirements: '1' + extra_name: ', no test-requirements' + continue-on-error: >- + ${{ + ( + endsWith(matrix.python, '-dev') + || endsWith(matrix.python, '-nightly') + ) + && true + || false + }} + steps: + - name: Retrieve the project source from an sdist inside the GHA artifact + if: matrix.check_formatting != '1' + uses: re-actors/checkout-python-sdist@release/v2 + with: + source-tarball-name: ${{ needs.build.outputs.sdist-artifact-name }} + workflow-artifact-name: ${{ env.dists-artifact-name }} + - name: Grab the source from Git + if: matrix.check_formatting == '1' + uses: actions/checkout@v7 + with: + persist-credentials: false + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: '${{ matrix.python }}' + cache: pip + cache-dependency-path: test-requirements.txt + allow-prereleases: true + check-latest: true + - name: Setup minimum Python version + if: matrix.check_formatting == '1' + uses: actions/setup-python@v6 + with: + python-version: '3.10' + cache: pip + cache-dependency-path: test-requirements.txt + allow-prereleases: true + - name: Check Formatting + if: matrix.check_formatting == '1' + run: + python -m pip install tox && + tox -m check + - name: Run tests + if: matrix.check_formatting == '0' + run: ./ci.sh + env: + NO_TEST_REQUIREMENTS: '${{ matrix.no_test_requirements }}' + - if: >- + always() + && matrix.check_formatting != '1' + uses: codecov/codecov-action@v7 + with: + name: Ubuntu (${{ matrix.python }}${{ matrix.extra_name }}) + flags: Ubuntu,${{ matrix.python }} + fail_ci_if_error: true + + macOS: + name: 'macOS (${{ matrix.python }})' + needs: + - build + + timeout-minutes: 15 + runs-on: 'macos-latest' + strategy: + fail-fast: false + matrix: + python: ['pypy-3.11', '3.10', '3.11', '3.12', '3.13', '3.14', '3.14t', '3.15-dev'] + continue-on-error: >- + ${{ + ( + endsWith(matrix.python, '-dev') + || endsWith(matrix.python, '-nightly') + ) + && true + || false + }} + steps: + - name: Retrieve the project source from an sdist inside the GHA artifact + uses: re-actors/checkout-python-sdist@release/v2 + with: + source-tarball-name: ${{ needs.build.outputs.sdist-artifact-name }} + workflow-artifact-name: ${{ env.dists-artifact-name }} + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: '${{ matrix.python }}' + cache: pip + cache-dependency-path: test-requirements.txt + allow-prereleases: true + check-latest: true + - name: Run tests + run: ./ci.sh + - if: always() + uses: codecov/codecov-action@v7 + with: + name: macOS (${{ matrix.python }}) + flags: macOS,${{ matrix.python }} + fail_ci_if_error: true + + # run CI on a musl linux + Alpine: + name: "Alpine" + needs: + - build + + runs-on: ubuntu-latest + container: alpine:3 + steps: + - name: Install necessary packages + # can't use setup-python because that python doesn't seem to work; + # `python3-dev` (rather than `python:alpine`) for some ctypes reason, + # `curl`, `gpg`, `git` for codecov-action v4/v5 to work (https://github.com/codecov/codecov-action/issues/1320). + # `nodejs` for pyright (`node-env` pulls in nodejs but that takes a while and can time out the test). + # `perl` for a platform independent `sed -i` alternative + run: apk update && apk add python3-dev bash curl gpg git nodejs perl + + - name: Retrieve the project source from an sdist inside the GHA artifact + # must be after `apk add` because it relies on `bash` existing + uses: re-actors/checkout-python-sdist@release/v2 + with: + source-tarball-name: ${{ needs.build.outputs.sdist-artifact-name }} + workflow-artifact-name: ${{ env.dists-artifact-name }} + + - name: Enter virtual environment + run: python -m venv .venv + + - name: Run tests + run: source .venv/bin/activate && ./ci.sh + + - name: Get Python version for codecov flag + id: get-version + shell: python + run: | + import sys, os + with open(os.environ["GITHUB_OUTPUT"], "a") as f: + f.write("version=" + ".".join(map(str, sys.version_info[:2]))) + f.write("\n") + + - if: always() + uses: codecov/codecov-action@v7 + with: + name: Alpine + flags: Alpine,${{ steps.get-version.outputs.version }} + fail_ci_if_error: true + + Cython: + name: "Cython" + needs: + - build + + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + # Cython 2 supports 3.10 and 3.11 and Cython 3 supports all versions we do, + # so test both the lowest and higher version for both + - python: '3.10' + cython: '<3' + - python: '3.11' + cython: '<3' + # TODO: technically we should pin cython versions + - python: '3.10' + cython: '>=3' + - python: '3.14' + cython: '>=3' + steps: + - name: Retrieve the project source from an sdist inside the GHA artifact + uses: re-actors/checkout-python-sdist@release/v2 + with: + source-tarball-name: ${{ needs.build.outputs.sdist-artifact-name }} + workflow-artifact-name: ${{ env.dists-artifact-name }} + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: '${{ matrix.python }}' + cache: pip + # setuptools is needed to get distutils on 3.12, which cythonize requires + - name: install trio and setuptools + run: python -m pip install --upgrade pip . setuptools 'coverage[toml]' + + - name: add cython plugin to the coveragepy config + run: >- + sed -i 's#plugins\s=\s\[\]#plugins = ["Cython.Coverage"]#' + pyproject.toml + + - name: install cython & compile pyx file + env: + CFLAGS: ${{ env.CFLAGS }} -DCYTHON_TRACE_NOGIL=1 + run: | + python -m pip install "cython${{ matrix.cython }}" + cythonize --inplace -X linetrace=True tests/cython/test_cython.pyx + + - name: import & run module + run: coverage run -m tests.cython.run_test_cython + + - name: Get Python version for codecov flag + id: get-version + shell: python + run: | + import sys, os + with open(os.environ["GITHUB_OUTPUT"], "a") as f: + f.write("version=" + ".".join(map(str, sys.version_info[:2]))) + f.write("\n") + + - run: | + coverage combine + coverage report + + - if: always() + uses: codecov/codecov-action@v7 + with: + name: Cython + flags: Cython,${{ steps.get-version.outputs.version }} + fail_ci_if_error: true + + zizmor: + name: Run zizmor 🌈 + runs-on: ubuntu-latest + permissions: + security-events: write # Required for upload-sarif (used by zizmor-action) to upload SARIF files. + steps: + - name: Checkout repository + uses: actions/checkout@v7 + with: + persist-credentials: false + + - name: Run zizmor 🌈 + uses: zizmorcore/zizmor-action@v0.6.1 + + # https://github.com/marketplace/actions/alls-green#why + check: # This job does nothing and is only used for the branch protection + + if: always() + + needs: + - smoke-tests + - Windows + - Ubuntu + - macOS + - Alpine + - Cython + - zizmor + + runs-on: ubuntu-latest + + steps: + - name: Decide whether the needed jobs succeeded or failed + uses: re-actors/alls-green@release/v1 + with: + jobs: ${{ toJSON(needs) }} diff --git a/ci.sh b/ci.sh index bd86bf5454..1ea2c49219 100755 --- a/ci.sh +++ b/ci.sh @@ -1,157 +1,157 @@ -#!/bin/bash - -set -ex -o pipefail - -# disable warnings about pyright being out of date -# used in test_exports and in check.sh -export PYRIGHT_PYTHON_IGNORE_WARNINGS=1 - -# Log some general info about the environment -echo "::group::Environment" -uname -a -env | sort -echo "::endgroup::" - -# Curl's built-in retry system is not very robust; it gives up on lots of -# network errors that we want to retry on. Wget might work better, but it's -# not installed on azure pipelines's windows boxes. So... let's try some good -# old-fashioned brute force. (This is also a convenient place to put options -# we always want, like -f to tell curl to give an error if the server sends an -# error response, and -L to follow redirects.) -function curl-harder() { - for BACKOFF in 0 1 2 4 8 15 15 15 15; do - sleep $BACKOFF - if curl -fL --connect-timeout 5 "$@"; then - return 0 - fi - done - return 1 -} - -################################################################ -# We have a Python environment! -################################################################ - -echo "::group::Versions" -python -c "import sys, struct, ssl; print('python:', sys.version); print('version_info:', sys.version_info); print('bits:', struct.calcsize('P') * 8); print('openssl:', ssl.OPENSSL_VERSION, ssl.OPENSSL_VERSION_INFO)" -echo "::endgroup::" - -echo "::group::Install dependencies" -python -m pip install -U pip uv -c test-requirements.txt -python -m pip --version -python -m uv --version - -python -m uv pip install build - -python -m build -wheel_package=$(ls dist/*.whl) -python -m uv pip install "trio @ $wheel_package" -c test-requirements.txt - -# Actual tests -# expands to 0 != 1 if NO_TEST_REQUIREMENTS is not set, if set the `-0` has no effect -# https://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html#tag_18_06_02 -if [ "${NO_TEST_REQUIREMENTS-0}" == 1 ]; then - python -m uv pip install pytest coverage -c test-requirements.txt - flags="--skip-optional-imports" -else - flags="" - if [ "${SKIP_SSL_INSTALLATION-0}" == 1 ]; then - echo cryptography >excluded_deps - PIP_ARGS="--exclude excluded_deps" - flags="--skip-ssl-imports" - fi - python -m uv pip install -r test-requirements.txt -v --no-deps $PIP_ARGS -fi - -# If we're testing with a LSP installed, then it might break network -# stuff, so wait until after we've finished setting everything else -# up. -if [ "$LSP" != "" ]; then - echo "Installing LSP from ${LSP}" - # We use --insecure because one of the LSP's has been observed to give - # cert verification errors: - # - # https://github.com/python-trio/trio/issues/1478 - # - # *Normally*, you should never ever use --insecure, especially when - # fetching an executable! But *in this case*, we're intentionally - # installing some untrustworthy quasi-malware onto into a sandboxed - # machine for testing. So MITM attacks are really the least of our - # worries. - if [ "$LSP_EXTRACT_FILE" != "" ]; then - # We host the Astrill VPN installer ourselves, and encrypt it - # so as to decrease the chances of becoming an inadvertent - # public redistributor. - curl-harder -o lsp-installer.zip "$LSP" - unzip -P "not very secret trio ci key" lsp-installer.zip "$LSP_EXTRACT_FILE" - mv "$LSP_EXTRACT_FILE" lsp-installer.exe - else - curl-harder --insecure -o lsp-installer.exe "$LSP" - fi - # This is only needed for the Astrill LSP, but there's no harm in - # doing it all the time. The cert was manually extracted by installing - # the package in a VM, clicking "Always trust from this publisher" - # when installing, and then running 'certmgr.msc' and exporting the - # certificate. See: - # http://www.migee.com/2010/09/24/solution-for-unattendedsilent-installs-and-would-you-like-to-install-this-device-software/ - certutil -addstore "TrustedPublisher" src/trio/_tests/astrill-codesigning-cert.cer - # Double-slashes are how you tell windows-bash that you want a single - # slash, and don't treat this as a unix-style filename that needs to - # be replaced by a windows-style filename. - # http://www.mingw.org/wiki/Posix_path_conversion - ./lsp-installer.exe //silent //norestart - echo "Waiting for LSP to appear in Winsock catalog" - while ! netsh winsock show catalog | grep "Layered Chain Entry"; do - sleep 1 - done - netsh winsock show catalog -fi -echo "::endgroup::" - -echo "::group::Setup for tests" - -# We run the tests from inside an empty directory, to make sure Python -# doesn't pick up any .py files from our working dir. Might have already -# been created by a previous run. -mkdir empty || true -cd empty - -INSTALLDIR=$(python -c "import os, trio; print(os.path.dirname(trio.__file__))") -cp ../pyproject.toml "$INSTALLDIR" # TODO: remove this - -# get mypy tests a nice cache -MYPYPATH=".." mypy --config-file= --cache-dir=./.mypy_cache -c "import trio" >/dev/null 2>/dev/null || true - -# support subprocess spawning with coverage.py -echo "import coverage; coverage.process_startup()" | tee -a "$INSTALLDIR/../sitecustomize.py" - -perl -i -pe 's/-p trio\._tests\.pytest_plugin//' "$INSTALLDIR/pyproject.toml" - -echo "::endgroup::" -echo "::group:: Run Tests" -if PYTHONPATH=../tests COVERAGE_PROCESS_START=$(pwd)/../pyproject.toml \ - coverage run --rcfile=../pyproject.toml -m \ - pytest -ra --junitxml=../test-results.xml \ - -p _trio_check_attrs_aliases --verbose --durations=10 \ - -p trio._tests.pytest_plugin --run-slow $flags "${INSTALLDIR}"; then - PASSED=true -else - PASSED=false -fi -echo "::endgroup::" -echo "::group::Coverage" - -coverage combine --rcfile ../pyproject.toml -cd .. # coverage needs to be in the folder containing src/trio -cp empty/.coverage . -coverage report -m --rcfile ./pyproject.toml -coverage xml --rcfile ./pyproject.toml - -# Remove the LSP again; again we want to do this ASAP to avoid -# accidentally breaking other stuff. -if [ "$LSP" != "" ]; then - netsh winsock reset -fi - -echo "::endgroup::" -$PASSED +#!/bin/bash + +set -ex -o pipefail + +# disable warnings about pyright being out of date +# used in test_exports and in check.sh +export PYRIGHT_PYTHON_IGNORE_WARNINGS=1 + +# Log some general info about the environment +echo "::group::Environment" +uname -a +env | sort +echo "::endgroup::" + +# Curl's built-in retry system is not very robust; it gives up on lots of +# network errors that we want to retry on. Wget might work better, but it's +# not installed on azure pipelines's windows boxes. So... let's try some good +# old-fashioned brute force. (This is also a convenient place to put options +# we always want, like -f to tell curl to give an error if the server sends an +# error response, and -L to follow redirects.) +function curl-harder() { + for BACKOFF in 0 1 2 4 8 15 15 15 15; do + sleep $BACKOFF + if curl -fL --connect-timeout 5 "$@"; then + return 0 + fi + done + return 1 +} + +################################################################ +# We have a Python environment! +################################################################ + +echo "::group::Versions" +python -c "import sys, struct, ssl; print('python:', sys.version); print('version_info:', sys.version_info); print('bits:', struct.calcsize('P') * 8); print('openssl:', ssl.OPENSSL_VERSION, ssl.OPENSSL_VERSION_INFO)" +echo "::endgroup::" + +echo "::group::Install dependencies" +python -m pip install -U pip uv -c test-requirements.txt +python -m pip --version +python -m uv --version + +python -m uv pip install build + +python -m build +wheel_package=$(ls dist/*.whl) +python -m uv pip install "trio @ $wheel_package" -c test-requirements.txt + +# Actual tests +# expands to 0 != 1 if NO_TEST_REQUIREMENTS is not set, if set the `-0` has no effect +# https://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html#tag_18_06_02 +if [ "${NO_TEST_REQUIREMENTS-0}" == 1 ]; then + python -m uv pip install pytest coverage -c test-requirements.txt + flags="--skip-optional-imports" +else + flags="" + if [ "${SKIP_SSL_INSTALLATION-0}" == 1 ]; then + echo cryptography >excluded_deps + PIP_ARGS="--exclude excluded_deps" + flags="--skip-ssl-imports" + fi + python -m uv pip install -r test-requirements.txt -vvv --no-deps $PIP_ARGS +fi + +# If we're testing with a LSP installed, then it might break network +# stuff, so wait until after we've finished setting everything else +# up. +if [ "$LSP" != "" ]; then + echo "Installing LSP from ${LSP}" + # We use --insecure because one of the LSP's has been observed to give + # cert verification errors: + # + # https://github.com/python-trio/trio/issues/1478 + # + # *Normally*, you should never ever use --insecure, especially when + # fetching an executable! But *in this case*, we're intentionally + # installing some untrustworthy quasi-malware onto into a sandboxed + # machine for testing. So MITM attacks are really the least of our + # worries. + if [ "$LSP_EXTRACT_FILE" != "" ]; then + # We host the Astrill VPN installer ourselves, and encrypt it + # so as to decrease the chances of becoming an inadvertent + # public redistributor. + curl-harder -o lsp-installer.zip "$LSP" + unzip -P "not very secret trio ci key" lsp-installer.zip "$LSP_EXTRACT_FILE" + mv "$LSP_EXTRACT_FILE" lsp-installer.exe + else + curl-harder --insecure -o lsp-installer.exe "$LSP" + fi + # This is only needed for the Astrill LSP, but there's no harm in + # doing it all the time. The cert was manually extracted by installing + # the package in a VM, clicking "Always trust from this publisher" + # when installing, and then running 'certmgr.msc' and exporting the + # certificate. See: + # http://www.migee.com/2010/09/24/solution-for-unattendedsilent-installs-and-would-you-like-to-install-this-device-software/ + certutil -addstore "TrustedPublisher" src/trio/_tests/astrill-codesigning-cert.cer + # Double-slashes are how you tell windows-bash that you want a single + # slash, and don't treat this as a unix-style filename that needs to + # be replaced by a windows-style filename. + # http://www.mingw.org/wiki/Posix_path_conversion + ./lsp-installer.exe //silent //norestart + echo "Waiting for LSP to appear in Winsock catalog" + while ! netsh winsock show catalog | grep "Layered Chain Entry"; do + sleep 1 + done + netsh winsock show catalog +fi +echo "::endgroup::" + +echo "::group::Setup for tests" + +# We run the tests from inside an empty directory, to make sure Python +# doesn't pick up any .py files from our working dir. Might have already +# been created by a previous run. +mkdir empty || true +cd empty + +INSTALLDIR=$(python -c "import os, trio; print(os.path.dirname(trio.__file__))") +cp ../pyproject.toml "$INSTALLDIR" # TODO: remove this + +# get mypy tests a nice cache +MYPYPATH=".." mypy --config-file= --cache-dir=./.mypy_cache -c "import trio" >/dev/null 2>/dev/null || true + +# support subprocess spawning with coverage.py +echo "import coverage; coverage.process_startup()" | tee -a "$INSTALLDIR/../sitecustomize.py" + +perl -i -pe 's/-p trio\._tests\.pytest_plugin//' "$INSTALLDIR/pyproject.toml" + +echo "::endgroup::" +echo "::group:: Run Tests" +if PYTHONPATH=../tests COVERAGE_PROCESS_START=$(pwd)/../pyproject.toml \ + coverage run --rcfile=../pyproject.toml -m \ + pytest -ra --junitxml=../test-results.xml \ + -p _trio_check_attrs_aliases --verbose --durations=10 \ + -p trio._tests.pytest_plugin --run-slow $flags "${INSTALLDIR}"; then + PASSED=true +else + PASSED=false +fi +echo "::endgroup::" +echo "::group::Coverage" + +coverage combine --rcfile ../pyproject.toml +cd .. # coverage needs to be in the folder containing src/trio +cp empty/.coverage . +coverage report -m --rcfile ./pyproject.toml +coverage xml --rcfile ./pyproject.toml + +# Remove the LSP again; again we want to do this ASAP to avoid +# accidentally breaking other stuff. +if [ "$LSP" != "" ]; then + netsh winsock reset +fi + +echo "::endgroup::" +$PASSED diff --git a/src/trio/_tests/pytest_plugin.py b/src/trio/_tests/pytest_plugin.py index a67b8a85a6..b929ae3a73 100644 --- a/src/trio/_tests/pytest_plugin.py +++ b/src/trio/_tests/pytest_plugin.py @@ -1,60 +1,60 @@ -from __future__ import annotations - -import inspect -from typing import NoReturn - -import pytest - -from ..testing import MockClock, trio_test - -RUN_SLOW = True -SKIP_OPTIONAL_IMPORTS = False -SKIP_SSL_IMPORTS = False - - -def pytest_addoption(parser: pytest.Parser) -> None: - parser.addoption("--run-slow", action="store_true", help="run slow tests") - parser.addoption( - "--skip-optional-imports", - action="store_true", - help="skip tests that rely on libraries not required by trio itself", - ) - parser.addoption( - "--skip-ssl-imports", - action="store_true", - help="skip tests that rely on cryptography", - ) - - -def pytest_configure(config: pytest.Config) -> None: - global RUN_SLOW, SKIP_OPTIONAL_IMPORTS, SKIP_SSL_IMPORTS - RUN_SLOW = config.getoption("--run-slow", default=True) - SKIP_OPTIONAL_IMPORTS = config.getoption("--skip-optional-imports", default=False) - SKIP_SSL_IMPORTS = config.getoption("--skip-ssl-imports", default=False) - - -@pytest.fixture -def mock_clock() -> MockClock: - return MockClock() - - -@pytest.fixture -def autojump_clock() -> MockClock: - return MockClock(autojump_threshold=0) - - -# FIXME: split off into a package (or just make part of Trio's public -# interface?), with config file to enable? and I guess a mark option too; I -# guess it's useful with the class- and file-level marking machinery (where -# the raw @trio_test decorator isn't enough). -@pytest.hookimpl(tryfirst=True) -def pytest_pyfunc_call(pyfuncitem: pytest.Function) -> None: - if inspect.iscoroutinefunction(pyfuncitem.obj): - pyfuncitem.obj = trio_test(pyfuncitem.obj) - - -def maybe_ignore_import_error(error: ImportError) -> NoReturn: - if SKIP_OPTIONAL_IMPORTS or (SKIP_SSL_IMPORTS and error.name == "cryptography"): - pytest.skip(error.msg, allow_module_level=True) - else: # pragma: no cover - raise error +from __future__ import annotations + +import inspect +from typing import NoReturn + +import pytest + +from ..testing import MockClock, trio_test + +RUN_SLOW = True +SKIP_OPTIONAL_IMPORTS = False +SKIP_SSL_IMPORTS = False + + +def pytest_addoption(parser: pytest.Parser) -> None: + parser.addoption("--run-slow", action="store_true", help="run slow tests") + parser.addoption( + "--skip-optional-imports", + action="store_true", + help="skip tests that rely on libraries not required by trio itself", + ) + parser.addoption( + "--skip-ssl-imports", + action="store_true", + help="skip tests that rely on cryptography", + ) + + +def pytest_configure(config: pytest.Config) -> None: + global RUN_SLOW, SKIP_OPTIONAL_IMPORTS, SKIP_SSL_IMPORTS + RUN_SLOW = config.getoption("--run-slow", default=True) + SKIP_OPTIONAL_IMPORTS = config.getoption("--skip-optional-imports", default=False) + SKIP_SSL_IMPORTS = config.getoption("--skip-ssl-imports", default=False) + + +@pytest.fixture +def mock_clock() -> MockClock: + return MockClock() + + +@pytest.fixture +def autojump_clock() -> MockClock: + return MockClock(autojump_threshold=0) + + +# FIXME: split off into a package (or just make part of Trio's public +# interface?), with config file to enable? and I guess a mark option too; I +# guess it's useful with the class- and file-level marking machinery (where +# the raw @trio_test decorator isn't enough). +@pytest.hookimpl(tryfirst=True) +def pytest_pyfunc_call(pyfuncitem: pytest.Function) -> None: + if inspect.iscoroutinefunction(pyfuncitem.obj): + pyfuncitem.obj = trio_test(pyfuncitem.obj) + + +def maybe_ignore_import_error(error: ImportError) -> NoReturn: + if SKIP_OPTIONAL_IMPORTS or (SKIP_SSL_IMPORTS and error.name == "cryptography"): + pytest.skip(error.msg, allow_module_level=True) + else: # pragma: no cover + raise error diff --git a/src/trio/_tests/test_dtls.py b/src/trio/_tests/test_dtls.py index 7b3bc205c2..5a34d8c811 100644 --- a/src/trio/_tests/test_dtls.py +++ b/src/trio/_tests/test_dtls.py @@ -1,952 +1,952 @@ -from __future__ import annotations - -import random -from contextlib import asynccontextmanager -from itertools import count -from typing import TYPE_CHECKING, NoReturn - -import attrs -import pytest - -from trio._tests.pytest_plugin import maybe_ignore_import_error - -try: - import trustme - from OpenSSL import SSL -except ImportError as error: - maybe_ignore_import_error(error) - - -import trio -import trio.testing -from trio import DTLSChannel, DTLSEndpoint -from trio.testing._fake_net import FakeNet, UDPPacket - -from .._core._tests.tutil import binds_ipv6, gc_collect_harder, slow - -if TYPE_CHECKING: - from collections.abc import AsyncGenerator - -ca = trustme.CA() -server_cert = ca.issue_cert("example.com") - - -@pytest.fixture -def server_ctx() -> SSL.Context: - ctx = SSL.Context(SSL.DTLS_METHOD) - server_cert.configure_cert(ctx) - return ctx - - -def client_ctx_fn() -> SSL.Context: - ctx = SSL.Context(SSL.DTLS_METHOD) - ca.configure_trust(ctx) - return ctx - - -@pytest.fixture -def client_ctx() -> SSL.Context: - return client_ctx_fn() - - -parametrize_ipv6 = pytest.mark.parametrize( - "ipv6", - [False, pytest.param(True, marks=binds_ipv6)], - ids=["ipv4", "ipv6"], -) - - -def endpoint(**kwargs: int | bool) -> DTLSEndpoint: - ipv6 = kwargs.pop("ipv6", False) - family = trio.socket.AF_INET6 if ipv6 else trio.socket.AF_INET - sock = trio.socket.socket(type=trio.socket.SOCK_DGRAM, family=family) - return DTLSEndpoint(sock, **kwargs) - - -@asynccontextmanager -async def dtls_echo_server( - *, - server_ctx: SSL.Context, - autocancel: bool = True, - mtu: int | None = None, - ipv6: bool = False, -) -> AsyncGenerator[tuple[DTLSEndpoint, tuple[str, int]], None]: - with endpoint(ipv6=ipv6) as server: - localhost = "::1" if ipv6 else "127.0.0.1" - await server.socket.bind((localhost, 0)) - async with trio.open_nursery() as nursery: - - async def echo_handler(dtls_channel: DTLSChannel) -> None: - print( - "echo handler started: " - f"server {dtls_channel.endpoint.socket.getsockname()!r} " - f"client {dtls_channel.peer_address!r}", - ) - if mtu is not None: - dtls_channel.set_ciphertext_mtu(mtu) - try: - print("server starting do_handshake") - await dtls_channel.do_handshake() - print("server finished do_handshake") - # no branch for leaving this for loop because we only leave - # a channel by cancellation. - async for packet in dtls_channel: # pragma: no branch - print(f"echoing {packet!r} -> {dtls_channel.peer_address!r}") - await dtls_channel.send(packet) - except trio.BrokenResourceError: # pragma: no cover - print("echo handler channel broken") - - await nursery.start(server.serve, server_ctx, echo_handler) - - yield server, server.socket.getsockname() # noqa: RUF075 - - if autocancel: - nursery.cancel_scope.cancel() - - -@parametrize_ipv6 -async def test_smoke( - ipv6: bool, server_ctx: SSL.Context, client_ctx: SSL.Context -) -> None: - async with dtls_echo_server(ipv6=ipv6, server_ctx=server_ctx) as ( - _server_endpoint, - address, - ): - with endpoint(ipv6=ipv6) as client_endpoint: - client_channel = client_endpoint.connect(address, client_ctx) - with pytest.raises(trio.NeedHandshakeError): - client_channel.get_cleartext_mtu() - - await client_channel.do_handshake() - await client_channel.send(b"hello") - assert await client_channel.receive() == b"hello" - await client_channel.send(b"goodbye") - assert await client_channel.receive() == b"goodbye" - - with pytest.raises( - ValueError, - match=r"^openssl doesn't support sending empty DTLS packets$", - ): - await client_channel.send(b"") - - client_channel.set_ciphertext_mtu(1234) - cleartext_mtu_1234 = client_channel.get_cleartext_mtu() - client_channel.set_ciphertext_mtu(4321) - assert client_channel.get_cleartext_mtu() > cleartext_mtu_1234 - client_channel.set_ciphertext_mtu(1234) - assert client_channel.get_cleartext_mtu() == cleartext_mtu_1234 - - -@slow -async def test_handshake_over_terrible_network( - autojump_clock: trio.testing.MockClock, - server_ctx: SSL.Context, -) -> None: - HANDSHAKES = 100 - r = random.Random(0) - fn = FakeNet() - fn.enable() - # avoid spurious timeouts on slow machines - autojump_clock.autojump_threshold = 0.001 - - async with dtls_echo_server(server_ctx=server_ctx) as (_, address): - async with trio.open_nursery() as nursery: - - async def route_packet(packet: UDPPacket) -> None: - while True: - op = r.choices( - ["deliver", "drop", "dupe", "delay"], - weights=[0.7, 0.1, 0.1, 0.1], - )[0] - print(f"{packet.source} -> {packet.destination}: {op}") - if op == "drop": - return - elif op == "dupe": - fn.send_packet(packet) - elif op == "delay": - await trio.sleep(r.random() * 3) - # I wanted to test random packet corruption too, but it turns out - # openssl has a bug in the following scenario: - # - # - client sends ClientHello - # - server sends HelloVerifyRequest with cookie -- but cookie is - # invalid b/c either the ClientHello or HelloVerifyRequest was - # corrupted - # - client re-sends ClientHello with invalid cookie - # - server replies with new HelloVerifyRequest and correct cookie - # - # At this point, the client *should* switch to the new, valid - # cookie. But OpenSSL doesn't; it stubbornly insists on re-sending - # the original, invalid cookie over and over. In theory we could - # work around this by detecting cookie changes and starting over - # with a whole new SSL object, but (a) it doesn't seem worth it, (b) - # when I tried then I ran into another issue where OpenSSL got stuck - # in an infinite loop sending alerts over and over, which I didn't - # dig into because see (a). - # - # elif op == "distort": - # payload = bytearray(packet.payload) - # payload[r.randrange(len(payload))] ^= 1 << r.randrange(8) - # packet = attrs.evolve(packet, payload=payload) - else: - assert op == "deliver" - print( - f"{packet.source} -> {packet.destination}: delivered" - f" {packet.payload.hex()}", - ) - fn.deliver_packet(packet) - break - - def route_packet_wrapper(packet: UDPPacket) -> None: - try: # noqa: SIM105 # suppressible-exception - nursery.start_soon(route_packet, packet) - except RuntimeError: # pragma: no cover - # We're exiting the nursery, so any remaining packets can just get - # dropped - pass - - fn.route_packet = route_packet_wrapper # type: ignore[assignment] # TODO: Fix FakeNet typing - - for i in range(HANDSHAKES): - print("#" * 80) - print("#" * 80) - print("#" * 80) - with endpoint() as client_endpoint: - client = client_endpoint.connect(address, client_ctx_fn()) - print("client starting do_handshake") - await client.do_handshake() - print("client finished do_handshake") - msg = str(i).encode() - # Make multiple attempts to send data, because the network might - # drop it - while True: - with trio.move_on_after(10) as cscope: - await client.send(msg) - assert await client.receive() == msg - if not cscope.cancelled_caught: - break - - -async def test_implicit_handshake( - server_ctx: SSL.Context, client_ctx: SSL.Context -) -> None: - async with dtls_echo_server(server_ctx=server_ctx) as (_, address): - with endpoint() as client_endpoint: - client = client_endpoint.connect(address, client_ctx) - - # Implicit handshake - await client.send(b"xyz") - assert await client.receive() == b"xyz" - - -async def test_full_duplex(server_ctx: SSL.Context, client_ctx: SSL.Context) -> None: - # Tests simultaneous send/receive, and also multiple methods implicitly invoking - # do_handshake simultaneously. - with endpoint() as server_endpoint, endpoint() as client_endpoint: - await server_endpoint.socket.bind(("127.0.0.1", 0)) - async with trio.open_nursery() as server_nursery: - - async def handler(channel: DTLSChannel) -> None: - async with trio.open_nursery() as nursery: - nursery.start_soon(channel.send, b"from server") - nursery.start_soon(channel.receive) - - await server_nursery.start(server_endpoint.serve, server_ctx, handler) - - client = client_endpoint.connect( - server_endpoint.socket.getsockname(), - client_ctx, - ) - async with trio.open_nursery() as nursery: - nursery.start_soon(client.send, b"from client") - nursery.start_soon(client.receive) - - server_nursery.cancel_scope.cancel() - - -async def test_channel_closing( - server_ctx: SSL.Context, client_ctx: SSL.Context -) -> None: - async with dtls_echo_server(server_ctx=server_ctx) as (_, address): - with endpoint() as client_endpoint: - client = client_endpoint.connect(address, client_ctx) - await client.do_handshake() - client.close() - - with pytest.raises(trio.ClosedResourceError): - await client.send(b"abc") - with pytest.raises(trio.ClosedResourceError): - await client.receive() - - # close is idempotent - client.close() - # can also aclose - await client.aclose() - - -async def test_serve_exits_cleanly_on_close(server_ctx: SSL.Context) -> None: - async with dtls_echo_server(autocancel=False, server_ctx=server_ctx) as ( - server_endpoint, - _address, - ): - server_endpoint.close() - # Testing that the nursery exits even without being cancelled - # close is idempotent - server_endpoint.close() - - -async def test_client_multiplex(server_ctx: SSL.Context) -> None: - async with ( - dtls_echo_server(server_ctx=server_ctx) as (_, address1), - dtls_echo_server(server_ctx=server_ctx) as (_, address2), - ): - with endpoint() as client_endpoint: - client1 = client_endpoint.connect(address1, client_ctx_fn()) - client2 = client_endpoint.connect(address2, client_ctx_fn()) - - await client1.send(b"abc") - await client2.send(b"xyz") - assert await client2.receive() == b"xyz" - assert await client1.receive() == b"abc" - - client_endpoint.close() - - with pytest.raises(trio.ClosedResourceError): - await client1.send(b"xxx") - with pytest.raises(trio.ClosedResourceError): - await client2.receive() - with pytest.raises(trio.ClosedResourceError): - client_endpoint.connect(address1, client_ctx_fn()) - - async def null_handler(_: object) -> None: # pragma: no cover - pass - - async with trio.open_nursery() as nursery: - with pytest.raises(trio.ClosedResourceError): - await nursery.start(client_endpoint.serve, server_ctx, null_handler) - - -async def test_dtls_over_dgram_only() -> None: - with trio.socket.socket() as s: - with pytest.raises(ValueError, match=r"^DTLS requires a SOCK_DGRAM socket$"): - DTLSEndpoint(s) - - -async def test_double_serve(server_ctx: SSL.Context) -> None: - async def null_handler(_: object) -> None: # pragma: no cover - pass - - with endpoint() as server_endpoint: - await server_endpoint.socket.bind(("127.0.0.1", 0)) - async with trio.open_nursery() as nursery: - await nursery.start(server_endpoint.serve, server_ctx, null_handler) - with pytest.raises(trio.BusyResourceError): - await nursery.start(server_endpoint.serve, server_ctx, null_handler) - - nursery.cancel_scope.cancel() - - async with trio.open_nursery() as nursery: - await nursery.start(server_endpoint.serve, server_ctx, null_handler) - nursery.cancel_scope.cancel() - - -async def test_connect_to_non_server( - autojump_clock: trio.abc.Clock, client_ctx: SSL.Context -) -> None: - fn = FakeNet() - fn.enable() - with endpoint() as client1, endpoint() as client2: - await client1.socket.bind(("127.0.0.1", 0)) - # This should just time out - with trio.move_on_after(100) as cscope: - channel = client2.connect(client1.socket.getsockname(), client_ctx) - await channel.do_handshake() - assert cscope.cancelled_caught - - -@pytest.mark.parametrize("buffer_size", [10, 20]) -async def test_incoming_buffer_overflow( - autojump_clock: trio.abc.Clock, - server_ctx: SSL.Context, - client_ctx: SSL.Context, - buffer_size: int, -) -> None: - fn = FakeNet() - fn.enable() - async with dtls_echo_server(server_ctx=server_ctx) as (_, address): - with endpoint(incoming_packets_buffer=buffer_size) as client_endpoint: - assert client_endpoint.incoming_packets_buffer == buffer_size - client = client_endpoint.connect(address, client_ctx) - for i in range(buffer_size + 15): - await client.send(str(i).encode()) - await trio.sleep(1) - stats = client.statistics() - assert stats.incoming_packets_dropped_in_trio == 15 - for i in range(buffer_size): - assert await client.receive() == str(i).encode() - await client.send(b"buffer clear now") - assert await client.receive() == b"buffer clear now" - - -async def test_server_socket_doesnt_crash_on_garbage( - autojump_clock: trio.abc.Clock, server_ctx: SSL.Context -) -> None: - fn = FakeNet() - fn.enable() - - from trio._dtls import ( - ContentType, - HandshakeFragment, - HandshakeType, - ProtocolVersion, - Record, - encode_handshake_fragment, - encode_record, - ) - - client_hello = encode_record( - Record( - content_type=ContentType.handshake, - version=ProtocolVersion.DTLS10, - epoch_seqno=0, - payload=encode_handshake_fragment( - HandshakeFragment( - msg_type=HandshakeType.client_hello, - msg_len=10, - msg_seq=0, - frag_offset=0, - frag_len=10, - frag=bytes(10), - ), - ), - ), - ) - - client_hello_extended = client_hello + b"\x00" - client_hello_short = client_hello[:-1] - # cuts off in middle of handshake message header - client_hello_really_short = client_hello[:14] - client_hello_corrupt_record_len = bytearray(client_hello) - client_hello_corrupt_record_len[11] = 0xFF - - client_hello_fragmented = encode_record( - Record( - content_type=ContentType.handshake, - version=ProtocolVersion.DTLS10, - epoch_seqno=0, - payload=encode_handshake_fragment( - HandshakeFragment( - msg_type=HandshakeType.client_hello, - msg_len=20, - msg_seq=0, - frag_offset=0, - frag_len=10, - frag=bytes(10), - ), - ), - ), - ) - - client_hello_trailing_data_in_record = encode_record( - Record( - content_type=ContentType.handshake, - version=ProtocolVersion.DTLS10, - epoch_seqno=0, - payload=encode_handshake_fragment( - HandshakeFragment( - msg_type=HandshakeType.client_hello, - msg_len=20, - msg_seq=0, - frag_offset=0, - frag_len=10, - frag=bytes(10), - ), - ) - + b"\x00", - ), - ) - - handshake_empty = encode_record( - Record( - content_type=ContentType.handshake, - version=ProtocolVersion.DTLS10, - epoch_seqno=0, - payload=b"", - ), - ) - - client_hello_truncated_in_cookie = encode_record( - Record( - content_type=ContentType.handshake, - version=ProtocolVersion.DTLS10, - epoch_seqno=0, - payload=bytes(2 + 32 + 1) + b"\xff", - ), - ) - - async with dtls_echo_server(server_ctx=server_ctx) as (_, address): - with trio.socket.socket(type=trio.socket.SOCK_DGRAM) as sock: - bad_packets: list[bytes | bytearray | memoryview[int]] = [ - b"", - b"xyz", - client_hello_extended, - client_hello_short, - client_hello_really_short, - client_hello_corrupt_record_len, - client_hello_fragmented, - client_hello_trailing_data_in_record, - handshake_empty, - client_hello_truncated_in_cookie, - ] - - for bad_packet in bad_packets: - await sock.sendto(bad_packet, address) - await trio.sleep(1) - - -async def test_invalid_cookie_rejected( - autojump_clock: trio.abc.Clock, server_ctx: SSL.Context, client_ctx: SSL.Context -) -> None: - fn = FakeNet() - fn.enable() - - from trio._dtls import BadPacket, decode_client_hello_untrusted - - with trio.CancelScope() as cscope: - # the first 11 bytes of ClientHello aren't protected by the cookie, so only test - # corrupting bytes after that. - offset_to_corrupt = count(11) - - def route_packet(packet: UDPPacket) -> None: - try: - _, cookie, _ = decode_client_hello_untrusted(packet.payload) - except BadPacket: - pass - else: - if len(cookie) != 0: - # this is a challenge response packet - # let's corrupt the next offset so the handshake should fail - payload = bytearray(packet.payload) - offset = next(offset_to_corrupt) - if offset >= len(payload): - # We've tried all offsets. Clamp offset to the end of the - # payload, and terminate the test. - offset = len(payload) - 1 - cscope.cancel() - payload[offset] ^= 0x01 - packet = attrs.evolve(packet, payload=bytes(payload)) - - fn.deliver_packet(packet) - - fn.route_packet = route_packet # type: ignore[assignment] # TODO: Fix FakeNet typing - - async with dtls_echo_server(server_ctx=server_ctx) as (_, address): - while True: - with endpoint() as client: - channel = client.connect(address, client_ctx) - await channel.do_handshake() - assert cscope.cancelled_caught - - -async def test_client_cancels_handshake_and_starts_new_one( - autojump_clock: trio.abc.Clock, server_ctx: SSL.Context -) -> None: - # if a client disappears during the handshake, and then starts a new handshake from - # scratch, then the first handler's channel should fail, and a new handler get - # started - fn = FakeNet() - fn.enable() - - with endpoint() as server, endpoint() as client: - await server.socket.bind(("127.0.0.1", 0)) - async with trio.open_nursery() as nursery: - first_time = True - - async def handler(channel: DTLSChannel) -> None: - nonlocal first_time - if first_time: - first_time = False - print("handler: first time, cancelling connect") - connect_cscope.cancel() - await trio.sleep(0.5) - print("handler: handshake should fail now") - with pytest.raises(trio.BrokenResourceError): - await channel.do_handshake() - else: - print("handler: not first time, sending hello") - await channel.send(b"hello") - - await nursery.start(server.serve, server_ctx, handler) - - print("client: starting first connect") - with trio.CancelScope() as connect_cscope: - channel = client.connect(server.socket.getsockname(), client_ctx_fn()) - await channel.do_handshake() - assert connect_cscope.cancelled_caught - - print("client: starting second connect") - channel = client.connect(server.socket.getsockname(), client_ctx_fn()) - assert await channel.receive() == b"hello" - - # Give handlers a chance to finish - await trio.sleep(10) - nursery.cancel_scope.cancel() - - -async def test_swap_client_server(server_ctx: SSL.Context) -> None: - with endpoint() as a, endpoint() as b: - await a.socket.bind(("127.0.0.1", 0)) - await b.socket.bind(("127.0.0.1", 0)) - - async def echo_handler(channel: DTLSChannel) -> None: - async for packet in channel: - await channel.send(packet) - - async def crashing_echo_handler(channel: DTLSChannel) -> None: - with pytest.raises(trio.BrokenResourceError): - await echo_handler(channel) - - async with trio.open_nursery() as nursery: - await nursery.start(a.serve, server_ctx, crashing_echo_handler) - await nursery.start(b.serve, server_ctx, echo_handler) - - b_to_a = b.connect(a.socket.getsockname(), client_ctx_fn()) - await b_to_a.send(b"b as client") - assert await b_to_a.receive() == b"b as client" - - a_to_b = a.connect(b.socket.getsockname(), client_ctx_fn()) - await a_to_b.do_handshake() - with pytest.raises(trio.BrokenResourceError): - await b_to_a.send(b"association broken") - await a_to_b.send(b"a as client") - assert await a_to_b.receive() == b"a as client" - - nursery.cancel_scope.cancel() - - -@slow -async def test_openssl_retransmit_doesnt_break_stuff( - server_ctx: SSL.Context, client_ctx: SSL.Context -) -> None: - # can't use autojump_clock here, because the point of the test is to wait for - # openssl's built-in retransmit timer to expire, which is hard-coded to use - # wall-clock time. - fn = FakeNet() - fn.enable() - - blackholed = True - - def route_packet(packet: UDPPacket) -> None: - if blackholed: - print("dropped packet", packet) - return - print("delivered packet", packet) - # packets.append( - # scapy.all.IP( - # src=packet.source.ip.compressed, dst=packet.destination.ip.compressed - # ) - # / scapy.all.UDP(sport=packet.source.port, dport=packet.destination.port) - # / packet.payload - # ) - fn.deliver_packet(packet) - - fn.route_packet = route_packet # type: ignore[assignment] # TODO add type annotations for FakeNet - - async with dtls_echo_server(server_ctx=server_ctx) as (server_endpoint, address): - with endpoint() as client_endpoint: - async with trio.open_nursery() as nursery: - - async def connecter() -> None: - client = client_endpoint.connect(address, client_ctx) - await client.do_handshake(initial_retransmit_timeout=1.5) - await client.send(b"hi") - assert await client.receive() == b"hi" - - nursery.start_soon(connecter) - - # openssl's default timeout is 1 second, so this ensures that it thinks - # the timeout has expired - await trio.sleep(1.1) - # disable blackholing and send a garbage packet to wake up openssl so it - # notices the timeout has expired - blackholed = False - await server_endpoint.socket.sendto( - b"xxx", - client_endpoint.socket.getsockname(), - ) - # now the client task should finish connecting and exit cleanly - - # scapy.all.wrpcap("/tmp/trace.pcap", packets) - - -async def test_initial_retransmit_timeout_configuration( - autojump_clock: trio.abc.Clock, server_ctx: SSL.Context -) -> None: - fn = FakeNet() - fn.enable() - - blackholed = True - - def route_packet(packet: UDPPacket) -> None: - nonlocal blackholed - if blackholed: - blackholed = False - else: - fn.deliver_packet(packet) - - fn.route_packet = route_packet # type: ignore[assignment] # TODO: add type annotations for FakeNet - - async with dtls_echo_server(server_ctx=server_ctx) as (_, address): - for t in [1, 2, 4]: - with endpoint() as client: - before = trio.current_time() - blackholed = True - channel = client.connect(address, client_ctx_fn()) - await channel.do_handshake(initial_retransmit_timeout=t) - after = trio.current_time() - assert after - before == t - - -async def test_explicit_tiny_mtu_is_respected( - server_ctx: SSL.Context, client_ctx: SSL.Context -) -> None: - # ClientHello is ~240 bytes, and it can't be fragmented, so our mtu has to - # be larger than that. (300 is still smaller than any real network though.) - MTU = 300 - - fn = FakeNet() - fn.enable() - - def route_packet(packet: UDPPacket) -> None: - print(f"delivering {packet}") - print(f"payload size: {len(packet.payload)}") - assert len(packet.payload) <= MTU - fn.deliver_packet(packet) - - fn.route_packet = route_packet # type: ignore[assignment] # TODO add type annotations for FakeNet - - async with dtls_echo_server(mtu=MTU, server_ctx=server_ctx) as (_server, address): - with endpoint() as client: - channel = client.connect(address, client_ctx) - channel.set_ciphertext_mtu(MTU) - await channel.do_handshake() - await channel.send(b"hi") - assert await channel.receive() == b"hi" - - -@parametrize_ipv6 -async def test_handshake_handles_minimum_network_mtu( - ipv6: bool, - autojump_clock: trio.abc.Clock, - server_ctx: SSL.Context, - client_ctx: SSL.Context, -) -> None: - # Fake network that has the minimum allowable MTU for whatever protocol we're using. - fn = FakeNet() - fn.enable() - - mtu = 1280 - 48 if ipv6 else 576 - 28 - - def route_packet(packet: UDPPacket) -> None: - if len(packet.payload) > mtu: - print(f"dropping {packet}") - else: - print(f"delivering {packet}") - fn.deliver_packet(packet) - - fn.route_packet = route_packet # type: ignore[assignment] # TODO: add type annotations for FakeNet - - # See if we can successfully do a handshake -- some of the volleys will get dropped, - # and the retransmit logic should detect this and back off the MTU to something - # smaller until it succeeds. - async with dtls_echo_server(ipv6=ipv6, server_ctx=server_ctx) as (_, address): - with endpoint(ipv6=ipv6) as client_endpoint: - client = client_endpoint.connect(address, client_ctx) - # the handshake mtu backoff shouldn't affect the return value from - # get_cleartext_mtu, b/c that's under the user's control via - # set_ciphertext_mtu - client.set_ciphertext_mtu(9999) - await client.send(b"xyz") - assert await client.receive() == b"xyz" - assert client.get_cleartext_mtu() > 9000 # as vegeta said - - -@pytest.mark.filterwarnings("always:unclosed DTLS:ResourceWarning") -async def test_system_task_cleaned_up_on_gc(client_ctx: SSL.Context) -> None: - before_tasks = trio.lowlevel.current_statistics().tasks_living - - # We put this into a sub-function so that everything automatically becomes garbage - # when the frame exits. For some reason just doing 'del e' wasn't enough on pypy - # with coverage enabled -- I think we were hitting this bug: - # https://foss.heptapod.net/pypy/pypy/-/issues/3656 - async def start_and_forget_endpoint() -> int: - e = endpoint() - - # This connection/handshake attempt can't succeed. The only purpose is to force - # the endpoint to set up a receive loop. - with trio.socket.socket(type=trio.socket.SOCK_DGRAM) as s: - await s.bind(("127.0.0.1", 0)) - c = e.connect(s.getsockname(), client_ctx) - async with trio.open_nursery() as nursery: - nursery.start_soon(c.do_handshake) - await trio.testing.wait_all_tasks_blocked() - nursery.cancel_scope.cancel() - - during_tasks = trio.lowlevel.current_statistics().tasks_living - return during_tasks - - with pytest.warns(ResourceWarning): # noqa: PT031 - during_tasks = await start_and_forget_endpoint() - await trio.testing.wait_all_tasks_blocked() - gc_collect_harder() - - await trio.testing.wait_all_tasks_blocked() - - after_tasks = trio.lowlevel.current_statistics().tasks_living - assert before_tasks < during_tasks - assert before_tasks == after_tasks - - -@pytest.mark.filterwarnings("always:unclosed DTLS:ResourceWarning") -async def test_gc_before_system_task_starts() -> None: - e = endpoint() - - with pytest.warns(ResourceWarning): # noqa: PT031 - del e - gc_collect_harder() - - await trio.testing.wait_all_tasks_blocked() - - -@pytest.mark.filterwarnings("always:unclosed DTLS:ResourceWarning") -async def test_gc_as_packet_received() -> None: - fn = FakeNet() - fn.enable() - - e = endpoint() - await e.socket.bind(("127.0.0.1", 0)) - e._ensure_receive_loop() - - await trio.testing.wait_all_tasks_blocked() - - with trio.socket.socket(type=trio.socket.SOCK_DGRAM) as s: - await s.sendto(b"xxx", e.socket.getsockname()) - # At this point, the endpoint's receive loop has been marked runnable because it - # just received a packet; closing the endpoint socket won't interrupt that. But by - # the time it wakes up to process the packet, the endpoint will be gone. - with pytest.warns(ResourceWarning): # noqa: PT031 - del e - gc_collect_harder() - - -@pytest.mark.filterwarnings("always:unclosed DTLS:ResourceWarning") -def test_gc_after_trio_exits() -> None: - async def main() -> DTLSEndpoint: - # We use fakenet just to make sure no real sockets can leak out of the test - # case - on pypy somehow the socket was outliving the gc_collect_harder call - # below. Since the test is just making sure DTLSEndpoint.__del__ doesn't explode - # when called after trio exits, it doesn't need a real socket. - fn = FakeNet() - fn.enable() - return endpoint() - - e = trio.run(main) - with pytest.warns(ResourceWarning): # noqa: PT031 - del e - gc_collect_harder() - - -async def test_already_closed_socket_doesnt_crash() -> None: - with endpoint() as e: - # We close the socket before checkpointing, so the socket will already be closed - # when the system task starts up - e.socket.close() - # Now give it a chance to start up, and hopefully not crash - await trio.testing.wait_all_tasks_blocked() - - -async def test_socket_closed_while_processing_clienthello( - autojump_clock: trio.abc.Clock, server_ctx: SSL.Context, client_ctx: SSL.Context -) -> None: - fn = FakeNet() - fn.enable() - - # Check what happens if the socket is discovered to be closed when sending a - # HelloVerifyRequest, since that has its own sending logic - async with dtls_echo_server(server_ctx=server_ctx) as (server, address): - - def route_packet(packet: UDPPacket) -> None: - fn.deliver_packet(packet) - server.socket.close() - - fn.route_packet = route_packet # type: ignore[assignment] # TODO add type annotations for FakeNet - - with endpoint() as client_endpoint: - with trio.move_on_after(10): - client = client_endpoint.connect(address, client_ctx) - await client.do_handshake() - - -async def test_association_replaced_while_handshake_running( - autojump_clock: trio.abc.Clock, server_ctx: SSL.Context -) -> None: - fn = FakeNet() - fn.enable() - - def route_packet(packet: UDPPacket) -> None: - pass - - fn.route_packet = route_packet # type: ignore[assignment] # TODO: add type annotations for FakeNet - - async with dtls_echo_server(server_ctx=server_ctx) as (_, address): - with endpoint() as client_endpoint: - # TODO: should this have the same exact client_ctx? - c1 = client_endpoint.connect(address, client_ctx_fn()) - async with trio.open_nursery() as nursery: - - async def doomed_handshake() -> None: - with pytest.raises(trio.BrokenResourceError): - await c1.do_handshake() - - nursery.start_soon(doomed_handshake) - - await trio.sleep(10) - - client_endpoint.connect(address, client_ctx_fn()) - - -async def test_association_replaced_before_handshake_starts( - server_ctx: SSL.Context, -) -> None: - fn = FakeNet() - fn.enable() - - # This test shouldn't send any packets - def route_packet(packet: UDPPacket) -> NoReturn: # pragma: no cover - raise AssertionError() - - fn.route_packet = route_packet # type: ignore[assignment] # TODO add type annotations for FakeNet - - async with dtls_echo_server(server_ctx=server_ctx) as (_, address): - with endpoint() as client_endpoint: - # TODO: should this use the same client_ctx? - c1 = client_endpoint.connect(address, client_ctx_fn()) - client_endpoint.connect(address, client_ctx_fn()) - with pytest.raises(trio.BrokenResourceError): - await c1.do_handshake() - - -async def test_send_to_closed_local_port(server_ctx: SSL.Context) -> None: - # On Windows, sending a UDP packet to a closed local port can cause a weird - # ECONNRESET error later, inside the receive task. Make sure we're handling it - # properly. - async with dtls_echo_server(server_ctx=server_ctx) as (_, address): - with endpoint() as client_endpoint: - async with trio.open_nursery() as nursery: - for i in range(1, 10): - channel = client_endpoint.connect(("127.0.0.1", i), client_ctx_fn()) - nursery.start_soon(channel.do_handshake) - channel = client_endpoint.connect(address, client_ctx_fn()) - await channel.send(b"xxx") - assert await channel.receive() == b"xxx" - nursery.cancel_scope.cancel() +from __future__ import annotations + +import random +from contextlib import asynccontextmanager +from itertools import count +from typing import TYPE_CHECKING, NoReturn + +import attrs +import pytest + +from trio._tests.pytest_plugin import maybe_ignore_import_error + +try: + import trustme + from OpenSSL import SSL +except ImportError as error: + maybe_ignore_import_error(error) + + +import trio +import trio.testing +from trio import DTLSChannel, DTLSEndpoint +from trio.testing._fake_net import FakeNet, UDPPacket + +from .._core._tests.tutil import binds_ipv6, gc_collect_harder, slow + +if TYPE_CHECKING: + from collections.abc import AsyncGenerator + +ca = trustme.CA() +server_cert = ca.issue_cert("example.com") + + +@pytest.fixture +def server_ctx() -> SSL.Context: + ctx = SSL.Context(SSL.DTLS_METHOD) + server_cert.configure_cert(ctx) + return ctx + + +def client_ctx_fn() -> SSL.Context: + ctx = SSL.Context(SSL.DTLS_METHOD) + ca.configure_trust(ctx) + return ctx + + +@pytest.fixture +def client_ctx() -> SSL.Context: + return client_ctx_fn() + + +parametrize_ipv6 = pytest.mark.parametrize( + "ipv6", + [False, pytest.param(True, marks=binds_ipv6)], + ids=["ipv4", "ipv6"], +) + + +def endpoint(**kwargs: int | bool) -> DTLSEndpoint: + ipv6 = kwargs.pop("ipv6", False) + family = trio.socket.AF_INET6 if ipv6 else trio.socket.AF_INET + sock = trio.socket.socket(type=trio.socket.SOCK_DGRAM, family=family) + return DTLSEndpoint(sock, **kwargs) + + +@asynccontextmanager +async def dtls_echo_server( + *, + server_ctx: SSL.Context, + autocancel: bool = True, + mtu: int | None = None, + ipv6: bool = False, +) -> AsyncGenerator[tuple[DTLSEndpoint, tuple[str, int]], None]: + with endpoint(ipv6=ipv6) as server: + localhost = "::1" if ipv6 else "127.0.0.1" + await server.socket.bind((localhost, 0)) + async with trio.open_nursery() as nursery: + + async def echo_handler(dtls_channel: DTLSChannel) -> None: + print( + "echo handler started: " + f"server {dtls_channel.endpoint.socket.getsockname()!r} " + f"client {dtls_channel.peer_address!r}", + ) + if mtu is not None: + dtls_channel.set_ciphertext_mtu(mtu) + try: + print("server starting do_handshake") + await dtls_channel.do_handshake() + print("server finished do_handshake") + # no branch for leaving this for loop because we only leave + # a channel by cancellation. + async for packet in dtls_channel: # pragma: no branch + print(f"echoing {packet!r} -> {dtls_channel.peer_address!r}") + await dtls_channel.send(packet) + except trio.BrokenResourceError: # pragma: no cover + print("echo handler channel broken") + + await nursery.start(server.serve, server_ctx, echo_handler) + + yield server, server.socket.getsockname() # noqa: RUF075 + + if autocancel: + nursery.cancel_scope.cancel() + + +@parametrize_ipv6 +async def test_smoke( + ipv6: bool, server_ctx: SSL.Context, client_ctx: SSL.Context +) -> None: + async with dtls_echo_server(ipv6=ipv6, server_ctx=server_ctx) as ( + _server_endpoint, + address, + ): + with endpoint(ipv6=ipv6) as client_endpoint: + client_channel = client_endpoint.connect(address, client_ctx) + with pytest.raises(trio.NeedHandshakeError): + client_channel.get_cleartext_mtu() + + await client_channel.do_handshake() + await client_channel.send(b"hello") + assert await client_channel.receive() == b"hello" + await client_channel.send(b"goodbye") + assert await client_channel.receive() == b"goodbye" + + with pytest.raises( + ValueError, + match=r"^openssl doesn't support sending empty DTLS packets$", + ): + await client_channel.send(b"") + + client_channel.set_ciphertext_mtu(1234) + cleartext_mtu_1234 = client_channel.get_cleartext_mtu() + client_channel.set_ciphertext_mtu(4321) + assert client_channel.get_cleartext_mtu() > cleartext_mtu_1234 + client_channel.set_ciphertext_mtu(1234) + assert client_channel.get_cleartext_mtu() == cleartext_mtu_1234 + + +@slow +async def test_handshake_over_terrible_network( + autojump_clock: trio.testing.MockClock, + server_ctx: SSL.Context, +) -> None: + HANDSHAKES = 100 + r = random.Random(0) + fn = FakeNet() + fn.enable() + # avoid spurious timeouts on slow machines + autojump_clock.autojump_threshold = 0.001 + + async with dtls_echo_server(server_ctx=server_ctx) as (_, address): + async with trio.open_nursery() as nursery: + + async def route_packet(packet: UDPPacket) -> None: + while True: + op = r.choices( + ["deliver", "drop", "dupe", "delay"], + weights=[0.7, 0.1, 0.1, 0.1], + )[0] + print(f"{packet.source} -> {packet.destination}: {op}") + if op == "drop": + return + elif op == "dupe": + fn.send_packet(packet) + elif op == "delay": + await trio.sleep(r.random() * 3) + # I wanted to test random packet corruption too, but it turns out + # openssl has a bug in the following scenario: + # + # - client sends ClientHello + # - server sends HelloVerifyRequest with cookie -- but cookie is + # invalid b/c either the ClientHello or HelloVerifyRequest was + # corrupted + # - client re-sends ClientHello with invalid cookie + # - server replies with new HelloVerifyRequest and correct cookie + # + # At this point, the client *should* switch to the new, valid + # cookie. But OpenSSL doesn't; it stubbornly insists on re-sending + # the original, invalid cookie over and over. In theory we could + # work around this by detecting cookie changes and starting over + # with a whole new SSL object, but (a) it doesn't seem worth it, (b) + # when I tried then I ran into another issue where OpenSSL got stuck + # in an infinite loop sending alerts over and over, which I didn't + # dig into because see (a). + # + # elif op == "distort": + # payload = bytearray(packet.payload) + # payload[r.randrange(len(payload))] ^= 1 << r.randrange(8) + # packet = attrs.evolve(packet, payload=payload) + else: + assert op == "deliver" + print( + f"{packet.source} -> {packet.destination}: delivered" + f" {packet.payload.hex()}", + ) + fn.deliver_packet(packet) + break + + def route_packet_wrapper(packet: UDPPacket) -> None: + try: # noqa: SIM105 # suppressible-exception + nursery.start_soon(route_packet, packet) + except RuntimeError: # pragma: no cover + # We're exiting the nursery, so any remaining packets can just get + # dropped + pass + + fn.route_packet = route_packet_wrapper # type: ignore[assignment] # TODO: Fix FakeNet typing + + for i in range(HANDSHAKES): + print("#" * 80) + print("#" * 80) + print("#" * 80) + with endpoint() as client_endpoint: + client = client_endpoint.connect(address, client_ctx_fn()) + print("client starting do_handshake") + await client.do_handshake() + print("client finished do_handshake") + msg = str(i).encode() + # Make multiple attempts to send data, because the network might + # drop it + while True: + with trio.move_on_after(10) as cscope: + await client.send(msg) + assert await client.receive() == msg + if not cscope.cancelled_caught: + break + + +async def test_implicit_handshake( + server_ctx: SSL.Context, client_ctx: SSL.Context +) -> None: + async with dtls_echo_server(server_ctx=server_ctx) as (_, address): + with endpoint() as client_endpoint: + client = client_endpoint.connect(address, client_ctx) + + # Implicit handshake + await client.send(b"xyz") + assert await client.receive() == b"xyz" + + +async def test_full_duplex(server_ctx: SSL.Context, client_ctx: SSL.Context) -> None: + # Tests simultaneous send/receive, and also multiple methods implicitly invoking + # do_handshake simultaneously. + with endpoint() as server_endpoint, endpoint() as client_endpoint: + await server_endpoint.socket.bind(("127.0.0.1", 0)) + async with trio.open_nursery() as server_nursery: + + async def handler(channel: DTLSChannel) -> None: + async with trio.open_nursery() as nursery: + nursery.start_soon(channel.send, b"from server") + nursery.start_soon(channel.receive) + + await server_nursery.start(server_endpoint.serve, server_ctx, handler) + + client = client_endpoint.connect( + server_endpoint.socket.getsockname(), + client_ctx, + ) + async with trio.open_nursery() as nursery: + nursery.start_soon(client.send, b"from client") + nursery.start_soon(client.receive) + + server_nursery.cancel_scope.cancel() + + +async def test_channel_closing( + server_ctx: SSL.Context, client_ctx: SSL.Context +) -> None: + async with dtls_echo_server(server_ctx=server_ctx) as (_, address): + with endpoint() as client_endpoint: + client = client_endpoint.connect(address, client_ctx) + await client.do_handshake() + client.close() + + with pytest.raises(trio.ClosedResourceError): + await client.send(b"abc") + with pytest.raises(trio.ClosedResourceError): + await client.receive() + + # close is idempotent + client.close() + # can also aclose + await client.aclose() + + +async def test_serve_exits_cleanly_on_close(server_ctx: SSL.Context) -> None: + async with dtls_echo_server(autocancel=False, server_ctx=server_ctx) as ( + server_endpoint, + _address, + ): + server_endpoint.close() + # Testing that the nursery exits even without being cancelled + # close is idempotent + server_endpoint.close() + + +async def test_client_multiplex(server_ctx: SSL.Context) -> None: + async with ( + dtls_echo_server(server_ctx=server_ctx) as (_, address1), + dtls_echo_server(server_ctx=server_ctx) as (_, address2), + ): + with endpoint() as client_endpoint: + client1 = client_endpoint.connect(address1, client_ctx_fn()) + client2 = client_endpoint.connect(address2, client_ctx_fn()) + + await client1.send(b"abc") + await client2.send(b"xyz") + assert await client2.receive() == b"xyz" + assert await client1.receive() == b"abc" + + client_endpoint.close() + + with pytest.raises(trio.ClosedResourceError): + await client1.send(b"xxx") + with pytest.raises(trio.ClosedResourceError): + await client2.receive() + with pytest.raises(trio.ClosedResourceError): + client_endpoint.connect(address1, client_ctx_fn()) + + async def null_handler(_: object) -> None: # pragma: no cover + pass + + async with trio.open_nursery() as nursery: + with pytest.raises(trio.ClosedResourceError): + await nursery.start(client_endpoint.serve, server_ctx, null_handler) + + +async def test_dtls_over_dgram_only() -> None: + with trio.socket.socket() as s: + with pytest.raises(ValueError, match=r"^DTLS requires a SOCK_DGRAM socket$"): + DTLSEndpoint(s) + + +async def test_double_serve(server_ctx: SSL.Context) -> None: + async def null_handler(_: object) -> None: # pragma: no cover + pass + + with endpoint() as server_endpoint: + await server_endpoint.socket.bind(("127.0.0.1", 0)) + async with trio.open_nursery() as nursery: + await nursery.start(server_endpoint.serve, server_ctx, null_handler) + with pytest.raises(trio.BusyResourceError): + await nursery.start(server_endpoint.serve, server_ctx, null_handler) + + nursery.cancel_scope.cancel() + + async with trio.open_nursery() as nursery: + await nursery.start(server_endpoint.serve, server_ctx, null_handler) + nursery.cancel_scope.cancel() + + +async def test_connect_to_non_server( + autojump_clock: trio.abc.Clock, client_ctx: SSL.Context +) -> None: + fn = FakeNet() + fn.enable() + with endpoint() as client1, endpoint() as client2: + await client1.socket.bind(("127.0.0.1", 0)) + # This should just time out + with trio.move_on_after(100) as cscope: + channel = client2.connect(client1.socket.getsockname(), client_ctx) + await channel.do_handshake() + assert cscope.cancelled_caught + + +@pytest.mark.parametrize("buffer_size", [10, 20]) +async def test_incoming_buffer_overflow( + autojump_clock: trio.abc.Clock, + server_ctx: SSL.Context, + client_ctx: SSL.Context, + buffer_size: int, +) -> None: + fn = FakeNet() + fn.enable() + async with dtls_echo_server(server_ctx=server_ctx) as (_, address): + with endpoint(incoming_packets_buffer=buffer_size) as client_endpoint: + assert client_endpoint.incoming_packets_buffer == buffer_size + client = client_endpoint.connect(address, client_ctx) + for i in range(buffer_size + 15): + await client.send(str(i).encode()) + await trio.sleep(1) + stats = client.statistics() + assert stats.incoming_packets_dropped_in_trio == 15 + for i in range(buffer_size): + assert await client.receive() == str(i).encode() + await client.send(b"buffer clear now") + assert await client.receive() == b"buffer clear now" + + +async def test_server_socket_doesnt_crash_on_garbage( + autojump_clock: trio.abc.Clock, server_ctx: SSL.Context +) -> None: + fn = FakeNet() + fn.enable() + + from trio._dtls import ( + ContentType, + HandshakeFragment, + HandshakeType, + ProtocolVersion, + Record, + encode_handshake_fragment, + encode_record, + ) + + client_hello = encode_record( + Record( + content_type=ContentType.handshake, + version=ProtocolVersion.DTLS10, + epoch_seqno=0, + payload=encode_handshake_fragment( + HandshakeFragment( + msg_type=HandshakeType.client_hello, + msg_len=10, + msg_seq=0, + frag_offset=0, + frag_len=10, + frag=bytes(10), + ), + ), + ), + ) + + client_hello_extended = client_hello + b"\x00" + client_hello_short = client_hello[:-1] + # cuts off in middle of handshake message header + client_hello_really_short = client_hello[:14] + client_hello_corrupt_record_len = bytearray(client_hello) + client_hello_corrupt_record_len[11] = 0xFF + + client_hello_fragmented = encode_record( + Record( + content_type=ContentType.handshake, + version=ProtocolVersion.DTLS10, + epoch_seqno=0, + payload=encode_handshake_fragment( + HandshakeFragment( + msg_type=HandshakeType.client_hello, + msg_len=20, + msg_seq=0, + frag_offset=0, + frag_len=10, + frag=bytes(10), + ), + ), + ), + ) + + client_hello_trailing_data_in_record = encode_record( + Record( + content_type=ContentType.handshake, + version=ProtocolVersion.DTLS10, + epoch_seqno=0, + payload=encode_handshake_fragment( + HandshakeFragment( + msg_type=HandshakeType.client_hello, + msg_len=20, + msg_seq=0, + frag_offset=0, + frag_len=10, + frag=bytes(10), + ), + ) + + b"\x00", + ), + ) + + handshake_empty = encode_record( + Record( + content_type=ContentType.handshake, + version=ProtocolVersion.DTLS10, + epoch_seqno=0, + payload=b"", + ), + ) + + client_hello_truncated_in_cookie = encode_record( + Record( + content_type=ContentType.handshake, + version=ProtocolVersion.DTLS10, + epoch_seqno=0, + payload=bytes(2 + 32 + 1) + b"\xff", + ), + ) + + async with dtls_echo_server(server_ctx=server_ctx) as (_, address): + with trio.socket.socket(type=trio.socket.SOCK_DGRAM) as sock: + bad_packets: list[bytes | bytearray | memoryview[int]] = [ + b"", + b"xyz", + client_hello_extended, + client_hello_short, + client_hello_really_short, + client_hello_corrupt_record_len, + client_hello_fragmented, + client_hello_trailing_data_in_record, + handshake_empty, + client_hello_truncated_in_cookie, + ] + + for bad_packet in bad_packets: + await sock.sendto(bad_packet, address) + await trio.sleep(1) + + +async def test_invalid_cookie_rejected( + autojump_clock: trio.abc.Clock, server_ctx: SSL.Context, client_ctx: SSL.Context +) -> None: + fn = FakeNet() + fn.enable() + + from trio._dtls import BadPacket, decode_client_hello_untrusted + + with trio.CancelScope() as cscope: + # the first 11 bytes of ClientHello aren't protected by the cookie, so only test + # corrupting bytes after that. + offset_to_corrupt = count(11) + + def route_packet(packet: UDPPacket) -> None: + try: + _, cookie, _ = decode_client_hello_untrusted(packet.payload) + except BadPacket: + pass + else: + if len(cookie) != 0: + # this is a challenge response packet + # let's corrupt the next offset so the handshake should fail + payload = bytearray(packet.payload) + offset = next(offset_to_corrupt) + if offset >= len(payload): + # We've tried all offsets. Clamp offset to the end of the + # payload, and terminate the test. + offset = len(payload) - 1 + cscope.cancel() + payload[offset] ^= 0x01 + packet = attrs.evolve(packet, payload=bytes(payload)) + + fn.deliver_packet(packet) + + fn.route_packet = route_packet # type: ignore[assignment] # TODO: Fix FakeNet typing + + async with dtls_echo_server(server_ctx=server_ctx) as (_, address): + while True: + with endpoint() as client: + channel = client.connect(address, client_ctx) + await channel.do_handshake() + assert cscope.cancelled_caught + + +async def test_client_cancels_handshake_and_starts_new_one( + autojump_clock: trio.abc.Clock, server_ctx: SSL.Context +) -> None: + # if a client disappears during the handshake, and then starts a new handshake from + # scratch, then the first handler's channel should fail, and a new handler get + # started + fn = FakeNet() + fn.enable() + + with endpoint() as server, endpoint() as client: + await server.socket.bind(("127.0.0.1", 0)) + async with trio.open_nursery() as nursery: + first_time = True + + async def handler(channel: DTLSChannel) -> None: + nonlocal first_time + if first_time: + first_time = False + print("handler: first time, cancelling connect") + connect_cscope.cancel() + await trio.sleep(0.5) + print("handler: handshake should fail now") + with pytest.raises(trio.BrokenResourceError): + await channel.do_handshake() + else: + print("handler: not first time, sending hello") + await channel.send(b"hello") + + await nursery.start(server.serve, server_ctx, handler) + + print("client: starting first connect") + with trio.CancelScope() as connect_cscope: + channel = client.connect(server.socket.getsockname(), client_ctx_fn()) + await channel.do_handshake() + assert connect_cscope.cancelled_caught + + print("client: starting second connect") + channel = client.connect(server.socket.getsockname(), client_ctx_fn()) + assert await channel.receive() == b"hello" + + # Give handlers a chance to finish + await trio.sleep(10) + nursery.cancel_scope.cancel() + + +async def test_swap_client_server(server_ctx: SSL.Context) -> None: + with endpoint() as a, endpoint() as b: + await a.socket.bind(("127.0.0.1", 0)) + await b.socket.bind(("127.0.0.1", 0)) + + async def echo_handler(channel: DTLSChannel) -> None: + async for packet in channel: + await channel.send(packet) + + async def crashing_echo_handler(channel: DTLSChannel) -> None: + with pytest.raises(trio.BrokenResourceError): + await echo_handler(channel) + + async with trio.open_nursery() as nursery: + await nursery.start(a.serve, server_ctx, crashing_echo_handler) + await nursery.start(b.serve, server_ctx, echo_handler) + + b_to_a = b.connect(a.socket.getsockname(), client_ctx_fn()) + await b_to_a.send(b"b as client") + assert await b_to_a.receive() == b"b as client" + + a_to_b = a.connect(b.socket.getsockname(), client_ctx_fn()) + await a_to_b.do_handshake() + with pytest.raises(trio.BrokenResourceError): + await b_to_a.send(b"association broken") + await a_to_b.send(b"a as client") + assert await a_to_b.receive() == b"a as client" + + nursery.cancel_scope.cancel() + + +@slow +async def test_openssl_retransmit_doesnt_break_stuff( + server_ctx: SSL.Context, client_ctx: SSL.Context +) -> None: + # can't use autojump_clock here, because the point of the test is to wait for + # openssl's built-in retransmit timer to expire, which is hard-coded to use + # wall-clock time. + fn = FakeNet() + fn.enable() + + blackholed = True + + def route_packet(packet: UDPPacket) -> None: + if blackholed: + print("dropped packet", packet) + return + print("delivered packet", packet) + # packets.append( + # scapy.all.IP( + # src=packet.source.ip.compressed, dst=packet.destination.ip.compressed + # ) + # / scapy.all.UDP(sport=packet.source.port, dport=packet.destination.port) + # / packet.payload + # ) + fn.deliver_packet(packet) + + fn.route_packet = route_packet # type: ignore[assignment] # TODO add type annotations for FakeNet + + async with dtls_echo_server(server_ctx=server_ctx) as (server_endpoint, address): + with endpoint() as client_endpoint: + async with trio.open_nursery() as nursery: + + async def connecter() -> None: + client = client_endpoint.connect(address, client_ctx) + await client.do_handshake(initial_retransmit_timeout=1.5) + await client.send(b"hi") + assert await client.receive() == b"hi" + + nursery.start_soon(connecter) + + # openssl's default timeout is 1 second, so this ensures that it thinks + # the timeout has expired + await trio.sleep(1.1) + # disable blackholing and send a garbage packet to wake up openssl so it + # notices the timeout has expired + blackholed = False + await server_endpoint.socket.sendto( + b"xxx", + client_endpoint.socket.getsockname(), + ) + # now the client task should finish connecting and exit cleanly + + # scapy.all.wrpcap("/tmp/trace.pcap", packets) + + +async def test_initial_retransmit_timeout_configuration( + autojump_clock: trio.abc.Clock, server_ctx: SSL.Context +) -> None: + fn = FakeNet() + fn.enable() + + blackholed = True + + def route_packet(packet: UDPPacket) -> None: + nonlocal blackholed + if blackholed: + blackholed = False + else: + fn.deliver_packet(packet) + + fn.route_packet = route_packet # type: ignore[assignment] # TODO: add type annotations for FakeNet + + async with dtls_echo_server(server_ctx=server_ctx) as (_, address): + for t in [1, 2, 4]: + with endpoint() as client: + before = trio.current_time() + blackholed = True + channel = client.connect(address, client_ctx_fn()) + await channel.do_handshake(initial_retransmit_timeout=t) + after = trio.current_time() + assert after - before == t + + +async def test_explicit_tiny_mtu_is_respected( + server_ctx: SSL.Context, client_ctx: SSL.Context +) -> None: + # ClientHello is ~240 bytes, and it can't be fragmented, so our mtu has to + # be larger than that. (300 is still smaller than any real network though.) + MTU = 300 + + fn = FakeNet() + fn.enable() + + def route_packet(packet: UDPPacket) -> None: + print(f"delivering {packet}") + print(f"payload size: {len(packet.payload)}") + assert len(packet.payload) <= MTU + fn.deliver_packet(packet) + + fn.route_packet = route_packet # type: ignore[assignment] # TODO add type annotations for FakeNet + + async with dtls_echo_server(mtu=MTU, server_ctx=server_ctx) as (_server, address): + with endpoint() as client: + channel = client.connect(address, client_ctx) + channel.set_ciphertext_mtu(MTU) + await channel.do_handshake() + await channel.send(b"hi") + assert await channel.receive() == b"hi" + + +@parametrize_ipv6 +async def test_handshake_handles_minimum_network_mtu( + ipv6: bool, + autojump_clock: trio.abc.Clock, + server_ctx: SSL.Context, + client_ctx: SSL.Context, +) -> None: + # Fake network that has the minimum allowable MTU for whatever protocol we're using. + fn = FakeNet() + fn.enable() + + mtu = 1280 - 48 if ipv6 else 576 - 28 + + def route_packet(packet: UDPPacket) -> None: + if len(packet.payload) > mtu: + print(f"dropping {packet}") + else: + print(f"delivering {packet}") + fn.deliver_packet(packet) + + fn.route_packet = route_packet # type: ignore[assignment] # TODO: add type annotations for FakeNet + + # See if we can successfully do a handshake -- some of the volleys will get dropped, + # and the retransmit logic should detect this and back off the MTU to something + # smaller until it succeeds. + async with dtls_echo_server(ipv6=ipv6, server_ctx=server_ctx) as (_, address): + with endpoint(ipv6=ipv6) as client_endpoint: + client = client_endpoint.connect(address, client_ctx) + # the handshake mtu backoff shouldn't affect the return value from + # get_cleartext_mtu, b/c that's under the user's control via + # set_ciphertext_mtu + client.set_ciphertext_mtu(9999) + await client.send(b"xyz") + assert await client.receive() == b"xyz" + assert client.get_cleartext_mtu() > 9000 # as vegeta said + + +@pytest.mark.filterwarnings("always:unclosed DTLS:ResourceWarning") +async def test_system_task_cleaned_up_on_gc(client_ctx: SSL.Context) -> None: + before_tasks = trio.lowlevel.current_statistics().tasks_living + + # We put this into a sub-function so that everything automatically becomes garbage + # when the frame exits. For some reason just doing 'del e' wasn't enough on pypy + # with coverage enabled -- I think we were hitting this bug: + # https://foss.heptapod.net/pypy/pypy/-/issues/3656 + async def start_and_forget_endpoint() -> int: + e = endpoint() + + # This connection/handshake attempt can't succeed. The only purpose is to force + # the endpoint to set up a receive loop. + with trio.socket.socket(type=trio.socket.SOCK_DGRAM) as s: + await s.bind(("127.0.0.1", 0)) + c = e.connect(s.getsockname(), client_ctx) + async with trio.open_nursery() as nursery: + nursery.start_soon(c.do_handshake) + await trio.testing.wait_all_tasks_blocked() + nursery.cancel_scope.cancel() + + during_tasks = trio.lowlevel.current_statistics().tasks_living + return during_tasks + + with pytest.warns(ResourceWarning): # noqa: PT031 + during_tasks = await start_and_forget_endpoint() + await trio.testing.wait_all_tasks_blocked() + gc_collect_harder() + + await trio.testing.wait_all_tasks_blocked() + + after_tasks = trio.lowlevel.current_statistics().tasks_living + assert before_tasks < during_tasks + assert before_tasks == after_tasks + + +@pytest.mark.filterwarnings("always:unclosed DTLS:ResourceWarning") +async def test_gc_before_system_task_starts() -> None: + e = endpoint() + + with pytest.warns(ResourceWarning): # noqa: PT031 + del e + gc_collect_harder() + + await trio.testing.wait_all_tasks_blocked() + + +@pytest.mark.filterwarnings("always:unclosed DTLS:ResourceWarning") +async def test_gc_as_packet_received() -> None: + fn = FakeNet() + fn.enable() + + e = endpoint() + await e.socket.bind(("127.0.0.1", 0)) + e._ensure_receive_loop() + + await trio.testing.wait_all_tasks_blocked() + + with trio.socket.socket(type=trio.socket.SOCK_DGRAM) as s: + await s.sendto(b"xxx", e.socket.getsockname()) + # At this point, the endpoint's receive loop has been marked runnable because it + # just received a packet; closing the endpoint socket won't interrupt that. But by + # the time it wakes up to process the packet, the endpoint will be gone. + with pytest.warns(ResourceWarning): # noqa: PT031 + del e + gc_collect_harder() + + +@pytest.mark.filterwarnings("always:unclosed DTLS:ResourceWarning") +def test_gc_after_trio_exits() -> None: + async def main() -> DTLSEndpoint: + # We use fakenet just to make sure no real sockets can leak out of the test + # case - on pypy somehow the socket was outliving the gc_collect_harder call + # below. Since the test is just making sure DTLSEndpoint.__del__ doesn't explode + # when called after trio exits, it doesn't need a real socket. + fn = FakeNet() + fn.enable() + return endpoint() + + e = trio.run(main) + with pytest.warns(ResourceWarning): # noqa: PT031 + del e + gc_collect_harder() + + +async def test_already_closed_socket_doesnt_crash() -> None: + with endpoint() as e: + # We close the socket before checkpointing, so the socket will already be closed + # when the system task starts up + e.socket.close() + # Now give it a chance to start up, and hopefully not crash + await trio.testing.wait_all_tasks_blocked() + + +async def test_socket_closed_while_processing_clienthello( + autojump_clock: trio.abc.Clock, server_ctx: SSL.Context, client_ctx: SSL.Context +) -> None: + fn = FakeNet() + fn.enable() + + # Check what happens if the socket is discovered to be closed when sending a + # HelloVerifyRequest, since that has its own sending logic + async with dtls_echo_server(server_ctx=server_ctx) as (server, address): + + def route_packet(packet: UDPPacket) -> None: + fn.deliver_packet(packet) + server.socket.close() + + fn.route_packet = route_packet # type: ignore[assignment] # TODO add type annotations for FakeNet + + with endpoint() as client_endpoint: + with trio.move_on_after(10): + client = client_endpoint.connect(address, client_ctx) + await client.do_handshake() + + +async def test_association_replaced_while_handshake_running( + autojump_clock: trio.abc.Clock, server_ctx: SSL.Context +) -> None: + fn = FakeNet() + fn.enable() + + def route_packet(packet: UDPPacket) -> None: + pass + + fn.route_packet = route_packet # type: ignore[assignment] # TODO: add type annotations for FakeNet + + async with dtls_echo_server(server_ctx=server_ctx) as (_, address): + with endpoint() as client_endpoint: + # TODO: should this have the same exact client_ctx? + c1 = client_endpoint.connect(address, client_ctx_fn()) + async with trio.open_nursery() as nursery: + + async def doomed_handshake() -> None: + with pytest.raises(trio.BrokenResourceError): + await c1.do_handshake() + + nursery.start_soon(doomed_handshake) + + await trio.sleep(10) + + client_endpoint.connect(address, client_ctx_fn()) + + +async def test_association_replaced_before_handshake_starts( + server_ctx: SSL.Context, +) -> None: + fn = FakeNet() + fn.enable() + + # This test shouldn't send any packets + def route_packet(packet: UDPPacket) -> NoReturn: # pragma: no cover + raise AssertionError() + + fn.route_packet = route_packet # type: ignore[assignment] # TODO add type annotations for FakeNet + + async with dtls_echo_server(server_ctx=server_ctx) as (_, address): + with endpoint() as client_endpoint: + # TODO: should this use the same client_ctx? + c1 = client_endpoint.connect(address, client_ctx_fn()) + client_endpoint.connect(address, client_ctx_fn()) + with pytest.raises(trio.BrokenResourceError): + await c1.do_handshake() + + +async def test_send_to_closed_local_port(server_ctx: SSL.Context) -> None: + # On Windows, sending a UDP packet to a closed local port can cause a weird + # ECONNRESET error later, inside the receive task. Make sure we're handling it + # properly. + async with dtls_echo_server(server_ctx=server_ctx) as (_, address): + with endpoint() as client_endpoint: + async with trio.open_nursery() as nursery: + for i in range(1, 10): + channel = client_endpoint.connect(("127.0.0.1", i), client_ctx_fn()) + nursery.start_soon(channel.do_handshake) + channel = client_endpoint.connect(address, client_ctx_fn()) + await channel.send(b"xxx") + assert await channel.receive() == b"xxx" + nursery.cancel_scope.cancel() diff --git a/src/trio/_tests/test_exports.py b/src/trio/_tests/test_exports.py index a16556fe93..5328c88adc 100644 --- a/src/trio/_tests/test_exports.py +++ b/src/trio/_tests/test_exports.py @@ -1,3 +1,4 @@ +<<<<<<< HEAD from __future__ import annotations # isort: split import __future__ # Regular import, not special! @@ -245,10 +246,7 @@ def no_underscores(symbols: Iterable[str]) -> set[str]: # ignore warnings about deprecated module tests missing_names -= {"tests"} - missing_names -= { - "ALG_SET_PUB_KEY", - "WSA_FLAG_NO_HANDLE_INHERIT", - } # https://github.com/pypy/pypy/issues/5594 + missing_names -= {"ALG_SET_PUB_KEY", "WSA_FLAG_NO_HANDLE_INHERIT"} # https://github.com/pypy/pypy/issues/5594 if missing_names: # pragma: no cover print(f"{tool} can't see the following names in {modname}:") @@ -639,3 +637,643 @@ def test_pyright_recognizes_init_attributes() -> None: ] assert attributes == [], class_ +======= +from __future__ import annotations # isort: split + +import __future__ # Regular import, not special! + +import enum +import functools +import importlib +import inspect +import json +import socket as stdlib_socket +import sys +import types +from pathlib import Path, PurePath +from types import ModuleType +from typing import TYPE_CHECKING, Any, Protocol + +import attrs +import pytest + +import trio +import trio.testing +from trio._tests.pytest_plugin import RUN_SLOW, maybe_ignore_import_error + +from .. import _core, _util +from .._core._tests.tutil import slow + +if TYPE_CHECKING: + from collections.abc import Iterable, Iterator + +mypy_cache_updated = False + + +try: # If installed, check both versions of this class. + from typing_extensions import Protocol as Protocol_ext +except ImportError: # pragma: no cover + Protocol_ext = Protocol + + +def _ensure_mypy_cache_updated() -> None: + # This pollutes the `empty` dir. Should this be changed? + try: + from mypy.api import run + except ImportError as error: + maybe_ignore_import_error(error) + + global mypy_cache_updated + if not mypy_cache_updated: + # mypy cache was *probably* already updated by the other tests, + # but `pytest -k ...` might run just this test on its own + result = run( + [ + "--config-file=", + "--cache-dir=./.mypy_cache", + "--no-error-summary", + # TODO: update our tests to use the exposed APIs + # instead of reading JSON... or use dmypy? + "--no-fixed-format-cache", + "--no-sqlite-cache", + "-c", + "import trio", + ], + ) + assert not result[1] # stderr + assert not result[0] # stdout + mypy_cache_updated = True + + +def test_core_is_properly_reexported() -> None: + # Each export from _core should be re-exported by exactly one of these + # three modules: + sources = [trio, trio.lowlevel, trio.testing] + for symbol in dir(_core): + if symbol.startswith("_"): + continue + found = 0 + for source in sources: + if symbol in dir(source) and getattr(source, symbol) is getattr( + _core, + symbol, + ): + found += 1 + print(symbol, found) + assert found == 1 + + +def class_is_final(cls: type) -> bool: + """Check if a class cannot be subclassed.""" + try: + # new_class() handles metaclasses properly, type(...) does not. + types.new_class("SubclassTester", (cls,)) + except TypeError: + return True + else: + return False + + +def iter_modules( + module: types.ModuleType, + only_public: bool, +) -> Iterator[types.ModuleType]: + yield module + for name, class_ in module.__dict__.items(): + if name.startswith("_") and only_public: + continue + if not isinstance(class_, ModuleType): + continue + if not class_.__name__.startswith(module.__name__): # pragma: no cover + continue + if class_ is module: # pragma: no cover + continue + yield from iter_modules(class_, only_public) + + +PUBLIC_MODULES = list(iter_modules(trio, only_public=True)) +ALL_MODULES = list(iter_modules(trio, only_public=False)) +PUBLIC_MODULE_NAMES = [m.__name__ for m in PUBLIC_MODULES] + + +# It doesn't make sense for downstream redistributors to run this test, since +# they might be using a newer version of Python with additional symbols which +# won't be reflected in trio.socket, and this shouldn't cause downstream test +# runs to start failing. +@pytest.mark.redistributors_should_skip +@pytest.mark.skipif( + sys.version_info[:4] == (3, 14, 0, "beta"), + # 12 pass, 16 fail + reason="several tools don't support 3.14", +) +# Static analysis tools often have trouble with alpha releases, where Python's +# internals are in flux, grammar may not have settled down, etc. +@pytest.mark.skipif( + sys.version_info.releaselevel == "alpha", + reason="skip static introspection tools on Python dev/alpha releases", +) +@pytest.mark.parametrize("modname", PUBLIC_MODULE_NAMES) +@pytest.mark.parametrize( + "tool", + ["pylint", "jedi", "mypy", "pyright_verifytypes"], +) +@pytest.mark.filterwarnings( + # https://github.com/pypa/setuptools/issues/3274 + "ignore:module 'sre_constants' is deprecated:DeprecationWarning", +) +def test_static_tool_sees_all_symbols(tool: str, modname: str, tmp_path: Path) -> None: + module = importlib.import_module(modname) + + def no_underscores(symbols: Iterable[str]) -> set[str]: + return {symbol for symbol in symbols if not symbol.startswith("_")} + + runtime_names = no_underscores(dir(module)) + + # ignore deprecated module `tests` being invisible + if modname == "trio": + runtime_names.discard("tests") + + # Ignore any __future__ feature objects, if imported under that name. + for name in __future__.all_feature_names: + if getattr(module, name, None) is getattr(__future__, name): + runtime_names.remove(name) + + if tool == "pylint": + try: + from pylint.lint import PyLinter + except ImportError as error: + maybe_ignore_import_error(error) + + linter = PyLinter() + assert module.__file__ is not None + ast = linter.get_ast(module.__file__, modname) + static_names = no_underscores(ast) # type: ignore[arg-type] + elif tool == "jedi": + if sys.implementation.name != "cpython": + pytest.skip("jedi does not support pypy") + + try: + import jedi + except ImportError as error: + maybe_ignore_import_error(error) + + # Simulate typing "import trio; trio." + script = jedi.Script(f"import {modname}; {modname}.") + completions = script.complete() + static_names = no_underscores(c.name for c in completions) + elif tool == "mypy": + if sys.implementation.name != "cpython": + # https://github.com/python/mypy/issues/20329 + pytest.skip("mypy does not support pypy") + + if not RUN_SLOW: # pragma: no cover + pytest.skip("use --run-slow to check against mypy") + + cache = Path.cwd() / ".mypy_cache" + + _ensure_mypy_cache_updated() + + trio_cache = next(cache.glob("*/trio")) + _, modname = (modname + ".").split(".", 1) + modname = modname[:-1] + mod_cache = trio_cache / modname if modname else trio_cache + if mod_cache.is_dir(): # pragma: no coverage + mod_cache = mod_cache / "__init__.data.json" + else: + mod_cache = trio_cache / (modname + ".data.json") + + assert mod_cache.exists() + assert mod_cache.is_file() + with mod_cache.open() as cache_file: + cache_json = json.loads(cache_file.read()) + static_names = no_underscores( + key + for key, value in cache_json["names"].items() + if not key.startswith(".") and value["kind"] == "Gdef" + ) + elif tool == "pyright_verifytypes": + if not RUN_SLOW: # pragma: no cover + pytest.skip("use --run-slow to check against pyright") + + try: + import pyright # noqa: F401 + except ImportError as error: + maybe_ignore_import_error(error) + import subprocess + + res = subprocess.run( + ["pyright", f"--verifytypes={modname}", "--outputjson"], + capture_output=True, + ) + current_result = json.loads(res.stdout) + + static_names = { + x["name"][len(modname) + 1 :] + for x in current_result["typeCompleteness"]["symbols"] + if x["name"].startswith(modname) + } + else: # pragma: no cover + raise AssertionError() + + # It's expected that the static set will contain more names than the + # runtime set: + # - static tools are sometimes sloppy and include deleted names + # - some symbols are platform-specific at runtime, but always show up in + # static analysis (e.g. in trio.socket or trio.lowlevel) + # So we check that the runtime names are a subset of the static names. + missing_names = runtime_names - static_names + + # ignore warnings about deprecated module tests + missing_names -= {"tests"} + missing_names -= {"ALG_SET_PUB_KEY"} # https://github.com/pypy/pypy/issues/5594 + + if missing_names: # pragma: no cover + print(f"{tool} can't see the following names in {modname}:") + print() + for name in sorted(missing_names): + print(f" {name}") + raise AssertionError() + + +@slow +# see comment on test_static_tool_sees_all_symbols +@pytest.mark.redistributors_should_skip +# Static analysis tools often have trouble with alpha releases, where Python's +# internals are in flux, grammar may not have settled down, etc. +@pytest.mark.skipif( + sys.version_info.releaselevel == "alpha", + reason="skip static introspection tools on Python dev/alpha releases", +) +@pytest.mark.parametrize("module_name", PUBLIC_MODULE_NAMES) +@pytest.mark.parametrize("tool", ["jedi", "mypy"]) +def test_static_tool_sees_class_members( + tool: str, + module_name: str, + tmp_path: Path, +) -> None: + module = PUBLIC_MODULES[PUBLIC_MODULE_NAMES.index(module_name)] + + # ignore hidden, but not dunder, symbols + def no_hidden(symbols: Iterable[str]) -> set[str]: + return { + symbol + for symbol in symbols + if (not symbol.startswith("_")) or symbol.startswith("__") + } + + if tool == "jedi" and sys.implementation.name != "cpython": + pytest.skip("jedi does not support pypy") + + if tool == "mypy" and sys.implementation.name != "cpython": + # https://github.com/python/mypy/issues/20329 + pytest.skip("mypy does not support pypy") + + if tool == "mypy": + cache = Path.cwd() / ".mypy_cache" + + _ensure_mypy_cache_updated() + + trio_cache = next(cache.glob("*/trio")) + modname = module_name + _, modname = (modname + ".").split(".", 1) + modname = modname[:-1] + mod_cache = trio_cache / modname if modname else trio_cache + if mod_cache.is_dir(): + mod_cache = mod_cache / "__init__.data.json" + else: + mod_cache = trio_cache / (modname + ".data.json") + + assert mod_cache.exists() + assert mod_cache.is_file() + with mod_cache.open() as cache_file: + cache_json = json.loads(cache_file.read()) + + # skip a bunch of file-system activity (probably can un-memoize?) + @functools.lru_cache + def lookup_symbol(symbol: str) -> dict[str, Any]: # type: ignore[misc, explicit-any] + topname, *modname, name = symbol.split(".") + version = next(cache.glob("3.*/")) + mod_cache = version / topname + if not mod_cache.is_dir(): + mod_cache = version / (topname + ".data.json") + + if modname: + for piece in modname[:-1]: + mod_cache /= piece + next_cache = mod_cache / modname[-1] + if next_cache.is_dir(): # pragma: no coverage + mod_cache = next_cache / "__init__.data.json" + else: + mod_cache = mod_cache / (modname[-1] + ".data.json") + elif mod_cache.is_dir(): + mod_cache /= "__init__.data.json" + with mod_cache.open() as f: + return json.loads(f.read())["names"][name] # type: ignore[no-any-return] + + errors: dict[str, object] = {} + for class_name, class_ in module.__dict__.items(): + if not isinstance(class_, type): + continue + if module_name == "trio.socket" and class_name in dir(stdlib_socket): + continue + + # Ignore classes that don't use attrs, they only define their members once + # __init__ is called (and reason they don't use attrs is because they're going + # to be reimplemented in pytest). + # Not 100% that's the case, and it works locally, so whatever /shrug + if module_name == "trio.testing" and class_name in ("_RaisesGroup", "_Matcher"): + continue + + # dir() and inspect.getmembers doesn't display properties from the metaclass + # also ignore some dunder methods that tend to differ but are of no consequence + ignore_names = set(dir(type(class_))) | { + "__annotations__", + "__attrs_attrs__", + "__attrs_own_setattr__", + "__callable_proto_members_only__", + "__class_getitem__", + "__final__", + "__getstate__", + "__match_args__", + "__order__", + "__orig_bases__", + "__parameters__", + "__protocol_attrs__", + "__setstate__", + "__slots__", + "__weakref__", + # ignore errors about dunders inherited from stdlib that tools might + # not see + "__copy__", + "__deepcopy__", + } + + if type(class_) is type: + # C extension classes don't have these dunders, but Python classes do + ignore_names.add("__firstlineno__") + ignore_names.add("__static_attributes__") + + # inspect.getmembers sees `name` and `value` in Enums, otherwise + # it behaves the same way as `dir` + # runtime_names = no_underscores(dir(class_)) + runtime_names = ( + no_hidden(x[0] for x in inspect.getmembers(class_)) - ignore_names + ) + + if tool == "jedi": + try: + import jedi + except ImportError as error: + maybe_ignore_import_error(error) + + script = jedi.Script( + f"from {module_name} import {class_name}; {class_name}.", + ) + completions = script.complete() + static_names = no_hidden(c.name for c in completions) - ignore_names + + elif tool == "mypy": + # load the cached type information + cached_type_info = cache_json["names"][class_name] + assert ( + "node" not in cached_type_info + ), "previously this was an 'if' but it seems it's no longer possible for this cache to contain 'node', if this assert raises for you please let us know!" + cached_type_info = lookup_symbol(cached_type_info["cross_ref"]) + + assert "node" in cached_type_info + node = cached_type_info["node"] + static_names = no_hidden( + k for k in node.get("names", ()) if not k.startswith(".") + ) + for symbol in node["mro"][1:]: + node = lookup_symbol(symbol)["node"] + static_names |= no_hidden( + k for k in node.get("names", ()) if not k.startswith(".") + ) + static_names -= ignore_names + + else: # pragma: no cover + raise AssertionError("unknown tool") + + missing = runtime_names - static_names + extra = static_names - runtime_names + + # using .remove() instead of .delete() to get an error in case they start not + # being missing + + if BaseException in class_.__mro__ and sys.version_info >= (3, 11): + extra.remove("__notes__") + + if tool == "mypy" and attrs.has(class_): + # e.g. __trio__core__run_CancelScope_AttrsAttributes__ + before = len(extra) + extra = {e for e in extra if not e.endswith("AttrsAttributes__")} + assert len(extra) == before - 1 + + if ( + tool == "jedi" + and issubclass(class_, trio.Path) + and sys.version_info >= (3, 15) + ): + # needs a typeshed update for https://github.com/python/typeshed/pull/15737 + extra.remove("is_reserved") + + if attrs.has(class_): + # dynamically created attribute by attrs? + missing.remove("__attrs_props__") + + # dir does not see `__signature__` on enums until 3.14 + if ( + enum.Enum in class_.__mro__ + and sys.version_info >= (3, 12) + and sys.version_info < (3, 14) + ): + extra.remove("__signature__") + + # TODO: this *should* be visible via `dir`!! + if tool == "mypy" and class_ == trio.Nursery: + extra.remove("cancel_scope") + + # These are (mostly? solely?) *runtime* attributes, often set in + # __init__, which doesn't show up with dir() or inspect.getmembers, + # but we get them in the way we query mypy & jedi + EXTRAS = { + trio.DTLSChannel: {"peer_address", "endpoint"}, + trio.DTLSEndpoint: {"socket", "incoming_packets_buffer"}, + trio.Process: {"args", "pid", "stderr", "stdin", "stdio", "stdout"}, + trio.SSLListener: {"transport_listener"}, + trio.SSLStream: {"transport_stream"}, + trio.SocketListener: {"socket"}, + trio.SocketStream: {"socket"}, + trio.testing.MemoryReceiveStream: {"close_hook", "receive_some_hook"}, + trio.testing.MemorySendStream: { + "close_hook", + "send_all_hook", + "wait_send_all_might_not_block_hook", + }, + } + if tool == "mypy" and class_ in EXTRAS: + before = len(extra) + extra -= EXTRAS[class_] + assert len(extra) == before - len(EXTRAS[class_]) + + # TODO: why is this? Is it a problem? + # see https://github.com/python-trio/trio/pull/2631#discussion_r1185615916 + if class_ == trio.StapledStream: + extra.remove("receive_stream") + extra.remove("send_stream") + + # I have not researched why these are missing, should maybe create an issue + # upstream with jedi + if tool == "jedi" and sys.version_info >= (3, 11): + if class_ in ( + trio.DTLSChannel, + trio.MemoryReceiveChannel, + trio.MemorySendChannel, + trio.SSLListener, + trio.SocketListener, + ): + missing.remove("__aenter__") + missing.remove("__aexit__") + if class_ in (trio.DTLSChannel, trio.MemoryReceiveChannel): + missing.remove("__aiter__") + missing.remove("__anext__") + + if class_ in (trio.Path, trio.WindowsPath, trio.PosixPath): + # These are from inherited subclasses. + missing -= PurePath.__dict__.keys() + # These are unix-only. + if tool == "mypy" and sys.platform == "win32": + missing -= {"owner", "is_mount", "group"} + if tool == "jedi" and sys.platform == "win32": + extra -= {"owner", "is_mount", "group"} + + # tuple subclasses are weird + if tool == "mypy" and issubclass(class_, tuple): + extra.remove("__reversed__") + missing.remove("__getnewargs__") + + if sys.version_info >= (3, 13) and attrs.has(class_): + missing.remove("__replace__") + + if sys.version_info >= (3, 14): + # these depend on whether a class has processed deferred annotations. + # (which might or might not happen and we don't know) + missing.discard("__annotate_func__") + missing.discard("__annotations_cache__") + + if tool == "jedi" and class_ == trio.open_memory_channel: + # something is seriously wrong with jedi's understanding of open_memory_channel... + for attrib in ( + "__add__", + "__contains__", + "__getitem__", + "__getnewargs__", + "__iter__", + "__len__", + "__mul__", + "__rmul__", + "count", + "index", + ): + missing.remove(attrib) + + if missing or extra: # pragma: no cover + errors[f"{module_name}.{class_name}"] = { + "missing": missing, + "extra": extra, + } + + # `assert not errors` will not print the full content of errors, even with + # `--verbose`, so we manually print it + if errors: # pragma: no cover + from pprint import pprint + + print(f"\n{tool} can't see the following symbols in {module_name}:") + pprint(errors) + assert not errors + + +def test_nopublic_is_final() -> None: + """Check all NoPublicConstructor classes are also @final.""" + assert class_is_final(_util.NoPublicConstructor) # This is itself final. + + for module in ALL_MODULES: + for class_ in module.__dict__.values(): + if isinstance(class_, _util.NoPublicConstructor): + assert class_is_final(class_) + + +def test_classes_are_final() -> None: + # Sanity checks. + assert not class_is_final(object) + assert class_is_final(bool) + + for module in PUBLIC_MODULES: + for name, class_ in module.__dict__.items(): + if not isinstance(class_, type): + continue + # Deprecated classes are exported with a leading underscore + if name.startswith("_"): # pragma: no cover + continue + + # Abstract classes can be subclassed, because that's the whole + # point of ABCs + if inspect.isabstract(class_): + continue + # Same with protocols, but only direct children. + if Protocol in class_.__bases__ or Protocol_ext in class_.__bases__: + continue + # Exceptions are allowed to be subclassed, because exception + # subclassing isn't used to inherit behavior. + if issubclass(class_, BaseException): + continue + # These are classes that are conceptually abstract, but + # inspect.isabstract returns False for boring reasons. + if class_ is trio.abc.Instrument or class_ is trio.socket.SocketType: + continue + # ... insert other special cases here ... + + # The `Path` class needs to support inheritance to allow `WindowsPath` and `PosixPath`. + if class_ is trio.Path: + continue + # don't care about the *Statistics classes + if name.endswith("Statistics"): + continue + + assert class_is_final(class_) + + +# Plugin might not be running, especially if running from an installed version. +@pytest.mark.skipif( + not hasattr(attrs.field, "trio_modded"), + reason="Pytest plugin not installed.", +) +def test_pyright_recognizes_init_attributes() -> None: + """Check whether we provide `alias` for all underscore prefixed attributes. + + Attrs always sets the `alias` attribute on fields, so a pytest plugin is used + to monkeypatch `field()` to record whether an alias was defined in the metadata. + See `_trio_check_attrs_aliases`. + """ + for module in PUBLIC_MODULES: + for class_ in module.__dict__.values(): + if not attrs.has(class_): + continue + if isinstance(class_, _util.NoPublicConstructor): + continue + + attributes = [ + attr + for attr in attrs.fields(class_) + if attr.init + if attr.alias + not in ( + attr.name, + # trio_original_args may not be present in autoattribs + attr.metadata.get("trio_original_args", {}).get("alias"), + ) + ] + + assert attributes == [], class_ +>>>>>>> parent of fdc413bb (merge, bump crypto) diff --git a/src/trio/_tests/test_ssl.py b/src/trio/_tests/test_ssl.py index 0e30eccfd0..f0dd884c64 100644 --- a/src/trio/_tests/test_ssl.py +++ b/src/trio/_tests/test_ssl.py @@ -1,1372 +1,1372 @@ -from __future__ import annotations - -import os -import socket as stdlib_socket -import ssl -import sys -import threading -from contextlib import asynccontextmanager, contextmanager, suppress -from functools import partial -from ssl import SSLContext -from typing import TYPE_CHECKING, Any, NoReturn, TypeAlias - -import pytest - -from trio import StapledStream -from trio._tests.pytest_plugin import maybe_ignore_import_error -from trio.abc import ReceiveStream, SendStream -from trio.testing import MemoryReceiveStream, MemorySendStream - -try: - import trustme - from OpenSSL import SSL -except ImportError as error: - maybe_ignore_import_error(error) - -import trio - -from .. import _core, socket as tsocket -from .._abc import Stream -from .._core import BrokenResourceError, ClosedResourceError -from .._core._tests.tutil import slow -from .._highlevel_generic import aclose_forcefully -from .._highlevel_open_tcp_stream import open_tcp_stream -from .._highlevel_socket import SocketListener, SocketStream -from .._ssl import NeedHandshakeError, SSLListener, SSLStream, _is_eof -from .._util import ConflictDetector -from ..testing import ( - Sequencer, - assert_checkpoints, - check_two_way_stream, - lockstep_stream_pair, - memory_stream_pair, -) - -if TYPE_CHECKING: - from collections.abc import AsyncIterator, Awaitable, Callable, Iterator - - from trio._core import MockClock - from trio._ssl import T_Stream - - from .._core._run import CancelScope - -# We have two different kinds of echo server fixtures we use for testing. The -# first is a real server written using the stdlib ssl module and blocking -# sockets. It runs in a thread and we talk to it over a real socketpair(), to -# validate interoperability in a semi-realistic setting. -# -# The second is a very weird virtual echo server that lives inside a custom -# Stream class. It lives entirely inside the Python object space; there are no -# operating system calls in it at all. No threads, no I/O, nothing. It's -# 'send_all' call takes encrypted data from a client and feeds it directly into -# the server-side TLS state engine to decrypt, then takes that data, feeds it -# back through to get the encrypted response, and returns it from 'receive_some'. This -# gives us full control and reproducibility. This server is written using -# PyOpenSSL, so that we can trigger renegotiations on demand. It also allows -# us to insert random (virtual) delays, to really exercise all the weird paths -# in SSLStream's state engine. -# -# Both present a certificate for "trio-test-1.example.org". - -TRIO_TEST_CA = trustme.CA() -TRIO_TEST_1_CERT = TRIO_TEST_CA.issue_server_cert("trio-test-1.example.org") - -SERVER_CTX = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) -if hasattr(ssl, "OP_IGNORE_UNEXPECTED_EOF"): - SERVER_CTX.options &= ~ssl.OP_IGNORE_UNEXPECTED_EOF - -TRIO_TEST_1_CERT.configure_cert(SERVER_CTX) - - -# TLS 1.3 has a lot of changes from previous versions. So we want to run tests -# with both TLS 1.3, and TLS 1.2. -# "tls13" means that we're willing to negotiate TLS 1.3. Usually that's -# what will happen, but the renegotiation tests explicitly force a -# downgrade on the server side. "tls12" means we refuse to negotiate TLS -# 1.3, so we'll almost certainly use TLS 1.2. -@pytest.fixture(scope="module", params=["tls13", "tls12"]) -def client_ctx(request: pytest.FixtureRequest) -> ssl.SSLContext: - ctx = ssl.create_default_context() - - if hasattr(ssl, "OP_IGNORE_UNEXPECTED_EOF"): - ctx.options &= ~ssl.OP_IGNORE_UNEXPECTED_EOF - - TRIO_TEST_CA.configure_trust(ctx) - if request.param in ["default", "tls13"]: - return ctx - elif request.param == "tls12": - ctx.maximum_version = ssl.TLSVersion.TLSv1_2 - return ctx - else: # pragma: no cover - raise AssertionError() - - -# The blocking socket server. -def ssl_echo_serve_sync( - sock: stdlib_socket.socket, - *, - expect_fail: bool = False, -) -> None: - try: - wrapped = SERVER_CTX.wrap_socket( - sock, - server_side=True, - suppress_ragged_eofs=False, - ) - with wrapped: - wrapped.do_handshake() - while True: - data = wrapped.recv(4096) - if not data: - # other side has initiated a graceful shutdown; we try to - # respond in kind but it's legal for them to have already - # gone away. - with suppress(BrokenPipeError, ssl.SSLZeroReturnError): - wrapped.unwrap() - return - wrapped.sendall(data) - # This is an obscure workaround for an openssl bug. In server mode, in - # some versions, openssl sends some extra data at the end of do_handshake - # that it shouldn't send. Normally this is harmless, but, if the other - # side shuts down the connection before it reads that data, it might cause - # the OS to report a ECONNREST or even ECONNABORTED (which is just wrong, - # since ECONNABORTED is supposed to mean that connect() failed, but what - # can you do). In this case the other side did nothing wrong, but there's - # no way to recover, so we let it pass, and just cross our fingers its not - # hiding any (other) real bugs. For more details see: - # - # https://github.com/python-trio/trio/issues/1293 - # - # Also, this happens frequently but non-deterministically, so we have to - # 'no cover' it to avoid coverage flapping. - except (ConnectionResetError, ConnectionAbortedError): # pragma: no cover - return - except Exception as exc: - if expect_fail: - print("ssl_echo_serve_sync got error as expected:", exc) - else: # pragma: no cover - print("ssl_echo_serve_sync got unexpected error:", exc) - raise - else: - if expect_fail: # pragma: no cover - raise RuntimeError("failed to fail?") - finally: - sock.close() - - -# Fixture that gives a raw socket connected to a trio-test-1 echo server -# (running in a thread). Useful for testing making connections with different -# SSLContexts. -@asynccontextmanager -async def ssl_echo_server_raw(expect_fail: bool = False) -> AsyncIterator[SocketStream]: - a, b = stdlib_socket.socketpair() - async with trio.open_nursery() as nursery: - # Exiting the 'with a, b' context manager closes the sockets, which - # causes the thread to exit (possibly with an error), which allows the - # nursery context manager to exit too. - with a, b: - nursery.start_soon( - trio.to_thread.run_sync, - partial(ssl_echo_serve_sync, b, expect_fail=expect_fail), - ) - - yield SocketStream(tsocket.from_stdlib_socket(a)) - - -# Fixture that gives a properly set up SSLStream connected to a trio-test-1 -# echo server (running in a thread) -@asynccontextmanager -async def ssl_echo_server( - client_ctx: SSLContext, - expect_fail: bool = False, -) -> AsyncIterator[SSLStream[Stream]]: - async with ssl_echo_server_raw(expect_fail=expect_fail) as sock: - yield SSLStream(sock, client_ctx, server_hostname="trio-test-1.example.org") - - -# The weird in-memory server ... thing. -# Doesn't inherit from Stream because I left out the methods that we don't -# actually need. -# jakkdl: it seems to implement all the abstract methods (now), so I made it inherit -# from Stream for the sake of typechecking. -class PyOpenSSLEchoStream(Stream): - def __init__( - self, - sleeper: Callable[[str], Awaitable[None]] | None = None, - ) -> None: - ctx = SSL.Context(SSL.SSLv23_METHOD) - # TLS 1.3 removes renegotiation support. Which is great for them, but - # we still have to support versions before that, and that means we - # need to test renegotiation support, which means we need to force this - # to use a lower version where this test server can trigger - # renegotiations. - from cryptography.hazmat.bindings.openssl.binding import Binding - - b = Binding() - ctx.set_options(b.lib.SSL_OP_NO_TLSv1_3) - - # Unfortunately there's currently no way to say "use 1.3 or worse", we - # can only disable specific versions. And if the two sides start - # negotiating 1.4 at some point in the future, it *might* mean that - # our tests silently stop working properly. So the next line is a - # tripwire to remind us we need to revisit this stuff in 5 years or - # whatever when the next TLS version is released: - assert not hasattr(SSL, "OP_NO_TLSv1_4") - TRIO_TEST_1_CERT.configure_cert(ctx) - self._conn = SSL.Connection(ctx, None) - self._conn.set_accept_state() - self._lot = _core.ParkingLot() - self._pending_cleartext = bytearray() - - self._send_all_conflict_detector = ConflictDetector( - "simultaneous calls to PyOpenSSLEchoStream.send_all", - ) - self._receive_some_conflict_detector = ConflictDetector( - "simultaneous calls to PyOpenSSLEchoStream.receive_some", - ) - - self.sleeper: Callable[[str], Awaitable[None]] - if sleeper is None: - - async def no_op_sleeper(_: object) -> None: - return - - self.sleeper = no_op_sleeper - else: - self.sleeper = sleeper - - async def aclose(self) -> None: - self._conn.bio_shutdown() - - def renegotiate_pending(self) -> bool: - return self._conn.renegotiate_pending() - - def renegotiate(self) -> None: - # Returns false if a renegotiation is already in progress, meaning - # nothing happens. - assert self._conn.renegotiate() - - async def wait_send_all_might_not_block(self) -> None: - with self._send_all_conflict_detector: - await _core.checkpoint() - await _core.checkpoint() - await self.sleeper("wait_send_all_might_not_block") - - async def send_all(self, data: bytes | bytearray | memoryview[int]) -> None: - print(" --> transport_stream.send_all") - data_ = bytes(data) - with self._send_all_conflict_detector: - await _core.checkpoint() - await _core.checkpoint() - await self.sleeper("send_all") - self._conn.bio_write(data_) - while True: - await self.sleeper("send_all") - try: - data_ = self._conn.recv(1) - except SSL.ZeroReturnError: - self._conn.shutdown() - print("renegotiations:", self._conn.total_renegotiations()) - break - except SSL.WantReadError: - break - else: - self._pending_cleartext += data_ - self._lot.unpark_all() - await self.sleeper("send_all") - print(" <-- transport_stream.send_all finished") - - async def receive_some(self, nbytes: int | None = None) -> bytes: - print(" --> transport_stream.receive_some") - if nbytes is None: - nbytes = 65536 # arbitrary - with self._receive_some_conflict_detector: - try: - await _core.checkpoint() - await _core.checkpoint() - while True: - await self.sleeper("receive_some") - try: - return self._conn.bio_read(nbytes) - except SSL.WantReadError: - # No data in our ciphertext buffer; try to generate - # some. - if self._pending_cleartext: - # We have some cleartext; maybe we can encrypt it - # and then return it. - print(" trying", self._pending_cleartext) - try: - # PyOpenSSL bug: doesn't accept bytearray - # https://github.com/pyca/pyopenssl/issues/621 - next_byte = self._pending_cleartext[0:1] - self._conn.send(bytes(next_byte)) - # Apparently this next bit never gets hit in the - # test suite, but it's not an interesting omission - # so let's pragma it. - except SSL.WantReadError: # pragma: no cover - # We didn't manage to send the cleartext (and - # in particular we better leave it there to - # try again, due to openssl's retry - # semantics), but it's possible we pushed a - # renegotiation forward and *now* we have data - # to send. - try: - return self._conn.bio_read(nbytes) - except SSL.WantReadError: - # Nope. We're just going to have to wait - # for someone to call send_all() to give - # use more data. - print("parking (a)") - await self._lot.park() - else: - # We successfully sent that byte, so we don't - # have to again. - del self._pending_cleartext[0:1] - else: - # no pending cleartext; nothing to do but wait for - # someone to call send_all - print("parking (b)") - await self._lot.park() - finally: - await self.sleeper("receive_some") - print(" <-- transport_stream.receive_some finished") - - -async def test_PyOpenSSLEchoStream_gives_resource_busy_errors() -> None: - # Make sure that PyOpenSSLEchoStream complains if two tasks call send_all - # at the same time, or ditto for receive_some. The tricky cases where SSLStream - # might accidentally do this are during renegotiation, which we test using - # PyOpenSSLEchoStream, so this makes sure that if we do have a bug then - # PyOpenSSLEchoStream will notice and complain. - - async def do_test( - func1: str, - args1: tuple[object, ...], - func2: str, - args2: tuple[object, ...], - ) -> None: - s = PyOpenSSLEchoStream() - with pytest.RaisesGroup( - pytest.RaisesExc(_core.BusyResourceError, match="simultaneous") - ): - async with _core.open_nursery() as nursery: - nursery.start_soon(getattr(s, func1), *args1) - nursery.start_soon(getattr(s, func2), *args2) - - await do_test("send_all", (b"x",), "send_all", (b"x",)) - await do_test("send_all", (b"x",), "wait_send_all_might_not_block", ()) - await do_test( - "wait_send_all_might_not_block", - (), - "wait_send_all_might_not_block", - (), - ) - await do_test("receive_some", (1,), "receive_some", (1,)) - - -@contextmanager -def virtual_ssl_echo_server( - client_ctx: SSLContext, - sleeper: Callable[[str], Awaitable[None]] | None = None, -) -> Iterator[SSLStream[PyOpenSSLEchoStream]]: - fakesock = PyOpenSSLEchoStream(sleeper=sleeper) - yield SSLStream(fakesock, client_ctx, server_hostname="trio-test-1.example.org") - - -def ssl_wrap_pair( # type: ignore[explicit-any] - client_ctx: SSLContext, - client_transport: T_Stream, - server_transport: T_Stream, - *, - client_kwargs: dict[str, Any] | None = None, - server_kwargs: dict[str, Any] | None = None, -) -> tuple[SSLStream[T_Stream], SSLStream[T_Stream]]: - if server_kwargs is None: - server_kwargs = {} - if client_kwargs is None: - client_kwargs = {} - client_ssl = SSLStream( - client_transport, - client_ctx, - server_hostname="trio-test-1.example.org", - **client_kwargs, - ) - server_ssl = SSLStream( - server_transport, - SERVER_CTX, - server_side=True, - **server_kwargs, - ) - return client_ssl, server_ssl - - -MemoryStapledStream: TypeAlias = StapledStream[MemorySendStream, MemoryReceiveStream] - - -def ssl_memory_stream_pair( - client_ctx: SSLContext, - client_kwargs: dict[str, str | bytes | bool | None] | None = None, - server_kwargs: dict[str, str | bytes | bool | None] | None = None, -) -> tuple[ - SSLStream[MemoryStapledStream], - SSLStream[MemoryStapledStream], -]: - client_transport, server_transport = memory_stream_pair() - return ssl_wrap_pair( - client_ctx, - client_transport, - server_transport, - client_kwargs=client_kwargs, - server_kwargs=server_kwargs, - ) - - -MyStapledStream: TypeAlias = StapledStream[SendStream, ReceiveStream] - - -def ssl_lockstep_stream_pair( - client_ctx: SSLContext, - client_kwargs: dict[str, str | bytes | bool | None] | None = None, - server_kwargs: dict[str, str | bytes | bool | None] | None = None, -) -> tuple[ - SSLStream[MyStapledStream], - SSLStream[MyStapledStream], -]: - client_transport, server_transport = lockstep_stream_pair() - return ssl_wrap_pair( - client_ctx, - client_transport, - server_transport, - client_kwargs=client_kwargs, - server_kwargs=server_kwargs, - ) - - -# Simple smoke test for handshake/send/receive/shutdown talking to a -# synchronous server, plus make sure that we do the bare minimum of -# certificate checking (even though this is really Python's responsibility) -async def test_ssl_client_basics(client_ctx: SSLContext) -> None: - # Everything OK - async with ssl_echo_server(client_ctx) as s: - assert not s.server_side - await s.send_all(b"x") - assert await s.receive_some(1) == b"x" - await s.aclose() - - # Didn't configure the CA file, should fail - async with ssl_echo_server_raw(expect_fail=True) as sock: - bad_client_ctx = ssl.create_default_context() - s = SSLStream(sock, bad_client_ctx, server_hostname="trio-test-1.example.org") - assert not s.server_side - with pytest.raises(BrokenResourceError) as excinfo: - await s.send_all(b"x") - assert isinstance(excinfo.value.__cause__, ssl.SSLError) - - # Trusted CA, but wrong host name - async with ssl_echo_server_raw(expect_fail=True) as sock: - s = SSLStream(sock, client_ctx, server_hostname="trio-test-2.example.org") - assert not s.server_side - with pytest.raises(BrokenResourceError) as excinfo: - await s.send_all(b"x") - assert isinstance(excinfo.value.__cause__, ssl.CertificateError) - - -async def test_ssl_server_basics(client_ctx: SSLContext) -> None: - a, b = stdlib_socket.socketpair() - with a, b: - server_sock = tsocket.from_stdlib_socket(b) - server_transport = SSLStream( - SocketStream(server_sock), - SERVER_CTX, - server_side=True, - ) - assert server_transport.server_side - - def client() -> None: - with client_ctx.wrap_socket( - a, - server_hostname="trio-test-1.example.org", - ) as client_sock: - client_sock.sendall(b"x") - assert client_sock.recv(1) == b"y" - client_sock.sendall(b"z") - client_sock.unwrap() - - t = threading.Thread(target=client) - t.start() - - assert await server_transport.receive_some(1) == b"x" - await server_transport.send_all(b"y") - assert await server_transport.receive_some(1) == b"z" - assert await server_transport.receive_some(1) == b"" - await server_transport.aclose() - - t.join() - - -async def test_attributes(client_ctx: SSLContext) -> None: - async with ssl_echo_server_raw(expect_fail=True) as sock: - good_ctx = client_ctx - bad_ctx = ssl.create_default_context() - s = SSLStream(sock, good_ctx, server_hostname="trio-test-1.example.org") - - assert s.transport_stream is sock - - # Forwarded attribute getting - assert s.context is good_ctx - assert s.server_side == False # noqa - assert s.server_hostname == "trio-test-1.example.org" - with pytest.raises(AttributeError): - s.asfdasdfsa # noqa: B018 # "useless expression" - - # __dir__ - assert "transport_stream" in dir(s) - assert "context" in dir(s) - - # Setting the attribute goes through to the underlying object - - # most attributes on SSLObject are read-only - with pytest.raises(AttributeError): - s.server_side = True - with pytest.raises(AttributeError): - s.server_hostname = "asdf" - - # but .context is *not*. Check that we forward attribute setting by - # making sure that after we set the bad context our handshake indeed - # fails: - s.context = bad_ctx - assert s.context is bad_ctx - with pytest.raises(BrokenResourceError) as excinfo: - await s.do_handshake() - assert isinstance(excinfo.value.__cause__, ssl.SSLError) - - -# Note: this test fails horribly if we force TLS 1.2 and trigger a -# renegotiation at the beginning (e.g. by switching to the pyopenssl -# server). Usually the client crashes in SSLObject.write with "UNEXPECTED -# RECORD"; sometimes we get something more exotic like a SyscallError. This is -# odd because openssl isn't doing any syscalls, but so it goes. After lots of -# websearching I'm pretty sure this is due to a bug in OpenSSL, where it just -# can't reliably handle full-duplex communication combined with -# renegotiation. Nice, eh? -# -# https://rt.openssl.org/Ticket/Display.html?id=3712 -# https://rt.openssl.org/Ticket/Display.html?id=2481 -# http://openssl.6102.n7.nabble.com/TLS-renegotiation-failure-on-receiving-application-data-during-handshake-td48127.html -# https://stackoverflow.com/questions/18728355/ssl-renegotiation-with-full-duplex-socket-communication -# -# In some variants of this test (maybe only against the java server?) I've -# also seen cases where our send_all blocks waiting to write, and then our receive_some -# also blocks waiting to write, and they never wake up again. It looks like -# some kind of deadlock. I suspect there may be an issue where we've filled up -# the send buffers, and the remote side is trying to handle the renegotiation -# from inside a write() call, so it has a problem: there's all this application -# data clogging up the pipe, but it can't process and return it to the -# application because it's in write(), and it doesn't want to buffer infinite -# amounts of data, and... actually I guess those are the only two choices. -# -# NSS even documents that you shouldn't try to do a renegotiation except when -# the connection is idle: -# -# https://developer.mozilla.org/en-US/docs/Mozilla/Projects/NSS/SSL_functions/sslfnc.html#1061582 -# -# I begin to see why HTTP/2 forbids renegotiation and TLS 1.3 removes it... - - -async def test_full_duplex_basics(client_ctx: SSLContext) -> None: - CHUNKS = 30 - CHUNK_SIZE = 32768 - EXPECTED = CHUNKS * CHUNK_SIZE - - sent = bytearray() - received = bytearray() - - async def sender(s: Stream) -> None: - nonlocal sent - for i in range(CHUNKS): - print(i) - chunk = bytes([i] * CHUNK_SIZE) - sent += chunk - await s.send_all(chunk) - - async def receiver(s: Stream) -> None: - nonlocal received - while len(received) < EXPECTED: - chunk = await s.receive_some(CHUNK_SIZE // 2) - received += chunk - - async with ssl_echo_server(client_ctx) as s: - async with _core.open_nursery() as nursery: - nursery.start_soon(sender, s) - nursery.start_soon(receiver, s) - # And let's have some doing handshakes too, everyone - # simultaneously - nursery.start_soon(s.do_handshake) - nursery.start_soon(s.do_handshake) - - await s.aclose() - - assert len(sent) == len(received) == EXPECTED - assert sent == received - - -async def test_renegotiation_simple(client_ctx: SSLContext) -> None: - with virtual_ssl_echo_server(client_ctx) as s: - await s.do_handshake() - s.transport_stream.renegotiate() - await s.send_all(b"a") - assert await s.receive_some(1) == b"a" - - # Have to send some more data back and forth to make sure the - # renegotiation is finished before shutting down the - # connection... otherwise openssl raises an error. I think this is a - # bug in openssl but what can ya do. - await s.send_all(b"b") - assert await s.receive_some(1) == b"b" - - await s.aclose() - - -@slow -async def test_renegotiation_randomized( - mock_clock: MockClock, - client_ctx: SSLContext, -) -> None: - # The only blocking things in this function are our random sleeps, so 0 is - # a good threshold. - mock_clock.autojump_threshold = 0 - - import random - - r = random.Random(0) - - async def sleeper(_: object) -> None: - await trio.sleep(r.uniform(0, 10)) - - async def clear() -> None: - while s.transport_stream.renegotiate_pending(): - with assert_checkpoints(): - await send(b"-") - with assert_checkpoints(): - await expect(b"-") - print("-- clear --") - - async def send(byte: bytes) -> None: - await s.transport_stream.sleeper("outer send") - print("calling SSLStream.send_all", byte) - with assert_checkpoints(): - await s.send_all(byte) - - async def expect(expected: bytes) -> None: - await s.transport_stream.sleeper("expect") - print("calling SSLStream.receive_some, expecting", expected) - assert len(expected) == 1 - with assert_checkpoints(): - assert await s.receive_some(1) == expected - - with virtual_ssl_echo_server(client_ctx, sleeper=sleeper) as s: - await s.do_handshake() - - await send(b"a") - s.transport_stream.renegotiate() - await expect(b"a") - - await clear() - - for i in range(100): - b1 = bytes([i % 0xFF]) - b2 = bytes([(2 * i) % 0xFF]) - s.transport_stream.renegotiate() - async with _core.open_nursery() as nursery: - nursery.start_soon(send, b1) - nursery.start_soon(expect, b1) - async with _core.open_nursery() as nursery: - nursery.start_soon(expect, b2) - nursery.start_soon(send, b2) - await clear() - - for i in range(100): - b1 = bytes([i % 0xFF]) - b2 = bytes([(2 * i) % 0xFF]) - await send(b1) - s.transport_stream.renegotiate() - await expect(b1) - async with _core.open_nursery() as nursery: - nursery.start_soon(expect, b2) - nursery.start_soon(send, b2) - await clear() - - # Checking that wait_send_all_might_not_block and receive_some don't - # conflict: - - # 1) Set up a situation where expect (receive_some) is blocked sending, - # and wait_send_all_might_not_block comes in. - - # Our receive_some() call will get stuck when it hits send_all - async def sleeper_with_slow_send_all(method: str) -> None: - if method == "send_all": - # ignore ASYNC116, not sleep_forever, trying to test a large but finite sleep - await trio.sleep(100000) # noqa: ASYNC116 - - # And our wait_send_all_might_not_block call will give it time to get - # stuck, and then start - async def sleep_then_wait_writable() -> None: - await trio.sleep(1000) - await s.wait_send_all_might_not_block() - - with virtual_ssl_echo_server(client_ctx, sleeper=sleeper_with_slow_send_all) as s: - await send(b"x") - s.transport_stream.renegotiate() - async with _core.open_nursery() as nursery: - nursery.start_soon(expect, b"x") - nursery.start_soon(sleep_then_wait_writable) - - await clear() - - await s.aclose() - - # 2) Same, but now wait_send_all_might_not_block is stuck when - # receive_some tries to send. - - async def sleeper_with_slow_wait_writable_and_expect(method: str) -> None: - if method == "wait_send_all_might_not_block": - # ignore ASYNC116, not sleep_forever, trying to test a large but finite sleep - await trio.sleep(100000) # noqa: ASYNC116 - elif method == "expect": - await trio.sleep(1000) - - with virtual_ssl_echo_server( - client_ctx, - sleeper=sleeper_with_slow_wait_writable_and_expect, - ) as s: - await send(b"x") - s.transport_stream.renegotiate() - async with _core.open_nursery() as nursery: - nursery.start_soon(expect, b"x") - nursery.start_soon(s.wait_send_all_might_not_block) - - await clear() - - await s.aclose() - - -async def test_resource_busy_errors(client_ctx: SSLContext) -> None: - S: TypeAlias = trio.SSLStream[ - trio.StapledStream[trio.abc.SendStream, trio.abc.ReceiveStream] - ] - - async def do_send_all(s: S) -> None: - with assert_checkpoints(): - await s.send_all(b"x") - - async def do_receive_some(s: S) -> None: - with assert_checkpoints(): - await s.receive_some(1) - - async def do_wait_send_all_might_not_block(s: S) -> None: - with assert_checkpoints(): - await s.wait_send_all_might_not_block() - - async def do_test( - func1: Callable[[S], Awaitable[None]], - func2: Callable[[S], Awaitable[None]], - ) -> None: - s, _ = ssl_lockstep_stream_pair(client_ctx) - with pytest.RaisesGroup( - pytest.RaisesExc(_core.BusyResourceError, match="another task") - ): - async with _core.open_nursery() as nursery: - nursery.start_soon(func1, s) - nursery.start_soon(func2, s) - - await do_test(do_send_all, do_send_all) - await do_test(do_receive_some, do_receive_some) - await do_test(do_send_all, do_wait_send_all_might_not_block) - await do_test(do_wait_send_all_might_not_block, do_wait_send_all_might_not_block) - - -async def test_wait_writable_calls_underlying_wait_writable() -> None: - record = [] - - class NotAStream(Stream): - async def wait_send_all_might_not_block(self) -> None: - record.append("ok") - - # define methods that are abstract in Stream - async def aclose(self) -> None: - raise AssertionError("Should not get called") # pragma: no cover - - async def receive_some(self, max_bytes: int | None = None) -> bytes | bytearray: - raise AssertionError("Should not get called") # pragma: no cover - - async def send_all(self, data: bytes | bytearray | memoryview) -> None: - raise AssertionError("Should not get called") # pragma: no cover - - ctx = ssl.create_default_context() - s = SSLStream(NotAStream(), ctx, server_hostname="x") - await s.wait_send_all_might_not_block() - assert record == ["ok"] - - -@pytest.mark.skipif( - os.name == "nt" and sys.version_info >= (3, 10), - reason="frequently fails on Windows + Python 3.10", -) -async def test_checkpoints(client_ctx: SSLContext) -> None: - async with ssl_echo_server(client_ctx) as s: - with assert_checkpoints(): - await s.do_handshake() - with assert_checkpoints(): - await s.do_handshake() - with assert_checkpoints(): - await s.wait_send_all_might_not_block() - with assert_checkpoints(): - await s.send_all(b"xxx") - with assert_checkpoints(): - await s.receive_some(1) - # These receive_some's in theory could return immediately, because the - # "xxx" was sent in a single record and after the first - # receive_some(1) the rest are sitting inside the SSLObject's internal - # buffers. - with assert_checkpoints(): - await s.receive_some(1) - with assert_checkpoints(): - await s.receive_some(1) - with assert_checkpoints(): - await s.unwrap() - - async with ssl_echo_server(client_ctx) as s: - await s.do_handshake() - with assert_checkpoints(): - await s.aclose() - - -async def test_send_all_empty_string(client_ctx: SSLContext) -> None: - async with ssl_echo_server(client_ctx) as s: - await s.do_handshake() - - # underlying SSLObject interprets writing b"" as indicating an EOF, - # for some reason. Make sure we don't inherit this. - with assert_checkpoints(): - await s.send_all(b"") - with assert_checkpoints(): - await s.send_all(b"") - await s.send_all(b"x") - assert await s.receive_some(1) == b"x" - - await s.aclose() - - -@pytest.mark.parametrize("https_compatible", [False, True]) -async def test_SSLStream_generic( - client_ctx: SSLContext, - https_compatible: bool, -) -> None: - async def stream_maker() -> tuple[ - SSLStream[MemoryStapledStream], - SSLStream[MemoryStapledStream], - ]: - return ssl_memory_stream_pair( - client_ctx, - client_kwargs={"https_compatible": https_compatible}, - server_kwargs={"https_compatible": https_compatible}, - ) - - async def clogged_stream_maker() -> tuple[ - SSLStream[MyStapledStream], - SSLStream[MyStapledStream], - ]: - client, server = ssl_lockstep_stream_pair(client_ctx) - # If we don't do handshakes up front, then we run into a problem in - # the following situation: - # - server does wait_send_all_might_not_block - # - client does receive_some to unclog it - # Then the client's receive_some will actually send some data to start - # the handshake, and itself get stuck. - async with _core.open_nursery() as nursery: - nursery.start_soon(client.do_handshake) - nursery.start_soon(server.do_handshake) - return client, server - - await check_two_way_stream(stream_maker, clogged_stream_maker) - - -async def test_unwrap(client_ctx: SSLContext) -> None: - client_ssl, server_ssl = ssl_memory_stream_pair(client_ctx) - client_transport = client_ssl.transport_stream - server_transport = server_ssl.transport_stream - - seq = Sequencer() - - async def client() -> None: - await client_ssl.do_handshake() - await client_ssl.send_all(b"x") - assert await client_ssl.receive_some(1) == b"y" - await client_ssl.send_all(b"z") - - # After sending that, disable outgoing data from our end, to make - # sure the server doesn't see our EOF until after we've sent some - # trailing data - async with seq(0): - send_all_hook = client_transport.send_stream.send_all_hook - client_transport.send_stream.send_all_hook = None - - assert await client_ssl.receive_some(1) == b"" - assert client_ssl.transport_stream is client_transport - # We just received EOF. Unwrap the connection and send some more. - raw, trailing = await client_ssl.unwrap() - assert raw is client_transport - assert trailing == b"" - assert client_ssl.transport_stream is None - await raw.send_all(b"trailing") - - # Reconnect the streams. Now the server will receive both our shutdown - # acknowledgement + the trailing data in a single lump. - client_transport.send_stream.send_all_hook = send_all_hook - await client_transport.send_stream.send_all_hook() - - async def server() -> None: - await server_ssl.do_handshake() - assert await server_ssl.receive_some(1) == b"x" - await server_ssl.send_all(b"y") - assert await server_ssl.receive_some(1) == b"z" - # Now client is blocked waiting for us to send something, but - # instead we close the TLS connection (with sequencer to make sure - # that the client won't see and automatically respond before we've had - # a chance to disable the client->server transport) - async with seq(1): - raw, trailing = await server_ssl.unwrap() - assert raw is server_transport - assert trailing == b"trailing" - assert server_ssl.transport_stream is None - - async with _core.open_nursery() as nursery: - nursery.start_soon(client) - nursery.start_soon(server) - - -async def test_closing_nice_case(client_ctx: SSLContext) -> None: - # the nice case: graceful closes all around - - client_ssl, server_ssl = ssl_memory_stream_pair(client_ctx) - client_transport = client_ssl.transport_stream - - # Both the handshake and the close require back-and-forth discussion, so - # we need to run them concurrently - async def client_closer() -> None: - with assert_checkpoints(): - await client_ssl.aclose() - - async def server_closer() -> None: - assert await server_ssl.receive_some(10) == b"" - assert await server_ssl.receive_some(10) == b"" - with assert_checkpoints(): - await server_ssl.aclose() - - async with _core.open_nursery() as nursery: - nursery.start_soon(client_closer) - nursery.start_soon(server_closer) - - # closing the SSLStream also closes its transport - with pytest.raises(ClosedResourceError): - await client_transport.send_all(b"123") - - # once closed, it's OK to close again - with assert_checkpoints(): - await client_ssl.aclose() - with assert_checkpoints(): - await client_ssl.aclose() - - # Trying to send more data does not work - with pytest.raises(ClosedResourceError): - await server_ssl.send_all(b"123") - - # And once the connection is has been closed *locally*, then instead of - # getting empty bytestrings we get a proper error - with pytest.raises(ClosedResourceError): - assert await client_ssl.receive_some(10) == b"" - - with pytest.raises(ClosedResourceError): - await client_ssl.unwrap() - - with pytest.raises(ClosedResourceError): - await client_ssl.do_handshake() - - # Check that a graceful close *before* handshaking gives a clean EOF on - # the other side - client_ssl, server_ssl = ssl_memory_stream_pair(client_ctx) - - async def expect_eof_server() -> None: - with assert_checkpoints(): - assert await server_ssl.receive_some(10) == b"" - with assert_checkpoints(): - await server_ssl.aclose() - - async with _core.open_nursery() as nursery: - nursery.start_soon(client_ssl.aclose) - nursery.start_soon(expect_eof_server) - - -async def test_send_all_fails_in_the_middle(client_ctx: SSLContext) -> None: - client, server = ssl_memory_stream_pair(client_ctx) - - async with _core.open_nursery() as nursery: - nursery.start_soon(client.do_handshake) - nursery.start_soon(server.do_handshake) - - async def bad_hook() -> NoReturn: - raise KeyError - - client.transport_stream.send_stream.send_all_hook = bad_hook - - with pytest.raises(KeyError): - await client.send_all(b"x") - - with pytest.raises(BrokenResourceError): - await client.wait_send_all_might_not_block() - - closed = 0 - - def close_hook() -> None: - nonlocal closed - closed += 1 - - client.transport_stream.send_stream.close_hook = close_hook - client.transport_stream.receive_stream.close_hook = close_hook - await client.aclose() - - assert closed == 2 - - -async def test_ssl_over_ssl(client_ctx: SSLContext) -> None: - client_0, server_0 = memory_stream_pair() - - client_1 = SSLStream( - client_0, - client_ctx, - server_hostname="trio-test-1.example.org", - ) - server_1 = SSLStream(server_0, SERVER_CTX, server_side=True) - - client_2 = SSLStream( - client_1, - client_ctx, - server_hostname="trio-test-1.example.org", - ) - server_2 = SSLStream(server_1, SERVER_CTX, server_side=True) - - async def client() -> None: - await client_2.send_all(b"hi") - assert await client_2.receive_some(10) == b"bye" - - async def server() -> None: - assert await server_2.receive_some(10) == b"hi" - await server_2.send_all(b"bye") - - async with _core.open_nursery() as nursery: - nursery.start_soon(client) - nursery.start_soon(server) - - -async def test_ssl_bad_shutdown(client_ctx: SSLContext) -> None: - client, server = ssl_memory_stream_pair(client_ctx) - - async with _core.open_nursery() as nursery: - nursery.start_soon(client.do_handshake) - nursery.start_soon(server.do_handshake) - - await trio.aclose_forcefully(client) - # now the server sees a broken stream - with pytest.raises(BrokenResourceError): - await server.receive_some(10) - with pytest.raises(BrokenResourceError): - await server.send_all(b"x" * 10) - - await server.aclose() - - -async def test_ssl_bad_shutdown_but_its_ok(client_ctx: SSLContext) -> None: - client, server = ssl_memory_stream_pair( - client_ctx, - server_kwargs={"https_compatible": True}, - client_kwargs={"https_compatible": True}, - ) - - async with _core.open_nursery() as nursery: - nursery.start_soon(client.do_handshake) - nursery.start_soon(server.do_handshake) - - await trio.aclose_forcefully(client) - # the server sees that as a clean shutdown - assert await server.receive_some(10) == b"" - with pytest.raises(BrokenResourceError): - await server.send_all(b"x" * 10) - - await server.aclose() - - -async def test_ssl_handshake_failure_during_aclose() -> None: - # Weird scenario: aclose() triggers an automatic handshake, and this - # fails. This also exercises a bit of code in aclose() that was otherwise - # uncovered, for re-raising exceptions after calling aclose_forcefully on - # the underlying transport. - async with ssl_echo_server_raw(expect_fail=True) as sock: - # Don't configure trust correctly - client_ctx = ssl.create_default_context() - s = SSLStream(sock, client_ctx, server_hostname="trio-test-1.example.org") - # It's a little unclear here whether aclose should swallow the error - # or let it escape. We *do* swallow the error if it arrives when we're - # sending close_notify, because both sides closing the connection - # simultaneously is allowed. But I guess when https_compatible=False - # then it's bad if we can get through a whole connection with a peer - # that has no valid certificate, and never raise an error. - with pytest.raises(BrokenResourceError): - await s.aclose() - - -async def test_ssl_only_closes_stream_once(client_ctx: SSLContext) -> None: - # We used to have a bug where if transport_stream.aclose() raised an - # error, we would call it again. This checks that that's fixed. - client, server = ssl_memory_stream_pair(client_ctx) - - async with _core.open_nursery() as nursery: - nursery.start_soon(client.do_handshake) - nursery.start_soon(server.do_handshake) - - client_orig_close_hook = client.transport_stream.send_stream.close_hook - transport_close_count = 0 - - def close_hook() -> NoReturn: - nonlocal transport_close_count - assert client_orig_close_hook is not None - client_orig_close_hook() - transport_close_count += 1 - raise KeyError - - client.transport_stream.send_stream.close_hook = close_hook - - with pytest.raises(KeyError): - await client.aclose() - assert transport_close_count == 1 - - -async def test_ssl_https_compatibility_disagreement(client_ctx: SSLContext) -> None: - client, server = ssl_memory_stream_pair( - client_ctx, - server_kwargs={"https_compatible": False}, - client_kwargs={"https_compatible": True}, - ) - - async with _core.open_nursery() as nursery: - nursery.start_soon(client.do_handshake) - nursery.start_soon(server.do_handshake) - - # client is in HTTPS-mode, server is not - # so client doing graceful_shutdown causes an error on server - async def receive_and_expect_error() -> None: - with pytest.raises(BrokenResourceError) as excinfo: - await server.receive_some(10) - - assert _is_eof(excinfo.value.__cause__) - - async with _core.open_nursery() as nursery: - nursery.start_soon(client.aclose) - nursery.start_soon(receive_and_expect_error) - - -async def test_https_mode_eof_before_handshake(client_ctx: SSLContext) -> None: - client, server = ssl_memory_stream_pair( - client_ctx, - server_kwargs={"https_compatible": True}, - client_kwargs={"https_compatible": True}, - ) - - async def server_expect_clean_eof() -> None: - assert await server.receive_some(10) == b"" - - async with _core.open_nursery() as nursery: - nursery.start_soon(client.aclose) - nursery.start_soon(server_expect_clean_eof) - - -async def test_send_error_during_handshake(client_ctx: SSLContext) -> None: - client, _server = ssl_memory_stream_pair(client_ctx) - - async def bad_hook() -> NoReturn: - raise KeyError - - client.transport_stream.send_stream.send_all_hook = bad_hook - - with pytest.raises(KeyError): - with assert_checkpoints(): - await client.do_handshake() - - with pytest.raises(BrokenResourceError): - with assert_checkpoints(): - await client.do_handshake() - - -async def test_receive_error_during_handshake(client_ctx: SSLContext) -> None: - client, server = ssl_memory_stream_pair(client_ctx) - - async def bad_hook() -> NoReturn: - raise KeyError - - client.transport_stream.receive_stream.receive_some_hook = bad_hook - - async def client_side(cancel_scope: CancelScope) -> None: - with pytest.raises(KeyError): - with assert_checkpoints(): - await client.do_handshake() - cancel_scope.cancel() - - async with _core.open_nursery() as nursery: - nursery.start_soon(client_side, nursery.cancel_scope) - nursery.start_soon(server.do_handshake) - - with pytest.raises(BrokenResourceError): - with assert_checkpoints(): - await client.do_handshake() - - -def test_selected_alpn_protocol_before_handshake(client_ctx: SSLContext) -> None: - client, server = ssl_memory_stream_pair(client_ctx) - - with pytest.raises(NeedHandshakeError): - client.selected_alpn_protocol() - - with pytest.raises(NeedHandshakeError): - server.selected_alpn_protocol() - - -async def test_selected_alpn_protocol_when_not_set(client_ctx: SSLContext) -> None: - # ALPN protocol still returns None when it's not set, - # instead of raising an exception - client, server = ssl_memory_stream_pair(client_ctx) - - async with _core.open_nursery() as nursery: - nursery.start_soon(client.do_handshake) - nursery.start_soon(server.do_handshake) - - assert client.selected_alpn_protocol() is None - assert server.selected_alpn_protocol() is None - - assert client.selected_alpn_protocol() == server.selected_alpn_protocol() - - -def test_selected_npn_protocol_before_handshake(client_ctx: SSLContext) -> None: - client, server = ssl_memory_stream_pair(client_ctx) - - with pytest.raises(NeedHandshakeError): - client.selected_npn_protocol() - - with pytest.raises(NeedHandshakeError): - server.selected_npn_protocol() - - -@pytest.mark.filterwarnings( - r"ignore: ssl module. NPN is deprecated, use ALPN instead:UserWarning", - r"ignore:ssl NPN is deprecated, use ALPN instead:DeprecationWarning", -) -async def test_selected_npn_protocol_when_not_set(client_ctx: SSLContext) -> None: - # NPN protocol still returns None when it's not set, - # instead of raising an exception - client, server = ssl_memory_stream_pair(client_ctx) - - async with _core.open_nursery() as nursery: - nursery.start_soon(client.do_handshake) - nursery.start_soon(server.do_handshake) - - assert client.selected_npn_protocol() is None - assert server.selected_npn_protocol() is None - - assert client.selected_npn_protocol() == server.selected_npn_protocol() - - -def test_get_channel_binding_before_handshake(client_ctx: SSLContext) -> None: - client, server = ssl_memory_stream_pair(client_ctx) - - with pytest.raises(NeedHandshakeError): - client.get_channel_binding() - - with pytest.raises(NeedHandshakeError): - server.get_channel_binding() - - -async def test_get_channel_binding_after_handshake(client_ctx: SSLContext) -> None: - client, server = ssl_memory_stream_pair(client_ctx) - - async with _core.open_nursery() as nursery: - nursery.start_soon(client.do_handshake) - nursery.start_soon(server.do_handshake) - - assert client.get_channel_binding() is not None - assert server.get_channel_binding() is not None - - assert client.get_channel_binding() == server.get_channel_binding() - - -async def test_getpeercert(client_ctx: SSLContext) -> None: - # Make sure we're not affected by https://bugs.python.org/issue29334 - client, server = ssl_memory_stream_pair(client_ctx) - - async with _core.open_nursery() as nursery: - nursery.start_soon(client.do_handshake) - nursery.start_soon(server.do_handshake) - - assert server.getpeercert() is None - print(client.getpeercert()) - assert ("DNS", "trio-test-1.example.org") in client.getpeercert()["subjectAltName"] - - -async def test_SSLListener(client_ctx: SSLContext) -> None: - async def setup( - https_compatible: bool = False, - ) -> tuple[tsocket.SocketType, SSLListener[SocketStream], SSLStream[SocketStream]]: - listen_sock = tsocket.socket() - await listen_sock.bind(("127.0.0.1", 0)) - listen_sock.listen(1) - socket_listener = SocketListener(listen_sock) - ssl_listener = SSLListener( - socket_listener, - SERVER_CTX, - https_compatible=https_compatible, - ) - - transport_client = await open_tcp_stream(*listen_sock.getsockname()) - ssl_client = SSLStream( - transport_client, - client_ctx, - server_hostname="trio-test-1.example.org", - ) - return listen_sock, ssl_listener, ssl_client - - listen_sock, ssl_listener, ssl_client = await setup() - - async with ssl_client: - ssl_server = await ssl_listener.accept() - - async with ssl_server: - assert not ssl_server._https_compatible - - # Make sure the connection works - async with _core.open_nursery() as nursery: - nursery.start_soon(ssl_client.do_handshake) - nursery.start_soon(ssl_server.do_handshake) - - # Test SSLListener.aclose - await ssl_listener.aclose() - assert listen_sock.fileno() == -1 - - ################ - - # Test https_compatible - _, ssl_listener, ssl_client = await setup(https_compatible=True) - - ssl_server = await ssl_listener.accept() - - assert ssl_server._https_compatible - - await aclose_forcefully(ssl_listener) - await aclose_forcefully(ssl_client) - await aclose_forcefully(ssl_server) +from __future__ import annotations + +import os +import socket as stdlib_socket +import ssl +import sys +import threading +from contextlib import asynccontextmanager, contextmanager, suppress +from functools import partial +from ssl import SSLContext +from typing import TYPE_CHECKING, Any, NoReturn, TypeAlias + +import pytest + +from trio import StapledStream +from trio._tests.pytest_plugin import maybe_ignore_import_error +from trio.abc import ReceiveStream, SendStream +from trio.testing import MemoryReceiveStream, MemorySendStream + +try: + import trustme + from OpenSSL import SSL +except ImportError as error: + maybe_ignore_import_error(error) + +import trio + +from .. import _core, socket as tsocket +from .._abc import Stream +from .._core import BrokenResourceError, ClosedResourceError +from .._core._tests.tutil import slow +from .._highlevel_generic import aclose_forcefully +from .._highlevel_open_tcp_stream import open_tcp_stream +from .._highlevel_socket import SocketListener, SocketStream +from .._ssl import NeedHandshakeError, SSLListener, SSLStream, _is_eof +from .._util import ConflictDetector +from ..testing import ( + Sequencer, + assert_checkpoints, + check_two_way_stream, + lockstep_stream_pair, + memory_stream_pair, +) + +if TYPE_CHECKING: + from collections.abc import AsyncIterator, Awaitable, Callable, Iterator + + from trio._core import MockClock + from trio._ssl import T_Stream + + from .._core._run import CancelScope + +# We have two different kinds of echo server fixtures we use for testing. The +# first is a real server written using the stdlib ssl module and blocking +# sockets. It runs in a thread and we talk to it over a real socketpair(), to +# validate interoperability in a semi-realistic setting. +# +# The second is a very weird virtual echo server that lives inside a custom +# Stream class. It lives entirely inside the Python object space; there are no +# operating system calls in it at all. No threads, no I/O, nothing. It's +# 'send_all' call takes encrypted data from a client and feeds it directly into +# the server-side TLS state engine to decrypt, then takes that data, feeds it +# back through to get the encrypted response, and returns it from 'receive_some'. This +# gives us full control and reproducibility. This server is written using +# PyOpenSSL, so that we can trigger renegotiations on demand. It also allows +# us to insert random (virtual) delays, to really exercise all the weird paths +# in SSLStream's state engine. +# +# Both present a certificate for "trio-test-1.example.org". + +TRIO_TEST_CA = trustme.CA() +TRIO_TEST_1_CERT = TRIO_TEST_CA.issue_server_cert("trio-test-1.example.org") + +SERVER_CTX = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) +if hasattr(ssl, "OP_IGNORE_UNEXPECTED_EOF"): + SERVER_CTX.options &= ~ssl.OP_IGNORE_UNEXPECTED_EOF + +TRIO_TEST_1_CERT.configure_cert(SERVER_CTX) + + +# TLS 1.3 has a lot of changes from previous versions. So we want to run tests +# with both TLS 1.3, and TLS 1.2. +# "tls13" means that we're willing to negotiate TLS 1.3. Usually that's +# what will happen, but the renegotiation tests explicitly force a +# downgrade on the server side. "tls12" means we refuse to negotiate TLS +# 1.3, so we'll almost certainly use TLS 1.2. +@pytest.fixture(scope="module", params=["tls13", "tls12"]) +def client_ctx(request: pytest.FixtureRequest) -> ssl.SSLContext: + ctx = ssl.create_default_context() + + if hasattr(ssl, "OP_IGNORE_UNEXPECTED_EOF"): + ctx.options &= ~ssl.OP_IGNORE_UNEXPECTED_EOF + + TRIO_TEST_CA.configure_trust(ctx) + if request.param in ["default", "tls13"]: + return ctx + elif request.param == "tls12": + ctx.maximum_version = ssl.TLSVersion.TLSv1_2 + return ctx + else: # pragma: no cover + raise AssertionError() + + +# The blocking socket server. +def ssl_echo_serve_sync( + sock: stdlib_socket.socket, + *, + expect_fail: bool = False, +) -> None: + try: + wrapped = SERVER_CTX.wrap_socket( + sock, + server_side=True, + suppress_ragged_eofs=False, + ) + with wrapped: + wrapped.do_handshake() + while True: + data = wrapped.recv(4096) + if not data: + # other side has initiated a graceful shutdown; we try to + # respond in kind but it's legal for them to have already + # gone away. + with suppress(BrokenPipeError, ssl.SSLZeroReturnError): + wrapped.unwrap() + return + wrapped.sendall(data) + # This is an obscure workaround for an openssl bug. In server mode, in + # some versions, openssl sends some extra data at the end of do_handshake + # that it shouldn't send. Normally this is harmless, but, if the other + # side shuts down the connection before it reads that data, it might cause + # the OS to report a ECONNREST or even ECONNABORTED (which is just wrong, + # since ECONNABORTED is supposed to mean that connect() failed, but what + # can you do). In this case the other side did nothing wrong, but there's + # no way to recover, so we let it pass, and just cross our fingers its not + # hiding any (other) real bugs. For more details see: + # + # https://github.com/python-trio/trio/issues/1293 + # + # Also, this happens frequently but non-deterministically, so we have to + # 'no cover' it to avoid coverage flapping. + except (ConnectionResetError, ConnectionAbortedError): # pragma: no cover + return + except Exception as exc: + if expect_fail: + print("ssl_echo_serve_sync got error as expected:", exc) + else: # pragma: no cover + print("ssl_echo_serve_sync got unexpected error:", exc) + raise + else: + if expect_fail: # pragma: no cover + raise RuntimeError("failed to fail?") + finally: + sock.close() + + +# Fixture that gives a raw socket connected to a trio-test-1 echo server +# (running in a thread). Useful for testing making connections with different +# SSLContexts. +@asynccontextmanager +async def ssl_echo_server_raw(expect_fail: bool = False) -> AsyncIterator[SocketStream]: + a, b = stdlib_socket.socketpair() + async with trio.open_nursery() as nursery: + # Exiting the 'with a, b' context manager closes the sockets, which + # causes the thread to exit (possibly with an error), which allows the + # nursery context manager to exit too. + with a, b: + nursery.start_soon( + trio.to_thread.run_sync, + partial(ssl_echo_serve_sync, b, expect_fail=expect_fail), + ) + + yield SocketStream(tsocket.from_stdlib_socket(a)) + + +# Fixture that gives a properly set up SSLStream connected to a trio-test-1 +# echo server (running in a thread) +@asynccontextmanager +async def ssl_echo_server( + client_ctx: SSLContext, + expect_fail: bool = False, +) -> AsyncIterator[SSLStream[Stream]]: + async with ssl_echo_server_raw(expect_fail=expect_fail) as sock: + yield SSLStream(sock, client_ctx, server_hostname="trio-test-1.example.org") + + +# The weird in-memory server ... thing. +# Doesn't inherit from Stream because I left out the methods that we don't +# actually need. +# jakkdl: it seems to implement all the abstract methods (now), so I made it inherit +# from Stream for the sake of typechecking. +class PyOpenSSLEchoStream(Stream): + def __init__( + self, + sleeper: Callable[[str], Awaitable[None]] | None = None, + ) -> None: + ctx = SSL.Context(SSL.SSLv23_METHOD) + # TLS 1.3 removes renegotiation support. Which is great for them, but + # we still have to support versions before that, and that means we + # need to test renegotiation support, which means we need to force this + # to use a lower version where this test server can trigger + # renegotiations. + from cryptography.hazmat.bindings.openssl.binding import Binding + + b = Binding() + ctx.set_options(b.lib.SSL_OP_NO_TLSv1_3) + + # Unfortunately there's currently no way to say "use 1.3 or worse", we + # can only disable specific versions. And if the two sides start + # negotiating 1.4 at some point in the future, it *might* mean that + # our tests silently stop working properly. So the next line is a + # tripwire to remind us we need to revisit this stuff in 5 years or + # whatever when the next TLS version is released: + assert not hasattr(SSL, "OP_NO_TLSv1_4") + TRIO_TEST_1_CERT.configure_cert(ctx) + self._conn = SSL.Connection(ctx, None) + self._conn.set_accept_state() + self._lot = _core.ParkingLot() + self._pending_cleartext = bytearray() + + self._send_all_conflict_detector = ConflictDetector( + "simultaneous calls to PyOpenSSLEchoStream.send_all", + ) + self._receive_some_conflict_detector = ConflictDetector( + "simultaneous calls to PyOpenSSLEchoStream.receive_some", + ) + + self.sleeper: Callable[[str], Awaitable[None]] + if sleeper is None: + + async def no_op_sleeper(_: object) -> None: + return + + self.sleeper = no_op_sleeper + else: + self.sleeper = sleeper + + async def aclose(self) -> None: + self._conn.bio_shutdown() + + def renegotiate_pending(self) -> bool: + return self._conn.renegotiate_pending() + + def renegotiate(self) -> None: + # Returns false if a renegotiation is already in progress, meaning + # nothing happens. + assert self._conn.renegotiate() + + async def wait_send_all_might_not_block(self) -> None: + with self._send_all_conflict_detector: + await _core.checkpoint() + await _core.checkpoint() + await self.sleeper("wait_send_all_might_not_block") + + async def send_all(self, data: bytes | bytearray | memoryview[int]) -> None: + print(" --> transport_stream.send_all") + data_ = bytes(data) + with self._send_all_conflict_detector: + await _core.checkpoint() + await _core.checkpoint() + await self.sleeper("send_all") + self._conn.bio_write(data_) + while True: + await self.sleeper("send_all") + try: + data_ = self._conn.recv(1) + except SSL.ZeroReturnError: + self._conn.shutdown() + print("renegotiations:", self._conn.total_renegotiations()) + break + except SSL.WantReadError: + break + else: + self._pending_cleartext += data_ + self._lot.unpark_all() + await self.sleeper("send_all") + print(" <-- transport_stream.send_all finished") + + async def receive_some(self, nbytes: int | None = None) -> bytes: + print(" --> transport_stream.receive_some") + if nbytes is None: + nbytes = 65536 # arbitrary + with self._receive_some_conflict_detector: + try: + await _core.checkpoint() + await _core.checkpoint() + while True: + await self.sleeper("receive_some") + try: + return self._conn.bio_read(nbytes) + except SSL.WantReadError: + # No data in our ciphertext buffer; try to generate + # some. + if self._pending_cleartext: + # We have some cleartext; maybe we can encrypt it + # and then return it. + print(" trying", self._pending_cleartext) + try: + # PyOpenSSL bug: doesn't accept bytearray + # https://github.com/pyca/pyopenssl/issues/621 + next_byte = self._pending_cleartext[0:1] + self._conn.send(bytes(next_byte)) + # Apparently this next bit never gets hit in the + # test suite, but it's not an interesting omission + # so let's pragma it. + except SSL.WantReadError: # pragma: no cover + # We didn't manage to send the cleartext (and + # in particular we better leave it there to + # try again, due to openssl's retry + # semantics), but it's possible we pushed a + # renegotiation forward and *now* we have data + # to send. + try: + return self._conn.bio_read(nbytes) + except SSL.WantReadError: + # Nope. We're just going to have to wait + # for someone to call send_all() to give + # use more data. + print("parking (a)") + await self._lot.park() + else: + # We successfully sent that byte, so we don't + # have to again. + del self._pending_cleartext[0:1] + else: + # no pending cleartext; nothing to do but wait for + # someone to call send_all + print("parking (b)") + await self._lot.park() + finally: + await self.sleeper("receive_some") + print(" <-- transport_stream.receive_some finished") + + +async def test_PyOpenSSLEchoStream_gives_resource_busy_errors() -> None: + # Make sure that PyOpenSSLEchoStream complains if two tasks call send_all + # at the same time, or ditto for receive_some. The tricky cases where SSLStream + # might accidentally do this are during renegotiation, which we test using + # PyOpenSSLEchoStream, so this makes sure that if we do have a bug then + # PyOpenSSLEchoStream will notice and complain. + + async def do_test( + func1: str, + args1: tuple[object, ...], + func2: str, + args2: tuple[object, ...], + ) -> None: + s = PyOpenSSLEchoStream() + with pytest.RaisesGroup( + pytest.RaisesExc(_core.BusyResourceError, match="simultaneous") + ): + async with _core.open_nursery() as nursery: + nursery.start_soon(getattr(s, func1), *args1) + nursery.start_soon(getattr(s, func2), *args2) + + await do_test("send_all", (b"x",), "send_all", (b"x",)) + await do_test("send_all", (b"x",), "wait_send_all_might_not_block", ()) + await do_test( + "wait_send_all_might_not_block", + (), + "wait_send_all_might_not_block", + (), + ) + await do_test("receive_some", (1,), "receive_some", (1,)) + + +@contextmanager +def virtual_ssl_echo_server( + client_ctx: SSLContext, + sleeper: Callable[[str], Awaitable[None]] | None = None, +) -> Iterator[SSLStream[PyOpenSSLEchoStream]]: + fakesock = PyOpenSSLEchoStream(sleeper=sleeper) + yield SSLStream(fakesock, client_ctx, server_hostname="trio-test-1.example.org") + + +def ssl_wrap_pair( # type: ignore[explicit-any] + client_ctx: SSLContext, + client_transport: T_Stream, + server_transport: T_Stream, + *, + client_kwargs: dict[str, Any] | None = None, + server_kwargs: dict[str, Any] | None = None, +) -> tuple[SSLStream[T_Stream], SSLStream[T_Stream]]: + if server_kwargs is None: + server_kwargs = {} + if client_kwargs is None: + client_kwargs = {} + client_ssl = SSLStream( + client_transport, + client_ctx, + server_hostname="trio-test-1.example.org", + **client_kwargs, + ) + server_ssl = SSLStream( + server_transport, + SERVER_CTX, + server_side=True, + **server_kwargs, + ) + return client_ssl, server_ssl + + +MemoryStapledStream: TypeAlias = StapledStream[MemorySendStream, MemoryReceiveStream] + + +def ssl_memory_stream_pair( + client_ctx: SSLContext, + client_kwargs: dict[str, str | bytes | bool | None] | None = None, + server_kwargs: dict[str, str | bytes | bool | None] | None = None, +) -> tuple[ + SSLStream[MemoryStapledStream], + SSLStream[MemoryStapledStream], +]: + client_transport, server_transport = memory_stream_pair() + return ssl_wrap_pair( + client_ctx, + client_transport, + server_transport, + client_kwargs=client_kwargs, + server_kwargs=server_kwargs, + ) + + +MyStapledStream: TypeAlias = StapledStream[SendStream, ReceiveStream] + + +def ssl_lockstep_stream_pair( + client_ctx: SSLContext, + client_kwargs: dict[str, str | bytes | bool | None] | None = None, + server_kwargs: dict[str, str | bytes | bool | None] | None = None, +) -> tuple[ + SSLStream[MyStapledStream], + SSLStream[MyStapledStream], +]: + client_transport, server_transport = lockstep_stream_pair() + return ssl_wrap_pair( + client_ctx, + client_transport, + server_transport, + client_kwargs=client_kwargs, + server_kwargs=server_kwargs, + ) + + +# Simple smoke test for handshake/send/receive/shutdown talking to a +# synchronous server, plus make sure that we do the bare minimum of +# certificate checking (even though this is really Python's responsibility) +async def test_ssl_client_basics(client_ctx: SSLContext) -> None: + # Everything OK + async with ssl_echo_server(client_ctx) as s: + assert not s.server_side + await s.send_all(b"x") + assert await s.receive_some(1) == b"x" + await s.aclose() + + # Didn't configure the CA file, should fail + async with ssl_echo_server_raw(expect_fail=True) as sock: + bad_client_ctx = ssl.create_default_context() + s = SSLStream(sock, bad_client_ctx, server_hostname="trio-test-1.example.org") + assert not s.server_side + with pytest.raises(BrokenResourceError) as excinfo: + await s.send_all(b"x") + assert isinstance(excinfo.value.__cause__, ssl.SSLError) + + # Trusted CA, but wrong host name + async with ssl_echo_server_raw(expect_fail=True) as sock: + s = SSLStream(sock, client_ctx, server_hostname="trio-test-2.example.org") + assert not s.server_side + with pytest.raises(BrokenResourceError) as excinfo: + await s.send_all(b"x") + assert isinstance(excinfo.value.__cause__, ssl.CertificateError) + + +async def test_ssl_server_basics(client_ctx: SSLContext) -> None: + a, b = stdlib_socket.socketpair() + with a, b: + server_sock = tsocket.from_stdlib_socket(b) + server_transport = SSLStream( + SocketStream(server_sock), + SERVER_CTX, + server_side=True, + ) + assert server_transport.server_side + + def client() -> None: + with client_ctx.wrap_socket( + a, + server_hostname="trio-test-1.example.org", + ) as client_sock: + client_sock.sendall(b"x") + assert client_sock.recv(1) == b"y" + client_sock.sendall(b"z") + client_sock.unwrap() + + t = threading.Thread(target=client) + t.start() + + assert await server_transport.receive_some(1) == b"x" + await server_transport.send_all(b"y") + assert await server_transport.receive_some(1) == b"z" + assert await server_transport.receive_some(1) == b"" + await server_transport.aclose() + + t.join() + + +async def test_attributes(client_ctx: SSLContext) -> None: + async with ssl_echo_server_raw(expect_fail=True) as sock: + good_ctx = client_ctx + bad_ctx = ssl.create_default_context() + s = SSLStream(sock, good_ctx, server_hostname="trio-test-1.example.org") + + assert s.transport_stream is sock + + # Forwarded attribute getting + assert s.context is good_ctx + assert s.server_side == False # noqa + assert s.server_hostname == "trio-test-1.example.org" + with pytest.raises(AttributeError): + s.asfdasdfsa # noqa: B018 # "useless expression" + + # __dir__ + assert "transport_stream" in dir(s) + assert "context" in dir(s) + + # Setting the attribute goes through to the underlying object + + # most attributes on SSLObject are read-only + with pytest.raises(AttributeError): + s.server_side = True + with pytest.raises(AttributeError): + s.server_hostname = "asdf" + + # but .context is *not*. Check that we forward attribute setting by + # making sure that after we set the bad context our handshake indeed + # fails: + s.context = bad_ctx + assert s.context is bad_ctx + with pytest.raises(BrokenResourceError) as excinfo: + await s.do_handshake() + assert isinstance(excinfo.value.__cause__, ssl.SSLError) + + +# Note: this test fails horribly if we force TLS 1.2 and trigger a +# renegotiation at the beginning (e.g. by switching to the pyopenssl +# server). Usually the client crashes in SSLObject.write with "UNEXPECTED +# RECORD"; sometimes we get something more exotic like a SyscallError. This is +# odd because openssl isn't doing any syscalls, but so it goes. After lots of +# websearching I'm pretty sure this is due to a bug in OpenSSL, where it just +# can't reliably handle full-duplex communication combined with +# renegotiation. Nice, eh? +# +# https://rt.openssl.org/Ticket/Display.html?id=3712 +# https://rt.openssl.org/Ticket/Display.html?id=2481 +# http://openssl.6102.n7.nabble.com/TLS-renegotiation-failure-on-receiving-application-data-during-handshake-td48127.html +# https://stackoverflow.com/questions/18728355/ssl-renegotiation-with-full-duplex-socket-communication +# +# In some variants of this test (maybe only against the java server?) I've +# also seen cases where our send_all blocks waiting to write, and then our receive_some +# also blocks waiting to write, and they never wake up again. It looks like +# some kind of deadlock. I suspect there may be an issue where we've filled up +# the send buffers, and the remote side is trying to handle the renegotiation +# from inside a write() call, so it has a problem: there's all this application +# data clogging up the pipe, but it can't process and return it to the +# application because it's in write(), and it doesn't want to buffer infinite +# amounts of data, and... actually I guess those are the only two choices. +# +# NSS even documents that you shouldn't try to do a renegotiation except when +# the connection is idle: +# +# https://developer.mozilla.org/en-US/docs/Mozilla/Projects/NSS/SSL_functions/sslfnc.html#1061582 +# +# I begin to see why HTTP/2 forbids renegotiation and TLS 1.3 removes it... + + +async def test_full_duplex_basics(client_ctx: SSLContext) -> None: + CHUNKS = 30 + CHUNK_SIZE = 32768 + EXPECTED = CHUNKS * CHUNK_SIZE + + sent = bytearray() + received = bytearray() + + async def sender(s: Stream) -> None: + nonlocal sent + for i in range(CHUNKS): + print(i) + chunk = bytes([i] * CHUNK_SIZE) + sent += chunk + await s.send_all(chunk) + + async def receiver(s: Stream) -> None: + nonlocal received + while len(received) < EXPECTED: + chunk = await s.receive_some(CHUNK_SIZE // 2) + received += chunk + + async with ssl_echo_server(client_ctx) as s: + async with _core.open_nursery() as nursery: + nursery.start_soon(sender, s) + nursery.start_soon(receiver, s) + # And let's have some doing handshakes too, everyone + # simultaneously + nursery.start_soon(s.do_handshake) + nursery.start_soon(s.do_handshake) + + await s.aclose() + + assert len(sent) == len(received) == EXPECTED + assert sent == received + + +async def test_renegotiation_simple(client_ctx: SSLContext) -> None: + with virtual_ssl_echo_server(client_ctx) as s: + await s.do_handshake() + s.transport_stream.renegotiate() + await s.send_all(b"a") + assert await s.receive_some(1) == b"a" + + # Have to send some more data back and forth to make sure the + # renegotiation is finished before shutting down the + # connection... otherwise openssl raises an error. I think this is a + # bug in openssl but what can ya do. + await s.send_all(b"b") + assert await s.receive_some(1) == b"b" + + await s.aclose() + + +@slow +async def test_renegotiation_randomized( + mock_clock: MockClock, + client_ctx: SSLContext, +) -> None: + # The only blocking things in this function are our random sleeps, so 0 is + # a good threshold. + mock_clock.autojump_threshold = 0 + + import random + + r = random.Random(0) + + async def sleeper(_: object) -> None: + await trio.sleep(r.uniform(0, 10)) + + async def clear() -> None: + while s.transport_stream.renegotiate_pending(): + with assert_checkpoints(): + await send(b"-") + with assert_checkpoints(): + await expect(b"-") + print("-- clear --") + + async def send(byte: bytes) -> None: + await s.transport_stream.sleeper("outer send") + print("calling SSLStream.send_all", byte) + with assert_checkpoints(): + await s.send_all(byte) + + async def expect(expected: bytes) -> None: + await s.transport_stream.sleeper("expect") + print("calling SSLStream.receive_some, expecting", expected) + assert len(expected) == 1 + with assert_checkpoints(): + assert await s.receive_some(1) == expected + + with virtual_ssl_echo_server(client_ctx, sleeper=sleeper) as s: + await s.do_handshake() + + await send(b"a") + s.transport_stream.renegotiate() + await expect(b"a") + + await clear() + + for i in range(100): + b1 = bytes([i % 0xFF]) + b2 = bytes([(2 * i) % 0xFF]) + s.transport_stream.renegotiate() + async with _core.open_nursery() as nursery: + nursery.start_soon(send, b1) + nursery.start_soon(expect, b1) + async with _core.open_nursery() as nursery: + nursery.start_soon(expect, b2) + nursery.start_soon(send, b2) + await clear() + + for i in range(100): + b1 = bytes([i % 0xFF]) + b2 = bytes([(2 * i) % 0xFF]) + await send(b1) + s.transport_stream.renegotiate() + await expect(b1) + async with _core.open_nursery() as nursery: + nursery.start_soon(expect, b2) + nursery.start_soon(send, b2) + await clear() + + # Checking that wait_send_all_might_not_block and receive_some don't + # conflict: + + # 1) Set up a situation where expect (receive_some) is blocked sending, + # and wait_send_all_might_not_block comes in. + + # Our receive_some() call will get stuck when it hits send_all + async def sleeper_with_slow_send_all(method: str) -> None: + if method == "send_all": + # ignore ASYNC116, not sleep_forever, trying to test a large but finite sleep + await trio.sleep(100000) # noqa: ASYNC116 + + # And our wait_send_all_might_not_block call will give it time to get + # stuck, and then start + async def sleep_then_wait_writable() -> None: + await trio.sleep(1000) + await s.wait_send_all_might_not_block() + + with virtual_ssl_echo_server(client_ctx, sleeper=sleeper_with_slow_send_all) as s: + await send(b"x") + s.transport_stream.renegotiate() + async with _core.open_nursery() as nursery: + nursery.start_soon(expect, b"x") + nursery.start_soon(sleep_then_wait_writable) + + await clear() + + await s.aclose() + + # 2) Same, but now wait_send_all_might_not_block is stuck when + # receive_some tries to send. + + async def sleeper_with_slow_wait_writable_and_expect(method: str) -> None: + if method == "wait_send_all_might_not_block": + # ignore ASYNC116, not sleep_forever, trying to test a large but finite sleep + await trio.sleep(100000) # noqa: ASYNC116 + elif method == "expect": + await trio.sleep(1000) + + with virtual_ssl_echo_server( + client_ctx, + sleeper=sleeper_with_slow_wait_writable_and_expect, + ) as s: + await send(b"x") + s.transport_stream.renegotiate() + async with _core.open_nursery() as nursery: + nursery.start_soon(expect, b"x") + nursery.start_soon(s.wait_send_all_might_not_block) + + await clear() + + await s.aclose() + + +async def test_resource_busy_errors(client_ctx: SSLContext) -> None: + S: TypeAlias = trio.SSLStream[ + trio.StapledStream[trio.abc.SendStream, trio.abc.ReceiveStream] + ] + + async def do_send_all(s: S) -> None: + with assert_checkpoints(): + await s.send_all(b"x") + + async def do_receive_some(s: S) -> None: + with assert_checkpoints(): + await s.receive_some(1) + + async def do_wait_send_all_might_not_block(s: S) -> None: + with assert_checkpoints(): + await s.wait_send_all_might_not_block() + + async def do_test( + func1: Callable[[S], Awaitable[None]], + func2: Callable[[S], Awaitable[None]], + ) -> None: + s, _ = ssl_lockstep_stream_pair(client_ctx) + with pytest.RaisesGroup( + pytest.RaisesExc(_core.BusyResourceError, match="another task") + ): + async with _core.open_nursery() as nursery: + nursery.start_soon(func1, s) + nursery.start_soon(func2, s) + + await do_test(do_send_all, do_send_all) + await do_test(do_receive_some, do_receive_some) + await do_test(do_send_all, do_wait_send_all_might_not_block) + await do_test(do_wait_send_all_might_not_block, do_wait_send_all_might_not_block) + + +async def test_wait_writable_calls_underlying_wait_writable() -> None: + record = [] + + class NotAStream(Stream): + async def wait_send_all_might_not_block(self) -> None: + record.append("ok") + + # define methods that are abstract in Stream + async def aclose(self) -> None: + raise AssertionError("Should not get called") # pragma: no cover + + async def receive_some(self, max_bytes: int | None = None) -> bytes | bytearray: + raise AssertionError("Should not get called") # pragma: no cover + + async def send_all(self, data: bytes | bytearray | memoryview) -> None: + raise AssertionError("Should not get called") # pragma: no cover + + ctx = ssl.create_default_context() + s = SSLStream(NotAStream(), ctx, server_hostname="x") + await s.wait_send_all_might_not_block() + assert record == ["ok"] + + +@pytest.mark.skipif( + os.name == "nt" and sys.version_info >= (3, 10), + reason="frequently fails on Windows + Python 3.10", +) +async def test_checkpoints(client_ctx: SSLContext) -> None: + async with ssl_echo_server(client_ctx) as s: + with assert_checkpoints(): + await s.do_handshake() + with assert_checkpoints(): + await s.do_handshake() + with assert_checkpoints(): + await s.wait_send_all_might_not_block() + with assert_checkpoints(): + await s.send_all(b"xxx") + with assert_checkpoints(): + await s.receive_some(1) + # These receive_some's in theory could return immediately, because the + # "xxx" was sent in a single record and after the first + # receive_some(1) the rest are sitting inside the SSLObject's internal + # buffers. + with assert_checkpoints(): + await s.receive_some(1) + with assert_checkpoints(): + await s.receive_some(1) + with assert_checkpoints(): + await s.unwrap() + + async with ssl_echo_server(client_ctx) as s: + await s.do_handshake() + with assert_checkpoints(): + await s.aclose() + + +async def test_send_all_empty_string(client_ctx: SSLContext) -> None: + async with ssl_echo_server(client_ctx) as s: + await s.do_handshake() + + # underlying SSLObject interprets writing b"" as indicating an EOF, + # for some reason. Make sure we don't inherit this. + with assert_checkpoints(): + await s.send_all(b"") + with assert_checkpoints(): + await s.send_all(b"") + await s.send_all(b"x") + assert await s.receive_some(1) == b"x" + + await s.aclose() + + +@pytest.mark.parametrize("https_compatible", [False, True]) +async def test_SSLStream_generic( + client_ctx: SSLContext, + https_compatible: bool, +) -> None: + async def stream_maker() -> tuple[ + SSLStream[MemoryStapledStream], + SSLStream[MemoryStapledStream], + ]: + return ssl_memory_stream_pair( + client_ctx, + client_kwargs={"https_compatible": https_compatible}, + server_kwargs={"https_compatible": https_compatible}, + ) + + async def clogged_stream_maker() -> tuple[ + SSLStream[MyStapledStream], + SSLStream[MyStapledStream], + ]: + client, server = ssl_lockstep_stream_pair(client_ctx) + # If we don't do handshakes up front, then we run into a problem in + # the following situation: + # - server does wait_send_all_might_not_block + # - client does receive_some to unclog it + # Then the client's receive_some will actually send some data to start + # the handshake, and itself get stuck. + async with _core.open_nursery() as nursery: + nursery.start_soon(client.do_handshake) + nursery.start_soon(server.do_handshake) + return client, server + + await check_two_way_stream(stream_maker, clogged_stream_maker) + + +async def test_unwrap(client_ctx: SSLContext) -> None: + client_ssl, server_ssl = ssl_memory_stream_pair(client_ctx) + client_transport = client_ssl.transport_stream + server_transport = server_ssl.transport_stream + + seq = Sequencer() + + async def client() -> None: + await client_ssl.do_handshake() + await client_ssl.send_all(b"x") + assert await client_ssl.receive_some(1) == b"y" + await client_ssl.send_all(b"z") + + # After sending that, disable outgoing data from our end, to make + # sure the server doesn't see our EOF until after we've sent some + # trailing data + async with seq(0): + send_all_hook = client_transport.send_stream.send_all_hook + client_transport.send_stream.send_all_hook = None + + assert await client_ssl.receive_some(1) == b"" + assert client_ssl.transport_stream is client_transport + # We just received EOF. Unwrap the connection and send some more. + raw, trailing = await client_ssl.unwrap() + assert raw is client_transport + assert trailing == b"" + assert client_ssl.transport_stream is None + await raw.send_all(b"trailing") + + # Reconnect the streams. Now the server will receive both our shutdown + # acknowledgement + the trailing data in a single lump. + client_transport.send_stream.send_all_hook = send_all_hook + await client_transport.send_stream.send_all_hook() + + async def server() -> None: + await server_ssl.do_handshake() + assert await server_ssl.receive_some(1) == b"x" + await server_ssl.send_all(b"y") + assert await server_ssl.receive_some(1) == b"z" + # Now client is blocked waiting for us to send something, but + # instead we close the TLS connection (with sequencer to make sure + # that the client won't see and automatically respond before we've had + # a chance to disable the client->server transport) + async with seq(1): + raw, trailing = await server_ssl.unwrap() + assert raw is server_transport + assert trailing == b"trailing" + assert server_ssl.transport_stream is None + + async with _core.open_nursery() as nursery: + nursery.start_soon(client) + nursery.start_soon(server) + + +async def test_closing_nice_case(client_ctx: SSLContext) -> None: + # the nice case: graceful closes all around + + client_ssl, server_ssl = ssl_memory_stream_pair(client_ctx) + client_transport = client_ssl.transport_stream + + # Both the handshake and the close require back-and-forth discussion, so + # we need to run them concurrently + async def client_closer() -> None: + with assert_checkpoints(): + await client_ssl.aclose() + + async def server_closer() -> None: + assert await server_ssl.receive_some(10) == b"" + assert await server_ssl.receive_some(10) == b"" + with assert_checkpoints(): + await server_ssl.aclose() + + async with _core.open_nursery() as nursery: + nursery.start_soon(client_closer) + nursery.start_soon(server_closer) + + # closing the SSLStream also closes its transport + with pytest.raises(ClosedResourceError): + await client_transport.send_all(b"123") + + # once closed, it's OK to close again + with assert_checkpoints(): + await client_ssl.aclose() + with assert_checkpoints(): + await client_ssl.aclose() + + # Trying to send more data does not work + with pytest.raises(ClosedResourceError): + await server_ssl.send_all(b"123") + + # And once the connection is has been closed *locally*, then instead of + # getting empty bytestrings we get a proper error + with pytest.raises(ClosedResourceError): + assert await client_ssl.receive_some(10) == b"" + + with pytest.raises(ClosedResourceError): + await client_ssl.unwrap() + + with pytest.raises(ClosedResourceError): + await client_ssl.do_handshake() + + # Check that a graceful close *before* handshaking gives a clean EOF on + # the other side + client_ssl, server_ssl = ssl_memory_stream_pair(client_ctx) + + async def expect_eof_server() -> None: + with assert_checkpoints(): + assert await server_ssl.receive_some(10) == b"" + with assert_checkpoints(): + await server_ssl.aclose() + + async with _core.open_nursery() as nursery: + nursery.start_soon(client_ssl.aclose) + nursery.start_soon(expect_eof_server) + + +async def test_send_all_fails_in_the_middle(client_ctx: SSLContext) -> None: + client, server = ssl_memory_stream_pair(client_ctx) + + async with _core.open_nursery() as nursery: + nursery.start_soon(client.do_handshake) + nursery.start_soon(server.do_handshake) + + async def bad_hook() -> NoReturn: + raise KeyError + + client.transport_stream.send_stream.send_all_hook = bad_hook + + with pytest.raises(KeyError): + await client.send_all(b"x") + + with pytest.raises(BrokenResourceError): + await client.wait_send_all_might_not_block() + + closed = 0 + + def close_hook() -> None: + nonlocal closed + closed += 1 + + client.transport_stream.send_stream.close_hook = close_hook + client.transport_stream.receive_stream.close_hook = close_hook + await client.aclose() + + assert closed == 2 + + +async def test_ssl_over_ssl(client_ctx: SSLContext) -> None: + client_0, server_0 = memory_stream_pair() + + client_1 = SSLStream( + client_0, + client_ctx, + server_hostname="trio-test-1.example.org", + ) + server_1 = SSLStream(server_0, SERVER_CTX, server_side=True) + + client_2 = SSLStream( + client_1, + client_ctx, + server_hostname="trio-test-1.example.org", + ) + server_2 = SSLStream(server_1, SERVER_CTX, server_side=True) + + async def client() -> None: + await client_2.send_all(b"hi") + assert await client_2.receive_some(10) == b"bye" + + async def server() -> None: + assert await server_2.receive_some(10) == b"hi" + await server_2.send_all(b"bye") + + async with _core.open_nursery() as nursery: + nursery.start_soon(client) + nursery.start_soon(server) + + +async def test_ssl_bad_shutdown(client_ctx: SSLContext) -> None: + client, server = ssl_memory_stream_pair(client_ctx) + + async with _core.open_nursery() as nursery: + nursery.start_soon(client.do_handshake) + nursery.start_soon(server.do_handshake) + + await trio.aclose_forcefully(client) + # now the server sees a broken stream + with pytest.raises(BrokenResourceError): + await server.receive_some(10) + with pytest.raises(BrokenResourceError): + await server.send_all(b"x" * 10) + + await server.aclose() + + +async def test_ssl_bad_shutdown_but_its_ok(client_ctx: SSLContext) -> None: + client, server = ssl_memory_stream_pair( + client_ctx, + server_kwargs={"https_compatible": True}, + client_kwargs={"https_compatible": True}, + ) + + async with _core.open_nursery() as nursery: + nursery.start_soon(client.do_handshake) + nursery.start_soon(server.do_handshake) + + await trio.aclose_forcefully(client) + # the server sees that as a clean shutdown + assert await server.receive_some(10) == b"" + with pytest.raises(BrokenResourceError): + await server.send_all(b"x" * 10) + + await server.aclose() + + +async def test_ssl_handshake_failure_during_aclose() -> None: + # Weird scenario: aclose() triggers an automatic handshake, and this + # fails. This also exercises a bit of code in aclose() that was otherwise + # uncovered, for re-raising exceptions after calling aclose_forcefully on + # the underlying transport. + async with ssl_echo_server_raw(expect_fail=True) as sock: + # Don't configure trust correctly + client_ctx = ssl.create_default_context() + s = SSLStream(sock, client_ctx, server_hostname="trio-test-1.example.org") + # It's a little unclear here whether aclose should swallow the error + # or let it escape. We *do* swallow the error if it arrives when we're + # sending close_notify, because both sides closing the connection + # simultaneously is allowed. But I guess when https_compatible=False + # then it's bad if we can get through a whole connection with a peer + # that has no valid certificate, and never raise an error. + with pytest.raises(BrokenResourceError): + await s.aclose() + + +async def test_ssl_only_closes_stream_once(client_ctx: SSLContext) -> None: + # We used to have a bug where if transport_stream.aclose() raised an + # error, we would call it again. This checks that that's fixed. + client, server = ssl_memory_stream_pair(client_ctx) + + async with _core.open_nursery() as nursery: + nursery.start_soon(client.do_handshake) + nursery.start_soon(server.do_handshake) + + client_orig_close_hook = client.transport_stream.send_stream.close_hook + transport_close_count = 0 + + def close_hook() -> NoReturn: + nonlocal transport_close_count + assert client_orig_close_hook is not None + client_orig_close_hook() + transport_close_count += 1 + raise KeyError + + client.transport_stream.send_stream.close_hook = close_hook + + with pytest.raises(KeyError): + await client.aclose() + assert transport_close_count == 1 + + +async def test_ssl_https_compatibility_disagreement(client_ctx: SSLContext) -> None: + client, server = ssl_memory_stream_pair( + client_ctx, + server_kwargs={"https_compatible": False}, + client_kwargs={"https_compatible": True}, + ) + + async with _core.open_nursery() as nursery: + nursery.start_soon(client.do_handshake) + nursery.start_soon(server.do_handshake) + + # client is in HTTPS-mode, server is not + # so client doing graceful_shutdown causes an error on server + async def receive_and_expect_error() -> None: + with pytest.raises(BrokenResourceError) as excinfo: + await server.receive_some(10) + + assert _is_eof(excinfo.value.__cause__) + + async with _core.open_nursery() as nursery: + nursery.start_soon(client.aclose) + nursery.start_soon(receive_and_expect_error) + + +async def test_https_mode_eof_before_handshake(client_ctx: SSLContext) -> None: + client, server = ssl_memory_stream_pair( + client_ctx, + server_kwargs={"https_compatible": True}, + client_kwargs={"https_compatible": True}, + ) + + async def server_expect_clean_eof() -> None: + assert await server.receive_some(10) == b"" + + async with _core.open_nursery() as nursery: + nursery.start_soon(client.aclose) + nursery.start_soon(server_expect_clean_eof) + + +async def test_send_error_during_handshake(client_ctx: SSLContext) -> None: + client, _server = ssl_memory_stream_pair(client_ctx) + + async def bad_hook() -> NoReturn: + raise KeyError + + client.transport_stream.send_stream.send_all_hook = bad_hook + + with pytest.raises(KeyError): + with assert_checkpoints(): + await client.do_handshake() + + with pytest.raises(BrokenResourceError): + with assert_checkpoints(): + await client.do_handshake() + + +async def test_receive_error_during_handshake(client_ctx: SSLContext) -> None: + client, server = ssl_memory_stream_pair(client_ctx) + + async def bad_hook() -> NoReturn: + raise KeyError + + client.transport_stream.receive_stream.receive_some_hook = bad_hook + + async def client_side(cancel_scope: CancelScope) -> None: + with pytest.raises(KeyError): + with assert_checkpoints(): + await client.do_handshake() + cancel_scope.cancel() + + async with _core.open_nursery() as nursery: + nursery.start_soon(client_side, nursery.cancel_scope) + nursery.start_soon(server.do_handshake) + + with pytest.raises(BrokenResourceError): + with assert_checkpoints(): + await client.do_handshake() + + +def test_selected_alpn_protocol_before_handshake(client_ctx: SSLContext) -> None: + client, server = ssl_memory_stream_pair(client_ctx) + + with pytest.raises(NeedHandshakeError): + client.selected_alpn_protocol() + + with pytest.raises(NeedHandshakeError): + server.selected_alpn_protocol() + + +async def test_selected_alpn_protocol_when_not_set(client_ctx: SSLContext) -> None: + # ALPN protocol still returns None when it's not set, + # instead of raising an exception + client, server = ssl_memory_stream_pair(client_ctx) + + async with _core.open_nursery() as nursery: + nursery.start_soon(client.do_handshake) + nursery.start_soon(server.do_handshake) + + assert client.selected_alpn_protocol() is None + assert server.selected_alpn_protocol() is None + + assert client.selected_alpn_protocol() == server.selected_alpn_protocol() + + +def test_selected_npn_protocol_before_handshake(client_ctx: SSLContext) -> None: + client, server = ssl_memory_stream_pair(client_ctx) + + with pytest.raises(NeedHandshakeError): + client.selected_npn_protocol() + + with pytest.raises(NeedHandshakeError): + server.selected_npn_protocol() + + +@pytest.mark.filterwarnings( + r"ignore: ssl module. NPN is deprecated, use ALPN instead:UserWarning", + r"ignore:ssl NPN is deprecated, use ALPN instead:DeprecationWarning", +) +async def test_selected_npn_protocol_when_not_set(client_ctx: SSLContext) -> None: + # NPN protocol still returns None when it's not set, + # instead of raising an exception + client, server = ssl_memory_stream_pair(client_ctx) + + async with _core.open_nursery() as nursery: + nursery.start_soon(client.do_handshake) + nursery.start_soon(server.do_handshake) + + assert client.selected_npn_protocol() is None + assert server.selected_npn_protocol() is None + + assert client.selected_npn_protocol() == server.selected_npn_protocol() + + +def test_get_channel_binding_before_handshake(client_ctx: SSLContext) -> None: + client, server = ssl_memory_stream_pair(client_ctx) + + with pytest.raises(NeedHandshakeError): + client.get_channel_binding() + + with pytest.raises(NeedHandshakeError): + server.get_channel_binding() + + +async def test_get_channel_binding_after_handshake(client_ctx: SSLContext) -> None: + client, server = ssl_memory_stream_pair(client_ctx) + + async with _core.open_nursery() as nursery: + nursery.start_soon(client.do_handshake) + nursery.start_soon(server.do_handshake) + + assert client.get_channel_binding() is not None + assert server.get_channel_binding() is not None + + assert client.get_channel_binding() == server.get_channel_binding() + + +async def test_getpeercert(client_ctx: SSLContext) -> None: + # Make sure we're not affected by https://bugs.python.org/issue29334 + client, server = ssl_memory_stream_pair(client_ctx) + + async with _core.open_nursery() as nursery: + nursery.start_soon(client.do_handshake) + nursery.start_soon(server.do_handshake) + + assert server.getpeercert() is None + print(client.getpeercert()) + assert ("DNS", "trio-test-1.example.org") in client.getpeercert()["subjectAltName"] + + +async def test_SSLListener(client_ctx: SSLContext) -> None: + async def setup( + https_compatible: bool = False, + ) -> tuple[tsocket.SocketType, SSLListener[SocketStream], SSLStream[SocketStream]]: + listen_sock = tsocket.socket() + await listen_sock.bind(("127.0.0.1", 0)) + listen_sock.listen(1) + socket_listener = SocketListener(listen_sock) + ssl_listener = SSLListener( + socket_listener, + SERVER_CTX, + https_compatible=https_compatible, + ) + + transport_client = await open_tcp_stream(*listen_sock.getsockname()) + ssl_client = SSLStream( + transport_client, + client_ctx, + server_hostname="trio-test-1.example.org", + ) + return listen_sock, ssl_listener, ssl_client + + listen_sock, ssl_listener, ssl_client = await setup() + + async with ssl_client: + ssl_server = await ssl_listener.accept() + + async with ssl_server: + assert not ssl_server._https_compatible + + # Make sure the connection works + async with _core.open_nursery() as nursery: + nursery.start_soon(ssl_client.do_handshake) + nursery.start_soon(ssl_server.do_handshake) + + # Test SSLListener.aclose + await ssl_listener.aclose() + assert listen_sock.fileno() == -1 + + ################ + + # Test https_compatible + _, ssl_listener, ssl_client = await setup(https_compatible=True) + + ssl_server = await ssl_listener.accept() + + assert ssl_server._https_compatible + + await aclose_forcefully(ssl_listener) + await aclose_forcefully(ssl_client) + await aclose_forcefully(ssl_server) diff --git a/src/trio/_tests/tools/test_gen_exports.py b/src/trio/_tests/tools/test_gen_exports.py index 2c31ac1ec9..3b923ed83d 100644 --- a/src/trio/_tests/tools/test_gen_exports.py +++ b/src/trio/_tests/tools/test_gen_exports.py @@ -1,179 +1,179 @@ -import ast -import sys -from pathlib import Path - -import pytest - -from trio._tests.pytest_plugin import maybe_ignore_import_error - -# imports in gen_exports that are not in `install_requires` in setup.py -try: - import astor # noqa: F401 - import isort # noqa: F401 -except ImportError as error: - maybe_ignore_import_error(error) - - -from trio._tools.gen_exports import ( - File, - create_passthrough_args, - get_public_methods, - process, - run_linters, - run_ruff, -) - -from ..._core._tests.tutil import slow - -SOURCE = '''from _run import _public -from collections import Counter - -class Test: - @_public - def public_func(self): - """With doc string""" - - @ignore_this - @_public - @another_decorator - async def public_async_func(self) -> Counter: - pass # no doc string - - def not_public(self): - pass - - async def not_public_async(self): - pass -''' - -IMPORT_1 = """\ -from collections import Counter -""" - -IMPORT_2 = """\ -from collections import Counter -import os -""" - -IMPORT_3 = """\ -from typing import TYPE_CHECKING -if TYPE_CHECKING: - from collections import Counter -""" - - -def test_get_public_methods() -> None: - methods = list(get_public_methods(ast.parse(SOURCE))) - assert {m.name for m in methods} == {"public_func", "public_async_func"} - - -def test_create_pass_through_args() -> None: - testcases = [ - ("def f()", "()"), - ("def f(one)", "(one)"), - ("def f(one, two)", "(one, two)"), - ("def f(one, *args)", "(one, *args)"), - ( - "def f(one, *args, kw1, kw2=None, **kwargs)", - "(one, *args, kw1=kw1, kw2=kw2, **kwargs)", - ), - ] - - for funcdef, expected in testcases: - func_node = ast.parse(funcdef + ":\n pass").body[0] - assert isinstance(func_node, ast.FunctionDef) - assert create_passthrough_args(func_node) == expected - - -skip_lints = pytest.mark.skipif( - sys.implementation.name != "cpython", - reason="gen_exports is internal, black/isort only runs on CPython", -) - - -@slow -@skip_lints -@pytest.mark.parametrize("imports", [IMPORT_1, IMPORT_2, IMPORT_3]) -def test_process( - tmp_path: Path, - imports: str, - capsys: pytest.CaptureFixture[str], -) -> None: - try: - import black # noqa: F401 - # there's no dedicated CI run that has astor+isort, but lacks black. - except ImportError as error: # pragma: no cover - maybe_ignore_import_error(error) - - modpath = tmp_path / "_module.py" - genpath = tmp_path / "_generated_module.py" - modpath.write_text(SOURCE, encoding="utf-8") - file = File(modpath, "runner", platform="linux", imports=imports) - assert not genpath.exists() - with pytest.raises(SystemExit) as excinfo: - process([file], do_test=True) - assert excinfo.value.code == 1 - captured = capsys.readouterr() - assert "Generated sources are outdated. Please regenerate." in captured.out - with pytest.raises(SystemExit) as excinfo: - process([file], do_test=False) - assert excinfo.value.code == 1 - captured = capsys.readouterr() - assert "Regenerated sources successfully." in captured.out - assert genpath.exists() - process([file], do_test=True) - # But if we change the lookup path it notices - with pytest.raises(SystemExit) as excinfo: - process( - [File(modpath, "runner.io_manager", platform="linux", imports=imports)], - do_test=True, - ) - assert excinfo.value.code == 1 - # Also if the platform is changed. - with pytest.raises(SystemExit) as excinfo: - process([File(modpath, "runner", imports=imports)], do_test=True) - assert excinfo.value.code == 1 - - -@skip_lints -def test_run_ruff(tmp_path: Path) -> None: - """Test that processing properly fails if ruff does.""" - try: - import ruff # noqa: F401 - except ImportError as error: # pragma: no cover - maybe_ignore_import_error(error) - - file = File(tmp_path / "module.py", "module") - - success, _ = run_ruff(file, "class not valid code ><") - assert not success - - test_function = '''def combine_and(data: list[str]) -> str: - """Join values of text, and have 'and' with the last one properly.""" - if len(data) >= 2: - data[-1] = 'and ' + data[-1] - if len(data) > 2: - return ', '.join(data) - return ' '.join(data)''' - - success, response = run_ruff(file, test_function) - assert success - assert response == test_function - - -@skip_lints -def test_lint_failure(tmp_path: Path) -> None: - """Test that processing properly fails if black or ruff does.""" - try: - import black # noqa: F401 - import ruff # noqa: F401 - except ImportError as error: # pragma: no cover - maybe_ignore_import_error(error) - - file = File(tmp_path / "module.py", "module") - - with pytest.raises(SystemExit): - run_linters(file, "class not valid code ><") - - with pytest.raises(SystemExit): - run_linters(file, "import waffle\n;import trio") +import ast +import sys +from pathlib import Path + +import pytest + +from trio._tests.pytest_plugin import maybe_ignore_import_error + +# imports in gen_exports that are not in `install_requires` in setup.py +try: + import astor # noqa: F401 + import isort # noqa: F401 +except ImportError as error: + maybe_ignore_import_error(error) + + +from trio._tools.gen_exports import ( + File, + create_passthrough_args, + get_public_methods, + process, + run_linters, + run_ruff, +) + +from ..._core._tests.tutil import slow + +SOURCE = '''from _run import _public +from collections import Counter + +class Test: + @_public + def public_func(self): + """With doc string""" + + @ignore_this + @_public + @another_decorator + async def public_async_func(self) -> Counter: + pass # no doc string + + def not_public(self): + pass + + async def not_public_async(self): + pass +''' + +IMPORT_1 = """\ +from collections import Counter +""" + +IMPORT_2 = """\ +from collections import Counter +import os +""" + +IMPORT_3 = """\ +from typing import TYPE_CHECKING +if TYPE_CHECKING: + from collections import Counter +""" + + +def test_get_public_methods() -> None: + methods = list(get_public_methods(ast.parse(SOURCE))) + assert {m.name for m in methods} == {"public_func", "public_async_func"} + + +def test_create_pass_through_args() -> None: + testcases = [ + ("def f()", "()"), + ("def f(one)", "(one)"), + ("def f(one, two)", "(one, two)"), + ("def f(one, *args)", "(one, *args)"), + ( + "def f(one, *args, kw1, kw2=None, **kwargs)", + "(one, *args, kw1=kw1, kw2=kw2, **kwargs)", + ), + ] + + for funcdef, expected in testcases: + func_node = ast.parse(funcdef + ":\n pass").body[0] + assert isinstance(func_node, ast.FunctionDef) + assert create_passthrough_args(func_node) == expected + + +skip_lints = pytest.mark.skipif( + sys.implementation.name != "cpython", + reason="gen_exports is internal, black/isort only runs on CPython", +) + + +@slow +@skip_lints +@pytest.mark.parametrize("imports", [IMPORT_1, IMPORT_2, IMPORT_3]) +def test_process( + tmp_path: Path, + imports: str, + capsys: pytest.CaptureFixture[str], +) -> None: + try: + import black # noqa: F401 + # there's no dedicated CI run that has astor+isort, but lacks black. + except ImportError as error: # pragma: no cover + maybe_ignore_import_error(error) + + modpath = tmp_path / "_module.py" + genpath = tmp_path / "_generated_module.py" + modpath.write_text(SOURCE, encoding="utf-8") + file = File(modpath, "runner", platform="linux", imports=imports) + assert not genpath.exists() + with pytest.raises(SystemExit) as excinfo: + process([file], do_test=True) + assert excinfo.value.code == 1 + captured = capsys.readouterr() + assert "Generated sources are outdated. Please regenerate." in captured.out + with pytest.raises(SystemExit) as excinfo: + process([file], do_test=False) + assert excinfo.value.code == 1 + captured = capsys.readouterr() + assert "Regenerated sources successfully." in captured.out + assert genpath.exists() + process([file], do_test=True) + # But if we change the lookup path it notices + with pytest.raises(SystemExit) as excinfo: + process( + [File(modpath, "runner.io_manager", platform="linux", imports=imports)], + do_test=True, + ) + assert excinfo.value.code == 1 + # Also if the platform is changed. + with pytest.raises(SystemExit) as excinfo: + process([File(modpath, "runner", imports=imports)], do_test=True) + assert excinfo.value.code == 1 + + +@skip_lints +def test_run_ruff(tmp_path: Path) -> None: + """Test that processing properly fails if ruff does.""" + try: + import ruff # noqa: F401 + except ImportError as error: # pragma: no cover + maybe_ignore_import_error(error) + + file = File(tmp_path / "module.py", "module") + + success, _ = run_ruff(file, "class not valid code ><") + assert not success + + test_function = '''def combine_and(data: list[str]) -> str: + """Join values of text, and have 'and' with the last one properly.""" + if len(data) >= 2: + data[-1] = 'and ' + data[-1] + if len(data) > 2: + return ', '.join(data) + return ' '.join(data)''' + + success, response = run_ruff(file, test_function) + assert success + assert response == test_function + + +@skip_lints +def test_lint_failure(tmp_path: Path) -> None: + """Test that processing properly fails if black or ruff does.""" + try: + import black # noqa: F401 + import ruff # noqa: F401 + except ImportError as error: # pragma: no cover + maybe_ignore_import_error(error) + + file = File(tmp_path / "module.py", "module") + + with pytest.raises(SystemExit): + run_linters(file, "class not valid code ><") + + with pytest.raises(SystemExit): + run_linters(file, "import waffle\n;import trio") diff --git a/src/trio/_tests/tools/test_sync_requirements.py b/src/trio/_tests/tools/test_sync_requirements.py index 544b79bd0f..5beb707558 100644 --- a/src/trio/_tests/tools/test_sync_requirements.py +++ b/src/trio/_tests/tools/test_sync_requirements.py @@ -1,79 +1,79 @@ -from __future__ import annotations - -from typing import TYPE_CHECKING - -from trio._tests.pytest_plugin import maybe_ignore_import_error - -# imports in gen_exports that are not in `install_requires` in requirements -try: - import yaml # noqa: F401 -except ImportError as error: - maybe_ignore_import_error(error) - -from trio._tools.sync_requirements import ( - update_requirements, - yield_pre_commit_version_data, -) - -if TYPE_CHECKING: - from pathlib import Path - - -def test_yield_pre_commit_version_data() -> None: - text = """ -repos: - - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.11.0 - - repo: https://github.com/psf/black-pre-commit-mirror - rev: 25.1.0 - - bad: data -""" - results = tuple(yield_pre_commit_version_data(text)) - assert results == ( - ("ruff-pre-commit", "0.11.0"), - ("black-pre-commit-mirror", "25.1.0"), - ) - - -def test_update_requirements( - tmp_path: Path, -) -> None: - requirements_file = tmp_path / "requirements.txt" - assert not requirements_file.exists() - requirements_file.write_text( - """# comment - # also comment but spaces line start -waffles are delicious no equals -black==3.1.4 ; specific version thingy -mypy==1.15.0 -ruff==1.2.5 -# required by soupy cat""", - encoding="utf-8", - ) - assert update_requirements(requirements_file, {"black": "3.1.5", "ruff": "1.2.7"}) - assert requirements_file.read_text(encoding="utf-8") == """# comment - # also comment but spaces line start -waffles are delicious no equals -black==3.1.5 ; specific version thingy -mypy==1.15.0 -ruff==1.2.7 -# required by soupy cat""" - - -def test_update_requirements_no_changes( - tmp_path: Path, -) -> None: - requirements_file = tmp_path / "requirements.txt" - assert not requirements_file.exists() - original = """# comment - # also comment but spaces line start -waffles are delicious no equals -black==3.1.4 ; specific version thingy -mypy==1.15.0 -ruff==1.2.5 -# required by soupy cat""" - requirements_file.write_text(original, encoding="utf-8") - assert not update_requirements( - requirements_file, {"black": "3.1.4", "ruff": "1.2.5"} - ) - assert requirements_file.read_text(encoding="utf-8") == original +from __future__ import annotations + +from typing import TYPE_CHECKING + +from trio._tests.pytest_plugin import maybe_ignore_import_error + +# imports in gen_exports that are not in `install_requires` in requirements +try: + import yaml # noqa: F401 +except ImportError as error: + maybe_ignore_import_error(error) + +from trio._tools.sync_requirements import ( + update_requirements, + yield_pre_commit_version_data, +) + +if TYPE_CHECKING: + from pathlib import Path + + +def test_yield_pre_commit_version_data() -> None: + text = """ +repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.11.0 + - repo: https://github.com/psf/black-pre-commit-mirror + rev: 25.1.0 + - bad: data +""" + results = tuple(yield_pre_commit_version_data(text)) + assert results == ( + ("ruff-pre-commit", "0.11.0"), + ("black-pre-commit-mirror", "25.1.0"), + ) + + +def test_update_requirements( + tmp_path: Path, +) -> None: + requirements_file = tmp_path / "requirements.txt" + assert not requirements_file.exists() + requirements_file.write_text( + """# comment + # also comment but spaces line start +waffles are delicious no equals +black==3.1.4 ; specific version thingy +mypy==1.15.0 +ruff==1.2.5 +# required by soupy cat""", + encoding="utf-8", + ) + assert update_requirements(requirements_file, {"black": "3.1.5", "ruff": "1.2.7"}) + assert requirements_file.read_text(encoding="utf-8") == """# comment + # also comment but spaces line start +waffles are delicious no equals +black==3.1.5 ; specific version thingy +mypy==1.15.0 +ruff==1.2.7 +# required by soupy cat""" + + +def test_update_requirements_no_changes( + tmp_path: Path, +) -> None: + requirements_file = tmp_path / "requirements.txt" + assert not requirements_file.exists() + original = """# comment + # also comment but spaces line start +waffles are delicious no equals +black==3.1.4 ; specific version thingy +mypy==1.15.0 +ruff==1.2.5 +# required by soupy cat""" + requirements_file.write_text(original, encoding="utf-8") + assert not update_requirements( + requirements_file, {"black": "3.1.4", "ruff": "1.2.5"} + ) + assert requirements_file.read_text(encoding="utf-8") == original diff --git a/test-requirements.txt b/test-requirements.txt index 3314ac8663..80c2bf538b 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -39,7 +39,7 @@ colorama==0.4.6 ; sys_platform == 'win32' # sphinx coverage==7.16.0 # via -r test-requirements.in -cryptography==50.0.2 +cryptography==50.0.1 # via # -r test-requirements.in # pyopenssl From 557cea1b4f772f2b544a650bf1dbe45edfd4576d Mon Sep 17 00:00:00 2001 From: CheViana <656043+CheViana@users.noreply.github.com> Date: Thu, 1 Oct 2026 08:49:04 -0400 Subject: [PATCH 23/34] Revert "remove debugs from test_Exports.py" This reverts commit ff19be6a5436b01c95134787ece5a15c776454c8. --- .github/workflows/ci.yml | 1 - ci.sh | 7 +--- src/trio/_tests/pytest_plugin.py | 14 +++----- src/trio/_tests/test_dtls.py | 4 +-- src/trio/_tests/test_exports.py | 32 +++++++++++++------ src/trio/_tests/test_ssl.py | 4 +-- src/trio/_tests/tools/test_gen_exports.py | 10 +++--- .../_tests/tools/test_sync_requirements.py | 4 +-- test-requirements.txt | 4 +-- 9 files changed, 41 insertions(+), 39 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5e2e758ecd..ef26a0b2bb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -223,7 +223,6 @@ jobs: env: LSP: '${{ matrix.lsp }}' LSP_EXTRACT_FILE: '${{ matrix.lsp_extract_file }}' - SKIP_SSL_INSTALLATION: "${{ matrix.arch == 'x86' && 1 || 0 }}" - if: always() uses: codecov/codecov-action@v7 with: diff --git a/ci.sh b/ci.sh index 1ea2c49219..9362dbc727 100755 --- a/ci.sh +++ b/ci.sh @@ -54,13 +54,8 @@ if [ "${NO_TEST_REQUIREMENTS-0}" == 1 ]; then python -m uv pip install pytest coverage -c test-requirements.txt flags="--skip-optional-imports" else + python -m uv pip install -r test-requirements.txt --no-deps flags="" - if [ "${SKIP_SSL_INSTALLATION-0}" == 1 ]; then - echo cryptography >excluded_deps - PIP_ARGS="--exclude excluded_deps" - flags="--skip-ssl-imports" - fi - python -m uv pip install -r test-requirements.txt -vvv --no-deps $PIP_ARGS fi # If we're testing with a LSP installed, then it might break network diff --git a/src/trio/_tests/pytest_plugin.py b/src/trio/_tests/pytest_plugin.py index b929ae3a73..bbb5093be5 100644 --- a/src/trio/_tests/pytest_plugin.py +++ b/src/trio/_tests/pytest_plugin.py @@ -9,7 +9,6 @@ RUN_SLOW = True SKIP_OPTIONAL_IMPORTS = False -SKIP_SSL_IMPORTS = False def pytest_addoption(parser: pytest.Parser) -> None: @@ -19,18 +18,13 @@ def pytest_addoption(parser: pytest.Parser) -> None: action="store_true", help="skip tests that rely on libraries not required by trio itself", ) - parser.addoption( - "--skip-ssl-imports", - action="store_true", - help="skip tests that rely on cryptography", - ) def pytest_configure(config: pytest.Config) -> None: - global RUN_SLOW, SKIP_OPTIONAL_IMPORTS, SKIP_SSL_IMPORTS + global RUN_SLOW RUN_SLOW = config.getoption("--run-slow", default=True) + global SKIP_OPTIONAL_IMPORTS SKIP_OPTIONAL_IMPORTS = config.getoption("--skip-optional-imports", default=False) - SKIP_SSL_IMPORTS = config.getoption("--skip-ssl-imports", default=False) @pytest.fixture @@ -53,8 +47,8 @@ def pytest_pyfunc_call(pyfuncitem: pytest.Function) -> None: pyfuncitem.obj = trio_test(pyfuncitem.obj) -def maybe_ignore_import_error(error: ImportError) -> NoReturn: - if SKIP_OPTIONAL_IMPORTS or (SKIP_SSL_IMPORTS and error.name == "cryptography"): +def skip_if_optional_else_raise(error: ImportError) -> NoReturn: + if SKIP_OPTIONAL_IMPORTS: pytest.skip(error.msg, allow_module_level=True) else: # pragma: no cover raise error diff --git a/src/trio/_tests/test_dtls.py b/src/trio/_tests/test_dtls.py index 5a34d8c811..99cb68f11f 100644 --- a/src/trio/_tests/test_dtls.py +++ b/src/trio/_tests/test_dtls.py @@ -8,13 +8,13 @@ import attrs import pytest -from trio._tests.pytest_plugin import maybe_ignore_import_error +from trio._tests.pytest_plugin import skip_if_optional_else_raise try: import trustme from OpenSSL import SSL except ImportError as error: - maybe_ignore_import_error(error) + skip_if_optional_else_raise(error) import trio diff --git a/src/trio/_tests/test_exports.py b/src/trio/_tests/test_exports.py index 5328c88adc..80954e91f3 100644 --- a/src/trio/_tests/test_exports.py +++ b/src/trio/_tests/test_exports.py @@ -659,7 +659,7 @@ def test_pyright_recognizes_init_attributes() -> None: import trio import trio.testing -from trio._tests.pytest_plugin import RUN_SLOW, maybe_ignore_import_error +from trio._tests.pytest_plugin import RUN_SLOW, skip_if_optional_else_raise from .. import _core, _util from .._core._tests.tutil import slow @@ -681,7 +681,7 @@ def _ensure_mypy_cache_updated() -> None: try: from mypy.api import run except ImportError as error: - maybe_ignore_import_error(error) + skip_if_optional_else_raise(error) global mypy_cache_updated if not mypy_cache_updated: @@ -772,17 +772,29 @@ def iter_modules( sys.version_info.releaselevel == "alpha", reason="skip static introspection tools on Python dev/alpha releases", ) -@pytest.mark.parametrize("modname", PUBLIC_MODULE_NAMES) +@pytest.mark.parametrize( + "modname", + # PUBLIC_MODULE_NAMES + ["trio.socket"], +) @pytest.mark.parametrize( "tool", - ["pylint", "jedi", "mypy", "pyright_verifytypes"], + [ + "pylint", + # "jedi", + # "mypy", + # "pyright_verifytypes" + ], ) @pytest.mark.filterwarnings( # https://github.com/pypa/setuptools/issues/3274 "ignore:module 'sre_constants' is deprecated:DeprecationWarning", ) def test_static_tool_sees_all_symbols(tool: str, modname: str, tmp_path: Path) -> None: + print(sys.version_info) module = importlib.import_module(modname) + print(dir(module)) + assert False def no_underscores(symbols: Iterable[str]) -> set[str]: return {symbol for symbol in symbols if not symbol.startswith("_")} @@ -802,7 +814,7 @@ def no_underscores(symbols: Iterable[str]) -> set[str]: try: from pylint.lint import PyLinter except ImportError as error: - maybe_ignore_import_error(error) + skip_if_optional_else_raise(error) linter = PyLinter() assert module.__file__ is not None @@ -815,7 +827,7 @@ def no_underscores(symbols: Iterable[str]) -> set[str]: try: import jedi except ImportError as error: - maybe_ignore_import_error(error) + skip_if_optional_else_raise(error) # Simulate typing "import trio; trio." script = jedi.Script(f"import {modname}; {modname}.") @@ -858,7 +870,7 @@ def no_underscores(symbols: Iterable[str]) -> set[str]: try: import pyright # noqa: F401 except ImportError as error: - maybe_ignore_import_error(error) + skip_if_optional_else_raise(error) import subprocess res = subprocess.run( @@ -882,10 +894,12 @@ def no_underscores(symbols: Iterable[str]) -> set[str]: # static analysis (e.g. in trio.socket or trio.lowlevel) # So we check that the runtime names are a subset of the static names. missing_names = runtime_names - static_names + print(f"{missing_names=}") + print(f"{runtime_names=}") + print(f"{static_names=}") # ignore warnings about deprecated module tests missing_names -= {"tests"} - missing_names -= {"ALG_SET_PUB_KEY"} # https://github.com/pypy/pypy/issues/5594 if missing_names: # pragma: no cover print(f"{tool} can't see the following names in {modname}:") @@ -1024,7 +1038,7 @@ def lookup_symbol(symbol: str) -> dict[str, Any]: # type: ignore[misc, explicit try: import jedi except ImportError as error: - maybe_ignore_import_error(error) + skip_if_optional_else_raise(error) script = jedi.Script( f"from {module_name} import {class_name}; {class_name}.", diff --git a/src/trio/_tests/test_ssl.py b/src/trio/_tests/test_ssl.py index f0dd884c64..8eb0659270 100644 --- a/src/trio/_tests/test_ssl.py +++ b/src/trio/_tests/test_ssl.py @@ -13,7 +13,7 @@ import pytest from trio import StapledStream -from trio._tests.pytest_plugin import maybe_ignore_import_error +from trio._tests.pytest_plugin import skip_if_optional_else_raise from trio.abc import ReceiveStream, SendStream from trio.testing import MemoryReceiveStream, MemorySendStream @@ -21,7 +21,7 @@ import trustme from OpenSSL import SSL except ImportError as error: - maybe_ignore_import_error(error) + skip_if_optional_else_raise(error) import trio diff --git a/src/trio/_tests/tools/test_gen_exports.py b/src/trio/_tests/tools/test_gen_exports.py index 3b923ed83d..81e6d5f18f 100644 --- a/src/trio/_tests/tools/test_gen_exports.py +++ b/src/trio/_tests/tools/test_gen_exports.py @@ -4,14 +4,14 @@ import pytest -from trio._tests.pytest_plugin import maybe_ignore_import_error +from trio._tests.pytest_plugin import skip_if_optional_else_raise # imports in gen_exports that are not in `install_requires` in setup.py try: import astor # noqa: F401 import isort # noqa: F401 except ImportError as error: - maybe_ignore_import_error(error) + skip_if_optional_else_raise(error) from trio._tools.gen_exports import ( @@ -103,7 +103,7 @@ def test_process( import black # noqa: F401 # there's no dedicated CI run that has astor+isort, but lacks black. except ImportError as error: # pragma: no cover - maybe_ignore_import_error(error) + skip_if_optional_else_raise(error) modpath = tmp_path / "_module.py" genpath = tmp_path / "_generated_module.py" @@ -141,7 +141,7 @@ def test_run_ruff(tmp_path: Path) -> None: try: import ruff # noqa: F401 except ImportError as error: # pragma: no cover - maybe_ignore_import_error(error) + skip_if_optional_else_raise(error) file = File(tmp_path / "module.py", "module") @@ -168,7 +168,7 @@ def test_lint_failure(tmp_path: Path) -> None: import black # noqa: F401 import ruff # noqa: F401 except ImportError as error: # pragma: no cover - maybe_ignore_import_error(error) + skip_if_optional_else_raise(error) file = File(tmp_path / "module.py", "module") diff --git a/src/trio/_tests/tools/test_sync_requirements.py b/src/trio/_tests/tools/test_sync_requirements.py index 5beb707558..92b3bfdd97 100644 --- a/src/trio/_tests/tools/test_sync_requirements.py +++ b/src/trio/_tests/tools/test_sync_requirements.py @@ -2,13 +2,13 @@ from typing import TYPE_CHECKING -from trio._tests.pytest_plugin import maybe_ignore_import_error +from trio._tests.pytest_plugin import skip_if_optional_else_raise # imports in gen_exports that are not in `install_requires` in requirements try: import yaml # noqa: F401 except ImportError as error: - maybe_ignore_import_error(error) + skip_if_optional_else_raise(error) from trio._tools.sync_requirements import ( update_requirements, diff --git a/test-requirements.txt b/test-requirements.txt index 80c2bf538b..96e65ac31d 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -39,7 +39,7 @@ colorama==0.4.6 ; sys_platform == 'win32' # sphinx coverage==7.16.0 # via -r test-requirements.in -cryptography==50.0.1 +cryptography==48.0.0 # via # -r test-requirements.in # pyopenssl @@ -126,7 +126,7 @@ pygments==2.21.0 # sphinx pylint==4.0.8 # via -r test-requirements.in -pyopenssl==26.4.0 +pyopenssl==26.2.0 # via -r test-requirements.in pyright==1.1.411 # via -r test-requirements.in From 614d6ab812639cf5a0e6ca5b1c2c3cab6f259b24 Mon Sep 17 00:00:00 2001 From: A5rocks Date: Tue, 29 Sep 2026 23:18:48 -0400 Subject: [PATCH 24/34] Avoid SSL tests on x86 Windows as cryptography doesn't support it (#3515) --- .github/workflows/ci.yml | 1 + ci.sh | 7 ++++++- src/trio/_tests/pytest_plugin.py | 14 ++++++++++---- src/trio/_tests/test_dtls.py | 4 ++-- src/trio/_tests/test_exports.py | 12 ++++++------ src/trio/_tests/test_ssl.py | 4 ++-- src/trio/_tests/tools/test_gen_exports.py | 10 +++++----- src/trio/_tests/tools/test_sync_requirements.py | 4 ++-- test-requirements.txt | 6 +++++- 9 files changed, 39 insertions(+), 23 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ef26a0b2bb..5e2e758ecd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -223,6 +223,7 @@ jobs: env: LSP: '${{ matrix.lsp }}' LSP_EXTRACT_FILE: '${{ matrix.lsp_extract_file }}' + SKIP_SSL_INSTALLATION: "${{ matrix.arch == 'x86' && 1 || 0 }}" - if: always() uses: codecov/codecov-action@v7 with: diff --git a/ci.sh b/ci.sh index 9362dbc727..d8d1094699 100755 --- a/ci.sh +++ b/ci.sh @@ -54,8 +54,13 @@ if [ "${NO_TEST_REQUIREMENTS-0}" == 1 ]; then python -m uv pip install pytest coverage -c test-requirements.txt flags="--skip-optional-imports" else - python -m uv pip install -r test-requirements.txt --no-deps flags="" + if [ "${SKIP_SSL_INSTALLATION-0}" == 1 ]; then + echo cryptography >excluded_deps + PIP_ARGS="--exclude excluded_deps" + flags="--skip-ssl-imports" + fi + python -m uv pip install -r test-requirements.txt --no-deps $PIP_ARGS fi # If we're testing with a LSP installed, then it might break network diff --git a/src/trio/_tests/pytest_plugin.py b/src/trio/_tests/pytest_plugin.py index bbb5093be5..b929ae3a73 100644 --- a/src/trio/_tests/pytest_plugin.py +++ b/src/trio/_tests/pytest_plugin.py @@ -9,6 +9,7 @@ RUN_SLOW = True SKIP_OPTIONAL_IMPORTS = False +SKIP_SSL_IMPORTS = False def pytest_addoption(parser: pytest.Parser) -> None: @@ -18,13 +19,18 @@ def pytest_addoption(parser: pytest.Parser) -> None: action="store_true", help="skip tests that rely on libraries not required by trio itself", ) + parser.addoption( + "--skip-ssl-imports", + action="store_true", + help="skip tests that rely on cryptography", + ) def pytest_configure(config: pytest.Config) -> None: - global RUN_SLOW + global RUN_SLOW, SKIP_OPTIONAL_IMPORTS, SKIP_SSL_IMPORTS RUN_SLOW = config.getoption("--run-slow", default=True) - global SKIP_OPTIONAL_IMPORTS SKIP_OPTIONAL_IMPORTS = config.getoption("--skip-optional-imports", default=False) + SKIP_SSL_IMPORTS = config.getoption("--skip-ssl-imports", default=False) @pytest.fixture @@ -47,8 +53,8 @@ def pytest_pyfunc_call(pyfuncitem: pytest.Function) -> None: pyfuncitem.obj = trio_test(pyfuncitem.obj) -def skip_if_optional_else_raise(error: ImportError) -> NoReturn: - if SKIP_OPTIONAL_IMPORTS: +def maybe_ignore_import_error(error: ImportError) -> NoReturn: + if SKIP_OPTIONAL_IMPORTS or (SKIP_SSL_IMPORTS and error.name == "cryptography"): pytest.skip(error.msg, allow_module_level=True) else: # pragma: no cover raise error diff --git a/src/trio/_tests/test_dtls.py b/src/trio/_tests/test_dtls.py index 99cb68f11f..5a34d8c811 100644 --- a/src/trio/_tests/test_dtls.py +++ b/src/trio/_tests/test_dtls.py @@ -8,13 +8,13 @@ import attrs import pytest -from trio._tests.pytest_plugin import skip_if_optional_else_raise +from trio._tests.pytest_plugin import maybe_ignore_import_error try: import trustme from OpenSSL import SSL except ImportError as error: - skip_if_optional_else_raise(error) + maybe_ignore_import_error(error) import trio diff --git a/src/trio/_tests/test_exports.py b/src/trio/_tests/test_exports.py index c9b29f2ce6..774d3ea68d 100644 --- a/src/trio/_tests/test_exports.py +++ b/src/trio/_tests/test_exports.py @@ -19,7 +19,7 @@ import trio import trio.testing -from trio._tests.pytest_plugin import RUN_SLOW, skip_if_optional_else_raise +from trio._tests.pytest_plugin import RUN_SLOW, maybe_ignore_import_error from .. import _core, _util from .._core._tests.tutil import slow @@ -41,7 +41,7 @@ def _ensure_mypy_cache_updated() -> None: try: from mypy.api import run except ImportError as error: - skip_if_optional_else_raise(error) + maybe_ignore_import_error(error) global mypy_cache_updated if not mypy_cache_updated: @@ -174,7 +174,7 @@ def no_underscores(symbols: Iterable[str]) -> set[str]: try: from pylint.lint import PyLinter except ImportError as error: - skip_if_optional_else_raise(error) + maybe_ignore_import_error(error) linter = PyLinter() assert module.__file__ is not None @@ -187,7 +187,7 @@ def no_underscores(symbols: Iterable[str]) -> set[str]: try: import jedi except ImportError as error: - skip_if_optional_else_raise(error) + maybe_ignore_import_error(error) # Simulate typing "import trio; trio." script = jedi.Script(f"import {modname}; {modname}.") @@ -230,7 +230,7 @@ def no_underscores(symbols: Iterable[str]) -> set[str]: try: import pyright # noqa: F401 except ImportError as error: - skip_if_optional_else_raise(error) + maybe_ignore_import_error(error) import subprocess res = subprocess.run( @@ -402,7 +402,7 @@ def lookup_symbol(symbol: str) -> dict[str, Any]: # type: ignore[misc, explicit try: import jedi except ImportError as error: - skip_if_optional_else_raise(error) + maybe_ignore_import_error(error) script = jedi.Script( f"from {module_name} import {class_name}; {class_name}.", diff --git a/src/trio/_tests/test_ssl.py b/src/trio/_tests/test_ssl.py index 8eb0659270..f0dd884c64 100644 --- a/src/trio/_tests/test_ssl.py +++ b/src/trio/_tests/test_ssl.py @@ -13,7 +13,7 @@ import pytest from trio import StapledStream -from trio._tests.pytest_plugin import skip_if_optional_else_raise +from trio._tests.pytest_plugin import maybe_ignore_import_error from trio.abc import ReceiveStream, SendStream from trio.testing import MemoryReceiveStream, MemorySendStream @@ -21,7 +21,7 @@ import trustme from OpenSSL import SSL except ImportError as error: - skip_if_optional_else_raise(error) + maybe_ignore_import_error(error) import trio diff --git a/src/trio/_tests/tools/test_gen_exports.py b/src/trio/_tests/tools/test_gen_exports.py index 81e6d5f18f..3b923ed83d 100644 --- a/src/trio/_tests/tools/test_gen_exports.py +++ b/src/trio/_tests/tools/test_gen_exports.py @@ -4,14 +4,14 @@ import pytest -from trio._tests.pytest_plugin import skip_if_optional_else_raise +from trio._tests.pytest_plugin import maybe_ignore_import_error # imports in gen_exports that are not in `install_requires` in setup.py try: import astor # noqa: F401 import isort # noqa: F401 except ImportError as error: - skip_if_optional_else_raise(error) + maybe_ignore_import_error(error) from trio._tools.gen_exports import ( @@ -103,7 +103,7 @@ def test_process( import black # noqa: F401 # there's no dedicated CI run that has astor+isort, but lacks black. except ImportError as error: # pragma: no cover - skip_if_optional_else_raise(error) + maybe_ignore_import_error(error) modpath = tmp_path / "_module.py" genpath = tmp_path / "_generated_module.py" @@ -141,7 +141,7 @@ def test_run_ruff(tmp_path: Path) -> None: try: import ruff # noqa: F401 except ImportError as error: # pragma: no cover - skip_if_optional_else_raise(error) + maybe_ignore_import_error(error) file = File(tmp_path / "module.py", "module") @@ -168,7 +168,7 @@ def test_lint_failure(tmp_path: Path) -> None: import black # noqa: F401 import ruff # noqa: F401 except ImportError as error: # pragma: no cover - skip_if_optional_else_raise(error) + maybe_ignore_import_error(error) file = File(tmp_path / "module.py", "module") diff --git a/src/trio/_tests/tools/test_sync_requirements.py b/src/trio/_tests/tools/test_sync_requirements.py index 92b3bfdd97..5beb707558 100644 --- a/src/trio/_tests/tools/test_sync_requirements.py +++ b/src/trio/_tests/tools/test_sync_requirements.py @@ -2,13 +2,13 @@ from typing import TYPE_CHECKING -from trio._tests.pytest_plugin import skip_if_optional_else_raise +from trio._tests.pytest_plugin import maybe_ignore_import_error # imports in gen_exports that are not in `install_requires` in requirements try: import yaml # noqa: F401 except ImportError as error: - skip_if_optional_else_raise(error) + maybe_ignore_import_error(error) from trio._tools.sync_requirements import ( update_requirements, diff --git a/test-requirements.txt b/test-requirements.txt index e812a60f0d..184b0ca79d 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -39,7 +39,11 @@ colorama==0.4.6 ; sys_platform == 'win32' # sphinx coverage==7.16.2 # via -r test-requirements.in +<<<<<<< HEAD cryptography==50.0.2 +======= +cryptography==50.0.1 +>>>>>>> ada9c778 (Avoid SSL tests on x86 Windows as cryptography doesn't support it (#3515)) # via # -r test-requirements.in # pyopenssl @@ -127,7 +131,7 @@ pygments==2.21.0 # sphinx pylint==4.1.1 # via -r test-requirements.in -pyopenssl==26.2.0 +pyopenssl==26.4.0 # via -r test-requirements.in pyright==1.1.414 # via -r test-requirements.in From d0bb1a4e308bda74f8951cf094cf6646d58b2c89 Mon Sep 17 00:00:00 2001 From: CheViana <656043+CheViana@users.noreply.github.com> Date: Thu, 1 Oct 2026 09:29:04 -0400 Subject: [PATCH 25/34] remove debug code and fix reqs --- src/trio/_tests/test_exports.py | 22 ++-------------------- test-requirements.txt | 4 ---- 2 files changed, 2 insertions(+), 24 deletions(-) diff --git a/src/trio/_tests/test_exports.py b/src/trio/_tests/test_exports.py index 774d3ea68d..838f5162b7 100644 --- a/src/trio/_tests/test_exports.py +++ b/src/trio/_tests/test_exports.py @@ -132,29 +132,14 @@ def iter_modules( sys.version_info.releaselevel == "alpha", reason="skip static introspection tools on Python dev/alpha releases", ) -@pytest.mark.parametrize( - "modname", - # PUBLIC_MODULE_NAMES - ["trio.socket"], -) -@pytest.mark.parametrize( - "tool", - [ - "pylint", - # "jedi", - # "mypy", - # "pyright_verifytypes" - ], -) +@pytest.mark.parametrize("modname", PUBLIC_MODULE_NAMES) +@pytest.mark.parametrize("tool", ["pylint", "jedi", "mypy", "pyright_verifytypes"]) @pytest.mark.filterwarnings( # https://github.com/pypa/setuptools/issues/3274 "ignore:module 'sre_constants' is deprecated:DeprecationWarning", ) def test_static_tool_sees_all_symbols(tool: str, modname: str, tmp_path: Path) -> None: - print(sys.version_info) module = importlib.import_module(modname) - print(dir(module)) - assert False def no_underscores(symbols: Iterable[str]) -> set[str]: return {symbol for symbol in symbols if not symbol.startswith("_")} @@ -254,9 +239,6 @@ def no_underscores(symbols: Iterable[str]) -> set[str]: # static analysis (e.g. in trio.socket or trio.lowlevel) # So we check that the runtime names are a subset of the static names. missing_names = runtime_names - static_names - print(f"{missing_names=}") - print(f"{runtime_names=}") - print(f"{static_names=}") # ignore warnings about deprecated module tests missing_names -= {"tests"} diff --git a/test-requirements.txt b/test-requirements.txt index 184b0ca79d..2e4cad4b0f 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -39,11 +39,7 @@ colorama==0.4.6 ; sys_platform == 'win32' # sphinx coverage==7.16.2 # via -r test-requirements.in -<<<<<<< HEAD cryptography==50.0.2 -======= -cryptography==50.0.1 ->>>>>>> ada9c778 (Avoid SSL tests on x86 Windows as cryptography doesn't support it (#3515)) # via # -r test-requirements.in # pyopenssl From 3cd8d7fbe73c8176517bd77085a1dcf11d16fb09 Mon Sep 17 00:00:00 2001 From: Jane R Date: Thu, 1 Oct 2026 09:43:40 -0400 Subject: [PATCH 26/34] fix rst formatting --- src/trio/_core/_io_epoll.py | 3 ++- src/trio/_core/_io_kqueue.py | 3 ++- src/trio/_core/_io_windows.py | 3 ++- src/trio/_core/_local.py | 6 +++--- src/trio/_core/_run.py | 8 ++++---- src/trio/_file_io.py | 2 +- src/trio/_sync.py | 4 ++-- 7 files changed, 16 insertions(+), 13 deletions(-) diff --git a/src/trio/_core/_io_epoll.py b/src/trio/_core/_io_epoll.py index 9b13da428c..3995d6399c 100644 --- a/src/trio/_core/_io_epoll.py +++ b/src/trio/_core/_io_epoll.py @@ -35,13 +35,14 @@ class EpollWaiters: class _EpollStatistics: """ DTO class that holds information on current epoll status: + - amount of tasks waiting on read - amount of tasks waiting on write - backend = "epoll" This class is used with platform "Linux" or "Darwin" (or any other platform that supports epoll functionality). - It has a sibling :class:`~trio._core._io_windows._WindowsStatistics` + It has a sibling ``_WindowsStatistics`` that is used with platform "Windows". """ diff --git a/src/trio/_core/_io_kqueue.py b/src/trio/_core/_io_kqueue.py index e1979a6a4f..1982bde9cb 100644 --- a/src/trio/_core/_io_kqueue.py +++ b/src/trio/_core/_io_kqueue.py @@ -28,13 +28,14 @@ class _KqueueStatistics: """ DTO class that holds information on current kqueue status: + - amount of tasks waiting - amount of monitors - backend = "kqueue" This class is used with platform "Linux" or "Darwin" (or any other platform that supports epoll functionality). - It has a sibling :class:`~trio._core._io_windows._WindowsStatistics` + It has a sibling ``_WindowsStatistics`` that is used with platform "Windows". """ diff --git a/src/trio/_core/_io_windows.py b/src/trio/_core/_io_windows.py index c30d4e101b..d447cc0993 100644 --- a/src/trio/_core/_io_windows.py +++ b/src/trio/_core/_io_windows.py @@ -296,6 +296,7 @@ class _WindowsStatistics: """ DTO class that holds information on current status that pertains to IO waits: + - amount of tasks waiting on read - amount of tasks waiting on write - amount of overlapped tasks @@ -303,7 +304,7 @@ class _WindowsStatistics: - backend = "windows" This class is used with platform "Windows". - It has a sibling :class:`~trio._core._io_epoll._EpollStatistics` + It has a sibling ``_EpollStatistics`` that is used with platform "Linux" or "Darwin". """ diff --git a/src/trio/_core/_local.py b/src/trio/_core/_local.py index 8f8751a5d7..fe5afd69d8 100644 --- a/src/trio/_core/_local.py +++ b/src/trio/_core/_local.py @@ -14,9 +14,9 @@ @final class _NoValue: """ - Indicates `RunVar` variable without a value - (e.g. when variable is not initialized with value yet). - Like `undefined` in JS (but better). + Indicates a :class:`~trio.lowlevel.RunVar` variable without a value + (e.g. when the variable is not initialized with a value yet). + Like ``undefined`` in JS (but better). """ diff --git a/src/trio/_core/_run.py b/src/trio/_core/_run.py index e9b3d05a99..f40d8e0611 100644 --- a/src/trio/_core/_run.py +++ b/src/trio/_core/_run.py @@ -1487,12 +1487,12 @@ def __del__(self) -> None: @attrs.define(eq=False, repr=False) class Task(metaclass=NoPublicConstructor): # type: ignore[explicit-any] """ - Class for Trio's `Task` primitive. + Class for Trio's ``Task`` primitive. Wraps coroutine, adding more functionality for stopping and resuming tasks, - for scheduling and canceling tasks, for associating a `Task` with `Runner` and + for scheduling and canceling tasks, for associating a ``Task`` with ``Runner`` and context (`contextvars.Context`), among other things. - `Task` can belong to `trio._core._run.Nursery` and can spawn child nurseries. - See `trio._core._run.Nursery` docs for more information. + A ``Task`` can belong to a :class:`~trio.Nursery` and can spawn child nurseries. + See :class:`~trio.Nursery` docs for more information. """ _parent_nursery: Nursery | None diff --git a/src/trio/_file_io.py b/src/trio/_file_io.py index fe7319d774..344fd2416d 100644 --- a/src/trio/_file_io.py +++ b/src/trio/_file_io.py @@ -119,7 +119,7 @@ class _HasFileNo(Protocol): def fileno(self) -> int: """ Integer that corresponds to file descriptor, as in - `sys.stdout.fileno()` + ``sys.stdout.fileno()`` """ ... diff --git a/src/trio/_sync.py b/src/trio/_sync.py index d0364a2b99..b0b32a6d3b 100644 --- a/src/trio/_sync.py +++ b/src/trio/_sync.py @@ -140,8 +140,8 @@ def release(self) -> object: class AsyncContextManagerMixin: """ An async context manager base class. - Should be used with :class:`~trio._sync._HasAcquireRelease` Protocol. - Calls `await self.acquire()` on entry and `self.release` on exit, + Should be used with the ``_HasAcquireRelease`` Protocol. + Calls ``await self.acquire()`` on entry and ``self.release()`` on exit, adding KeyboardInterrupt protection support. """ From f1561ee45ba87041e99a56bb512b2ceb60e97460 Mon Sep 17 00:00:00 2001 From: Jane R Date: Thu, 1 Oct 2026 10:46:38 -0400 Subject: [PATCH 27/34] beautify IOStatistics docstrings --- src/trio/_core/_io_epoll.py | 31 +++++++++++++++++------- src/trio/_core/_io_kqueue.py | 31 +++++++++++++++++------- src/trio/_core/_io_windows.py | 44 ++++++++++++++++++++++++----------- 3 files changed, 75 insertions(+), 31 deletions(-) diff --git a/src/trio/_core/_io_epoll.py b/src/trio/_core/_io_epoll.py index 3995d6399c..7ab3f3aca8 100644 --- a/src/trio/_core/_io_epoll.py +++ b/src/trio/_core/_io_epoll.py @@ -33,17 +33,30 @@ class EpollWaiters: @attrs.frozen(eq=False) class _EpollStatistics: - """ - DTO class that holds information on current epoll status: + """DTO class that holds information on current epoll status. + + This class can be used on any platform that supports + epoll functionality (e.g. Linux). + Trio also defines other similar classes for statistics reporting: + ``_WindowsStatistics`` and ``_KqueueStatistics``. + They are similar in function but some fields differ. + All of them have a ``backend`` attribute. + The statistics class best suited for the runtime platform is + imported as the ``IOStatistics`` type. See ``trio._core._run``. + + .. attribute:: tasks_waiting_read + + Number of tasks waiting on read. + + .. attribute:: tasks_waiting_write + + Number of tasks waiting on write. - - amount of tasks waiting on read - - amount of tasks waiting on write - - backend = "epoll" + .. attribute:: backend - This class is used with platform "Linux" or "Darwin" - (or any other platform that supports epoll functionality). - It has a sibling ``_WindowsStatistics`` - that is used with platform "Windows". + This attribute holds a string value that corresponds to the statistics type. + This class has ``backend == "epoll"``. + Use ``backend`` attribute to distinguish statistics types at runtime. """ tasks_waiting_read: int diff --git a/src/trio/_core/_io_kqueue.py b/src/trio/_core/_io_kqueue.py index 1982bde9cb..514434f16d 100644 --- a/src/trio/_core/_io_kqueue.py +++ b/src/trio/_core/_io_kqueue.py @@ -26,17 +26,30 @@ @attrs.frozen(eq=False) class _KqueueStatistics: - """ - DTO class that holds information on current kqueue status: + """DTO class that holds information on current kqueue status. + + This class can be used on any platform that supports + kqueue functionality (e.g. BSD/Darwin). + Trio also defines other similar classes for statistics reporting: + ``_WindowsStatistics`` and ``_EpollStatistics``. + They are similar in function but some fields differ. + All of them have a ``backend`` attribute. + The statistics class best suited for the runtime platform is + imported as the ``IOStatistics`` type. See ``trio._core._run``. + + .. attribute:: tasks_waiting + + Number of tasks that are currently in the waiting state. + + .. attribute:: monitors + + Number of monitors. - - amount of tasks waiting - - amount of monitors - - backend = "kqueue" + .. attribute:: backend - This class is used with platform "Linux" or "Darwin" - (or any other platform that supports epoll functionality). - It has a sibling ``_WindowsStatistics`` - that is used with platform "Windows". + This attribute holds a string value that corresponds to the statistics type. + This class has ``backend == "kqueue"``. + Use ``backend`` attribute to distinguish statistics types at runtime. """ tasks_waiting: int diff --git a/src/trio/_core/_io_windows.py b/src/trio/_core/_io_windows.py index d447cc0993..9c3403d937 100644 --- a/src/trio/_core/_io_windows.py +++ b/src/trio/_core/_io_windows.py @@ -293,19 +293,37 @@ class AFDGroup: @attrs.frozen(eq=False) class _WindowsStatistics: - """ - DTO class that holds information on current status - that pertains to IO waits: - - - amount of tasks waiting on read - - amount of tasks waiting on write - - amount of overlapped tasks - - amount of completion monitors - - backend = "windows" - - This class is used with platform "Windows". - It has a sibling ``_EpollStatistics`` - that is used with platform "Linux" or "Darwin". + """DTO class that holds information on current I/O waits status. + + This class is used on the Windows platform. + Trio also defines other similar classes for statistics reporting: + ``_KqueueStatistics`` and ``_EpollStatistics``. + They are similar in function but some fields differ. + All of them have a ``backend`` attribute. + The statistics class best suited for the runtime platform is + imported as the ``IOStatistics`` type. See ``trio._core._run``. + + .. attribute:: tasks_waiting_read + + Number of tasks waiting on read. + + .. attribute:: tasks_waiting_write + + Number of tasks waiting on write. + + .. attribute:: tasks_waiting_overlapped + + Number of tasks waiting on overlapped I/O operations. + + .. attribute:: completion_key_monitors + + Number of completion key monitors. + + .. attribute:: backend + + This attribute holds a string value that corresponds to the statistics type. + This class has ``backend == "windows"``. + Use ``backend`` attribute to distinguish statistics types at runtime. """ tasks_waiting_read: int From 268ea76c29d8b921431c71b8b5ee0a727bb4f66d Mon Sep 17 00:00:00 2001 From: Jane R Date: Thu, 1 Oct 2026 11:49:33 -0400 Subject: [PATCH 28/34] beautify other docstrings --- src/trio/_core/_local.py | 9 +++++---- src/trio/_core/_run.py | 16 +++++++++------- src/trio/_file_io.py | 3 +-- src/trio/_sync.py | 12 +++++------- src/trio/_tests/check_type_completeness.py | 6 ++++-- 5 files changed, 24 insertions(+), 22 deletions(-) diff --git a/src/trio/_core/_local.py b/src/trio/_core/_local.py index fe5afd69d8..5d7885eb33 100644 --- a/src/trio/_core/_local.py +++ b/src/trio/_core/_local.py @@ -13,10 +13,11 @@ @final class _NoValue: - """ - Indicates a :class:`~trio.lowlevel.RunVar` variable without a value - (e.g. when the variable is not initialized with a value yet). - Like ``undefined`` in JS (but better). + """Sentinel class object, used as the "undefined" variable value. + + A :class:`~trio.lowlevel.RunVar` variable has this "stub" value + until the variable is assigned an actual value. + It is distinct from ``None``, which is a legitimate value a variable can hold. """ diff --git a/src/trio/_core/_run.py b/src/trio/_core/_run.py index f40d8e0611..93dca680d0 100644 --- a/src/trio/_core/_run.py +++ b/src/trio/_core/_run.py @@ -1486,13 +1486,15 @@ def __del__(self) -> None: @final @attrs.define(eq=False, repr=False) class Task(metaclass=NoPublicConstructor): # type: ignore[explicit-any] - """ - Class for Trio's ``Task`` primitive. - Wraps coroutine, adding more functionality for stopping and resuming tasks, - for scheduling and canceling tasks, for associating a ``Task`` with ``Runner`` and - context (`contextvars.Context`), among other things. - A ``Task`` can belong to a :class:`~trio.Nursery` and can spawn child nurseries. - See :class:`~trio.Nursery` docs for more information. + """A ``Task`` object represents a concurrent "thread" of execution. + + See the :class:`~trio.lowlevel.Task` entry in Trio's lowlevel module docs for a detailed description. + Trio's task primitive wraps a coroutine (`types.CoroutineType`), + adding more functionality for stopping and resuming tasks, + for scheduling and canceling tasks, + for associating a ``Task`` with a ``Runner`` and context variables. + A ``Task`` can belong to a :class:`~trio.Nursery` and can spawn + its own child nurseries. See :class:`~trio.Nursery` docs for more information. """ _parent_nursery: Nursery | None diff --git a/src/trio/_file_io.py b/src/trio/_file_io.py index 344fd2416d..281a177640 100644 --- a/src/trio/_file_io.py +++ b/src/trio/_file_io.py @@ -119,9 +119,8 @@ class _HasFileNo(Protocol): def fileno(self) -> int: """ Integer that corresponds to file descriptor, as in - ``sys.stdout.fileno()`` + ``sys.stdout.fileno()``. """ - ... class _HasIsATTY(Protocol): def isatty(self) -> bool: ... diff --git a/src/trio/_sync.py b/src/trio/_sync.py index b0b32a6d3b..cba2958f98 100644 --- a/src/trio/_sync.py +++ b/src/trio/_sync.py @@ -125,24 +125,22 @@ class _HasAcquireRelease(Protocol): async def acquire(self) -> object: """ Method for resource locking. - Protocol implies this is similar to `threading.Semaphore.acquire`. + See docs for :class:`~trio.Lock` and :class:`~trio.StrictFIFOLock` for usage details. """ - ... def release(self) -> object: """ Method for resource releasing. - Protocol implies this is similar to `threading.Semaphore.release`. + See docs for :class:`~trio.Lock` and :class:`~trio.StrictFIFOLock` for usage details. """ - ... class AsyncContextManagerMixin: - """ - An async context manager base class. + """An async context manager base class. + Should be used with the ``_HasAcquireRelease`` Protocol. Calls ``await self.acquire()`` on entry and ``self.release()`` on exit, - adding KeyboardInterrupt protection support. + adding :exc:`KeyboardInterrupt` protection support. """ @enable_ki_protection diff --git a/src/trio/_tests/check_type_completeness.py b/src/trio/_tests/check_type_completeness.py index d43d80fc04..8867e9017c 100755 --- a/src/trio/_tests/check_type_completeness.py +++ b/src/trio/_tests/check_type_completeness.py @@ -76,8 +76,7 @@ def has_docstring_at_runtime(name: str) -> bool: # darwin "trio._core._io_kqueue._KqueueStatistics", # confirmed to have docstring # windows - "trio._socket.SocketType.share", # doesn't have docstring, like other - # SocketType methods. Shows up here bc method def is wrapped in `if not TYPE_CHECKING` + "trio._socket.SocketType.share", # gets docstring copied from socket stdlib "trio._core._io_windows._WindowsStatistics", # confirmed to have docstring "trio._core._windows_cffi.Handle", # doesn't have docstring, it's basically type alias # linux @@ -101,6 +100,9 @@ def has_docstring_at_runtime(name: str) -> bool: # if TYPE_CHECKING: # from .._file_io import _HasFileNo # ``` + # The situation can be dramatically remedied if pyright fixes bug #11803 + # and trio's Path file vague typing is fixed/ignored. Then can + # decommision this file and put plain ``pyright`` check in tox.ini/ci.yaml. "trio._file_io._HasFileNo", "trio._file_io._HasFileNo.fileno", # verified that `HasFileNo` has docstrings From 7c81d784fd07efb6bba15953c3668916242ef9ad Mon Sep 17 00:00:00 2001 From: CheViana <656043+CheViana@users.noreply.github.com> Date: Thu, 1 Oct 2026 11:53:31 -0400 Subject: [PATCH 29/34] shorter newsfragment --- newsfragments/3226.bugfix.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newsfragments/3226.bugfix.rst b/newsfragments/3226.bugfix.rst index 6aa4e8c525..6a79334f25 100644 --- a/newsfragments/3226.bugfix.rst +++ b/newsfragments/3226.bugfix.rst @@ -1 +1 @@ -Add docstrings for the classes and functions that ``pyright --verifytypes`` complains about. +Add docstrings that ``pyright --verifytypes`` complains about. From 563c1b2a643ac5ecab859d3c5808666c1381d18d Mon Sep 17 00:00:00 2001 From: CheViana <656043+CheViana@users.noreply.github.com> Date: Thu, 1 Oct 2026 12:35:58 -0400 Subject: [PATCH 30/34] touch to rerun CI --- src/trio/_file_io.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/trio/_file_io.py b/src/trio/_file_io.py index 281a177640..cd341cfc24 100644 --- a/src/trio/_file_io.py +++ b/src/trio/_file_io.py @@ -118,8 +118,7 @@ class _HasFileNo(Protocol): def fileno(self) -> int: """ - Integer that corresponds to file descriptor, as in - ``sys.stdout.fileno()``. + Integer that corresponds to file descriptor, as in ``sys.stdout.fileno()``. """ class _HasIsATTY(Protocol): From ac8efc5ed2acdd18de39794590a99e5e5bcdfe9d Mon Sep 17 00:00:00 2001 From: CheViana <656043+CheViana@users.noreply.github.com> Date: Thu, 1 Oct 2026 21:35:23 -0400 Subject: [PATCH 31/34] revert check_type_completeness.py and bring back empty json --- src/trio/_tests/check_type_completeness.py | 99 +++++++++++++++------- 1 file changed, 67 insertions(+), 32 deletions(-) diff --git a/src/trio/_tests/check_type_completeness.py b/src/trio/_tests/check_type_completeness.py index 8867e9017c..bbf38080d3 100755 --- a/src/trio/_tests/check_type_completeness.py +++ b/src/trio/_tests/check_type_completeness.py @@ -69,43 +69,37 @@ def has_docstring_at_runtime(name: str) -> bool: if "AsyncIOWrapper" in str(exc) or name in ( # Symbols not existing on all platforms, so we can't dynamically inspect them. # Manually confirmed to have docstrings but pyright doesn't see them due to - # export shenanigans. + # export shenanigans. TODO: actually manually confirm that. # In theory we could verify these at runtime, probably by running the script separately # on separate platforms. It might also be a decent idea to work the other way around, # a la test_static_tool_sees_class_members # darwin - "trio._core._io_kqueue._KqueueStatistics", # confirmed to have docstring + "trio.lowlevel.current_kqueue", + "trio.lowlevel.monitor_kevent", + "trio.lowlevel.wait_kevent", + "trio._core._io_kqueue._KqueueStatistics", # windows - "trio._socket.SocketType.share", # gets docstring copied from socket stdlib - "trio._core._io_windows._WindowsStatistics", # confirmed to have docstring - "trio._core._windows_cffi.Handle", # doesn't have docstring, it's basically type alias + "trio._socket.SocketType.share", + "trio._core._io_windows._WindowsStatistics", + "trio._core._windows_cffi.Handle", + "trio.lowlevel.current_iocp", + "trio.lowlevel.monitor_completion_key", + "trio.lowlevel.readinto_overlapped", + "trio.lowlevel.register_with_iocp", + "trio.lowlevel.wait_overlapped", + "trio.lowlevel.write_overlapped", + "trio.lowlevel.WaitForSingleObject", + "trio.socket.fromshare", # linux # this test will fail on linux, but I don't develop on linux. So the next # person to do so is very welcome to open a pull request and populate with # objects # TODO: these are erroring on all platforms, why? - # It could be due to the fact that this file runs with - # `typing.TYPE_CHECKING=False` flag, whereas `pyright` executes - # with flag `typing.TYPE_CHECKING=True`. - # `AsyncIOWrapper` has conditional method stubs, only for type checking, - # and docstring is in real method. - # In other places, private types are imported (or publicly reexported) - # when `not TYPE_CHECKING`. - # This could be why this the whole of this exception handler is like this. - # If one puts `typing.TYPE_CHECKING=True` before trio imports on top - # of this file, there's circular import error. - # - # _io_poll.py and _io_windows.py and _io_kqueue.py: - # ``` - # if TYPE_CHECKING: - # from .._file_io import _HasFileNo - # ``` - # The situation can be dramatically remedied if pyright fixes bug #11803 - # and trio's Path file vague typing is fixed/ignored. Then can - # decommision this file and put plain ``pyright`` check in tox.ini/ci.yaml. + "trio._highlevel_generic.StapledStream.send_stream", + "trio._highlevel_generic.StapledStream.receive_stream", + "trio._ssl.SSLStream.transport_stream", "trio._file_io._HasFileNo", "trio._file_io._HasFileNo.fileno", - # verified that `HasFileNo` has docstrings ): return True @@ -122,6 +116,7 @@ def has_docstring_at_runtime(name: str) -> bool: def check_type( platform: str, full_diagnostics_file: Path | None, + expected_errors: list[object], ) -> list[object]: # convince isort we use the trio import assert trio is not None @@ -166,7 +161,7 @@ def check_type( # Missing docstring messages include the name of the object. # Other errors don't, so we add it. message = f"{name}: {message}" - if message not in printed_diagnostics: + if message not in expected_errors and message not in printed_diagnostics: print(f"new error: {message}", file=sys.stderr) errors.append(message) printed_diagnostics.add(message) @@ -183,25 +178,65 @@ def main(args: argparse.Namespace) -> int: else: full_diagnostics_file = None - has_errors = False + errors_by_platform_file = Path(__file__).parent / "_check_type_completeness.json" + if errors_by_platform_file.exists(): + with open(errors_by_platform_file) as f: + errors_by_platform = json.load(f) + else: + errors_by_platform = {"Linux": [], "Windows": [], "Darwin": [], "all": []} + + changed = False for platform in "Linux", "Windows", "Darwin": + platform_errors = errors_by_platform[platform] + errors_by_platform["all"] print("*" * 20, f"\nChecking {platform}...") - errors = check_type(platform, full_diagnostics_file) + errors = check_type(platform, full_diagnostics_file, platform_errors) + + new_errors = [e for e in errors if e not in platform_errors] + missing_errors = [e for e in platform_errors if e not in errors] - if errors: + if new_errors: print( - "New errors introduced in `pyright --verifytypes`. Fix them.", + f"New errors introduced in `pyright --verifytypes`. Fix them, or ignore them by modifying {errors_by_platform_file}, either manually or with '--overwrite-file'.", file=sys.stderr, ) - has_errors = True + changed = True + if missing_errors: + print( + f"Congratulations, you have resolved existing errors! Please remove them from {errors_by_platform_file}, either manually or with '--overwrite-file'.", + file=sys.stderr, + ) + changed = True + print(missing_errors, file=sys.stderr) + errors_by_platform[platform] = errors print("*" * 20) + # cut down the size of the json file by a lot, and make it easier to parse for + # humans, by moving errors that appear on all platforms to a separate category + errors_by_platform["all"] = [] + for e in errors_by_platform["Linux"].copy(): + if e in errors_by_platform["Darwin"] and e in errors_by_platform["Windows"]: + for platform in "Linux", "Windows", "Darwin": + errors_by_platform[platform].remove(e) + errors_by_platform["all"].append(e) + + if changed and args.overwrite_file: + with open(errors_by_platform_file, "w") as f: + json.dump(errors_by_platform, f, indent=4, sort_keys=True) + # newline at end of file + f.write("\n") + # True -> 1 -> non-zero exit value -> error - return has_errors + return changed parser = argparse.ArgumentParser() +parser.add_argument( + "--overwrite-file", + action="store_true", + default=False, + help="Use this flag to overwrite the current stored results. Either in CI together with a diff check, or to avoid having to manually correct it.", +) parser.add_argument( "--full-diagnostics-file", type=Path, From 5d1104a959e7df9bda142a3022a88983073176ad Mon Sep 17 00:00:00 2001 From: CheViana <656043+CheViana@users.noreply.github.com> Date: Thu, 1 Oct 2026 21:36:28 -0400 Subject: [PATCH 32/34] revert check_type_completeness.py and bring back empty json --- src/trio/_tests/_check_type_completeness.json | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 src/trio/_tests/_check_type_completeness.json diff --git a/src/trio/_tests/_check_type_completeness.json b/src/trio/_tests/_check_type_completeness.json new file mode 100644 index 0000000000..3521950c44 --- /dev/null +++ b/src/trio/_tests/_check_type_completeness.json @@ -0,0 +1,6 @@ +{ + "Darwin": [], + "Linux": [], + "Windows": [], + "all": [] +} \ No newline at end of file From a8851f4d67d4ddb4586adf4de56cc09cb0c0f02d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 2 Oct 2026 01:36:57 +0000 Subject: [PATCH 33/34] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/trio/_tests/_check_type_completeness.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/trio/_tests/_check_type_completeness.json b/src/trio/_tests/_check_type_completeness.json index 3521950c44..ee3668aca4 100644 --- a/src/trio/_tests/_check_type_completeness.json +++ b/src/trio/_tests/_check_type_completeness.json @@ -3,4 +3,4 @@ "Linux": [], "Windows": [], "all": [] -} \ No newline at end of file +} From c307cab6388c634e419fd3b52bbaf05b4aaf8197 Mon Sep 17 00:00:00 2001 From: CheViana <656043+CheViana@users.noreply.github.com> Date: Fri, 2 Oct 2026 09:03:21 -0400 Subject: [PATCH 34/34] polish docs, remove outdated entries --- src/trio/_file_io.py | 4 +--- src/trio/_socket.py | 6 +++--- src/trio/_subprocess.py | 4 +--- src/trio/_sync.py | 8 ++++---- src/trio/_tests/check_type_completeness.py | 14 -------------- src/trio/_unix_pipes.py | 18 +++++++++--------- 6 files changed, 18 insertions(+), 36 deletions(-) diff --git a/src/trio/_file_io.py b/src/trio/_file_io.py index cd341cfc24..a947d9c04f 100644 --- a/src/trio/_file_io.py +++ b/src/trio/_file_io.py @@ -117,9 +117,7 @@ class _HasFileNo(Protocol): """Represents any file-like object that has a file descriptor.""" def fileno(self) -> int: - """ - Integer that corresponds to file descriptor, as in ``sys.stdout.fileno()``. - """ + """Return the file descriptor.""" class _HasIsATTY(Protocol): def isatty(self) -> bool: ... diff --git a/src/trio/_socket.py b/src/trio/_socket.py index 4ba9d2cbb1..36a84c96a5 100644 --- a/src/trio/_socket.py +++ b/src/trio/_socket.py @@ -544,9 +544,9 @@ async def _resolve_address_nocp( class SocketType: - """ - Trio's version of standard library's :class:`socket.socket`. - Encompasses some of Trio-specific platform handling logic. + """Trio's version of the standard library's :class:`socket.socket`. + + Encompasses some Trio-specific platform handling logic. """ def __init__(self) -> None: diff --git a/src/trio/_subprocess.py b/src/trio/_subprocess.py index d4966b8d91..4bb7018e8e 100644 --- a/src/trio/_subprocess.py +++ b/src/trio/_subprocess.py @@ -99,9 +99,7 @@ class HasFileno(Protocol): """Represents any file-like object that has a file descriptor.""" def fileno(self) -> int: - """ - Integer that corresponds to file descriptor. - """ + """Return the file descriptor.""" ... diff --git a/src/trio/_sync.py b/src/trio/_sync.py index cba2958f98..90cdac30f6 100644 --- a/src/trio/_sync.py +++ b/src/trio/_sync.py @@ -123,14 +123,14 @@ class _HasAcquireRelease(Protocol): """Only classes with acquire() and release() can use the mixin's implementations.""" async def acquire(self) -> object: - """ - Method for resource locking. + """Acquire the resource (lock). + See docs for :class:`~trio.Lock` and :class:`~trio.StrictFIFOLock` for usage details. """ def release(self) -> object: - """ - Method for resource releasing. + """Release the resource (unlock). + See docs for :class:`~trio.Lock` and :class:`~trio.StrictFIFOLock` for usage details. """ diff --git a/src/trio/_tests/check_type_completeness.py b/src/trio/_tests/check_type_completeness.py index bbf38080d3..c6911da357 100755 --- a/src/trio/_tests/check_type_completeness.py +++ b/src/trio/_tests/check_type_completeness.py @@ -74,30 +74,16 @@ def has_docstring_at_runtime(name: str) -> bool: # on separate platforms. It might also be a decent idea to work the other way around, # a la test_static_tool_sees_class_members # darwin - "trio.lowlevel.current_kqueue", - "trio.lowlevel.monitor_kevent", - "trio.lowlevel.wait_kevent", "trio._core._io_kqueue._KqueueStatistics", # windows "trio._socket.SocketType.share", "trio._core._io_windows._WindowsStatistics", "trio._core._windows_cffi.Handle", - "trio.lowlevel.current_iocp", - "trio.lowlevel.monitor_completion_key", - "trio.lowlevel.readinto_overlapped", - "trio.lowlevel.register_with_iocp", - "trio.lowlevel.wait_overlapped", - "trio.lowlevel.write_overlapped", - "trio.lowlevel.WaitForSingleObject", - "trio.socket.fromshare", # linux # this test will fail on linux, but I don't develop on linux. So the next # person to do so is very welcome to open a pull request and populate with # objects # TODO: these are erroring on all platforms, why? - "trio._highlevel_generic.StapledStream.send_stream", - "trio._highlevel_generic.StapledStream.receive_stream", - "trio._ssl.SSLStream.transport_stream", "trio._file_io._HasFileNo", "trio._file_io._HasFileNo.fileno", ): diff --git a/src/trio/_unix_pipes.py b/src/trio/_unix_pipes.py index 6c60e069cb..167dd56098 100644 --- a/src/trio/_unix_pipes.py +++ b/src/trio/_unix_pipes.py @@ -187,23 +187,23 @@ async def receive_some(self, max_bytes: int | None = None) -> bytes: return data def close(self) -> None: - """ - Close the stream and close file descriptor it's wrapping. - See class docstring for details. + """Close the stream and close file descriptor it's wrapping. + + See the class docstring for details. """ self._fd_holder.close() async def aclose(self) -> None: - """ - Close (synchronously) the stream and close file descriptor - it's wrapping. See class docstring for details. + """Close the stream synchronously, then execute a checkpoint. + + See the class docstring for details. """ self.close() await trio.lowlevel.checkpoint() def fileno(self) -> int: - """ - Returns integer representing the file descriptor this `FdStream` class - instance is wrapping. See class docstring for details. + """Return the file descriptor this `FdStream` is wrapping. + + See the class docstring for details. """ return self._fd_holder.fd