From 24874814371a46edd70964a67fa97c999729ef2b Mon Sep 17 00:00:00 2001 From: David Leong Date: Tue, 3 Nov 2026 00:41:10 +0000 Subject: [PATCH 1/5] fix: Retry session directory cleanup on Windows while handles are released Signed-off-by: David Leong --- src/openjd/sessions/_tempdir.py | 42 ++++- .../sessions_v0/test_tempdir_hardening.py | 155 ++++++++++++++++++ 2 files changed, 192 insertions(+), 5 deletions(-) diff --git a/src/openjd/sessions/_tempdir.py b/src/openjd/sessions/_tempdir.py index 92d46ce2..1abc89a2 100644 --- a/src/openjd/sessions/_tempdir.py +++ b/src/openjd/sessions/_tempdir.py @@ -3,6 +3,7 @@ import os import stat import sys +import time from ._logging import LoggerAdapter, LogContent, LogExtraInfo from pathlib import Path from shutil import rmtree @@ -32,6 +33,15 @@ mode does not depend on how the embedding application happens to be configured. """ +_CLEANUP_RETRY_DELAYS_SECONDS: tuple[float, ...] = (1.0, 2.0) +"""Seconds to wait before each retry of a failed Windows cleanup. + +Windows refuses to delete a file or directory while any process holds a handle to +it, and a child process that has exited or been terminated can still hold handles +for a short time. POSIX does not retry: a failure there is a permission problem +that waiting will not fix. +""" + def _prepare_temp_dir_root_posix(temp_dir: str) -> None: """Validate the shared temporary root and set its mode, operating on a single @@ -333,10 +343,36 @@ def __init__( def cleanup(self) -> None: """Deletes the temporary directory and all of its contents. + + On Windows a failed deletion is retried after each delay in + ``_CLEANUP_RETRY_DELAYS_SECONDS``, because a process that has just exited, + or been terminated, can hold its file handles for a short time afterwards. + Raises: RuntimeError - If not all files could be deleted. The message names each path that could not be deleted along with the reason. """ + existed = os.path.lexists(self.path) + failures = self._delete_tree() + retry_delays = _CLEANUP_RETRY_DELAYS_SECONDS if is_windows() and existed else () + for delay in retry_delays: + if not failures: + break + time.sleep(delay) + if not os.path.lexists(self.path): + # The pass that reported a failure still removed the directory. + failures = [] + break + failures = self._delete_tree() + + if failures: + raise RuntimeError( + f"Files within temporary directory {str(self.path)} could not be deleted.\n" + + "\n".join(failures) + ) + + def _delete_tree(self) -> list[str]: + """One recursive deletion pass. Returns the paths that could not be deleted, with reasons.""" failures: list[str] = [] def _record(path: Any, error: BaseException) -> None: @@ -367,8 +403,4 @@ def onerror( rmtree(self.path, onerror=onerror) - if failures: - raise RuntimeError( - f"Files within temporary directory {str(self.path)} could not be deleted.\n" - + "\n".join(failures) - ) + return failures diff --git a/test/openjd/sessions_v0/test_tempdir_hardening.py b/test/openjd/sessions_v0/test_tempdir_hardening.py index 950ab5f0..d63a455b 100644 --- a/test/openjd/sessions_v0/test_tempdir_hardening.py +++ b/test/openjd/sessions_v0/test_tempdir_hardening.py @@ -7,7 +7,9 @@ """ import os +import shutil import stat +import threading from contextlib import contextmanager from pathlib import Path from typing import Any, Iterator @@ -17,6 +19,7 @@ from openjd.sessions._os_checker import is_posix, is_windows from openjd.sessions._tempdir import ( + _CLEANUP_RETRY_DELAYS_SECONDS, OPENJD_TEMPDIR_MODE, TempDir, _prepare_temp_dir_root, @@ -293,6 +296,158 @@ def test_successful_cleanup_still_removes_everything(self, tmp_path: Path) -> No assert not d.path.exists() +def _rmtree_failing_first(fail_count: int) -> Any: + """A stand-in for `shutil.rmtree` whose first `fail_count` calls report an error + for the root without deleting anything, then behave like the real one.""" + real_rmtree = shutil.rmtree + calls = {"n": 0} + + def fake(path: Any, *args: Any, **kwargs: Any) -> None: + calls["n"] += 1 + if calls["n"] <= fail_count: + error = PermissionError(32, "The process cannot access the file", str(path)) + if "onexc" in kwargs: + kwargs["onexc"](os.unlink, path, error) + else: + kwargs["onerror"](os.unlink, path, (type(error), error, None)) + return + real_rmtree(path, *args, **kwargs) + + fake.calls = calls # type: ignore[attr-defined] + return fake + + +class TestCleanupRetry: + """A process that has just exited can still hold handles in the session directory, + and Windows refuses to delete a file while a handle to it is open.""" + + def test_windows_retries_until_the_directory_is_deleted(self, tmp_path: Path) -> None: + # GIVEN: a deletion that fails twice, then succeeds + d = TempDir(dir=tmp_path) + (d.path / "a.txt").write_text("x") + fake = _rmtree_failing_first(2) + + # WHEN + with ( + patch("openjd.sessions._tempdir.is_windows", return_value=True), + patch("openjd.sessions._tempdir.rmtree", side_effect=fake), + patch("openjd.sessions._tempdir._CLEANUP_RETRY_DELAYS_SECONDS", (0.0, 0.0)), + ): + d.cleanup() + + # THEN + assert fake.calls["n"] == 3 + assert not d.path.exists() + + def test_windows_raises_once_the_retries_are_exhausted(self, tmp_path: Path) -> None: + # GIVEN: a deletion that never succeeds + d = TempDir(dir=tmp_path) + fake = _rmtree_failing_first(99) + + # WHEN + with ( + patch("openjd.sessions._tempdir.is_windows", return_value=True), + patch("openjd.sessions._tempdir.rmtree", side_effect=fake), + patch("openjd.sessions._tempdir._CLEANUP_RETRY_DELAYS_SECONDS", (0.0, 0.0)), + ): + with pytest.raises(RuntimeError) as excinfo: + d.cleanup() + + # THEN: one attempt plus one per delay, and the last failure is reported + assert fake.calls["n"] == 3 + assert "could not be deleted" in str(excinfo.value) + assert "PermissionError" in str(excinfo.value) + + def test_windows_waits_the_configured_delays(self, tmp_path: Path) -> None: + # GIVEN + d = TempDir(dir=tmp_path) + fake = _rmtree_failing_first(99) + + # WHEN + with ( + patch("openjd.sessions._tempdir.is_windows", return_value=True), + patch("openjd.sessions._tempdir.rmtree", side_effect=fake), + patch("openjd.sessions._tempdir.time.sleep") as sleep, + ): + with pytest.raises(RuntimeError): + d.cleanup() + + # THEN + assert [c.args[0] for c in sleep.call_args_list] == list(_CLEANUP_RETRY_DELAYS_SECONDS) + + def test_posix_does_not_retry(self, tmp_path: Path) -> None: + # GIVEN + d = TempDir(dir=tmp_path) + fake = _rmtree_failing_first(99) + + # WHEN + with ( + patch("openjd.sessions._tempdir.is_windows", return_value=False), + patch("openjd.sessions._tempdir.rmtree", side_effect=fake), + patch("openjd.sessions._tempdir.time.sleep") as sleep, + ): + with pytest.raises(RuntimeError): + d.cleanup() + + # THEN + assert fake.calls["n"] == 1 + sleep.assert_not_called() + + def test_missing_directory_is_still_an_error(self, tmp_path: Path) -> None: + # GIVEN: a directory removed by someone else before cleanup + d = TempDir(dir=tmp_path) + os.rmdir(d.path) + + # WHEN / THEN: unchanged by the retry, on either platform + for windows in (True, False): + d2 = TempDir.__new__(TempDir) + d2.path = d.path + with ( + patch("openjd.sessions._tempdir.is_windows", return_value=windows), + patch("openjd.sessions._tempdir.time.sleep") as sleep, + ): + with pytest.raises(RuntimeError): + d2.cleanup() + sleep.assert_not_called() + + @pytest.mark.skipif(not is_windows(), reason="Windows mandatory file locking") + def test_windows_file_held_open_briefly_is_deleted(self, tmp_path: Path) -> None: + # GIVEN: a file in the session directory held open by another thread for + # less time than the retries cover. Python's open() does not grant + # FILE_SHARE_DELETE, so deleting it fails with a sharing violation. + d = TempDir(dir=tmp_path) + held = open(d.path / "locked.bin", "wb") + release = threading.Timer(0.5, held.close) + release.start() + + # WHEN + try: + d.cleanup() + finally: + release.cancel() + held.close() + + # THEN + assert not d.path.exists() + + @pytest.mark.skipif(not is_windows(), reason="Windows mandatory file locking") + def test_windows_file_held_open_throughout_is_reported(self, tmp_path: Path) -> None: + # GIVEN + d = TempDir(dir=tmp_path) + held = open(d.path / "locked.bin", "wb") + + # WHEN + try: + with patch("openjd.sessions._tempdir._CLEANUP_RETRY_DELAYS_SECONDS", (0.1, 0.1)): + with pytest.raises(RuntimeError) as excinfo: + d.cleanup() + finally: + held.close() + + # THEN + assert "locked.bin" in str(excinfo.value) + + @pytest.mark.skipif(not is_posix(), reason="symlink swap and fchmod semantics are POSIX here") class TestTempRootCheckThenUse: """The first R5-3 implementation validated with `lstat(path)` and then called From e82c7d025466ac634da31113c1ed63ed0e5c75d7 Mon Sep 17 00:00:00 2001 From: David Leong Date: Wed, 4 Nov 2026 02:24:54 +0000 Subject: [PATCH 2/5] fix: Retry Windows cleanup up to 16 s and log which processes hold files open Signed-off-by: David Leong --- src/openjd/sessions/_session.py | 2 +- src/openjd/sessions/_tempdir.py | 105 +++++++++-- src/openjd/sessions/_win32/_file_locks.py | 125 ++++++++++++++ .../sessions_v0/test_tempdir_hardening.py | 163 +++++++++++++++++- 4 files changed, 380 insertions(+), 15 deletions(-) create mode 100644 src/openjd/sessions/_win32/_file_locks.py diff --git a/src/openjd/sessions/_session.py b/src/openjd/sessions/_session.py index 71bcba70..f0a23d36 100644 --- a/src/openjd/sessions/_session.py +++ b/src/openjd/sessions/_session.py @@ -603,7 +603,7 @@ def cleanup(self) -> None: # Note: Blocking call until the process has exited _subprocess.run() - self._working_dir.cleanup() + self._working_dir.cleanup(logger=self._logger) except RuntimeError as exc: # Warn if we couldn't cleanup the temporary files for some reason. self._logger.exception( diff --git a/src/openjd/sessions/_tempdir.py b/src/openjd/sessions/_tempdir.py index 1abc89a2..87e22df9 100644 --- a/src/openjd/sessions/_tempdir.py +++ b/src/openjd/sessions/_tempdir.py @@ -4,7 +4,7 @@ import stat import sys import time -from ._logging import LoggerAdapter, LogContent, LogExtraInfo +from ._logging import LOG, LoggerAdapter, LogContent, LogExtraInfo from pathlib import Path from shutil import rmtree from ._embedded_files import chown_group @@ -33,15 +33,65 @@ mode does not depend on how the embedding application happens to be configured. """ -_CLEANUP_RETRY_DELAYS_SECONDS: tuple[float, ...] = (1.0, 2.0) -"""Seconds to wait before each retry of a failed Windows cleanup. +_CLEANUP_RETRY_DELAYS_SECONDS: tuple[float, ...] = (1.0, 2.0, 4.0, 8.0, 16.0) +"""Seconds to wait before each retry of a failed Windows cleanup: about 31 s in total. Windows refuses to delete a file or directory while any process holds a handle to it, and a child process that has exited or been terminated can still hold handles -for a short time. POSIX does not retry: a failure there is a permission problem -that waiting will not fix. +for a while. POSIX does not retry: a failure there is a permission problem that +waiting will not fix. """ +_MAX_FILES_TO_INSPECT = 256 +"""Most files a failed cleanup inspects to find what is still holding them.""" + +_MAX_FILES_TO_NAME = 3 +"""Most undeletable file names a failed-attempt log line lists.""" + + +def _find_lock_holders(files: list[str]) -> tuple[list[tuple[int, str]], Optional[str]]: + """The (pid, name) of each process holding any of ``files`` open, and the + reason the lookup failed if it did. Windows only.""" + from ._win32._file_locks import processes_holding # type: ignore + + try: + return [(h.pid, h.name) for h in processes_holding(files)], None + except OSError as err: + return [], str(err) + + +def _describe_failed_attempt( + dir: Path, + error: BaseException, + files: list[str], + holders: list[tuple[int, str]], + holders_error: Optional[str], + retry_in: Optional[float], +) -> str: + """The text of a failed-attempt log line. ``files`` are those that could not be deleted.""" + reason = getattr(error, "strerror", None) or str(error) + msg = f"Could not delete temporary directory {dir}: {reason}." + if files: + names = [] + for p in files[:_MAX_FILES_TO_NAME]: + try: + names.append(str(Path(p).relative_to(dir))) + except ValueError: + names.append(str(p)) + msg += f" {len(files)} file(s) could not be deleted, including {', '.join(names)}." + if holders: + seen: dict[int, str] = {} + for pid, name in holders: + seen.setdefault(pid, name) + listed = ", ".join(f"{name} (pid {pid})" for pid, name in seen.items()) + msg += f" Files are still locked open by: {listed}." + elif holders_error is not None: + msg += f" The processes holding them open could not be determined: {holders_error}." + else: + msg += " No running process was found holding them open." + msg += f" Retrying in {retry_in:g} s." if retry_in is not None else " Giving up." + return msg + def _prepare_temp_dir_root_posix(temp_dir: str) -> None: """Validate the shared temporary root and set its mode, operating on a single @@ -341,12 +391,17 @@ def __init__( rmtree(self.path, ignore_errors=True) raise - def cleanup(self) -> None: + def cleanup(self, logger: Optional[LoggerAdapter] = None) -> None: """Deletes the temporary directory and all of its contents. On Windows a failed deletion is retried after each delay in ``_CLEANUP_RETRY_DELAYS_SECONDS``, because a process that has just exited, - or been terminated, can hold its file handles for a short time afterwards. + or been terminated, can hold its file handles for a while afterwards. Each + failed attempt is logged as a warning naming the files that could not be + deleted and the processes holding them open. + + Args: + logger: Where to log failed attempts. Defaults to the module logger. Raises: RuntimeError - If not all files could be deleted. The message names @@ -354,10 +409,12 @@ def cleanup(self) -> None: """ existed = os.path.lexists(self.path) failures = self._delete_tree() - retry_delays = _CLEANUP_RETRY_DELAYS_SECONDS if is_windows() and existed else () + retry_windows = is_windows() and existed + retry_delays = _CLEANUP_RETRY_DELAYS_SECONDS if retry_windows else () for delay in retry_delays: if not failures: break + self._log_failed_attempt(failures, delay, logger) time.sleep(delay) if not os.path.lexists(self.path): # The pass that reported a failure still removed the directory. @@ -366,14 +423,36 @@ def cleanup(self) -> None: failures = self._delete_tree() if failures: + if retry_windows: + self._log_failed_attempt(failures, None, logger) raise RuntimeError( f"Files within temporary directory {str(self.path)} could not be deleted.\n" - + "\n".join(failures) + + "\n".join(f"{path}: {error!r}" for path, error in failures) ) - def _delete_tree(self) -> list[str]: - """One recursive deletion pass. Returns the paths that could not be deleted, with reasons.""" - failures: list[str] = [] + def _log_failed_attempt( + self, + failures: list[tuple[str, BaseException]], + retry_in: Optional[float], + logger: Optional[LoggerAdapter], + ) -> None: + files = [p for p, _ in failures if os.path.isfile(p)][:_MAX_FILES_TO_INSPECT] + holders, holders_error = _find_lock_holders(files) if files else ([], None) + message = _describe_failed_attempt( + self.path, failures[0][1], files, holders, holders_error, retry_in + ) + (logger if logger is not None else LOG).warning( + message, + extra=LogExtraInfo( + openjd_log_content=LogContent.FILE_PATH + | LogContent.PROCESS_CONTROL + | LogContent.EXCEPTION_INFO + ), + ) + + def _delete_tree(self) -> list[tuple[str, BaseException]]: + """One recursive deletion pass. Returns each path that could not be deleted, with the error.""" + failures: list[tuple[str, BaseException]] = [] def _record(path: Any, error: BaseException) -> None: # R5-7 fix: keep the reason. Cleanup is exactly where the difference @@ -381,7 +460,7 @@ def _record(path: Any, error: BaseException) -> None: # decides what the operator should do next, and the old handler # accepted the exception and discarded it -- leaving a list of bare # paths with no cause. - failures.append(f"{path}: {error!r}") + failures.append((str(path), error)) if sys.version_info >= (3, 12): # `onerror` is deprecated from 3.12 and slated for removal; `onexc` diff --git a/src/openjd/sessions/_win32/_file_locks.py b/src/openjd/sessions/_win32/_file_locks.py new file mode 100644 index 00000000..50217d34 --- /dev/null +++ b/src/openjd/sessions/_win32/_file_locks.py @@ -0,0 +1,125 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""Find the processes holding files open, via the Windows Restart Manager. + +Used only to explain a failed cleanup in the log, so every failure here is +raised as OSError for the caller to report, and never stops the cleanup. +""" + +import ctypes +import sys +from ctypes import POINTER, Structure, byref +from ctypes.wintypes import BOOL, DWORD, FILETIME, LPCWSTR, LPDWORD, LPWSTR, UINT, ULONG, WCHAR +from typing import NamedTuple, Sequence + +import psutil + +# This assertion short-circuits mypy from type checking this module on platforms other than Windows +# https://mypy.readthedocs.io/en/stable/common_issues.html#python-version-and-system-platform-checks +assert sys.platform == "win32" + +CCH_RM_SESSION_KEY = 32 +CCH_RM_MAX_APP_NAME = 255 +CCH_RM_MAX_SVC_NAME = 63 +ERROR_SUCCESS = 0 +ERROR_MORE_DATA = 234 + + +class RM_UNIQUE_PROCESS(Structure): + _fields_ = [("dwProcessId", DWORD), ("ProcessStartTime", FILETIME)] + + +class RM_PROCESS_INFO(Structure): + _fields_ = [ + ("Process", RM_UNIQUE_PROCESS), + ("strAppName", WCHAR * (CCH_RM_MAX_APP_NAME + 1)), + ("strServiceShortName", WCHAR * (CCH_RM_MAX_SVC_NAME + 1)), + ("ApplicationType", ctypes.c_int), + ("AppStatus", ULONG), + ("TSSessionId", DWORD), + ("bRestartable", BOOL), + ] + + +rstrtmgr = ctypes.WinDLL("rstrtmgr") + +# https://learn.microsoft.com/en-us/windows/win32/api/restartmanager/nf-restartmanager-rmstartsession +rstrtmgr.RmStartSession.restype = DWORD +rstrtmgr.RmStartSession.argtypes = [LPDWORD, DWORD, LPWSTR] +# https://learn.microsoft.com/en-us/windows/win32/api/restartmanager/nf-restartmanager-rmregisterresources +rstrtmgr.RmRegisterResources.restype = DWORD +rstrtmgr.RmRegisterResources.argtypes = [ + DWORD, + UINT, + POINTER(LPCWSTR), + UINT, + POINTER(RM_UNIQUE_PROCESS), + UINT, + POINTER(LPCWSTR), +] +# https://learn.microsoft.com/en-us/windows/win32/api/restartmanager/nf-restartmanager-rmgetlist +rstrtmgr.RmGetList.restype = DWORD +rstrtmgr.RmGetList.argtypes = [ + DWORD, + POINTER(UINT), + POINTER(UINT), + POINTER(RM_PROCESS_INFO), + LPDWORD, +] +# https://learn.microsoft.com/en-us/windows/win32/api/restartmanager/nf-restartmanager-rmendsession +rstrtmgr.RmEndSession.restype = DWORD +rstrtmgr.RmEndSession.argtypes = [DWORD] + + +class FileHolder(NamedTuple): + pid: int + name: str + + +def processes_holding(files: Sequence[str]) -> list[FileHolder]: + """Returns the processes that hold any of ``files`` open. + + Raises: + OSError: if the Restart Manager could not be queried. + """ + if not files: + return [] + handle = DWORD() + key = ctypes.create_unicode_buffer(CCH_RM_SESSION_KEY + 1) + rc = rstrtmgr.RmStartSession(byref(handle), 0, key) + if rc != ERROR_SUCCESS: + raise OSError(f"RmStartSession failed ({rc})") + try: + return _list_holders(handle, files) + finally: + rstrtmgr.RmEndSession(handle) + + +def _list_holders(handle: DWORD, files: Sequence[str]) -> list[FileHolder]: + names = (LPCWSTR * len(files))(*files) + rc = rstrtmgr.RmRegisterResources(handle, len(files), names, 0, None, 0, None) + if rc != ERROR_SUCCESS: + raise OSError(f"RmRegisterResources failed ({rc})") + infos = (RM_PROCESS_INFO * 0)() + # The list can grow between calls, so ask again a few times on ERROR_MORE_DATA. + for _ in range(3): + needed = UINT(0) + count = UINT(len(infos)) + reasons = DWORD(0) + rc = rstrtmgr.RmGetList(handle, byref(needed), byref(count), infos, byref(reasons)) + if rc == ERROR_SUCCESS: + return [_holder_from(infos[i]) for i in range(count.value)] + if rc != ERROR_MORE_DATA: + raise OSError(f"RmGetList failed ({rc})") + infos = (RM_PROCESS_INFO * needed.value)() + raise OSError("RmGetList kept reporting more processes") + + +def _holder_from(info: RM_PROCESS_INFO) -> FileHolder: + pid = int(info.Process.dwProcessId) + try: + # The executable's file name, such as UnrealShaderCompileWorker.exe. + name = psutil.Process(pid).name() + except psutil.Error: + name = info.strAppName + return FileHolder(pid=pid, name=name) diff --git a/test/openjd/sessions_v0/test_tempdir_hardening.py b/test/openjd/sessions_v0/test_tempdir_hardening.py index d63a455b..1699a15f 100644 --- a/test/openjd/sessions_v0/test_tempdir_hardening.py +++ b/test/openjd/sessions_v0/test_tempdir_hardening.py @@ -13,10 +13,11 @@ from contextlib import contextmanager from pathlib import Path from typing import Any, Iterator -from unittest.mock import patch +from unittest.mock import MagicMock, patch import pytest +from openjd.sessions._logging import LogContent from openjd.sessions._os_checker import is_posix, is_windows from openjd.sessions._tempdir import ( _CLEANUP_RETRY_DELAYS_SECONDS, @@ -273,6 +274,7 @@ def boom(path: Any, *a: Any, **k: Any) -> None: with ( patch("openjd.sessions._tempdir.os.unlink", side_effect=boom), patch("openjd.sessions._tempdir.os.remove", side_effect=boom, create=True), + patch("openjd.sessions._tempdir._CLEANUP_RETRY_DELAYS_SECONDS", ()), ): with pytest.raises(RuntimeError) as excinfo: d.cleanup() @@ -447,6 +449,165 @@ def test_windows_file_held_open_throughout_is_reported(self, tmp_path: Path) -> # THEN assert "locked.bin" in str(excinfo.value) + @pytest.mark.skipif(not is_windows(), reason="Windows mandatory file locking") + def test_windows_failed_attempt_names_this_process_as_the_holder(self, tmp_path: Path) -> None: + # GIVEN: a file this process holds open for the whole cleanup + d = TempDir(dir=tmp_path) + held = open(d.path / "locked.bin", "wb") + logger = MagicMock() + + # WHEN + try: + with patch("openjd.sessions._tempdir._CLEANUP_RETRY_DELAYS_SECONDS", (0.1,)): + with pytest.raises(RuntimeError): + d.cleanup(logger=logger) + finally: + held.close() + + # THEN: one warning per failed attempt, each naming the file and this process + messages = [c.args[0] for c in logger.warning.call_args_list] + assert len(messages) == 2 + for message in messages: + assert "locked.bin" in message + assert "locked open by: " in message + assert f"(pid {os.getpid()})" in message + assert messages[0].endswith("Retrying in 0.1 s.") + assert messages[1].endswith("Giving up.") + + +def _rmtree_failing_on(path: Path, fail_count: int) -> Any: + """Like `_rmtree_failing_first`, but reports `path` (a real file) as undeletable.""" + real_rmtree = shutil.rmtree + calls = {"n": 0} + + def fake(root: Any, *args: Any, **kwargs: Any) -> None: + calls["n"] += 1 + if calls["n"] <= fail_count: + error = PermissionError(32, "The process cannot access the file", str(path)) + if "onexc" in kwargs: + kwargs["onexc"](os.unlink, str(path), error) + else: + kwargs["onerror"](os.unlink, str(path), (type(error), error, None)) + return + real_rmtree(root, *args, **kwargs) + + return fake + + +class TestCleanupLockLogging: + """Each failed Windows attempt is logged so an operator can see which files are + still locked and which process holds them.""" + + def test_retry_delays_double_from_one_second(self) -> None: + assert _CLEANUP_RETRY_DELAYS_SECONDS == (1.0, 2.0, 4.0, 8.0, 16.0) + + def test_each_failed_attempt_is_logged_with_files_holders_and_next_delay( + self, tmp_path: Path + ) -> None: + # GIVEN: a file that cannot be deleted for two passes, held by two processes + d = TempDir(dir=tmp_path) + locked = d.path / "sub" / "locked.bin" + locked.parent.mkdir() + locked.write_text("x") + holders = [(42, "UnrealShaderCompileWorker.exe"), (42, "dup"), (7, "p4.exe")] + logger = MagicMock() + + # WHEN + with ( + patch("openjd.sessions._tempdir.is_windows", return_value=True), + patch("openjd.sessions._tempdir.rmtree", side_effect=_rmtree_failing_on(locked, 2)), + patch("openjd.sessions._tempdir._CLEANUP_RETRY_DELAYS_SECONDS", (0.0, 4.0, 8.0)), + patch("openjd.sessions._tempdir.time.sleep"), + patch( + "openjd.sessions._tempdir._find_lock_holders", return_value=(holders, None) + ) as find, + ): + d.cleanup(logger=logger) + + # THEN: two warnings, then success; the lookup was given the locked file + assert not d.path.exists() + find.assert_called_with([str(locked)]) + messages = [c.args[0] for c in logger.warning.call_args_list] + sub_locked = str(Path("sub") / "locked.bin") + assert messages == [ + f"Could not delete temporary directory {d.path}: The process cannot access the file." + f" 1 file(s) could not be deleted, including {sub_locked}. Files are still locked" + f" open by: UnrealShaderCompileWorker.exe (pid 42), p4.exe (pid 7). Retrying in {r} s." + for r in ("0", "4") + ] + extra = logger.warning.call_args.kwargs["extra"] + assert LogContent.PROCESS_CONTROL in extra["openjd_log_content"] + + def test_last_failed_attempt_says_giving_up_and_reports_lookup_errors( + self, tmp_path: Path + ) -> None: + # GIVEN: a file that can never be deleted, and a lookup that fails + d = TempDir(dir=tmp_path) + locked = d.path / "locked.bin" + locked.write_text("x") + logger = MagicMock() + + # WHEN + with ( + patch("openjd.sessions._tempdir.is_windows", return_value=True), + patch("openjd.sessions._tempdir.rmtree", side_effect=_rmtree_failing_on(locked, 99)), + patch("openjd.sessions._tempdir._CLEANUP_RETRY_DELAYS_SECONDS", (0.0,)), + patch( + "openjd.sessions._tempdir._find_lock_holders", + return_value=([], "RmGetList failed (5)"), + ), + ): + with pytest.raises(RuntimeError): + d.cleanup(logger=logger) + + # THEN + messages = [c.args[0] for c in logger.warning.call_args_list] + assert len(messages) == 2 + assert "could not be determined: RmGetList failed (5)." in messages[1] + assert messages[1].endswith("Giving up.") + + def test_no_holder_lookup_when_only_directories_failed(self, tmp_path: Path) -> None: + # GIVEN: a deletion failing on the directory itself, not a file + d = TempDir(dir=tmp_path) + logger = MagicMock() + + # WHEN + with ( + patch("openjd.sessions._tempdir.is_windows", return_value=True), + patch("openjd.sessions._tempdir.rmtree", side_effect=_rmtree_failing_first(1)), + patch("openjd.sessions._tempdir._CLEANUP_RETRY_DELAYS_SECONDS", (0.0,)), + patch("openjd.sessions._tempdir._find_lock_holders") as find, + ): + d.cleanup(logger=logger) + + # THEN + find.assert_not_called() + (message,) = [c.args[0] for c in logger.warning.call_args_list] + assert "locked" not in message + assert message.endswith("Retrying in 0 s.") + + def test_posix_logs_nothing_extra(self, tmp_path: Path) -> None: + d = TempDir(dir=tmp_path) + logger = MagicMock() + with ( + patch("openjd.sessions._tempdir.is_windows", return_value=False), + patch("openjd.sessions._tempdir.rmtree", side_effect=_rmtree_failing_first(99)), + ): + with pytest.raises(RuntimeError): + d.cleanup(logger=logger) + logger.warning.assert_not_called() + + def test_defaults_to_the_module_logger(self, tmp_path: Path) -> None: + d = TempDir(dir=tmp_path) + with ( + patch("openjd.sessions._tempdir.is_windows", return_value=True), + patch("openjd.sessions._tempdir.rmtree", side_effect=_rmtree_failing_first(1)), + patch("openjd.sessions._tempdir._CLEANUP_RETRY_DELAYS_SECONDS", (0.0,)), + patch("openjd.sessions._tempdir.LOG") as log, + ): + d.cleanup() + log.warning.assert_called_once() + @pytest.mark.skipif(not is_posix(), reason="symlink swap and fchmod semantics are POSIX here") class TestTempRootCheckThenUse: From efb2a739f0e676d8188aba4daf952893c955fdb2 Mon Sep 17 00:00:00 2001 From: David Leong Date: Wed, 4 Nov 2026 03:41:34 +0000 Subject: [PATCH 3/5] fix: Never let the lock-holder lookup abort session cleanup Signed-off-by: David Leong --- src/openjd/sessions/_tempdir.py | 12 ++++++---- .../sessions_v0/test_tempdir_hardening.py | 23 +++++++++++++++++++ 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/src/openjd/sessions/_tempdir.py b/src/openjd/sessions/_tempdir.py index 87e22df9..a28a9b41 100644 --- a/src/openjd/sessions/_tempdir.py +++ b/src/openjd/sessions/_tempdir.py @@ -51,13 +51,17 @@ def _find_lock_holders(files: list[str]) -> tuple[list[tuple[int, str]], Optional[str]]: """The (pid, name) of each process holding any of ``files`` open, and the - reason the lookup failed if it did. Windows only.""" - from ._win32._file_locks import processes_holding # type: ignore + reason the lookup failed if it did. Windows only. + Never raises: the result only feeds a log line, and an exception here would + abort the cleanup it is explaining. + """ try: + from ._win32._file_locks import processes_holding # type: ignore + return [(h.pid, h.name) for h in processes_holding(files)], None - except OSError as err: - return [], str(err) + except Exception as err: + return [], f"{type(err).__name__}: {err}" def _describe_failed_attempt( diff --git a/test/openjd/sessions_v0/test_tempdir_hardening.py b/test/openjd/sessions_v0/test_tempdir_hardening.py index 1699a15f..8b7e2ba7 100644 --- a/test/openjd/sessions_v0/test_tempdir_hardening.py +++ b/test/openjd/sessions_v0/test_tempdir_hardening.py @@ -9,6 +9,7 @@ import os import shutil import stat +import sys import threading from contextlib import contextmanager from pathlib import Path @@ -586,6 +587,28 @@ def test_no_holder_lookup_when_only_directories_failed(self, tmp_path: Path) -> assert "locked" not in message assert message.endswith("Retrying in 0 s.") + def test_a_lookup_that_cannot_even_import_does_not_abort_cleanup(self, tmp_path: Path) -> None: + # GIVEN: the Restart Manager module fails to import, as when rstrtmgr.dll + # cannot be loaded, and a file that is undeletable for one pass + d = TempDir(dir=tmp_path) + locked = d.path / "locked.bin" + locked.write_text("x") + logger = MagicMock() + + # WHEN + with ( + patch.dict(sys.modules, {"openjd.sessions._win32._file_locks": None}), + patch("openjd.sessions._tempdir.is_windows", return_value=True), + patch("openjd.sessions._tempdir.rmtree", side_effect=_rmtree_failing_on(locked, 1)), + patch("openjd.sessions._tempdir._CLEANUP_RETRY_DELAYS_SECONDS", (0.0,)), + ): + d.cleanup(logger=logger) + + # THEN: the retry still ran, and the warning says why holders are unknown + assert not d.path.exists() + (message,) = [c.args[0] for c in logger.warning.call_args_list] + assert "could not be determined: ModuleNotFoundError" in message + def test_posix_logs_nothing_extra(self, tmp_path: Path) -> None: d = TempDir(dir=tmp_path) logger = MagicMock() From 4976cda649c415f11390143442afaf8ba1aa53e9 Mon Sep 17 00:00:00 2001 From: David Leong Date: Thu, 5 Nov 2026 23:32:41 +0000 Subject: [PATCH 4/5] refactor: Import ctypes names one way in the Restart Manager module Signed-off-by: David Leong --- src/openjd/sessions/_win32/_file_locks.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/openjd/sessions/_win32/_file_locks.py b/src/openjd/sessions/_win32/_file_locks.py index 50217d34..ffa7147c 100644 --- a/src/openjd/sessions/_win32/_file_locks.py +++ b/src/openjd/sessions/_win32/_file_locks.py @@ -6,9 +6,8 @@ raised as OSError for the caller to report, and never stops the cleanup. """ -import ctypes import sys -from ctypes import POINTER, Structure, byref +from ctypes import POINTER, Structure, WinDLL, byref, c_int, create_unicode_buffer from ctypes.wintypes import BOOL, DWORD, FILETIME, LPCWSTR, LPDWORD, LPWSTR, UINT, ULONG, WCHAR from typing import NamedTuple, Sequence @@ -34,14 +33,14 @@ class RM_PROCESS_INFO(Structure): ("Process", RM_UNIQUE_PROCESS), ("strAppName", WCHAR * (CCH_RM_MAX_APP_NAME + 1)), ("strServiceShortName", WCHAR * (CCH_RM_MAX_SVC_NAME + 1)), - ("ApplicationType", ctypes.c_int), + ("ApplicationType", c_int), ("AppStatus", ULONG), ("TSSessionId", DWORD), ("bRestartable", BOOL), ] -rstrtmgr = ctypes.WinDLL("rstrtmgr") +rstrtmgr = WinDLL("rstrtmgr") # https://learn.microsoft.com/en-us/windows/win32/api/restartmanager/nf-restartmanager-rmstartsession rstrtmgr.RmStartSession.restype = DWORD @@ -85,7 +84,7 @@ def processes_holding(files: Sequence[str]) -> list[FileHolder]: if not files: return [] handle = DWORD() - key = ctypes.create_unicode_buffer(CCH_RM_SESSION_KEY + 1) + key = create_unicode_buffer(CCH_RM_SESSION_KEY + 1) rc = rstrtmgr.RmStartSession(byref(handle), 0, key) if rc != ERROR_SUCCESS: raise OSError(f"RmStartSession failed ({rc})") From 45a4f724053c32abe6388fe7e0925e25657a2bb3 Mon Sep 17 00:00:00 2001 From: David Leong Date: Thu, 5 Nov 2026 23:33:36 +0000 Subject: [PATCH 5/5] fix: Mark the Windows-only ctypes import for mypy Signed-off-by: David Leong --- src/openjd/sessions/_win32/_file_locks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/openjd/sessions/_win32/_file_locks.py b/src/openjd/sessions/_win32/_file_locks.py index ffa7147c..0bd51a80 100644 --- a/src/openjd/sessions/_win32/_file_locks.py +++ b/src/openjd/sessions/_win32/_file_locks.py @@ -7,7 +7,7 @@ """ import sys -from ctypes import POINTER, Structure, WinDLL, byref, c_int, create_unicode_buffer +from ctypes import POINTER, Structure, WinDLL, byref, c_int, create_unicode_buffer # type: ignore from ctypes.wintypes import BOOL, DWORD, FILETIME, LPCWSTR, LPDWORD, LPWSTR, UINT, ULONG, WCHAR from typing import NamedTuple, Sequence