Skip to content

Commit 4c91ded

Browse files
authored
test(protocols): bound the #1428 deadline test so a lost alarm cannot hang the worker (#1889) (#1891)
1 parent aa05d96 commit 4c91ded

2 files changed

Lines changed: 341 additions & 54 deletions

File tree

‎examples/generators/options.py‎

Lines changed: 90 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,12 @@
102102
Python and holds the GIL for the whole of an iteration, so no watchdog thread
103103
would ever get to run. The deadline is wall-clock time, so a stalled runner
104104
can trip it on a case that normally takes milliseconds (#1428):
105-
:data:`TIMEOUT_SCALE_ENV` stretches it without disarming it.
105+
:data:`TIMEOUT_SCALE_ENV` stretches it without disarming it. It also repeats
106+
every second until :func:`roundtrip` has the :exc:`TimeoutError`, because one
107+
raise can be lost: a finaliser the cyclic collector runs reports it to
108+
:func:`sys.unraisablehook` and drops it, and an ``except OSError`` drops it too,
109+
:exc:`TimeoutError` being one. A single alarm then left the case running for good
110+
(#1889).
106111
107112
"""
108113

@@ -1744,52 +1749,70 @@ def roundtrip(case: 'Case', deadline: 'int' = DEADLINE) -> 'Outcome':
17441749
octets = b''
17451750
again = b''
17461751

1747-
with _deadline(deadline), warnings.catch_warnings(record=True) as caught:
1752+
# The deadline is entered last, so it is torn down first: nothing of
1753+
# catch_warnings runs on the way out while it can still fire. The body
1754+
# settles it on every way out before :meth:`_deadline.__exit__` is entered
1755+
# (#1889): the ``finally`` covers finishing and a :exc:`BaseException`, and
1756+
# each ``except`` clause settles it first too, because a clause builds its
1757+
# outcome before the ``finally`` runs. A repeat firing after that, at
1758+
# ``__exit__``'s entry included, is a no-op. Attribute stores rather than a
1759+
# method call, which would give a repeat already due one more place to land.
1760+
with warnings.catch_warnings(record=True) as caught, _deadline(deadline) as limit:
17481761
warnings.simplefilter('always')
1749-
1750-
def seen() -> 'tuple[str, ...]':
1751-
"""Warning messages so far, deduplicated but in order.
1752-
1753-
:exc:`ResourceWarning` is dropped. It is raised by the garbage
1754-
collector rather than by the code under test, so whichever case
1755-
happens to be running when a file object from somewhere else is
1756-
finalised would otherwise be blamed for it -- which is exactly what
1757-
happened: a ``.pcapng`` handle left open by a sibling generator was
1758-
reported against ``hopopt-option/Router_Alert``.
1759-
"""
1760-
return tuple(dict.fromkeys(
1761-
str(entry.message) for entry in caught
1762-
if not issubclass(entry.category, ResourceWarning)))
1763-
1764-
try:
1765-
octets = bytes(family.build(case.code, case.kwargs))
1766-
except TimeoutError as exc:
1767-
return Outcome(case, 'TIMEOUT', str(exc), None, seen())
1768-
except Exception as exc: # pylint: disable=broad-except
1769-
# Broad on purpose: the point is to find out *what* a constructor
1770-
# does when it is wrong, and narrowing this to ProtocolError would
1771-
# let an AttributeError or a KeyError -- which is what most of the
1772-
# current defects raise -- abort the sweep rather than be recorded
1773-
# as this case's result.
1774-
return Outcome(case, 'CONSTRUCT', f'{type(exc).__name__}: {exc}', None, seen())
1775-
17761762
try:
1777-
parsed = family.extract(family.parse(octets))
1778-
except TimeoutError as exc:
1779-
return Outcome(case, 'TIMEOUT', str(exc), octets, seen())
1780-
except Exception as exc: # pylint: disable=broad-except
1781-
return Outcome(case, 'PARSE', f'{type(exc).__name__}: {exc}', octets, seen())
1782-
1783-
try:
1784-
again = bytes(family.rebuild(parsed))
1785-
except TimeoutError as exc:
1786-
return Outcome(case, 'TIMEOUT', str(exc), octets, seen())
1787-
except Exception as exc: # pylint: disable=broad-except
1788-
return Outcome(case, 'RECONSTRUCT', f'{type(exc).__name__}: {exc}',
1789-
octets, seen())
1790-
1791-
warned = seen()
1792-
1763+
def seen() -> 'tuple[str, ...]':
1764+
"""Warning messages so far, deduplicated but in order.
1765+
1766+
:exc:`ResourceWarning` is dropped. It is raised by the garbage
1767+
collector rather than by the code under test, so whichever case
1768+
happens to be running when a file object from somewhere else is
1769+
finalised would otherwise be blamed for it -- which is exactly
1770+
what happened: a ``.pcapng`` handle left open by a sibling
1771+
generator was reported against ``hopopt-option/Router_Alert``.
1772+
"""
1773+
return tuple(dict.fromkeys(
1774+
str(entry.message) for entry in caught
1775+
if not issubclass(entry.category, ResourceWarning)))
1776+
1777+
try:
1778+
octets = bytes(family.build(case.code, case.kwargs))
1779+
except TimeoutError as exc:
1780+
limit.settled = True
1781+
return Outcome(case, 'TIMEOUT', str(exc), None, seen())
1782+
except Exception as exc: # pylint: disable=broad-except
1783+
limit.settled = True
1784+
# Broad on purpose: the point is to find out *what* a
1785+
# constructor does when it is wrong, and narrowing this to
1786+
# ProtocolError would let an AttributeError or a KeyError --
1787+
# which is what most of the current defects raise -- abort the
1788+
# sweep rather than be recorded as this case's result.
1789+
return Outcome(case, 'CONSTRUCT', f'{type(exc).__name__}: {exc}', None,
1790+
seen())
1791+
1792+
try:
1793+
parsed = family.extract(family.parse(octets))
1794+
except TimeoutError as exc:
1795+
limit.settled = True
1796+
return Outcome(case, 'TIMEOUT', str(exc), octets, seen())
1797+
except Exception as exc: # pylint: disable=broad-except
1798+
limit.settled = True
1799+
return Outcome(case, 'PARSE', f'{type(exc).__name__}: {exc}', octets,
1800+
seen())
1801+
1802+
try:
1803+
again = bytes(family.rebuild(parsed))
1804+
except TimeoutError as exc:
1805+
limit.settled = True
1806+
return Outcome(case, 'TIMEOUT', str(exc), octets, seen())
1807+
except Exception as exc: # pylint: disable=broad-except
1808+
limit.settled = True
1809+
return Outcome(case, 'RECONSTRUCT', f'{type(exc).__name__}: {exc}',
1810+
octets, seen())
1811+
finally:
1812+
limit.settled = True
1813+
1814+
# Read after the deadline is gone; ``caught`` outlives the ``with``.
1815+
warned = seen()
17931816
if octets != again:
17941817
return Outcome(case, 'MISMATCH', f'{octets.hex()} != {again.hex()}',
17951818
octets, warned)
@@ -1842,6 +1865,13 @@ class _deadline: # pylint: disable=invalid-name
18421865
:exc:`TimeoutError` names the stretched value -- the seconds actually
18431866
allowed.
18441867
1868+
Each raise re-arms the alarm for one more second, so the raise repeats until
1869+
:attr:`settled` is set (#1889). One raise is not enough: one that lands in a
1870+
finaliser, or in an ``except OSError``, never reaches the body, and a single
1871+
alarm then left a hung case hung. The body must set :attr:`settled` before
1872+
it leaves: :meth:`__exit__` sets it too, but a firing handled as
1873+
:meth:`__exit__` is entered runs before it can.
1874+
18451875
Written as a class rather than a :func:`contextlib.contextmanager` so that
18461876
it can be entered alongside :func:`warnings.catch_warnings` in one ``with``
18471877
statement without the generator machinery swallowing the alarm.
@@ -1862,6 +1892,10 @@ def __init__(self, seconds: 'int') -> 'None':
18621892
self.armed = False
18631893
#: Handler displaced by :meth:`__enter__`, restored by :meth:`__exit__`.
18641894
self.previous = None # type: Any
1895+
#: Set by the body before it leaves, and by :meth:`__exit__` as a
1896+
#: backstop: a firing after that does nothing, so it cannot interrupt
1897+
#: the way out.
1898+
self.settled = False
18651899

18661900
def __enter__(self) -> '_deadline':
18671901
if self.seconds <= 0 or not hasattr(signal, 'SIGALRM'):
@@ -1870,7 +1904,12 @@ def __enter__(self) -> '_deadline':
18701904
# nothing installed that would need restoring.
18711905
self.seconds = scale_deadline(self.seconds)
18721906

1873-
def expire(signum: 'int', frame: 'Any') -> 'None':
1907+
def expire(signum: 'int', frame: 'Any') -> 'None': # pylint: disable=unused-argument
1908+
if self.settled:
1909+
return
1910+
# Re-armed before raising, so a raise that never reaches the body
1911+
# is followed by another one.
1912+
signal.alarm(1)
18741913
raise TimeoutError(f'did not finish within {self.seconds}s')
18751914

18761915
self.previous = signal.signal(signal.SIGALRM, expire)
@@ -1879,6 +1918,11 @@ def expire(signum: 'int', frame: 'Any') -> 'None':
18791918
return self
18801919

18811920
def __exit__(self, *exc_info: 'Any') -> 'None':
1921+
# For a body that left without settling: a firing from here to the
1922+
# cancel below must not re-arm and raise, which would skip the restore.
1923+
# One handled on entry, before this line runs, is not stopped by it --
1924+
# only a body that settles before leaving stops that, as roundtrip does.
1925+
self.settled = True
18821926
if self.armed:
18831927
# Cancel before restoring, so an alarm firing between the two
18841928
# cannot be delivered to whatever handler was installed before.

0 commit comments

Comments
 (0)