Skip to content

Commit 33dddb9

Browse files
committed
fix(foundation): back the abandoned-extraction close with weakref.finalize (#1997)
GraalPy never calls `__del__`, so an `auto=False` extraction dropped before EOF kept the file it opened. `Extractor.__init__` now registers a `weakref.finalize` over the input stream alone, and only for a stream it owns, so a caller's stream is never closed by it. `_cleanup` and `__exit__` detach it, and `__del__` stays, sharing the same guarded close. New tests check the finaliser on CPython too: it is registered over the stream rather than the extractor, closes an owned handle with `__del__` patched out, is detached on a finished extraction, and spares a caller's stream, plain or wrapped.
1 parent 279f3f0 commit 33dddb9

3 files changed

Lines changed: 197 additions & 9 deletions

File tree

‎docs/source/pcapkit/foundation/extraction.rst‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@ and extracts parameters from a PCAP file.
7272
.. autoattribute:: _flag_s
7373

7474
.. autoattribute:: _ifile
75+
.. autoattribute:: _finalizer
7576
.. autoattribute:: _ofile
7677
.. autoattribute:: _ifnm
7778
.. autoattribute:: _ofnm

‎pcapkit/foundation/extraction.py‎

Lines changed: 71 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import os
2121
import sys
2222
import types
23+
import weakref
2324
from typing import TYPE_CHECKING, Generic, TypeVar, cast
2425

2526
from dictdumper.dumper import Dumper
@@ -106,6 +107,35 @@
106107
_PATH_ENGINES = frozenset({'scapy', 'pyshark', 'pypcap', 'pcap_ct'})
107108

108109

110+
def _release_input(ifile: 'IO[bytes]') -> 'None':
111+
"""Close an input stream from a finaliser, if it is still open.
112+
113+
The one close both backstops of :issue:`606` perform --
114+
:meth:`Extractor.__del__ <pcapkit.foundation.extraction.Extractor.__del__>`
115+
and the :class:`weakref.finalize` held in
116+
:attr:`Extractor._finalizer <pcapkit.foundation.extraction.Extractor._finalizer>`.
117+
It takes the stream rather than the extractor, so that a finaliser built on
118+
it holds no reference that would keep the extractor alive; deciding whether
119+
the stream is the extractor's to close is the caller's business, cf.
120+
:meth:`Extractor._owns_input <pcapkit.foundation.extraction.Extractor._owns_input>`.
121+
122+
Args:
123+
ifile: The stream to close.
124+
125+
Note:
126+
A finaliser must not raise, so the close is guarded: that is what a close
127+
during interpreter shutdown can fail at.
128+
129+
"""
130+
try:
131+
if not ifile.closed:
132+
ifile.close()
133+
except (OSError, ValueError): # pragma: no cover
134+
# Nothing useful can be reported from a finaliser, and raising here
135+
# would only produce an "Exception ignored in __del__" on stderr.
136+
pass
137+
138+
109139
class Extractor(Generic[_P]):
110140
"""Extractor for PCAP files.
111141
@@ -229,6 +259,12 @@ class Extractor(Generic[_P]):
229259

230260
#: Input file object.
231261
_ifile: 'BufferedReader'
262+
#: Backstop that closes :attr:`_ifile` when the extractor is collected
263+
#: before :meth:`_cleanup` has run, registered only when the stream is
264+
#: this class's own to close (:issue:`1997`), else :data:`None`.
265+
#: :meth:`__del__` alone is not enough, as GraalPy never calls it; a
266+
#: :class:`weakref.finalize` runs there and on CPython alike.
267+
_finalizer: 'Optional[weakref.finalize]'
232268
#: Output writer. It holds the writer class until :meth:`_open_output`
233269
#: replaces it with the writer, or under split mode with a factory.
234270
_ofile: 'Dumper | Type[Dumper]'
@@ -1439,6 +1475,15 @@ def __init__(self,
14391475
logger.debug('input stream has no peek(), wrapping it in PeekableStream')
14401476
self._ifile = cast('BufferedReader', PeekableStream(self._ifile))
14411477

1478+
# NOTE: the #606 backstop for an extraction abandoned before EOF. GraalPy
1479+
# never calls ``__del__``, so it is a ``weakref.finalize`` as well (#1997).
1480+
# It closes over the stream alone -- a reference to ``self`` would keep
1481+
# the extractor alive for as long as the finaliser is -- and it is only
1482+
# registered for a stream this class owns, so a caller's is never closed.
1483+
self._finalizer = None
1484+
if self._owns_input():
1485+
self._finalizer = weakref.finalize(self, _release_input, self._ifile)
1486+
14421487
if not self._flag_q:
14431488
output, ext = self.__output__[fmt]
14441489
if ext is None:
@@ -1548,6 +1593,7 @@ def __exit__(self, exc_type: 'Type[BaseException] | None', exc_value: 'BaseExcep
15481593
"""Close the input file when exits."""
15491594
logger.debug('closing %s on context exit after %d frame(s)', self._ifnm, self._frnum)
15501595
self._ifile.close()
1596+
self._detach_finalizer()
15511597
self._exeng.close()
15521598

15531599
def __del__(self) -> 'None':
@@ -1561,6 +1607,13 @@ def __del__(self) -> 'None':
15611607
``ResourceWarning`` raised wherever collection happens to run, which can be
15621608
far from the code that abandoned the handle (:issue:`606`).
15631609
1610+
GraalPy never calls this method, so the same close is also registered as
1611+
the :class:`weakref.finalize` in :attr:`_finalizer`, which runs on either
1612+
interpreter (:issue:`1997`). This method stays for whoever calls it
1613+
directly, a subclass extending it through :func:`super` among them, and
1614+
both are safe to run for one stream: whichever comes second finds it
1615+
already closed.
1616+
15641617
This is a backstop and not the recommended route: collection is not
15651618
deterministic, and :class:`Extractor` takes part in a reference cycle
15661619
through its engine, so a handle can outlive its last reference until the
@@ -1580,14 +1633,7 @@ def __del__(self) -> 'None':
15801633
"""
15811634
if not self._owns_input():
15821635
return
1583-
try:
1584-
ifile = self.__dict__['_ifile']
1585-
if not ifile.closed:
1586-
ifile.close()
1587-
except (OSError, ValueError): # pragma: no cover
1588-
# Nothing useful can be reported from a finaliser, and raising here
1589-
# would only produce an "Exception ignored in __del__" on stderr.
1590-
pass
1636+
_release_input(self.__dict__['_ifile'])
15911637

15921638
##########################################################################
15931639
# Utilities.
@@ -1597,7 +1643,8 @@ def _owns_input(self) -> 'bool':
15971643
"""Whether the input stream is this class's to close.
15981644
15991645
The one place the ownership rule of :issue:`610` is written down, so that
1600-
:meth:`_cleanup` and :meth:`__del__` cannot drift apart on it.
1646+
:meth:`_cleanup`, :meth:`__del__` and the registration of
1647+
:attr:`_finalizer` cannot drift apart on it.
16011648
16021649
Returns:
16031650
:data:`True` when the input stream was opened by this class and is
@@ -1633,6 +1680,20 @@ def _owns_input(self) -> 'bool':
16331680
return True
16341681
return isinstance(ifile, SeekableReader)
16351682

1683+
def _detach_finalizer(self) -> 'None':
1684+
"""Unregister :attr:`_finalizer`, once the input has been closed on purpose.
1685+
1686+
A finished extraction then leaves nothing registered against its stream:
1687+
neither a second close at collection nor a reference that keeps the stream
1688+
alive until the extractor is collected.
1689+
1690+
"""
1691+
# NOTE: read through ``__dict__``, as ``_owns_input`` does, for an
1692+
# instance whose constructor did not get as far as registering one.
1693+
finalizer = self.__dict__.get('_finalizer')
1694+
if finalizer is not None:
1695+
finalizer.detach()
1696+
16361697
def _note_eof_progress(self) -> 'bool':
16371698
"""Record this end of stream, and say whether the input advanced to reach it.
16381699
@@ -1734,4 +1795,5 @@ def _cleanup(self) -> 'None':
17341795
# supplied and still needs *is* closed.
17351796
if self._owns_input():
17361797
self._ifile.close()
1798+
self._detach_finalizer()
17371799
self._exeng.close()

‎tests/foundation/test_extraction_ownership.py‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -337,6 +337,131 @@ def test_non_seekable_stream_survives_its_seekable_wrapper(self) -> None:
337337
stream.close()
338338

339339

340+
@unittest.skipUnless(HAS_RUNTIME, 'runtime dependencies not installed')
341+
class FinaliserBackstopWithoutDel(unittest.TestCase):
342+
"""#1997: the backstop is a :class:`weakref.finalize`, not ``__del__`` alone.
343+
344+
GraalPy never calls ``__del__``, so on that interpreter the #606 backstop
345+
only ever ran if something else did the closing. CPython does call it, which
346+
is why a regression here would go unnoticed on every CPython leg unless the
347+
finaliser is checked on its own: these tests do that, either by inspecting it
348+
or by taking ``__del__`` out of the way, so that they fail on CPython too.
349+
350+
"""
351+
352+
def test_abandoned_extraction_registers_a_finaliser_over_its_own_handle(self) -> None:
353+
"""The finaliser is alive mid-extraction and holds the stream, not ``self``.
354+
355+
A finaliser that referenced the extractor would keep it alive for as long
356+
as the finaliser is, which is to say for ever.
357+
358+
"""
359+
import weakref
360+
361+
import pcapkit
362+
363+
extractor = pcapkit.extract(fin=sample_path('in.pcap'), nofile=True,
364+
store=False, auto=False)
365+
next(extractor)
366+
367+
finalizer = extractor._finalizer
368+
self.assertIsInstance(finalizer, weakref.finalize)
369+
self.assertTrue(finalizer.alive,
370+
'an extraction still in flight should have its backstop '
371+
'registered')
372+
373+
target, _, args, kwargs = finalizer.peek()
374+
self.assertIs(target, extractor)
375+
self.assertEqual(len(args), 1)
376+
self.assertIs(args[0], extractor._ifile)
377+
self.assertFalse(any(value is extractor for value in kwargs.values()),
378+
'the finaliser must not hold the extractor itself')
379+
380+
self.assertFalse(extractor._ifile.closed)
381+
finalizer()
382+
self.assertTrue(extractor._ifile.closed,
383+
'invoking the finaliser should release the handle')
384+
self.assertFalse(finalizer.alive)
385+
386+
def test_finaliser_alone_releases_an_abandoned_handle(self) -> None:
387+
"""With ``__del__`` a no-op, as on GraalPy, collection still closes the file."""
388+
import pcapkit
389+
from pcapkit.foundation.extraction import Extractor
390+
391+
capture = sample_path('in.pcap')
392+
recorder = _OpenRecorder(capture)
393+
with mock.patch.object(Extractor, '__del__', lambda self: None):
394+
with mock.patch.object(builtins, 'open', recorder):
395+
extractor = pcapkit.extract(fin=capture, nofile=True, store=False,
396+
auto=False)
397+
next(extractor)
398+
self.assertEqual(recorder.still_open, 1)
399+
400+
del extractor
401+
gc.collect()
402+
403+
self.assertEqual(recorder.still_open, 0,
404+
'without __del__ the handle outlived the extractor')
405+
406+
def test_finished_extraction_detaches_its_finaliser(self) -> None:
407+
"""``_cleanup`` and the context manager unregister it, so nothing closes twice."""
408+
import pcapkit
409+
410+
capture = sample_path('in.pcap')
411+
with self.subTest(route='_cleanup at end of file'):
412+
extractor = pcapkit.extract(fin=capture, nofile=True, store=False)
413+
self.assertTrue(extractor._ifile.closed)
414+
self.assertFalse(extractor._finalizer.alive)
415+
416+
with self.subTest(route='context manager exit'):
417+
with pcapkit.extract(fin=capture, nofile=True, store=False,
418+
auto=False) as extractor:
419+
next(extractor)
420+
self.assertTrue(extractor._finalizer.alive)
421+
self.assertTrue(extractor._ifile.closed)
422+
self.assertFalse(extractor._finalizer.alive)
423+
424+
def test_no_finaliser_for_a_callers_stream(self) -> None:
425+
"""A stream the caller passed in gets no finaliser, so collection leaves it open."""
426+
import pcapkit
427+
from pcapkit.foundation.extraction import Extractor
428+
429+
with open(sample_path('in.pcap'), 'rb') as handle:
430+
with mock.patch.object(Extractor, '__del__', lambda self: None):
431+
extractor = pcapkit.extract(fin=handle, nofile=True, store=False,
432+
auto=False)
433+
next(extractor)
434+
self.assertIsNone(extractor._finalizer)
435+
436+
del extractor
437+
gc.collect()
438+
439+
self.assertFalse(handle.closed,
440+
"collecting the extractor closed the caller's stream")
441+
442+
def test_finaliser_on_a_wrapper_spares_the_stream_beneath(self) -> None:
443+
"""A wrapped stream's finaliser closes the wrapper, not the caller's stream."""
444+
import pcapkit
445+
from pcapkit.corekit.io import SeekableReader
446+
447+
with open(sample_path('in.pcap'), 'rb') as source:
448+
payload = source.read()
449+
stream = io.BufferedReader(_NonSeekableRaw(payload))
450+
self.addCleanup(stream.close)
451+
452+
extractor = pcapkit.extract(fin=stream, nofile=True, store=False,
453+
auto=False)
454+
next(extractor)
455+
self.assertIsInstance(extractor._ifile, SeekableReader)
456+
self.assertTrue(extractor._finalizer.alive)
457+
458+
extractor._finalizer()
459+
self.assertTrue(extractor._ifile.closed)
460+
self.assertFalse(stream.closed,
461+
"the finaliser closed the caller's stream through the "
462+
'wrapper')
463+
464+
340465
class OwnershipRuleInIsolation(unittest.TestCase):
341466
"""``_owns_input`` and ``__del__`` without constructing a real extraction.
342467

0 commit comments

Comments
 (0)