Skip to content

Commit cf06e9d

Browse files
authored
fix(reassembly)!: move TCP's gap cap into config as PCAPKIT_TCP_GAP_LIMIT (#2546) (#2579)
1 parent e0989d6 commit cf06e9d

10 files changed

Lines changed: 140 additions & 45 deletions

File tree

‎docs/source/contributing/conventions/reassembly-integration.rst‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,12 @@ The Rule
2828
message, accept overlaps -- the engine does the same and adds no timeout, cap or
2929
ordering rule the RFC does not call for.
3030

31+
**TCP's gap cap is a safety limit, not a reassembly rule.** No RFC calls for
32+
:data:`~pcapkit.utilities.config.TCP_GAP_LIMIT`, the widest gap TCP reassembly
33+
zero-fills to merge a segment. It bounds the memory a capture can have zero-filled, as
34+
:data:`~pcapkit.utilities.config.MAX_ZERO_PAD_LENGTH` does for a field, and ``-1``
35+
lifts it to give the uncapped reassembly point 4 asks for. Ruled on :issue:`2546`.
36+
3137
Compliance, Measured
3238
~~~~~~~~~~~~~~~~~~~~~
3339

‎docs/source/pcapkit/foundation/reassembly/tcp.rst‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@ which reconstructs fragmented TCP packets back to origin.
2121
.. autoattribute:: __callback_fn__
2222
:no-value:
2323
.. autoattribute:: __timeout__
24-
.. autoattribute:: __window__
2524

2625
Algorithm
2726
=========

‎docs/source/pcapkit/utilities/config.rst‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,11 @@ Limits
6161
- ``32``
6262
- ``>= 0``, or ``-1``
6363
- each capture remembered pins one packet in memory
64+
* - :envvar:`PCAPKIT_TCP_GAP_LIMIT`
65+
- ``16777216``
66+
- ``>= 0``, or ``-1``
67+
- a TCP gap that wide is zero-filled in memory, up to 2 GiB at ``-1``
68+
(:issue:`2546`)
6469
* - :envvar:`PCAPKIT_FALLBACK_LIMIT`
6570
- ``10000``
6671
- ``>= 0``, or ``-1``
@@ -99,6 +104,10 @@ Limits
99104

100105
Sets :data:`~pcapkit.utilities.config.IPV6_TCP_WARNED_LIMIT`.
101106

107+
.. envvar:: PCAPKIT_TCP_GAP_LIMIT
108+
109+
Sets :data:`~pcapkit.utilities.config.TCP_GAP_LIMIT`.
110+
102111
.. envvar:: PCAPKIT_FALLBACK_LIMIT
103112

104113
Sets :data:`~pcapkit.utilities.config.FALLBACK_LIMIT`.
@@ -127,6 +136,7 @@ Limits
127136
.. autodata:: pcapkit.utilities.config.FRAME_LAYER_LIMIT
128137
.. autodata:: pcapkit.utilities.config.EXTENSION_HEADER_LIMIT
129138
.. autodata:: pcapkit.utilities.config.IPV6_TCP_WARNED_LIMIT
139+
.. autodata:: pcapkit.utilities.config.TCP_GAP_LIMIT
130140
.. autodata:: pcapkit.utilities.config.FALLBACK_LIMIT
131141
.. autodata:: pcapkit.utilities.config.DETECT_CACHE_SIZE
132142
.. autodata:: pcapkit.utilities.config.READ_CHUNK

‎examples/generators/legacy.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,8 +47,9 @@
4747
each direction is a buffer of its own.
4848
:meth:`~pcapkit.foundation.reassembly.tcp.TCP.submit` runs on that
4949
direction's RST; once its FIN has been seen and every octet below the FIN
50-
has arrived; when a SYN, or a segment farther than ``__window__`` from the
51-
buffer, opens a new one on the same four-tuple; on a timeout, if one is
50+
has arrived; when a SYN, or a segment farther than
51+
:data:`~pcapkit.utilities.config.TCP_GAP_LIMIT` from the buffer, opens a
52+
new one on the same four-tuple; on a timeout, if one is
5253
set; and, for whatever is still buffered, when the datagrams are read --
5354
``extraction.reassembly`` flushes every open buffer through
5455
:meth:`~pcapkit.foundation.reassembly.reassembly.Reassembly.fetch`. So a

‎pcapkit/foundation/reassembly/data/tcp.py‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -236,9 +236,8 @@ class Fragment(Info):
236236
#: merge only ever shrinks or removes one, filling it from the arriving
237237
#: segment. The one exception lies wholly *outside* :attr:`raw`: when a
238238
#: stream whose SYN was captured is split rather than padded across a gap
239-
#: wider than :attr:`TCP.__window__
240-
#: <pcapkit.foundation.reassembly.tcp.TCP.__window__>`, both halves record
241-
#: the range left between them. Any entry at all makes the fragment
239+
#: wider than :data:`~pcapkit.utilities.config.TCP_GAP_LIMIT`, both halves
240+
#: record the range left between them. Any entry at all makes the fragment
242241
#: incomplete.
243242
#:
244243
#: This is deliberately **not** derived from

‎pcapkit/foundation/reassembly/tcp.py‎

Lines changed: 32 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
DatagramID, Fragment, HoleDescriptor, Packet)
1919
from pcapkit.foundation.reassembly.reassembly import ReassemblyBase
2020
from pcapkit.protocols.transport.tcp import TCP as TCP_Protocol
21+
from pcapkit.utilities import config
2122

2223
if TYPE_CHECKING:
2324
from typing import Type
@@ -99,6 +100,11 @@ class TCP(ReassemblyBase[Packet, Datagram, BufferID, Buffer]):
99100
<pcapkit.foundation.reassembly.data.tcp.Datagram.conflict>`; 1.4.x
100101
kept the later octets without a record (:issue:`443`).
101102
103+
.. versionremoved:: 1.6.0
104+
105+
``__window__``, now :data:`~pcapkit.utilities.config.TCP_GAP_LIMIT`
106+
(:issue:`2546`).
107+
102108
"""
103109
if TYPE_CHECKING:
104110
protocol: 'Type[TCP_Protocol]'
@@ -129,41 +135,33 @@ class TCP(ReassemblyBase[Packet, Datagram, BufferID, Buffer]):
129135
#: ask for one, e.g. ``2 * 120`` for 2·MSL if that is the policy wanted.
130136
__timeout__ = math.inf
131137

132-
#: int: Widest gap, in octets, that merging a segment into the payload
133-
#: buffer it is read against may zero-fill: 16 MiB. The gap is measured on
134-
#: whichever side the segment lands, from the buffer's end to the
135-
#: segment's start, or from the segment's end to the buffer's start.
136-
#:
137-
#: A segment can only legitimately open a gap as wide as the data in flight,
138-
#: which the receive window bounds. :rfc:`7323` allows windows up to
139-
#: ``2 ** 30``, but zero-filling a gap that wide costs 1 GiB, and real
140-
#: windows are far smaller -- Linux's default ``net.ipv4.tcp_rmem`` ceiling
141-
#: is 6 MiB -- so 16 MiB clears them with room to spare. A segment that
142-
#: would open a wider gap is not padded to; the buffer held so far is
143-
#: submitted and a fresh one opened for the segment instead. What that
144-
#: means depends on whether the buffer's SYN was captured:
145-
#:
146-
#: * **No SYN** -- the segment is taken to start a new stream on a reused
147-
#: 4-tuple, and the two buffers are reported independently.
148-
#: * **SYN captured** -- the SYN proves both are the same stream with the
149-
#: range in between lost, so that range is recorded in the
150-
#: :attr:`~pcapkit.foundation.reassembly.data.tcp.Fragment.gap` of the
151-
#: payload buffer the segment was read against and of the new one, and
152-
#: both are reported
153-
#: :attr:`~pcapkit.foundation.reassembly.data.data.Completion.PARTIAL`
154-
#: (in strict mode, as a single run each). The new buffer keeps the
155-
#: SYN's header and counts as having seen it.
156-
#:
157-
#: Override in a subclass to change the bound.
158-
__window__ = 1 << 24
159-
160138
##########################################################################
161139
# Methods.
162140
##########################################################################
163141

164142
def reassembly(self, info: 'Packet') -> 'None':
165143
"""Reassembly procedure.
166144
145+
A segment is merged into the payload buffer it is read against by
146+
zero-filling the gap between them, measured on whichever side the
147+
segment lands: from the buffer's end to the segment's start, or from
148+
the segment's end to the buffer's start. A gap wider than
149+
:data:`~pcapkit.utilities.config.TCP_GAP_LIMIT` is not padded to; the
150+
buffer held so far is submitted and a fresh one opened for the segment
151+
instead. What that means depends on whether the buffer's SYN was
152+
captured:
153+
154+
* **No SYN** -- the segment is taken to start a new stream on a reused
155+
4-tuple, and the two buffers are reported independently.
156+
* **SYN captured** -- the SYN proves both are the same stream with the
157+
range in between lost, so that range is recorded in the
158+
:attr:`~pcapkit.foundation.reassembly.data.tcp.Fragment.gap` of the
159+
payload buffer the segment was read against and of the new one, and
160+
both are reported
161+
:attr:`~pcapkit.foundation.reassembly.data.data.Completion.PARTIAL`
162+
(in strict mode, as a single run each). The new buffer keeps the
163+
SYN's header and counts as having seen it.
164+
167165
Arguments:
168166
info: :term:`info <reasm.tcp.packet>` dict of packets to be reassembled
169167
@@ -203,8 +201,8 @@ def reassembly(self, info: 'Packet') -> 'None':
203201
# it most plausibly continues -- this ACK's own payload buffer, or else
204202
# the most recently opened one -- per :rfc:`9293#section-3.4`. Plain
205203
# integer comparison reads a stream crossing 2**32 as a gap of about
206-
# 4 GiB. A segment landing farther than ``__window__`` outside that
207-
# buffer starts a new stream instead -- see ``__window__``.
204+
# 4 GiB. A segment landing farther than ``config.TCP_GAP_LIMIT``
205+
# outside that buffer splits the stream instead -- see the docstring.
208206
SHIFT = 0
209207
HDR = info.header if SYN else b'' # header of the buffer opened below, if any
210208
ANCHORED = SYN # whether that buffer's stream start is known
@@ -217,7 +215,8 @@ def reassembly(self, info: 'Packet') -> 'None':
217215
END = fragment.isn + fragment.len
218216
UNWRAPPED = self._unwrap(PSN, END)
219217
# the zero-filled gap merging this segment would open, either side
220-
if max(UNWRAPPED - END, fragment.isn - (UNWRAPPED + info.len)) > self.__window__:
218+
limit = config.TCP_GAP_LIMIT
219+
if limit != -1 and max(UNWRAPPED - END, fragment.isn - (UNWRAPPED + info.len)) > limit:
221220
old = self._buffer.pop(BUFID)
222221
if self._anchored(old):
223222
# The SYN proves this is still the same stream, with the
@@ -552,7 +551,7 @@ def _trim_gaps(gap: 'list[tuple[int, int]]', first: 'int', last: 'int') -> 'None
552551
"""Remove a received range from a fragment's gap list, in place.
553552
554553
Only an entry lying outside :attr:`~pcapkit.foundation.reassembly.data.tcp.Fragment.raw`
555-
-- the range a split stream left out, see :attr:`TCP.__window__` -- can
554+
-- the range a split stream left out, see :meth:`TCP.reassembly` -- can
556555
still overlap a segment once it is merged; :meth:`_merge_overlap` has
557556
already closed every entry inside ``raw`` that the segment covers.
558557
@@ -806,7 +805,7 @@ def submit(self, buf: 'Buffer', *, bufid: 'BufferID', # type: ignore[override]
806805
# How completely this buffer came out, and why it stopped. Derived
807806
# once per buffer, so the two branches cannot disagree about it.
808807
# Any gap entry counts, including one lying wholly outside ``raw``:
809-
# that is the range a split stream left out (see ``__window__``).
808+
# that is the range a split stream left out (see ``reassembly``).
810809
completion = Completion.COMPLETE if not buffer.gap else (
811810
Completion.TIMEOUT if timeout else Completion.PARTIAL
812811
)

‎pcapkit/utilities/config.py‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@
5252
__all__ = [
5353
# limits
5454
'FRAME_LAYER_LIMIT', 'EXTENSION_HEADER_LIMIT', 'IPV6_TCP_WARNED_LIMIT',
55-
'FALLBACK_LIMIT', 'DETECT_CACHE_SIZE', 'READ_CHUNK',
55+
'TCP_GAP_LIMIT', 'FALLBACK_LIMIT', 'DETECT_CACHE_SIZE', 'READ_CHUNK',
5656
'MAX_ZERO_PAD_LENGTH', 'MAX_ZERO_PAD_SHORTFALL', 'ZERO_PAD_BUDGET_RATIO',
5757
# flags
5858
'CHECKSUM_VERIFY', 'CHECKSUM_COMPUTE',
@@ -218,6 +218,31 @@ def read_flag(name: 'str', default: 'bool' = False) -> 'bool':
218218
IPV6_TCP_WARNED_LIMIT = read_limit('IPV6_TCP_WARNED_LIMIT', 32, risk=(
219219
'each capture remembered pins one of its packets in memory'))
220220

221+
#: int: Widest gap, in octets, that
222+
#: :meth:`TCP.reassembly <pcapkit.foundation.reassembly.tcp.TCP.reassembly>`
223+
#: zero-fills to merge a segment into the payload buffer it is read against,
224+
#: :envvar:`PCAPKIT_TCP_GAP_LIMIT`; ``-1`` always merges.
225+
#:
226+
#: This is a safety limit, not a reassembly rule: no RFC calls for one
227+
#: (:issue:`2546`). A segment can only legitimately open a gap as wide as the
228+
#: data in flight, which the receive window bounds. :rfc:`7323` allows windows
229+
#: up to ``2 ** 30``, but zero-filling a gap that wide costs 1 GiB, and real
230+
#: windows are far smaller -- Linux's default ``net.ipv4.tcp_rmem`` ceiling is
231+
#: 6 MiB -- so the default of 16 MiB clears them with room to spare. Past it,
232+
#: the stream is split rather than padded, as :meth:`TCP.reassembly
233+
#: <pcapkit.foundation.reassembly.tcp.TCP.reassembly>` describes.
234+
#:
235+
#: At ``-1`` only the sequence space bounds the gap: a sequence number is read
236+
#: within ``2 ** 31`` of the data buffered (:rfc:`9293#section-3.4`), so one
237+
#: segment can have up to 2 GiB zero-filled.
238+
#:
239+
#: .. versionadded:: 1.6.0
240+
#:
241+
#: Moved from ``pcapkit.foundation.reassembly.tcp.TCP.__window__`` (:issue:`2546`).
242+
TCP_GAP_LIMIT = read_limit('TCP_GAP_LIMIT', 1 << 24, risk=(
243+
'merging a segment that far from the data buffered zero-fills the gap in memory, '
244+
'up to 2 GiB per segment at -1'))
245+
221246
#: int: Most objects the ``_append_fallback`` method of a dumper
222247
#: :func:`~pcapkit.dumpkit.common.make_dumper` builds
223248
#: expands for one top-level object, counting an object once per time it is met,

‎tests/foundation/reassembly/test_tcp_reassembly_roundtrip_unit.py‎

Lines changed: 54 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,9 @@
1313
* #1356 -- a FIN that submits the buffer at once splits data reordered behind
1414
it off into a buffer of its own.
1515
16+
The widest gap a segment is padded across is ``config.TCP_GAP_LIMIT``, a safety
17+
limit that ``-1`` lifts (#2546).
18+
1619
Every case feeds hand-built segments and reads no capture. Classes are imported
1720
inside each test, after :func:`~tests._support.reimport_once_per_class`.
1821
@@ -174,14 +177,62 @@ def test_a_segment_far_below_the_buffer_starts_a_new_stream(self) -> None:
174177
reasm = self._run(self._seg(0x30000000, b'A' * 5), self._seg(0x10000000, b'B' * 5))
175178
self.assertEqual(sorted(p for (_, _, _, p) in self._summary(reasm)), [b'A' * 5, b'B' * 5])
176179

177-
def test_a_gap_within_the_window_is_still_padded(self) -> None:
178-
from pcapkit.foundation.reassembly.tcp import TCP
180+
def test_a_gap_at_the_default_limit_is_still_padded(self) -> None:
181+
from pcapkit.utilities import config
179182

180-
far = TCP.__window__
183+
far = config.TCP_GAP_LIMIT
181184
reasm = self._run(self._seg(0, b'A' * 5), self._seg(5 + far, b'B' * 5), strict=False)
182185
(completed, _, index, payload), = self._summary(reasm)
183186
self.assertEqual((completed, index, len(payload)), ('PARTIAL', (1, 2), 10 + far))
184187

188+
# #2546 ------------------------------------------------------------------
189+
190+
def _gap_limit(self, limit):
191+
"""Patch ``config.TCP_GAP_LIMIT`` to ``limit`` for the rest of the test."""
192+
from unittest import mock
193+
194+
from pcapkit.utilities import config
195+
196+
patcher = mock.patch.object(config, 'TCP_GAP_LIMIT', limit)
197+
patcher.start()
198+
self.addCleanup(patcher.stop)
199+
200+
def test_a_gap_up_to_the_limit_is_padded(self) -> None:
201+
self._gap_limit(100)
202+
# the gap is measured on either side of the buffer
203+
for side, segs, runs in (('after', (self._seg(0, b'A' * 5), self._seg(105, b'B' * 5)),
204+
(b'A' * 5, b'B' * 5)),
205+
('before', (self._seg(200, b'A' * 5), self._seg(95, b'B' * 5)),
206+
(b'B' * 5, b'A' * 5))):
207+
with self.subTest(side=side):
208+
self.assertEqual(self._summary(self._run(*segs)), [('PARTIAL', 500, (1, 2), runs)])
209+
210+
def test_a_gap_past_the_limit_splits_the_stream(self) -> None:
211+
self._gap_limit(100)
212+
for side, seq in (('after', 106), ('before', 94)):
213+
with self.subTest(side=side, syn=False):
214+
reasm = self._run(self._seg(200 if side == 'before' else 0, b'A' * 5),
215+
self._seg(seq, b'B' * 5))
216+
self.assertEqual(sorted((c, p) for (c, _, _, p) in self._summary(reasm)),
217+
[('COMPLETE', b'A' * 5), ('COMPLETE', b'B' * 5)])
218+
# a captured SYN keeps both halves one stream, each missing the gap
219+
reasm = self._run(self._seg(0, syn=True), self._seg(1, b'A' * 5), self._seg(107, b'B' * 5))
220+
self.assertEqual(self._summary(reasm), [('PARTIAL', 500, (3,), (b'B' * 5,)),
221+
('PARTIAL', 500, (1, 2), (b'A' * 5,))])
222+
self.assertEqual(reasm._buffer[BUFID].ack[500].gap, [(6, 106)])
223+
224+
def test_minus_one_pads_a_gap_past_the_default(self) -> None:
225+
self._gap_limit(-1)
226+
far = (1 << 24) + 1
227+
start = far + 10
228+
for side, segs, runs in (('after', (self._seg(0, b'A' * 5), self._seg(5 + far, b'B' * 5)),
229+
(b'A' * 5, b'B' * 5)),
230+
('before', (self._seg(start, b'A' * 5),
231+
self._seg(start - far - 5, b'B' * 5)),
232+
(b'B' * 5, b'A' * 5))):
233+
with self.subTest(side=side):
234+
self.assertEqual(self._summary(self._run(*segs)), [('PARTIAL', 500, (1, 2), runs)])
235+
185236
def test_data_reordered_behind_the_fin_joins_the_stream(self) -> None:
186237
reasm = self._run(self._seg(100, syn=True, ack=0), self._seg(101, b'AAAAA'),
187238
self._seg(111, fin=True), self._seg(106, b'BBBBB'))

‎tests/project/test_rfc_constants_one_home_1739_unit.py‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -350,8 +350,6 @@
350350
'an if_tsoffset option: 4 octets of header and 8 of value',
351351
('pcapkit/foundation/engines/scapy.py', 'options[4:12]'):
352352
'the 8-octet value of an if_tsoffset option',
353-
('pcapkit/foundation/reassembly/tcp.py', '1 << 24'):
354-
'a bit width, not a PCAP global header',
355353
('pcapkit/toolkit/dpkt.py', '_fragment(6)'):
356354
'the IP version number',
357355
('pcapkit/toolkit/dpkt.py', 'version == 6'):

‎tests/utilities/test_config_unit.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@
7272
('FRAME_LAYER_LIMIT', 64, 0, 'RecursionError (#1610)'),
7373
('EXTENSION_HEADER_LIMIT', 32, 0, 'RecursionError (#1610)'),
7474
('IPV6_TCP_WARNED_LIMIT', 32, 0, 'pins one of its packets in memory'),
75+
('TCP_GAP_LIMIT', 16_777_216, 0, 'zero-fills the gap in memory'),
7576
('FALLBACK_LIMIT', 10_000, 0, 'a large or endless object graph'),
7677
('DETECT_CACHE_SIZE', 1024, 0, 'never-repeating text'),
7778
('READ_CHUNK', 1_048_576, 1, 'a bogus record length allocates'),
@@ -318,6 +319,12 @@ def test_a_moved_name_has_no_re_export(self) -> None:
318319
with self.subTest(module=module, name=name):
319320
self.assertFalse(hasattr(loaded, name))
320321

322+
def test_the_tcp_gap_limit_left_no_class_attribute(self) -> None:
323+
# #2546: TCP.__window__ moved here as TCP_GAP_LIMIT.
324+
from pcapkit.foundation.reassembly.tcp import TCP
325+
326+
self.assertFalse(hasattr(TCP, '__window__'))
327+
321328
def test_every_variable_is_documented(self) -> None:
322329
from pcapkit.utilities import config
323330

0 commit comments

Comments
 (0)