Skip to content

Commit a96aea3

Browse files
authored
fix(reassembly): IP reassembly reports only the octets it holds (truncated fragments, strict runs, zero-length datagrams) (#1368)
- A fragment writes and marks only the octets the capture holds, clipped to its declared length and to the data buffer, which never changes length; a truncated fragment's partial last block stays a hole (#1357). - Strict runs stop at the total data length, and the last run is emitted even when it reaches the end of the bit table (#1358). - TDL is tested against -1 rather than for truthiness, so a zero-length datagram completes; completion is checked against the buffer's TDL on every fragment, not only the one with MF clear (#1359).
1 parent 6455cbc commit a96aea3

2 files changed

Lines changed: 163 additions & 18 deletions

File tree

‎pcapkit/foundation/reassembly/ip.py‎

Lines changed: 49 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,12 @@ class IP(ReassemblyBase[Packet[_AT], Datagram[_AT], BufferID, Buffer[_AT]], Gene
3737
strict: if :data:`True`, report a datagram that is not completely
3838
reassembled as the tuple of its received runs; otherwise as one
3939
contiguous payload, its holes zero-filled -- or, while the total
40-
length is unknown, only the prefix up to the first hole
40+
length is unknown, only the prefix up to the first hole. Receipt
41+
is tracked in 8-octet blocks, so the partial last block of a
42+
snaplen-truncated fragment is not counted as received: it is left
43+
out of the runs and ends the prefix, but once the total length is
44+
known the contiguous payload keeps its real octets, with only the
45+
rest of the hole zero-filled
4146
store: if store reassembled datagram in memory, i.e.,
4247
:attr:`self._dtgram <pcapkit.foundation.reassembly.reassembly.Reassembly._dtgram>`
4348
(if not, datagram will be discarded after callback)
@@ -132,15 +137,24 @@ def reassembly(self, info: 'Packet[_AT]') -> 'None':
132137
# append packet index
133138
buf.index.append(info.num)
134139

140+
# The fragment's data length is what its header declares, but only the
141+
# octets the capture actually holds can be written: a snaplen-truncated
142+
# fragment carries fewer, and anything past the declared length is not
143+
# fragment data. The data buffer is preallocated, so the write is also
144+
# clipped to it -- its length never changes.
145+
length = TL - IHL
146+
held = max(min(len(info.payload), length, len(buf.datagram) - FO), 0)
147+
payload = info.payload[:held]
148+
135149
# put data into data buffer
136150
start = FO
137-
stop = TL - IHL + FO
151+
stop = FO + held
138152

139153
# Find where this fragment disagrees with what the buffer already
140154
# holds *before* writing it -- ``buf.RCVBT`` and ``buf.TDL`` still
141155
# describe the state as every earlier fragment left it, which is
142156
# exactly what :meth:`_detect_conflicts` needs.
143-
conflicts = self._detect_conflicts(buf.RCVBT, buf.TDL, buf.datagram, info.payload, start, stop)
157+
conflicts = self._detect_conflicts(buf.RCVBT, buf.TDL, buf.datagram, payload, start, stop)
144158
if conflicts:
145159
buf.conflict.extend(conflicts)
146160

@@ -150,23 +164,31 @@ def reassembly(self, info: 'Packet[_AT]') -> 'None':
150164
# payload always overwrites here; ``conflicts`` above is what records
151165
# that it *disagreed* with what it overwrote, which is the part RFC 791
152166
# leaves unrecorded.
153-
buf.datagram[start:stop] = info.payload
167+
buf.datagram[start:stop] = payload
154168

155-
# set RCVBT bits (in 8 octets)
169+
# Set RCVBT bits (in 8 octets) for the blocks this fragment actually
170+
# filled. A fragment held in full marks its last block even when it is
171+
# short, as the final fragment's is; a truncated one leaves its partial
172+
# last block clear, since the octets after the cut are a real hole.
156173
start = FO // 8
157-
stop = FO // 8 + (TL - IHL + 7) // 8
158-
buf.RCVBT[start:stop] = b'\x01' * (stop - start)
174+
if held == length:
175+
stop = (FO + held + 7) // 8
176+
else:
177+
stop = (FO + held) // 8
178+
stop = min(stop, len(buf.RCVBT))
179+
if stop > start:
180+
buf.RCVBT[start:stop] = b'\x01' * (stop - start)
159181

160-
# get total data length (header excludes)
161-
TDL = 0
182+
# get total data length (header excludes); it stays ``-1`` until the
183+
# fragment with MF clear has arrived, and ``0`` is a valid length
162184
if not MF:
163-
TDL = TL - IHL + FO
164-
buf.__update__(TDL=TDL)
185+
buf.__update__(TDL=TL - IHL + FO)
186+
TDL = buf.TDL
165187

166188
# when datagram is reassembled in whole
167189
start = 0
168190
stop = (TDL + 7) // 8
169-
if TDL and all(buf.RCVBT[start:stop]):
191+
if TDL >= 0 and all(buf.RCVBT[start:stop]):
170192
self._dtgram.extend(
171193
self.submit(self._buffer.pop(BUFID), bufid=BUFID, checked=True)
172194
)
@@ -270,7 +292,7 @@ def submit(self, buf: 'Buffer[_AT]', *, bufid: 'tuple[_AT, _AT, int, TransType]'
270292

271293
start = 0
272294
stop = (TDL + 7) // 8
273-
flag = checked or (TDL > 0 and all(RCVBT[start:stop]))
295+
flag = checked or (TDL >= 0 and all(RCVBT[start:stop]))
274296
ret = [] # type: list[Datagram[_AT]]
275297

276298
# How completely this datagram came out, and why it stopped. Derived once,
@@ -283,16 +305,21 @@ def submit(self, buf: 'Buffer[_AT]', *, bufid: 'tuple[_AT, _AT, int, TransType]'
283305
if not flag and self._flag_s:
284306
data = [] # type: list[bytes]
285307
byte = bytearray()
286-
# extract received payload
308+
# extract received payload; a run never extends past the total
309+
# data length once it is known
310+
limit = TDL if TDL >= 0 else len(datagram)
287311
for (bctr, bit) in enumerate(RCVBT):
288312
if bit: # received bit
289313
this = bctr * 8
290-
that = this + 8
314+
that = min(this + 8, limit)
291315
byte += datagram[this:that]
292316
else: # missing bit
293317
if byte: # strip empty payload
294318
data.append(bytes(byte))
295319
byte = bytearray()
320+
# the last run may reach the end of the bit table
321+
if byte:
322+
data.append(bytes(byte))
296323
# strip empty packets
297324
if data or header:
298325
packet = Datagram(
@@ -317,12 +344,14 @@ def submit(self, buf: 'Buffer[_AT]', *, bufid: 'tuple[_AT, _AT, int, TransType]'
317344
# fragment with **MF** clear has arrived -- until then it is still its
318345
# initial ``-1``. Which case this is decides how much of the buffer
319346
# there is to report.
320-
if TDL > 0:
347+
if TDL >= 0:
321348
# The length is known. Report it, holes and all: the gaps read as
322349
# zeros, exactly as they do in the TCP reassembler's loose mode.
323350
# This is the reason ``strict=False`` exists -- a caller who
324351
# wants the gaps *marked* rather than
325-
# zero-filled uses ``strict=True`` and gets the runs.
352+
# zero-filled uses ``strict=True`` and gets the runs. A truncated
353+
# fragment's partial last block is not marked received, but the
354+
# octets it did write are real and are reported here as they are.
326355
stop = TDL
327356
else:
328357
# The length is not known. Slicing ``datagram[:-1]`` here would
@@ -341,7 +370,9 @@ def submit(self, buf: 'Buffer[_AT]', *, bufid: 'tuple[_AT, _AT, int, TransType]'
341370
# precisely because their offsets cannot be conveyed in a blob.
342371
#
343372
# ``RCVBT`` records receipt in 8-octet units, so the prefix ends at
344-
# the first clear bit.
373+
# the first clear bit -- which leaves out a truncated fragment's
374+
# partial last block, octets that the known-length case above
375+
# does report.
345376
received = 0
346377
for bit in RCVBT:
347378
if not bit:
Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
# -*- coding: utf-8 -*-
2+
"""IP reassembly reports exactly the octets it received.
3+
4+
- GitHub issue #1357: a snaplen-truncated fragment shrank the data buffer and
5+
marked its whole declared range received, so the datagram came out
6+
``COMPLETE`` with zeros in it and the rest shifted.
7+
- GitHub issue #1358: ``strict`` runs copied whole 8-octet blocks past the
8+
total data length, and dropped a run that reached the end of the bit table.
9+
- GitHub issue #1359: a datagram whose total data length is ``0`` never
10+
completed, because the length was tested for truthiness.
11+
12+
Every case builds its fragments in memory and reads no capture. Classes are
13+
imported inside each test, after :func:`~tests._support.reimport_once_per_class`.
14+
15+
"""
16+
17+
from ipaddress import ip_address
18+
import unittest
19+
20+
from tests._support import reimport_once_per_class
21+
22+
#: A 2000-octet payload with no zero octet in it, so a zero-filled hole shows.
23+
FULL = (bytes(range(1, 256)) * 8)[:2000]
24+
25+
26+
class TestIPReassemblyRoundTrip(unittest.TestCase):
27+
"""Pin what the IP reassembler reports for truncated, sparse and empty datagrams."""
28+
29+
def setUp(self) -> None:
30+
reimport_once_per_class(self)
31+
32+
@staticmethod
33+
def _packet(num, fo, mf, payload, *, tl=None):
34+
from pcapkit.const.reg.transtype import TransType
35+
from pcapkit.foundation.reassembly.data.ip import Packet
36+
37+
tl = 20 + len(payload) if tl is None else tl
38+
bufid = (ip_address('192.0.2.1'), ip_address('198.51.100.2'), 42, TransType.UDP)
39+
return Packet(bufid, num, fo, 20, mf, tl, b'ip-header' if fo == 0 else b'',
40+
bytearray(payload), 1000.0)
41+
42+
@staticmethod
43+
def _reassembler(strict):
44+
from pcapkit.foundation.reassembly.ipv4 import IPv4
45+
46+
return IPv4(strict=strict)
47+
48+
def _truncated(self, order, strict):
49+
frags = [(0, True, FULL[0:800]), (800, True, FULL[800:1600]), (1600, False, FULL[1600:])]
50+
reasm = self._reassembler(strict)
51+
for num, idx in enumerate(order, 1):
52+
fo, mf, payload = frags[idx]
53+
declared = len(payload)
54+
if idx == 1:
55+
payload = payload[:500] # snaplen cut: 500 of 800 octets held
56+
reasm(self._packet(num, fo, mf, payload, tl=20 + declared))
57+
return reasm
58+
59+
def test_truncated_fragment_leaves_a_hole_strict(self) -> None:
60+
for order in ([0, 1, 2], [2, 1, 0], [0, 2, 1]):
61+
with self.subTest(order=order):
62+
datagram, = self._truncated(order, strict=True).datagram
63+
self.assertEqual(datagram.completed.name, 'PARTIAL')
64+
# 800 + 500 = 1300 octets held, of which 162 whole blocks
65+
self.assertEqual(datagram.payload, (FULL[:1296], FULL[1600:]))
66+
67+
def test_truncated_fragment_keeps_buffer_length_loose(self) -> None:
68+
for order in ([0, 1, 2], [2, 1, 0], [0, 2, 1]):
69+
with self.subTest(order=order):
70+
reasm = self._truncated(order, strict=False)
71+
datagram, = reasm.datagram
72+
self.assertEqual(datagram.completed.name, 'PARTIAL')
73+
self.assertEqual(len(datagram.payload), 2000)
74+
self.assertEqual(datagram.payload[:1300], FULL[:1300])
75+
self.assertEqual(datagram.payload[1300:1600], bytes(300))
76+
self.assertEqual(datagram.payload[1600:], FULL[1600:])
77+
78+
def test_strict_run_clipped_at_total_data_length(self) -> None:
79+
reasm = self._reassembler(strict=True)
80+
reasm(self._packet(1, 0, True, b'A' * 1480))
81+
reasm(self._packet(2, 2960, False, b'C' * 43))
82+
datagram, = reasm.datagram
83+
self.assertEqual(datagram.completed.name, 'PARTIAL')
84+
self.assertEqual(datagram.payload, (b'A' * 1480, b'C' * 43))
85+
86+
def test_strict_run_at_end_of_bit_table_is_kept(self) -> None:
87+
reasm = self._reassembler(strict=True)
88+
reasm(self._packet(1, 0, True, b'A' * 8))
89+
reasm(self._packet(2, 65520, False, b'Z' * 7))
90+
datagram, = reasm.datagram
91+
self.assertEqual(datagram.payload, (b'A' * 8, b'Z' * 7))
92+
93+
def test_zero_length_unfragmented_datagram_completes(self) -> None:
94+
for strict in (True, False):
95+
with self.subTest(strict=strict):
96+
reasm = self._reassembler(strict)
97+
reasm(self._packet(1, 0, False, b''))
98+
self.assertEqual(dict(reasm._buffer), {})
99+
datagram, = reasm.datagram
100+
self.assertEqual(datagram.completed.name, 'COMPLETE')
101+
self.assertEqual(datagram.payload, b'')
102+
103+
def test_completes_when_the_last_hole_fills_after_the_final_fragment(self) -> None:
104+
reasm = self._reassembler(strict=True)
105+
reasm(self._packet(1, 8, False, b'ijkl'))
106+
reasm(self._packet(2, 0, True, b'abcdefgh'))
107+
self.assertEqual(dict(reasm._buffer), {})
108+
datagram, = reasm.datagram
109+
self.assertEqual(datagram.completed.name, 'COMPLETE')
110+
self.assertEqual(datagram.payload, b'abcdefghijkl')
111+
112+
113+
if __name__ == '__main__':
114+
unittest.main()

0 commit comments

Comments
 (0)