From 41a67cc1828b815f8df2f18f64835dd32a7effdc Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Tue, 6 Oct 2026 20:07:34 -0400 Subject: [PATCH] fix(protocols): from_data keeps declared lengths on truncated captures IPv4, IPv6 and UDP computed their length field from the payload `make()` was handed, so a `from_data` rebuild of a capture cut short by the snaplen wrote the recorded length instead of the declared one -- 1186 became 82 on `big_endian.pcap` frame 3, and the UDP length 0x048e became 0x003e. Each `make()` now takes the length explicitly (`total_length` for IPv4 and UDP, `payload_length` for IPv6) and writes it as given; `_make_data` hands over the parsed one. Omitting it computes the length as before, so a plain `IPv4(payload=...)` is unchanged. Closes #1155 --- pcapkit/protocols/internet/ipv4.py | 24 ++- pcapkit/protocols/internet/ipv6.py | 18 ++- pcapkit/protocols/transport/udp.py | 17 +- tests/protocols/internet/test_ipv4_unit.py | 2 + tests/protocols/internet/test_ipv6_unit.py | 2 + .../test_declared_length_roundtrip_runtime.py | 152 ++++++++++++++++++ .../protocols/transport/test_tcp_udp_unit.py | 4 +- 7 files changed, 211 insertions(+), 8 deletions(-) create mode 100644 tests/protocols/test_declared_length_roundtrip_runtime.py diff --git a/pcapkit/protocols/internet/ipv4.py b/pcapkit/protocols/internet/ipv4.py index 7ce2dc9c52..bc1a84a386 100644 --- a/pcapkit/protocols/internet/ipv4.py +++ b/pcapkit/protocols/internet/ipv4.py @@ -366,6 +366,7 @@ def make(self, src: 'IPv4Address | str | int | bytes' = '127.0.0.1', dst: 'IPv4Address | str | int | bytes' = '0.0.0.0', # nosec: B104 options: 'Optional[list[Schema_Option | tuple[Enum_OptionNumber, dict[str, Any]] | bytes] | Option]' = None, # pylint: disable=line-too-long + total_length: 'Optional[int]' = None, payload: 'bytes | ProtocolBase | Schema' = b'', **kwargs: 'Any') -> 'Schema_IPv4': """Make (construct) packet data. @@ -404,6 +405,8 @@ def make(self, src: Source address of the packet. dst: Destination address of the packet. options: Options of the packet. + total_length: Total length of the packet, header included; computed + from the header and the payload when omitted. payload: Payload of the packet. **kwargs: Arbitrary keyword arguments. @@ -427,12 +430,19 @@ def make(self, ttl_val = ttl if isinstance(ttl, int) else math.ceil(ttl.total_seconds()) if options is not None: - options_value, total_length = self._make_ipv4_options(options) + options_value, options_length = self._make_ipv4_options(options) else: - options_value, total_length = [], 0 + options_value, options_length = [], 0 - ihl = 5 + math.ceil(total_length / 4) - total_len = ihl * 4 + len(payload) + ihl = 5 + math.ceil(options_length / 4) + + # NOTE: An explicit ``total_length`` is written as given, so that a + # ``from_data`` rebuild of a capture truncated by snaplen keeps the + # length the sender declared instead of the one that was recorded + # (:issue:`1155`). Computed from the octets at hand otherwise, which is + # what a plain ``IPv4(payload=...)`` wants. + if total_length is None: + total_length = ihl * 4 + len(payload) return Schema_IPv4( vihl={ @@ -446,7 +456,7 @@ def make(self, 'rel': tos_rel_val, 'ecn': tos_ecn_val, }, - length=total_len, + length=total_length, id=id, flags={ 'df': df, @@ -543,6 +553,10 @@ def _make_data(cls, data: 'Data_IPv4') -> 'dict[str, Any]': # type: ignore[over # ``hdr_len`` exceeds the fixed 20-octet header (see :meth:`read`), # so read it defensively rather than assuming it always exists. 'options': getattr(data, 'options', None), + # NOTE: The parsed total length is passed through rather than left + # to ``make`` to recompute, so that a truncated capture rebuilds + # with the length that was on the wire (:issue:`1155`). + 'total_length': data.len, 'payload': cls._make_payload(data), } diff --git a/pcapkit/protocols/internet/ipv6.py b/pcapkit/protocols/internet/ipv6.py index f41d1258f4..1cb9c82f2d 100644 --- a/pcapkit/protocols/internet/ipv6.py +++ b/pcapkit/protocols/internet/ipv6.py @@ -227,6 +227,7 @@ def make(self, hop_limit: 'int' = 64, # reasonable default src: 'IPv6Address | str | bytes | int' = '::1', dst: 'IPv6Address | str | bytes | int' = '::', + payload_length: 'Optional[int]' = None, payload: 'bytes | ProtocolBase | Schema' = b'', **kwargs: 'Any') -> 'Schema_IPv6': """Make (construct) packet data. @@ -241,6 +242,8 @@ def make(self, hop_limit: Hop limit. src: Source IP address. dst: Destination IP address. + payload_length: Length of the payload, extension headers included; + computed from the payload when omitted. payload: Payload data. **kwargs: Arbitrary keyword arguments. @@ -251,13 +254,21 @@ def make(self, next_val = self._make_index(next, next_default, namespace=next_namespace, reversed=next_reversed, pack=False) + # NOTE: An explicit ``payload_length`` is written as given, so that a + # ``from_data`` rebuild of a capture truncated by snaplen keeps the + # length the sender declared instead of the one that was recorded + # (:issue:`1155`). Computed from the octets at hand otherwise, which is + # what a plain ``IPv6(payload=...)`` wants. + if payload_length is None: + payload_length = len(payload) + return Schema_IPv6( hextet={ 'version': 6, 'class': traffic_class, 'label': flow_label, }, - length=len(payload), + length=payload_length, next=next_val, # type: ignore[arg-type] limit=hop_limit, src=src, @@ -317,6 +328,11 @@ def _make_data(cls, data: 'Data_IPv6') -> 'dict[str, Any]': # type: ignore[over 'hop_limit': data.limit, 'src': data.src, 'dst': data.dst, + # NOTE: ``data.payload`` is the parsed payload length, passed through + # rather than left to ``make`` to recompute, so that a truncated + # capture rebuilds with the length that was on the wire + # (:issue:`1155`). + 'payload_length': data.payload, 'payload': cls._make_payload(data) } diff --git a/pcapkit/protocols/transport/udp.py b/pcapkit/protocols/transport/udp.py index f0e06e3941..af67c01610 100644 --- a/pcapkit/protocols/transport/udp.py +++ b/pcapkit/protocols/transport/udp.py @@ -179,6 +179,7 @@ def make(self, srcport: 'Enum_AppType | int' = 0, dstport: 'Enum_AppType | int' = 0, checksum: 'bytes' = b'\x00\x00', + total_length: 'Optional[int]' = None, payload: 'bytes | Schema | ProtocolBase' = b'', **kwargs: 'Any') -> 'Schema_UDP': """Make (construct) packet data. @@ -187,6 +188,8 @@ def make(self, srcport: Source port. dstport: Destination port. checksum: Checksum. + total_length: Length of the datagram, header included; computed from + the header and the payload when omitted. payload: Payload data. **kwargs: Arbitrary keyword arguments. @@ -194,10 +197,18 @@ def make(self, Constructed packet data. """ + # NOTE: An explicit ``total_length`` is written as given, so that a + # ``from_data`` rebuild of a capture truncated by snaplen keeps the + # length the sender declared instead of the one that was recorded + # (:issue:`1155`). Computed from the octets at hand otherwise, which is + # what a plain ``UDP(payload=...)`` wants. + if total_length is None: + total_length = 8 + len(payload) + return Schema_UDP( srcport=self._make_port(srcport, Enum_TransportProtocol.udp), dstport=self._make_port(dstport, Enum_TransportProtocol.udp), - len=8 + len(payload), + len=total_length, checksum=checksum, payload=payload, ) @@ -241,5 +252,9 @@ def _make_data(cls, data: 'Data_UDP') -> 'dict[str, Any]': # type: ignore[overr 'srcport': data.srcport, 'dstport': data.dstport, 'checksum': data.checksum, + # NOTE: The parsed length is passed through rather than left to + # ``make`` to recompute, so that a truncated capture rebuilds with + # the length that was on the wire (:issue:`1155`). + 'total_length': data.len, 'payload': cls._make_payload(data), } diff --git a/tests/protocols/internet/test_ipv4_unit.py b/tests/protocols/internet/test_ipv4_unit.py index a1c1292ae2..b84481a137 100644 --- a/tests/protocols/internet/test_ipv4_unit.py +++ b/tests/protocols/internet/test_ipv4_unit.py @@ -37,6 +37,7 @@ class DummyDict(dict): data = DummyDict( tos=DummyDict(pre=0, thr=False, rel=False, ecn=0, **{'del': False}), + len=1186, id=7, flags=DummyDict(df=True, mf=False), offset=0, @@ -50,6 +51,7 @@ class DummyDict(dict): ) values = IPv4._make_data(data) + self.assertEqual(values['total_length'], 1186) self.assertEqual(values['id'], 7) self.assertEqual(values['df'], True) self.assertEqual(values['mf'], False) diff --git a/tests/protocols/internet/test_ipv6_unit.py b/tests/protocols/internet/test_ipv6_unit.py index 23dfd9711a..fb05731816 100644 --- a/tests/protocols/internet/test_ipv6_unit.py +++ b/tests/protocols/internet/test_ipv6_unit.py @@ -41,12 +41,14 @@ def test_ipv6_make_data_preserves_selected_fields(self) -> None: 'label': 0x12345, 'next': TransType.TCP, 'limit': 32, + 'payload': 1146, 'src': ip_address('2001:db8::1'), 'dst': ip_address('2001:db8::2'), '__next_type__': None, }) values = IPv6._make_data(data) + self.assertEqual(values['payload_length'], 1146) self.assertEqual(values['traffic_class'], 0x2a) self.assertEqual(values['flow_label'], 0x12345) self.assertEqual(values['next'], TransType.TCP) diff --git a/tests/protocols/test_declared_length_roundtrip_runtime.py b/tests/protocols/test_declared_length_roundtrip_runtime.py new file mode 100644 index 0000000000..2d868af986 --- /dev/null +++ b/tests/protocols/test_declared_length_roundtrip_runtime.py @@ -0,0 +1,152 @@ +# -*- coding: utf-8 -*- +"""``from_data`` keeps the length a truncated capture declared. + +GitHub issue #1155. :meth:`IPv4.make `, +:meth:`UDP.make ` and :meth:`IPv6.make +` computed their length field from the +payload they were handed. On a capture cut short by the snaplen that is the +*recorded* length rather than the declared one, so a +:meth:`~pcapkit.protocols.protocol.ProtocolBase.from_data` rebuild of +:file:`big_endian.pcap` frame 3 rewrote the IPv4 total length from 1186 to 82 and +the UDP length from 0x048e to 0x003e. Each ``make`` now takes the length +explicitly, and ``_make_data`` hands it the parsed one; a construction that gives +no length still computes it. + +The round-trip cases read generated captures, so this module belongs to the +fixture-dependent tier. The in-memory cases pin that construction is unchanged. + +""" +from __future__ import annotations + +import importlib.util +import unittest + +from tests._support import close_extractor, reimport_once_per_class, sample_path + +RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') +HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) + +#: Captures whose last frame is truncated by the snaplen: 1186 octets declared, +#: 82 captured. +TRUNCATED_SAMPLES = ('big_endian.pcap', 'big_endian_nanosecond.pcap', 'little_endian.pcap') + +#: Captures every one of whose IPv4 and UDP packets is complete, so that +#: preserving the declared length cannot change what they rebuild to. +COMPLETE_SAMPLES = ('ipv4.pcap', 'dhcp.pcapng', 'options-ipv4.pcap') + + +@unittest.skipUnless(HAS_RUNTIME, 'runtime dependencies not installed') +class DeclaredLengthRoundTripTests(unittest.TestCase): + """A ``from_data`` rebuild reproduces the octets it was parsed from.""" + + def setUp(self) -> None: + reimport_once_per_class(self) + + def _frames(self, sample: str) -> 'list': + from pcapkit.interface import extract + + extractor = extract(fin=sample_path(sample), fout='/tmp/out', format='tree', + store=True, nofile=True) + self.addCleanup(close_extractor, extractor) + return list(extractor.frame) + + def assertRoundTrips(self, sample: str) -> None: + """Every IPv4 and UDP packet of ``sample`` rebuilds byte for byte.""" + from pcapkit.protocols.internet.ipv4 import IPv4 + from pcapkit.protocols.transport.udp import UDP + + seen = 0 + for index, frame in enumerate(self._frames(sample)): + for protocol in (IPv4, UDP): + if protocol not in frame: + continue + parsed = frame[protocol] + with self.subTest(sample=sample, frame=index, protocol=protocol.__name__): + self.assertEqual(protocol.from_data(parsed.info).data, parsed.data) + seen += 1 + self.assertGreater(seen, 0, f'{sample} carries no IPv4 or UDP packet') + + def test_truncated_frames_keep_their_declared_lengths(self) -> None: + for sample in TRUNCATED_SAMPLES: + frame = self._frames(sample)[2] + + from pcapkit.protocols.internet.ipv4 import IPv4 + from pcapkit.protocols.transport.udp import UDP + + with self.subTest(sample=sample): + # The frame as recorded: 82 of the 1186 declared octets. + self.assertEqual(frame[IPv4].info.len, 1186) + self.assertEqual(len(frame[IPv4].data), 82) + self.assertEqual(frame[UDP].info.len, 0x048e) + + rebuilt = IPv4.from_data(frame[IPv4].info) + self.assertEqual(rebuilt.info.len, 1186) + self.assertEqual(rebuilt.data, frame[IPv4].data) + + rebuilt_udp = UDP.from_data(frame[UDP].info) + self.assertEqual(rebuilt_udp.info.len, 0x048e) + self.assertEqual(rebuilt_udp.data, frame[UDP].data) + + def test_truncated_frames_round_trip(self) -> None: + for sample in TRUNCATED_SAMPLES: + self.assertRoundTrips(sample) + + def test_complete_frames_still_round_trip(self) -> None: + for sample in COMPLETE_SAMPLES: + self.assertRoundTrips(sample) + + def test_header_truncated_frame_keeps_its_total_length(self) -> None: + """``test.pcapng`` frame 3 is cut inside the IPv4 header itself. + + Only 18 of the 20 header octets were recorded, so the rebuild cannot be + byte for byte -- it writes a whole header. What it must not do is replace + the declared 300 with the 18 that were kept. + + """ + from pcapkit.protocols.internet.ipv4 import IPv4 + + parsed = self._frames('test.pcapng')[2][IPv4] + self.assertEqual(parsed.info.len, 300) + self.assertEqual(len(parsed.data), 18) + + rebuilt = IPv4.from_data(parsed.info) + self.assertEqual(rebuilt.info.len, 300) + self.assertEqual(rebuilt.data[:18], parsed.data) + + +@unittest.skipUnless(HAS_RUNTIME, 'runtime dependencies not installed') +class DeclaredLengthConstructionTests(unittest.TestCase): + """Construction reads no capture, and is unchanged by the override.""" + + def setUp(self) -> None: + reimport_once_per_class(self) + + def test_length_is_computed_when_not_given(self) -> None: + from pcapkit.protocols.internet.ipv4 import IPv4 + from pcapkit.protocols.internet.ipv6 import IPv6 + from pcapkit.protocols.transport.udp import UDP + + self.assertEqual(IPv4(payload=b'\x00' * 10).info.len, 30) + self.assertEqual(UDP(payload=b'\x00' * 10).info.len, 18) + self.assertEqual(IPv6(payload=b'\x00' * 10).info.payload, 10) + + def test_explicit_length_is_written_as_given(self) -> None: + from pcapkit.protocols.internet.ipv4 import IPv4 + from pcapkit.protocols.internet.ipv6 import IPv6 + from pcapkit.protocols.transport.udp import UDP + + ipv4 = IPv4(total_length=1186, payload=b'\x00' * 10) + self.assertEqual(ipv4.info.len, 1186) + self.assertEqual(ipv4.data[2:4], (1186).to_bytes(2, 'big')) + + udp = UDP(total_length=0x048e, payload=b'\x00' * 10) + self.assertEqual(udp.info.len, 0x048e) + self.assertEqual(udp.data[4:6], (0x048e).to_bytes(2, 'big')) + + ipv6 = IPv6(payload_length=1146, payload=b'\x00' * 10) + self.assertEqual(ipv6.info.payload, 1146) + self.assertEqual(ipv6.data[4:6], (1146).to_bytes(2, 'big')) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/protocols/transport/test_tcp_udp_unit.py b/tests/protocols/transport/test_tcp_udp_unit.py index 07efc13d81..70a95acea8 100644 --- a/tests/protocols/transport/test_tcp_udp_unit.py +++ b/tests/protocols/transport/test_tcp_udp_unit.py @@ -79,7 +79,8 @@ def test_udp_index_make_and_make_data_preserve_core_fields(self) -> None: udp = object.__new__(UDP) schema = UDP.make(udp, srcport=53, dstport=5353, checksum=b'\x12\x34', payload=b'data') - data = DummyData(srcport=53, dstport=5353, checksum=b'\xab\xcd', __next_type__=None) + data = DummyData(srcport=53, dstport=5353, len=0x048e, checksum=b'\xab\xcd', + __next_type__=None) self.assertEqual(UDP.__index__(), TransType.UDP) self.assertEqual(schema.srcport, 53) @@ -91,6 +92,7 @@ def test_udp_index_make_and_make_data_preserve_core_fields(self) -> None: self.assertEqual(values['srcport'], 53) self.assertEqual(values['dstport'], 5353) self.assertEqual(values['checksum'], b'\xab\xcd') + self.assertEqual(values['total_length'], 0x048e) self.assertIn('payload', values) udp._info = DummyData(srcport=AppType.get(53, proto=TransportProtocol.udp),