Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 19 additions & 5 deletions pcapkit/protocols/internet/ipv4.py
Original file line number Diff line number Diff line change
Expand Up @@ -366,6 +366,7 @@ def make(self,
src: 'IPv4Address | str | int | bytes' = '127.0.0.1',
dst: 'IPv4Address | str | int | bytes' = '0.0.0.0', # nosec: B104
options: 'Optional[list[Schema_Option | tuple[Enum_OptionNumber, dict[str, Any]] | bytes] | Option]' = None, # pylint: disable=line-too-long
total_length: 'Optional[int]' = None,
payload: 'bytes | ProtocolBase | Schema' = b'',
**kwargs: 'Any') -> 'Schema_IPv4':
"""Make (construct) packet data.
Expand Down Expand Up @@ -404,6 +405,8 @@ def make(self,
src: Source address of the packet.
dst: Destination address of the packet.
options: Options of the packet.
total_length: Total length of the packet, header included; computed
from the header and the payload when omitted.
payload: Payload of the packet.
**kwargs: Arbitrary keyword arguments.

Expand All @@ -427,12 +430,19 @@ def make(self,
ttl_val = ttl if isinstance(ttl, int) else math.ceil(ttl.total_seconds())

if options is not None:
options_value, total_length = self._make_ipv4_options(options)
options_value, options_length = self._make_ipv4_options(options)
else:
options_value, total_length = [], 0
options_value, options_length = [], 0

ihl = 5 + math.ceil(total_length / 4)
total_len = ihl * 4 + len(payload)
ihl = 5 + math.ceil(options_length / 4)

# NOTE: An explicit ``total_length`` is written as given, so that a
# ``from_data`` rebuild of a capture truncated by snaplen keeps the
# length the sender declared instead of the one that was recorded
# (:issue:`1155`). Computed from the octets at hand otherwise, which is
# what a plain ``IPv4(payload=...)`` wants.
if total_length is None:
total_length = ihl * 4 + len(payload)

return Schema_IPv4(
vihl={
Expand All @@ -446,7 +456,7 @@ def make(self,
'rel': tos_rel_val,
'ecn': tos_ecn_val,
},
length=total_len,
length=total_length,
id=id,
flags={
'df': df,
Expand Down Expand Up @@ -543,6 +553,10 @@ def _make_data(cls, data: 'Data_IPv4') -> 'dict[str, Any]': # type: ignore[over
# ``hdr_len`` exceeds the fixed 20-octet header (see :meth:`read`),
# so read it defensively rather than assuming it always exists.
'options': getattr(data, 'options', None),
# NOTE: The parsed total length is passed through rather than left
# to ``make`` to recompute, so that a truncated capture rebuilds
# with the length that was on the wire (:issue:`1155`).
'total_length': data.len,
'payload': cls._make_payload(data),
}

Expand Down
18 changes: 17 additions & 1 deletion pcapkit/protocols/internet/ipv6.py
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,7 @@ def make(self,
hop_limit: 'int' = 64, # reasonable default
src: 'IPv6Address | str | bytes | int' = '::1',
dst: 'IPv6Address | str | bytes | int' = '::',
payload_length: 'Optional[int]' = None,
payload: 'bytes | ProtocolBase | Schema' = b'',
**kwargs: 'Any') -> 'Schema_IPv6':
"""Make (construct) packet data.
Expand All @@ -241,6 +242,8 @@ def make(self,
hop_limit: Hop limit.
src: Source IP address.
dst: Destination IP address.
payload_length: Length of the payload, extension headers included;
computed from the payload when omitted.
payload: Payload data.
**kwargs: Arbitrary keyword arguments.

Expand All @@ -251,13 +254,21 @@ def make(self,
next_val = self._make_index(next, next_default, namespace=next_namespace,
reversed=next_reversed, pack=False)

# NOTE: An explicit ``payload_length`` is written as given, so that a
# ``from_data`` rebuild of a capture truncated by snaplen keeps the
# length the sender declared instead of the one that was recorded
# (:issue:`1155`). Computed from the octets at hand otherwise, which is
# what a plain ``IPv6(payload=...)`` wants.
if payload_length is None:
payload_length = len(payload)

return Schema_IPv6(
hextet={
'version': 6,
'class': traffic_class,
'label': flow_label,
},
length=len(payload),
length=payload_length,
next=next_val, # type: ignore[arg-type]
limit=hop_limit,
src=src,
Expand Down Expand Up @@ -317,6 +328,11 @@ def _make_data(cls, data: 'Data_IPv6') -> 'dict[str, Any]': # type: ignore[over
'hop_limit': data.limit,
'src': data.src,
'dst': data.dst,
# NOTE: ``data.payload`` is the parsed payload length, passed through
# rather than left to ``make`` to recompute, so that a truncated
# capture rebuilds with the length that was on the wire
# (:issue:`1155`).
'payload_length': data.payload,
'payload': cls._make_payload(data)
}

Expand Down
17 changes: 16 additions & 1 deletion pcapkit/protocols/transport/udp.py
Original file line number Diff line number Diff line change
Expand Up @@ -179,6 +179,7 @@ def make(self,
srcport: 'Enum_AppType | int' = 0,
dstport: 'Enum_AppType | int' = 0,
checksum: 'bytes' = b'\x00\x00',
total_length: 'Optional[int]' = None,
payload: 'bytes | Schema | ProtocolBase' = b'',
**kwargs: 'Any') -> 'Schema_UDP':
"""Make (construct) packet data.
Expand All @@ -187,17 +188,27 @@ def make(self,
srcport: Source port.
dstport: Destination port.
checksum: Checksum.
total_length: Length of the datagram, header included; computed from
the header and the payload when omitted.
payload: Payload data.
**kwargs: Arbitrary keyword arguments.

Returns:
Constructed packet data.

"""
# NOTE: An explicit ``total_length`` is written as given, so that a
# ``from_data`` rebuild of a capture truncated by snaplen keeps the
# length the sender declared instead of the one that was recorded
# (:issue:`1155`). Computed from the octets at hand otherwise, which is
# what a plain ``UDP(payload=...)`` wants.
if total_length is None:
total_length = 8 + len(payload)

return Schema_UDP(
srcport=self._make_port(srcport, Enum_TransportProtocol.udp),
dstport=self._make_port(dstport, Enum_TransportProtocol.udp),
len=8 + len(payload),
len=total_length,
checksum=checksum,
payload=payload,
)
Expand Down Expand Up @@ -241,5 +252,9 @@ def _make_data(cls, data: 'Data_UDP') -> 'dict[str, Any]': # type: ignore[overr
'srcport': data.srcport,
'dstport': data.dstport,
'checksum': data.checksum,
# NOTE: The parsed length is passed through rather than left to
# ``make`` to recompute, so that a truncated capture rebuilds with
# the length that was on the wire (:issue:`1155`).
'total_length': data.len,
'payload': cls._make_payload(data),
}
2 changes: 2 additions & 0 deletions tests/protocols/internet/test_ipv4_unit.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ class DummyDict(dict):

data = DummyDict(
tos=DummyDict(pre=0, thr=False, rel=False, ecn=0, **{'del': False}),
len=1186,
id=7,
flags=DummyDict(df=True, mf=False),
offset=0,
Expand All @@ -50,6 +51,7 @@ class DummyDict(dict):
)

values = IPv4._make_data(data)
self.assertEqual(values['total_length'], 1186)
self.assertEqual(values['id'], 7)
self.assertEqual(values['df'], True)
self.assertEqual(values['mf'], False)
Expand Down
2 changes: 2 additions & 0 deletions tests/protocols/internet/test_ipv6_unit.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,12 +41,14 @@ def test_ipv6_make_data_preserves_selected_fields(self) -> None:
'label': 0x12345,
'next': TransType.TCP,
'limit': 32,
'payload': 1146,
'src': ip_address('2001:db8::1'),
'dst': ip_address('2001:db8::2'),
'__next_type__': None,
})

values = IPv6._make_data(data)
self.assertEqual(values['payload_length'], 1146)
self.assertEqual(values['traffic_class'], 0x2a)
self.assertEqual(values['flow_label'], 0x12345)
self.assertEqual(values['next'], TransType.TCP)
Expand Down
152 changes: 152 additions & 0 deletions tests/protocols/test_declared_length_roundtrip_runtime.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,152 @@
# -*- coding: utf-8 -*-
"""``from_data`` keeps the length a truncated capture declared.

GitHub issue #1155. :meth:`IPv4.make <pcapkit.protocols.internet.ipv4.IPv4.make>`,
:meth:`UDP.make <pcapkit.protocols.transport.udp.UDP.make>` and :meth:`IPv6.make
<pcapkit.protocols.internet.ipv6.IPv6.make>` computed their length field from the
payload they were handed. On a capture cut short by the snaplen that is the
*recorded* length rather than the declared one, so a
:meth:`~pcapkit.protocols.protocol.ProtocolBase.from_data` rebuild of
:file:`big_endian.pcap` frame 3 rewrote the IPv4 total length from 1186 to 82 and
the UDP length from 0x048e to 0x003e. Each ``make`` now takes the length
explicitly, and ``_make_data`` hands it the parsed one; a construction that gives
no length still computes it.

The round-trip cases read generated captures, so this module belongs to the
fixture-dependent tier. The in-memory cases pin that construction is unchanged.

"""
from __future__ import annotations

import importlib.util
import unittest

from tests._support import close_extractor, reimport_once_per_class, sample_path

RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper')
HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS)

#: Captures whose last frame is truncated by the snaplen: 1186 octets declared,
#: 82 captured.
TRUNCATED_SAMPLES = ('big_endian.pcap', 'big_endian_nanosecond.pcap', 'little_endian.pcap')

#: Captures every one of whose IPv4 and UDP packets is complete, so that
#: preserving the declared length cannot change what they rebuild to.
COMPLETE_SAMPLES = ('ipv4.pcap', 'dhcp.pcapng', 'options-ipv4.pcap')


@unittest.skipUnless(HAS_RUNTIME, 'runtime dependencies not installed')
class DeclaredLengthRoundTripTests(unittest.TestCase):
"""A ``from_data`` rebuild reproduces the octets it was parsed from."""

def setUp(self) -> None:
reimport_once_per_class(self)

def _frames(self, sample: str) -> 'list':
from pcapkit.interface import extract

extractor = extract(fin=sample_path(sample), fout='/tmp/out', format='tree',
store=True, nofile=True)
self.addCleanup(close_extractor, extractor)
return list(extractor.frame)

def assertRoundTrips(self, sample: str) -> None:
"""Every IPv4 and UDP packet of ``sample`` rebuilds byte for byte."""
from pcapkit.protocols.internet.ipv4 import IPv4
from pcapkit.protocols.transport.udp import UDP

seen = 0
for index, frame in enumerate(self._frames(sample)):
for protocol in (IPv4, UDP):
if protocol not in frame:
continue
parsed = frame[protocol]
with self.subTest(sample=sample, frame=index, protocol=protocol.__name__):
self.assertEqual(protocol.from_data(parsed.info).data, parsed.data)
seen += 1
self.assertGreater(seen, 0, f'{sample} carries no IPv4 or UDP packet')

def test_truncated_frames_keep_their_declared_lengths(self) -> None:
for sample in TRUNCATED_SAMPLES:
frame = self._frames(sample)[2]

from pcapkit.protocols.internet.ipv4 import IPv4
from pcapkit.protocols.transport.udp import UDP

with self.subTest(sample=sample):
# The frame as recorded: 82 of the 1186 declared octets.
self.assertEqual(frame[IPv4].info.len, 1186)
self.assertEqual(len(frame[IPv4].data), 82)
self.assertEqual(frame[UDP].info.len, 0x048e)

rebuilt = IPv4.from_data(frame[IPv4].info)
self.assertEqual(rebuilt.info.len, 1186)
self.assertEqual(rebuilt.data, frame[IPv4].data)

rebuilt_udp = UDP.from_data(frame[UDP].info)
self.assertEqual(rebuilt_udp.info.len, 0x048e)
self.assertEqual(rebuilt_udp.data, frame[UDP].data)

def test_truncated_frames_round_trip(self) -> None:
for sample in TRUNCATED_SAMPLES:
self.assertRoundTrips(sample)

def test_complete_frames_still_round_trip(self) -> None:
for sample in COMPLETE_SAMPLES:
self.assertRoundTrips(sample)

def test_header_truncated_frame_keeps_its_total_length(self) -> None:
"""``test.pcapng`` frame 3 is cut inside the IPv4 header itself.

Only 18 of the 20 header octets were recorded, so the rebuild cannot be
byte for byte -- it writes a whole header. What it must not do is replace
the declared 300 with the 18 that were kept.

"""
from pcapkit.protocols.internet.ipv4 import IPv4

parsed = self._frames('test.pcapng')[2][IPv4]
self.assertEqual(parsed.info.len, 300)
self.assertEqual(len(parsed.data), 18)

rebuilt = IPv4.from_data(parsed.info)
self.assertEqual(rebuilt.info.len, 300)
self.assertEqual(rebuilt.data[:18], parsed.data)


@unittest.skipUnless(HAS_RUNTIME, 'runtime dependencies not installed')
class DeclaredLengthConstructionTests(unittest.TestCase):
"""Construction reads no capture, and is unchanged by the override."""

def setUp(self) -> None:
reimport_once_per_class(self)

def test_length_is_computed_when_not_given(self) -> None:
from pcapkit.protocols.internet.ipv4 import IPv4
from pcapkit.protocols.internet.ipv6 import IPv6
from pcapkit.protocols.transport.udp import UDP

self.assertEqual(IPv4(payload=b'\x00' * 10).info.len, 30)
self.assertEqual(UDP(payload=b'\x00' * 10).info.len, 18)
self.assertEqual(IPv6(payload=b'\x00' * 10).info.payload, 10)

def test_explicit_length_is_written_as_given(self) -> None:
from pcapkit.protocols.internet.ipv4 import IPv4
from pcapkit.protocols.internet.ipv6 import IPv6
from pcapkit.protocols.transport.udp import UDP

ipv4 = IPv4(total_length=1186, payload=b'\x00' * 10)
self.assertEqual(ipv4.info.len, 1186)
self.assertEqual(ipv4.data[2:4], (1186).to_bytes(2, 'big'))

udp = UDP(total_length=0x048e, payload=b'\x00' * 10)
self.assertEqual(udp.info.len, 0x048e)
self.assertEqual(udp.data[4:6], (0x048e).to_bytes(2, 'big'))

ipv6 = IPv6(payload_length=1146, payload=b'\x00' * 10)
self.assertEqual(ipv6.info.payload, 1146)
self.assertEqual(ipv6.data[4:6], (1146).to_bytes(2, 'big'))


if __name__ == '__main__':
unittest.main()
4 changes: 3 additions & 1 deletion tests/protocols/transport/test_tcp_udp_unit.py
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,8 @@ def test_udp_index_make_and_make_data_preserve_core_fields(self) -> None:

udp = object.__new__(UDP)
schema = UDP.make(udp, srcport=53, dstport=5353, checksum=b'\x12\x34', payload=b'data')
data = DummyData(srcport=53, dstport=5353, checksum=b'\xab\xcd', __next_type__=None)
data = DummyData(srcport=53, dstport=5353, len=0x048e, checksum=b'\xab\xcd',
__next_type__=None)

self.assertEqual(UDP.__index__(), TransType.UDP)
self.assertEqual(schema.srcport, 53)
Expand All @@ -91,6 +92,7 @@ def test_udp_index_make_and_make_data_preserve_core_fields(self) -> None:
self.assertEqual(values['srcport'], 53)
self.assertEqual(values['dstport'], 5353)
self.assertEqual(values['checksum'], b'\xab\xcd')
self.assertEqual(values['total_length'], 0x048e)
self.assertIn('payload', values)

udp._info = DummyData(srcport=AppType.get(53, proto=TransportProtocol.udp),
Expand Down
Loading