From 2ade9f4b3b92286c8732651a0f2a48cdf05b1aa9 Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Mon, 2 Nov 2026 21:15:58 -0500 Subject: [PATCH] fix(protocols): report a rebuilt frame's messages once and record cuts on the layer (#2711) - Rebuilding an outer layer rebuilt its payload by from_data, whose own parse reported what the octets carry, and then parsed them again itself, so Ethernet over a cut IGMPv3 reported its messages twice. Under rebuilding, ProtocolBase.__post_init__ drops the payload's parse warnings with hold_warnings, and beholder builds the errors it absorbs quiet through CaughtByCaller and logs none; the enclosing layer's parse reports them once. The owner ruled on #2711 that this carve-out from the handled-refusals rule stands, as protocol-family-grouping.rst now records. - An IGMPv3 Report parsed on its own and cut on a field boundary before its counted records read INVALID. ListField now takes a count: a list whose octets hold fewer items than it counts reports the shortfall, and Schema.unpack records it as a short read, the items absent never made up; the fields after it find the record in packet, as #2727 gives them. Report.records counts num_group_records for a message parsed on its own, so such a cut reads TRUNCATED whatever the context parsed before; inside a frame the count sizes nothing. A count no IP payload could carry raises ProtocolError. - A message cut inside the length its context gives it read TRUNCATED in its frame but INVALID rebuilt on its own. Schema.unpack records a cut in a payload of a declared length; keep_short_read records a frame cut on a layer only where Schema._frame_end_field finds its last field sized by what is left of the length it was given, so a layer whose header declares its end (IPv4, IPv6 and its extension headers, UDP, an HTTP/2 frame) records none at or past it, nor one ending in a list, as SCTP's chunks, whose buffer holds only what its items took. The TSO IPv4 payload is sized that way, and ICMPv4 reads a recorded cut as TRUNCATED, as GRE and IGMP do. - Tests that pinned no record for those cuts now pin the record. The new modules fail without the fix; the protocols, corekit, foundation, utilities and project modules pass. Sample captures give the same octets and messages; cut frames gain __short_read__ on Ethernet, IPv4 and UDP. Closes #2711 --- .../conventions/protocol-family-grouping.rst | 5 +- docs/source/contributing/pep.rst | 5 +- pcapkit/corekit/decorators.py | 19 +- pcapkit/corekit/fields/collections.py | 39 +- pcapkit/corekit/schema/shortread.py | 18 + pcapkit/protocols/internet/icmpv4.py | 16 +- pcapkit/protocols/link/gre.py | 3 +- pcapkit/protocols/protocol.py | 29 +- pcapkit/protocols/schema/internet/igmp.py | 6 +- pcapkit/protocols/schema/internet/igmpv3.py | 43 ++- pcapkit/protocols/schema/internet/ipv4.py | 6 +- pcapkit/protocols/schema/schema.py | 62 +++- pcapkit/utilities/contextvars.py | 3 +- .../test_list_count_shortfall_2711_unit.py | 88 +++++ .../test_igmp_header_cut_2689_unit.py | 4 +- tests/protocols/internet/test_igmp_unit.py | 11 +- .../test_igmpv3_report_count_cut_2711_unit.py | 256 +++++++++++++ .../test_ipv4_tso_cut_checksum_2689_unit.py | 7 +- ...est_cut_recorded_on_the_layer_2711_unit.py | 338 ++++++++++++++++++ .../test_rebuild_reports_once_2711_unit.py | 197 ++++++++++ ...test_short_read_rebuild_quiet_2703_unit.py | 6 +- 21 files changed, 1127 insertions(+), 34 deletions(-) create mode 100644 tests/corekit/test_list_count_shortfall_2711_unit.py create mode 100644 tests/protocols/internet/test_igmpv3_report_count_cut_2711_unit.py create mode 100644 tests/protocols/test_cut_recorded_on_the_layer_2711_unit.py create mode 100644 tests/protocols/test_rebuild_reports_once_2711_unit.py diff --git a/docs/source/contributing/conventions/protocol-family-grouping.rst b/docs/source/contributing/conventions/protocol-family-grouping.rst index 81ae964559..29bf136682 100644 --- a/docs/source/contributing/conventions/protocol-family-grouping.rst +++ b/docs/source/contributing/conventions/protocol-family-grouping.rst @@ -229,7 +229,10 @@ What stays loud: mode, installs the hooks. * **Any other error class.** An error the fallback absorbs is as loud as it was constructed, and :func:`~pcapkit.corekit.decorators.beholder` logs it at - :data:`logging.WARNING` as well. + :data:`logging.WARNING` as well. Inside a payload + :meth:`Protocol._make_payload ` + rebuilds, it is quiet, as the owner ruled on :issue:`2711`: the layer the payload + is rebuilt for parses the same octets again and reports it there, once. * **A refusal built under an inner** ``CaughtByCaller()``. A block naming no class, such as the one :meth:`~pcapkit.corekit.enum.EnumRegistry.register` enters, makes errors loud again inside the fallback's. diff --git a/docs/source/contributing/pep.rst b/docs/source/contributing/pep.rst index b7c595e61b..6547b20853 100644 --- a/docs/source/contributing/pep.rst +++ b/docs/source/contributing/pep.rst @@ -969,7 +969,10 @@ offload heuristic on :issue:`2470`. ICMP and IGMP go by the original length as well (:issue:`2689`). A GRE or IGMP header the data ends inside is :attr:`~pcapkit.corekit.checksum.internet.ChecksumStatus.TRUNCATED` too - (:issue:`2689`). + (:issue:`2689`). Each of these cuts is recorded on the message's own header, so + the message rebuilt on its own reads + :attr:`~pcapkit.corekit.checksum.internet.ChecksumStatus.TRUNCATED` as well, an + ICMP one included (:issue:`2711`). * **DCCP covers what its Checksum Coverage gives** (:rfc:`4340#section-9.2`), over a pseudo-header that carries the whole packet's length, as TCP's does, so a fragment is diff --git a/pcapkit/corekit/decorators.py b/pcapkit/corekit/decorators.py index 4fb55af3c3..0f212a25e4 100644 --- a/pcapkit/corekit/decorators.py +++ b/pcapkit/corekit/decorators.py @@ -30,7 +30,7 @@ import traceback from typing import TYPE_CHECKING, cast -from pcapkit.utilities.contextvars import beholding +from pcapkit.utilities.contextvars import beholding, rebuilding from pcapkit.utilities.exceptions import (CaughtByCaller, ProtocolError, StreamEOFError, StructError, stacklevel) from pcapkit.utilities.logging import DEVMODE, VERBOSE, get_logger @@ -124,6 +124,13 @@ def beholder( exception is logged here at :data:`logging.WARNING`, and is as loud as it was constructed. + Inside a payload :meth:`Protocol._make_payload + ` rebuilds, the block + names every :exc:`Exception`, which is what the ``except`` below catches, + and nothing is logged here: the layer the payload is rebuilt for parses + the same octets again, outside the rebuild, and its own :func:`beholder` + reports the error there (:issue:`2711`). + The recovery runs after the block has ended. Under the outermost :func:`beholder` of a parse, a :exc:`~pcapkit.utilities.exceptions.ProtocolError` it raises therefore @@ -155,7 +162,8 @@ def beholder( A :exc:`~pcapkit.utilities.exceptions.ProtocolError` the decorated function builds is quiet, and one this decorator absorbs is not logged - here either (:issue:`2588`). + here either (:issue:`2588`). Inside a payload being rebuilt, so is every + other error (:issue:`2711`). :param func: decorated function :meta decorator: @@ -173,13 +181,16 @@ def behold(*args: 'P.args', **kwargs: 'P.kwargs') -> 'R_beholder': except IndexError: length = None + # NOTE: A rebuilt payload's parse is repeated by the layer it is rebuilt + # for, which reports what this one would (#2711). + rebuilt = rebuilding.get() try: # call method token = beholding.set(True) try: # NOTE: The whole call is quiet for a ProtocolError, which the ``except`` below # records; the recovery runs after it, loud unless an outer block is active (#2588). - with CaughtByCaller(ProtocolError): + with CaughtByCaller(Exception if rebuilt else ProtocolError): return func(*args, **kwargs) finally: beholding.reset(token) @@ -192,7 +203,7 @@ def behold(*args: 'P.args', **kwargs: 'P.kwargs') -> 'R_beholder': # NOTE: Keyed on the class, not on quietness: a quiet end-of-file StructError is still # logged here, and a ProtocolError built loud has logged at CRITICAL already (#2588). - if not isinstance(exc, ProtocolError): + if not isinstance(exc, ProtocolError) and not rebuilt: logger.warning(str(exc), stack_info=DEVMODE, stacklevel=stacklevel()) if VERBOSE: diff --git a/pcapkit/corekit/fields/collections.py b/pcapkit/corekit/fields/collections.py index 0d0a787a63..5f9fde4e2f 100644 --- a/pcapkit/corekit/fields/collections.py +++ b/pcapkit/corekit/fields/collections.py @@ -76,10 +76,23 @@ class ListField(FieldBase[List[_TL]], Generic[_TL]): item_type: Field type of the contained items. callback: Callback function to be called upon :meth:`self.__call__ `. + count: Number of items a field before this one counts, for a list + running to the end of the data; if a callable is given, it should + return an integer value and accept the current packet as its only + argument. A negative value, the default, counts none. This field is used to represent a list of fields, as in the case of lists of constant-length-field items in a protocol. + A list whose octets hold fewer items than ``count`` ended with the data + before the rest, and reports them as :attr:`shortfall`, so that + :meth:`Schema.unpack ` + records the cut; the items absent are not made up. + + .. versionchanged:: 1.6.0 + + Takes ``count`` (:issue:`2711`). + """ @property @@ -92,9 +105,20 @@ def optional(self) -> 'bool': """Field is optional.""" return True + @property + def count(self) -> 'int': + """Number of items counted; negative where none are.""" + return self._count + + @property + def shortfall(self) -> 'int': + """Number of the items counted that the octets last unpacked did not hold.""" + return self._shortfall + def __init__(self, length: 'int | Callable[[dict[str, Any]], int]' = lambda _: -1, item_type: 'Optional[FieldBase]' = None, - callback: 'Callable[[Self, dict[str, Any]], None]' = lambda *_: None) -> 'None': + callback: 'Callable[[Self, dict[str, Any]], None]' = lambda *_: None, + count: 'int | Callable[[dict[str, Any]], int]' = -1) -> 'None': #self._name = '' if not hasattr(self, '_name'): self._name = f'<{type(self).__name__[:-5].lower()}>' @@ -108,6 +132,12 @@ def __init__(self, length: 'int | Callable[[dict[str, Any]], int]' = lambda _: - self._length = length self._template = '0s' + self._count_callback = None + if not isinstance(count, int): + self._count_callback, count = count, -1 + self._count = count + self._shortfall = 0 + def __call__(self, packet: 'dict[str, Any]') -> 'Self': """Update field attributes. @@ -126,6 +156,8 @@ def __call__(self, packet: 'dict[str, Any]') -> 'Self': if new_self._length_callback is not None: new_self._length = new_self._length_callback(packet) new_self._template = f'{new_self._length}s' + if new_self._count_callback is not None: + new_self._count = new_self._count_callback(packet) return new_self def pack(self, value: 'Optional[list[_TL]]', packet: 'dict[str, Any]') -> 'bytes': @@ -306,6 +338,11 @@ def _unpack_items(self, file: 'IO[bytes]', packet: 'dict[str, Any]', is_schema: data = field.unpack(buffer, packet) temp.append(data) + + # NOTE: Items are read to the end of the octets, so those counted past them are + # not there to read: the data ended first, and the field reads short (#2711). + if self._count >= 0: + self._shortfall = max(self._count - len(temp), 0) return temp diff --git a/pcapkit/corekit/schema/shortread.py b/pcapkit/corekit/schema/shortread.py index b7b26ef9f4..4bbe542e07 100644 --- a/pcapkit/corekit/schema/shortread.py +++ b/pcapkit/corekit/schema/shortread.py @@ -22,6 +22,8 @@ import io from typing import TYPE_CHECKING, cast +from pcapkit.utilities.contextvars import frame_length + if TYPE_CHECKING: from typing import Any, Optional, TypeVar @@ -53,10 +55,26 @@ def keep_short_read(protocol: 'ProtocolBase') -> 'None': and ``from_data(info.to_dict())`` cuts its rebuild back too; it is absent for a header that was captured whole. + Where the capture cut the frame short, as + :data:`~pcapkit.utilities.contextvars.frame_length` gives its lengths, a + layer whose message runs to the end of the frame records the cut too, on the + field it runs to the end in, through :meth:`Schema._note_short_read + `, so its rebuild + reads as cut, as the layer did inside the frame (:issue:`2711`). A layer + whose own header declares where its message ends records none, whether the + cut falls at that end or past it; see :meth:`Schema._frame_end_field + `. + """ header = cast('Optional[Schema]', getattr(protocol, '__header__', None)) if header is None: return + lengths = frame_length.get() + if lengths is not None and lengths[1] > lengths[0]: + name = header._frame_end_field() # pylint: disable=protected-access + if name is not None: + octets = header.__buffer__[name] + header._note_short_read(name, len(octets) + lengths[1] - lengths[0], octets) # pylint: disable=protected-access record = header.__dict__.get('__short_read__') if record is not None: protocol._info.__update__({'__short_read__': record}) # pylint: disable=protected-access diff --git a/pcapkit/protocols/internet/icmpv4.py b/pcapkit/protocols/internet/icmpv4.py index dc693f3ad4..3545a50ce5 100644 --- a/pcapkit/protocols/internet/icmpv4.py +++ b/pcapkit/protocols/internet/icmpv4.py @@ -97,7 +97,10 @@ class ICMPv4(ICMP[Data_ICMPv4, Schema_ICMPv4], The checksum is computed when :meth:`make` is given none, and verified on read under :data:`~pcapkit.utilities.config.CHECKSUM_VERIFY`, - unless the enclosing IP layer is a fragment (:issue:`2470`). + unless the enclosing IP layer is a fragment (:issue:`2470`). A message + whose header records a cut reads + :attr:`~pcapkit.corekit.checksum.internet.ChecksumStatus.TRUNCATED` + (:issue:`2711`). """ @@ -202,9 +205,16 @@ def read(self, length: 'Optional[int]' = None, *, # NOTE: The checksum covers the whole message, with no pseudo-header (RFC 792); # an IP fragment carries only part of one. if config.CHECKSUM_VERIFY: - from pcapkit.corekit.checksum.internet import verify_message # isort: skip # pylint: disable=import-outside-toplevel + from pcapkit.corekit.checksum.internet import ( # isort: skip # pylint: disable=import-outside-toplevel + ChecksumStatus, verify_message) + # NOTE: A message whose header records a cut leaves part of what the + # checksum covers uncaptured, as for GRE and IGMP (#2711). + if '__short_read__' in schema.__dict__: + status = ChecksumStatus.TRUNCATED + else: + status = verify_message(self.alias, self._data, schema.checksum, length=length) icmp.__update__([ - ('checksum_status', verify_message(self.alias, self._data, schema.checksum, length=length)), + ('checksum_status', status), ]) # NOTE: The quoted original datagram stays raw in the message body, diff --git a/pcapkit/protocols/link/gre.py b/pcapkit/protocols/link/gre.py index 1f3a26ad02..50bdb78a9d 100644 --- a/pcapkit/protocols/link/gre.py +++ b/pcapkit/protocols/link/gre.py @@ -111,7 +111,8 @@ class GRE(Link[Data_GRE, Schema_GRE], as :attr:`~pcapkit.corekit.checksum.internet.ChecksumStatus.TRUNCATED`, as :data:`~pcapkit.utilities.contextvars.frame_length` tells it (:issue:`2653`), and so does a header the data ends inside - (:issue:`2689`). + (:issue:`2689`). The header records such a cut, so the message rebuilt on + its own reads as cut too (:issue:`2711`). """ diff --git a/pcapkit/protocols/protocol.py b/pcapkit/protocols/protocol.py index d981460ae7..8df4ed4473 100644 --- a/pcapkit/protocols/protocol.py +++ b/pcapkit/protocols/protocol.py @@ -1424,6 +1424,13 @@ def __post_init__(self, file: 'Optional[IO[bytes] | bytes]' = None, For construction arguments, please refer to :meth:`self.make `. + Note: + Inside a payload :meth:`_make_payload` rebuilds, the warnings of the + parse are dropped, as the layer the payload is rebuilt for parses + the same octets again and reports them there (:issue:`2711`). Those + of a parse that raises are reported as the error leaves, unless + :func:`~pcapkit.corekit.decorators.beholder` replaces the layer. + """ if file is None: _data = self.pack(**kwargs) @@ -1439,8 +1446,20 @@ def __post_init__(self, file: 'Optional[IO[bytes] | bytes]' = None, self._data = _data #: io.BytesIO: Source packet stream. self._file = io.BytesIO(self._data) - #: pcapkit.protocols.data.data.Data: Parsed packet data. - self._info = self.unpack(length, **kwargs) + if not rebuilding.get(): + #: pcapkit.protocols.data.data.Data: Parsed packet data. + self._info = self.unpack(length, **kwargs) + return + + # NOTE: The enclosing layer parses these octets again and reports what + # they carry; as in ``_parse_next_layer``, a raise keeps them (#2711). + with hold_warnings() as held: + try: + self._info = self.unpack(length, **kwargs) + except Exception: + held.discard = beholding.get() + raise + held.discard = True def __init_subclass__(cls, /, schema: 'Optional[Type[_ST]]' = None, data: 'Optional[Type[_PT]]' = None, @@ -2561,8 +2580,10 @@ def _make_payload(cls, data: 'Data') -> 'ProtocolBase': parses its own octets again. So rebuilding a frame stops at :data:`~pcapkit.utilities.config.FRAME_LAYER_LIMIT` where parsing it did, rather than that many layers below each layer of it (:issue:`1610`). Nor does the payload's - parse warn of the limit: this protocol's own parse goes over the same - octets, so the rebuilt frame warns of it once, as the parsed one did. + parse report anything, the limit included: this protocol's own parse + goes over the same octets, so the rebuilt frame reports each warning, + and each error a next-layer fallback absorbs, once, as the parsed one + did (:issue:`2711`). Args: data: protocol data diff --git a/pcapkit/protocols/schema/internet/igmp.py b/pcapkit/protocols/schema/internet/igmp.py index 4313171458..37a1d0ba5c 100644 --- a/pcapkit/protocols/schema/internet/igmp.py +++ b/pcapkit/protocols/schema/internet/igmp.py @@ -43,8 +43,10 @@ class IGMP(Schema): #: Group address. group: 'IPv4Address' = IPv4AddressField() #: Payload, empty for a standards-conformant IGMPv1/v2 message, since the - #: 8-octet header is the whole message. - payload: 'bytes' = PayloadField() + #: 8-octet header is the whole message. It is the rest of the IP payload, + #: which the checksum covers too, so a cut inside it is recorded + #: (:issue:`2711`). + payload: 'bytes' = PayloadField(length=lambda pkt: pkt['__length__']) if TYPE_CHECKING: def __init__(self, type: 'Enum_Type', max_resp_time: 'int', checksum: 'bytes', # pylint: disable=unused-argument,super-init-not-called,multiple-statements,redefined-builtin diff --git a/pcapkit/protocols/schema/internet/igmpv3.py b/pcapkit/protocols/schema/internet/igmpv3.py index 637b6cfaf1..c2c007cdf1 100644 --- a/pcapkit/protocols/schema/internet/igmpv3.py +++ b/pcapkit/protocols/schema/internet/igmpv3.py @@ -11,7 +11,9 @@ from pcapkit.corekit.fields.misc import ConditionalField, PayloadField, SchemaField, SwitchField from pcapkit.corekit.fields.numbers import EnumField, UInt8Field, UInt16Field from pcapkit.corekit.fields.strings import BitField, BytesField +from pcapkit.protocols.schema.internet.ipv6 import IP_MAX_LENGTH from pcapkit.protocols.schema.schema import Schema, schema_final +from pcapkit.utilities.contextvars import frame_depth from pcapkit.utilities.exceptions import ProtocolError __all__ = ['IGMPv3', 'Query', 'QueryExtension', 'Report', 'GroupRecord'] @@ -90,6 +92,43 @@ def query_extension_present(pkt: 'dict[str, Any]') -> 'bool': return pkt['__length__'] >= 4 +def report_record_count(pkt: 'dict[str, Any]') -> 'int': + """Count callback for :attr:`Report.records`. + + Args: + pkt: Packet data. + + Returns: + :attr:`Report.num_group_records`, for a message parsed on its own; + otherwise ``-1``, counting none: inside a frame, the layer before the + message or the frame's own lengths say where it ends, and on the + packing path ``'records'`` is already in ``pkt``, as no count is + without :attr:`Report.num_group_records`. + + No layer before a message parsed on its own gives its length, and none + tells a cut from the end of the message, so the count does: records it + counts past the data are a cut, however the cut falls, on a field boundary + too, and :meth:`Schema.unpack + ` records it, the records + present alone being read (:issue:`2711`). + + Raises: + ProtocolError: If the records counted, at the 8 fixed octets of each, + behind the 8 octets ahead of them, reach past the + :data:`~pcapkit.protocols.schema.internet.ipv6.IP_MAX_LENGTH` octets + an IP payload can hold, so the count is no cut but one the message + cannot carry (:issue:`2711`). + + """ + count = pkt.get('num_group_records') # type: Optional[int] + if count is None or 'records' in pkt or frame_depth.get(): + return -1 + # NOTE: The 8 octets ahead of the records count against the IP payload too. + if 8 + 8 * count > IP_MAX_LENGTH: + raise ProtocolError(f'IGMPv3: {count} group records cannot fit in one IP payload') + return count + + @schema_final class QueryExtension(Schema): """Header schema for the IGMPv3 Query's source-list extension. @@ -230,7 +269,8 @@ class Report(Schema): :class:`~pcapkit.protocols.schema.internet.hip.LocatorSetParameter`). :meth:`post_process` keeps only the first :attr:`num_group_records` records and gives the rest to :attr:`payload`, which is - :rfc:`3376#section-4.2.11`'s "Additional Data". + :rfc:`3376#section-4.2.11`'s "Additional Data". Where the data ends + before the records counted, see :func:`report_record_count`. """ @@ -247,6 +287,7 @@ class Report(Schema): records: 'list[GroupRecord]' = ListField( length=lambda pkt: pkt['__length__'], item_type=SchemaField(schema=GroupRecord), + count=report_record_count, ) #: Payload; empty for a standards-conformant message with #: :attr:`num_group_records` matching the octets actually present. diff --git a/pcapkit/protocols/schema/internet/ipv4.py b/pcapkit/protocols/schema/internet/ipv4.py index 1d5e3ddeff..93b8f3c89d 100644 --- a/pcapkit/protocols/schema/internet/ipv4.py +++ b/pcapkit/protocols/schema/internet/ipv4.py @@ -755,9 +755,11 @@ class IPv4(Schema): #: Payload. A Total Length of 0, as TCP segmentation offload leaves it in a #: capture taken on the sending host, takes the rest of the data and leaves #: no trailer; any other Total Length shorter than the header leaves the - #: payload empty and the rest to the trailer (:issue:`1547`). + #: payload empty and the rest to the trailer (:issue:`1547`). The rest is + #: sized as what is left of the length given, so a frame the capture cut + #: records its cut on it (:issue:`2711`). payload: 'bytes' = PayloadField(length=lambda pkt: ( - -1 if pkt['length'] == 0 else max(pkt['length'] - pkt['vihl']['ihl'] * 4, 0))) + pkt['__length__'] if pkt['length'] == 0 else max(pkt['length'] - pkt['vihl']['ihl'] * 4, 0))) #: Octets captured past the Total Length, such as Ethernet minimum-frame #: padding. A :class:`~pcapkit.corekit.fields.misc.PayloadField` with no #: length takes whatever the payload leaves on unpack and writes its value diff --git a/pcapkit/protocols/schema/schema.py b/pcapkit/protocols/schema/schema.py index 2d21a96a1f..7b03f81a45 100644 --- a/pcapkit/protocols/schema/schema.py +++ b/pcapkit/protocols/schema/schema.py @@ -738,6 +738,17 @@ def unpack(cls, data: 'bytes | IO[bytes]', remainder itself rather than the same number of octets from beyond it, and record a cut inside it (:issue:`2685`). + A payload field of a declared length the data ends inside is a cut + like any other field's, so a layer whose octets fall short of the + length it declares records it, and its rebuild reads as cut + (:issue:`2711`). One declaring no length takes what the message + leaves, as a trailer or the next record does, which is no part of + the layer, and records nothing. A + :class:`~pcapkit.corekit.fields.collections.ListField` whose octets + hold fewer items than it counts records the cut too, the items + absent being its + :attr:`~pcapkit.corekit.fields.collections.ListField.shortfall`. + The fields after the one the data ends inside find it in ``packet`` as ``__short_read__``: the ``(name, octets read)`` pair recorded as the short read, or that of a @@ -775,7 +786,7 @@ def unpack(cls, data: 'bytes | IO[bytes]', # the declared length leaves, never less than nothing; a # computed zero is an empty payload, not an unset one, or it # would swallow the fields after it (#1275). - length = field.length + length = declared = field.length if length < 0: length = max(cast('int', packet['__length__']), 0) @@ -786,6 +797,10 @@ def unpack(cls, data: 'bytes | IO[bytes]', # that, so the bounded read is the same short read (#1975). payload = data.read(min(length, sys.maxsize)) self.__buffer__[field.name] = payload + # NOTE: A payload of a declared length the data ends inside is recorded as + # any field is; one taking what is left, as a trailer does, is not (#2711). + if declared >= 0: + self._note_short_read(field.name, length, payload, packet) packet['__length__'] -= length packet[field.name] = payload @@ -823,6 +838,10 @@ def unpack(cls, data: 'bytes | IO[bytes]', self._note_short_read(field.name, length, byte, packet) value = field.unpack(byte, packet.copy()) + if isinstance(field, ListField) and field.shortfall > 0: + # NOTE: The data ended before the items the list counts, each at least an + # octet, so it declares that many past those read (#2711). + self._note_short_read(field.name, len(byte) + field.shortfall, byte, packet) if isinstance(field, SchemaField) and isinstance(value, Schema): # NOTE: Kept on the nested schema rather than only in # ``__buffer__``, so that the octets it left unread go wherever @@ -1256,6 +1275,47 @@ def _cut_short_read(self) -> 'None': self.__buffer__[key] = self.__buffer__[key][:count] cut = True + def _frame_end_field(self) -> 'Optional[str]': + """The field this header's message runs to the end of its frame in, if any. + + Returns: + The last field sized by what is left of the length the layer was + given, as :class:`~pcapkit.protocols.schema.link.gre.GRE`'s payload + is: the message runs on in it to the end of whatever holds it. + :data:`None` where the last field is instead fixed or sized by the + header, as UDP's payload is by its Length and an HTTP/2 frame's by + its own, since the message then ends where its header says + (:issue:`2711`), and where it is a list, as SCTP's chunks are, whose + buffer holds only what its items took. A payload field declaring no + length at all, as a trailer or the next record does, takes what the + message leaves, and is passed over. + + Each field's length is resolved from this schema's own values under two + lengths given; the one that changes with them is sized by what was left. + :func:`~pcapkit.corekit.schema.shortread.keep_short_read` records a + frame the capture cut short on that field. + + """ + packet = dict(self.__dict__) + for name, field in reversed(self.__fields__.items()): + try: + if isinstance(field, ConditionalField) and not field.test(packet): + continue + # NOTE: Lengths far past any message, so a ``max(..., 0)`` callback still moves. + resolved = field(dict(packet, __length__=0x7FFF_FFFE)) + near, far = resolved.length, field(dict(packet, __length__=0x7FFF_FFFF)).length + except Exception: # a length its own values cannot resolve + return None + if isinstance(field, PayloadField) and far < 0: + continue + # NOTE: A list keeps in its buffer only the octets its items took, not the + # rest of the layer's, so a cut recorded on it would drop them (#2711). + if any(isinstance(kind, (ListField, ForwardMatchField)) + for kind in (resolved, getattr(resolved, 'field', None))): + return None + return name if near != far else None + return None + class _EnumRegistry(collections.defaultdict): """A registry :class:`collections.defaultdict` that never inserts a miss. diff --git a/pcapkit/utilities/contextvars.py b/pcapkit/utilities/contextvars.py index 88b9ec606a..f17dffda34 100644 --- a/pcapkit/utilities/contextvars.py +++ b/pcapkit/utilities/contextvars.py @@ -51,6 +51,7 @@ #: contextvars.ContextVar[bool]: Whether a payload is being rebuilt by #: :meth:`Protocol._make_payload #: `, so that the layer -#: it is rebuilt for parses the same octets again; see +#: it is rebuilt for parses the same octets again and reports what they carry, +#: which the payload's own parse then does not (:issue:`2711`); see #: :data:`~pcapkit.utilities.config.FRAME_LAYER_LIMIT`. rebuilding = contextvars.ContextVar('pcapkit.utilities.contextvars.rebuilding', default=False) diff --git a/tests/corekit/test_list_count_shortfall_2711_unit.py b/tests/corekit/test_list_count_shortfall_2711_unit.py new file mode 100644 index 0000000000..6e6950dbe3 --- /dev/null +++ b/tests/corekit/test_list_count_shortfall_2711_unit.py @@ -0,0 +1,88 @@ +# -*- coding: utf-8 -*- +"""GitHub issue #2711: a cut a list short of its count, or a declared payload, records. + +:class:`~pcapkit.corekit.fields.collections.ListField` takes a ``count``, and a list +whose octets hold fewer items than it counts reports the rest as its ``shortfall``, +which :meth:`Schema.unpack ` records +as a short read. A payload field of a declared length the data ends inside records +one too. Either way the fields after it find the ``(name, octets read)`` pair in +their packet as ``__short_read__``, as :issue:`2727` gives them any other field's. + +Each test imports :mod:`pcapkit` itself, after +:func:`~tests._support.reimport_once_per_class`. + +""" + +import unittest +import warnings + +from tests._support import reimport_once_per_class + +#: Test tier (#1958): hand-picked cases of one field type. +__tier__ = 'ALL' + + +class ShortReadSeenAfterTests(unittest.TestCase): + """A list short of its count, or a cut declared payload, is recorded and seen after it.""" + + def setUp(self) -> None: + reimport_once_per_class(self) + from pcapkit.corekit.fields.collections import ListField + from pcapkit.corekit.fields.misc import PayloadField, SchemaField + from pcapkit.corekit.fields.numbers import UInt8Field, UInt16Field + from pcapkit.corekit.fields.strings import BytesField + from pcapkit.protocols.schema.schema import Schema, schema_final + + seen = self.seen = [] # type: list + + def look(pkt): # type: ignore[no-untyped-def] + seen.append(pkt.get('__short_read__')) + return 0 + + @schema_final + class Item(Schema): + value = UInt16Field() + + @schema_final + class Counted(Schema): + count = UInt8Field() + items = ListField(length=lambda pkt: pkt['__length__'], item_type=SchemaField(schema=Item), + count=lambda pkt: pkt['count']) + tail = BytesField(length=look) + + @schema_final + class Sized(Schema): + size = UInt8Field() + payload = PayloadField(length=lambda pkt: pkt['size']) + tail = BytesField(length=look) + + self.Counted, self.Sized = Counted, Sized + + def test_a_list_short_of_its_count_is_seen_after_it(self) -> None: + for count, data, record in ((3, b'\x00\x01', ('items', 2)), (2, b'', ('items', 0)), + (1, b'\x00\x01', None), (1, b'\x00\x01\x00\x02', None)): + with self.subTest(count=count, data=data): + del self.seen[:] + raw = bytes([count]) + data + parsed = self.Counted.unpack(raw, len(raw), {}) + self.assertEqual(parsed.__dict__.get('__short_read__'), record) + self.assertEqual(self.seen, [record]) + self.assertEqual(len(parsed.items), len(data) // 2) + self.assertEqual(parsed.pack(), raw) + + def test_a_cut_declared_payload_is_seen_after_it(self) -> None: + for size, data, record in ((5, b'ab', ('payload', 2)), (2, b'ab', None)): + with self.subTest(size=size, data=data): + del self.seen[:] + raw = bytes([size]) + data + with warnings.catch_warnings(): + # the payload's declared length runs past the data + warnings.simplefilter('ignore') + parsed = self.Sized.unpack(raw, len(raw), {}) + self.assertEqual(parsed.__dict__.get('__short_read__'), record) + self.assertEqual(self.seen, [record]) + self.assertEqual(parsed.payload, data) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/protocols/internet/test_igmp_header_cut_2689_unit.py b/tests/protocols/internet/test_igmp_header_cut_2689_unit.py index 488010c62c..42dcd30456 100644 --- a/tests/protocols/internet/test_igmp_header_cut_2689_unit.py +++ b/tests/protocols/internet/test_igmp_header_cut_2689_unit.py @@ -42,13 +42,13 @@ def message(head: bytes, rest: bytes) -> bytes: #: that leave a short read on its header. Shorter than eight octets, an IGMPv2 or #: IGMPv3 Query is refused by its class, which classifies a Query by its length. #: Eight octets of the Report are a whole header whose record count no octet -#: backs, which records no short read. +#: backs, which reads short as well (#2711). MESSAGES = { 'v1-report': ('IGMPv1', message(b'\x12\x00', bytes([224, 0, 0, 1])), range(1, 8)), 'v2-report': ('IGMPv2', message(b'\x16\x00', bytes([239, 1, 2, 3])), range(1, 8)), 'v2-leave': ('IGMPv2', message(b'\x17\x00', bytes([239, 1, 2, 3])), range(1, 8)), 'v3-report': ('IGMPv3', message(b'\x22\x00', bytes.fromhex('00000001') + RECORD), - [*range(1, 8), *range(9, 20)]), + range(1, 20)), # A General Query with QRV 2, QQIC 125 and one source, 10.0.0.1. 'v3-query': ('IGMPv3', message(b'\x11\x64', bytes(4) + bytes.fromhex('027d00010a000001')), range(12, 16)), diff --git a/tests/protocols/internet/test_igmp_unit.py b/tests/protocols/internet/test_igmp_unit.py index c9c84b6b0a..238e7d49a3 100644 --- a/tests/protocols/internet/test_igmp_unit.py +++ b/tests/protocols/internet/test_igmp_unit.py @@ -1296,7 +1296,8 @@ class IGMPv3DirectCutTests(unittest.TestCase): """A message cut short and parsed on its own, as ``IGMPv3(raw, len(raw))``, rebuilds as captured, as it does inside IPv4 (#2660). IGMP has no length field, so the body's own read is never short there: the cut is recorded on - the body, after the octets it holds, wherever the data ends inside a field.""" + the body, after the octets it holds, wherever the data ends inside a field + or before the records a Report counts (#2711).""" def setUp(self) -> None: reimport_once_per_class(self) @@ -1355,8 +1356,9 @@ def test_a_query_cut_at_every_offset(self) -> None: self.assert_rebuilds(raw, igmp, ('body', keep - 1)) def test_a_report_cut_at_every_offset(self) -> None: - """A cut at the end of the header or of the first record ends inside no - field, so it is not marked, and the count as parsed rebuilds it (#2650).""" + """Every cut is marked: one at the end of the header or of the first + record ends inside no field, but before the records the count declares + (#2711), and the count as parsed rebuilds it (#2650).""" packet = IGMPV3_TWO_RECORD_REPORT for keep in range(1, len(packet)): raw = packet[:keep] @@ -1364,8 +1366,7 @@ def test_a_report_cut_at_every_offset(self) -> None: igmp = self.parse(raw) if keep >= 8: self.assertEqual(igmp.info.num_group_records, 2) - record = None if keep in (8, 24) else ('body', keep - 1) - self.assert_rebuilds(raw, igmp, record) + self.assert_rebuilds(raw, igmp, ('body', keep - 1)) def test_a_cut_is_recorded_as_inside_ipv4(self) -> None: """The record a direct parse makes is the one the enclosing IPv4 layer's diff --git a/tests/protocols/internet/test_igmpv3_report_count_cut_2711_unit.py b/tests/protocols/internet/test_igmpv3_report_count_cut_2711_unit.py new file mode 100644 index 0000000000..c54886dbf2 --- /dev/null +++ b/tests/protocols/internet/test_igmpv3_report_count_cut_2711_unit.py @@ -0,0 +1,256 @@ +# -*- coding: utf-8 -*- +"""An IGMPv3 Report the data ends before its counted records reads ``TRUNCATED``. + +GitHub issue #2711: built directly on octets that end on a field boundary, at the end +of the header or of a group record, an IGMPv3 Report recorded no short read, since its +record list was read with the rest of the message, which ended there too. So its +checksum, covering records never captured, read ``INVALID`` with a ``ChecksumWarning``, +where behind IPv4, whose length runs past the data, the same cut read ``TRUNCATED``. The +record list now counts the records the Number of Group Records gives wherever the data +ends with the message (:rfc:`3376#section-4.2`), and one holding fewer records than it +counts records the cut through ``Schema``'s own short read, the records absent never made +up. + +Each Report below is cut at every field boundary short of its end and built directly. +It must record the cut, read ``TRUNCATED`` with no ``ChecksumWarning``, wrong checksum or +right, parse the records captured as the same cut does behind IPv4, and rebuild from its +data model and its dict as captured, reporting nothing. It must read so whatever the +context parsed before it. Whole Reports, Additional Data included, are the control. + +The messages are built here; this module is unit-tier and reads no capture file. + +""" +from __future__ import annotations + +import contextlib +import logging +import unittest +import warnings +from typing import TYPE_CHECKING +from unittest import mock + +from tests._support import reimport_once_per_class +from tests.foundation import _roundtrip as wire + +if TYPE_CHECKING: + from typing import Any, Iterator + +#: Test tier (#1958): hand-picked parse -> rebuild round trips. +__tier__ = 'ALL' + + +def record(sources: int, aux: int = 0) -> bytes: + """A MODE_IS_INCLUDE record for 239.1.1.1 with ``sources`` sources and ``aux`` words of auxiliary data.""" + return (bytes([1, aux]) + sources.to_bytes(2, 'big') + bytes([239, 1, 1, 1]) + + bytes([10, 0, 0, 1]) * sources + b'AUX!' * aux) + + +def report(*records: bytes, extra: bytes = b'', count: 'int | None' = None) -> bytes: + """An IGMPv3 Report of ``records`` counting ``count``, then Additional Data ``extra``, checksummed.""" + rest = (len(records) if count is None else count).to_bytes(2, 'big') + b''.join(records) + extra + head = b'\x22\x00' + return head + wire.checksum(head + bytes(4) + rest).to_bytes(2, 'big') + b'\x00\x00' + rest + + +def boundaries(*records: bytes) -> 'list[int]': + """Every offset where a field of a Report of ``records`` ends, short of its end.""" + ends = [1, 2, 4, 6, 8] + start = 8 + for item in records: + ends.extend(start + width for width in (1, 2, 4, 8)) + ends.extend(range(start + 12, start + len(item) + 1, 4)) + start += len(item) + return [end for end in ends if end < start] + + +#: Each Report as a tuple of its records: the issue's two of one source each, two of +#: different sizes, and one carrying auxiliary data before one with no sources. +SHAPES = { + 'two records of one source': (record(1), record(1)), + 'records of two and three sources': (record(2), record(3)), + 'auxiliary data, then no sources': (record(1, aux=1), record(0)), +} + + +def starts(records: 'tuple[bytes, ...]') -> 'list[int]': + """Where each of ``records`` begins in its Report.""" + offsets, start = [], 8 + for item in records: + offsets.append(start) + start += len(item) + return offsets + + +def wrong(msg: bytes) -> bytes: + """``msg`` with its checksum flipped in both octets.""" + return msg[:2] + (int.from_bytes(msg[2:4], 'big') ^ 0x0101).to_bytes(2, 'big') + msg[4:] + + +class _Logged(logging.Handler): + """Collect what the ``pcapkit`` logger reports.""" + + def __init__(self) -> None: + super().__init__(logging.DEBUG) + self.messages = [] # type: list[str] + + def emit(self, record: 'logging.LogRecord') -> None: # pylint: disable=redefined-outer-name + self.messages.append(f'logged: {record.getMessage()}') + + +@contextlib.contextmanager +def _reported() -> 'Iterator[tuple[list[str], list[warnings.WarningMessage]]]': + """Collect what the block reports, on the logger and the :mod:`warnings` channel.""" + handler = _Logged() + logger = logging.getLogger('pcapkit') + logger.addHandler(handler) + try: + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter('always') + yield handler.messages, caught + finally: + logger.removeHandler(handler) + + +class ReportCountCutTests(unittest.TestCase): + """A Report the data ends before its counted records is cut, wherever the cut falls.""" + + def setUp(self) -> None: + reimport_once_per_class(self) + from pcapkit.corekit.checksum.internet import ChecksumStatus + from pcapkit.protocols.internet.igmpv3 import IGMPv3 + from pcapkit.protocols.internet.ipv4 import IPv4 + from pcapkit.utilities import config + from pcapkit.utilities.warnings import ChecksumWarning + + self.IGMPv3 = IGMPv3 + self.IPv4 = IPv4 + self.Status = ChecksumStatus + self.ChecksumWarning = ChecksumWarning + patch = mock.patch.object(config, 'CHECKSUM_VERIFY', True) + patch.start() + self.addCleanup(patch.stop) + + def cuts(self) -> 'Iterator[tuple[str, bytes, int]]': + """Every Report with its octets, and each field boundary it is cut at.""" + for label, records in SHAPES.items(): + for keep in boundaries(*records): + yield label, report(*records), keep + + def direct(self, data: bytes) -> 'tuple[Any, list[str]]': + """``data`` built directly as IGMPv3, and the checksum warnings it gave.""" + with _reported() as (_, caught): + layer = self.IGMPv3(data, len(data)) + return layer, [str(item.message) for item in caught if issubclass(item.category, self.ChecksumWarning)] + + def test_a_cut_on_a_field_boundary_reads_truncated(self) -> None: + for label, msg, keep in self.cuts(): + for bad in (False, True): + with self.subTest(report=label, captured=keep, bad=bad): + layer, caught = self.direct((wrong(msg) if bad else msg)[:keep]) + self.assertEqual(layer.info.get('__short_read__'), ('body', keep - 1)) + self.assertEqual((layer.info.checksum_status, caught), (self.Status.TRUNCATED, [])) + + def test_a_cut_parses_the_records_captured_as_behind_ipv4(self) -> None: + # Behind IPv4, whose length runs past the data, the records past it read as + # zeros (#2686); built directly, only the records the data reaches are read. + for label, msg, keep in self.cuts(): + if keep < 2: + # One octet is too few for IGMP to tell its version from, so IPv4 keeps it Raw. + continue + with self.subTest(report=label, captured=keep): + packet = wire.ipv4(msg, proto=2) + with _reported(): + carried = self.IPv4(packet[:len(packet) - len(msg) + keep]).payload + direct, _ = self.direct(msg[:keep]) + self.assertIsInstance(carried, self.IGMPv3) + ours, theirs = direct.info.to_dict(), carried.info.to_dict() + records, filled = ours.pop('records'), theirs.pop('records') + self.assertEqual(ours, theirs) + self.assertEqual(len(records), sum(start < keep for start in starts(SHAPES[label]))) + self.assertEqual(records, filled[:len(records)]) + self.assertEqual({str(item.multicast_address) for item in filled[len(records):]} - {'0.0.0.0'}, set()) + + def test_a_cut_rebuilds_as_captured_and_quietly(self) -> None: + for label, msg, keep in self.cuts(): + layer, _ = self.direct(msg[:keep]) + for path, data in (('model', layer.info), ('dict', layer.info.to_dict())): + with self.subTest(report=label, captured=keep, path=path): + with _reported() as (logged, caught): + rebuilt = self.IGMPv3.from_data(data) + self.assertEqual((rebuilt.data, rebuilt.info.checksum_status), (msg[:keep], self.Status.TRUNCATED)) + self.assertEqual((logged, [str(item.message) for item in caught]), ([], [])) + + def test_an_enclosing_length_still_sizes_the_records(self) -> None: + # IPv4 declares one record's worth, the count three, and the capture ends + # inside the record: the records span the declared length, as before. + msg = report(record(1), count=3) + packet = wire.ipv4(msg, proto=2) + with _reported(): + carried = self.IPv4(packet[:len(packet) - len(msg) + 14]).payload + self.assertIsInstance(carried, self.IGMPv3) + self.assertEqual(carried.info.get('__short_read__'), ('body', 13)) + self.assertEqual([str(item.multicast_address) for item in carried.info.records], ['239.1.0.0']) + + def test_a_count_no_ip_payload_can_carry_is_refused(self) -> None: + # Behind the 8-octet header, 8,190 records of 8 octets fit an IP payload; 8,191 do not. + from pcapkit.utilities.exceptions import ProtocolError + + for count in (8191, 0xFFFF): + msg = report(count=count) + with self.subTest(count=count), \ + self.assertRaisesRegex(ProtocolError, rf'{count} group records cannot fit'): + self.IGMPv3(msg, len(msg)) + + def test_a_count_past_the_data_reads_the_same_whatever_was_parsed_before(self) -> None: + # The same Report, in a padding budget of its own and again once the context has + # read enough for that budget to cover zero-filling its counted records: both read + # only the records present, as cut, and rebuild as captured (#2711). + from pcapkit.corekit.fields.field import _zero_pad_budget + from pcapkit.utilities import config + + bulk = report(*[record(255)] * 32) + for count in (2, 16, 17, 100, 8190): + for records in ((), (record(1),)): + msg = report(*records, count=count) + with self.subTest(count=count, records=len(records)), _zero_pad_budget() as ledger: + fresh, caught = self.direct(msg) + self.IGMPv3(bulk, len(bulk)) + self.assertGreater(config.ZERO_PAD_BUDGET_RATIO * ledger[0], 8 * count) + later, caught_later = self.direct(msg) + seen = [(layer.info.checksum_status, layer.info.get('__short_read__'), + layer.info.to_dict()['records'], self.IGMPv3.from_data(layer.info).data) + for layer in (fresh, later)] + self.assertEqual(seen[0], seen[1]) + self.assertEqual(seen[0][:2], (self.Status.TRUNCATED, ('body', len(msg) - 1))) + self.assertEqual((len(seen[0][2]), seen[0][3]), (len(records), msg)) + self.assertEqual((caught, caught_later), ([], [])) + + def test_inside_a_frame_the_count_sizes_nothing(self) -> None: + # Behind IPv4, whose length says where the message ends, a whole message whose + # count overstates its records is read as it was, not as a cut (#2711). + for count in (2, 100, 0xFFFF): + msg = report(record(1), count=count) + with self.subTest(count=count), _reported() as (logged, _): + layer = self.IPv4(wire.ipv4(msg, proto=2)).payload + self.assertIsInstance(layer, self.IGMPv3) + self.assertIsNone(layer.info.get('__short_read__')) + self.assertEqual((len(layer.info.records), layer.info.num_group_records), (1, count)) + self.assertEqual(logged, []) + + def test_a_whole_report_is_verified(self) -> None: + for label, records in SHAPES.items(): + for extra in (b'', b'\xca\xfe\xba\xbe'): + msg = report(*records, extra=extra) + with self.subTest(report=label, extra=extra): + layer, caught = self.direct(msg) + self.assertIsNone(layer.info.get('__short_read__')) + self.assertEqual((layer.info.checksum_status, caught), (self.Status.VALID, [])) + self.assertEqual(len(layer.info.records), len(records)) + self.assertEqual(self.IGMPv3.from_data(layer.info).data, msg) + layer, caught = self.direct(wrong(msg)) + self.assertIs(layer.info.checksum_status, self.Status.INVALID) + self.assertEqual(len(caught), 1) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/protocols/internet/test_ipv4_tso_cut_checksum_2689_unit.py b/tests/protocols/internet/test_ipv4_tso_cut_checksum_2689_unit.py index 2287c78a6e..3f12f1d741 100644 --- a/tests/protocols/internet/test_ipv4_tso_cut_checksum_2689_unit.py +++ b/tests/protocols/internet/test_ipv4_tso_cut_checksum_2689_unit.py @@ -220,10 +220,13 @@ def spy(*args: 'Any', **kwargs: 'Any') -> 'Any': wire.ethernet(ipv4(ipv4(msg, 1, tso=False), 4, tso=True), 0x0800), False), } for name, (whole, kept) in cases.items(): + # A cut inside a declared length is recorded on the message, which then reads + # TRUNCATED without verifying (#2711), so a declared case is read whole. + captured = whole[:-CUT] if kept else whole with self.subTest(case=name), mock.patch.object(checksum, 'verify_message', spy): seen.clear() - self.extract(pcap(ETHERNET, whole[:-CUT], len(whole))) - self.assertEqual(seen, [(len(whole) - CUT, len(whole)) if kept else None]) + self.extract(pcap(ETHERNET, captured, len(whole))) + self.assertEqual(seen, [(len(captured), len(whole)) if kept else None]) self.assertIsNone(frame_length.get()) def test_parse_rebuild_is_exact(self) -> None: diff --git a/tests/protocols/test_cut_recorded_on_the_layer_2711_unit.py b/tests/protocols/test_cut_recorded_on_the_layer_2711_unit.py new file mode 100644 index 0000000000..fa9ecc3f1e --- /dev/null +++ b/tests/protocols/test_cut_recorded_on_the_layer_2711_unit.py @@ -0,0 +1,338 @@ +# -*- coding: utf-8 -*- +"""A layer whose octets the capture cut short records the cut, and rebuilds as cut. + +GitHub issue #2711: inside a frame, a message whose captured octets fall short of the +length its context gives it reads ``TRUNCATED``, from the length an enclosing IP layer +declares or, where no layer declares one, from the frame's original length (#2653, +#2689). Nothing recorded the cut on the message's own header, though, so rebuilt on its +own from its data model the message verified the octets captured and read ``INVALID`` +with a ``ChecksumWarning``. Ruled on the issue: the layer records the cut, so its parse +and its rebuild agree. ``Schema``'s own short read now covers a payload the data ends +inside, and a layer running to the end of a frame the capture cut short records the cut +on the last of its fields holding octets. + +Each frame below is cut at every octet inside its message and read from a PCAP +savefile: GRE behind IPv4 and behind a Linux cooked-capture header, ICMP and IGMPv2 +behind an IPv4 Total Length of 0, IGMPv2 behind IPv4, and UDP behind IPv4 and IPv6. The +message must record the cut and read as its context reads it, and its rebuild from its +data model and its dict must give the octets captured, read the same, and report +nothing. Whole frames, a wrong checksum and a cut in the Ethernet padding past a +declared length are the controls. + +The octets and captures are built here; this module is unit-tier and reads no capture file. + +""" +from __future__ import annotations + +import contextlib +import io +import itertools +import logging +import struct +import unittest +import warnings +from typing import TYPE_CHECKING +from unittest import mock + +from tests._support import reimport_once_per_class +from tests.foundation import _roundtrip as wire + +if TYPE_CHECKING: + from typing import Any, Iterator + +#: Test tier (#1958): hand-picked parse -> rebuild round trips. +__tier__ = 'ALL' + +#: ``LINKTYPE_ETHERNET`` and ``LINKTYPE_LINUX_SLL``. +ETHERNET, LINUX_SLL = 1, 113 +#: An IPv4 header of its own 20 octets, protocol 253, as GRE's payload. +INNER = bytes.fromhex('450000141234000040fd7b83c0000201c6336401') + + +def message(head: bytes, rest: bytes) -> bytes: + """``head``, then the checksum over the whole message, then ``rest``.""" + return head + wire.checksum(head + b'\x00\x00' + rest).to_bytes(2, 'big') + rest + + +def gre(payload: bytes) -> bytes: + """A GRE message with C set, carrying IPv4 ``payload``, checksummed.""" + head = struct.pack('>HH', 0x8000, 0x0800) + return head + struct.pack('>HH', wire.checksum(head + bytes(4) + payload), 0) + payload + + +def tso(packet: bytes) -> bytes: + """IPv4 ``packet`` with its Total Length 0, as segmentation offload leaves it.""" + head = bytearray(packet[:20]) + head[2:4], head[10:12] = bytes(2), bytes(2) + head[10:12] = wire.checksum(bytes(head)).to_bytes(2, 'big') + return bytes(head) + packet[20:] + + +def sll(packet: bytes) -> bytes: + """``packet`` behind a Linux cooked-capture header naming ``ARPHRD_IPGRE``.""" + return struct.pack('>HHH', 0, 778, 0) + bytes(8) + struct.pack('>H', 0x0800) + packet + + +def pcap(linktype: int, packet: bytes, original: int) -> bytes: + """A PCAP savefile of one frame holding ``packet`` of ``original`` octets on the wire.""" + return (struct.pack(' bytes: + """A UDP datagram of ``data``, checksummed over the IPv4 or IPv6 pseudo-header.""" + src, dst, nxt = ((wire.IPV4_A, wire.IPV4_B, b'\x00\x11') if version == 4 + else (wire.IPV6_A, wire.IPV6_B, b'\x00\x00\x00\x11')) + head = struct.pack('>HHH', 40000, 9, 8 + len(data)) + length = (8 + len(data)).to_bytes(2 if version == 4 else 4, 'big') + pseudo = src + dst + (nxt + length if version == 4 else length + nxt) + value = wire.checksum(pseudo + head + bytes(2) + data) or 0xFFFF + return head + value.to_bytes(2, 'big') + data + + +ICMP = message(b'\x08\x00', b'\x00\x01\x00\x01' + b'echo-data-octets') +IGMP = message(b'\x16\x00', bytes([239, 1, 2, 3]) + b'trailing') +GRE = gre(INNER) +#: A SETTINGS frame of three settings, then a connection-level WINDOW_UPDATE: each a +#: frame whose 3-octet Length counts its payload alone (RFC 9113, section 4.1). +SETTINGS = bytes.fromhex('000012' '04' '00' '00000000' '000100001000' '000300000064' '000400010000') +WINDOW_UPDATE = bytes.fromhex('000004' '08' '00' '00000000' '0000ffff') + +#: Each frame: its link type, its octets, the layer holding its message, and the +#: octets of its message a cut must leave for the message to be its own layer. +FRAMES = { + 'GRE behind IPv4': (ETHERNET, wire.ethernet(wire.ipv4(GRE, proto=47), 0x0800), 'GRE', 8), + 'GRE behind LINUX_SLL': (LINUX_SLL, sll(GRE), 'GRE', 8), + 'GRE behind a Total Length of 0': (ETHERNET, wire.ethernet(tso(wire.ipv4(GRE, proto=47)), 0x0800), 'GRE', 8), + 'ICMP behind a Total Length of 0': (ETHERNET, wire.ethernet(tso(wire.ipv4(ICMP, proto=1)), 0x0800), 'ICMP', 8), + 'IGMPv2 behind IPv4': (ETHERNET, wire.ethernet(wire.ipv4(IGMP, proto=2), 0x0800), 'IGMPv2', 8), + 'IGMPv2 behind a Total Length of 0': (ETHERNET, wire.ethernet(tso(wire.ipv4(IGMP, proto=2)), 0x0800), + 'IGMPv2', 8), +} + +#: Each UDP frame: its octets, and the octets of the datagram past its header. +DATAGRAMS = { + 'UDP behind IPv4': (wire.ethernet(wire.ipv4(udp(b'udp-payload-octets', version=4)), 0x0800), 18), + 'UDP behind IPv6': (wire.ethernet(wire.ipv6(udp(b'udp-payload-octets', version=6), nxt=17), 0x86DD), 18), +} + + +def checksums(messages: 'list[str]') -> 'list[str]': + """The checksum warnings among ``messages``; the savefile's own end is reported too.""" + return [line for line in messages if 'checksum' in line] + + +class _Logged(logging.Handler): + """Collect what the ``pcapkit`` logger reports.""" + + def __init__(self) -> None: + super().__init__(logging.DEBUG) + self.messages = [] # type: list[str] + + def emit(self, record: 'logging.LogRecord') -> None: + self.messages.append(f'logged: {record.getMessage()}') + + +@contextlib.contextmanager +def _reported() -> 'Iterator[list[str]]': + """Collect what the block reports, on the :mod:`warnings` channel and the logger.""" + handler = _Logged() + logger = logging.getLogger('pcapkit') + logger.addHandler(handler) + try: + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter('always') + yield handler.messages + finally: + logger.removeHandler(handler) + handler.messages.extend(f'warned: {item.message}' for item in caught) + + +class CutRecordedOnTheLayerTests(unittest.TestCase): + """A message the capture cut short records it, and its rebuild reads as cut.""" + + def setUp(self) -> None: + reimport_once_per_class(self) + import pcapkit + from pcapkit.corekit.checksum.internet import ChecksumStatus + from pcapkit.utilities import config + + self.pcapkit = pcapkit + self.Status = ChecksumStatus + patch = mock.patch.object(config, 'CHECKSUM_VERIFY', True) + patch.start() + self.addCleanup(patch.stop) + + def extract(self, linktype: int, packet: bytes, original: int) -> 'tuple[Any, list[str]]': + """The frame a savefile of ``packet`` holds, and what reading it reported.""" + with _reported() as messages: + frame = self.pcapkit.extract(fin=io.BytesIO(pcap(linktype, packet, original)), nofile=True).frame[0] + return frame, messages + + def assertRebuildsAsCut(self, layer: 'Any') -> 'None': # pylint: disable=invalid-name + """``layer`` rebuilt from its data model and its dict: its octets and status, quietly.""" + for path, data in (('model', layer.info), ('dict', layer.info.to_dict())): + with self.subTest(path=path), _reported() as messages: + rebuilt = type(layer).from_data(data) + self.assertEqual((rebuilt.data, rebuilt.info.get('checksum_status'), messages), + (layer.data, layer.info.get('checksum_status'), [])) + + def test_a_cut_message_records_it_and_rebuilds_as_cut(self) -> None: + for label, (linktype, whole, name, header) in FRAMES.items(): + span = len(whole) - whole.index(GRE if name == 'GRE' else ICMP if name == 'ICMP' else IGMP) + for cut in range(1, span - header + 1): + with self.subTest(frame=label, cut=cut): + frame, _ = self.extract(linktype, whole[:len(whole) - cut], len(whole)) + layer = frame[name] + self.assertIsNotNone(layer.info.get('__short_read__')) + self.assertIs(layer.info.checksum_status, self.Status.TRUNCATED) + self.assertRebuildsAsCut(layer) + if 'Total Length of 0' in label: + # The cut is in the payload the IPv4 layer runs to the frame's end + # with, not in its empty trailer. + self.assertEqual(frame['IPv4'].info.get('__short_read__'), + ('payload', span - cut)) + + def test_a_layer_declaring_a_length_records_its_cut(self) -> None: + for label, (whole, data) in DATAGRAMS.items(): + for cut in range(1, data + 1): + with self.subTest(frame=label, cut=cut): + frame, _ = self.extract(ETHERNET, whole[:len(whole) - cut], len(whole)) + layer = frame['UDP'] + self.assertEqual(layer.info.get('__short_read__'), ('payload', data - cut)) + self.assertEqual(type(layer).from_data(layer.info).data, layer.data) + + def test_a_whole_message_is_verified(self) -> None: + for label, (linktype, whole, name, _) in FRAMES.items(): + with self.subTest(frame=label): + frame, messages = self.extract(linktype, whole, len(whole)) + layer = frame[name] + self.assertIsNone(layer.info.get('__short_read__')) + self.assertEqual((layer.info.checksum_status, checksums(messages)), (self.Status.VALID, [])) + self.assertRebuildsAsCut(layer) + + def test_a_wrong_checksum_still_warns(self) -> None: + bad = GRE[:4] + (int.from_bytes(GRE[4:6], 'big') ^ 0x0101).to_bytes(2, 'big') + GRE[6:] + for label, linktype, whole in (('IPv4', ETHERNET, wire.ethernet(wire.ipv4(bad, proto=47), 0x0800)), + ('LINUX_SLL', LINUX_SLL, sll(bad))): + with self.subTest(frame=label): + frame, messages = self.extract(linktype, whole, len(whole)) + self.assertIs(frame['GRE'].info.checksum_status, self.Status.INVALID) + self.assertRegex(' '.join(messages), r'GRE: checksum 0x[0-9a-f]{4} is wrong;') + + def test_a_cut_past_a_declared_length_is_not_the_message(self) -> None: + # The frame is padded past the datagram IPv4 declares, and cut in that padding only. + whole = wire.ethernet(wire.ipv4(ICMP, proto=1), 0x0800) + bytes(8) + frame, messages = self.extract(ETHERNET, whole[:-4], len(whole)) + layer = frame['ICMP'] + self.assertIsNone(layer.info.get('__short_read__')) + self.assertEqual((layer.info.checksum_status, checksums(messages)), (self.Status.VALID, [])) + self.assertRebuildsAsCut(layer) + + def test_a_layer_declaring_its_length_records_no_frame_cut_past_it(self) -> None: + # Behind an IPv4 Total Length of 0 the frame's cut reaches every layer, so + # IPv4 and Ethernet, declaring no length, record it. UDP declares its Length + # and an HTTP/2 frame its own, so a cut at or past those ends is none of theirs. + datagram = wire.ethernet(tso(wire.ipv4(udp(b'hello', version=4))), 0x0800) + whole = datagram + bytes(20) + for where, keep in (('at its end', len(datagram)), ('3 into the padding', len(datagram) + 3), + ('19 into the padding', len(datagram) + 19), ('inside its data', len(datagram) - 2)): + with self.subTest(layer='UDP', cut=where): + frame, _ = self.extract(ETHERNET, whole[:keep], len(whole)) + record = ('payload', 3) if keep < len(datagram) else None + self.assertEqual(frame['UDP'].info.get('__short_read__'), record) + self.assertEqual(frame['IPv4'].info.get('__short_read__'), ('payload', keep - 34)) + self.assertEqual(frame['Ethernet'].info.get('__short_read__'), ('payload', keep - 14)) + self.assertEqual(type(frame['UDP']).from_data(frame['UDP'].info).data, frame['UDP'].data) + + segment = wire.ethernet(tso(wire.ipv4(wire.tcp(SETTINGS + WINDOW_UPDATE, seq=1, dport=80), proto=6)), + 0x0800) + for where, keep in (('at the first frame\'s end', len(segment) - len(WINDOW_UPDATE)), + ('inside the next frame', len(segment) - 1)): + with self.subTest(layer='HTTP/2', cut=where): + frame, _ = self.extract(ETHERNET, segment[:keep], len(segment)) + first = frame['TCP'].payload + self.assertEqual((type(first).__name__, first.info.get('__short_read__')), ('HTTPv2', None)) + with _reported() as messages: + rebuilt = type(first).from_data(first.info) + self.assertEqual((rebuilt.data, messages), (first.data, [])) + + def test_a_layer_ending_in_a_list_records_no_frame_cut(self) -> None: + # SCTP behind an IPv4 Total Length of 0 ends in its chunk list, whose buffer holds + # only the chunks it took: a cut recorded there would drop the trailer a cut + # chunk is kept in. So SCTP records nothing, and rebuilds as captured. + packets = { + # a DATA chunk of three octets and its pad octet + 'DATA': bytes.fromhex('13881389112233441eca615f0003001300000007000100020000000001020300'), + # an INIT with four parameters, a DATA chunk of five octets, then a HEARTBEAT + 'INIT, DATA, HEARTBEAT': bytes.fromhex( + '1388138911223344f8c903be0100003caabbccdd00010000000a000a0000000100050008c00002' + '01000c000600050000000b000e6c6f63616c686f737400000000090008000003e8000300150000' + '0007000100020000000061626364650000000400000c0001000778797a00'), + } + for (label, packet), (where, cut) in itertools.product( + packets.items(), (('at its end', 0), ('3 into the padding', 3), ('2 inside the last chunk', -2))): + segment = wire.ethernet(tso(wire.ipv4(packet, proto=132)), 0x0800) + whole, keep = segment + bytes(20), len(segment) + cut + with self.subTest(packet=label, cut=where): + frame, _ = self.extract(ETHERNET, whole[:keep], len(whole)) + sctp = frame['SCTP'] + self.assertEqual(sctp.data, whole[34:keep]) + self.assertIsNone(sctp.info.get('__short_read__')) + with _reported() as messages: + rebuilt = type(sctp).from_data(sctp.info) + self.assertEqual((rebuilt.data, messages), (sctp.data, [])) + + def test_an_ipv6_header_past_its_payload_length_stays_raw_in_a_cut_frame(self) -> None: + # With #2694's capture-cut test on each header IPv6 walks: a padded frame cut at + # the IP end or in the padding records nothing on IPv6, whose Payload Length + # its octets cover, and a 16-octet Hop-by-Hop header under a Payload Length of 8 + # runs past it, so it stays Raw, quietly, whether or not the frame was cut. + hop = bytes([59, 1, 0, 1, 4, 0, 0, 0, 1, 5, 0, 0, 0, 0, 0, 0]) + packet = wire.ethernet(b'\x60\x00\x00\x00\x00\x08\x00\x40' + wire.IPV6_A + wire.IPV6_B + hop[:8], 0x86DD) + whole = packet + bytes(12) + for where, keep in (('whole', len(whole)), ('at the packet end', len(packet)), + ('inside the padding', len(packet) + 4)): + with self.subTest(cut=where): + frame, messages = self.extract(ETHERNET, whole[:keep], len(whole)) + ipv6 = frame['IPv6'] + self.assertEqual(type(ipv6.payload).__name__, 'Raw') + self.assertNotIn('hopopt', ipv6.info) + self.assertIsNone(ipv6.info.get('__short_read__')) + self.assertEqual([line for line in messages if 'EOF' not in line], []) + with _reported() as rebuilt_messages: + rebuilt = type(ipv6).from_data(ipv6.info) + self.assertEqual((rebuilt.data, rebuilt_messages), (ipv6.data, [])) + + def test_an_ip_layer_covered_to_its_length_records_no_cut(self) -> None: + # The frame is padded past the packet, and cut at the packet's end or inside + # the padding: the IP layer, a Hop-by-Hop header it carries and its UDP + # datagram are whole, while Ethernet, declaring no length, is cut with the frame. + hop = bytes([17, 0, 1, 4, 0, 0, 0, 0]) + packets = { + 'IPv4': (wire.ethernet(wire.ipv4(udp(b'hello', version=4)), 0x0800), ()), + 'IPv6': (wire.ethernet(wire.ipv6(udp(b'hello', version=6), nxt=17), 0x86DD), ()), + 'IPv6 with HOPOPT': (wire.ethernet(wire.ipv6(hop + udp(b'hello', version=6), nxt=0), 0x86DD), + ('HOPOPT',)), + } + for label, (packet, headers) in packets.items(): + whole = packet + bytes(max(60 - len(packet), 8)) + for where, keep in (('at the packet end', len(packet)), ('inside the padding', len(packet) + 2)): + with self.subTest(packet=label, cut=where): + frame, _ = self.extract(ETHERNET, whole[:keep], len(whole)) + names = ('IPv6' if 'IPv6' in label else 'IPv4', 'UDP') + self.assertEqual({name: frame[name].info.get('__short_read__') for name in names}, + dict.fromkeys(names)) + # an IPv6 extension header is held in the IPv6 layer's info, not as a layer + self.assertEqual([frame['IPv6'].info[name.lower()].get('__short_read__') for name in headers], + [None] * len(headers)) + self.assertEqual(frame['Ethernet'].info.get('__short_read__'), ('payload', keep - 14)) + for name in names: + # UDP rebuilt alone has no pseudo-header to verify against + with self.subTest(layer=name), _reported() as messages: + rebuilt = type(frame[name]).from_data(frame[name].info) + self.assertEqual((rebuilt.data, messages), (frame[name].data, [])) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/protocols/test_rebuild_reports_once_2711_unit.py b/tests/protocols/test_rebuild_reports_once_2711_unit.py new file mode 100644 index 0000000000..42c953e1e8 --- /dev/null +++ b/tests/protocols/test_rebuild_reports_once_2711_unit.py @@ -0,0 +1,197 @@ +# -*- coding: utf-8 -*- +"""A rebuilt frame reports nothing its parse did not, and a new problem once. + +GitHub issue #2711: rebuilding an outer layer with ``from_data`` rebuilds its payload +the same way, whose own parse went over the payload's octets and reported what they +carry; the outer layer then parsed those octets again and reported it all once more. +So Ethernet over IPv4 over a cut IGMPv3 Report logged every message of the Report +twice, where IPv4 alone logged it once. The payload's parse is now quiet, warnings and +the errors a next-layer fallback absorbs alike, since the outer layer's own parse +reports them. + +Ethernet frames carrying an IGMPv3 Report over IPv4 and over IPv6 are cut at every +offset, read from their octets and from a PCAP savefile, and every layer is rebuilt +from its data model and its dict: it must give the octets parsed and report nothing +beyond what the parse reported, on the :mod:`warnings` channel or the ``pcapkit`` +logger. A zero-fill ceiling low enough that the Report's in-frame parse raises gives +the errors. The controls are a wrong checksum, reported once on rebuild as on parse, +and a problem the parse never had, which the rebuild reports once. + +The octets and captures are built here; this module is unit-tier and reads no capture file. + +""" +from __future__ import annotations + +import collections +import contextlib +import io +import logging +import struct +import unittest +import warnings +from typing import TYPE_CHECKING +from unittest import mock + +from tests._support import reimport_once_per_class +from tests.foundation import _roundtrip as wire + +if TYPE_CHECKING: + from typing import Any, Callable, Iterator + +#: Test tier (#1958): hand-picked parse -> rebuild round trips. +__tier__ = 'ALL' + +#: A group record of type MODE_IS_INCLUDE for 239.1.1.1 with one source, 10.0.0.1. +RECORD = bytes.fromhex('01000001ef0101010a000001') + + +def message(head: bytes, rest: bytes) -> bytes: + """``head``, then the checksum over the whole message, then ``rest``.""" + return head + wire.checksum(head + b'\x00\x00' + rest).to_bytes(2, 'big') + rest + + +#: An IGMPv3 Report of two group records. +REPORT = message(b'\x22\x00', bytes.fromhex('00000002') + RECORD * 2) + +#: Each frame: Ethernet over IPv4, then over IPv6, carrying the Report as IP protocol 2. +FRAMES = { + 'IPv4': wire.ethernet(wire.ipv4(REPORT, proto=2), 0x0800), + 'IPv6': wire.ethernet(wire.ipv6(REPORT, nxt=2), 0x86DD), +} + + +def pcap(packet: bytes, original: int) -> bytes: + """A PCAP savefile of one Ethernet frame holding ``packet`` of ``original`` octets on the wire.""" + return (struct.pack(' None: + super().__init__(logging.DEBUG) + self.messages = [] # type: list[str] + + def emit(self, record: 'logging.LogRecord') -> None: + self.messages.append(f'{record.levelname}: {record.getMessage()}') + + +@contextlib.contextmanager +def _reported() -> 'Iterator[list[str]]': + """Collect what the block reports, on the :mod:`warnings` channel and the logger.""" + handler = _Logged() + logger = logging.getLogger('pcapkit') + logger.addHandler(handler) + try: + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter('always') + yield handler.messages + finally: + logger.removeHandler(handler) + handler.messages.extend(f'warned: {item.message}' for item in caught) + + +def _layers(frame: 'Any') -> 'Iterator[Any]': + """``frame`` and every layer it carries, NoPayload excluded.""" + layer = frame + while type(layer).__name__ != 'NoPayload': + yield layer + layer = layer.payload + + +class RebuildReportsOnceTests(unittest.TestCase): + """A rebuild reports what its octets carry once, and nothing the parse did not.""" + + def setUp(self) -> None: + reimport_once_per_class(self) + import pcapkit + from pcapkit.protocols.link.ethernet import Ethernet + from pcapkit.utilities import config + + self.pcapkit = pcapkit + self.Ethernet = Ethernet + self.config = config + patch = mock.patch.object(config, 'CHECKSUM_VERIFY', True) + patch.start() + self.addCleanup(patch.stop) + + def parse(self, build: 'Callable[[], Any]') -> 'tuple[Any, list[str]]': + """What ``build`` returns, and what it reported.""" + with _reported() as messages: + layer = build() + return layer, messages + + def cuts(self) -> 'Iterator[tuple[str, str, Any, list[str]]]': + """Every frame cut at every offset, read from its octets and from a savefile.""" + for label, whole in FRAMES.items(): + for cut in range(len(whole)): + data = whole[:len(whole) - cut] + frame, messages = self.parse(lambda: self.Ethernet(data, len(data))) # pylint: disable=cell-var-from-loop + yield label, f'octets, cut {cut}', frame, messages + frame, messages = self.parse(lambda: self.pcapkit.extract( # pylint: disable=cell-var-from-loop + fin=io.BytesIO(pcap(data, len(whole))), nofile=True).frame[0].payload) + yield label, f'savefile, cut {cut}', frame, messages + + def assertRebuildsWithin(self, frame: 'Any', parsed: 'list[str]') -> 'None': # pylint: disable=invalid-name + """Every layer of ``frame`` rebuilds its octets, reporting nothing ``parsed`` lacks.""" + for layer in _layers(frame): + for path, data in (('model', layer.info), ('dict', layer.info.to_dict())): + with self.subTest(layer=type(layer).__name__, path=path): + rebuilt, messages = self.parse(lambda: type(layer).from_data(data)) # pylint: disable=cell-var-from-loop + self.assertEqual(rebuilt.data, layer.data) + beyond = collections.Counter(messages) - collections.Counter(parsed) + self.assertEqual(sorted(beyond.elements()), []) + + def test_a_cut_frame_rebuilds_within_its_parse(self) -> None: + for label, how, frame, parsed in self.cuts(): + with self.subTest(frame=label, read=how): + self.assertRebuildsWithin(frame, parsed) + + def test_errors_a_fallback_absorbs_are_reported_once(self) -> None: + # Too low a ceiling for the Report's records, zero-filled to the length IP + # declares, so its parse inside the frame raises and becomes Raw. + with mock.patch.object(self.config, 'MAX_ZERO_PAD_LENGTH', 8): + seen = 0 + for label, how, frame, parsed in self.cuts(): + with self.subTest(frame=label, read=how): + self.assertRebuildsWithin(frame, parsed) + seen += any(line.startswith('CRITICAL: FieldValueError') for line in parsed) + self.assertTrue(seen) + + def test_a_wrong_checksum_is_reported_once_on_rebuild(self) -> None: + bad = REPORT[:2] + (int.from_bytes(REPORT[2:4], 'big') ^ 0x0101).to_bytes(2, 'big') + REPORT[4:] + for label, whole in (('IPv4', wire.ethernet(wire.ipv4(bad, proto=2), 0x0800)), + ('IPv6', wire.ethernet(wire.ipv6(bad, nxt=2), 0x86DD))): + frame, parsed = self.parse(lambda: self.Ethernet(whole, len(whole))) # pylint: disable=cell-var-from-loop + self.assertRegex(' '.join(parsed), r'IGMPv3: checksum 0x[0-9a-f]{4} is wrong;') + for layer in list(_layers(frame))[:2]: + with self.subTest(frame=label, layer=type(layer).__name__): + rebuilt, messages = self.parse(lambda: type(layer).from_data(layer.info)) # pylint: disable=cell-var-from-loop + self.assertEqual((rebuilt.data, messages), (layer.data, parsed)) + + def test_a_problem_new_to_the_rebuild_is_reported_once(self) -> None: + for label, whole in FRAMES.items(): + frame, parsed = self.parse(lambda: self.Ethernet(whole, len(whole))) # pylint: disable=cell-var-from-loop + self.assertEqual(parsed, []) + edited = frame.info.to_dict() + report = edited[edited['__next_name__']] + report = report[report['__next_name__']] + report['checksum'] = b'\xbe\xef' + with self.subTest(frame=label, problem='checksum'): + _, messages = self.parse(lambda: self.Ethernet.from_data(edited)) # pylint: disable=cell-var-from-loop + self.assertEqual(len(messages), 2) + self.assertRegex(' '.join(messages), r'IGMPv3: checksum 0xbeef is wrong;') + + cut = whole[:len(whole) - 8] + frame, parsed = self.parse(lambda: self.Ethernet(cut, len(cut))) # pylint: disable=cell-var-from-loop + self.assertNotRegex(' '.join(parsed), 'FieldValueError') + with self.subTest(frame=label, problem='zero-fill ceiling'), \ + mock.patch.object(self.config, 'MAX_ZERO_PAD_LENGTH', 8): + _, messages = self.parse(lambda: self.Ethernet.from_data(frame.info)) # pylint: disable=cell-var-from-loop + self.assertEqual([line.split(':')[0] for line in messages], ['CRITICAL', 'WARNING']) + self.assertRegex(messages[0], r'^CRITICAL: FieldValueError: Field records declares') + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/protocols/test_short_read_rebuild_quiet_2703_unit.py b/tests/protocols/test_short_read_rebuild_quiet_2703_unit.py index d682d654f5..569aac2bbe 100644 --- a/tests/protocols/test_short_read_rebuild_quiet_2703_unit.py +++ b/tests/protocols/test_short_read_rebuild_quiet_2703_unit.py @@ -75,15 +75,15 @@ def wrong(msg: bytes, at: int) -> bytes: #: Each message built on its octets: the module and class it is read as, its octets, #: where its checksum sits, and the cuts that leave its header a short read. Shorter #: than eight octets, an IGMPv3 Query is refused by its class, which classifies a -#: Query by its length, and a Report cut after its header or its first group record -#: ends on a whole field, which records no short read. +#: Query by its length. A Report cut after its header or its first group record ends +#: on a whole field, before the records its count declares, which reads short (#2711). DIRECT = { 'GRE C': ('link.gre', 'GRE', gre(0x8000, b''), 4, range(1, 8)), 'GRE C, K, S': ('link.gre', 'GRE', gre(0xB000, struct.pack('>II', 0x11223344, 7)), 4, range(1, 16)), 'IGMPv1 report': ('internet.igmpv1', 'IGMPv1', message(b'\x12\x00', bytes([224, 0, 0, 1])), 2, range(1, 8)), 'IGMPv2 leave': ('internet.igmpv2', 'IGMPv2', message(b'\x17\x00', bytes([239, 1, 2, 3])), 2, range(1, 8)), 'IGMPv3 report': ('internet.igmpv3', 'IGMPv3', message(b'\x22\x00', bytes.fromhex('00000002') + RECORD * 2), - 2, [*range(1, 8), *range(9, 20), *range(21, 32)]), + 2, range(1, 32)), 'IGMPv3 query': ('internet.igmpv3', 'IGMPv3', message(b'\x11\x64', bytes(4) + bytes.fromhex('027d00010a000001')), 2, range(12, 16)), 'OSPFv2': ('application.ospf', 'OSPF', ospf(), 12, range(1, 24)),