fix(internet)!: read a cut after an IPv6 option's type octet as a short read (#2727)

docs/source/contributing/conventions/utilities-or-corekit.rst

@@ -84,6 +84,14 @@ through them, never through an ``unpack`` override of its own or a length callab
 clamped to the octets present, which hides the cut from the record. Ruled in review
 of :pr:`2682` (:issue:`2680`), and applied to option areas on :issue:`2694`.
 
+A field callback tells a value the cut left as zero from one read through the same
+record: :meth:`Schema.unpack <pcapkit.protocols.schema.schema.Schema.unpack>` gives
+it to the fields after the one the data ends inside, as ``__short_read__`` in their
+packet. A callback that would refuse such a zero leaves it to the short read, and
+refuses a value read with a :exc:`~pcapkit.utilities.exceptions.ProtocolError`, as
+the ``Opt Data Len`` helpers of
+:mod:`pcapkit.protocols.schema.internet.ipv6_option` do (:issue:`2727`).
+
 **A helper in one protocol's own schema module moves there too once code outside that
 protocol uses it and it is generic enough to reuse**, so that another implementor can
 find it; a helper only its own protocol calls stays beside the schema. Ruled on

pcapkit/protocols/schema/internet/ipv6_option.py

@@ -23,7 +23,7 @@
 from pcapkit.corekit.fields.ipaddress import IPv4AddressField, IPv6AddressField
 from pcapkit.corekit.fields.misc import NoValueField, SchemaField
 from pcapkit.corekit.fields.strings import BytesField
-from pcapkit.utilities.exceptions import FieldValueError
+from pcapkit.utilities.exceptions import FieldValueError, ProtocolError
 
 __all__ = [
     'rpl_opt_sub_tlv_len', 'mpl_opt_seed_id_len', 'smf_i_dpd_id_len',
@@ -38,6 +38,23 @@
     from pcapkit.protocols.schema.schema import EnumMeta, Schema
 
 
+def _opt_data_len_cut(pkt: 'dict[str, Any]') -> 'bool':
+    """Whether the data ends before an option's ``Opt Data Len``.
+
+    Args:
+        pkt: Option unpacked schema.
+
+    Returns:
+        Whether the field :meth:`Schema.unpack
+        <pcapkit.protocols.schema.schema.Schema.unpack>` gives as the one the
+        data ends inside is ``len``, so that its value is the zero a capture cut
+        leaves rather than a length read (:issue:`2727`).
+
+    """
+    record = pkt.get('__short_read__')
+    return record is not None and record[0] == 'len'
+
+
 def rpl_opt_sub_tlv_len(pkt: 'dict[str, Any]') -> 'int':
     """Return RPL option sub-TLV length.
 
@@ -87,17 +104,27 @@ def smf_i_dpd_id_len(pkt: 'dict[str, Any]', *, prefix: 'str') -> 'int':
         prefix: Name of the extension header, leading the error message.
 
     Returns:
-        SMF I-DPD identifier length.
+        SMF I-DPD identifier length, or zero if the data ends before
+        ``Opt Data Len``, which leaves the option to the short read.
 
     Raises:
-        FieldValueError: If ``Opt Data Len`` on the wire is too short to hold
+        ProtocolError: If ``Opt Data Len`` on the wire is too short to hold
             the TaggerID it declares, which would otherwise underflow the
             identifier length below zero.
 
+    .. versionchanged:: 1.6.0
+
+       A length the data ends before is left to the short read, and one too
+       short raises :exc:`~pcapkit.utilities.exceptions.ProtocolError` rather
+       than :exc:`~pcapkit.utilities.exceptions.FieldValueError`
+       (:issue:`2727`).
+
     """
     length = pkt['len'] - (1 if pkt['info']['type'] == 0 else (pkt['info']['len'] + 2))
     if length < 0:
-        raise FieldValueError(f'{prefix}: invalid SMF I-DPD option length: {pkt["len"]}')
+        if _opt_data_len_cut(pkt):
+            return 0
+        raise ProtocolError(f'{prefix}: invalid SMF I-DPD option length: {pkt["len"]}')
     return length
 
 
@@ -120,8 +147,8 @@ def smf_dpd_data_selector(pkt: 'dict[str, Any]', *, prefix: 'str',
           wrapped ``SMFHashBasedDPDOption`` instance.
 
     Raises:
-        FieldValueError: If ``Opt Data Len`` is ``0``, or if no schema is
-            registered for the DPD mode.
+        ProtocolError: If ``Opt Data Len`` is ``0`` on the wire.
+        FieldValueError: If no schema is registered for the DPD mode.
 
     Note:
         The field is sized ``Opt Data Len + 2`` rather than ``Opt Data Len``.
@@ -133,11 +160,26 @@ def smf_dpd_data_selector(pkt: 'dict[str, Any]', *, prefix: 'str',
         :class:`~pcapkit.corekit.fields.collections.OptionField` would then
         mis-count against the option area.
 
+        Where the data ends before ``Opt Data Len``, the forward match reads
+        it as zero, and the option is the short read of the schema chosen by
+        the mode bit, likewise zero.
+
+    .. versionchanged:: 1.6.0
+
+       A length the data ends before is left to the short read, and a zero
+       on the wire raises :exc:`~pcapkit.utilities.exceptions.ProtocolError`
+       rather than :exc:`~pcapkit.utilities.exceptions.FieldValueError`
+       (:issue:`2727`).
+
     """
     # NOTE: With ``Opt Data Len`` 0 the option has no data octet, so the mode
-    # bit forward-matched above belongs to whatever follows the option.
-    if pkt['test']['len'] == 0:
-        raise FieldValueError(f'{prefix}: invalid SMF DPD option length: 0')
+    # bit forward-matched above belongs to whatever follows the option. Unless
+    # the data ends inside the forward match before ``Opt Data Len``, the
+    # option's second octet, which then reads as zero (#2727).
+    record = pkt.get('__short_read__')
+    cut = record is not None and record[0] == 'test' and record[1] < 2
+    if pkt['test']['len'] == 0 and not cut:
+        raise ProtocolError(f'{prefix}: invalid SMF DPD option length: 0')
 
     mode = Enum_SMFDPDMode.get(pkt['test']['mode'])
     schema: 'Optional[Type[Schema]]' = base.registry[mode]
@@ -268,18 +310,28 @@ def calipso_pad_len(pkt: 'dict[str, Any]', *, prefix: 'str') -> 'int':
         prefix: Name of the extension header, leading the error message.
 
     Returns:
-        CALIPSO option padding length.
+        CALIPSO option padding length, or zero if the data ends before
+        ``Opt Data Len``, which leaves the option to the short read.
 
     Raises:
-        FieldValueError: If ``Opt Data Len`` on the wire is too short to hold
+        ProtocolError: If ``Opt Data Len`` on the wire is too short to hold
             the fixed header and the compartment bitmap declared by
             ``cmpt_len``, which would otherwise underflow the padding length
             below zero.
 
+    .. versionchanged:: 1.6.0
+
+       A length the data ends before is left to the short read, and one too
+       short raises :exc:`~pcapkit.utilities.exceptions.ProtocolError` rather
+       than :exc:`~pcapkit.utilities.exceptions.FieldValueError`
+       (:issue:`2727`).
+
     """
     length = pkt['len'] - 8 - pkt['cmpt_len'] * 4
     if length < 0:
-        raise FieldValueError(f'{prefix}: invalid CALIPSO option length: {pkt["len"]}')
+        if _opt_data_len_cut(pkt):
+            return 0
+        raise ProtocolError(f'{prefix}: invalid CALIPSO option length: {pkt["len"]}')
     return length
 
 
@@ -291,16 +343,26 @@ def mpl_opt_pad_len(pkt: 'dict[str, Any]', *, prefix: 'str') -> 'int':
         prefix: Name of the extension header, leading the error message.
 
     Returns:
-        MPL option padding length.
+        MPL option padding length, or zero if the data ends before
+        ``Opt Data Len``, which leaves the option to the short read.
 
     Raises:
-        FieldValueError: If ``Opt Data Len`` on the wire is too short to hold
+        ProtocolError: If ``Opt Data Len`` on the wire is too short to hold
             the fixed header and the Seed-ID declared by ``flags.type``, which
             would otherwise underflow the padding length below zero.
 
+    .. versionchanged:: 1.6.0
+
+       A length the data ends before is left to the short read, and one too
+       short raises :exc:`~pcapkit.utilities.exceptions.ProtocolError` rather
+       than :exc:`~pcapkit.utilities.exceptions.FieldValueError`
+       (:issue:`2727`).
+
     """
     length = pkt['len'] - 2 - (0 if pkt['flags']['type'] == 0
                                else mpl_opt_seed_id_len(pkt, prefix=prefix))
     if length < 0:
-        raise FieldValueError(f'{prefix}: invalid MPL option length: {pkt["len"]}')
+        if _opt_data_len_cut(pkt):
+            return 0
+        raise ProtocolError(f'{prefix}: invalid MPL option length: {pkt["len"]}')
     return length

pcapkit/protocols/schema/schema.py

@@ -738,13 +738,32 @@ def unpack(cls, data: 'bytes | IO[bytes]',
             remainder itself rather than the same number of octets from beyond
             it, and record a cut inside it (:issue:`2685`).
 
+            The fields after the one the data ends inside find it in ``packet``
+            as ``__short_read__``: the ``(name, octets read)`` pair recorded as
+            the short read, or that of a
+            :class:`~pcapkit.corekit.fields.misc.ForwardMatchField` the data
+            ends inside first, which the record leaves out as it consumes
+            nothing. So a field callback can tell a value the end of the data
+            left as zero from one read, and leave such a value to the short
+            read rather than refuse it. The key is cleared as unpacking starts,
+            so a nested schema finds only its own.
+
+        .. versionchanged:: 1.6.0
+
+           The field the data ends inside is given to the fields after it, as
+           ``__short_read__`` in ``packet`` (:issue:`2727`).
+
         """
         # force cast arg type since decorator changed their signatures
         if TYPE_CHECKING:
             data = cast('IO[bytes]', data)
             length = cast('int', length)
             packet = cast('dict[str, Any]', packet)
 
+        # NOTE: ``packet`` may be an enclosing schema's, copied into a nested
+        # context, so its short read is not this schema's (#2727).
+        packet.pop('__short_read__', None)
+
         self = cls.__new__(cls)
         for field in self.__fields__.values():
             field = field(packet)
@@ -779,7 +798,7 @@ def unpack(cls, data: 'bytes | IO[bytes]',
 
                 byte = data.read(length)
                 self.__buffer__[field.name] = byte
-                self._note_short_read(field.name, length, byte)
+                self._note_short_read(field.name, length, byte, packet)
 
                 packet[field.name] = byte
                 packet['__length__'] -= length
@@ -801,7 +820,7 @@ def unpack(cls, data: 'bytes | IO[bytes]',
             byte = data.read(min(length, sys.maxsize))
             self.__buffer__[field.name] = byte
             if not isinstance(field, (ForwardMatchField, OptionField)):
-                self._note_short_read(field.name, length, byte)
+                self._note_short_read(field.name, length, byte, packet)
 
             value = field.unpack(byte, packet.copy())
             if isinstance(field, SchemaField) and isinstance(value, Schema):
@@ -843,6 +862,12 @@ def unpack(cls, data: 'bytes | IO[bytes]',
                 # preceding octet as their first.
                 data.seek(-len(byte), io.SEEK_CUR)
                 self.__buffer__[field.name] = b''
+
+                # NOTE: Left out of the record, but the values it matched past the
+                # end of the data are zeros all the same, so the fields after it
+                # are told (#2727).
+                if len(byte) < length:
+                    packet.setdefault('__short_read__', (field.name, len(byte)))
             elif isinstance(field, OptionField) and field.option_padding > 0:
                 # NOTE: The option list ended before the declared area did, and
                 # ``option_padding`` is the rest of that area; give back as much
@@ -856,10 +881,10 @@ def unpack(cls, data: 'bytes | IO[bytes]',
                 self.__buffer__[field.name] = byte[:kept]
                 packet['__length__'] -= consumed
                 if kept == len(byte):
-                    self._note_short_read(field.name, length, byte)
+                    self._note_short_read(field.name, length, byte, packet)
             else:
                 if isinstance(field, OptionField):
-                    self._note_short_read(field.name, length, byte)
+                    self._note_short_read(field.name, length, byte, packet)
                 packet['__length__'] -= length
 
             if packet['__length__'] < 0:
@@ -1187,17 +1212,24 @@ def __delattr__(self, name: 'str') -> 'None':
             return
         return super().__delattr__(name)
 
-    def _note_short_read(self, name: 'str', length: 'int', byte: 'bytes') -> 'None':
+    def _note_short_read(self, name: 'str', length: 'int', byte: 'bytes',
+                         packet: 'Optional[dict[str, Any]]' = None) -> 'None':
         """Record the first field the data ends inside, for :meth:`_cut_short_read`.
 
         Args:
             name: Field name.
             length: Octets the field declares.
             byte: Octets actually read for it.
+            packet: Unpacked data, where the fields after it find the record,
+                unless a forward match the data ended inside is there first
+                (see :meth:`unpack`). Without it, the record is kept on the
+                schema alone.
 
         """
         if len(byte) < length and '__short_read__' not in self.__dict__:
             self.__dict__['__short_read__'] = (name, len(byte))
+            if packet is not None:
+                packet.setdefault('__short_read__', self.__dict__['__short_read__'])
 
     def _cut_short_read(self) -> 'None':
         """Cut the packed buffers back to the octets a short read consumed.

tests/protocols/internet/test_ipv6_extension_unit.py

@@ -1422,7 +1422,7 @@ def _assert_option_schema_helpers_and_post_process_branches(self, schema: types.
         from pcapkit.corekit.fields.ipaddress import IPv4AddressField, IPv6AddressField
         from pcapkit.corekit.fields.misc import NoValueField, SchemaField
         from pcapkit.corekit.fields.strings import BytesField
-        from pcapkit.utilities.exceptions import FieldValueError
+        from pcapkit.utilities.exceptions import FieldValueError, ProtocolError
 
         self.assertEqual(schema.mpl_opt_seed_id_len({'flags': {'type': 0}}), 0)
         self.assertEqual(schema.mpl_opt_seed_id_len({'flags': {'type': 1}}), 2)
@@ -1439,10 +1439,11 @@ def _assert_option_schema_helpers_and_post_process_branches(self, schema: types.
         self.assertEqual(schema.smf_i_dpd_id_len({'len': 7, 'info': {'type': 2, 'len': 3}}), 2)
         # ``Opt Data Len`` too short to hold the TaggerID it declares must raise
         # rather than drive the identifier length negative -- c.f. #438, where the
-        # unguarded subtraction reached :func:`struct.calcsize` as ``'-1s'``.
-        with self.assertRaisesRegex(FieldValueError, 'invalid SMF I-DPD option length'):
+        # unguarded subtraction reached :func:`struct.calcsize` as ``'-1s'``. A
+        # ``ProtocolError``, so that a header refused for it is refused quietly (#2727).
+        with self.assertRaisesRegex(ProtocolError, 'invalid SMF I-DPD option length'):
             schema.smf_i_dpd_id_len({'len': 0, 'info': {'type': 0, 'len': 0}})
-        with self.assertRaisesRegex(FieldValueError, 'invalid SMF I-DPD option length'):
+        with self.assertRaisesRegex(ProtocolError, 'invalid SMF I-DPD option length'):
             schema.smf_i_dpd_id_len({'len': 0, 'info': {'type': 2, 'len': 3}})
 
         for mode in (SMFDPDMode.I_DPD, SMFDPDMode.H_DPD):
@@ -2110,14 +2111,15 @@ def _assert_identification_based_dpd_option_rejects_underflowing_length(self, pr
         identifier length to ``-1``. That reached :func:`struct.calcsize` as the
         template ``'-1s'`` and raised a bare ``struct.error: bad char in struct
         format`` -- not one of pcapkit's own exception types, and uncatchable
-        through :mod:`pcapkit.utilities.exceptions`.
+        through :mod:`pcapkit.utilities.exceptions`. It is a ``ProtocolError``, so
+        that a header refused for it is refused quietly (#2727).
 
         """
-        from pcapkit.utilities.exceptions import FieldValueError
+        from pcapkit.utilities.exceptions import ProtocolError
 
         raw = bytes.fromhex('3b00080000000000')
 
-        with self.assertRaisesRegex(FieldValueError, 'invalid SMF (I-)?DPD option length'):
+        with self.assertRaisesRegex(ProtocolError, 'invalid SMF (I-)?DPD option length'):
             with time_limit():
                 protocol_cls(raw, extension=True)
 
@@ -2143,17 +2145,18 @@ def _assert_calipso_option_rejects_underflowing_length(self, protocol_cls: type)
         length to ``-8``. That reached :func:`struct.calcsize` as the template
         ``'-8s'`` and raised a bare ``struct.error: bad char in struct
         format`` -- not one of pcapkit's own exception types, and uncatchable
-        through :mod:`pcapkit.utilities.exceptions`.
+        through :mod:`pcapkit.utilities.exceptions`. It is a ``ProtocolError``, so
+        that a header refused for it is refused quietly (#2727).
 
         """
-        from pcapkit.utilities.exceptions import FieldValueError
+        from pcapkit.utilities.exceptions import ProtocolError
 
         # next(1) hdr_ext_len(1)=1 -> 16-octet extension header; option
         # type(1)=CALIPSO(0x07) len(1)=0, then domain(4) cmpt_len(1) level(1)
         # checksum(2) all zero, with four trailing zero octets.
         raw = bytes.fromhex('3b0007000000000000000000000000')
 
-        with self.assertRaisesRegex(FieldValueError, 'invalid CALIPSO option length'):
+        with self.assertRaisesRegex(ProtocolError, 'invalid CALIPSO option length'):
             with time_limit():
                 protocol_cls(raw, extension=True)
 
@@ -2178,17 +2181,18 @@ def _assert_mpl_option_rejects_underflowing_length(self, protocol_cls: type) ->
         :func:`struct.calcsize` as the template ``'-2s'`` and raised a bare
         ``struct.error: bad char in struct format`` -- not one of pcapkit's
         own exception types, and uncatchable through
-        :mod:`pcapkit.utilities.exceptions`.
+        :mod:`pcapkit.utilities.exceptions`. It is a ``ProtocolError``, so that a
+        header refused for it is refused quietly (#2727).
 
         """
-        from pcapkit.utilities.exceptions import FieldValueError
+        from pcapkit.utilities.exceptions import ProtocolError
 
         # next(1) hdr_ext_len(1)=0 -> 8-octet extension header; option
         # type(1)=MPL_Option(0x6d) len(1)=0, flags(1)=0 (Seed-ID type 0),
         # seq(1)=0, with two trailing zero octets.
         raw = bytes.fromhex('3b006d0000000000')
 
-        with self.assertRaisesRegex(FieldValueError, 'invalid MPL option length'):
+        with self.assertRaisesRegex(ProtocolError, 'invalid MPL option length'):
             with time_limit():
                 protocol_cls(raw, extension=True)