fix(toolkit): reassemble the first fragmented IP layer, so tunnelled fragments reassemble (#2307)

Problem: the IPv4 and IPv6 reassembly adapters read the frame's outermost
IPv4 or IPv6 layer, so in an IP-in-IP tunnel a fragmented inner datagram was
reassembled whole under the tunnel's header, or dropped (tunnel DF set, or no
Fragment header in the tunnel's IPv6), and never reassembled itself.

Fix: take the first layer of that IP version, from the outside in, that is a
fragment (IPv4 MF or a non-zero offset; an IPv6 Fragment header), else the
outermost as before. Applied to the default engine's PCAP and PCAP-NG
adapters, and to scapy and dpkt, which picked the outermost too.

Test: tests/toolkit/test_tunnelled_ip_reassembly_2307_unit.py reassembles a
fragmented inner datagram (4in4 with DF clear and set, 6in6) and a fragmented
tunnel carrying a whole one, per engine; the inner cases fail on main. The
scapy 6in6 pin now expects the inner packet reassembled on its own header.

Author: JarryShaw

docs/source/pcapkit/toolkit/3rdparty.rst

@@ -46,6 +46,10 @@ Auxiliary Functions
 
 .. autofunction:: pcapkit.toolkit.scapy._within_datagrams
 
+.. autofunction:: pcapkit.toolkit.scapy._fragment
+
+.. autofunction:: pcapkit.toolkit.scapy._fragment_header
+
 .. autofunction:: pcapkit.toolkit.scapy._tcp_carrier
 
 .. autofunction:: pcapkit.toolkit.scapy._tcp_segment
@@ -91,6 +95,8 @@ Auxiliary Functions
 
 .. autofunction:: pcapkit.toolkit.dpkt._capture_time
 
+.. autofunction:: pcapkit.toolkit.dpkt._fragment
+
 Auxiliary Data
 --------------
 

docs/source/pcapkit/toolkit/builtin.rst

@@ -27,6 +27,8 @@ Auxiliary Functions
 
 .. autofunction:: pcapkit.toolkit.pcap.tcp_segment
 
+.. autofunction:: pcapkit.toolkit.pcap._fragment
+
 .. autoclass:: pcapkit.toolkit.pcap.TCPSegment
    :members:
    :show-inheritance:

pcapkit/toolkit/dpkt.py

@@ -529,13 +529,56 @@ def wrapper(packet: 'Packet') -> 'dict[str, Any]':
     }
 
 
+def _fragment(packet: 'Packet', version: 'int') -> 'Optional[IP | IP6]':
+    """Fetch the IPv4 or IPv6 layer of a DPKT packet that IP reassembly reads.
+
+    Args:
+        packet: DPKT packet, i.e. the outermost layer.
+        version: ``4`` or ``6``, the IP version of the layer to fetch.
+
+    Returns:
+        The first layer of that version, from the network layer in, that is a
+        fragment -- for IPv4, one with **MF** set or a non-zero fragment
+        offset, for IPv6, one with an IPv6 Fragment header; with none, the
+        network layer if it is of that version; otherwise :data:`None`.
+
+    Note:
+        As with :func:`pcapkit.toolkit.pcap._fragment`, either datagram of an
+        IP-in-IP tunnel may be the fragmented one, and each is reassembled on
+        its own header (:issue:`2307`). An IPv6 layer is the one with extension
+        headers. The walk goes on past an IP layer only where its payload
+        protocol is IPv4 or IPv6, i.e. a tunnel, so a datagram an ICMP error
+        quotes, which `DPKT`_ decodes as an IP layer below the ICMP one, is not
+        walked to.
+
+    """
+    import dpkt  # isort:skip
+
+    network = _network(packet)
+    layer = network  # type: Any
+    while isinstance(layer, (dpkt.ip.IP, dpkt.ip6.IP6)):
+        if isinstance(layer, dpkt.ip6.IP6):
+            if version == 6 and layer.extension_hdrs.get(44) is not None:
+                return layer
+        elif version == 4 and (layer.mf or layer.offset):
+            return layer
+        # ``IP6.p`` is the protocol after the extension headers
+        if getattr(layer, 'p', None) not in (Enum_TransType.IPv4, Enum_TransType.IPv6):
+            break
+        layer = layer.data
+    if network is None or hasattr(network, 'extension_hdrs') is not (version == 6):
+        return None
+    return network
+
+
 def ipv4_reassembly(packet: 'Packet', timestamp: 'float', *,
                     count: 'int' = -1) -> 'IP_Packet[IPv4Address] | None':
     """Make data for IPv4 reassembly.
 
-    A packet can be reassembled if its network layer is IPv4 (:class:`dpkt.ip.IP`) --
-    one carried in IPv6 is not read -- whose header was captured whole, and the **DF**
-    (:attr:`dpkt.ip.IP.df`) flag is :data:`False`.
+    A packet can be reassembled if it has an IPv4 layer (:class:`dpkt.ip.IP`) whose
+    header was captured whole, and the **DF** (:attr:`dpkt.ip.IP.df`) flag is
+    :data:`False`. The layer is the first that is a fragment, or with none, the network
+    layer if it is IPv4 (see :func:`_fragment`).
 
     Args:
         packet: DPKT packet.
@@ -581,9 +624,7 @@ def ipv4_reassembly(packet: 'Packet', timestamp: 'float', *,
        with ``packet`` alone fails.
 
     """
-    network = _network(packet)
-    # an IPv6 layer is the one with extension headers, as in ``_wire_hdr_len``
-    ipv4 = None if hasattr(network, 'extension_hdrs') else network
+    ipv4 = cast('Optional[IP]', _fragment(packet, 4))
     if ipv4 is not None:
         if ipv4.df:     # dismiss not fragmented packet
             return None
@@ -628,9 +669,10 @@ def ipv6_reassembly(packet: 'Packet', timestamp: 'float', *,
                     count: 'int' = -1) -> 'IP_Packet[IPv6Address] | None':
     """Make data for IPv6 reassembly.
 
-    A packet can be reassembled if its network layer is IPv6 (:class:`dpkt.ip6.IP6`) --
-    one carried in IPv4 is not read -- and has an IPv6 Fragment header
-    (:rfc:`2460#section-4.5`, i.e., :class:`dpkt.ip6.IP6FragmentHeader`).
+    A packet can be reassembled if it has an IPv6 layer (:class:`dpkt.ip6.IP6`) with an
+    IPv6 Fragment header (:rfc:`2460#section-4.5`, i.e.,
+    :class:`dpkt.ip6.IP6FragmentHeader`). The layer is the first that has one (see
+    :func:`_fragment`).
 
     Args:
         packet: DPKT packet.
@@ -663,8 +705,7 @@ def ipv6_reassembly(packet: 'Packet', timestamp: 'float', *,
        with ``packet`` alone fails.
 
     """
-    network = _network(packet)
-    ipv6 = cast('IP6', network) if hasattr(network, 'extension_hdrs') else None
+    ipv6 = cast('Optional[IP6]', _fragment(packet, 6))
     if ipv6 is not None:
         # dpkt keeps each extension header under its protocol number, so 44 is a
         # Fragment header; the dict's value type cannot say so

pcapkit/toolkit/pcap.py

@@ -21,6 +21,7 @@
 from pcapkit.foundation.traceflow.data.tcp import Packet as TF_TCP_Packet
 from pcapkit.protocols.internet.ip import IP as Protocol_IP
 from pcapkit.protocols.internet.ipv4 import IPv4
+from pcapkit.protocols.internet.ipv6 import IPv6
 from pcapkit.protocols.misc.null import NoPayload
 from pcapkit.protocols.misc.raw import Raw
 from pcapkit.protocols.protocol import ProtocolBase
@@ -34,7 +35,6 @@
     from pcapkit.const.reg.linktype import LinkType
     from pcapkit.protocols.data.internet.ipv4 import IPv4 as Data_IPv4
     from pcapkit.protocols.data.internet.ipv6 import IPv6 as Data_IPv6
-    from pcapkit.protocols.internet.ipv6 import IPv6
     from pcapkit.protocols.internet.ipv6_frag import IPv6_Frag
     from pcapkit.protocols.misc.pcap import Frame
 
@@ -193,13 +193,51 @@ def tcp_segment(frame: 'ProtocolBase') -> 'Optional[TCPSegment]':
     )
 
 
+def _fragment(frame: 'ProtocolBase', version: 'type[IPv4 | IPv6]') -> 'Optional[IPv4 | IPv6]':
+    """Fetch the IPv4 or IPv6 layer of a frame that IP reassembly reads.
+
+    Args:
+        frame: PCAP frame or PCAP-NG block.
+        version: :class:`~pcapkit.protocols.internet.ipv4.IPv4` or
+            :class:`~pcapkit.protocols.internet.ipv6.IPv6`, the layer to fetch.
+
+    Returns:
+        The first ``version`` layer, from the outside in, that is a fragment --
+        for IPv4, one with **MF** set or a non-zero Fragment Offset, for IPv6,
+        one with an IPv6 Fragment header; with none, the outermost ``version``
+        layer; with no ``version`` layer, :data:`None`.
+
+    Note:
+        In an IP-in-IP tunnel (:rfc:`2003`, :rfc:`2473`, :rfc:`4213`) either
+        datagram may be the fragmented one: the inner, each fragment of it
+        carried whole by the tunnel, or the tunnel's own, whose first fragment
+        alone carries the inner header. Each is reassembled on its own header
+        (:issue:`2307`). A frame of no tunnel has one layer to choose from.
+
+    """
+    if version.__name__ not in frame:
+        return None
+    outermost = cast('IPv4 | IPv6', frame[version.__name__])
+    layer = outermost  # type: ProtocolBase
+    while isinstance(layer, ProtocolBase) and not isinstance(layer, NoPayload):
+        if isinstance(layer, IPv4) and version is IPv4:
+            if layer.info.flags.mf or layer.info.offset:
+                return layer
+        elif isinstance(layer, IPv6) and version is IPv6:
+            if Enum_ExtensionHeader.IPv6_Frag in layer.extension_headers:
+                return layer
+        layer = layer.payload
+    return outermost
+
+
 def ipv4_reassembly(frame: 'Frame') -> 'IP_Packet[IPv4Address] | None':
     """Make data for IPv4 reassembly.
 
     A frame can be reassembled if it contains an IPv4 layer
     (:class:`~pcapkit.protocols.internet.ipv4.IPv4`) and the **DF**
     (:attr:`IPv4.flags.df <pcapkit.protocols.data.internet.ipv4.Flags.df>`) flag is
-    :data:`False`.
+    :data:`False`. In an IP-in-IP tunnel, the layer is the first that is a fragment
+    (see :func:`_fragment`).
 
     Args:
         frame: PCAP frame.
@@ -212,8 +250,7 @@ def ipv4_reassembly(frame: 'Frame') -> 'IP_Packet[IPv4Address] | None':
         :class:`pcapkit.foundation.reassembly.ipv4.IPv4`
 
     """
-    if 'IPv4' in frame:
-        ipv4 = cast('IPv4', frame['IPv4'])
+    if (ipv4 := cast('Optional[IPv4]', _fragment(frame, IPv4))) is not None:
         ipv4_info = ipv4.info
         if ipv4_info.flags.df:       # dismiss not fragmented frame
             return None
@@ -244,7 +281,8 @@ def ipv6_reassembly(frame: 'Frame') -> 'IP_Packet[IPv6Address] | None':
     A frame can be reassembled if it contains an IPv6 layer
     (:class:`~pcapkit.protocols.internet.ipv6.IPv6`) and an IPv6 Fragment header
     (:rfc:`2460#section-4.5`, i.e.,
-    :class:`~pcapkit.protocols.internet.ipv6_frag.IPv6_Frag`).
+    :class:`~pcapkit.protocols.internet.ipv6_frag.IPv6_Frag`). In an IP-in-IP tunnel,
+    the layer is the first that has one (see :func:`_fragment`).
 
     Args:
         frame: PCAP frame.
@@ -257,8 +295,7 @@ def ipv6_reassembly(frame: 'Frame') -> 'IP_Packet[IPv6Address] | None':
         :class:`pcapkit.foundation.reassembly.ipv6.IPv6`
 
     """
-    if 'IPv6' in frame:
-        ipv6 = cast('IPv6', frame['IPv6'])
+    if (ipv6 := cast('Optional[IPv6]', _fragment(frame, IPv6))) is not None:
         ipv6_info = ipv6.info
         if (ipv6_frag := ipv6.extension_headers.get(  # type: ignore[call-overload]
             Enum_ExtensionHeader.IPv6_Frag

pcapkit/toolkit/pcapng.py

@@ -18,14 +18,15 @@
 from pcapkit.foundation.traceflow.data.tcp import Packet as TF_TCP_Packet
 from pcapkit.protocols.data.misc.pcap.frame import Frame as Data_Frame
 from pcapkit.protocols.data.misc.pcap.frame import FrameInfo as Data_FrameInfo
+from pcapkit.protocols.internet.ipv4 import IPv4
+from pcapkit.protocols.internet.ipv6 import IPv6
 from pcapkit.protocols.schema.misc.pcap.header import RESOLUTION
-from pcapkit.toolkit.pcap import tcp_segment
+from pcapkit.toolkit.pcap import _fragment, tcp_segment
 
 if TYPE_CHECKING:
     from ipaddress import IPv4Address, IPv6Address
+    from typing import Optional
 
-    from pcapkit.protocols.internet.ipv4 import IPv4
-    from pcapkit.protocols.internet.ipv6 import IPv6
     from pcapkit.protocols.internet.ipv6_frag import IPv6_Frag
     from pcapkit.protocols.misc.pcapng import PCAPNG, Packet
 
@@ -39,7 +40,8 @@ def ipv4_reassembly(frame: 'PCAPNG') -> 'IP_Packet[IPv4Address] | None':
     A frame can be reassembled if it contains an IPv4 layer
     (:class:`~pcapkit.protocols.internet.ipv4.IPv4`) and the **DF**
     (:attr:`IPv4.flags.df <pcapkit.protocols.data.internet.ipv4.Flags.df>`) flag is
-    :data:`False`.
+    :data:`False`. In an IP-in-IP tunnel, the layer is the first that is a fragment
+    (see :func:`pcapkit.toolkit.pcap._fragment`).
 
     Args:
         frame: PCAP-NG frame.
@@ -52,8 +54,7 @@ def ipv4_reassembly(frame: 'PCAPNG') -> 'IP_Packet[IPv4Address] | None':
         :class:`pcapkit.foundation.reassembly.ipv4.IPv4`
 
     """
-    if 'IPv4' in frame:
-        ipv4 = cast('IPv4', frame['IPv4'])
+    if (ipv4 := cast('Optional[IPv4]', _fragment(frame, IPv4))) is not None:
         ipv4_info = ipv4.info
         if ipv4_info.flags.df:       # dismiss not fragmented frame
             return None
@@ -85,7 +86,8 @@ def ipv6_reassembly(frame: 'PCAPNG') -> 'IP_Packet[IPv6Address] | None':
     A frame can be reassembled if it contains an IPv6 layer
     (:class:`~pcapkit.protocols.internet.ipv6.IPv6`) and an IPv6 Fragment header
     (:rfc:`2460#section-4.5`, i.e.,
-    :class:`~pcapkit.protocols.internet.ipv6_frag.IPv6_Frag`).
+    :class:`~pcapkit.protocols.internet.ipv6_frag.IPv6_Frag`). In an IP-in-IP tunnel,
+    the layer is the first that has one (see :func:`pcapkit.toolkit.pcap._fragment`).
 
     Args:
         frame: PCAP-NG frame.
@@ -98,8 +100,7 @@ def ipv6_reassembly(frame: 'PCAPNG') -> 'IP_Packet[IPv6Address] | None':
         :class:`pcapkit.foundation.reassembly.ipv6.IPv6`
 
     """
-    if 'IPv6' in frame:
-        ipv6 = cast('IPv6', frame['IPv6'])
+    if (ipv6 := cast('Optional[IPv6]', _fragment(frame, IPv6))) is not None:
         ipv6_info = ipv6.info
         if (ipv6_frag := ipv6.extension_headers.get(  # type: ignore[call-overload]
             Enum_ExtensionHeader.IPv6_Frag