Skip to content

Commit ca5f6db

Browse files
authored
fix(protocols): rebuild the payload from info.to_dict() in from_data (#1447) (#1452)
to_dict() leaves out the dispatch keys, so _make_payload fell back to NoPayload and a layer rebuilt from its dict lost everything past its own header. The dict is left as it is, since the dumps are written from it; the payload is found from its structure instead. - ProtocolBase._lookup_payload: the payload is the last key the data model does not declare whose value is a mapping, matched against the info_name of each protocol in __proto__ and its fallback. - IPv6._lookup_exthdr: recover the extension header chain by walking next from the keys the headers were written under. - HTTP.from_data: rebuild a dict into the HTTP/1.* or HTTP/2 data model by its keys, since the dispatcher's own model carries no version. - test_capture_roundtrip_runtime: drop the #1447 DICT_GAPS rows.
1 parent ef878e4 commit ca5f6db

5 files changed

Lines changed: 367 additions & 28 deletions

File tree

‎pcapkit/protocols/application/http.py‎

Lines changed: 30 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,18 +16,19 @@
1616
"""
1717
import contextlib
1818
import struct
19-
from typing import TYPE_CHECKING, Generic
19+
from typing import TYPE_CHECKING, Generic, cast
2020

2121
from pcapkit.corekit.protochain import ProtoChain
2222
from pcapkit.protocols.application.application import Application
23+
from pcapkit.protocols.data.data import Data
2324
from pcapkit.protocols.misc.null import NoPayload
2425
from pcapkit.protocols.protocol import _PT, _ST
2526
from pcapkit.utilities.exceptions import ProtocolError
2627

2728
if TYPE_CHECKING:
2829
from typing import Any, Optional
2930

30-
from typing_extensions import Literal
31+
from typing_extensions import Literal, Self
3132

3233
__all__ = ['HTTP']
3334

@@ -296,6 +297,33 @@ def pack(self, **kwargs: 'Any') -> 'bytes':
296297
# Utilities.
297298
##########################################################################
298299

300+
@classmethod
301+
def from_data(cls, data: '_PT | dict[str, Any]', **kwargs: 'Any') -> 'Self':
302+
"""Create protocol instance from data.
303+
304+
Args:
305+
data: Protocol data. A :obj:`dict`, as :meth:`Info.to_dict
306+
<pcapkit.corekit.infoclass.Info.to_dict>` returns it, is rebuilt
307+
into the versioned data model its keys belong to -- ``receipt``
308+
is HTTP/1.*'s and ``sid`` HTTP/2's -- since this class's own data
309+
model carries no version.
310+
**kwargs: As :meth:`ProtocolBase.from_data
311+
<pcapkit.protocols.protocol.ProtocolBase.from_data>`.
312+
313+
Returns:
314+
Protocol instance.
315+
316+
"""
317+
from pcapkit.protocols.data.application.httpv1 import HTTP as Data_HTTPv1 # isort: skip # pylint: disable=line-too-long,import-outside-toplevel
318+
from pcapkit.protocols.data.application.httpv2 import HTTP as Data_HTTPv2 # isort: skip # pylint: disable=line-too-long,import-outside-toplevel
319+
320+
if not isinstance(data, Data) and not issubclass(cls.__data__, (Data_HTTPv1, Data_HTTPv2)):
321+
if 'receipt' in data:
322+
data = cast('_PT', Data_HTTPv1.from_dict(data))
323+
elif 'sid' in data:
324+
data = cast('_PT', Data_HTTPv2.from_dict(data))
325+
return super().from_data(data, **kwargs)
326+
299327
@classmethod
300328
def _make_data(cls, data: '_PT') -> 'dict[str, Any]': # type: ignore[override]
301329
"""Create key-value pairs from ``data`` for protocol construction.

‎pcapkit/protocols/internet/ipv6.py‎

Lines changed: 51 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -359,12 +359,18 @@ def _make_data(cls, data: 'Data_IPv6') -> 'dict[str, Any]': # type: ignore[over
359359
Key-value pairs for protocol construction.
360360
361361
"""
362-
upper = cls._make_payload(data)
362+
exthdr = data.get('__exthdr__') # type: Optional[tuple[tuple[Type[ProtocolBase], Any], ...]] # pylint: disable=line-too-long
363+
exclude = () # type: tuple[str, ...]
364+
if exthdr is None:
365+
# NOTE: ``data`` was rebuilt from :meth:`Info.to_dict`, which leaves
366+
# ``__exthdr__`` out, so the chain is recovered from the keys the
367+
# extension headers were written under; those are no payload.
368+
exclude, exthdr = cls._lookup_exthdr(data)
369+
upper = cls._make_payload(data, exclude=exclude)
363370
payload = upper # type: bytes | ProtocolBase
364371

365372
# NOTE: The extension headers sit between this header and the upper
366373
# layer payload, so they are rebuilt in wire order ahead of it.
367-
exthdr = data.get('__exthdr__')
368374
if exthdr:
369375
payload = b''.join(proto.from_data(info).data for proto, info in exthdr) + upper.data
370376

@@ -385,6 +391,49 @@ def _make_data(cls, data: 'Data_IPv6') -> 'dict[str, Any]': # type: ignore[over
385391
'trailer': data.get('trailer', b''),
386392
}
387393

394+
@classmethod
395+
def _lookup_exthdr(cls, data: 'Data_IPv6') -> 'tuple[tuple[str, ...], tuple[tuple[Type[ProtocolBase], Any], ...]]': # pylint: disable=line-too-long
396+
"""Recover the extension header chain of ``data`` without ``__exthdr__``.
397+
398+
:meth:`_decode_next_layer` writes each extension header's info under a
399+
key of its own, in wire order, ahead of the upper layer. Walking the
400+
``next`` field from :attr:`data.next <Data_IPv6.next>` names the parser
401+
of each in turn, exactly as the walk that parsed them did.
402+
403+
Args:
404+
data: protocol data, without ``__exthdr__``
405+
406+
Returns:
407+
The keys the extension headers are under, and the chain in the
408+
shape of ``__exthdr__``, i.e. the parser class and info of each.
409+
410+
Note:
411+
A header repeated in the chain is written under the key of its
412+
predecessor, so only the last of them is left in ``data``.
413+
414+
"""
415+
names = [] # type: list[str]
416+
chain = [] # type: list[tuple[Type[ProtocolBase], Any]]
417+
proto = data.get('next')
418+
for key in cls._lookup_nested(data):
419+
try:
420+
ex_proto = Enum_ExtensionHeader(proto)
421+
except ValueError:
422+
break
423+
424+
# NOTE: As in :meth:`_decode_next_layer`, a code with no parser
425+
# carrying ``next`` ends the chain, and so does a fragment header.
426+
info = data[key]
427+
if 'next' not in info:
428+
break
429+
names.append(key)
430+
chain.append((cls._lookup_next_layer(cls.__proto__, int(ex_proto)), info))
431+
432+
proto = info['next']
433+
if ex_proto == Enum_ExtensionHeader.IPv6_Frag:
434+
break
435+
return tuple(names), tuple(chain)
436+
388437
def _read_ip_hextet(self) -> 'tuple[int, int, int]':
389438
"""Read the first four octets of IPv6.
390439

‎pcapkit/protocols/protocol.py‎

Lines changed: 95 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
import abc
1515
import ast
1616
import collections
17+
import collections.abc
1718
import contextlib
1819
import difflib
1920
import enum
@@ -1907,7 +1908,7 @@ def _make_data(cls, data: 'Data') -> 'dict[str, Any]':
19071908
return data.to_dict()
19081909

19091910
@classmethod
1910-
def _make_payload(cls, data: 'Data') -> 'ProtocolBase':
1911+
def _make_payload(cls, data: 'Data', exclude: 'tuple[str, ...]' = ()) -> 'ProtocolBase':
19111912
"""Create payload from ``data`` for protocol construction.
19121913
19131914
This method uses ``__next_type__`` and ``__next_name__`` to
@@ -1916,13 +1917,28 @@ def _make_payload(cls, data: 'Data') -> 'ProtocolBase':
19161917
instance will be returned. Otherwise, the payload will be
19171918
constructed by :meth:`Protocol.from_data <pcapkit.protocols.protocol.Protocol.from_data>`.
19181919
1920+
:meth:`Info.to_dict <pcapkit.corekit.infoclass.Info.to_dict>` leaves
1921+
both keys out, so ``data`` rebuilt from such a :obj:`dict` carries
1922+
neither. The payload is then found from the structure of ``data``
1923+
instead, by :meth:`_lookup_payload`.
1924+
19191925
Args:
19201926
data: protocol data
1927+
exclude: keys of ``data`` that are not the payload, though they may
1928+
look like it, passed on to :meth:`_lookup_payload`
19211929
19221930
Returns:
19231931
Payload for protocol construction.
19241932
19251933
"""
1934+
if '__next_type__' not in data and '__next_name__' not in data:
1935+
proto, name = cls._lookup_payload(data, exclude)
1936+
if proto is None or name is None:
1937+
from pcapkit.protocols.misc.null import \
1938+
NoPayload # pylint: disable=import-outside-toplevel
1939+
return NoPayload()
1940+
return proto.from_data(data[name])
1941+
19261942
proto = cast('Optional[Type[Protocol]]', data.get('__next_type__'))
19271943
if proto is None or not (isinstance(proto, type) and issubclass(proto, ProtocolBase)):
19281944
from pcapkit.protocols.misc.null import \
@@ -1937,6 +1953,82 @@ def _make_payload(cls, data: 'Data') -> 'ProtocolBase':
19371953

19381954
return proto.from_data(data[name])
19391955

1956+
@classmethod
1957+
def _lookup_payload(cls, data: 'Data', exclude: 'tuple[str, ...]' = ()) -> 'tuple[Optional[Type[ProtocolBase]], Optional[str]]': # pylint: disable=line-too-long
1958+
"""Find the payload of ``data`` that carries no ``__next_type__``.
1959+
1960+
:meth:`_decode_next_layer` writes the payload's info under the payload's
1961+
:attr:`info_name`, after every field of the data model, so the payload is
1962+
the *last* key that the data model does not declare and whose value is a
1963+
mapping. Its protocol is the one of :attr:`__proto__` -- or its fallback
1964+
-- whose :attr:`info_name` the key is. That is the lowercased class name
1965+
by default, but a protocol may override it: every version of
1966+
:class:`~pcapkit.protocols.link.l2tp.L2TP` is ``l2tp``.
1967+
1968+
Args:
1969+
data: protocol data
1970+
exclude: keys of ``data`` to pass over, such as the IPv6 extension
1971+
headers, which are written like a payload but are not one
1972+
1973+
Returns:
1974+
The payload protocol and the key of its info in ``data``, or
1975+
:data:`None` for both if ``data`` has no payload this protocol can
1976+
dispatch to.
1977+
1978+
"""
1979+
keys = [key for key in cls._lookup_nested(data) if key not in exclude]
1980+
if not keys:
1981+
return None, None
1982+
name = keys[-1]
1983+
1984+
registry = cls.__proto__
1985+
candidates = list(registry.values())
1986+
if registry.default_factory is not None:
1987+
candidates.append(registry.default_factory())
1988+
for entry in candidates:
1989+
proto = entry.klass if isinstance(entry, ModuleDescriptor) else entry
1990+
if not (isinstance(proto, type) and issubclass(proto, ProtocolBase)):
1991+
continue
1992+
1993+
# NOTE: :attr:`info_name` is an instance property, so it is read off
1994+
# a bare instance; ``__new__`` only sets up the per-instance caches.
1995+
# An abstract class cannot be instantiated, and is never a payload.
1996+
try:
1997+
label = proto.__new__(proto).info_name
1998+
except TypeError:
1999+
continue
2000+
if label == name:
2001+
return proto, name
2002+
return None, None
2003+
2004+
@staticmethod
2005+
def _lookup_nested(data: 'Data') -> 'list[str]':
2006+
"""Keys of ``data`` that its data model does not declare and that hold a mapping.
2007+
2008+
These are where the next layer protocols wrote their info, in the order
2009+
they wrote it -- the payload, and the IPv6 extension headers.
2010+
2011+
Args:
2012+
data: protocol data
2013+
2014+
Returns:
2015+
The keys, in the order of ``data``.
2016+
2017+
"""
2018+
# NOTE: :func:`inspect.get_annotations` is new in Python 3.10; below it
2019+
# a class's own annotations are in its ``__dict__``, where from 3.14 on
2020+
# they are evaluated lazily and may not be.
2021+
get_annotations = getattr(inspect, 'get_annotations', None)
2022+
2023+
declared = set() # type: set[str]
2024+
for klass in type(data).mro():
2025+
if get_annotations is None:
2026+
declared.update(klass.__dict__.get('__annotations__', {}))
2027+
else:
2028+
declared.update(get_annotations(klass))
2029+
return [key for key in data
2030+
if key not in declared and isinstance(data[key], collections.abc.Mapping)]
2031+
19402032
@staticmethod
19412033
def _lookup_registry(registry: 'DefaultDict[Any, _VT]', code: 'Any') -> '_VT':
19422034
"""Look up a dispatch registry entry without recording a miss.
@@ -2041,7 +2133,8 @@ def _decode_next_layer(self, dict_: '_PT', proto: 'int', length: 'Optional[int]'
20412133
We added a new key ``__next_type__`` to ``dict_`` to store the
20422134
next layer protocol type, and a new key ``__next_name__`` to
20432135
store the next layer protocol name. These two keys will **NOT**
2044-
be included when :meth:`Info.to_dict <pcapkit.corekit.infoclass.Info.to_dict>` is called.
2136+
be included when :meth:`Info.to_dict <pcapkit.corekit.infoclass.Info.to_dict>` is called;
2137+
:meth:`_lookup_payload` finds the payload of such a :obj:`dict` instead.
20452138
20462139
"""
20472140
next_ = cast('ProtocolBase', self._import_next_layer(proto, length, packet=packet)) # type: ignore[misc,call-arg,redundant-cast]

‎tests/protocols/test_capture_roundtrip_runtime.py‎

Lines changed: 2 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -63,17 +63,6 @@ class Gap(NamedTuple):
6363
#: IPv6 extension-header rebuild (#1446).
6464
LAYER_GAPS: 'dict[str, Gap]' = {}
6565

66-
_DICT_DROPS_PAYLOAD = (
67-
'from_data(info.to_dict()) rebuilds the header only: to_dict() leaves out the '
68-
'__next_type__ and __next_name__ keys _make_payload dispatches on '
69-
'(pcapkit/protocols/protocol.py:1926), so the payload becomes NoPayload. '
70-
'IPv6 also loses its extension headers, which _make_data reads from the '
71-
'equally dropped __exthdr__ key (pcapkit/protocols/internet/ipv6.py:367). '
72-
'Repro: Ethernet.from_data(eth.info.to_dict()).data is 14 of 1870 octets '
73-
'for ipv4.pcap frame 1.'
74-
)
75-
76-
7766
class DictGap(NamedTuple):
7867
"""A protocol whose dict-form rebuild does not close."""
7968

@@ -89,17 +78,8 @@ class DictGap(NamedTuple):
8978

9079

9180
#: Every protocol, by class name, whose rebuild from ``info.to_dict()`` fails.
92-
DICT_GAPS = {
93-
**{name: DictGap(1447, 'TRUNCATED', 'payload', _DICT_DROPS_PAYLOAD)
94-
for name in ('Frame', 'Ethernet', 'ARP', 'IPv4', 'IPv6', 'TCP', 'UDP')},
95-
'HTTP': DictGap(
96-
1447, 'ProtocolError', 'always',
97-
"HTTP.from_data(info.to_dict()) raises ProtocolError('invalid HTTP data: "
98-
"Raw'): the dict is rebuilt into a generic data model that "
99-
'HTTP._make_data does not accept. Repro: the first HTTP layer of '
100-
'http.pcap.',
101-
),
102-
}
81+
#: Empty since the payload is found from the dict's structure (#1447).
82+
DICT_GAPS: 'dict[str, DictGap]' = {}
10383

10484

10585
def captures() -> 'list[str]':

0 commit comments

Comments
 (0)