Skip to content

Commit 07ceaf7

Browse files
authored
refactor!: hold mapping values in Info or Data, not dict subclasses (#2726) (#2734)
- FrameRecord is an Info rather than a slotted dict subclass, so it is immutable and no dict. Its mapping is the engine's dissection; packet, frame_info and time_epoch sit under hidden dunder keys, read through __getattribute__, so a mapping key of the same name never shadows them. It pickles through _frame_record, as the 1.5 releases do, and to_dict() exports the mapping alone. - TLSKeyLog.entries is a TLSKeyLogEntries data model keyed by label, not a plain dict, and a mapping given for it is stored as one. to_dict() exports it as an OrderedMultiDict, as #1484 requires of every level; TLSKeyLog pickles hold the entries as a dict and restore the model. - The traced Packet's frame annotation names FrameRecord, and its versionchanged notes what from_dict(to_dict()) gives back; the pickle docs drop FrameRecord as their slotted-dict example. Pickles load across main and 1.5.1 and this change both ways at protocols 0, 1, 2 and 5, except main's own FrameRecord pickles, a format no release wrote. Dumps and from_data rebuilds match main. New tests fail on main.
1 parent af69a79 commit 07ceaf7

12 files changed

Lines changed: 561 additions & 50 deletions

File tree

‎docs/source/pcapkit/protocols/misc/pcapng.rst‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -835,6 +835,10 @@ Data Models
835835
:members:
836836
:show-inheritance:
837837

838+
.. autoclass:: pcapkit.protocols.data.misc.pcapng.TLSKeyLogEntries
839+
:members:
840+
:show-inheritance:
841+
838842
.. autoclass:: pcapkit.protocols.data.misc.pcapng.TLSKeyLog
839843
:members:
840844
:show-inheritance:

‎docs/source/pcapkit/utilities/pickle.rst‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,8 @@ Pickle Reductions
55

66
:mod:`pcapkit.utilities.pickle` holds the :mod:`pickle` reductions several
77
:mod:`pcapkit` classes share, so each is written once (:issue:`2714`).
8-
:class:`ReduceAsProtocol2` lets a :class:`typing.Generic` subclass or a slotted
9-
:class:`dict` pickle under protocol 0 and 1, on GraalPy as on CPython;
8+
:class:`ReduceAsProtocol2` lets a :class:`typing.Generic` subclass or a class with
9+
``__slots__`` pickle under protocol 0 and 1, on GraalPy as on CPython;
1010
:class:`ReduceToName` reduces a singleton to the module-level name holding it;
1111
:class:`ReduceToItems` reduces a multi-valued mapping to a call of its class on its
1212
item pairs; and :func:`reduce_to` makes one instance reduce to a call of a given

‎pcapkit/foundation/traceflow/data/data.py‎

Lines changed: 79 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,12 @@
44
from typing import TYPE_CHECKING
55

66
from pcapkit.corekit.infoclass import Info, info_final
7-
from pcapkit.utilities.pickle import ReduceAsProtocol2
87

98
__all__ = ['TraceFlowData', 'Deferred', 'FrameRecord']
109

1110
if TYPE_CHECKING:
1211
from decimal import Decimal
13-
from typing import Any, Optional
12+
from typing import Any, Mapping, Optional, SupportsIndex
1413

1514
from pcapkit.foundation.reassembly.data.tcp import Datagram as TCP_Datagram
1615
from pcapkit.foundation.reassembly.tcp import TCP as TCP_Reassembly
@@ -67,7 +66,13 @@ def __call__(self) -> 'tuple[TCP_Datagram, ...]':
6766
return self.reassembly.datagram
6867

6968

70-
class FrameRecord(ReduceAsProtocol2, dict):
69+
#: Each attribute a :class:`FrameRecord` carries beside its mapping, to the
70+
#: dunder key holding it, which iteration, ``len()`` and the dumps leave out.
71+
_RECORD = {'packet': '__packet__', 'frame_info': '__frame_info__', 'time_epoch': '__time_epoch__'}
72+
73+
74+
@info_final
75+
class FrameRecord(Info):
7176
"""A frame as a third-party engine's flow tracing adapter reports it.
7277
7378
The mapping is the engine's own dissection of the frame, which every trace
@@ -76,44 +81,93 @@ class FrameRecord(ReduceAsProtocol2, dict):
7681
reads of a frame -- :meth:`PCAPIO._append_value
7782
<pcapkit.dumpkit.pcap.PCAPIO._append_value>` takes ``packet``,
7883
``frame_info`` and ``time_epoch`` -- so ``trace_format='pcap'`` writes the
79-
frame's record too (:issue:`1507`).
84+
frame's record too (:issue:`1507`). A key of the mapping and an attribute
85+
of the same name stay apart: ``record['packet']`` is the engine's, and
86+
``record.packet`` the frame's captured octets.
8087
8188
Args:
8289
mapping: The engine's dissection of the frame.
8390
packet: The frame's captured octets.
8491
frame_info: The frame's PCAP record header.
8592
time_epoch: The frame's exact UNIX timestamp.
8693
87-
"""
94+
.. versionchanged:: 1.6.0
95+
96+
An :class:`~pcapkit.corekit.infoclass.Info` rather than a :obj:`dict`
97+
subclass, so it is immutable and no :obj:`dict`. :meth:`to_dict`
98+
exports the mapping alone, as any :class:`~pcapkit.corekit.infoclass.Info`
99+
exports its keys, and drops the ``packet``, ``frame_info`` and
100+
``time_epoch`` attributes (:issue:`2726`).
88101
89-
__slots__ = ('packet', 'frame_info', 'time_epoch')
102+
"""
90103

91-
#: The frame's captured octets.
92-
packet: 'bytes'
93-
#: The frame's PCAP record header, with the fraction in the capture's own
94-
#: resolution.
95-
frame_info: 'Data_FrameInfo'
96-
#: The frame's exact UNIX timestamp, from which
97-
#: :meth:`PCAPIO._make_timestamp <pcapkit.dumpkit.pcap.PCAPIO._make_timestamp>`
98-
#: tells which resolution ``frame_info.ts_usec`` is in. A
99-
#: :class:`~decimal.Decimal`, since only that settles it exactly.
100-
time_epoch: 'Decimal'
104+
#: The dunder keys holding the attributes, which ``repr`` and ``str``
105+
#: leave out as well.
106+
__excluded__ = list(_RECORD.values())
101107

102-
def __init__(self, mapping: 'dict[str, Any]', *, packet: 'bytes',
108+
# NOTE: annotated for the type checker alone, since the attributes are no
109+
# fields of the mapping, which is the engine's.
110+
if TYPE_CHECKING:
111+
#: The frame's captured octets.
112+
packet: 'bytes'
113+
#: The frame's PCAP record header, with the fraction in the capture's own
114+
#: resolution.
115+
frame_info: 'Data_FrameInfo'
116+
#: The frame's exact UNIX timestamp, from which
117+
#: :meth:`PCAPIO._make_timestamp <pcapkit.dumpkit.pcap.PCAPIO._make_timestamp>`
118+
#: tells which resolution ``frame_info.ts_usec`` is in. A
119+
#: :class:`~decimal.Decimal`, since only that settles it exactly.
120+
time_epoch: 'Decimal'
121+
122+
def __init__(self, mapping: 'Mapping[str, Any]', *, packet: 'bytes',
103123
frame_info: 'Data_FrameInfo', time_epoch: 'Decimal') -> 'None':
104124
super().__init__(mapping)
105-
self.packet = packet
106-
self.frame_info = frame_info
107-
self.time_epoch = time_epoch
125+
self.__dict__.update(zip(_RECORD.values(), (packet, frame_info, time_epoch)))
126+
127+
def __getattribute__(self, name: 'str') -> 'Any':
128+
"""Read an attribute, taking :attr:`packet`, :attr:`frame_info` and
129+
:attr:`time_epoch` from the keys holding them.
130+
131+
So a mapping key of the same name never stands in for one of them.
132+
133+
Args:
134+
name: Attribute name.
135+
136+
Raises:
137+
AttributeError: If ``name`` is one of them and the record carries
138+
none, as one :meth:`from_dict` builds from a mapping.
139+
140+
"""
141+
key = _RECORD.get(name)
142+
if key is None:
143+
return super().__getattribute__(name)
144+
try:
145+
return super().__getattribute__('__dict__')[key]
146+
except KeyError:
147+
raise AttributeError(f'{type(self).__name__!r} object has no attribute {name!r}') from None
148+
149+
def __reduce_ex__(self, protocol: 'SupportsIndex') -> 'tuple[Any, ...]':
150+
"""Reduce to a call of :func:`_frame_record`, under every protocol.
151+
152+
A release before 1.6.0 reduces a record to it as well, so a pickle
153+
loads on either side of 1.6.0.
154+
155+
Args:
156+
protocol: Pickle protocol in use, which does not change the reduction.
157+
158+
Returns:
159+
``(_frame_record, (mapping, packet, frame_info, time_epoch))``.
160+
161+
"""
162+
return _frame_record, (dict(self), self.packet, self.frame_info, self.time_epoch)
108163

109164

110-
def _frame_record(mapping: 'dict[str, Any]', packet: 'bytes',
165+
def _frame_record(mapping: 'Mapping[str, Any]', packet: 'bytes',
111166
frame_info: 'Data_FrameInfo', time_epoch: 'Decimal') -> 'FrameRecord':
112-
"""Rebuild a :class:`FrameRecord` pickled by a release before 1.6.0.
167+
"""Rebuild a pickled :class:`FrameRecord`.
113168
114-
Those pickles call this; a :class:`FrameRecord` reduces through
115-
:class:`~pcapkit.utilities.pickle.ReduceAsProtocol2` instead, which carries
116-
the slots under every protocol (:issue:`2714`).
169+
Its pickles call this under every protocol, as those of a release before
170+
1.6.0 do.
117171
118172
Args:
119173
mapping: The engine's dissection of the frame.

‎pcapkit/foundation/traceflow/data/tcp.py‎

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
from pcapkit.const.reg.linktype import LinkType as Enum_LinkType
2121
from pcapkit.foundation.reassembly.data.tcp import Datagram as TCP_Datagram
2222
from pcapkit.foundation.reassembly.tcp import TCP as TCP_Reassembly
23+
from pcapkit.foundation.traceflow.data.data import FrameRecord
2324
from pcapkit.protocols.data.misc.pcap.frame import Frame as Data_Frame
2425

2526
_AT = TypeVar('_AT', 'IPv4Address', 'IPv6Address')
@@ -47,15 +48,28 @@ class Packet(Info, Generic[_AT]):
4748
* :meth:`pcapkit.foundation.traceflow.traceflow.TraceFlow.dump`
4849
* :term:`trace.tcp.packet`
4950
51+
.. versionchanged:: 1.6.0
52+
53+
A third-party engine's :attr:`frame` is a
54+
:class:`~pcapkit.foundation.traceflow.data.data.FrameRecord`, which
55+
:meth:`to_dict` exports as an
56+
:class:`~pcapkit.corekit.multidict.OrderedMultiDict`; so
57+
``Packet.from_dict(packet.to_dict())`` holds the frame as an
58+
:class:`~pcapkit.corekit.infoclass.OrderedMultiInfo`, not a
59+
:class:`~pcapkit.foundation.traceflow.data.data.FrameRecord`
60+
(:issue:`2726`).
61+
5062
"""
5163

5264
#: Data link type: the PCAP global header's, or that of the PCAP-NG
5365
#: interface the packet was captured on.
5466
protocol: 'Enum_LinkType'
5567
#: Frame number.
5668
index: 'int'
57-
#: Extracted frame info.
58-
frame: 'Data_Frame | dict[str, Any]'
69+
#: Extracted frame info: the default engine's frame, a third-party
70+
#: engine's :class:`~pcapkit.foundation.traceflow.data.data.FrameRecord`, or
71+
#: the :obj:`dict` PyShark's adapter reports, which carries no frame record.
72+
frame: 'Data_Frame | FrameRecord | dict[str, Any]'
5973
#: TCP synchronise (SYN) flag.
6074
syn: 'bool'
6175
#: TCP finish (FIN) flag.
@@ -91,7 +105,7 @@ class Packet(Info, Generic[_AT]):
91105
payload: 'bytearray'
92106

93107
if TYPE_CHECKING:
94-
def __init__(self, protocol: 'Enum_LinkType', index: 'int', frame: 'Data_Frame | dict[str, Any]', syn: 'bool', fin: 'bool', rst: 'bool', src: '_AT', dst: '_AT', srcport: 'int', dstport: 'int', timestamp: 'float', seq: 'int', ack: 'int', header: 'bytes', payload: 'bytearray') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long
108+
def __init__(self, protocol: 'Enum_LinkType', index: 'int', frame: 'Data_Frame | FrameRecord | dict[str, Any]', syn: 'bool', fin: 'bool', rst: 'bool', src: '_AT', dst: '_AT', srcport: 'int', dstport: 'int', timestamp: 'float', seq: 'int', ack: 'int', header: 'bytes', payload: 'bytearray') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long
95109

96110

97111
@info_final

‎pcapkit/protocols/data/__init__.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,8 +49,8 @@
4949
'PCAPNG_ISB_FilterAcceptOption', 'PCAPNG_ISB_OSDropOption', 'PCAPNG_ISB_UsrDelivOption',
5050
'PCAPNG_PACK_FlagsOption', 'PCAPNG_PACK_HashOption',
5151
'PCAPNG_NameResolutionRecord', 'PCAPNG_UnknownRecord', 'PCAPNG_EndRecord', 'PCAPNG_IPv4Record', 'PCAPNG_IPv6Record',
52-
'PCAPNG_DSBSecrets', 'PCAPNG_UnknownSecrets', 'PCAPNG_TLSKeyLog', 'PCAPNG_WireGuardKeyLog', 'PCAPNG_ZigBeeNWKKey',
53-
'PCAPNG_ZigBeeAPSKey',
52+
'PCAPNG_DSBSecrets', 'PCAPNG_UnknownSecrets', 'PCAPNG_TLSKeyLogEntries', 'PCAPNG_TLSKeyLog',
53+
'PCAPNG_WireGuardKeyLog', 'PCAPNG_ZigBeeNWKKey', 'PCAPNG_ZigBeeAPSKey',
5454
'PCAPNG_UnknownBlock', 'PCAPNG_SectionHeaderBlock', 'PCAPNG_InterfaceDescriptionBlock',
5555
'PCAPNG_EnhancedPacketBlock', 'PCAPNG_SimplePacketBlock', 'PCAPNG_NameResolutionBlock',
5656
'PCAPNG_InterfaceStatisticsBlock', 'PCAPNG_SystemdJournalExportBlock', 'PCAPNG_DecryptionSecretsBlock',

‎pcapkit/protocols/data/misc/__init__.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@
6565
from pcapkit.protocols.data.misc.pcapng import \
6666
SystemdJournalExportBlock as PCAPNG_SystemdJournalExportBlock
6767
from pcapkit.protocols.data.misc.pcapng import TLSKeyLog as PCAPNG_TLSKeyLog
68+
from pcapkit.protocols.data.misc.pcapng import TLSKeyLogEntries as PCAPNG_TLSKeyLogEntries
6869
from pcapkit.protocols.data.misc.pcapng import UnknownBlock as PCAPNG_UnknownBlock
6970
from pcapkit.protocols.data.misc.pcapng import UnknownOption as PCAPNG_UnknownOption
7071
from pcapkit.protocols.data.misc.pcapng import UnknownRecord as PCAPNG_UnknownRecord
@@ -98,8 +99,8 @@
9899
'PCAPNG_ISB_FilterAcceptOption', 'PCAPNG_ISB_OSDropOption', 'PCAPNG_ISB_UsrDelivOption',
99100
'PCAPNG_PACK_FlagsOption', 'PCAPNG_PACK_HashOption',
100101
'PCAPNG_NameResolutionRecord', 'PCAPNG_UnknownRecord', 'PCAPNG_EndRecord', 'PCAPNG_IPv4Record', 'PCAPNG_IPv6Record',
101-
'PCAPNG_DSBSecrets', 'PCAPNG_UnknownSecrets', 'PCAPNG_TLSKeyLog', 'PCAPNG_WireGuardKeyLog', 'PCAPNG_ZigBeeNWKKey',
102-
'PCAPNG_ZigBeeAPSKey',
102+
'PCAPNG_DSBSecrets', 'PCAPNG_UnknownSecrets', 'PCAPNG_TLSKeyLogEntries', 'PCAPNG_TLSKeyLog',
103+
'PCAPNG_WireGuardKeyLog', 'PCAPNG_ZigBeeNWKKey', 'PCAPNG_ZigBeeAPSKey',
103104
'PCAPNG_UnknownBlock', 'PCAPNG_SectionHeaderBlock', 'PCAPNG_InterfaceDescriptionBlock',
104105
'PCAPNG_EnhancedPacketBlock', 'PCAPNG_SimplePacketBlock', 'PCAPNG_NameResolutionBlock',
105106
'PCAPNG_InterfaceStatisticsBlock', 'PCAPNG_SystemdJournalExportBlock', 'PCAPNG_DecryptionSecretsBlock',

‎pcapkit/protocols/data/misc/pcapng.py‎

Lines changed: 74 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
# -*- coding: utf-8 -*-
22
"""data models for PCAP-NG file format"""
33

4+
import collections.abc
45
import datetime
56
import decimal
67
from typing import TYPE_CHECKING
@@ -29,8 +30,8 @@
2930

3031
'NameResolutionRecord', 'UnknownRecord', 'EndRecord', 'IPv4Record', 'IPv6Record',
3132

32-
'DSBSecrets', 'UnknownSecrets', 'TLSKeyLog', 'WireGuardKeyLog', 'ZigBeeNWKKey',
33-
'ZigBeeAPSKey',
33+
'DSBSecrets', 'UnknownSecrets', 'TLSKeyLogEntries', 'TLSKeyLog', 'WireGuardKeyLog',
34+
'ZigBeeNWKKey', 'ZigBeeAPSKey',
3435

3536
'UnknownBlock', 'SectionHeaderBlock', 'InterfaceDescriptionBlock',
3637
'EnhancedPacketBlock', 'SimplePacketBlock', 'NameResolutionBlock',
@@ -44,6 +45,7 @@
4445
from datetime import timezone as dt_timezone
4546
from decimal import Decimal
4647
from ipaddress import IPv4Address, IPv4Interface, IPv6Address, IPv6Interface
48+
from typing import Any, Mapping, Optional
4749

4850
from typing_extensions import Literal
4951

@@ -829,19 +831,86 @@ class UnknownSecrets(DSBSecrets):
829831
def __init__(self, data: 'bytes') -> 'None': ...
830832

831833

834+
@info_final
835+
class TLSKeyLogEntries(Data):
836+
"""Data model for the entries of a TLS key log, keyed by label.
837+
838+
A key is each :class:`~pcapkit.const.pcapng.tls_key_label.TLSKeyLabel`
839+
the log has an entry under, and its value that label's client random to
840+
secret pairs, in log order, as an
841+
:class:`~pcapkit.corekit.infoclass.OrderedMultiInfo`. A label also reads
842+
as an attribute, as ``entries.CLIENT_RANDOM``.
843+
844+
.. versionadded:: 1.6.0
845+
846+
"""
847+
848+
if TYPE_CHECKING:
849+
def __init__(self, dict_: 'Optional[Mapping[Enum_TLSKeyLabel, OrderedMultiDict[bytes, bytes]]]' = None) -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long
850+
851+
832852
@info_final
833853
class TLSKeyLog(DSBSecrets):
834-
"""Data model for TLS key log DSB secrets."""
854+
"""Data model for TLS key log DSB secrets.
855+
856+
.. versionchanged:: 1.6.0
857+
858+
:attr:`entries` is a :class:`TLSKeyLogEntries` rather than a
859+
:obj:`dict`, and a mapping given for it is stored as one; :meth:`to_dict`
860+
exports it as an :class:`~pcapkit.corekit.multidict.OrderedMultiDict`,
861+
as it does any nested data model. A pickle holds it as a :obj:`dict`, so
862+
pickles load across 1.6.0 in both directions (:issue:`2726`).
863+
864+
"""
835865

836866
#: TLS key log entries.
837-
entries: 'dict[Enum_TLSKeyLabel, OrderedMultiInfo[bytes, bytes]]'
867+
entries: 'TLSKeyLogEntries'
838868

839869
if TYPE_CHECKING:
840870
#: TLS key log text as the block carries it, comments and line order
841871
#: included. Set on parsing only, and written back verbatim on rebuild.
842872
data: 'str'
843873

844-
def __init__(self, entries: 'dict[Enum_TLSKeyLabel, OrderedMultiDict[bytes, bytes]]') -> 'None': ...
874+
def __init__(self, entries: 'TLSKeyLogEntries | Mapping[Enum_TLSKeyLabel, OrderedMultiDict[bytes, bytes]]') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long
875+
876+
def __post_init__(self) -> 'None':
877+
"""Store a mapping given for :attr:`entries` as a :class:`TLSKeyLogEntries`."""
878+
_hold_entries(self.__dict__)
879+
880+
def __getstate__(self) -> 'dict[str, Any]':
881+
"""Return the state to pickle, holding :attr:`entries` as a :obj:`dict`.
882+
883+
A release before 1.6.0 loads them as the :obj:`dict` it holds, and
884+
:meth:`__setstate__` loads them as a :class:`TLSKeyLogEntries`.
885+
886+
"""
887+
state = dict(self.__dict__)
888+
if isinstance(state.get('entries'), TLSKeyLogEntries):
889+
state['entries'] = dict(state['entries'])
890+
return state
891+
892+
def __setstate__(self, state: 'dict[str, Any]') -> 'None':
893+
"""Restore a pickled state, storing :attr:`entries` as a :class:`TLSKeyLogEntries`.
894+
895+
Args:
896+
state: The state :meth:`__getstate__` returns, or one holding the
897+
entries as a :obj:`dict`, as before 1.6.0.
898+
899+
"""
900+
self.__dict__.update(state)
901+
_hold_entries(self.__dict__)
902+
903+
904+
def _hold_entries(namespace: 'dict[str, Any]') -> 'None':
905+
"""Store the ``entries`` in ``namespace`` as a :class:`TLSKeyLogEntries`, unless they are one.
906+
907+
Args:
908+
namespace: The :attr:`~object.__dict__` of a :class:`TLSKeyLog`.
909+
910+
"""
911+
entries = namespace.get('entries')
912+
if isinstance(entries, collections.abc.Mapping) and not isinstance(entries, TLSKeyLogEntries):
913+
namespace['entries'] = TLSKeyLogEntries(entries)
845914

846915

847916
@info_final

0 commit comments

Comments
 (0)