Skip to content

Commit 50cd95f

Browse files
authored
fix(toolkit): accept tshark's True/False TCP flags in the pyshark trace (#1514) (#1527)
1 parent 2d088a7 commit 50cd95f

4 files changed

Lines changed: 492 additions & 26 deletions

File tree

‎pcapkit/foundation/engines/pyshark.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -261,7 +261,7 @@ def read_frame(self) -> 'PySharkPacket':
261261
# trace flows
262262
if ext._flag_t:
263263
if ext._tcp:
264-
data_tf_tcp = tcp_traceflow(packet)
264+
data_tf_tcp = tcp_traceflow(packet, count=ext._frnum)
265265
if data_tf_tcp is not None:
266266
ext._trace.tcp(data_tf_tcp)
267267

‎pcapkit/toolkit/pyshark.py‎

Lines changed: 150 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,13 @@
1616
the `PyShark`_ engine.
1717
1818
"""
19+
import decimal
1920
import ipaddress
2021
from typing import TYPE_CHECKING, cast
2122

2223
from pcapkit.const.reg.linktype import LinkType as Enum_LinkType
2324
from pcapkit.foundation.traceflow.data.tcp import Packet as TF_TCP_Packet
24-
from pcapkit.utilities.exceptions import MissingKeyError
25+
from pcapkit.utilities.exceptions import MissingKeyError, ProtocolError
2526

2627
if TYPE_CHECKING:
2728
from typing import Any
@@ -316,6 +317,107 @@
316317
'xra': Enum_LinkType.DOCSIS31_XRA31, # docsis31_xra31
317318
} # type: dict[str, Enum_LinkType]
318319

320+
#: PyShark's text for a boolean field -> its value. tshark 4.x spells one ``'True'``/``'False'``
321+
#: (measured for ``tcp.flags_syn``, ``tcp.flags_fin`` and ``tcp.flags_reset`` with PyShark 0.6
322+
#: on tshark 4.6.9); ``'1'``/``'0'`` is the spelling this module first parsed (#1514).
323+
_BOOLEAN_FIELD = {'1': True, '0': False, 'True': True, 'False': False} # type: dict[str, bool]
324+
325+
326+
def _parse_flag(value: 'Any', name: 'str') -> 'bool':
327+
"""Parse a boolean PyShark field, such as ``tcp.flags_syn``.
328+
329+
Args:
330+
value: The field as PyShark reports it, a ``LayerFieldsContainer``
331+
(a :obj:`str` subclass).
332+
name: The field's name, for the error message.
333+
334+
Returns:
335+
The flag's value.
336+
337+
Raises:
338+
ProtocolError: If ``value`` is not a :obj:`str` spelling ``'1'``,
339+
``'0'``, ``'True'`` or ``'False'``.
340+
341+
"""
342+
if isinstance(value, str):
343+
flag = _BOOLEAN_FIELD.get(str(value))
344+
if flag is not None:
345+
return flag
346+
raise ProtocolError(f'invalid PyShark boolean field {name}: {value!r}')
347+
348+
349+
def _parse_epoch(value: 'Any') -> 'float':
350+
"""Parse PyShark's ``frame.time_epoch``, such as ``'1500000000.000638000'``.
351+
352+
Args:
353+
value: The field as PyShark reports it, decimal text.
354+
355+
Returns:
356+
The timestamp, as the same :obj:`float` the default engine reports.
357+
358+
Note:
359+
The default engine computes the epoch as a :class:`~decimal.Decimal`
360+
and hands the tracer ``float()`` of it (:mod:`pcapkit.toolkit.pcap`).
361+
This takes the same route, so both are the double nearest the same
362+
decimal value: identical, not merely close.
363+
364+
Raises:
365+
ProtocolError: If ``value`` is not decimal text.
366+
367+
"""
368+
try:
369+
return float(decimal.Decimal(str(value)))
370+
except decimal.InvalidOperation:
371+
raise ProtocolError(f'invalid PyShark frame.time_epoch: {value!r}') from None
372+
373+
374+
def _absolute(tcp: 'Any', name: 'str') -> 'int':
375+
"""Read the absolute TCP sequence or acknowledgement number.
376+
377+
Args:
378+
tcp: PyShark TCP layer.
379+
name: ``'seq'`` or ``'ack'``.
380+
381+
Returns:
382+
``tcp.<name>_raw``, the number on the wire, or ``tcp.<name>`` where
383+
tshark does not report the former.
384+
385+
Note:
386+
tshark's ``tcp.seq`` and ``tcp.ack`` are *relative* to the flow's first
387+
number by default (``0`` where the wire says ``832175175``).
388+
``tcp.seq_raw`` and ``tcp.ack_raw`` exist from Wireshark 3.2: they are
389+
absent from 3.0.0's ``epan/dissectors/packet-tcp.c`` and present in
390+
3.2.0's and 4.2.2's. On an older tshark the fallback is relative unless
391+
its ``tcp.relative_sequence_numbers`` preference is off.
392+
393+
"""
394+
value = getattr(tcp, f'{name}_raw', None)
395+
if value is None:
396+
value = getattr(tcp, name)
397+
return int(value)
398+
399+
400+
def _frame_layer(packet: 'Packet') -> 'Any':
401+
"""Find the ``frame`` layer of a PyShark packet.
402+
403+
Args:
404+
packet: PyShark packet.
405+
406+
Returns:
407+
The layer named ``frame``, or :data:`None` if the record has none.
408+
409+
Note:
410+
PyShark takes the second PDML ``<proto>`` as ``frame_info``. On a packet
411+
with a comment that is ``pkt_comment``, and the frame layer is
412+
``layers[0]`` (measured on ``test.pcapng`` frames 1 and 6 with tshark
413+
4.6.9), so the layer is looked up by name, as the engine does (#1531).
414+
415+
"""
416+
for layer in (packet.frame_info, *packet.layers):
417+
if layer.layer_name == 'frame':
418+
return layer
419+
return None
420+
319421

320422
def packet2dict(packet: 'Packet') -> 'dict[str, Any]':
321423
"""Convert PyShark packet into :obj:`dict`.
@@ -324,16 +426,27 @@ def packet2dict(packet: 'Packet') -> 'dict[str, Any]':
324426
packet: PyShark packet.
325427
326428
Returns:
327-
A :obj:`dict` mapping of packet data.
429+
A :obj:`dict` mapping of packet data: the ``frame`` layer's fields at
430+
the top level, and each later layer nested in the one before it. A
431+
packet comment, which PyShark reports as ``frame_info`` in place of the
432+
frame layer, is kept under its own ``PKT_COMMENT`` key. A record with
433+
no ``frame`` layer keeps PyShark's ``frame_info`` at the top level.
328434
329435
"""
330436
dict_ = {} # type: dict[str, Any]
331-
frame = packet.frame_info
437+
frame = _frame_layer(packet)
438+
if frame is None:
439+
frame = packet.frame_info
332440
for field in frame.field_names:
333441
dict_[field] = getattr(frame, field)
442+
if packet.frame_info is not frame:
443+
note = packet.frame_info
444+
dict_[note.layer_name.upper()] = {field: getattr(note, field) for field in note.field_names}
334445

335446
tempdict = dict_
336447
for layer in packet.layers:
448+
if layer is frame:
449+
continue
337450
tempdict[layer.layer_name.upper()] = {}
338451
tempdict = tempdict[layer.layer_name.upper()]
339452
for field in layer.field_names:
@@ -342,11 +455,14 @@ def packet2dict(packet: 'Packet') -> 'dict[str, Any]':
342455
return dict_
343456

344457

345-
def tcp_traceflow(packet: 'Packet') -> 'TF_TCP_Packet | None':
458+
def tcp_traceflow(packet: 'Packet', *, count: 'int' = -1) -> 'TF_TCP_Packet | None':
346459
"""Trace packet flow for TCP.
347460
348461
Args:
349462
packet: PyShark packet.
463+
count: Packet index. If not provided, default to ``-1``. The engine
464+
passes its own frame count, not tshark's ``frame.number``, which
465+
also numbers records that are not packets (#1515).
350466
351467
Returns:
352468
Data for TCP flow tracing.
@@ -356,6 +472,12 @@ def tcp_traceflow(packet: 'Packet') -> 'TF_TCP_Packet | None':
356472
* If the ``packet`` can be reassembled, then the :obj:`dict` mapping of data for TCP
357473
flow tracing (:term:`trace.tcp.packet`) will be returned; otherwise, returns :data:`None`.
358474
475+
Raises:
476+
MissingKeyError: If the frame's link type cannot be resolved.
477+
ProtocolError: If the packet has no ``frame`` layer, a TCP flag field is
478+
not a boolean tshark spelling, or ``frame.time_epoch`` is not decimal
479+
text.
480+
359481
See Also:
360482
:class:`pcapkit.foundation.traceflow.tcp.TCP`
361483
@@ -395,9 +517,16 @@ def tcp_traceflow(packet: 'Packet') -> 'TF_TCP_Packet | None':
395517
# :exc:`~pcapkit.utilities.exceptions.MissingKeyError` -- this
396518
# package's own house exception for a lookup miss -- rather than
397519
# letting it escape this public function.
398-
encap_type = getattr(packet.frame_info, 'encap_type', None)
520+
#
521+
# The frame layer is found by name, not taken as ``frame_info``, which
522+
# is ``pkt_comment`` on a commented packet; the root layer is then the
523+
# first layer that is not the frame layer.
524+
frame = _frame_layer(packet)
525+
if frame is None:
526+
raise ProtocolError(f'PyShark packet {packet.number} has no frame layer')
527+
encap_type = getattr(frame, 'encap_type', None)
399528
if encap_type is None:
400-
name = packet.layers[0].layer_name
529+
name = [layer for layer in packet.layers if layer is not frame][0].layer_name
401530
try:
402531
protocol = FILTER_NAME_TO_LINKTYPE[name.lower()]
403532
except KeyError:
@@ -408,20 +537,28 @@ def tcp_traceflow(packet: 'Packet') -> 'TF_TCP_Packet | None':
408537
except (KeyError, ValueError):
409538
raise MissingKeyError('frame.encap_type=%s' % encap_type) from None
410539

540+
# A Simple Packet Block carries no timestamp, so tshark reports no
541+
# ``frame.time_epoch`` for it; the PDML ``geninfo`` timestamp PyShark
542+
# keeps as ``sniff_timestamp`` is then ``'0.000000000'``, which is what
543+
# the default engine reports (measured on ``test.pcapng`` frame 4).
544+
epoch = getattr(frame, 'time_epoch', None)
545+
if epoch is None:
546+
epoch = packet.sniff_timestamp
547+
411548
data = TF_TCP_Packet( # type: ignore[type-var]
412549
protocol=protocol, # data link type
413-
index=int(packet.number), # frame number
550+
index=count, # frame number
414551
frame=packet2dict(packet), # extracted packet
415-
syn=bool(int(tcp.flags_syn)), # TCP synchronise (SYN) flag
416-
fin=bool(int(tcp.flags_fin)), # TCP finish (FIN) flag
417-
rst=bool(int(tcp.flags_reset)), # TCP reset (RST) flag
552+
syn=_parse_flag(tcp.flags_syn, 'tcp.flags_syn'), # TCP synchronise (SYN) flag
553+
fin=_parse_flag(tcp.flags_fin, 'tcp.flags_fin'), # TCP finish (FIN) flag
554+
rst=_parse_flag(tcp.flags_reset, 'tcp.flags_reset'), # TCP reset (RST) flag
418555
src=ipaddress.ip_address(ip.src), # source IP
419556
dst=ipaddress.ip_address(ip.dst), # destination IP
420557
srcport=int(tcp.srcport), # TCP source port
421558
dstport=int(tcp.dstport), # TCP destination port
422-
timestamp=packet.frame_info.time_epoch, # timestamp
423-
seq=int(tcp.seq), # TCP sequence number
424-
ack=int(tcp.ack), # TCP acknowledgement number
559+
timestamp=_parse_epoch(epoch), # timestamp
560+
seq=_absolute(tcp, 'seq'), # TCP sequence number
561+
ack=_absolute(tcp, 'ack'), # TCP acknowledgement number
425562
# NOTE: PyShark reports dissected *fields*, not the octets behind
426563
# them, so there is no header or payload to hand over -- which is
427564
# the same reason this module carries no ``tcp_reassembly`` at all.

‎tests/foundation/engines/test_engine_agreement_runtime.py‎

Lines changed: 31 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,14 @@
1919
hands the reassembler, keyed by frame number -- every field;
2020
``trace``
2121
every :class:`~pcapkit.foundation.traceflow.data.tcp.Packet` handed to the
22-
flow tracer, keyed by frame number -- every field but ``frame``, which is
23-
the engine's own frame object;
22+
flow tracer, keyed by frame number -- every field except ``frame``, which
23+
is the engine's own frame object, and except ``header`` and ``payload`` for
24+
an engine in :data:`NO_TCP_OCTETS`;
2425
``datagrams``
2526
every reassembled datagram, every field but the lazily parsed ``packet``;
2627
``flows``
27-
every traced flow's label and index, and the octets of its PCAP file.
28+
every traced flow's label and index, and the octets of its PCAP file unless
29+
the engine is in :data:`JSON_TRACE`.
2830
2931
The inputs are captured by wrapping :meth:`ReassemblyBase.__call__
3032
<pcapkit.foundation.reassembly.reassembly.ReassemblyBase.__call__>` and
@@ -80,6 +82,20 @@
8082
'pyshark': 'pyshark reports dissected fields, not the frame octets',
8183
'pypcapfile': 'the pypcapfile engine replaces the frame octets with decoded layers',
8284
}
85+
#: Engines whose flow-tracing input carries no TCP octets, and why. Their ``trace``
86+
#: comparison leaves out ``header`` and ``payload`` and compares every other field.
87+
NO_TCP_OCTETS = {
88+
'pyshark': 'pyshark reports dissected fields, not the octets behind them, so tcp_traceflow '
89+
"hands the tracer empty ones and Extractor refuses trace_analyse for it "
90+
'(pcapkit/foundation/extraction.py:1446-1455; ruling on #1507)',
91+
}
92+
#: Engines whose flow files are JSON whatever ``trace_format`` asks for, and why. Their
93+
#: ``flows`` comparison checks each flow's label and frame indices, not the file's octets.
94+
JSON_TRACE = {
95+
'pyshark': 'pyshark cannot supply the frame bytes a PCAP trace needs, so Extractor writes '
96+
"JSON with a FormatWarning (pcapkit/foundation/extraction.py:1440-1444; ruling "
97+
'on #1507)',
98+
}
8399
#: Captures with at least one TCP flow, measured on ``00d232890``.
84100
TRACED = ('http.pcap', 'http6.cap', 'in.pcap', 'many_interfaces.pcapng', 'options-tcp.pcap',
85101
'options-transport.pcap', 'stream.pcap', 'tcp.pcap', 'test.pcap')
@@ -205,6 +221,12 @@ def _run(engine: 'str', capture: 'str') -> '_Run':
205221
return _RUNS[key]
206222

207223

224+
def _without_tcp_octets(inputs: 'dict[int, Any]') -> 'dict[int, Any]':
225+
"""``trace`` inputs without ``header`` and ``payload``, per :data:`NO_TCP_OCTETS`."""
226+
return {number: [{key: value for key, value in packet.items() if key not in ('header', 'payload')}
227+
for packet in packets] for number, packets in inputs.items()}
228+
229+
208230
def _first_difference(got: 'dict[int, Any]', want: 'dict[int, Any]') -> 'Optional[str]':
209231
for number in sorted(set(got) | set(want)):
210232
if number not in got:
@@ -251,7 +273,10 @@ def compare(engine: 'str', capture: 'str', aspect: 'str') -> 'Outcome':
251273
return Outcome('RECORDS', f'{len(mine)} {aspect} != {len(theirs)}')
252274
return Outcome('OK')
253275
if aspect in run.inputs:
254-
diff = _first_difference(run.inputs[aspect], base.inputs[aspect])
276+
mine, theirs = run.inputs[aspect], base.inputs[aspect]
277+
if aspect == 'trace' and engine in NO_TCP_OCTETS:
278+
mine, theirs = _without_tcp_octets(mine), _without_tcp_octets(theirs)
279+
diff = _first_difference(mine, theirs)
255280
return Outcome('INPUTS', diff) if diff else Outcome('OK')
256281
if aspect == 'datagrams':
257282
kinds = [kind for kind in ('ipv4', 'ipv6', 'tcp') if run.datagrams[kind] is not None]
@@ -273,6 +298,8 @@ def compare(engine: 'str', capture: 'str', aspect: 'str') -> 'Outcome':
273298
if [f[:2] for f in run.flows] != [f[:2] for f in base.flows]:
274299
return Outcome('FLOWS', f'flows {[f[:2] for f in run.flows]!r:.200} != '
275300
f'{[f[:2] for f in base.flows]!r:.200}')
301+
if engine in JSON_TRACE:
302+
return Outcome('OK')
276303
for (label, _, mine), (_, _, theirs) in zip(run.flows, base.flows):
277304
if mine != theirs:
278305
return Outcome('FLOWS', f'{label}: file of {len(mine)} octets != {len(theirs)} '
@@ -311,11 +338,6 @@ class TestEngineAgreement(harness.RoundTripBase):
311338
tuple(f'{capture}/pypcapfile/{aspect}'
312339
for capture in ('http6.cap', 'stream.pcap', 'tcp.pcap', 'test.pcap')
313340
for aspect in ('tcp', 'trace', 'datagrams', 'flows'))),
314-
Gap(1514, "the pyshark engine raises ValueError on the first TCP frame it traces: "
315-
'tcp_traceflow reads the flags with bool(int(tcp.flags_syn)) (pcapkit/toolkit/'
316-
"pyshark.py:415-417), and tshark 4.x reports them as 'True' and 'False'",
317-
'ERROR', "ValueError: invalid literal for int() with base 10: ",
318-
tuple(f'{capture}/pyshark/*' for capture in TRACED)),
319341
Gap(1515, 'the pyshark engine counts a Systemd Journal Export Block as a frame: tshark '
320342
'numbers it as one, and read_frame takes every tshark frame '
321343
'(pcapkit/foundation/engines/pyshark.py:245-248)',

0 commit comments

Comments
 (0)