Skip to content

Commit bec58be

Browse files
committed
fix(engines): the pyshark engine skips non-packet PCAP-NG blocks (#1515)
tshark numbers every record it reads. A systemd Journal Export Block, Custom Block or Sysdig Event Block is a frame to it, but not to the default engine, so test.pcapng read as 6 frames instead of 5. Every frame after the journal block was also off by one. - read_frame skips a record unless its frame layer carries frame.encap_type. Wireshark adds that field for packet records only, in 4.2.2 and 4.6.9. - The frame layer is looked up by name, because frame_info is pkt_comment on a packet that carries a comment. The verbose handler read protocols from frame_info and raised AttributeError on such a packet; it now reads the frame layer too. - Frames are numbered by count, as the default engine numbers them, not by tshark's frame number. - Drop the Gap(1515) rows from the engine agreement module. Closes #1515
1 parent 16fbea0 commit bec58be

5 files changed

Lines changed: 207 additions & 16 deletions

File tree

‎pcapkit/foundation/engines/pyshark.py‎

Lines changed: 55 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
from typing import Optional
2626

2727
from pyshark.capture.file_capture import FileCapture
28+
from pyshark.packet.layers.base import BaseLayer
2829
from pyshark.packet.packet import Packet as PySharkPacket
2930

3031
from pcapkit.foundation.extraction import Extractor
@@ -34,6 +35,45 @@
3435
logger = get_logger(__name__)
3536

3637

38+
def _frame_layer(packet: 'PySharkPacket') -> 'Optional[BaseLayer]':
39+
"""The ``frame`` layer of a `PyShark`_ packet, or :data:`None` when the record has none.
40+
41+
Looked up by name, because `PyShark`_ takes the second PDML ``<proto>`` as
42+
``frame_info``. On a packet with a comment, that is ``pkt_comment``, which has
43+
no ``protocols``, and the frame layer is ``layers[0]``. By its source,
44+
:program:`tshark` 4.2.2 roots a Sysdig event at ``syscall`` and a Netflix
45+
custom block at ``bblog``, so those records have no ``frame`` layer.
46+
47+
.. _PyShark: https://kiminewt.github.io/pyshark
48+
49+
"""
50+
for layer in (packet.frame_info, *packet.layers):
51+
if layer.layer_name == 'frame':
52+
return layer
53+
return None
54+
55+
56+
def _is_packet(packet: 'PySharkPacket') -> 'bool':
57+
"""Whether a `PyShark`_ packet is a packet record, rather than another kind of tshark record.
58+
59+
:program:`tshark` numbers every record it reads, not only packets: a PCAP-NG
60+
systemd Journal Export Block, Custom Block or Sysdig Event Block is a frame
61+
to it, measured with :program:`tshark` 4.6.9. The default engine counts
62+
Enhanced, Simple and obsolete Packet Blocks only.
63+
64+
The marker is ``frame.encap_type`` on the record's :func:`_frame_layer`.
65+
Wireshark adds it for ``REC_TYPE_PACKET`` records only:
66+
:file:`epan/dissectors/packet-frame.c` lines 1001-1002 in 4.2.2, and 828-890
67+
in 4.6.9. The summary line is not used, because it differs between those
68+
versions. A record with no ``frame`` layer is not a packet either.
69+
70+
.. _PyShark: https://kiminewt.github.io/pyshark
71+
72+
"""
73+
frame = _frame_layer(packet)
74+
return frame is not None and 'encap_type' in frame.field_names
75+
76+
3777
class PyShark(EngineBase['PySharkPacket']):
3878
"""PyShark engine support.
3979
@@ -214,10 +254,12 @@ def run(self) -> 'None':
214254
f"so 'ipv4={ext._ipv4}', 'ipv6={ext._ipv6}' and 'tcp={ext._tcp}' will be ignored",
215255
AttributeWarning, stacklevel=stacklevel())
216256

217-
# setup verbose handler
257+
# setup verbose handler; a packet always has a frame layer, since
258+
# read_frame skips every record that does not (#1515)
218259
if ext._flag_v:
219260
ext._vfunc = lambda e, f: print(
220-
f'Frame {e._frnum:>3d}: {f.frame_info.protocols}' # pylint: disable=protected-access
261+
f'Frame {e._frnum:>3d}: ' # pylint: disable=protected-access
262+
f'{cast("BaseLayer", _frame_layer(f)).protocols}'
221263
)
222264

223265
# extract global header
@@ -230,6 +272,10 @@ def run(self) -> 'None':
230272
def read_frame(self) -> 'PySharkPacket':
231273
"""Read frames with PyShark engine.
232274
275+
A tshark record that is not a packet, such as a PCAP-NG systemd Journal
276+
Export Block, is skipped. Frames are numbered by count, as the default
277+
engine numbers them, not by tshark's frame number.
278+
233279
Returns:
234280
Parsed frame instance.
235281
@@ -241,11 +287,15 @@ def read_frame(self) -> 'PySharkPacket':
241287
from pcapkit.toolkit.pyshark import packet2dict, tcp_traceflow
242288
ext = self._extractor
243289

244-
# fetch PyShark packet
245-
packet = cast('PySharkPacket', self._extmp.next())
290+
# fetch the next PyShark packet that is a packet record
291+
while True:
292+
packet = cast('PySharkPacket', self._extmp.next())
293+
if _is_packet(packet):
294+
break
295+
logger.debug('pyshark: skipping tshark frame %s, not a packet record', packet.number)
246296

247297
# verbose output
248-
ext._frnum = int(packet.number)
298+
ext._frnum += 1
249299
ext._vfunc(ext, packet)
250300

251301
# write plist

‎tests/foundation/engines/test_engine_agreement_runtime.py‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -349,11 +349,6 @@ class TestEngineAgreement(harness.RoundTripBase):
349349
tuple(f'{capture}/pypcapfile/{aspect}'
350350
for capture in ('http6.cap', 'stream.pcap', 'tcp.pcap', 'test.pcap')
351351
for aspect in ('tcp', 'trace', 'datagrams', 'flows'))),
352-
Gap(1515, 'the pyshark engine counts a Systemd Journal Export Block as a frame: tshark '
353-
'numbers it as one, and read_frame takes every tshark frame '
354-
'(pcapkit/foundation/engines/pyshark.py:245-248)',
355-
('FRAMES', 'RECORDS'), ('6 frames != 5', 'frame 5: timestamp '),
356-
('test.pcapng/pyshark/frames', 'test.pcapng/pyshark/timestamps')),
357352
Gap(1501, 'the dpkt engine hands reassembly the Decimal timestamp dpkt reads from a '
358353
'nanosecond PCAP (pcapkit/foundation/engines/dpkt.py:309), where every other '
359354
'engine hands it a float (pcapkit/toolkit/pcap.py:73)',
Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
# -*- coding: utf-8 -*-
2+
"""The PyShark engine reads packet records only, as the default engine does. C.f. #1515.
3+
4+
:program:`tshark` numbers every record wiretap hands it, and a PCAP-NG systemd
5+
Journal Export Block, Custom Block or Sysdig Event Block is one. The default
6+
engine counts Enhanced, Simple and obsolete Packet Blocks only, so a capture
7+
holding a journal block read as one frame more through ``pyshark``, and every
8+
later frame was off by one.
9+
10+
The same lookup serves the verbose handler. ``pyshark`` takes the second PDML
11+
``<proto>`` as ``frame_info``, and on a packet that carries a comment that is
12+
``pkt_comment``, which has no ``protocols``. So ``verbose=True`` raised
13+
:exc:`AttributeError` on frame 1 of :file:`examples/captures/test.pcapng`.
14+
15+
The packets here are stand-ins shaped like what ``pyshark`` 0.6 builds from
16+
:program:`tshark` 4.6.9's PDML. The field names are the ones measured on
17+
:file:`examples/captures/test.pcapng` and on a capture holding a Custom Block and
18+
a Sysdig Event Block. ``pyshark`` itself is never imported, so this runs where it
19+
is absent or cannot run.
20+
21+
"""
22+
23+
import contextlib
24+
import importlib.util
25+
import io
26+
import types
27+
import unittest
28+
29+
from tests._support import reimport_once_per_class
30+
31+
RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper')
32+
HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS)
33+
34+
#: ``frame`` layer fields of a packet record; only these carry ``encap_type``.
35+
PACKET_FRAME = ('section_number', 'interface_id', 'interface_name', 'encap_type', 'time',
36+
'time_epoch', 'number', 'len', 'cap_len', 'protocols')
37+
#: ``frame`` layer fields of a systemd Journal Export or Sysdig Event record.
38+
RECORD_FRAME = ('section_number', 'time', 'time_utc', 'time_epoch', 'number', 'len', 'cap_len',
39+
'protocols')
40+
#: ``frame.protocols`` of frame 1 of :file:`examples/captures/test.pcapng`.
41+
PROTOCOLS = 'eth:ethertype:ip:udp:dhcp'
42+
43+
44+
def _layer(name: str, fields: 'tuple[str, ...]' = (), **values: str) -> types.SimpleNamespace:
45+
return types.SimpleNamespace(layer_name=name, field_names=list(fields), **values)
46+
47+
48+
def _packet(number: int, comment: bool = False) -> types.SimpleNamespace:
49+
"""A packet record; with a comment, ``pyshark`` takes ``pkt_comment`` as ``frame_info``."""
50+
frame = _layer('frame', PACKET_FRAME, protocols=PROTOCOLS)
51+
eth = _layer('eth', ('dst', 'src', 'type'))
52+
if comment:
53+
note = _layer('pkt_comment', ('frame_comment',))
54+
return types.SimpleNamespace(number=str(number), frame_info=note, layers=[frame, eth])
55+
return types.SimpleNamespace(number=str(number), frame_info=frame, layers=[eth])
56+
57+
58+
def _record(number: int, frame: 'tuple[str, ...] | None', root: str) -> types.SimpleNamespace:
59+
"""A non-packet record; ``frame=None`` for a root that is not ``frame`` at all."""
60+
head = _layer('frame', frame) if frame is not None else _layer(root, ('num',))
61+
layers = [_layer(root, ('message',))] if frame is not None else []
62+
return types.SimpleNamespace(number=str(number), frame_info=head, layers=layers)
63+
64+
65+
#: Every non-packet record shape, and where it was seen.
66+
RECORDS = {
67+
'systemd journal (4.6.9)': ('systemd_journal', RECORD_FRAME),
68+
'custom block (4.6.9)': ('DATA', ('section_number', 'interface_id', 'time', 'number', 'len')),
69+
'sysdig event (4.6.9)': ('sysdig', RECORD_FRAME),
70+
# 4.2.2 roots a Sysdig event at ``syscall`` and a Netflix custom block at
71+
# ``bblog`` instead of ``frame`` (epan/dissectors/packet-frame.c:858-869, :884-890).
72+
'sysdig event (4.2.2)': ('syscall', None),
73+
'black box log (4.2.2)': ('bblog', None),
74+
}
75+
76+
77+
@unittest.skipUnless(HAS_RUNTIME, 'runtime dependencies not installed')
78+
class TestPySharkNonPacketRecords(unittest.TestCase):
79+
""":meth:`PyShark.read_frame` skips every record that is not a packet."""
80+
81+
def setUp(self) -> None:
82+
reimport_once_per_class(self)
83+
84+
@staticmethod
85+
def _engine(records: 'list[types.SimpleNamespace]') -> 'tuple[object, types.SimpleNamespace]':
86+
from pcapkit.foundation.engines.pyshark import PyShark
87+
88+
stream = iter(records)
89+
extractor = types.SimpleNamespace(_frnum=0, _vfunc=lambda e, f: None, _flag_q=True,
90+
_flag_t=False, _flag_d=True, _frame=[])
91+
engine = PyShark.__new__(PyShark) # skip __init__, which imports pyshark
92+
engine._extractor = extractor
93+
engine._extmp = types.SimpleNamespace(next=lambda: next(stream))
94+
return engine, extractor
95+
96+
def test_a_journal_record_is_not_a_frame(self) -> None:
97+
first, last = _packet(1, comment=True), _packet(3)
98+
engine, ext = self._engine([first, _record(2, RECORD_FRAME, 'systemd_journal'), last])
99+
100+
self.assertIs(engine.read_frame(), first)
101+
self.assertEqual(ext._frnum, 1)
102+
self.assertIs(engine.read_frame(), last)
103+
self.assertEqual(ext._frnum, 2, 'frames are numbered as the default engine numbers them')
104+
self.assertEqual(ext._frame, [first, last])
105+
with self.assertRaises(StopIteration):
106+
engine.read_frame()
107+
108+
def test_every_record_without_an_encapsulation_is_skipped(self) -> None:
109+
for label, (root, frame) in RECORDS.items():
110+
with self.subTest(record=label):
111+
record = _record(2, frame, root)
112+
engine, ext = self._engine([_packet(1), record, _packet(3)])
113+
frames = [engine.read_frame(), engine.read_frame()]
114+
self.assertNotIn(record, frames)
115+
self.assertEqual([f.number for f in frames], ['1', '3'])
116+
self.assertEqual(ext._frnum, 2)
117+
118+
def test_a_trailing_record_ends_the_capture(self) -> None:
119+
engine, ext = self._engine([_packet(1), _record(2, RECORD_FRAME, 'systemd_journal')])
120+
engine.read_frame()
121+
with self.assertRaises(StopIteration):
122+
engine.read_frame()
123+
self.assertEqual(ext._frnum, 1)
124+
125+
def test_verbose_mode_prints_a_commented_packet(self) -> None:
126+
engine, ext = self._engine([_packet(1, comment=True), _packet(2)])
127+
vars(ext).update(_exlyr='none', _exptl='null', _exctx=None, _flag_r=False, _flag_v=True,
128+
_ifnm='test.pcapng', record_header=lambda: None)
129+
stream = engine._extmp
130+
engine._expkg = types.SimpleNamespace(FileCapture=lambda *args, **kwargs: stream)
131+
engine.run() # installs the verbose handler
132+
133+
with contextlib.redirect_stdout(io.StringIO()) as out:
134+
engine.read_frame()
135+
engine.read_frame()
136+
self.assertEqual(out.getvalue().splitlines(),
137+
[f'Frame 1: {PROTOCOLS}', f'Frame 2: {PROTOCOLS}'])
138+
139+
140+
if __name__ == '__main__':
141+
unittest.main()

‎tests/foundation/engines/test_runtime_engines.py‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -432,8 +432,10 @@ def test_pyshark_run_read_frame_and_close(self) -> None:
432432
class FakeCapture:
433433
def __init__(self) -> None:
434434
self.closed = False
435-
self.packet = types.SimpleNamespace(number='7',
436-
frame_info=types.SimpleNamespace(protocols='eth:ip:tcp'))
435+
# a packet record: its frame layer carries frame.encap_type (#1515)
436+
frame = types.SimpleNamespace(layer_name='frame', field_names=['encap_type'],
437+
protocols='eth:ip:tcp')
438+
self.packet = types.SimpleNamespace(number='7', frame_info=frame, layers=[])
437439

438440
def next(self):
439441
return self.packet
@@ -466,8 +468,9 @@ def close(self) -> None:
466468
with mock.patch('pcapkit.toolkit.pyshark.tcp_traceflow', return_value='trace'):
467469
packet = engine.read_frame()
468470
self.assertIs(packet, capture.packet)
469-
self.assertEqual(extractor._frnum, 7)
470-
self.assertEqual(sink.paths[-1], 'out/Frame 7.json')
471+
# numbered by count, as the default engine numbers frames, not by tshark's '7'
472+
self.assertEqual(extractor._frnum, 1)
473+
self.assertEqual(sink.paths[-1], 'out/Frame 1.json')
471474
extractor._trace.tcp.assert_called_once_with('trace')
472475
self.assertEqual(extractor._frame, [packet])
473476

@@ -480,7 +483,7 @@ def close(self) -> None:
480483
with mock.patch('pcapkit.toolkit.pyshark.packet2dict', return_value={'packet': True}):
481484
with mock.patch('pcapkit.toolkit.pyshark.tcp_traceflow', return_value=None):
482485
self.assertIs(engine.read_frame(), capture.packet)
483-
self.assertEqual(stream_sink.records[-1][1], 'Frame 7')
486+
self.assertEqual(stream_sink.records[-1][1], 'Frame 1')
484487
stream_mode._trace.tcp.assert_not_called()
485488

486489
disabled, _ = make_extractor(_flag_q=True, _flag_t=False, _flag_d=False)

‎tests/foundation/test_extraction_record_header_unit.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,9 @@ def close(self) -> None:
134134
return fmt, records
135135

136136
def test_pcap_output_opens_with_the_global_header(self) -> None:
137-
packet = types.SimpleNamespace(number='1')
137+
# a packet record, as pyshark builds one: its frame layer carries frame.encap_type (#1515)
138+
frame = types.SimpleNamespace(layer_name='frame', field_names=['encap_type'])
139+
packet = types.SimpleNamespace(number='1', frame_info=frame, layers=[])
138140
fmt, records = self._run(sample_path('in.pcap'), [packet])
139141
self.assertEqual(fmt, 'json')
140142
self.assertEqual(list(records), ['Global Header', 'Frame 1'])

0 commit comments

Comments
 (0)