Skip to content

Commit ffca646

Browse files
authored
test(foundation): reassembly, traceflow, engine and registry round trips (#1202) (#1508)
- reassembly: IPv4, IPv6 and TCP payloads cut, reordered, duplicated and overlapped come back as one COMPLETE datagram with the original octets; a late duplicate IP fragment surfaces as a second PARTIAL one (#1507) - traceflow: every flow's PCAP file reads back as exactly its frames, over built captures (PCAP and PCAP-NG, both byte orders and resolutions) and over every sample capture - engines: dpkt, scapy, pyshark, pypcap, pcap_ct and pypcapfile agree with default field for field; an engine that falls back, or a documented refusal, is skipped - registry: override then restore leaves every public registry as shipped - failing cases are recorded per root cause in KNOWN_FAILURES, checked in both directions: #1500-#1505, #1507, #1512-#1515, #1520 Refs #1202
1 parent 5f4da23 commit ffca646

6 files changed

Lines changed: 1830 additions & 0 deletions

‎tests/foundation/_roundtrip.py‎

Lines changed: 307 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,307 @@
1+
# -*- coding: utf-8 -*-
2+
"""Shared harness for the foundation round-trip modules. C.f. #1202.
3+
4+
The owner's rule is that every round trip the library promises passes and
5+
matches exactly. Four foundation paths make such a promise, and each has a
6+
module beside this one that enumerates its cases:
7+
8+
* reassembly recovers the payload its fragments or segments were cut from
9+
(:file:`reassembly/test_reassembly_payload_roundtrip_unit.py`);
10+
* flow tracing writes each flow to a file that reads back as exactly that
11+
flow's frames (:file:`traceflow/test_traceflow_reread_roundtrip_unit.py`,
12+
and the ``_runtime`` twin over :file:`examples/captures/`);
13+
* every offline engine reports what the ``default`` engine reports, field for
14+
field (:file:`engines/test_engine_agreement_runtime.py`);
15+
* a registrar handed back the entry it shipped with leaves every registry it
16+
touches as it was (:file:`registry/test_registry_symmetry_unit.py`).
17+
18+
A case that does not close goes in its module's ``KNOWN_FAILURES``, one
19+
:class:`Gap` per root cause, in the style of
20+
:mod:`tests.protocols._edge_roundtrip`. The table is asserted in both
21+
directions: a case it does not list must pass, and a case it lists must still
22+
fail as recorded, so a fixed defect turns the module red until its entry goes.
23+
24+
Nothing here imports :mod:`pcapkit`.
25+
26+
"""
27+
28+
from __future__ import annotations
29+
30+
import fnmatch
31+
import struct
32+
import unittest
33+
from typing import TYPE_CHECKING, NamedTuple
34+
35+
from tests._support import time_limit
36+
37+
if TYPE_CHECKING:
38+
from typing import Optional
39+
40+
#: Whole seconds one case may take.
41+
CASE_TIMEOUT = 60
42+
43+
44+
class Outcome(NamedTuple):
45+
"""What one check found."""
46+
47+
#: ``'OK'``, ``'SKIP'`` (the case cannot run here, e.g. an engine's
48+
#: dependency is missing), or one of the module's failure statuses.
49+
status: 'str'
50+
#: What differed, observed against expected.
51+
detail: 'str' = ''
52+
53+
54+
class Gap(NamedTuple):
55+
"""One root cause, and every case it stops."""
56+
57+
#: GitHub issue tracking the defect.
58+
issue: 'Optional[int]'
59+
#: The defect, with the ``file:line`` that causes it.
60+
defect: 'str'
61+
#: Status every case below reports, or the statuses of which each must
62+
#: report one.
63+
status: 'str | tuple[str, ...]'
64+
#: Substring every failure detail contains, or a tuple of which each
65+
#: detail contains at least one.
66+
fragment: 'str | tuple[str, ...]'
67+
#: Labels of the cases this defect stops, as :mod:`fnmatch` patterns. Each
68+
#: pattern matches at least one case, and every case it matches fails.
69+
cases: 'tuple[str, ...]'
70+
71+
72+
def _as_tuple(value: 'str | tuple[str, ...]') -> 'tuple[str, ...]':
73+
return (value,) if isinstance(value, str) else value
74+
75+
76+
###############################################################################
77+
# Wire builders, so the unit-tier modules read no capture.
78+
###############################################################################
79+
80+
#: Ethernet addresses every built frame uses.
81+
MAC_A, MAC_B = bytes.fromhex('020000000001'), bytes.fromhex('020000000002')
82+
#: IPv4 and IPv6 endpoints every built packet uses, as octets.
83+
IPV4_A, IPV4_B = bytes([192, 0, 2, 1]), bytes([198, 51, 100, 2])
84+
IPV6_A = bytes.fromhex('20010db8000000000000000000000001')
85+
IPV6_B = bytes.fromhex('20010db8000000000000000000000002')
86+
87+
88+
def checksum(data: 'bytes') -> 'int':
89+
"""RFC 1071 Internet checksum."""
90+
if len(data) % 2:
91+
data += b'\x00'
92+
total = sum(int.from_bytes(data[i:i + 2], 'big') for i in range(0, len(data), 2))
93+
while total >> 16:
94+
total = (total & 0xFFFF) + (total >> 16)
95+
return ~total & 0xFFFF
96+
97+
98+
def ethernet(payload: 'bytes', ethertype: 'int', *, reverse: 'bool' = False) -> 'bytes':
99+
"""An Ethernet II frame."""
100+
src, dst = (MAC_B, MAC_A) if reverse else (MAC_A, MAC_B)
101+
return dst + src + ethertype.to_bytes(2, 'big') + payload
102+
103+
104+
def ipv4(payload: 'bytes', *, ident: 'int' = 0x1234, offset: 'int' = 0, mf: 'bool' = False,
105+
df: 'bool' = False, proto: 'int' = 17, reverse: 'bool' = False) -> 'bytes':
106+
"""An IPv4 packet; ``offset`` is in octets and must be a multiple of 8."""
107+
src, dst = (IPV4_B, IPV4_A) if reverse else (IPV4_A, IPV4_B)
108+
flags = (0x4000 if df else 0) | (0x2000 if mf else 0) | (offset // 8)
109+
header = bytearray(b'\x45\x00' + (20 + len(payload)).to_bytes(2, 'big')
110+
+ ident.to_bytes(2, 'big') + flags.to_bytes(2, 'big')
111+
+ bytes([64, proto]) + b'\x00\x00' + src + dst)
112+
header[10:12] = checksum(bytes(header)).to_bytes(2, 'big')
113+
return bytes(header) + payload
114+
115+
116+
def ipv6(payload: 'bytes', *, nxt: 'int', reverse: 'bool' = False) -> 'bytes':
117+
"""An IPv6 packet whose first extension header or upper layer is ``nxt``."""
118+
src, dst = (IPV6_B, IPV6_A) if reverse else (IPV6_A, IPV6_B)
119+
return (b'\x60\x00\x00\x00' + len(payload).to_bytes(2, 'big') + bytes([nxt, 64])
120+
+ src + dst + payload)
121+
122+
123+
def ipv6_fragment(payload: 'bytes', *, ident: 'int' = 0x12345678, offset: 'int' = 0,
124+
mf: 'bool' = False, nxt: 'int' = 17) -> 'bytes':
125+
"""An IPv6 packet carrying one Fragment header (RFC 8200 section 4.5)."""
126+
word = offset | (1 if mf else 0)
127+
header = bytes([nxt, 0]) + word.to_bytes(2, 'big') + ident.to_bytes(4, 'big')
128+
return ipv6(header + payload, nxt=44)
129+
130+
131+
def udp(data: 'bytes', *, sport: 'int' = 40000, dport: 'int' = 9) -> 'bytes':
132+
"""A UDP datagram; port 9 (discard) dispatches to no application layer."""
133+
return (sport.to_bytes(2, 'big') + dport.to_bytes(2, 'big')
134+
+ (8 + len(data)).to_bytes(2, 'big') + b'\x00\x00' + data)
135+
136+
137+
def tcp(payload: 'bytes', *, seq: 'int', ack: 'int' = 0, syn: 'bool' = False,
138+
fin: 'bool' = False, rst: 'bool' = False, sport: 'int' = 40000,
139+
dport: 'int' = 9) -> 'bytes':
140+
"""A TCP segment with no options; the ACK flag is set whenever ``ack`` is."""
141+
flags = (0x01 if fin else 0) | (0x02 if syn else 0) | (0x04 if rst else 0) | (0x10 if ack else 0)
142+
return (sport.to_bytes(2, 'big') + dport.to_bytes(2, 'big') + (seq % (1 << 32)).to_bytes(4, 'big')
143+
+ ack.to_bytes(4, 'big') + bytes([0x50, flags]) + b'\xff\xff\x00\x00\x00\x00' + payload)
144+
145+
146+
def pcap(frames: 'list[tuple[int, int, bytes]]', *, linktype: 'int' = 1,
147+
nanosecond: 'bool' = False, byteorder: 'str' = 'little') -> 'bytes':
148+
"""A PCAP savefile of ``(seconds, fraction, frame)`` records.
149+
150+
``fraction`` is microseconds, or nanoseconds when ``nanosecond`` is set.
151+
152+
"""
153+
order = '<' if byteorder == 'little' else '>'
154+
magic = 0xA1B23C4D if nanosecond else 0xA1B2C3D4
155+
out = [struct.pack(f'{order}IHHiIII', magic, 2, 4, 0, 0, 0xFFFF, linktype)]
156+
for seconds, fraction, frame in frames:
157+
out.append(struct.pack(f'{order}IIII', seconds, fraction, len(frame), len(frame)) + frame)
158+
return b''.join(out)
159+
160+
161+
def pcapng(frames: 'list[tuple[int, int, bytes]]', *, linktype: 'int' = 1,
162+
nanosecond: 'bool' = False, byteorder: 'str' = 'little') -> 'bytes':
163+
"""A PCAP-NG file of one section and one interface, one EPB per record.
164+
165+
``fraction`` is microseconds, or nanoseconds when ``nanosecond`` is set, in
166+
which case the interface carries ``if_tsresol`` 9.
167+
168+
"""
169+
order = '<' if byteorder == 'little' else '>'
170+
171+
def block(kind: 'int', body: 'bytes') -> 'bytes':
172+
body += b'\x00' * (-len(body) % 4)
173+
size = 12 + len(body)
174+
return struct.pack(f'{order}II', kind, size) + body + struct.pack(f'{order}I', size)
175+
176+
shb = block(0x0A0D0D0A, struct.pack(f'{order}IHHq', 0x1A2B3C4D, 1, 0, -1))
177+
options = b''
178+
if nanosecond: # if_tsresol = 9, then opt_endofopt
179+
options = struct.pack(f'{order}HH', 9, 1) + b'\x09\x00\x00\x00' + struct.pack(f'{order}HH', 0, 0)
180+
idb = block(0x00000001, struct.pack(f'{order}HHI', linktype, 0, 0xFFFF) + options)
181+
out = [shb, idb]
182+
for seconds, fraction, frame in frames:
183+
ticks = seconds * (10 ** 9 if nanosecond else 10 ** 6) + fraction
184+
out.append(block(0x00000006, struct.pack(f'{order}IIIII', 0, ticks >> 32, ticks & 0xFFFFFFFF,
185+
len(frame), len(frame)) + frame))
186+
return b''.join(out)
187+
188+
189+
class Savefile(NamedTuple):
190+
"""A PCAP savefile, read without :mod:`pcapkit`."""
191+
192+
#: ``'little'`` or ``'big'``.
193+
byteorder: 'str'
194+
nanosecond: 'bool'
195+
linktype: 'int'
196+
#: ``(seconds, fraction, captured length, original length, octets)``.
197+
records: 'tuple[tuple[int, int, int, int, bytes], ...]'
198+
199+
200+
def read_pcap(data: 'bytes') -> 'Savefile':
201+
"""Read a PCAP savefile with :mod:`struct` alone, as an independent reference."""
202+
magic = data[:4]
203+
table = {b'\xd4\xc3\xb2\xa1': ('little', False), b'\xa1\xb2\xc3\xd4': ('big', False),
204+
b'\x4d\x3c\xb2\xa1': ('little', True), b'\xa1\xb2\x3c\x4d': ('big', True)}
205+
byteorder, nanosecond = table[magic]
206+
order = '<' if byteorder == 'little' else '>'
207+
linktype, = struct.unpack_from(f'{order}I', data, 20)
208+
records, offset = [], 24
209+
while offset < len(data):
210+
sec, frac, incl, orig = struct.unpack_from(f'{order}IIII', data, offset)
211+
records.append((sec, frac, incl, orig, data[offset + 16:offset + 16 + incl]))
212+
offset += 16 + incl
213+
return Savefile(byteorder, nanosecond, linktype, tuple(records))
214+
215+
216+
def payload_pattern(size: 'int', seed: 'int' = 1) -> 'bytes':
217+
"""``size`` octets, none of them zero, so a zero-filled hole shows."""
218+
return bytes((seed + 7 * i) % 255 + 1 for i in range(size))
219+
220+
221+
class RoundTripBase(unittest.TestCase):
222+
"""The table checks, shared by the foundation round-trip modules.
223+
224+
A subclass sets :attr:`STATUSES` and :attr:`KNOWN_FAILURES`, and
225+
implements :meth:`labels` and :meth:`outcome`. This class is not collected
226+
itself, because :meth:`labels` is empty here.
227+
228+
"""
229+
230+
#: Failure statuses this module reports.
231+
STATUSES = () # type: tuple[str, ...]
232+
#: Known failures, one entry per root cause.
233+
KNOWN_FAILURES = () # type: tuple[Gap, ...]
234+
235+
def labels(self) -> 'list[str]':
236+
"""Every case label, in the order the cases run."""
237+
return []
238+
239+
def outcome(self, label: 'str') -> 'Outcome':
240+
"""Run one case."""
241+
raise NotImplementedError
242+
243+
def setUp(self) -> None:
244+
if not self.STATUSES:
245+
self.skipTest('base class')
246+
247+
# -- helpers ----------------------------------------------------------
248+
249+
def _gaps(self, labels: 'list[str]') -> 'dict[str, Gap]':
250+
table = {} # type: dict[str, Gap]
251+
for gap in self.KNOWN_FAILURES:
252+
for pattern in gap.cases:
253+
for label in fnmatch.filter(labels, pattern):
254+
self.assertIs(table.setdefault(label, gap), gap,
255+
f'{label} is listed under two root causes')
256+
return table
257+
258+
def _run(self, label: 'str') -> 'Outcome':
259+
try:
260+
with time_limit(CASE_TIMEOUT):
261+
return self.outcome(label)
262+
except TimeoutError as exc:
263+
return Outcome('TIMEOUT', str(exc))
264+
265+
# -- the tables -------------------------------------------------------
266+
267+
def test_labels_are_unique(self) -> None:
268+
labels = self.labels()
269+
self.assertTrue(labels, 'no cases')
270+
self.assertEqual(len(labels), len(set(labels)), 'duplicate case labels')
271+
272+
def test_known_failures_name_real_cases(self) -> None:
273+
labels = self.labels()
274+
stale = [pattern for gap in self.KNOWN_FAILURES for pattern in gap.cases
275+
if not fnmatch.filter(labels, pattern)]
276+
self.assertEqual(stale, [], 'KNOWN_FAILURES patterns that match no case')
277+
for gap in self.KNOWN_FAILURES:
278+
self.assertTrue(gap.cases, f'empty KNOWN_FAILURES entry: {gap.defect}')
279+
self.assertTrue(gap.fragment, f'no failure fragment: {gap.defect}')
280+
for status in _as_tuple(gap.status):
281+
self.assertIn(status, self.STATUSES + ('TIMEOUT',), gap.defect)
282+
self._gaps(labels)
283+
284+
def test_round_trip_is_exact_or_a_recorded_gap(self) -> None:
285+
labels = self.labels()
286+
gaps = self._gaps(labels)
287+
for label in labels:
288+
with self.subTest(case=label):
289+
outcome = self._run(label)
290+
if outcome.status == 'SKIP':
291+
self.skipTest(outcome.detail)
292+
gap = gaps.get(label)
293+
if gap is None:
294+
self.assertEqual(
295+
outcome.status, 'OK',
296+
f'{label} does not round-trip: {outcome.status}: {outcome.detail}. '
297+
'If this is a new defect, add it to KNOWN_FAILURES under its root cause.')
298+
continue
299+
self.assertIn(
300+
outcome.status, _as_tuple(gap.status),
301+
f'{label} was recorded as failing (#{gap.issue}: {gap.defect}) but came '
302+
f'back {outcome.status}: {outcome.detail}. If the defect is fixed, '
303+
'delete the label from its entry.')
304+
self.assertTrue(
305+
any(fragment in outcome.detail for fragment in _as_tuple(gap.fragment)),
306+
f'{label} fails, but not in the recorded way ({gap.defect}); '
307+
f'detail was {outcome.detail!r}')

0 commit comments

Comments
 (0)