Skip to content

Commit d0bbf31

Browse files
committed
test(protocols): edge-case round trips for every layer family (#1202)
Four unit-tier modules and a shared harness build minimum and maximum lengths, reserved bits, zero-length values, padding variants and unassigned codes for link, internet, transport and application protocols. Each packet must parse back to itself and rebuild from its info; one-short and half-length cuts must be refused in-library or round-trip too; make() must be stable through a re-parse. KNOWN_FAILURES groups what still fails by root cause and is asserted in both directions: #1455 (L2TPv2/OSPF length trailer), #1456 (RPL compression under IPv6), #1457 (truncated RPL address vectors), #1458 (zero-filled short reads) and #1451 (its IPv6 extension-header case).
1 parent b14d676 commit d0bbf31

5 files changed

Lines changed: 1110 additions & 0 deletions

File tree

‎tests/protocols/_edge_roundtrip.py‎

Lines changed: 336 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,336 @@
1+
# -*- coding: utf-8 -*-
2+
"""Shared harness for the protocol edge-case round-trip modules. C.f. #1202.
3+
4+
The owner's rule is that parse -> rebuild and make -> parse are byte-exact
5+
everywhere. The ``test_*_edge_roundtrip_unit`` modules beside this one hold
6+
hand-built edge-case packets -- minimum and maximum lengths, reserved bits set,
7+
zero-length values, padding variants, unassigned codes -- and this module runs
8+
them through three checks:
9+
10+
* a **parse case** must parse, its ``data`` must be the input, and
11+
``from_data(info).data`` must be the input too;
12+
* a **cut** of a parse case (the same octets one short, and half as long) must
13+
either be rejected with an in-library exception or meet the same two
14+
equalities -- a truncated packet may be refused, but not crash and not
15+
rebuild differently;
16+
* a **make case** must construct, its octets must parse back to themselves, and
17+
``from_data`` over the parsed ``info`` must construct the same octets again.
18+
19+
A case that does not close goes in its module's ``KNOWN_FAILURES``, grouped by
20+
root cause, and the table is asserted in both directions so that a fixed defect
21+
turns the module red. Malformed inputs a protocol is entitled to refuse go in
22+
``REJECTED`` instead, with the exception they raise.
23+
24+
Nothing here imports :mod:`pcapkit` at module level: classes are named by
25+
dotted path and resolved inside each test, after
26+
:func:`~tests._support.reimport_once_per_class`, for the reason
27+
:mod:`tests.protocols.link.test_ethernet_mac_roundtrip_unit` gives.
28+
29+
"""
30+
31+
from __future__ import annotations
32+
33+
import fnmatch
34+
import importlib
35+
import importlib.util
36+
import unittest
37+
import warnings
38+
from typing import TYPE_CHECKING, NamedTuple
39+
40+
from tests._support import reimport_once_per_class, time_limit
41+
42+
if TYPE_CHECKING:
43+
from typing import Any, Optional
44+
45+
RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper')
46+
HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS)
47+
48+
#: Whole seconds one case may take; a working case takes milliseconds.
49+
CASE_TIMEOUT = 30
50+
51+
#: Statuses a check can report. ``REJECTED`` is an in-library exception raised
52+
#: while parsing; it is a pass for a cut and for a ``REJECTED`` entry only.
53+
#: ``PADDED`` is a ``MISMATCH`` whose rebuild is the input followed by zero
54+
#: octets, the signature of a short read zero-filled on parse.
55+
STATUSES = ('OK', 'REJECTED', 'PARSE', 'SELF', 'REBUILD', 'MISMATCH', 'PADDED',
56+
'MAKE', 'REPARSE', 'REMAKE', 'TIMEOUT')
57+
58+
#: The defect behind every ``PADDED`` cut, shared by all the family modules.
59+
ZERO_FILLED_SHORT_READ = (
60+
'A header cut short inside a fixed-width field parses, because '
61+
'``FieldBase.unpack`` zero-fills a short read (pcapkit/corekit/fields/field.py:586), '
62+
'and nothing records the shortfall, so ``from_data`` rebuilds the field at full width.'
63+
)
64+
65+
#: The same read, where it is an IPv6 extension header rebuilding itself.
66+
ZERO_FILLED_EXTENSION_HEADER = (
67+
'A truncated IPv6 extension header rebuilds from its own ``info`` padded to its '
68+
'declared length: the IPv6 extension-header instance of #1458.'
69+
)
70+
71+
72+
class Case(NamedTuple):
73+
"""One packet to parse and rebuild."""
74+
75+
#: Unique label, ``family/what``.
76+
label: 'str'
77+
#: Protocol class as ``module:Class``.
78+
cls: 'str'
79+
#: Wire octets.
80+
raw: 'bytes'
81+
#: Extra parse keywords, e.g. ``{'extension': True}``.
82+
kwargs: 'dict[str, Any]' = {}
83+
#: Octets the layer itself owns, when fewer than ``raw``: an extension
84+
#: header parsed with ``extension=True`` covers only its own header in
85+
#: ``data`` (#1446). :data:`None` means all of ``raw``.
86+
owned: 'Optional[int]' = None
87+
88+
89+
class MakeCase(NamedTuple):
90+
"""One ``make`` call to construct, parse and construct again."""
91+
92+
#: Unique label, ``family/make/what``.
93+
label: 'str'
94+
#: Protocol class as ``module:Class``.
95+
cls: 'str'
96+
#: ``make`` keywords.
97+
kwargs: 'dict[str, Any]'
98+
#: Extra parse keywords for the re-parse.
99+
parse_kwargs: 'dict[str, Any]' = {}
100+
101+
102+
class Outcome(NamedTuple):
103+
"""What one check found."""
104+
105+
status: 'str'
106+
detail: 'str'
107+
108+
109+
class Gap(NamedTuple):
110+
"""One root cause, and every case it stops."""
111+
112+
#: GitHub issue tracking the defect, or :data:`None` if not filed yet.
113+
issue: 'Optional[int]'
114+
#: The defect, with the ``file:line`` that causes it.
115+
defect: 'str'
116+
#: Expected status of every case below, or the statuses one root cause
117+
#: shows as, of which each case must report one.
118+
status: 'str | tuple[str, ...]'
119+
#: Substring every failure detail must contain, or ``''``.
120+
fragment: 'str'
121+
#: Labels of the cases this defect stops, as :mod:`fnmatch` patterns. Each
122+
#: pattern must match at least one case, and every case it matches must
123+
#: fail as recorded, so a pattern is only as wide as the defect.
124+
cases: 'tuple[str, ...]'
125+
126+
127+
class Reject(NamedTuple):
128+
"""A malformed input the protocol is entitled to refuse."""
129+
130+
#: Name of the in-library exception class raised.
131+
exc: 'str'
132+
#: Substring the message must contain.
133+
fragment: 'str'
134+
135+
136+
def _statuses(gap: 'Gap') -> 'tuple[str, ...]':
137+
return (gap.status,) if isinstance(gap.status, str) else gap.status
138+
139+
140+
def resolve(path: 'str') -> 'type':
141+
"""Import ``module:Class`` from the current :mod:`pcapkit` import."""
142+
module, _, name = path.partition(':')
143+
return getattr(importlib.import_module(module), name)
144+
145+
146+
def _describe(exc: 'BaseException') -> 'str':
147+
return f'{type(exc).__name__}: {exc}'
148+
149+
150+
def _in_library(exc: 'BaseException') -> 'bool':
151+
base = importlib.import_module('pcapkit.utilities.exceptions').BaseError
152+
return isinstance(exc, base)
153+
154+
155+
def _diff(got: 'bytes', want: 'bytes') -> 'str':
156+
return f'{len(got)} octets {got.hex()} != {len(want)} octets {want.hex()}'
157+
158+
159+
def run_parse(case: 'Case') -> 'Outcome':
160+
"""Parse ``case.raw``, then rebuild it from ``info``."""
161+
cls = resolve(case.cls)
162+
raw = case.raw if case.owned is None else case.raw[:case.owned]
163+
with warnings.catch_warnings():
164+
warnings.simplefilter('ignore')
165+
try:
166+
proto = cls(case.raw, len(case.raw), **case.kwargs)
167+
except Exception as exc: # pylint: disable=broad-except
168+
return Outcome('REJECTED' if _in_library(exc) else 'PARSE', _describe(exc))
169+
if proto.data != raw:
170+
return Outcome('SELF', _diff(proto.data, raw))
171+
try:
172+
rebuilt = type(proto).from_data(proto.info).data
173+
except Exception as exc: # pylint: disable=broad-except
174+
return Outcome('REBUILD', _describe(exc))
175+
if rebuilt != raw:
176+
if len(rebuilt) > len(raw) and rebuilt == raw.ljust(len(rebuilt), b'\x00'):
177+
return Outcome('PADDED', _diff(rebuilt, raw))
178+
return Outcome('MISMATCH', _diff(rebuilt, raw))
179+
return Outcome('OK', '')
180+
181+
182+
def run_make(case: 'MakeCase') -> 'Outcome':
183+
"""Construct, parse the octets back, and construct from the parsed ``info``."""
184+
cls = resolve(case.cls)
185+
with warnings.catch_warnings():
186+
warnings.simplefilter('ignore')
187+
try:
188+
made = cls(**case.kwargs).data
189+
except Exception as exc: # pylint: disable=broad-except
190+
return Outcome('MAKE', _describe(exc))
191+
try:
192+
parsed = cls(made, len(made), **case.parse_kwargs)
193+
except Exception as exc: # pylint: disable=broad-except
194+
return Outcome('REPARSE', _describe(exc))
195+
if parsed.data != made:
196+
return Outcome('REPARSE', _diff(parsed.data, made))
197+
try:
198+
again = type(parsed).from_data(parsed.info).data
199+
except Exception as exc: # pylint: disable=broad-except
200+
return Outcome('REMAKE', _describe(exc))
201+
if again != made:
202+
return Outcome('REMAKE', _diff(again, made))
203+
return Outcome('OK', '')
204+
205+
206+
def cuts(cases: 'tuple[Case, ...]') -> 'tuple[Case, ...]':
207+
"""Each case one octet short, and half as long; at most two per case."""
208+
out = [] # type: list[Case]
209+
for case in cases:
210+
size = len(case.raw)
211+
for keep in sorted({size - 1, size // 2}):
212+
if keep > 0:
213+
owned = None if case.owned is None else min(case.owned, keep)
214+
out.append(case._replace(label=f'{case.label}/cut{keep}', raw=case.raw[:keep], owned=owned))
215+
return tuple(out)
216+
217+
218+
class EdgeRoundTripBase(unittest.TestCase):
219+
"""The assertions, shared by every family module.
220+
221+
A subclass sets the four tables; this class is not collected itself
222+
because it has no tables (:attr:`CASES` is empty).
223+
224+
"""
225+
226+
#: Parse cases.
227+
CASES = () # type: tuple[Case, ...]
228+
#: Make cases.
229+
MAKE_CASES = () # type: tuple[MakeCase, ...]
230+
#: Labels of malformed inputs that must be refused, and how.
231+
REJECTED = {} # type: dict[str, Reject]
232+
#: Known failures, one entry per root cause.
233+
KNOWN_FAILURES = () # type: tuple[Gap, ...]
234+
235+
def setUp(self) -> None:
236+
if not self.CASES:
237+
self.skipTest('base class')
238+
if not HAS_RUNTIME:
239+
self.skipTest('runtime dependencies not installed')
240+
reimport_once_per_class(self)
241+
242+
# -- helpers ----------------------------------------------------------
243+
244+
def _gaps(self) -> 'dict[str, Gap]':
245+
"""Every case label a ``KNOWN_FAILURES`` pattern matches, to its entry."""
246+
labels = sorted(self._labels())
247+
table = {} # type: dict[str, Gap]
248+
for gap in self.KNOWN_FAILURES:
249+
for pattern in gap.cases:
250+
for label in fnmatch.filter(labels, pattern):
251+
self.assertIs(table.setdefault(label, gap), gap,
252+
f'{label} is listed under two root causes')
253+
return table
254+
255+
def _all_cases(self) -> 'tuple[Case, ...]':
256+
return self.CASES + cuts(tuple(c for c in self.CASES if c.label not in self.REJECTED))
257+
258+
def _labels(self) -> 'set[str]':
259+
return {c.label for c in self._all_cases()} | {c.label for c in self.MAKE_CASES}
260+
261+
def _check(self, label: 'str', outcome: 'Outcome', passing: 'tuple[str, ...]',
262+
gaps: 'dict[str, Gap]') -> None:
263+
gap = gaps.get(label)
264+
if gap is None:
265+
self.assertIn(
266+
outcome.status, passing,
267+
f'{label} does not round-trip: {outcome.status}: {outcome.detail}. '
268+
f'If this is a new defect, add it to KNOWN_FAILURES under its root cause.')
269+
return
270+
self.assertIn(
271+
outcome.status, _statuses(gap),
272+
f'{label} was recorded as failing with {gap.status} '
273+
f'(#{gap.issue}: {gap.defect}) but came back {outcome.status}: '
274+
f'{outcome.detail}. If the defect is fixed, delete the label from its entry.')
275+
if gap.fragment:
276+
self.assertIn(gap.fragment, outcome.detail,
277+
f'{label} fails, but not in the recorded way ({gap.defect})')
278+
279+
def _run(self, func: 'Any', case: 'Any') -> 'Outcome':
280+
try:
281+
with time_limit(CASE_TIMEOUT):
282+
return func(case)
283+
except TimeoutError as exc:
284+
return Outcome('TIMEOUT', str(exc))
285+
286+
# -- the tables -------------------------------------------------------
287+
288+
def test_labels_are_unique(self) -> None:
289+
labels = [c.label for c in self.CASES] + [c.label for c in self.MAKE_CASES]
290+
self.assertEqual(len(labels), len(set(labels)), 'duplicate case labels')
291+
292+
def test_tables_name_real_cases(self) -> None:
293+
"""Every ``KNOWN_FAILURES`` pattern and ``REJECTED`` label names a case."""
294+
labels = sorted(self._labels())
295+
stale = [pattern for gap in self.KNOWN_FAILURES for pattern in gap.cases
296+
if not fnmatch.filter(labels, pattern)]
297+
self.assertEqual(stale, [], 'KNOWN_FAILURES patterns that match no case')
298+
self.assertEqual(sorted(set(self.REJECTED) - set(labels)), [], 'stale REJECTED labels')
299+
self.assertEqual(sorted(set(self._gaps()) & set(self.REJECTED)), [],
300+
'a label cannot be both rejected and a known failure')
301+
for gap in self.KNOWN_FAILURES:
302+
for status in _statuses(gap):
303+
self.assertIn(status, STATUSES)
304+
self.assertNotEqual(status, 'OK', gap.defect)
305+
self.assertTrue(gap.cases, f'empty KNOWN_FAILURES entry: {gap.defect}')
306+
307+
# -- the checks -------------------------------------------------------
308+
309+
def test_parse_then_rebuild_is_byte_exact(self) -> None:
310+
gaps = self._gaps()
311+
for case in self.CASES:
312+
if case.label in self.REJECTED:
313+
continue
314+
with self.subTest(case=case.label):
315+
self._check(case.label, self._run(run_parse, case), ('OK',), gaps)
316+
317+
def test_cut_is_rejected_or_byte_exact(self) -> None:
318+
gaps = self._gaps()
319+
for case in cuts(tuple(c for c in self.CASES if c.label not in self.REJECTED)):
320+
with self.subTest(case=case.label):
321+
self._check(case.label, self._run(run_parse, case), ('OK', 'REJECTED'), gaps)
322+
323+
def test_make_then_parse_is_stable(self) -> None:
324+
gaps = self._gaps()
325+
for case in self.MAKE_CASES:
326+
with self.subTest(case=case.label):
327+
self._check(case.label, self._run(run_make, case), ('OK',), gaps)
328+
329+
def test_malformed_input_is_rejected_in_library(self) -> None:
330+
cases = {c.label: c for c in self.CASES}
331+
for label, reject in self.REJECTED.items():
332+
with self.subTest(case=label):
333+
outcome = self._run(run_parse, cases[label])
334+
self.assertEqual(outcome.status, 'REJECTED', f'{label}: {outcome.detail}')
335+
self.assertTrue(outcome.detail.startswith(f'{reject.exc}: '), outcome.detail)
336+
self.assertIn(reject.fragment, outcome.detail)

0 commit comments

Comments
 (0)