|
| 1 | +# -*- coding: utf-8 -*- |
| 2 | +"""Shared harness for the protocol edge-case round-trip modules. C.f. #1202. |
| 3 | +
|
| 4 | +The owner's rule is that parse -> rebuild and make -> parse are byte-exact |
| 5 | +everywhere. The ``test_*_edge_roundtrip_unit`` modules beside this one hold |
| 6 | +hand-built edge-case packets -- minimum and maximum lengths, reserved bits set, |
| 7 | +zero-length values, padding variants, unassigned codes -- and this module runs |
| 8 | +them through three checks: |
| 9 | +
|
| 10 | +* a **parse case** must parse, its ``data`` must be the input, and |
| 11 | + ``from_data(info).data`` must be the input too; |
| 12 | +* a **cut** of a parse case (the same octets one short, and half as long) must |
| 13 | + either be rejected with an in-library exception or meet the same two |
| 14 | + equalities -- a truncated packet may be refused, but not crash and not |
| 15 | + rebuild differently; |
| 16 | +* a **make case** must construct, its octets must parse back to themselves, and |
| 17 | + ``from_data`` over the parsed ``info`` must construct the same octets again. |
| 18 | +
|
| 19 | +A case that does not close goes in its module's ``KNOWN_FAILURES``, grouped by |
| 20 | +root cause, and the table is asserted in both directions so that a fixed defect |
| 21 | +turns the module red. Malformed inputs a protocol is entitled to refuse go in |
| 22 | +``REJECTED`` instead, with the exception they raise. |
| 23 | +
|
| 24 | +Nothing here imports :mod:`pcapkit` at module level: classes are named by |
| 25 | +dotted path and resolved inside each test, after |
| 26 | +:func:`~tests._support.reimport_once_per_class`, for the reason |
| 27 | +:mod:`tests.protocols.link.test_ethernet_mac_roundtrip_unit` gives. |
| 28 | +
|
| 29 | +""" |
| 30 | + |
| 31 | +from __future__ import annotations |
| 32 | + |
| 33 | +import fnmatch |
| 34 | +import importlib |
| 35 | +import importlib.util |
| 36 | +import unittest |
| 37 | +import warnings |
| 38 | +from typing import TYPE_CHECKING, NamedTuple |
| 39 | + |
| 40 | +from tests._support import reimport_once_per_class, time_limit |
| 41 | + |
| 42 | +if TYPE_CHECKING: |
| 43 | + from typing import Any, Optional |
| 44 | + |
| 45 | +RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') |
| 46 | +HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) |
| 47 | + |
| 48 | +#: Whole seconds one case may take; a working case takes milliseconds. |
| 49 | +CASE_TIMEOUT = 30 |
| 50 | + |
| 51 | +#: Statuses a check can report. ``REJECTED`` is an in-library exception raised |
| 52 | +#: while parsing; it is a pass for a cut and for a ``REJECTED`` entry only. |
| 53 | +#: ``PADDED`` is a ``MISMATCH`` whose rebuild is the input followed by zero |
| 54 | +#: octets, the signature of a short read zero-filled on parse. |
| 55 | +STATUSES = ('OK', 'REJECTED', 'PARSE', 'SELF', 'REBUILD', 'MISMATCH', 'PADDED', |
| 56 | + 'MAKE', 'REPARSE', 'REMAKE', 'TIMEOUT') |
| 57 | + |
| 58 | +#: The defect behind every ``PADDED`` cut, shared by all the family modules. |
| 59 | +ZERO_FILLED_SHORT_READ = ( |
| 60 | + 'A header cut short inside a fixed-width field parses, because ' |
| 61 | + '``FieldBase.unpack`` zero-fills a short read (pcapkit/corekit/fields/field.py:586), ' |
| 62 | + 'and nothing records the shortfall, so ``from_data`` rebuilds the field at full width.' |
| 63 | +) |
| 64 | + |
| 65 | +#: The same read, where it is an IPv6 extension header rebuilding itself. |
| 66 | +ZERO_FILLED_EXTENSION_HEADER = ( |
| 67 | + 'A truncated IPv6 extension header rebuilds from its own ``info`` padded to its ' |
| 68 | + 'declared length: the IPv6 extension-header instance of #1458.' |
| 69 | +) |
| 70 | + |
| 71 | + |
| 72 | +class Case(NamedTuple): |
| 73 | + """One packet to parse and rebuild.""" |
| 74 | + |
| 75 | + #: Unique label, ``family/what``. |
| 76 | + label: 'str' |
| 77 | + #: Protocol class as ``module:Class``. |
| 78 | + cls: 'str' |
| 79 | + #: Wire octets. |
| 80 | + raw: 'bytes' |
| 81 | + #: Extra parse keywords, e.g. ``{'extension': True}``. |
| 82 | + kwargs: 'dict[str, Any]' = {} |
| 83 | + #: Octets the layer itself owns, when fewer than ``raw``: an extension |
| 84 | + #: header parsed with ``extension=True`` covers only its own header in |
| 85 | + #: ``data`` (#1446). :data:`None` means all of ``raw``. |
| 86 | + owned: 'Optional[int]' = None |
| 87 | + |
| 88 | + |
| 89 | +class MakeCase(NamedTuple): |
| 90 | + """One ``make`` call to construct, parse and construct again.""" |
| 91 | + |
| 92 | + #: Unique label, ``family/make/what``. |
| 93 | + label: 'str' |
| 94 | + #: Protocol class as ``module:Class``. |
| 95 | + cls: 'str' |
| 96 | + #: ``make`` keywords. |
| 97 | + kwargs: 'dict[str, Any]' |
| 98 | + #: Extra parse keywords for the re-parse. |
| 99 | + parse_kwargs: 'dict[str, Any]' = {} |
| 100 | + |
| 101 | + |
| 102 | +class Outcome(NamedTuple): |
| 103 | + """What one check found.""" |
| 104 | + |
| 105 | + status: 'str' |
| 106 | + detail: 'str' |
| 107 | + |
| 108 | + |
| 109 | +class Gap(NamedTuple): |
| 110 | + """One root cause, and every case it stops.""" |
| 111 | + |
| 112 | + #: GitHub issue tracking the defect, or :data:`None` if not filed yet. |
| 113 | + issue: 'Optional[int]' |
| 114 | + #: The defect, with the ``file:line`` that causes it. |
| 115 | + defect: 'str' |
| 116 | + #: Expected status of every case below, or the statuses one root cause |
| 117 | + #: shows as, of which each case must report one. |
| 118 | + status: 'str | tuple[str, ...]' |
| 119 | + #: Substring every failure detail must contain, or ``''``. |
| 120 | + fragment: 'str' |
| 121 | + #: Labels of the cases this defect stops, as :mod:`fnmatch` patterns. Each |
| 122 | + #: pattern must match at least one case, and every case it matches must |
| 123 | + #: fail as recorded, so a pattern is only as wide as the defect. |
| 124 | + cases: 'tuple[str, ...]' |
| 125 | + |
| 126 | + |
| 127 | +class Reject(NamedTuple): |
| 128 | + """A malformed input the protocol is entitled to refuse.""" |
| 129 | + |
| 130 | + #: Name of the in-library exception class raised. |
| 131 | + exc: 'str' |
| 132 | + #: Substring the message must contain. |
| 133 | + fragment: 'str' |
| 134 | + |
| 135 | + |
| 136 | +def _statuses(gap: 'Gap') -> 'tuple[str, ...]': |
| 137 | + return (gap.status,) if isinstance(gap.status, str) else gap.status |
| 138 | + |
| 139 | + |
| 140 | +def resolve(path: 'str') -> 'type': |
| 141 | + """Import ``module:Class`` from the current :mod:`pcapkit` import.""" |
| 142 | + module, _, name = path.partition(':') |
| 143 | + return getattr(importlib.import_module(module), name) |
| 144 | + |
| 145 | + |
| 146 | +def _describe(exc: 'BaseException') -> 'str': |
| 147 | + return f'{type(exc).__name__}: {exc}' |
| 148 | + |
| 149 | + |
| 150 | +def _in_library(exc: 'BaseException') -> 'bool': |
| 151 | + base = importlib.import_module('pcapkit.utilities.exceptions').BaseError |
| 152 | + return isinstance(exc, base) |
| 153 | + |
| 154 | + |
| 155 | +def _diff(got: 'bytes', want: 'bytes') -> 'str': |
| 156 | + return f'{len(got)} octets {got.hex()} != {len(want)} octets {want.hex()}' |
| 157 | + |
| 158 | + |
| 159 | +def run_parse(case: 'Case') -> 'Outcome': |
| 160 | + """Parse ``case.raw``, then rebuild it from ``info``.""" |
| 161 | + cls = resolve(case.cls) |
| 162 | + raw = case.raw if case.owned is None else case.raw[:case.owned] |
| 163 | + with warnings.catch_warnings(): |
| 164 | + warnings.simplefilter('ignore') |
| 165 | + try: |
| 166 | + proto = cls(case.raw, len(case.raw), **case.kwargs) |
| 167 | + except Exception as exc: # pylint: disable=broad-except |
| 168 | + return Outcome('REJECTED' if _in_library(exc) else 'PARSE', _describe(exc)) |
| 169 | + if proto.data != raw: |
| 170 | + return Outcome('SELF', _diff(proto.data, raw)) |
| 171 | + try: |
| 172 | + rebuilt = type(proto).from_data(proto.info).data |
| 173 | + except Exception as exc: # pylint: disable=broad-except |
| 174 | + return Outcome('REBUILD', _describe(exc)) |
| 175 | + if rebuilt != raw: |
| 176 | + if len(rebuilt) > len(raw) and rebuilt == raw.ljust(len(rebuilt), b'\x00'): |
| 177 | + return Outcome('PADDED', _diff(rebuilt, raw)) |
| 178 | + return Outcome('MISMATCH', _diff(rebuilt, raw)) |
| 179 | + return Outcome('OK', '') |
| 180 | + |
| 181 | + |
| 182 | +def run_make(case: 'MakeCase') -> 'Outcome': |
| 183 | + """Construct, parse the octets back, and construct from the parsed ``info``.""" |
| 184 | + cls = resolve(case.cls) |
| 185 | + with warnings.catch_warnings(): |
| 186 | + warnings.simplefilter('ignore') |
| 187 | + try: |
| 188 | + made = cls(**case.kwargs).data |
| 189 | + except Exception as exc: # pylint: disable=broad-except |
| 190 | + return Outcome('MAKE', _describe(exc)) |
| 191 | + try: |
| 192 | + parsed = cls(made, len(made), **case.parse_kwargs) |
| 193 | + except Exception as exc: # pylint: disable=broad-except |
| 194 | + return Outcome('REPARSE', _describe(exc)) |
| 195 | + if parsed.data != made: |
| 196 | + return Outcome('REPARSE', _diff(parsed.data, made)) |
| 197 | + try: |
| 198 | + again = type(parsed).from_data(parsed.info).data |
| 199 | + except Exception as exc: # pylint: disable=broad-except |
| 200 | + return Outcome('REMAKE', _describe(exc)) |
| 201 | + if again != made: |
| 202 | + return Outcome('REMAKE', _diff(again, made)) |
| 203 | + return Outcome('OK', '') |
| 204 | + |
| 205 | + |
| 206 | +def cuts(cases: 'tuple[Case, ...]') -> 'tuple[Case, ...]': |
| 207 | + """Each case one octet short, and half as long; at most two per case.""" |
| 208 | + out = [] # type: list[Case] |
| 209 | + for case in cases: |
| 210 | + size = len(case.raw) |
| 211 | + for keep in sorted({size - 1, size // 2}): |
| 212 | + if keep > 0: |
| 213 | + owned = None if case.owned is None else min(case.owned, keep) |
| 214 | + out.append(case._replace(label=f'{case.label}/cut{keep}', raw=case.raw[:keep], owned=owned)) |
| 215 | + return tuple(out) |
| 216 | + |
| 217 | + |
| 218 | +class EdgeRoundTripBase(unittest.TestCase): |
| 219 | + """The assertions, shared by every family module. |
| 220 | +
|
| 221 | + A subclass sets the four tables; this class is not collected itself |
| 222 | + because it has no tables (:attr:`CASES` is empty). |
| 223 | +
|
| 224 | + """ |
| 225 | + |
| 226 | + #: Parse cases. |
| 227 | + CASES = () # type: tuple[Case, ...] |
| 228 | + #: Make cases. |
| 229 | + MAKE_CASES = () # type: tuple[MakeCase, ...] |
| 230 | + #: Labels of malformed inputs that must be refused, and how. |
| 231 | + REJECTED = {} # type: dict[str, Reject] |
| 232 | + #: Known failures, one entry per root cause. |
| 233 | + KNOWN_FAILURES = () # type: tuple[Gap, ...] |
| 234 | + |
| 235 | + def setUp(self) -> None: |
| 236 | + if not self.CASES: |
| 237 | + self.skipTest('base class') |
| 238 | + if not HAS_RUNTIME: |
| 239 | + self.skipTest('runtime dependencies not installed') |
| 240 | + reimport_once_per_class(self) |
| 241 | + |
| 242 | + # -- helpers ---------------------------------------------------------- |
| 243 | + |
| 244 | + def _gaps(self) -> 'dict[str, Gap]': |
| 245 | + """Every case label a ``KNOWN_FAILURES`` pattern matches, to its entry.""" |
| 246 | + labels = sorted(self._labels()) |
| 247 | + table = {} # type: dict[str, Gap] |
| 248 | + for gap in self.KNOWN_FAILURES: |
| 249 | + for pattern in gap.cases: |
| 250 | + for label in fnmatch.filter(labels, pattern): |
| 251 | + self.assertIs(table.setdefault(label, gap), gap, |
| 252 | + f'{label} is listed under two root causes') |
| 253 | + return table |
| 254 | + |
| 255 | + def _all_cases(self) -> 'tuple[Case, ...]': |
| 256 | + return self.CASES + cuts(tuple(c for c in self.CASES if c.label not in self.REJECTED)) |
| 257 | + |
| 258 | + def _labels(self) -> 'set[str]': |
| 259 | + return {c.label for c in self._all_cases()} | {c.label for c in self.MAKE_CASES} |
| 260 | + |
| 261 | + def _check(self, label: 'str', outcome: 'Outcome', passing: 'tuple[str, ...]', |
| 262 | + gaps: 'dict[str, Gap]') -> None: |
| 263 | + gap = gaps.get(label) |
| 264 | + if gap is None: |
| 265 | + self.assertIn( |
| 266 | + outcome.status, passing, |
| 267 | + f'{label} does not round-trip: {outcome.status}: {outcome.detail}. ' |
| 268 | + f'If this is a new defect, add it to KNOWN_FAILURES under its root cause.') |
| 269 | + return |
| 270 | + self.assertIn( |
| 271 | + outcome.status, _statuses(gap), |
| 272 | + f'{label} was recorded as failing with {gap.status} ' |
| 273 | + f'(#{gap.issue}: {gap.defect}) but came back {outcome.status}: ' |
| 274 | + f'{outcome.detail}. If the defect is fixed, delete the label from its entry.') |
| 275 | + if gap.fragment: |
| 276 | + self.assertIn(gap.fragment, outcome.detail, |
| 277 | + f'{label} fails, but not in the recorded way ({gap.defect})') |
| 278 | + |
| 279 | + def _run(self, func: 'Any', case: 'Any') -> 'Outcome': |
| 280 | + try: |
| 281 | + with time_limit(CASE_TIMEOUT): |
| 282 | + return func(case) |
| 283 | + except TimeoutError as exc: |
| 284 | + return Outcome('TIMEOUT', str(exc)) |
| 285 | + |
| 286 | + # -- the tables ------------------------------------------------------- |
| 287 | + |
| 288 | + def test_labels_are_unique(self) -> None: |
| 289 | + labels = [c.label for c in self.CASES] + [c.label for c in self.MAKE_CASES] |
| 290 | + self.assertEqual(len(labels), len(set(labels)), 'duplicate case labels') |
| 291 | + |
| 292 | + def test_tables_name_real_cases(self) -> None: |
| 293 | + """Every ``KNOWN_FAILURES`` pattern and ``REJECTED`` label names a case.""" |
| 294 | + labels = sorted(self._labels()) |
| 295 | + stale = [pattern for gap in self.KNOWN_FAILURES for pattern in gap.cases |
| 296 | + if not fnmatch.filter(labels, pattern)] |
| 297 | + self.assertEqual(stale, [], 'KNOWN_FAILURES patterns that match no case') |
| 298 | + self.assertEqual(sorted(set(self.REJECTED) - set(labels)), [], 'stale REJECTED labels') |
| 299 | + self.assertEqual(sorted(set(self._gaps()) & set(self.REJECTED)), [], |
| 300 | + 'a label cannot be both rejected and a known failure') |
| 301 | + for gap in self.KNOWN_FAILURES: |
| 302 | + for status in _statuses(gap): |
| 303 | + self.assertIn(status, STATUSES) |
| 304 | + self.assertNotEqual(status, 'OK', gap.defect) |
| 305 | + self.assertTrue(gap.cases, f'empty KNOWN_FAILURES entry: {gap.defect}') |
| 306 | + |
| 307 | + # -- the checks ------------------------------------------------------- |
| 308 | + |
| 309 | + def test_parse_then_rebuild_is_byte_exact(self) -> None: |
| 310 | + gaps = self._gaps() |
| 311 | + for case in self.CASES: |
| 312 | + if case.label in self.REJECTED: |
| 313 | + continue |
| 314 | + with self.subTest(case=case.label): |
| 315 | + self._check(case.label, self._run(run_parse, case), ('OK',), gaps) |
| 316 | + |
| 317 | + def test_cut_is_rejected_or_byte_exact(self) -> None: |
| 318 | + gaps = self._gaps() |
| 319 | + for case in cuts(tuple(c for c in self.CASES if c.label not in self.REJECTED)): |
| 320 | + with self.subTest(case=case.label): |
| 321 | + self._check(case.label, self._run(run_parse, case), ('OK', 'REJECTED'), gaps) |
| 322 | + |
| 323 | + def test_make_then_parse_is_stable(self) -> None: |
| 324 | + gaps = self._gaps() |
| 325 | + for case in self.MAKE_CASES: |
| 326 | + with self.subTest(case=case.label): |
| 327 | + self._check(case.label, self._run(run_make, case), ('OK',), gaps) |
| 328 | + |
| 329 | + def test_malformed_input_is_rejected_in_library(self) -> None: |
| 330 | + cases = {c.label: c for c in self.CASES} |
| 331 | + for label, reject in self.REJECTED.items(): |
| 332 | + with self.subTest(case=label): |
| 333 | + outcome = self._run(run_parse, cases[label]) |
| 334 | + self.assertEqual(outcome.status, 'REJECTED', f'{label}: {outcome.detail}') |
| 335 | + self.assertTrue(outcome.detail.startswith(f'{reject.exc}: '), outcome.detail) |
| 336 | + self.assertIn(reject.fragment, outcome.detail) |
0 commit comments