Skip to content

Commit db3b2ec

Browse files
committed
fix(corekit): keep a NumberField's bits outside bit_length as captured (#1488)
- NumberField.post_process: when the octets carry bits outside bit_length that pre_process would not write back (zeros, or a negative signed value's sign extension), return an int subclass recording them with the capturing field's length, bit_length, sign and byte order. pre_process writes them back only for a field matching all four; any other field, and any other value, is truncated to bit_length bits as before. - EnumField.post_process: carry that record on the pseudo-member, or on an unregistered copy of a registry member, which stays isinstance and equal. - Delete the Gap(1488) row; relax one assertIs pin to isinstance/equality for an octet whose spare bits are set.
1 parent 6af87f6 commit db3b2ec

4 files changed

Lines changed: 357 additions & 16 deletions

File tree

‎pcapkit/corekit/fields/numbers.py‎

Lines changed: 147 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,92 @@
3131

3232
_T = TypeVar('_T', bound='int')
3333

34+
#: Instance attribute a parsed value records its bits outside
35+
#: :attr:`NumberField.bit_length` under, as ``(outside, length, bit_length,
36+
#: signed, byteorder)``. See GitHub issue :issue:`1488`.
37+
_OUTSIDE_BITS = '_outside_bits_'
38+
39+
40+
class _CapturedInt(int):
41+
"""An :obj:`int` that also records the bits outside its field's
42+
:attr:`~NumberField.bit_length` as they were captured.
43+
44+
It equals, hashes and renders as the plain :obj:`int` it holds, and any
45+
arithmetic on it yields a plain :obj:`int`, which is what lets a value a
46+
caller derives from a parsed one keep today's truncating semantics. See
47+
GitHub issue :issue:`1488`.
48+
49+
"""
50+
51+
52+
def _capture_key(field: 'NumberField') -> 'tuple[int, int, bool, str]':
53+
"""Return the shape of ``field`` a captured value's bits belong to.
54+
55+
Args:
56+
field: The field packing or unpacking the value.
57+
58+
Returns:
59+
``(length, bit_length, signed, byteorder)``.
60+
61+
"""
62+
return (field._length, field._bit_length, # pylint: disable=protected-access
63+
field._signed, field._byteorder) # pylint: disable=protected-access
64+
65+
66+
def _capture_record(value: 'Any') -> 'Optional[tuple[int, int, int, bool, str]]':
67+
"""Return the capture ``value`` records, or :data:`None` if it records none.
68+
69+
Args:
70+
value: Field value.
71+
72+
Returns:
73+
``(outside, length, bit_length, signed, byteorder)``, or :data:`None`.
74+
75+
The instance dictionary is read rather than :func:`getattr`, so that no
76+
class-level attribute -- an enumeration's member lookup among them -- can
77+
answer for it.
78+
79+
"""
80+
return getattr(value, '__dict__', {}).get(_OUTSIDE_BITS)
81+
82+
83+
def _outside_bits(value: 'Any', field: 'NumberField') -> 'Optional[int]':
84+
"""Return the captured bits ``value`` records outside ``field``'s
85+
:attr:`~NumberField.bit_length`, or :data:`None`.
86+
87+
Args:
88+
value: Field value.
89+
field: The field packing it.
90+
91+
Returns:
92+
The bits, in place (not shifted down), if ``value`` was captured by a
93+
field of the same length, bit length, sign and byte order as
94+
``field``; :data:`None` otherwise, so that any other field packs it
95+
exactly as it would the plain value.
96+
97+
"""
98+
record = _capture_record(value)
99+
if record is None or tuple(record[1:]) != _capture_key(field):
100+
return None
101+
return record[0]
102+
103+
104+
def _with_outside_bits(value: 'int', record: 'tuple[int, int, int, bool, str]') -> 'int':
105+
"""Return ``value`` recording ``record`` as its captured bits outside
106+
:attr:`~NumberField.bit_length`.
107+
108+
Args:
109+
value: Field value, the low :attr:`~NumberField.bit_length` bits.
110+
record: ``(outside, length, bit_length, signed, byteorder)``.
111+
112+
Returns:
113+
A :class:`_CapturedInt` equal to ``value``.
114+
115+
"""
116+
captured = _CapturedInt(value)
117+
setattr(captured, _OUTSIDE_BITS, record)
118+
return captured
119+
34120

35121
class NumberField(Field[int], Generic[_T]):
36122
"""Numerical value for protocol fields.
@@ -276,9 +362,26 @@ def pre_process(self, value: 'int', packet: 'dict[str, Any]') -> 'int | bytes':
276362
:meth:`int.to_bytes` and :func:`struct.pack` both refuse. See GitHub
277363
issue :issue:`599`.
278364
365+
A value :meth:`post_process` returned for octets whose bits outside
366+
:attr:`bit_length` were not what packing writes there carries those
367+
bits (see :func:`_outside_bits`), and they are written back as
368+
captured, so ``pack(unpack(b)) == b`` holds for every ``b``. Only a
369+
field of the same length, bit length, sign and byte order as the
370+
one that captured them honours them; any other field, and any other
371+
value -- one a caller built, or the result of arithmetic on a
372+
parsed one, which is a plain :obj:`int` -- is truncated to
373+
:attr:`bit_length` bits exactly as before. See GitHub issue
374+
:issue:`1488`.
375+
279376
"""
377+
outside = _outside_bits(value, self)
280378
value = value & self._bit_mask
281-
if self._signed and value > self._bit_mask >> 1:
379+
if outside is not None and self._length >= 0:
380+
value |= outside
381+
width = self._length * 8
382+
if self._signed and value >> (width - 1):
383+
value -= 1 << width
384+
elif self._signed and value > self._bit_mask >> 1:
282385
value -= self._bit_mask + 1
283386

284387
if self._need_process and self._length < 0:
@@ -313,15 +416,30 @@ def post_process(self, value: 'int | bytes', packet: 'dict[str, Any]') -> 'int':
313416
resolved signed field therefore decodes ``ffffffff`` as ``-1``
314417
rather than ``4294967295``. See GitHub issue :issue:`1274`.
315418
419+
The value is the low :attr:`bit_length` bits, but the octets hold
420+
more. When the bits outside it are not what :meth:`pre_process`
421+
would write there for this value -- zeros, or for a negative signed
422+
value the sign extension -- they are captured but uninterpreted,
423+
and the value records them (see :func:`_with_outside_bits`) so that
424+
packing it writes them back. Otherwise the value is a plain
425+
:obj:`int`, as before. See GitHub issue :issue:`1488`.
426+
316427
"""
317428
if not self._need_process:
318-
value = cast('int', value) & self._bit_mask
429+
raw = cast('int', value)
319430
else:
320-
value = int.from_bytes(
431+
raw = int.from_bytes(
321432
cast('bytes', value), self._byteorder, signed=self._signed
322-
) & self._bit_mask
433+
)
434+
value = raw & self._bit_mask
323435
if self._signed and value > self._bit_mask >> 1:
324436
value -= self._bit_mask + 1
437+
438+
if 0 <= self._bit_length < self._length * 8:
439+
spare = ((1 << (self._length * 8)) - 1) & ~self._bit_mask
440+
outside = raw & spare
441+
if outside != (spare if value < 0 else 0):
442+
return _with_outside_bits(value, (outside, *_capture_key(self)))
325443
return value
326444

327445

@@ -626,18 +744,41 @@ def post_process(self, value: 'int | bytes', packet: 'dict[str, Any]') -> 'Stdli
626744
a registry registered from outside :mod:`pcapkit.const`, which has no
627745
such obligation to stay quiet.
628746
747+
Octets carrying bits outside :attr:`~NumberField.bit_length` that
748+
:meth:`NumberField.post_process` recorded resolve to a member that
749+
records them too, so that packing it writes them back. A registry
750+
member is shared, so that one is an unregistered copy of it, built
751+
by :meth:`_unregistered_member`: ``isinstance`` against the
752+
registry and equality with the member still hold, identity does
753+
not. See GitHub issue :issue:`1488`.
754+
629755
"""
630756
value = super().post_process(value, packet)
757+
record = _capture_record(value)
631758
if self._namespace is not None:
632759
try:
633-
return self._namespace(value)
760+
member = self._namespace(value)
634761
except ValueError as error:
635762
# NOTE: An in-library rejection is pcapkit's own decision about
636763
# the value, rather than the enumeration library reporting that
637764
# no member carries it, so it is not this layer's to absorb.
638765
if isinstance(error, BaseError):
639766
raise
640-
return self._pseudo_member(value)
767+
else:
768+
if record is None:
769+
return member
770+
# NOTE: A registry member is shared, so the captured bits
771+
# outside ``bit_length`` go on an unregistered copy of it
772+
# rather than on the member itself (:issue:`1488`).
773+
attrs = {key: val for key, val in vars(member).items()
774+
if key not in ('_name_', '_value_', '__reduce_ex__')}
775+
attrs[_OUTSIDE_BITS] = record
776+
return self._unregistered_member(
777+
type(member), member._value_, member._name_, **attrs) # pylint: disable=protected-access
778+
pseudo = self._pseudo_member(int(value))
779+
if record is not None:
780+
setattr(pseudo, _OUTSIDE_BITS, record)
781+
return pseudo
641782

642783
def _pseudo_member(self, value: 'int') -> 'StdlibEnum | AenumEnum':
643784
"""Build the bounded nameless pseudo-member this method falls back to

‎tests/corekit/test_field_roundtrip_unit.py‎

Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@
3434
from typing import TYPE_CHECKING, NamedTuple
3535

3636
from tests._support import reimport_once_per_class
37-
from tests.corekit._roundtrip import (OK, Gap, KnownFailureTable, Outcome, describe, diff, run,
37+
from tests.corekit._roundtrip import (OK, KnownFailureTable, Outcome, describe, diff, run,
3838
skip_without_runtime)
3939

4040
if TYPE_CHECKING:
@@ -549,14 +549,7 @@ class FieldRoundTripTests(KnownFailureTable, unittest.TestCase):
549549

550550
STATUSES = ('OK', 'PACK', 'UNPACK', 'VALUE', 'REPACK', 'WIRE', 'DICT', 'REJECTED', 'TIMEOUT')
551551

552-
KNOWN_FAILURES = (
553-
Gap(1488, 'NumberField with bit_length narrower than its octets masks the unpacked value '
554-
'to bit_length bits, so set high bits are dropped and pack writes them as '
555-
'zeros (pcapkit/corekit/fields/numbers.py:318, :322). Shipped use: vlan.TCI '
556-
'(pcapkit/protocols/schema/link/vlan.py:47-51), which no protocol parses with',
557-
'WIRE', '', ('NumberField/bit-length-12/wire/ffff', 'NumberField/bit-length-12/wire/f000',
558-
'EnumField/bit-length-3/wire/ff')),
559-
)
552+
KNOWN_FAILURES = ()
560553

561554
def setUp(self) -> None:
562555
skip_without_runtime(self)

0 commit comments

Comments
 (0)