Skip to content

Commit 11cfd55

Browse files
authored
fix(dumpkit): drop redundant escape/date workarounds dictdumper 0.8.5 fixed (#2445)
1 parent 83c7b22 commit 11cfd55

8 files changed

Lines changed: 252 additions & 259 deletions

File tree

‎Pipfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ verify_ssl = true
66
[packages]
77
aenum = "*"
88
chardet = "*"
9-
dictdumper = "~=0.8.4"
9+
dictdumper = ">=0.8.5"
1010
emoji = "*"
1111
tbtrim = "~=0.3.1"
1212

‎pcapkit/dumpkit/common.py‎

Lines changed: 164 additions & 148 deletions
Large diffs are not rendered by default.

‎pyproject.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ classifiers = [
7474
"Typing :: Typed",
7575
]
7676
dependencies = [
77-
"dictdumper~=0.8.0", # for formatted output
77+
"dictdumper>=0.8.5", # for formatted output -- the floor fixing #772/#1448 upstream
7878
"chardet", # for bytes decode
7979
"aenum", # for const types
8080
"tbtrim>=0.2.1", # for refined exceptions

‎tests/dumpkit/test_common_unit.py‎

Lines changed: 57 additions & 96 deletions
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,6 @@ def __init__(self) -> None:
7777
#: because neither is special inside an XML text node.
7878
BYTES_KEY = bytes(range(0x20, 0x40))
7979
RAW_BYTES_KEY = """b' !"#$%&\\'()*+,-./0123456789:;<=>?'"""
80-
ESCAPED_BYTES_KEY = """b' !"#$%&amp;\\'()*+,-./0123456789:;&lt;=&gt;?'"""
8180

8281

8382
def plist_like_dumper() -> 'object':
@@ -226,123 +225,85 @@ def test_make_dumper_object_hook_conversions_and_fallbacks(self) -> None:
226225
dumper._append_fallback(123, file)
227226
self.assertEqual(file.getvalue(), "{'base': '123'}")
228227

229-
def test_make_dumper_escapes_strings_only_for_plist_like_output(self) -> None:
230-
"""GitHub issue #772: no entity escaping anywhere in :mod:`dictdumper`.
228+
def test_make_dumper_object_hook_leaves_strings_and_enum_renderings_unescaped(self) -> None:
229+
"""GitHub issue #772: escaping is now the writer's own job, not this hook's.
231230
232231
``dictdumper.plist.PLIST`` (which is also what ``'xml'`` maps to --
233-
see ``Extractor.__output__``) interpolates every ``<string>``/``<key>``
234-
value straight into its markup, so a value containing ``&``, ``<`` or
235-
``>`` breaks the document it is embedded in. Filed upstream as
236-
JarryShaw/DictDumper#125. ``json``/``tree``/``text`` have no such
237-
defect and all three characters are legal there, so the escaping in
238-
:func:`~pcapkit.dumpkit.common.make_dumper` must fire for a
239-
PLIST-rooted ``output`` and stay off for anything else -- a plain
240-
:class:`str`, an enum rendered by :func:`~pcapkit.dumpkit.common.render_enum`,
241-
and a :class:`~pcapkit.corekit.multidict.MultiDict` key built from one,
242-
since that key is written by the PLIST writer directly and never
243-
passed back through this hook the way a value is.
232+
see ``Extractor.__output__``) used to interpolate every
233+
``<string>``/``<key>`` value straight into its markup with no entity
234+
escaping at all, so a value containing ``&``, ``<`` or ``>`` broke the
235+
document it was embedded in. Filed upstream as JarryShaw/DictDumper#125
236+
and fixed there in ``dictdumper`` 0.8.5 (DictDumper#123):
237+
:func:`dictdumper.plist.encode_xml` now escapes a value itself. So
238+
:meth:`~pcapkit.dumpkit.common.make_dumper.DictDumper.object_hook` no
239+
longer escapes anything on the way out, for *any* ``output`` -- a plain
240+
:class:`str`, an enum rendered by
241+
:func:`~pcapkit.dumpkit.common.render_enum`, or a
242+
:class:`~pcapkit.corekit.multidict.MultiDict` key built from one, since
243+
that key is written by the writer directly and never passed back
244+
through this hook the way a value is.
244245
245246
"""
246247
from pcapkit.corekit.multidict import MultiDict
247248
from pcapkit.dumpkit.common import make_dumper
248249

249-
plist_dumper = plist_like_dumper()
250-
self.assertEqual(plist_dumper.object_hook('a & b <c>'), 'a &amp; b &lt;c&gt;')
251-
252250
unknown = enum.IntEnum('<unknown>', {'<unassigned>': 1})
253251
member = getattr(unknown, '<unassigned>')
254-
self.assertEqual(plist_dumper.object_hook(member),
255-
'&lt;unknown&gt;::&lt;unassigned&gt; [1]')
256252

257-
multidict = MultiDict()
258-
multidict.add(member, 'value')
259-
converted = plist_dumper.object_hook(multidict)
260-
self.assertEqual(converted['&lt;unknown&gt;::&lt;unassigned&gt; [1]'], ['value'])
253+
for name, dumper in (('plist-like', plist_like_dumper()), ('identity', make_dumper(IdentityDumper)())):
254+
with self.subTest(dumper=name):
255+
self.assertEqual(dumper.object_hook('a & b <c>'), 'a & b <c>')
256+
self.assertEqual(dumper.object_hook(member), '<unknown>::<unassigned> [1]')
261257

262-
plain_dumper = make_dumper(IdentityDumper)()
263-
self.assertEqual(plain_dumper.object_hook('a & b <c>'), 'a & b <c>')
264-
self.assertEqual(plain_dumper.object_hook(member), '<unknown>::<unassigned> [1]')
258+
multidict = MultiDict()
259+
multidict.add(member, 'value')
260+
converted = dumper.object_hook(multidict)
261+
self.assertEqual(converted['<unknown>::<unassigned> [1]'], ['value'])
265262

266-
def test_make_dumper_escapes_mapping_keys_for_plist_like_output(self) -> None:
263+
def test_make_dumper_never_rebuilds_or_escapes_a_mapping(self) -> None:
267264
"""GitHub issue #772, the half the hook above cannot reach.
268265
269266
``PLIST._append_dict`` in :mod:`dictdumper.plist` writes a key as
270-
``'<key>{item}</key>'.format(item=item)`` and then calls ``_encode_value``
271-
on the *value*, never on the key -- so :meth:`object_hook`
272-
is handed every value the writer interpolates and no key at all, and the
273-
escaping proven above cannot reach one. Both branches that build a
274-
mapping therefore escape their own keys:
275-
:class:`~pcapkit.corekit.multidict.MultiDict`, where without the fix
276-
only an enum-derived key is escaped, and a plain :class:`dict`, where
277-
nothing is.
278-
279-
The :class:`bytes` key is :file:`examples/captures/test.pcapng`'s own,
280-
and it is the reason that fixture's ``plist`` report did not parse:
281-
``&``, ``<`` and ``>`` all inside one key. It is rendered with
282-
:func:`format`, i.e. the conversion the writer's interpolation already
283-
applies to it, so the ``<key>`` text is unchanged apart from the
284-
escaping -- which is what the :data:`RAW_BYTES_KEY` assertion pins,
285-
since an expected escaping is only as good as the rendering it is
286-
derived from.
267+
``'<key>{item}</key>'.format(item=item)`` and
268+
``JSON._append_object`` into ``'"{item}": '``, then call
269+
``_encode_value`` on the *value*, never on the key -- so
270+
:meth:`object_hook` is handed every value the writer interpolates and
271+
no key at all. Since ``dictdumper`` 0.8.5 (DictDumper#123) both writers
272+
escape a key themselves, exactly as they escape the value beside it, so
273+
neither a plain :class:`dict` nor the single-key mappings a
274+
:class:`~pcapkit.corekit.multidict.MultiDict` or
275+
:class:`~pcapkit.corekit.multidict.OrderedMultiDict` becomes is rebuilt
276+
here any more: the writer is handed the caller's own mapping, with the
277+
caller's own key objects in it, exactly as given -- for *any*
278+
``output``, not only a non-PLIST one.
279+
280+
The :class:`bytes` key is :file:`examples/captures/test.pcapng`'s own
281+
-- it used to be the reason that fixture's ``plist`` report did not
282+
parse, since its repr carries ``&``, ``<`` and ``>``.
287283
288284
"""
289285
from pcapkit.corekit.multidict import MultiDict, OrderedMultiDict
286+
from pcapkit.dumpkit.common import make_dumper
290287

291-
plist_dumper = plist_like_dumper()
292-
293-
# A plain dict: str keys, and the bytes key of the fixture.
294-
self.assertEqual(format(BYTES_KEY, ''), RAW_BYTES_KEY)
295-
converted = plist_dumper.object_hook({'a & b <c>': 'value', BYTES_KEY: 'secret'})
296-
self.assertEqual(list(converted), ['a &amp; b &lt;c&gt;', ESCAPED_BYTES_KEY])
297-
self.assertEqual(converted['a &amp; b &lt;c&gt;'], 'value')
298-
self.assertEqual(converted[ESCAPED_BYTES_KEY], 'secret')
299-
300-
# An OrderedMultiDict: the bytes key of the fixture arrives through here
301-
# -- its TLS key log entries are an OrderedMultiDict. Each entry becomes
302-
# a single-key mapping, whose key is escaped when the writer hands that
303-
# mapping back to the hook.
304-
multidict = OrderedMultiDict()
305-
multidict.add('a & b <c>', 'value')
306-
multidict.add(BYTES_KEY, 'secret')
307-
converted = [plist_dumper.object_hook(entry) for entry in plist_dumper.object_hook(multidict)]
308-
self.assertEqual(converted, [{'a &amp; b &lt;c&gt;': 'value'}, {ESCAPED_BYTES_KEY: 'secret'}])
309-
310-
# An enum-derived key is escaped exactly once: this branch renders it
311-
# with ``render_enum`` and escapes the result through the one shared
312-
# ``escape_key`` helper, so escaping it twice is the mistake to rule out.
313288
unknown = enum.IntEnum('<unknown>', {'<unassigned>': 1})
314289
member = getattr(unknown, '<unassigned>')
315-
multidict = MultiDict()
316-
multidict.add(member, 'value')
317-
converted = plist_dumper.object_hook(multidict)
318-
self.assertEqual(list(converted), ['&lt;unknown&gt;::&lt;unassigned&gt; [1]'])
319-
self.assertNotIn('&amp;', ''.join(converted))
320-
321-
def test_make_dumper_hands_other_output_the_mapping_it_was_given(self) -> None:
322-
"""``json``, ``tree`` and ``text`` take all three characters literally.
323-
324-
So no key is escaped for them, and -- the stronger statement, and the
325-
one that keeps this branch free of any risk of double-escaping -- a
326-
plain :class:`dict` is not even rebuilt: the writer is handed the
327-
caller's own mapping, with the caller's own key objects in it, exactly
328-
as given. A :class:`bytes` key stays :class:`bytes` there,
329-
which is what ``dictdumper``'s ``json`` writer then breaks on, for
330-
reasons of its own.
331-
332-
"""
333-
from pcapkit.corekit.multidict import OrderedMultiDict
334-
from pcapkit.dumpkit.common import make_dumper
335-
336-
plain_dumper = make_dumper(IdentityDumper)()
337-
338-
mapping = {'a & b <c>': 'value', BYTES_KEY: 'secret'}
339-
self.assertIs(plain_dumper.object_hook(mapping), mapping)
340290

341-
multidict = OrderedMultiDict()
342-
multidict.add('a & b <c>', 'value')
343-
multidict.add(BYTES_KEY, 'secret')
344-
converted = plain_dumper.object_hook(multidict)
345-
self.assertEqual(converted, [{'a & b <c>': 'value'}, {BYTES_KEY: 'secret'}])
291+
for name, dumper in (('plist-like', plist_like_dumper()), ('identity', make_dumper(IdentityDumper)())):
292+
with self.subTest(dumper=name):
293+
mapping = {'a & b <c>': 'value', BYTES_KEY: 'secret'}
294+
self.assertIs(dumper.object_hook(mapping), mapping)
295+
self.assertEqual(format(BYTES_KEY, ''), RAW_BYTES_KEY)
296+
297+
multidict = OrderedMultiDict()
298+
multidict.add('a & b <c>', 'value')
299+
multidict.add(BYTES_KEY, 'secret')
300+
converted = dumper.object_hook(multidict)
301+
self.assertEqual(converted, [{'a & b <c>': 'value'}, {BYTES_KEY: 'secret'}])
302+
303+
key_multidict = MultiDict()
304+
key_multidict.add(member, 'value')
305+
converted_keys = dumper.object_hook(key_multidict)
306+
self.assertEqual(list(converted_keys), ['<unknown>::<unassigned> [1]'])
346307

347308
def test_an_unassigned_port_dumps_its_addon_keys_in_the_declared_order(self) -> None:
348309
"""GitHub issue #575's fallback must not reorder what it renders.

‎tests/dumpkit/test_dumper_roundtrip_unit.py‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -173,6 +173,19 @@ def test_json_writes_astral_characters_as_surrogate_pairs(self) -> None:
173173
self.assertIn(b'"a\\u007f\\u000b\\u00e9\\"\\\\\\n"', data)
174174
self.assertEqual(json.loads(data)['x'], value)
175175

176+
def test_json_writes_astral_characters_in_keys_as_surrogate_pairs(self) -> None:
177+
"""#1261: a mapping KEY above ``U+FFFF`` round-trips, not just a value.
178+
179+
``dictdumper.json.JSON._append_object`` escapes a key through the
180+
module-level :func:`dictdumper.json.encode_string` directly, bypassing
181+
the overridable :meth:`_append_string` the previous test covers, so the
182+
key side needed its own, separate fix.
183+
"""
184+
value = {'astral\U0001F600key': 'v', 'plain': 1}
185+
data = self.dump('JSON', value)
186+
self.assertIn(b'"astral\\ud83d\\ude00key"', data)
187+
self.assertEqual(json.loads(data)['x'], value)
188+
176189
def test_plist_writes_strings_xml_cannot_spell_as_data(self) -> None:
177190
"""#1262: a control character does not make the document unparseable."""
178191
value = {'c': 'a\x01b\x1bc', 'ok': 'tab\there & <there>'}

‎tests/integration/test_output_formats.py‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -117,10 +117,11 @@ class PlistRoundTripTests(EndToEndTestCase):
117117
def test_plist_report_round_trips_through_plistlib(self) -> None:
118118
"""A ``plist`` report is readable by :func:`plistlib.load` (GitHub issue #1448).
119119
120-
``dictdumper`` writes a ``<date>`` with fractional seconds, which
121-
:mod:`plistlib` rejects; :mod:`pcapkit.dumpkit` writes it to whole
122-
seconds instead, and the exact value stays in ``time_epoch``. The date
123-
is compared as naive UTC, since ``aware_datetime`` is new in Python 3.13.
120+
``dictdumper`` used to write a ``<date>`` with fractional seconds,
121+
which :mod:`plistlib` rejects; fixed upstream in ``dictdumper`` 0.8.5
122+
(DictDumper#124), which now writes whole seconds itself. The exact
123+
value stays in ``time_epoch`` regardless. The date is compared as
124+
naive UTC, since ``aware_datetime`` is new in Python 3.13.
124125
125126
"""
126127
import datetime

‎tests/integration/test_pcapng_end_to_end.py‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -171,9 +171,12 @@ class PcapngUnescapedKeyTests(EndToEndTestCase):
171171
"""The ``json`` report of :file:`test.pcapng`, whose mapping keys need escaping.
172172
173173
The fixture's decryption secrets block keys its TLS key log entries by a raw
174-
:class:`bytes` client random, and :mod:`dictdumper` writes a mapping key
175-
into a ``json`` report without escaping it. The report parses only because
176-
:mod:`pcapkit.dumpkit` escapes the keys itself (GitHub issue #1152).
174+
:class:`bytes` client random. ``dictdumper.json.JSON._append_object`` used
175+
to write a mapping key into a ``json`` report without escaping it (GitHub
176+
issue #1152), and the report parsed only because :mod:`pcapkit.dumpkit`
177+
escaped the key itself. Fixed upstream in ``dictdumper`` 0.8.5
178+
(DictDumper#123), so the report now parses on the writer's own escaping;
179+
this test pins that the behaviour survives either way.
177180
178181
"""
179182

@@ -184,12 +187,9 @@ def test_json_report_of_a_decryption_secrets_block_parses(self) -> None:
184187
<pcapkit.protocols.schema.misc.pcapng.TLSKeyLog.post_process>` keys the
185188
entries by ``bytes.fromhex(random)``, so the key the writer formats is a
186189
:class:`bytes` repr carrying ``"`` and ``\\``:
187-
``b' !"#$%&\\'()*+,-./0123456789:;<=>?'``.
188-
:meth:`dictdumper.json.JSON._append_object` writes a key into
189-
``'"{item}": '`` raw, so that ``"`` would end the JSON string early --
190-
still open upstream as JarryShaw/DictDumper#121. The ``escape_key``
191-
helper of :func:`~pcapkit.dumpkit.common.make_dumper` escapes it with
192-
:func:`json.dumps` first; without that, this test fails.
190+
``b' !"#$%&\\'()*+,-./0123456789:;<=>?'``, which a key written into
191+
``'"{item}": '`` raw -- as ``dictdumper`` 0.8.5 no longer does -- would
192+
close the JSON string early.
193193
194194
"""
195195
extractor = self.extract(fin=sample_path('test.pcapng'), fout=self.out('report'),

‎tests/project/test_rfc_constants_one_home_1739_unit.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -336,6 +336,8 @@
336336
'the IP version an IPv6-in-IP tunnel carries, not the Next Header offset',
337337
('pcapkit/dumpkit/common.py', 'ord(char) > 65535'):
338338
'the last code point of the Basic Multilingual Plane, not an IP length',
339+
('pcapkit/dumpkit/common.py', 'codepoint > 65535'):
340+
'above the Basic Multilingual Plane, needs a UTF-16 surrogate pair, not an IP length',
339341
}
340342

341343
#: Sweep hits of a homed value in a module whose own protocol owns no constant

0 commit comments

Comments
 (0)