From 78c99d6c1bf37a6589b901a214ac30b7ecdae8f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Furga=C5=82a?= <83299832+00200200@users.noreply.github.com> Date: Tue, 29 Sep 2026 13:24:49 +0200 Subject: [PATCH 1/4] =?UTF-8?q?=F0=9F=90=9B=20FIX:=20Don't=20crash=20on=20?= =?UTF-8?q?nested=20YAML=20dates=20in=20front=20matter=20(#1203)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit json.dumps of front-matter mappings/lists raised TypeError for nested date/datetime values. Stringify those with default=str, and catch TypeError so a YAML date used as a mapping key warns instead of aborting the build. --- CHANGELOG.md | 6 ++++ myst_parser/mdit_to_docutils/base.py | 7 +++-- tests/test_docutils.py | 42 ++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ff2d904f..c8c4d93e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## Unreleased + +### 🐛 Bug Fixes + +- 🐛 Don't crash when a YAML date/datetime is nested in front matter by + ## 5.1.0 - 2026-05-13 ### ✨ New Features diff --git a/myst_parser/mdit_to_docutils/base.py b/myst_parser/mdit_to_docutils/base.py index 68cfae72..fe54b5aa 100644 --- a/myst_parser/mdit_to_docutils/base.py +++ b/myst_parser/mdit_to_docutils/base.py @@ -1387,8 +1387,11 @@ def dict_to_fm_field_list( ) continue try: - value = json.dumps(value) - except (ValueError, RecursionError): + # Nested date/datetime values (allowed at top level) are + # not JSON-serializable; stringify them like scalars. + # TypeError also covers non-string keys such as a YAML date. + value = json.dumps(value, default=str) + except (ValueError, TypeError, RecursionError): # e.g. a self-referential structure via a YAML alias self.create_warning( f"Front matter field {key!r} could not be serialized", diff --git a/tests/test_docutils.py b/tests/test_docutils.py index 890badbc..43cb7baa 100644 --- a/tests/test_docutils.py +++ b/tests/test_docutils.py @@ -334,6 +334,48 @@ def test_topmatter_deeply_nested_yaml_warns(): assert "content" in doctree.pformat() +@pytest.mark.parametrize( + "yaml_block", + [ + "review:\n date: 2026-08-03", # date nested in a mapping + "reviewed: [2026-08-03]", # date nested in a sequence + "review:\n at: 2026-08-03T12:00:00", # datetime nested in a mapping + ], +) +def test_topmatter_nested_dates_do_not_crash(yaml_block): + """YAML dates nested in dicts/lists must not abort the build. + + Top-level ``date``/``datetime`` scalars skip JSON encoding, but a nested + date used to reach ``json.dumps`` and raise ``TypeError`` (#1203). + """ + stream = io.StringIO() + doctree = publish_doctree( + source=f"---\n{yaml_block}\n---\n\ncontent\n", + parser=Parser(), + settings_overrides={"warning_stream": stream}, + ) + assert "content" in doctree.pformat() + assert "2026-08-03" in doctree.astext() + assert "[myst.topmatter]" not in stream.getvalue() + + +def test_topmatter_date_mapping_key_warns(): + """A YAML date used as a mapping *key* cannot be JSON-serialized. + + ``json.dumps(..., default=str)`` does not convert keys, so this still + cannot be rendered as a field, but it must warn rather than crash. + """ + stream = io.StringIO() + doctree = publish_doctree( + source="---\nreview:\n 2026-08-03: done\n---\n\ncontent\n", + parser=Parser(), + settings_overrides={"warning_stream": stream}, + ) + assert "content" in doctree.pformat() + assert "could not be serialized" in stream.getvalue() + assert "[myst.topmatter]" in stream.getvalue() + + def test_topmatter_alias_expansion_bomb_warns(): """A YAML alias-expansion ("billion laughs") bomb warns, not hangs. From 7e6c99d47f57b9956c571dc61e2edaab68ecd61c Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Mon, 9 Nov 2026 20:29:40 +0100 Subject: [PATCH 2/4] =?UTF-8?q?=F0=9F=94=A7=20MAINTAIN:=20drop=20changelog?= =?UTF-8?q?=20entry=20(written=20at=20release)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c8c4d93e..ff2d904f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,5 @@ # Changelog -## Unreleased - -### 🐛 Bug Fixes - -- 🐛 Don't crash when a YAML date/datetime is nested in front matter by - ## 5.1.0 - 2026-05-13 ### ✨ New Features From 0f1961f600bf0a842b4a1b47d158bbcfa35ab00a Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Mon, 9 Nov 2026 20:32:44 +0100 Subject: [PATCH 3/4] =?UTF-8?q?=F0=9F=91=8C=20IMPROVE:=20render=20front=20?= =?UTF-8?q?matter=20dates=20as=20ISO,=20sets=20deterministically?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace `default=str` with `_fm_json_default` for front matter values that JSON has no type for: - a date or datetime nested in a mapping or list renders as ISO 8601 (`2026-08-03T12:00:00`); - a YAML `!!set` renders as a list sorted by each item's JSON text, so the output no longer changes with `PYTHONHASHSEED` (`str(set)` did), and a set of dates is in date order; - anything else (e.g. `!!binary`) still uses `str()`. Tests pin both, plus a mixed-type set, and check that a date used as a mapping key drops only that field. --- myst_parser/mdit_to_docutils/base.py | 28 ++++++++++-- tests/test_docutils.py | 65 ++++++++++++++++++++++++++-- 2 files changed, 86 insertions(+), 7 deletions(-) diff --git a/myst_parser/mdit_to_docutils/base.py b/myst_parser/mdit_to_docutils/base.py index 84423a9a..023e74cf 100644 --- a/myst_parser/mdit_to_docutils/base.py +++ b/myst_parser/mdit_to_docutils/base.py @@ -1409,10 +1409,10 @@ def dict_to_fm_field_list( ) continue try: - # Nested date/datetime values (allowed at top level) are - # not JSON-serializable; stringify them like scalars. + # Nested dates, sets, etc. are not JSON-serializable; + # see _fm_json_default for how each is rendered. # TypeError also covers non-string keys such as a YAML date. - value = json.dumps(value, default=str) + value = json.dumps(value, default=_fm_json_default) except (ValueError, TypeError, RecursionError): # e.g. a self-referential structure via a YAML alias self.create_warning( @@ -2049,6 +2049,28 @@ def render_substitution(self, token: SyntaxTreeNode, inline: bool) -> None: """Maximum number of items a front matter field may expand to when rendered.""" +def _fm_json_default(obj: Any) -> Any: + """Serialize a front matter value that JSON has no type for. + + Dates and datetimes (e.g. a YAML date nested in a mapping or list) + become ISO 8601 strings. + ``!!set`` values become a list sorted by each item's rendered JSON text + (a textual order, which for ISO dates is also chronological), + so the output does not depend on hash randomization. + Anything else (e.g. ``!!binary`` bytes) uses ``str()``. + """ + if isinstance(obj, date): # also datetime, a subclass of date + return obj.isoformat() + if isinstance(obj, set | frozenset): + return sorted(obj, key=_fm_json_text) + return str(obj) + + +def _fm_json_text(obj: Any) -> str: + """Return the JSON text a front matter value renders as.""" + return json.dumps(obj, default=_fm_json_default) + + def _expanded_length(value: Any, _memo: dict[int, int] | None = None) -> int: """Return the total number of items ``value`` expands to when serialized. diff --git a/tests/test_docutils.py b/tests/test_docutils.py index e04f9076..91796f0c 100644 --- a/tests/test_docutils.py +++ b/tests/test_docutils.py @@ -364,18 +364,75 @@ def test_topmatter_nested_dates_do_not_crash(yaml_block): def test_topmatter_date_mapping_key_warns(): """A YAML date used as a mapping *key* cannot be JSON-serialized. - ``json.dumps(..., default=str)`` does not convert keys, so this still - cannot be rendered as a field, but it must warn rather than crash. + ``json.dumps``'s ``default=`` hook is not called for keys, so this still + cannot be rendered as a field, but it must warn rather than crash, + and only that field is dropped. """ stream = io.StringIO() doctree = publish_doctree( - source="---\nreview:\n 2026-08-03: done\n---\n\ncontent\n", + source="---\nreview:\n 2026-08-03: done\nother: [2026-08-04]\n---\n\ncontent\n", parser=Parser(), settings_overrides={"warning_stream": stream}, ) assert "content" in doctree.pformat() - assert "could not be serialized" in stream.getvalue() + assert "Front matter field 'review' could not be serialized" in stream.getvalue() assert "[myst.topmatter]" in stream.getvalue() + assert _topmatter_fields(doctree) == {"other": '["2026-08-04"]'} + + +def _topmatter_fields(doctree: nodes.document) -> dict[str, str]: + """Return the rendered front matter fields, as ``{name: body text}``.""" + return { + field[0].astext(): field[1].astext() for field in doctree.findall(nodes.field) + } + + +@pytest.mark.parametrize( + ("yaml_block", "expected"), + [ + pytest.param( + "review:\n at: 2026-08-03T12:00:00", + '{"at": "2026-08-03T12:00:00"}', + id="nested-datetime-isoformat", + ), + pytest.param( + "review:\n at: 2026-08-03T12:00:00+02:00", + '{"at": "2026-08-03T12:00:00+02:00"}', + id="nested-tz-datetime-isoformat", + ), + pytest.param( + "tags: !!set {gamma, alpha, beta}", + '["alpha", "beta", "gamma"]', + id="set-sorted", + ), + pytest.param( + "review:\n d: !!set {2026-08-10, 2026-08-04, 2026-08-03}", + '{"d": ["2026-08-03", "2026-08-04", "2026-08-10"]}', + id="nested-set-of-dates", + ), + pytest.param( + "tags: !!set {1, b, 2026-08-03}", + '["2026-08-03", "b", 1]', + id="mixed-type-set", + ), + ], +) +def test_topmatter_json_rendering(yaml_block, expected): + """Values JSON has no type for render deterministically. + + Dates and datetimes nested in a mapping or list render as ISO 8601 strings, + and a YAML ``!!set`` renders as a sorted list, + so the output does not depend on hash randomization (``PYTHONHASHSEED``). + """ + stream = io.StringIO() + doctree = publish_doctree( + source=f"---\n{yaml_block}\n---\n\ncontent\n", + parser=Parser(), + settings_overrides={"warning_stream": stream}, + ) + key = yaml_block.split(":", 1)[0] + assert _topmatter_fields(doctree) == {key: expected} + assert "[myst.topmatter]" not in stream.getvalue() def test_topmatter_alias_expansion_bomb_warns(): From 3b3e698bde09445292cf16f9f246563ede31939f Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Mon, 9 Nov 2026 21:36:23 +0100 Subject: [PATCH 4/4] =?UTF-8?q?=F0=9F=90=9B=20FIX:=20count=20front=20matte?= =?UTF-8?q?r=20set=20items=20toward=20the=20size=20limit?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `!!set` values now render as lists, so the alias-expansion guard must measure their items too; otherwise a set repeated through YAML aliases stays under the limit and renders tens of MB from a few KB of source. --- myst_parser/mdit_to_docutils/base.py | 3 ++- tests/test_docutils.py | 27 +++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/myst_parser/mdit_to_docutils/base.py b/myst_parser/mdit_to_docutils/base.py index cdc7c3ce..a275cd71 100644 --- a/myst_parser/mdit_to_docutils/base.py +++ b/myst_parser/mdit_to_docutils/base.py @@ -2093,7 +2093,8 @@ def _expanded_length(value: Any, _memo: dict[int, int] | None = None) -> int: """ if isinstance(value, dict): children: Iterable[Any] = value.values() - elif isinstance(value, list | tuple): + elif isinstance(value, list | tuple | set | frozenset): + # sets are rendered as lists by _fm_json_default, so count their items too children = value else: return 1 diff --git a/tests/test_docutils.py b/tests/test_docutils.py index 42e3569e..809d4b38 100644 --- a/tests/test_docutils.py +++ b/tests/test_docutils.py @@ -488,6 +488,33 @@ def test_topmatter_alias_expansion_bomb_warns(): assert "content" in doctree.pformat() +def test_topmatter_set_alias_bomb_warns(): + """A ``!!set`` repeated through YAML aliases counts toward the size limit. + + Sets render as lists, so their items must be measured too; + otherwise a few KB of front matter renders to tens of MB. + Field ``c`` expands to about 20,000 items (rendered), + field ``d`` to about 200,000 (over the limit, so it warns). + """ + items = ", ".join(f"k{i}" for i in range(200)) + source = ( + f"---\na: &a !!set {{{items}}}\n" + f"b: &b [{', '.join(['*a'] * 10)}]\n" + f"c: &c [{', '.join(['*b'] * 10)}]\n" + f"d: [{', '.join(['*c'] * 10)}]\n---\n\ncontent\n" + ) + stream = io.StringIO() + doctree = publish_doctree( + source=source, + parser=Parser(), + settings_overrides={"warning_stream": stream}, + ) + assert "content" in doctree.pformat() + warnings = stream.getvalue() + assert "Front matter field 'd' is too large to render" in warnings + assert "Front matter field 'c'" not in warnings + + def test_heading_anchors_html_ids_disabled(): """``myst_heading_anchors_html_ids=False`` restores lookup-only slugs.""" settings: dict = {"myst_heading_anchors": 1, "doctitle_xform": False}