diff --git a/myst_parser/mdit_to_docutils/base.py b/myst_parser/mdit_to_docutils/base.py index 2c7ff32b..a275cd71 100644 --- a/myst_parser/mdit_to_docutils/base.py +++ b/myst_parser/mdit_to_docutils/base.py @@ -1421,8 +1421,11 @@ def dict_to_fm_field_list( ) continue try: - value = json.dumps(value) - except (ValueError, RecursionError): + # Nested dates, sets, etc. are not JSON-serializable; + # see _fm_json_default for how each is rendered. + # TypeError also covers non-string keys such as a YAML date. + value = json.dumps(value, default=_fm_json_default) + except (ValueError, TypeError, RecursionError): # e.g. a self-referential structure via a YAML alias self.create_warning( f"Front matter field {key!r} could not be serialized", @@ -2058,6 +2061,28 @@ def render_substitution(self, token: SyntaxTreeNode, inline: bool) -> None: """Maximum number of items a front matter field may expand to when rendered.""" +def _fm_json_default(obj: Any) -> Any: + """Serialize a front matter value that JSON has no type for. + + Dates and datetimes (e.g. a YAML date nested in a mapping or list) + become ISO 8601 strings. + ``!!set`` values become a list sorted by each item's rendered JSON text + (a textual order, which for ISO dates is also chronological), + so the output does not depend on hash randomization. + Anything else (e.g. ``!!binary`` bytes) uses ``str()``. + """ + if isinstance(obj, date): # also datetime, a subclass of date + return obj.isoformat() + if isinstance(obj, set | frozenset): + return sorted(obj, key=_fm_json_text) + return str(obj) + + +def _fm_json_text(obj: Any) -> str: + """Return the JSON text a front matter value renders as.""" + return json.dumps(obj, default=_fm_json_default) + + def _expanded_length(value: Any, _memo: dict[int, int] | None = None) -> int: """Return the total number of items ``value`` expands to when serialized. @@ -2068,7 +2093,8 @@ def _expanded_length(value: Any, _memo: dict[int, int] | None = None) -> int: """ if isinstance(value, dict): children: Iterable[Any] = value.values() - elif isinstance(value, list | tuple): + elif isinstance(value, list | tuple | set | frozenset): + # sets are rendered as lists by _fm_json_default, so count their items too children = value else: return 1 diff --git a/tests/test_docutils.py b/tests/test_docutils.py index 7fd7db95..809d4b38 100644 --- a/tests/test_docutils.py +++ b/tests/test_docutils.py @@ -367,6 +367,105 @@ def test_topmatter_deeply_nested_yaml_warns(): assert "content" in doctree.pformat() +@pytest.mark.parametrize( + "yaml_block", + [ + "review:\n date: 2026-08-03", # date nested in a mapping + "reviewed: [2026-08-03]", # date nested in a sequence + "review:\n at: 2026-08-03T12:00:00", # datetime nested in a mapping + ], +) +def test_topmatter_nested_dates_do_not_crash(yaml_block): + """YAML dates nested in dicts/lists must not abort the build. + + Top-level ``date``/``datetime`` scalars skip JSON encoding, but a nested + date used to reach ``json.dumps`` and raise ``TypeError`` (#1203). + """ + stream = io.StringIO() + doctree = publish_doctree( + source=f"---\n{yaml_block}\n---\n\ncontent\n", + parser=Parser(), + settings_overrides={"warning_stream": stream}, + ) + assert "content" in doctree.pformat() + assert "2026-08-03" in doctree.astext() + assert "[myst.topmatter]" not in stream.getvalue() + + +def test_topmatter_date_mapping_key_warns(): + """A YAML date used as a mapping *key* cannot be JSON-serialized. + + ``json.dumps``'s ``default=`` hook is not called for keys, so this still + cannot be rendered as a field, but it must warn rather than crash, + and only that field is dropped. + """ + stream = io.StringIO() + doctree = publish_doctree( + source="---\nreview:\n 2026-08-03: done\nother: [2026-08-04]\n---\n\ncontent\n", + parser=Parser(), + settings_overrides={"warning_stream": stream}, + ) + assert "content" in doctree.pformat() + assert "Front matter field 'review' could not be serialized" in stream.getvalue() + assert "[myst.topmatter]" in stream.getvalue() + assert _topmatter_fields(doctree) == {"other": '["2026-08-04"]'} + + +def _topmatter_fields(doctree: nodes.document) -> dict[str, str]: + """Return the rendered front matter fields, as ``{name: body text}``.""" + return { + field[0].astext(): field[1].astext() for field in doctree.findall(nodes.field) + } + + +@pytest.mark.parametrize( + ("yaml_block", "expected"), + [ + pytest.param( + "review:\n at: 2026-08-03T12:00:00", + '{"at": "2026-08-03T12:00:00"}', + id="nested-datetime-isoformat", + ), + pytest.param( + "review:\n at: 2026-08-03T12:00:00+02:00", + '{"at": "2026-08-03T12:00:00+02:00"}', + id="nested-tz-datetime-isoformat", + ), + pytest.param( + "tags: !!set {gamma, alpha, beta}", + '["alpha", "beta", "gamma"]', + id="set-sorted", + ), + pytest.param( + "review:\n d: !!set {2026-08-10, 2026-08-04, 2026-08-03}", + '{"d": ["2026-08-03", "2026-08-04", "2026-08-10"]}', + id="nested-set-of-dates", + ), + pytest.param( + "tags: !!set {1, b, 2026-08-03}", + '["2026-08-03", "b", 1]', + id="mixed-type-set", + ), + ], +) +def test_topmatter_json_rendering(yaml_block, expected): + """Values JSON has no type for render deterministically. + + Dates and datetimes nested in a mapping or list render as ISO 8601 strings, + and a YAML ``!!set`` renders as a sorted list, + so the output does not depend on hash randomization (``PYTHONHASHSEED``). + """ + stream = io.StringIO() + doctree = publish_doctree( + source=f"---\n{yaml_block}\n---\n\ncontent\n", + parser=Parser(), + settings_overrides={"warning_stream": stream}, + ) + key = yaml_block.split(":", 1)[0] + assert _topmatter_fields(doctree) == {key: expected} + assert "[myst.topmatter]" not in stream.getvalue() + + def test_topmatter_alias_expansion_bomb_warns(): """A YAML alias-expansion ("billion laughs") bomb warns, not hangs. @@ -389,6 +488,33 @@ def test_topmatter_alias_expansion_bomb_warns(): assert "content" in doctree.pformat() +def test_topmatter_set_alias_bomb_warns(): + """A ``!!set`` repeated through YAML aliases counts toward the size limit. + + Sets render as lists, so their items must be measured too; + otherwise a few KB of front matter renders to tens of MB. + Field ``c`` expands to about 20,000 items (rendered), + field ``d`` to about 200,000 (over the limit, so it warns). + """ + items = ", ".join(f"k{i}" for i in range(200)) + source = ( + f"---\na: &a !!set {{{items}}}\n" + f"b: &b [{', '.join(['*a'] * 10)}]\n" + f"c: &c [{', '.join(['*b'] * 10)}]\n" + f"d: [{', '.join(['*c'] * 10)}]\n---\n\ncontent\n" + ) + stream = io.StringIO() + doctree = publish_doctree( + source=source, + parser=Parser(), + settings_overrides={"warning_stream": stream}, + ) + assert "content" in doctree.pformat() + warnings = stream.getvalue() + assert "Front matter field 'd' is too large to render" in warnings + assert "Front matter field 'c'" not in warnings + + def test_heading_anchors_html_ids_disabled(): """``myst_heading_anchors_html_ids=False`` restores lookup-only slugs.""" settings: dict = {"myst_heading_anchors": 1, "doctitle_xform": False}