From 29c59fd44d5159481515309c820ed540e567b29a Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Tue, 10 Nov 2026 12:57:27 +0100 Subject: [PATCH 1/6] =?UTF-8?q?=F0=9F=90=9B=20FIX:=20section=5Fref=20no=20?= =?UTF-8?q?longer=20raises=20on=20very=20long=20digit=20runs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `§` followed by more than 4300 digits made `int()` raise ValueError (Python's integer string conversion limit), aborting the whole render. A level longer than nine digits now makes the rule decline, so the text renders literally and no shorter prefix is captured. Nine digits keeps every number below 10**9 (a signed 32-bit int) and far below the conversion limit, which cannot be configured below 640. --- mdit_py_plugins/section_ref/index.py | 25 ++++++++-- tests/fixtures/section_ref.md | 28 +++++++++++ tests/test_section_ref.py | 69 ++++++++++++++++++++++++++++ 3 files changed, 118 insertions(+), 4 deletions(-) diff --git a/mdit_py_plugins/section_ref/index.py b/mdit_py_plugins/section_ref/index.py index 5069123..f8ed7e3 100644 --- a/mdit_py_plugins/section_ref/index.py +++ b/mdit_py_plugins/section_ref/index.py @@ -29,6 +29,14 @@ # ASCII digits only (``[0-9]``, never ``\d`` which also matches unicode digits). _SECTION_REF_PATTERN = re.compile(r"§([0-9]+(?:\.[0-9]+)*)") +# Longest dotted part (in written digits, leading zeros included) that is still +# read as a section number. Nine digits keeps every number below 10**9, so it +# fits a signed 32-bit integer for downstream consumers, and keeps ``int()`` far +# below Python's integer string conversion limit (4300 digits by default, and +# never configurable below 640), which would otherwise raise ``ValueError`` and +# abort the whole render. No real document has a billion sections. +_MAX_PART_DIGITS = 9 + def section_ref_plugin(md: MarkdownIt) -> None: """Parse section-sign references such as ``§1``, ``§1.1`` and ``§2.3.4``. @@ -38,9 +46,12 @@ def section_ref_plugin(md: MarkdownIt) -> None: A trailing dot is not consumed, so ``see §1.`` captures ``§1``. A ``§`` directly followed by an ASCII letter (e.g. ``§1a``) is *not* a reference, but a following non-ASCII letter does not block a match, so ``见§3章`` - still captures ``§3``. Following CommonMark backslash-escape behaviour, - ``\\§1`` stays literal text, while ``\\\\§1`` renders a literal backslash - followed by a live reference. + still captures ``§3``. A reference with any level longer than nine + digits (e.g. ``§1.1234567890``) is not a reference either: the whole + text stays literal, rather than a shorter prefix being captured. + Following CommonMark backslash-escape behaviour, ``\\§1`` stays literal + text, while ``\\\\§1`` renders a literal backslash followed by a live + reference. Each reference becomes a ``section_ref`` token, with the matched text (e.g. ``"§1.1"``) as its ``content``, ``markup`` set to ``§`` and @@ -76,11 +87,17 @@ def _section_ref_rule(state: StateInline, silent: bool) -> bool: if end < state.posMax and state.src[end].isascii() and state.src[end].isalpha(): return False + # reject an over-long level outright, so hostile input can never raise in + # ``int()``; the whole text stays literal (no prefix match splits a number) + parts = match.group(1).split(".") + if any(len(part) > _MAX_PART_DIGITS for part in parts): + return False + if not silent: token = state.push("section_ref", "", 0) token.content = match.group(0) token.markup = "§" - token.meta = {"numbers": [int(part) for part in match.group(1).split(".")]} + token.meta = {"numbers": [int(part) for part in parts]} state.pos = end return True diff --git a/tests/fixtures/section_ref.md b/tests/fixtures/section_ref.md index f870d2a..2a1973f 100644 --- a/tests/fixtures/section_ref.md +++ b/tests/fixtures/section_ref.md @@ -149,6 +149,34 @@ Leading zeros preserved in content (meta numbers are normalized):

§01.02

. +Level at the nine-digit bound (captured): +. +§123456789 and §1.999999999 +. +

§123456789 and §1.999999999

+. + +Level one past the nine-digit bound (not captured): +. +§1234567890 and §1.1234567890 +. +

§1234567890 and §1.1234567890

+. + +Over-long middle level (not captured, no partial prefix): +. +§1.1234567890.2 then §3 +. +

§1.1234567890.2 then §3

+. + +Leading zeros count towards the bound (not captured): +. +§0000000001 +. +

§0000000001

+. + In a heading: . ## About §2 diff --git a/tests/test_section_ref.py b/tests/test_section_ref.py index 7b4d804..02a604d 100644 --- a/tests/test_section_ref.py +++ b/tests/test_section_ref.py @@ -1,4 +1,5 @@ from pathlib import Path +import time from types import SimpleNamespace from markdown_it import MarkdownIt @@ -126,3 +127,71 @@ def test_all(line, title, input, expected): text = md.render(input) print(text) assert text.rstrip() == expected.rstrip() + + +def test_huge_digit_run_renders_as_text(): + """``§`` + 5000 digits must not raise (``int()`` refuses > 4300 digits). + + The over-long level makes the rule decline, so the text renders literally. + """ + md = MarkdownIt().use(section_ref_plugin) + digits = "1" * 5000 + assert md.render(f"see §{digits}") == f"

see §{digits}

\n" + tokens = md.parseInline(f"see §{digits}")[0].children + assert tokens is not None + assert [t.type for t in tokens] == ["text"] + + +@pytest.mark.parametrize( + "src,numbers", + [ + ("§999999999", [999999999]), + ("§1.999999999.2", [1, 999999999, 2]), + ("§000000001", [1]), + ], +) +def test_numbers_at_bound(src, numbers): + """A level of exactly nine digits is still a reference, with its int value.""" + md = MarkdownIt().use(section_ref_plugin) + tokens = md.parseInline(src)[0].children + assert tokens is not None + assert [t.type for t in tokens] == ["section_ref"] + assert tokens[0].content == src + assert tokens[0].meta == {"numbers": numbers} + + +@pytest.mark.parametrize( + "src", + ["§1234567890", "§1.1234567890", "§1234567890.1", "§1.2.1234567890.3"], +) +def test_level_past_bound_declines(src): + """Any level past nine digits makes the whole text literal (no prefix match).""" + md = MarkdownIt().use(section_ref_plugin) + tokens = md.parseInline(src)[0].children + assert tokens is not None + assert [t.type for t in tokens] == ["text"] + assert tokens[0].content == src + + +@pytest.mark.timeout(10) +@pytest.mark.parametrize( + "src", + [ + "§" + "1" * 200_000, + "§" + "1." * 100_000 + "1" * 10, + ("§" + "1" * 5000 + " ") * 40, + "§1 " * 20_000, + ], + ids=["one-run", "dotted-run", "many-runs", "many-refs"], +) +def test_long_digit_runs_stay_fast(src): + """Long digit runs are scanned in linear time, with no backtracking blow-up. + + A quadratic scan of these inputs (up to 200k characters) would take far longer than + the budget, so staying under it shows the rule is linear in practice. + """ + md = MarkdownIt().use(section_ref_plugin) + start = time.perf_counter() + md.render(src) + elapsed = time.perf_counter() - start + assert elapsed < 1.0, f"render took {elapsed:.3f}s for {len(src)} characters" From 47a241cd422e1e4c1904e8d3e9e774fd732a0746 Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Tue, 10 Nov 2026 13:11:02 +0100 Subject: [PATCH 2/6] =?UTF-8?q?=F0=9F=A7=AA=20TEST:=20section=5Fref=20timi?= =?UTF-8?q?ng=20test=20robust=20on=20PyPy=20and=20under=20coverage?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drop the many-refs timing case: it goes through the valid-reference path this change does not touch, and on PyPy it mostly measures markdown-it-py's renderer, whose string concatenation is quadratic there (it failed the 1s budget, and took about 3s with --cov). Raise the budget to 3s, which still catches a quadratic scan (tens of seconds at 200k characters), and drop the redundant timeout marker (pyproject.toml already sets timeout = 10). Say in the docstring that leading zeros count towards the nine digits. --- mdit_py_plugins/section_ref/index.py | 2 ++ tests/test_section_ref.py | 11 +++++------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/mdit_py_plugins/section_ref/index.py b/mdit_py_plugins/section_ref/index.py index f8ed7e3..857a7ac 100644 --- a/mdit_py_plugins/section_ref/index.py +++ b/mdit_py_plugins/section_ref/index.py @@ -49,6 +49,8 @@ def section_ref_plugin(md: MarkdownIt) -> None: still captures ``§3``. A reference with any level longer than nine digits (e.g. ``§1.1234567890``) is not a reference either: the whole text stays literal, rather than a shorter prefix being captured. + Leading zeros count towards the nine digits, so ``§0000000001`` stays + literal too. Following CommonMark backslash-escape behaviour, ``\\§1`` stays literal text, while ``\\\\§1`` renders a literal backslash followed by a live reference. diff --git a/tests/test_section_ref.py b/tests/test_section_ref.py index 02a604d..1ba6339 100644 --- a/tests/test_section_ref.py +++ b/tests/test_section_ref.py @@ -173,25 +173,24 @@ def test_level_past_bound_declines(src): assert tokens[0].content == src -@pytest.mark.timeout(10) @pytest.mark.parametrize( "src", [ "§" + "1" * 200_000, "§" + "1." * 100_000 + "1" * 10, ("§" + "1" * 5000 + " ") * 40, - "§1 " * 20_000, ], - ids=["one-run", "dotted-run", "many-runs", "many-refs"], + ids=["one-run", "dotted-run", "many-runs"], ) def test_long_digit_runs_stay_fast(src): """Long digit runs are scanned in linear time, with no backtracking blow-up. - A quadratic scan of these inputs (up to 200k characters) would take far longer than - the budget, so staying under it shows the rule is linear in practice. + A quadratic scan of these inputs (up to 200k characters) would take tens of + seconds, far longer than the budget, so staying under it shows the rule is + linear in practice. The budget leaves room for slow CI runners and coverage. """ md = MarkdownIt().use(section_ref_plugin) start = time.perf_counter() md.render(src) elapsed = time.perf_counter() - start - assert elapsed < 1.0, f"render took {elapsed:.3f}s for {len(src)} characters" + assert elapsed < 3.0, f"render took {elapsed:.3f}s for {len(src)} characters" From 25c8d3189beed4b30aa0847e78547fd4682f0723 Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Tue, 10 Nov 2026 13:19:27 +0100 Subject: [PATCH 3/6] =?UTF-8?q?=F0=9F=93=9A=20DOCS:=20say=20what=20the=20s?= =?UTF-8?q?ection=5Fref=20timing=20test=20catches?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The budget catches a scan that does Python-level work per character, or a backtracking regex. A quadratic scan built from string slices can still pass it, so the docstring no longer claims every quadratic scan would take tens of seconds. --- tests/test_section_ref.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_section_ref.py b/tests/test_section_ref.py index 1ba6339..d83355b 100644 --- a/tests/test_section_ref.py +++ b/tests/test_section_ref.py @@ -185,9 +185,10 @@ def test_level_past_bound_declines(src): def test_long_digit_runs_stay_fast(src): """Long digit runs are scanned in linear time, with no backtracking blow-up. - A quadratic scan of these inputs (up to 200k characters) would take tens of - seconds, far longer than the budget, so staying under it shows the rule is - linear in practice. The budget leaves room for slow CI runners and coverage. + A scan that does Python-level work per character, quadratic in the input + (up to 200k characters), or a backtracking regex would take far longer than + the budget. The budget leaves room for slow CI runners and coverage, so a + quadratic scan built from string slices or copies can still pass it. """ md = MarkdownIt().use(section_ref_plugin) start = time.perf_counter() From 098895aa0ed7e2a605a001ed55126a741dbb93d3 Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Tue, 10 Nov 2026 13:55:02 +0100 Subject: [PATCH 4/6] =?UTF-8?q?=F0=9F=A7=AA=20TEST:=20pin=20the=20section?= =?UTF-8?q?=5Fref=20token=20shape=20as=20public=20API?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Assert every field of the section_ref token (type, nesting, children, tag, attrs, markup, content and meta) in one test, and say in the plugin docstring that the token shape is part of the public API from 0.7.0. --- mdit_py_plugins/section_ref/index.py | 14 ++++++++------ tests/test_section_ref.py | 21 +++++++++++++++++++++ 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/mdit_py_plugins/section_ref/index.py b/mdit_py_plugins/section_ref/index.py index 857a7ac..6d307af 100644 --- a/mdit_py_plugins/section_ref/index.py +++ b/mdit_py_plugins/section_ref/index.py @@ -55,12 +55,14 @@ def section_ref_plugin(md: MarkdownIt) -> None: text, while ``\\\\§1`` renders a literal backslash followed by a live reference. - Each reference becomes a ``section_ref`` token, with the matched text - (e.g. ``"§1.1"``) as its ``content``, ``markup`` set to ``§`` and - ``meta["numbers"]`` holding the parsed section number as a list of ints - (e.g. ``[1, 1]`` for ``§1.1``), so that downstream renderers can resolve - it to a heading target without re-parsing. Leading zeros are normalized - (``§01.02`` gives ``[1, 2]``); the original text remains in ``content``. + Each reference becomes a single ``section_ref`` leaf token + (``nesting=0``, ``children=None``, ``tag=""``, ``attrs={}``), + with ``markup`` set to ``"§"``, the text as written (e.g. ``"§01.02"``) + as its ``content``, and ``meta["numbers"]`` holding the integer value of + each level as a list of ints (``[1, 2]`` for ``§01.02``; every value is at + most 999,999,999), so that downstream renderers can resolve it to a + heading target without re-parsing. + The token shape is part of the public API from 0.7.0. The default rendering is ``§1.1``. Override it with ``md.add_render_rule("section_ref", ...)``. diff --git a/tests/test_section_ref.py b/tests/test_section_ref.py index d83355b..df777dc 100644 --- a/tests/test_section_ref.py +++ b/tests/test_section_ref.py @@ -112,6 +112,27 @@ def test_token(): ] +def test_token_shape_is_public_api(): + """Every field of the ``section_ref`` token, which is public API from 0.7.0. + + Downstream renderers (MyST-Parser) and ports of the grammar (ubCode) read + these fields directly, so changing any of them is a breaking change. + """ + md = MarkdownIt().use(section_ref_plugin) + children = md.parseInline("§01.02")[0].children + assert children is not None + assert len(children) == 1 + token = children[0] + assert token.type == "section_ref" + assert token.nesting == 0 + assert token.children is None + assert token.tag == "" + assert token.attrs == {} + assert token.markup == "§" + assert token.content == "§01.02" + assert token.meta == {"numbers": [1, 2]} + + def test_runtime_error(): """The plugin requires the ``add_terminator_char`` API (markdown-it-py >= 4.1.0).""" md = MarkdownIt() From 879f3977f8e354adc2f75e9fb624dce2f678cb31 Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Tue, 10 Nov 2026 13:56:08 +0100 Subject: [PATCH 5/6] =?UTF-8?q?=F0=9F=93=9A=20DOCS:=20make=20the=20section?= =?UTF-8?q?=5Fref=20grammar=20normative?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit State the grammar in the plugin docstring as rules: ASCII digits only, the ASCII-letter look-ahead (a non-ASCII letter does not block a match), no trailing dot, the nine-digit bound, the contexts where no reference forms, and that \§ is not a backslash escape (the backslash stays visible), with § and § giving a literal sign. Pin the statements the fixtures did not cover yet with fixture rows: non-ASCII digits, the two entities, an autolink and raw HTML. Note that markdown-it-py's renderInlineAsText keeps only text tokens, so a renderer that wants the reference in image alt text should use the section_ref token's content. --- mdit_py_plugins/section_ref/index.py | 43 +++++++++++++++++++--------- tests/fixtures/section_ref.md | 28 ++++++++++++++++++ 2 files changed, 58 insertions(+), 13 deletions(-) diff --git a/mdit_py_plugins/section_ref/index.py b/mdit_py_plugins/section_ref/index.py index 6d307af..6eacc63 100644 --- a/mdit_py_plugins/section_ref/index.py +++ b/mdit_py_plugins/section_ref/index.py @@ -41,19 +41,32 @@ def section_ref_plugin(md: MarkdownIt) -> None: """Parse section-sign references such as ``§1``, ``§1.1`` and ``§2.3.4``. - A reference is a ``§`` immediately followed by ASCII digits, optionally - grouped into further ``.``-separated levels (no spaces are allowed). - A trailing dot is not consumed, so ``see §1.`` captures ``§1``. A ``§`` - directly followed by an ASCII letter (e.g. ``§1a``) is *not* a reference, - but a following non-ASCII letter does not block a match, so ``见§3章`` - still captures ``§3``. A reference with any level longer than nine - digits (e.g. ``§1.1234567890``) is not a reference either: the whole - text stays literal, rather than a shorter prefix being captured. - Leading zeros count towards the nine digits, so ``§0000000001`` stays - literal too. - Following CommonMark backslash-escape behaviour, ``\\§1`` stays literal - text, while ``\\\\§1`` renders a literal backslash followed by a live - reference. + The grammar is normative: + + - A reference is ``§`` (U+00A7) immediately followed by one or more + ``.``-separated levels, with no spaces. + Each level is 1 to 9 ASCII digits, ``§([0-9]{1,9}(\\.[0-9]{1,9})*)``, + and the match is maximal. + Only ASCII ``0``-``9`` are digits: other Unicode digits, such as + Arabic-Indic (U+0660 to U+0669) or fullwidth (U+FF10 to U+FF19) digits, + do not form a reference, so a ``§`` followed by one stays literal text. + - A trailing ``.`` is not part of the reference, + so in ``see §1.`` the reference is ``§1`` and the ``.`` stays text. + - There is no reference, and the whole run stays literal text, + if the character after the match is an ASCII letter (``§1a``, ``§1.2b``), + or if any level has more than nine digits, counting leading zeros + (``§1234567890``, ``§0000000001``). + No shorter prefix is taken, so ``§1.1234567890.2`` stays literal too. + A non-ASCII letter after the match does not block it, + so ``见§3章`` captures ``§3``. + - References are recognised wherever inline text is parsed, + and not inside code spans, autolinks, raw HTML or inline math. + - ``\\§`` is not a CommonMark backslash escape, + because ``§`` is not ASCII punctuation: + in ``\\§1`` the backslash stays visible and no reference is formed. + ``\\\\§1`` is a literal backslash followed by the reference ``§1``. + To write a literal sign, use ``§`` or ``§``, + which never form a reference (``§1`` renders as the text ``§1``). Each reference becomes a single ``section_ref`` leaf token (``nesting=0``, ``children=None``, ``tag=""``, ``attrs={}``), @@ -66,6 +79,10 @@ def section_ref_plugin(md: MarkdownIt) -> None: The default rendering is ``§1.1``. Override it with ``md.add_render_rule("section_ref", ...)``. + markdown-it-py's ``renderInlineAsText``, which renders image ``alt`` text, + keeps only ``text`` tokens, so ``![a §7 b](x.png)`` renders ``alt="a b"`` + (as it does for inline math). A renderer that wants the reference in the + alt text should use the ``section_ref`` token's ``content``. .. versionadded:: 0.7.0 diff --git a/tests/fixtures/section_ref.md b/tests/fixtures/section_ref.md index 2a1973f..33af1d8 100644 --- a/tests/fixtures/section_ref.md +++ b/tests/fixtures/section_ref.md @@ -83,6 +83,20 @@ Inside inline code (not captured):

§1

. +Inside an autolink (not captured): +. + +. +

https://example.com/§1

+. + +Inside raw HTML (not captured), but captured in its content: +. +§2 +. +

§2

+. + Inside fenced code block (not captured): . ``` @@ -135,6 +149,13 @@ Escaped backslash before ref (captured):

\§1

. +Entity section signs give a literal sign (not captured): +. +§1 and §2 +. +

§1 and §2

+. + Followed by non-digit, non-space char (not captured): . §x and §.5 @@ -142,6 +163,13 @@ Followed by non-digit, non-space char (not captured):

§x and §.5

. +Non-ASCII digits (not captured): +. +§١ and §３ +. +

§١ and §３

+. + Leading zeros preserved in content (meta numbers are normalized): . §01.02 From d7e31ff3a62736b4619049df838101f3da564ceb Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Tue, 10 Nov 2026 14:00:50 +0100 Subject: [PATCH 6/6] =?UTF-8?q?=F0=9F=93=9A=20DOCS:=20section=5Fref=20text?= =?UTF-8?q?=20between=20raw=20HTML=20tags=20is=20inline=20text?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mdit_py_plugins/section_ref/index.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mdit_py_plugins/section_ref/index.py b/mdit_py_plugins/section_ref/index.py index 6eacc63..b616b14 100644 --- a/mdit_py_plugins/section_ref/index.py +++ b/mdit_py_plugins/section_ref/index.py @@ -60,7 +60,9 @@ def section_ref_plugin(md: MarkdownIt) -> None: A non-ASCII letter after the match does not block it, so ``见§3章`` captures ``§3``. - References are recognised wherever inline text is parsed, - and not inside code spans, autolinks, raw HTML or inline math. + and not inside code spans, autolinks, inline math or raw HTML tags + (an attribute value is not inline text, but the text between two tags is, + so ``§2`` captures ``§2`` only). - ``\\§`` is not a CommonMark backslash escape, because ``§`` is not ASCII punctuation: in ``\\§1`` the backslash stays visible and no reference is formed.