From 46023abaeddb901652db73db61f2b571ec386669 Mon Sep 17 00:00:00 2001 From: Kyle King Date: Mon, 20 Jul 2026 22:25:35 -0600 Subject: [PATCH 01/11] fix: address confirmed regex vulnerabilities --- mdit_py_plugins/admon/index.py | 6 +- mdit_py_plugins/texmath/index.py | 10 +-- tests/test_admon.py | 7 ++ tests/test_redos.py | 108 +++++++++++++++++++++++++++++++ tests/test_texmath.py | 15 +++++ 5 files changed, 140 insertions(+), 6 deletions(-) create mode 100644 tests/test_redos.py diff --git a/mdit_py_plugins/admon/index.py b/mdit_py_plugins/admon/index.py index 6e28940..e26b554 100644 --- a/mdit_py_plugins/admon/index.py +++ b/mdit_py_plugins/admon/index.py @@ -18,10 +18,12 @@ from markdown_it.utils import EnvType, OptionsDict +TAGS_RE = re.compile(r'^\s*(?P[^"\s](?:[^"]*[^"\s])?)\s+"(?P.*)"\S*$') + + def _get_multiple_tags(params: str) -> tuple[list[str], str]: """Check for multiple tags when the title is double quoted.""" - re_tags = re.compile(r'^\s*(?P<tokens>[^"]+)\s+"(?P<title>.*)"\S*$') - match = re_tags.match(params) + match = TAGS_RE.match(params) if match: tags = match["tokens"].strip().split(" ") return [tag.lower() for tag in tags], match["title"] diff --git a/mdit_py_plugins/texmath/index.py b/mdit_py_plugins/texmath/index.py index 67372fa..8bae929 100644 --- a/mdit_py_plugins/texmath/index.py +++ b/mdit_py_plugins/texmath/index.py @@ -228,14 +228,15 @@ def render(tex: str, displayMode: bool, macros: Any) -> str: { "name": "math_block_eqno", "rex": re.compile( - r"^`{3}math\s+?([^`]+?)\s+?`{3}\s*?\(([^)$\r\n]+?)\)", re.M + r"^`{3}math\s+([^`\s](?:[^`]*[^`\s])?)\s+`{3}\s*?\(([^)$\r\n]+?)\)", + re.M, ), "tmpl": '<section class="eqno">\n<eqn>{0}</eqn><span>({1})</span>\n</section>\n', "tag": "```math", }, { "name": "math_block", - "rex": re.compile(r"^`{3}math\s+?([^`]+?)\s+?`{3}", re.M), + "rex": re.compile(r"^`{3}math\s+([^`\s](?:[^`]*[^`\s])?)\s+`{3}", re.M), "tmpl": "<section>\n<eqn>{0}</eqn>\n</section>\n", "tag": "```math", }, @@ -270,14 +271,15 @@ def render(tex: str, displayMode: bool, macros: Any) -> str: { "name": "math_block_eqno", "rex": re.compile( - r"^`{3}math\s+?([^`]+?)\s+?`{3}\s*?\(([^)$\r\n]+?)\)", re.M + r"^`{3}math\s+([^`\s](?:[^`]*[^`\s])?)\s+`{3}\s*?\(([^)$\r\n]+?)\)", + re.M, ), "tmpl": '<section class="eqno"><eqn>{0}</eqn><span>({1})</span></section>', "tag": "```math", }, { "name": "math_block", - "rex": re.compile(r"^`{3}math\s+?([^`]+?)\s+?`{3}", re.M), + "rex": re.compile(r"^`{3}math\s+([^`\s](?:[^`]*[^`\s])?)\s+`{3}", re.M), "tmpl": "<section><eqn>{0}</eqn></section>", "tag": "```math", }, diff --git a/tests/test_admon.py b/tests/test_admon.py index f0ef479..a9f36bb 100644 --- a/tests/test_admon.py +++ b/tests/test_admon.py @@ -34,3 +34,10 @@ def test_plugin_parse(data_regression, text_idx): md = MarkdownIt().use(admon_plugin) tokens = md.parse(dedent(texts[text_idx])) data_regression.check([t.as_dict() for t in tokens]) + + +@pytest.mark.timeout(2) +def test_title_parsing_redos(): + """A long unquoted admonition header must not trigger catastrophic backtracking.""" + md = MarkdownIt("commonmark").use(admon_plugin) + md.render("!!! note " + "a " * 50000 + "b\n content") diff --git a/tests/test_redos.py b/tests/test_redos.py new file mode 100644 index 0000000..4993229 --- /dev/null +++ b/tests/test_redos.py @@ -0,0 +1,108 @@ +"""Categorical ReDoS guard for every regex shipped by the package. + +New plugins add regexes over time. Rather than trust review to catch catastrophic +backtracking, this test discovers every compiled pattern reachable from the +package and asserts it stays fast against adversarial homogeneous input (the +class of input that makes overlapping or nested quantifiers blow up). +""" + +from __future__ import annotations + +import importlib +import pkgutil +import re +import time + +import pytest + +import mdit_py_plugins + +# Homogeneous runs of a single class of character are what drive overlapping +# quantifiers (e.g. `\s+ [^x]+ \s+`) into super-linear backtracking. +_ADVERSARIAL_CHARS = [" ", "\t", "a", "1", "$", "`", "\\", '"', ".", "-"] +_LENGTH = 20_000 +_BUDGET_SECONDS = 0.5 + +# Backslash escapes that denote a character *class*, not a literal, so prefix +# extraction must stop rather than emit the letter that follows the backslash. +_CLASS_ESCAPES = set("sSdDwWbBAZ") + + +def _literal_prefix(source: str) -> str: + """Best-effort leading literal of a regex, so anchored patterns get exercised. + + Anchored rules like ``^```math\\s+...`` only backtrack once their literal + lead-in matches, so an adversarial run must be prefixed with that literal. + """ + if source.startswith("^"): + source = source[1:] + out: list[str] = [] + i = 0 + while i < len(source): + char = source[i] + if char == "\\" and i + 1 < len(source): + nxt = source[i + 1] + if nxt in _CLASS_ESCAPES: + break + out.append(nxt) + i += 2 + elif char in ".^$*+?()[]{}|": + break + else: + out.append(char) + i += 1 + if i < len(source) and source[i] == "{": + end = source.find("}", i) + if end == -1 or not out: + break + count = source[i + 1 : end].split(",")[0] + if not count.isdigit(): + break + out.append(out[-1] * (int(count) - 1)) + i = end + 1 + return "".join(out) + + +def _iter_patterns() -> list[tuple[str, re.Pattern[str]]]: + found: dict[int, tuple[str, re.Pattern[str]]] = {} + + def _collect(where: str, value: object) -> None: + if isinstance(value, re.Pattern): + found.setdefault(id(value), (where, value)) + elif isinstance(value, dict): + for item in value.values(): + _collect(where, item) + elif isinstance(value, (list, tuple)): + for item in value: + _collect(where, item) + + for info in pkgutil.walk_packages( + mdit_py_plugins.__path__, prefix=f"{mdit_py_plugins.__name__}." + ): + module = importlib.import_module(info.name) + for name, value in vars(module).items(): + _collect(f"{info.name}.{name}", value) + + return sorted(found.values(), key=lambda item: item[0]) + + +_PATTERNS = _iter_patterns() + + +@pytest.mark.timeout(10) +@pytest.mark.parametrize("where,pattern", _PATTERNS, ids=[w for w, _ in _PATTERNS]) +def test_regex_is_redos_safe(where: str, pattern: re.Pattern[str]) -> None: + prefix = _literal_prefix(pattern.pattern) + for char in _ADVERSARIAL_CHARS: + payload = prefix + char * _LENGTH + start = time.perf_counter() + pattern.search(payload) + elapsed = time.perf_counter() - start + assert elapsed < _BUDGET_SECONDS, ( + f"{where} took {elapsed:.3f}s on {prefix!r}+{_LENGTH}x{char!r}; " + "likely catastrophic backtracking (ReDoS)" + ) + + +def test_patterns_were_discovered() -> None: + assert len(_PATTERNS) > 10 diff --git a/tests/test_texmath.py b/tests/test_texmath.py index 42f7d9b..da601b6 100644 --- a/tests/test_texmath.py +++ b/tests/test_texmath.py @@ -106,3 +106,18 @@ def test_bracket_fixtures(line, title, input, expected): text = md.render(input) print(text) assert text.rstrip() == expected.rstrip() + + +@pytest.mark.parametrize("delimiters", ["gitlab", "julia"]) +def test_fenced_math_block(delimiters): + md = MarkdownIt("commonmark").use(texmath_plugin, delimiters=delimiters) + text = md.render("```math\n\\alpha = 1\n```\n") + assert "<eqn>\\alpha = 1</eqn>" in text + + +@pytest.mark.parametrize("delimiters", ["gitlab", "julia"]) +@pytest.mark.timeout(2) +def test_fenced_math_block_redos(delimiters): + """An unclosed ``` ```math ``` fence must not trigger catastrophic backtracking.""" + md = MarkdownIt("commonmark").use(texmath_plugin, delimiters=delimiters) + md.render("```math\n" + " " * 20000) From 8309e76e40268dbdc88361c4350d9eeffde5674f Mon Sep 17 00:00:00 2001 From: Kyle King <KyleKing@users.noreply.github.com> Date: Mon, 20 Jul 2026 22:46:43 -0600 Subject: [PATCH 02/11] refactor: check inline regex too Hoist inline and string regexes to module-level compiled patterns (amsmath, tasklists, dollarmath, anchors) so the ReDoS guard discovers them. Compiling is also faster for these hot paths. Inline re.match(str, ...) still compiles once and caches, but pays a cache lookup on every call, so a compiled object runs about 2.2x faster per call (102 vs 235 ns on py3.14). The one-time compile (5 to 59 us) is paid once either way, and module globals sidestep re._cache's 512-entry eviction. Benched with timeit (best-of-5, 2M calls) comparing compiled.match(text) against a cache-warm re.match(pattern, text). --- mdit_py_plugins/amsmath/__init__.py | 4 ++-- mdit_py_plugins/anchors/index.py | 5 +++- mdit_py_plugins/dollarmath/index.py | 5 +++- mdit_py_plugins/tasklists/__init__.py | 3 ++- tests/test_redos.py | 34 +++++++++++++++++---------- 5 files changed, 33 insertions(+), 18 deletions(-) diff --git a/mdit_py_plugins/amsmath/__init__.py b/mdit_py_plugins/amsmath/__init__.py index bad1b21..c5a9927 100644 --- a/mdit_py_plugins/amsmath/__init__.py +++ b/mdit_py_plugins/amsmath/__init__.py @@ -55,7 +55,7 @@ # whose total width is the actual width of the contents; # thus they can be used as a component in a containing expression -RE_OPEN = r"\\begin\{(" + "|".join(ENVIRONMENTS) + r")([\*]?)\}" +RE_OPEN = re.compile(r"\\begin\{(" + "|".join(ENVIRONMENTS) + r")([\*]?)\}") def amsmath_plugin( @@ -112,7 +112,7 @@ def amsmath_block( first_end = state.eMarks[startLine] first_text = state.src[first_start:first_end] - if not (match_open := re.match(RE_OPEN, first_text)): + if not (match_open := RE_OPEN.match(first_text)): return False # construct the closing tag diff --git a/mdit_py_plugins/anchors/index.py b/mdit_py_plugins/anchors/index.py index 66bf0aa..f00eb6d 100644 --- a/mdit_py_plugins/anchors/index.py +++ b/mdit_py_plugins/anchors/index.py @@ -115,8 +115,11 @@ def _anchor_func(state: StateCore) -> None: return _anchor_func +_NON_SLUG_CHARS_RE = re.compile(r"[^\w\u4e00-\u9fff\- ]") + + def slugify(title: str) -> str: - return re.sub(r"[^\w\u4e00-\u9fff\- ]", "", title.strip().lower().replace(" ", "-")) + return _NON_SLUG_CHARS_RE.sub("", title.strip().lower().replace(" ", "-")) def unique_slug(slug: str, slugs: set[str]) -> str: diff --git a/mdit_py_plugins/dollarmath/index.py b/mdit_py_plugins/dollarmath/index.py index 57ba590..7af9a6a 100644 --- a/mdit_py_plugins/dollarmath/index.py +++ b/mdit_py_plugins/dollarmath/index.py @@ -17,6 +17,9 @@ from markdown_it.utils import EnvType, OptionsDict +_LABEL_WHITESPACE_RE = re.compile(r"\s+") + + def dollarmath_plugin( md: MarkdownIt, *, @@ -54,7 +57,7 @@ def dollarmath_plugin( """ if label_normalizer is None: - label_normalizer = lambda label: re.sub(r"\s+", "-", label) # noqa: E731 + label_normalizer = lambda label: _LABEL_WHITESPACE_RE.sub("-", label) # noqa: E731 md.inline.ruler.before( "escape", diff --git a/mdit_py_plugins/tasklists/__init__.py b/mdit_py_plugins/tasklists/__init__.py index d80f475..d2ef09f 100644 --- a/mdit_py_plugins/tasklists/__init__.py +++ b/mdit_py_plugins/tasklists/__init__.py @@ -28,6 +28,7 @@ # https://github.github.com/gfm/#whitespace-character # (spec version 0.29-gfm (2019-04-06)) _GFM_WHITESPACE_RE = r"[ \t\n\v\f\r]" +_TASK_LIST_ITEM_RE = re.compile(rf"\[[ xX]]{_GFM_WHITESPACE_RE}+") def tasklists_plugin( @@ -148,4 +149,4 @@ def is_list_item(token: Token) -> bool: def starts_with_todo_markdown(token: Token) -> bool: # leading whitespace in a list item is already trimmed off by markdown-it - return re.match(rf"\[[ xX]]{_GFM_WHITESPACE_RE}+", token.content) is not None + return _TASK_LIST_ITEM_RE.match(token.content) is not None diff --git a/tests/test_redos.py b/tests/test_redos.py index 4993229..d86d517 100644 --- a/tests/test_redos.py +++ b/tests/test_redos.py @@ -23,6 +23,8 @@ class of input that makes overlapping or nested quantifiers blow up). _LENGTH = 20_000 _BUDGET_SECONDS = 0.5 +# Metacharacters that end the run of literal leading characters. +_METACHARS = set(".^$*+?()[]{}|") # Backslash escapes that denote a character *class*, not a literal, so prefix # extraction must stop rather than emit the letter that follows the backslash. _CLASS_ESCAPES = set("sSdDwWbBAZ") @@ -33,9 +35,11 @@ def _literal_prefix(source: str) -> str: Anchored rules like ``^```math\\s+...`` only backtrack once their literal lead-in matches, so an adversarial run must be prefixed with that literal. + Parsing walks the source until the first metacharacter or character class, + treating ``\\x`` as the literal ``x`` and expanding a ``{n}`` repeat of the + preceding literal (so ```` `{3} ```` yields three backticks). """ - if source.startswith("^"): - source = source[1:] + source = source.removeprefix("^") out: list[str] = [] i = 0 while i < len(source): @@ -46,26 +50,26 @@ def _literal_prefix(source: str) -> str: break out.append(nxt) i += 2 - elif char in ".^$*+?()[]{}|": + elif char == "{" and out: + end = source.find("}", i) + count = source[i + 1 : end] if end != -1 else "" + if not (repeats := count.split(",")[0]).isdigit(): + break + out.append(out[-1] * (int(repeats) - 1)) + i = end + 1 + elif char in _METACHARS: break else: out.append(char) i += 1 - if i < len(source) and source[i] == "{": - end = source.find("}", i) - if end == -1 or not out: - break - count = source[i + 1 : end].split(",")[0] - if not count.isdigit(): - break - out.append(out[-1] * (int(count) - 1)) - i = end + 1 return "".join(out) def _iter_patterns() -> list[tuple[str, re.Pattern[str]]]: found: dict[int, tuple[str, re.Pattern[str]]] = {} + # Patterns may sit directly in a module global or nested inside a global + # container (e.g. texmath's ``rules`` dict of delimiter flavors), so recurse. def _collect(where: str, value: object) -> None: if isinstance(value, re.Pattern): found.setdefault(id(value), (where, value)) @@ -88,9 +92,13 @@ def _collect(where: str, value: object) -> None: _PATTERNS = _iter_patterns() +# Several patterns share a ``where`` (e.g. every texmath flavor lives in one +# ``rules`` dict), so include the regex source to keep failing ids unambiguous. +_IDS = [f"{where} {pattern.pattern[:48]}" for where, pattern in _PATTERNS] + @pytest.mark.timeout(10) -@pytest.mark.parametrize("where,pattern", _PATTERNS, ids=[w for w, _ in _PATTERNS]) +@pytest.mark.parametrize("where,pattern", _PATTERNS, ids=_IDS) def test_regex_is_redos_safe(where: str, pattern: re.Pattern[str]) -> None: prefix = _literal_prefix(pattern.pattern) for char in _ADVERSARIAL_CHARS: From c290fd8f0309f11066f820892edb2513783636f4 Mon Sep 17 00:00:00 2001 From: Kyle King <KyleKing@users.noreply.github.com> Date: Tue, 21 Jul 2026 08:11:47 -0600 Subject: [PATCH 03/11] fix: address review comments/re-scope --- CHANGELOG.md | 17 ++ mdit_py_plugins/admon/index.py | 6 +- mdit_py_plugins/amsmath/__init__.py | 4 +- mdit_py_plugins/anchors/index.py | 5 +- mdit_py_plugins/dollarmath/index.py | 5 +- mdit_py_plugins/tasklists/__init__.py | 3 +- tests/test_admon.py | 7 - tests/test_redos.py | 238 ++++++++++++++++---------- 8 files changed, 167 insertions(+), 118 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a1b78b3..92c8da6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,22 @@ # Change Log +## Unreleased + +- 🐛 FIX: Resolve regex denial-of-service (ReDoS) in `texmath` and `admon` (#143, #148) + + Two shipped patterns backtracked super-linearly on adversarial input with no + closing delimiter, so a single crafted document could hang the parser: + + - `texmath`'s `` ```math `` block rules (all `` ```math `` flavors, i.e. the + `gitlab` and `julia` delimiters) blew up on a `` ```math `` fence followed by a + long whitespace run with no closing fence. + - `admon`'s multi-tag title parser blew up on an unquoted header containing a long + interior whitespace run. + + Both patterns were rewritten to anchor the capture on non-whitespace at both ends, + which stays flat on the same input. A behavioral guard (`tests/test_redos.py`) now + renders adversarial payloads through every plugin to catch the next such pattern. + ## 0.7.0 - 2026-07-19 - ✨ NEW: Add section reference plugin (`section_ref`) (#144) diff --git a/mdit_py_plugins/admon/index.py b/mdit_py_plugins/admon/index.py index e26b554..d603e1a 100644 --- a/mdit_py_plugins/admon/index.py +++ b/mdit_py_plugins/admon/index.py @@ -18,12 +18,10 @@ from markdown_it.utils import EnvType, OptionsDict -TAGS_RE = re.compile(r'^\s*(?P<tokens>[^"\s](?:[^"]*[^"\s])?)\s+"(?P<title>.*)"\S*$') - - def _get_multiple_tags(params: str) -> tuple[list[str], str]: """Check for multiple tags when the title is double quoted.""" - match = TAGS_RE.match(params) + re_tags = re.compile(r'^\s*(?P<tokens>[^"\s](?:[^"]*[^"\s])?)\s+"(?P<title>.*)"\S*$') + match = re_tags.match(params) if match: tags = match["tokens"].strip().split(" ") return [tag.lower() for tag in tags], match["title"] diff --git a/mdit_py_plugins/amsmath/__init__.py b/mdit_py_plugins/amsmath/__init__.py index c5a9927..bad1b21 100644 --- a/mdit_py_plugins/amsmath/__init__.py +++ b/mdit_py_plugins/amsmath/__init__.py @@ -55,7 +55,7 @@ # whose total width is the actual width of the contents; # thus they can be used as a component in a containing expression -RE_OPEN = re.compile(r"\\begin\{(" + "|".join(ENVIRONMENTS) + r")([\*]?)\}") +RE_OPEN = r"\\begin\{(" + "|".join(ENVIRONMENTS) + r")([\*]?)\}" def amsmath_plugin( @@ -112,7 +112,7 @@ def amsmath_block( first_end = state.eMarks[startLine] first_text = state.src[first_start:first_end] - if not (match_open := RE_OPEN.match(first_text)): + if not (match_open := re.match(RE_OPEN, first_text)): return False # construct the closing tag diff --git a/mdit_py_plugins/anchors/index.py b/mdit_py_plugins/anchors/index.py index f00eb6d..66bf0aa 100644 --- a/mdit_py_plugins/anchors/index.py +++ b/mdit_py_plugins/anchors/index.py @@ -115,11 +115,8 @@ def _anchor_func(state: StateCore) -> None: return _anchor_func -_NON_SLUG_CHARS_RE = re.compile(r"[^\w\u4e00-\u9fff\- ]") - - def slugify(title: str) -> str: - return _NON_SLUG_CHARS_RE.sub("", title.strip().lower().replace(" ", "-")) + return re.sub(r"[^\w\u4e00-\u9fff\- ]", "", title.strip().lower().replace(" ", "-")) def unique_slug(slug: str, slugs: set[str]) -> str: diff --git a/mdit_py_plugins/dollarmath/index.py b/mdit_py_plugins/dollarmath/index.py index 7af9a6a..57ba590 100644 --- a/mdit_py_plugins/dollarmath/index.py +++ b/mdit_py_plugins/dollarmath/index.py @@ -17,9 +17,6 @@ from markdown_it.utils import EnvType, OptionsDict -_LABEL_WHITESPACE_RE = re.compile(r"\s+") - - def dollarmath_plugin( md: MarkdownIt, *, @@ -57,7 +54,7 @@ def dollarmath_plugin( """ if label_normalizer is None: - label_normalizer = lambda label: _LABEL_WHITESPACE_RE.sub("-", label) # noqa: E731 + label_normalizer = lambda label: re.sub(r"\s+", "-", label) # noqa: E731 md.inline.ruler.before( "escape", diff --git a/mdit_py_plugins/tasklists/__init__.py b/mdit_py_plugins/tasklists/__init__.py index d2ef09f..d80f475 100644 --- a/mdit_py_plugins/tasklists/__init__.py +++ b/mdit_py_plugins/tasklists/__init__.py @@ -28,7 +28,6 @@ # https://github.github.com/gfm/#whitespace-character # (spec version 0.29-gfm (2019-04-06)) _GFM_WHITESPACE_RE = r"[ \t\n\v\f\r]" -_TASK_LIST_ITEM_RE = re.compile(rf"\[[ xX]]{_GFM_WHITESPACE_RE}+") def tasklists_plugin( @@ -149,4 +148,4 @@ def is_list_item(token: Token) -> bool: def starts_with_todo_markdown(token: Token) -> bool: # leading whitespace in a list item is already trimmed off by markdown-it - return _TASK_LIST_ITEM_RE.match(token.content) is not None + return re.match(rf"\[[ xX]]{_GFM_WHITESPACE_RE}+", token.content) is not None diff --git a/tests/test_admon.py b/tests/test_admon.py index a9f36bb..f0ef479 100644 --- a/tests/test_admon.py +++ b/tests/test_admon.py @@ -34,10 +34,3 @@ def test_plugin_parse(data_regression, text_idx): md = MarkdownIt().use(admon_plugin) tokens = md.parse(dedent(texts[text_idx])) data_regression.check([t.as_dict() for t in tokens]) - - -@pytest.mark.timeout(2) -def test_title_parsing_redos(): - """A long unquoted admonition header must not trigger catastrophic backtracking.""" - md = MarkdownIt("commonmark").use(admon_plugin) - md.render("!!! note " + "a " * 50000 + "b\n content") diff --git a/tests/test_redos.py b/tests/test_redos.py index d86d517..22f9650 100644 --- a/tests/test_redos.py +++ b/tests/test_redos.py @@ -1,116 +1,164 @@ -"""Categorical ReDoS guard for every regex shipped by the package. +"""Behavioral ReDoS guard: render adversarial markdown through each plugin. New plugins add regexes over time. Rather than trust review to catch catastrophic -backtracking, this test discovers every compiled pattern reachable from the -package and asserts it stays fast against adversarial homogeneous input (the -class of input that makes overlapping or nested quantifiers blow up). +backtracking, this drives each plugin end-to-end (``MarkdownIt().use(plugin)`` then +``md.render(payload)``) against adversarial input and asserts it stays fast. + +Two properties make this faithful to the real attack surface without introspecting +any pattern: + +- The payload embeds a long *homogeneous run* of a single character class, which is + what pushes overlapping or nested quantifiers (e.g. ``\\s+ [^x]+ \\s+``) into + super-linear backtracking. We fuzz every run character, so we never need to know + which regex is vulnerable or how it is structured. +- Rendering exercises the *whole* rule chain (the parser re-invoking a block rule + across offsets), so it also catches the O(n^2) amplification an isolated-regex + check would miss, and never false-positives on a pattern adversarial input can't + actually reach. + +The only per-plugin cost is one hand-written activation payload: the plugin's public +markdown syntax with a ``{run}`` placeholder for the homogeneous run. """ from __future__ import annotations -import importlib -import pkgutil -import re +from collections.abc import Callable +from dataclasses import dataclass import time +from markdown_it import MarkdownIt import pytest -import mdit_py_plugins - -# Homogeneous runs of a single class of character are what drive overlapping -# quantifiers (e.g. `\s+ [^x]+ \s+`) into super-linear backtracking. -_ADVERSARIAL_CHARS = [" ", "\t", "a", "1", "$", "`", "\\", '"', ".", "-"] -_LENGTH = 20_000 +from mdit_py_plugins.admon import admon_plugin +from mdit_py_plugins.amsmath import amsmath_plugin +from mdit_py_plugins.anchors import anchors_plugin +from mdit_py_plugins.colon_fence import colon_fence_plugin +from mdit_py_plugins.deflist import deflist_plugin +from mdit_py_plugins.dollarmath import dollarmath_plugin +from mdit_py_plugins.field_list import fieldlist_plugin +from mdit_py_plugins.footnote import footnote_plugin +from mdit_py_plugins.myst_blocks import myst_block_plugin +from mdit_py_plugins.myst_role import myst_role_plugin +from mdit_py_plugins.tasklists import tasklists_plugin +from mdit_py_plugins.texmath import texmath_plugin + +# Homogeneous runs of one character class are what drive overlapping/nested +# quantifiers into catastrophic backtracking; fuzz every plugin with each. +_FILLERS = [" ", "\t", "a", "1", "$", "`", "\\", '"', ".", "-"] +_RUN = 50_000 _BUDGET_SECONDS = 0.5 -# Metacharacters that end the run of literal leading characters. -_METACHARS = set(".^$*+?()[]{}|") -# Backslash escapes that denote a character *class*, not a literal, so prefix -# extraction must stop rather than emit the letter that follows the backslash. -_CLASS_ESCAPES = set("sSdDwWbBAZ") - - -def _literal_prefix(source: str) -> str: - """Best-effort leading literal of a regex, so anchored patterns get exercised. - - Anchored rules like ``^```math\\s+...`` only backtrack once their literal - lead-in matches, so an adversarial run must be prefixed with that literal. - Parsing walks the source until the first metacharacter or character class, - treating ``\\x`` as the literal ``x`` and expanding a ``{n}`` repeat of the - preceding literal (so ```` `{3} ```` yields three backticks). - """ - source = source.removeprefix("^") - out: list[str] = [] - i = 0 - while i < len(source): - char = source[i] - if char == "\\" and i + 1 < len(source): - nxt = source[i + 1] - if nxt in _CLASS_ESCAPES: - break - out.append(nxt) - i += 2 - elif char == "{" and out: - end = source.find("}", i) - count = source[i + 1 : end] if end != -1 else "" - if not (repeats := count.split(",")[0]).isdigit(): - break - out.append(out[-1] * (int(repeats) - 1)) - i = end + 1 - elif char in _METACHARS: - break - else: - out.append(char) - i += 1 - return "".join(out) - - -def _iter_patterns() -> list[tuple[str, re.Pattern[str]]]: - found: dict[int, tuple[str, re.Pattern[str]]] = {} - - # Patterns may sit directly in a module global or nested inside a global - # container (e.g. texmath's ``rules`` dict of delimiter flavors), so recurse. - def _collect(where: str, value: object) -> None: - if isinstance(value, re.Pattern): - found.setdefault(id(value), (where, value)) - elif isinstance(value, dict): - for item in value.values(): - _collect(where, item) - elif isinstance(value, (list, tuple)): - for item in value: - _collect(where, item) - for info in pkgutil.walk_packages( - mdit_py_plugins.__path__, prefix=f"{mdit_py_plugins.__name__}." - ): - module = importlib.import_module(info.name) - for name, value in vars(module).items(): - _collect(f"{info.name}.{name}", value) - - return sorted(found.values(), key=lambda item: item[0]) - - -_PATTERNS = _iter_patterns() - -# Several patterns share a ``where`` (e.g. every texmath flavor lives in one -# ``rules`` dict), so include the regex source to keep failing ids unambiguous. -_IDS = [f"{where} {pattern.pattern[:48]}" for where, pattern in _PATTERNS] +@dataclass(frozen=True) +class Case: + id: str + configure: Callable[[MarkdownIt], None] + payload: Callable[[str], str] + + +_CASES = [ + Case( + # An interior run with no closing quote: ``[^"]+\s+"`` backtracks the run + # exhaustively looking for a ``"`` that never comes. A trailing run would be + # stripped off before the regex sees it, so the run must sit between tokens. + "admon-header", + lambda md: md.use(admon_plugin), + lambda run: f"!!! note a{run}b\n content", + ), + Case( + "amsmath-open", + lambda md: md.use(amsmath_plugin), + lambda run: f"\\begin{{{run}", + ), + Case( + "amsmath-unclosed", + lambda md: md.use(amsmath_plugin), + lambda run: f"\\begin{{equation}}\n{run}", + ), + Case( + "anchors-heading", + lambda md: md.use(anchors_plugin), + lambda run: f"# {run}", + ), + Case( + "colon-fence", + lambda md: md.use(colon_fence_plugin), + lambda run: f"::: {run}\ncontent\n:::", + ), + Case( + "deflist", + lambda md: md.use(deflist_plugin), + lambda run: f"term\n:{run}", + ), + Case( + "dollarmath-inline", + lambda md: md.use(dollarmath_plugin), + lambda run: f"${run}$", + ), + Case( + "dollarmath-block", + lambda md: md.use(dollarmath_plugin, allow_labels=True), + lambda run: f"$$\n{run}", + ), + Case( + "dollarmath-label", + lambda md: md.use(dollarmath_plugin, allow_labels=True), + lambda run: f"$$a$$ ({run})", + ), + Case( + "fieldlist", + lambda md: md.use(fieldlist_plugin), + lambda run: f":name:{run}", + ), + Case( + "footnote-ref", + lambda md: md.use(footnote_plugin), + lambda run: f"[^{run}]: body", + ), + Case( + "myst-block-target", + lambda md: md.use(myst_block_plugin), + lambda run: f"({run})=", + ), + Case( + "myst-role", + lambda md: md.use(myst_role_plugin), + lambda run: f"{{role}}`{run}`", + ), + Case( + "tasklist", + lambda md: md.use(tasklists_plugin), + lambda run: f"- [ ]{run}x", + ), + Case( + "texmath-fence", + lambda md: md.use(texmath_plugin, delimiters="gitlab"), + lambda run: f"```math\n{run}", + ), + Case( + "texmath-fence-eqno", + lambda md: md.use(texmath_plugin, delimiters="gitlab"), + lambda run: f"```math\n{run}``` (1)", + ), + Case( + "texmath-dollar", + lambda md: md.use(texmath_plugin, delimiters="dollars"), + lambda run: f"${run}$", + ), +] @pytest.mark.timeout(10) -@pytest.mark.parametrize("where,pattern", _PATTERNS, ids=_IDS) -def test_regex_is_redos_safe(where: str, pattern: re.Pattern[str]) -> None: - prefix = _literal_prefix(pattern.pattern) - for char in _ADVERSARIAL_CHARS: - payload = prefix + char * _LENGTH +@pytest.mark.parametrize("case", _CASES, ids=lambda case: case.id) +def test_plugin_is_redos_safe(case: Case) -> None: + for filler in _FILLERS: + md = MarkdownIt("commonmark") + case.configure(md) + source = case.payload(filler * _RUN) start = time.perf_counter() - pattern.search(payload) + md.render(source) elapsed = time.perf_counter() - start assert elapsed < _BUDGET_SECONDS, ( - f"{where} took {elapsed:.3f}s on {prefix!r}+{_LENGTH}x{char!r}; " + f"{case.id} took {elapsed:.3f}s on a {_RUN}x{filler!r} run; " "likely catastrophic backtracking (ReDoS)" ) - - -def test_patterns_were_discovered() -> None: - assert len(_PATTERNS) > 10 From bdc7f414e6e184a2efc1127b3eb477bb9a183573 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 21 Jul 2026 15:54:39 +0000 Subject: [PATCH 04/11] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- mdit_py_plugins/admon/index.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mdit_py_plugins/admon/index.py b/mdit_py_plugins/admon/index.py index d603e1a..e54c1ae 100644 --- a/mdit_py_plugins/admon/index.py +++ b/mdit_py_plugins/admon/index.py @@ -20,7 +20,9 @@ def _get_multiple_tags(params: str) -> tuple[list[str], str]: """Check for multiple tags when the title is double quoted.""" - re_tags = re.compile(r'^\s*(?P<tokens>[^"\s](?:[^"]*[^"\s])?)\s+"(?P<title>.*)"\S*$') + re_tags = re.compile( + r'^\s*(?P<tokens>[^"\s](?:[^"]*[^"\s])?)\s+"(?P<title>.*)"\S*$' + ) match = re_tags.match(params) if match: tags = match["tokens"].strip().split(" ") From b254e54ce9d1510073ac4665fd0d5197f8c48224 Mon Sep 17 00:00:00 2001 From: Kyle King <KyleKing@users.noreply.github.com> Date: Tue, 21 Jul 2026 10:18:46 -0600 Subject: [PATCH 05/11] ci: relax time limit in CI and speed up --- tests/test_redos.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_redos.py b/tests/test_redos.py index 22f9650..cfc1320 100644 --- a/tests/test_redos.py +++ b/tests/test_redos.py @@ -43,10 +43,10 @@ from mdit_py_plugins.texmath import texmath_plugin # Homogeneous runs of one character class are what drive overlapping/nested -# quantifiers into catastrophic backtracking; fuzz every plugin with each. -_FILLERS = [" ", "\t", "a", "1", "$", "`", "\\", '"', ".", "-"] -_RUN = 50_000 -_BUDGET_SECONDS = 0.5 +# quantifiers into catastrophic backtracking; fuzz every plugin with each +_FILLERS = [" ", "\t", "$", "`", "\\", '"', "-"] +_RUN = 15_000 +_BUDGET_SECONDS = 1.0 # Safe threshold relaxed for CI @dataclass(frozen=True) From 4ba57660fd1fa623729b943f6cc5f6fe8416ba1f Mon Sep 17 00:00:00 2001 From: Chris Sewell <chrisj_sewell@hotmail.com> Date: Mon, 9 Nov 2026 20:25:28 +0100 Subject: [PATCH 06/11] =?UTF-8?q?=F0=9F=90=9B=20FIX:=20texmath:=20keep=20f?= =?UTF-8?q?irst-line=20indentation=20in=20```math=20blocks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ReDoS-safe pattern `\s+([^`\s]...)` also consumed the indentation of the body's first line, so "```math\n a = 1" rendered `<eqn>a = 1`. Match exactly one separator character and let the group start with `\s*[^`\s]`: `\s*` and `[^`\s]` are disjoint, so the pattern stays linear, and for every body that is not all whitespace it captures what the original pattern captured. --- mdit_py_plugins/texmath/index.py | 12 ++++++++---- tests/test_texmath.py | 20 ++++++++++++++++++++ 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/mdit_py_plugins/texmath/index.py b/mdit_py_plugins/texmath/index.py index 8bae929..3d4a1a4 100644 --- a/mdit_py_plugins/texmath/index.py +++ b/mdit_py_plugins/texmath/index.py @@ -228,7 +228,7 @@ def render(tex: str, displayMode: bool, macros: Any) -> str: { "name": "math_block_eqno", "rex": re.compile( - r"^`{3}math\s+([^`\s](?:[^`]*[^`\s])?)\s+`{3}\s*?\(([^)$\r\n]+?)\)", + r"^`{3}math\s(\s*[^`\s](?:[^`]*[^`\s])?)\s+`{3}\s*?\(([^)$\r\n]+?)\)", re.M, ), "tmpl": '<section class="eqno">\n<eqn>{0}</eqn><span>({1})</span>\n</section>\n', @@ -236,7 +236,9 @@ def render(tex: str, displayMode: bool, macros: Any) -> str: }, { "name": "math_block", - "rex": re.compile(r"^`{3}math\s+([^`\s](?:[^`]*[^`\s])?)\s+`{3}", re.M), + "rex": re.compile( + r"^`{3}math\s(\s*[^`\s](?:[^`]*[^`\s])?)\s+`{3}", re.M + ), "tmpl": "<section>\n<eqn>{0}</eqn>\n</section>\n", "tag": "```math", }, @@ -271,7 +273,7 @@ def render(tex: str, displayMode: bool, macros: Any) -> str: { "name": "math_block_eqno", "rex": re.compile( - r"^`{3}math\s+([^`\s](?:[^`]*[^`\s])?)\s+`{3}\s*?\(([^)$\r\n]+?)\)", + r"^`{3}math\s(\s*[^`\s](?:[^`]*[^`\s])?)\s+`{3}\s*?\(([^)$\r\n]+?)\)", re.M, ), "tmpl": '<section class="eqno"><eqn>{0}</eqn><span>({1})</span></section>', @@ -279,7 +281,9 @@ def render(tex: str, displayMode: bool, macros: Any) -> str: }, { "name": "math_block", - "rex": re.compile(r"^`{3}math\s+([^`\s](?:[^`]*[^`\s])?)\s+`{3}", re.M), + "rex": re.compile( + r"^`{3}math\s(\s*[^`\s](?:[^`]*[^`\s])?)\s+`{3}", re.M + ), "tmpl": "<section><eqn>{0}</eqn></section>", "tag": "```math", }, diff --git a/tests/test_texmath.py b/tests/test_texmath.py index da601b6..dfa1ae9 100644 --- a/tests/test_texmath.py +++ b/tests/test_texmath.py @@ -115,6 +115,26 @@ def test_fenced_math_block(delimiters): assert "<eqn>\\alpha = 1</eqn>" in text +@pytest.mark.parametrize("delimiters", ["gitlab", "julia"]) +@pytest.mark.parametrize( + "source,expected", + [ + ("```math\n a = 1\n b = 2\n```\n", "<eqn> a = 1\n b = 2</eqn>"), + ("```math\n a = 1\n``` (1)\n", "<eqn> a = 1</eqn><span>(1)</span>"), + ], + ids=["math_block", "math_block_eqno"], +) +def test_fenced_math_block_keeps_indent(delimiters, source, expected): + """The ReDoS fix must not change output: the first line keeps its indentation. + + One whitespace character (the newline after the opening fence) separates + the fence from the body; a pattern that skips all leading whitespace + would render ``<eqn>a = 1`` instead. + """ + md = MarkdownIt("commonmark").use(texmath_plugin, delimiters=delimiters) + assert expected in md.render(source) + + @pytest.mark.parametrize("delimiters", ["gitlab", "julia"]) @pytest.mark.timeout(2) def test_fenced_math_block_redos(delimiters): From 947441e8ab9d5049579a98624badc1638a1b18f4 Mon Sep 17 00:00:00 2001 From: Chris Sewell <chrisj_sewell@hotmail.com> Date: Mon, 9 Nov 2026 20:25:46 +0100 Subject: [PATCH 07/11] =?UTF-8?q?=F0=9F=A7=AA=20TEST:=20admon:=20guard=20t?= =?UTF-8?q?he=20header=20ReDoS=20fix=20at=20100k=20characters?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit At the 15k-character run that `test_redos` uses, the old admon title pattern takes about 0.2 s, inside that test's 1 s budget, so reverting the admon fix left the suite green. `test_header_redos` renders runs of 25k and 100k characters and asserts the time grows less than 8x (about 4x when linear, about 16x when quadratic), so it does not depend on the speed of the machine. --- tests/test_admon.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/tests/test_admon.py b/tests/test_admon.py index f0ef479..4f611d5 100644 --- a/tests/test_admon.py +++ b/tests/test_admon.py @@ -1,5 +1,6 @@ from pathlib import Path from textwrap import dedent +import time from markdown_it import MarkdownIt from markdown_it.utils import read_fixture_file @@ -34,3 +35,39 @@ def test_plugin_parse(data_regression, text_idx): md = MarkdownIt().use(admon_plugin) tokens = md.parse(dedent(texts[text_idx])) data_regression.check([t.as_dict() for t in tokens]) + + +def _best_render_seconds(md: MarkdownIt, source: str, limit: float) -> float: + """Fastest of three renders, stopping early once one exceeds ``limit``.""" + best = float("inf") + for _ in range(3): + start = time.perf_counter() + md.render(source) + best = min(best, time.perf_counter() - start) + if best > limit: + break + return best + + +@pytest.mark.timeout(60) +def test_header_redos(): + """A multi-tag header with a long unquoted space run parses in linear time. + + The title pattern ``[^"]+\\s+"`` backtracked quadratically on this input (#143), + taking seconds at 100k characters. + The guard compares two run lengths instead of asserting a time budget, + so it does not depend on how fast the machine is: + a run four times longer takes about 4x as long when parsing is linear, + and about 16x when it is quadratic. + """ + md = MarkdownIt().use(admon_plugin) + + def source(run: int) -> str: + return "!!! note a" + " " * run + "b\n content" + + small = _best_render_seconds(md, source(25_000), float("inf")) + large = _best_render_seconds(md, source(100_000), 8 * small) + assert large < 8 * small, ( + f"25k run: {small:.4f}s, 100k run: {large:.4f}s ({large / small:.1f}x); " + "the admon header parse is super-linear (ReDoS)" + ) From 30eedc14948b79a6e5bbae549ba6fed152a631c7 Mon Sep 17 00:00:00 2001 From: Chris Sewell <chrisj_sewell@hotmail.com> Date: Mon, 9 Nov 2026 20:25:50 +0100 Subject: [PATCH 08/11] =?UTF-8?q?=F0=9F=94=A7=20MAINTAIN:=20drop=20changel?= =?UTF-8?q?og=20entry=20(written=20at=20release)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 92c8da6..a1b78b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,22 +1,5 @@ # Change Log -## Unreleased - -- 🐛 FIX: Resolve regex denial-of-service (ReDoS) in `texmath` and `admon` (#143, #148) - - Two shipped patterns backtracked super-linearly on adversarial input with no - closing delimiter, so a single crafted document could hang the parser: - - - `texmath`'s `` ```math `` block rules (all `` ```math `` flavors, i.e. the - `gitlab` and `julia` delimiters) blew up on a `` ```math `` fence followed by a - long whitespace run with no closing fence. - - `admon`'s multi-tag title parser blew up on an unquoted header containing a long - interior whitespace run. - - Both patterns were rewritten to anchor the capture on non-whitespace at both ends, - which stays flat on the same input. A behavioral guard (`tests/test_redos.py`) now - renders adversarial payloads through every plugin to catch the next such pattern. - ## 0.7.0 - 2026-07-19 - ✨ NEW: Add section reference plugin (`section_ref`) (#144) From abe657b2e18ba1e2322ae9c5e0c13ad4c2b7b180 Mon Sep 17 00:00:00 2001 From: Chris Sewell <chrisj_sewell@hotmail.com> Date: Mon, 9 Nov 2026 21:16:02 +0100 Subject: [PATCH 09/11] =?UTF-8?q?=F0=9F=A7=AA=20TEST:=20steady=20the=20adm?= =?UTF-8?q?on=20ReDoS=20guard;=20check=20math=20tokens,=20not=20HTML?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit test_header_redos stopped retrying at its first slow sample, so one scheduler or GC pause failed it (seen at 38x on a loaded machine). It now times the 25k and 100k runs in alternation with the GC paused, keeps the fastest of each, stops once the ratio holds, and stops retrying after 15s of renders so a quadratic regression fails on the assertion rather than the timeout. test_fenced_math_block_keeps_indent asserted on the rendered HTML, but every math block renders with the same template, so the eqno case could never pass. It now checks the token's type, content and info. --- tests/test_admon.py | 37 +++++++++++++++++++++++++++---------- tests/test_texmath.py | 11 +++++++---- 2 files changed, 34 insertions(+), 14 deletions(-) diff --git a/tests/test_admon.py b/tests/test_admon.py index 4f611d5..20a7127 100644 --- a/tests/test_admon.py +++ b/tests/test_admon.py @@ -1,3 +1,4 @@ +import gc from pathlib import Path from textwrap import dedent import time @@ -37,16 +38,17 @@ def test_plugin_parse(data_regression, text_idx): data_regression.check([t.as_dict() for t in tokens]) -def _best_render_seconds(md: MarkdownIt, source: str, limit: float) -> float: - """Fastest of three renders, stopping early once one exceeds ``limit``.""" - best = float("inf") - for _ in range(3): +def _render_seconds(md: MarkdownIt, source: str) -> float: + """Time one render with the garbage collector paused.""" + gc_was_enabled = gc.isenabled() + gc.disable() + try: start = time.perf_counter() md.render(source) - best = min(best, time.perf_counter() - start) - if best > limit: - break - return best + return time.perf_counter() - start + finally: + if gc_was_enabled: + gc.enable() @pytest.mark.timeout(60) @@ -59,14 +61,29 @@ def test_header_redos(): so it does not depend on how fast the machine is: a run four times longer takes about 4x as long when parsing is linear, and about 16x when it is quadratic. + The two lengths are timed in alternation and the fastest of each kept, + stopping once the ratio holds, + so a single slow sample (a busy machine) cannot fail the test, + while a quadratic parse is slow on every sample. + Retries also stop once the renders have taken 15 seconds in all, + which a linear parse (milliseconds per round) never reaches, + so a regression fails on the assertion rather than the timeout. """ md = MarkdownIt().use(admon_plugin) def source(run: int) -> str: return "!!! note a" + " " * run + "b\n content" - small = _best_render_seconds(md, source(25_000), float("inf")) - large = _best_render_seconds(md, source(100_000), 8 * small) + small_source, large_source = source(25_000), source(100_000) + small = large = float("inf") + spent = 0.0 + for _ in range(5): + small_sample = _render_seconds(md, small_source) + large_sample = _render_seconds(md, large_source) + small, large = min(small, small_sample), min(large, large_sample) + spent += small_sample + large_sample + if large < 8 * small or spent > 15: + break assert large < 8 * small, ( f"25k run: {small:.4f}s, 100k run: {large:.4f}s ({large / small:.1f}x); " "the admon header parse is super-linear (ReDoS)" diff --git a/tests/test_texmath.py b/tests/test_texmath.py index dfa1ae9..d22ad5b 100644 --- a/tests/test_texmath.py +++ b/tests/test_texmath.py @@ -119,8 +119,8 @@ def test_fenced_math_block(delimiters): @pytest.mark.parametrize( "source,expected", [ - ("```math\n a = 1\n b = 2\n```\n", "<eqn> a = 1\n b = 2</eqn>"), - ("```math\n a = 1\n``` (1)\n", "<eqn> a = 1</eqn><span>(1)</span>"), + ("```math\n a = 1\n b = 2\n```\n", ("math_block", " a = 1\n b = 2")), + ("```math\n a = 1\n``` (1)\n", ("math_block_eqno", " a = 1", "1")), ], ids=["math_block", "math_block_eqno"], ) @@ -129,10 +129,13 @@ def test_fenced_math_block_keeps_indent(delimiters, source, expected): One whitespace character (the newline after the opening fence) separates the fence from the body; a pattern that skips all leading whitespace - would render ``<eqn>a = 1`` instead. + would give the content ``a = 1`` instead. + The tokens are checked rather than the HTML, + because every math block renders with the same template. """ md = MarkdownIt("commonmark").use(texmath_plugin, delimiters=delimiters) - assert expected in md.render(source) + [token] = md.parse(source) + assert (token.type, token.content, token.info)[: len(expected)] == expected @pytest.mark.parametrize("delimiters", ["gitlab", "julia"]) From 43db7ef7bb4368e8a3ef94e43062008046731d7b Mon Sep 17 00:00:00 2001 From: Chris Sewell <chrisj_sewell@hotmail.com> Date: Mon, 9 Nov 2026 21:46:20 +0100 Subject: [PATCH 10/11] =?UTF-8?q?=F0=9F=90=9B=20FIX:=20admon:=20match=20a?= =?UTF-8?q?=20title=20closed=20by=20a=20run=20of=20quotes=20in=20linear=20?= =?UTF-8?q?time?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The title tail "(?P<title>.*)"\S*$ tried every quote in a run as the closing one, and scanned the rest of the run for each, so a header such as `!!! note "` followed by tens of thousands of quotes took seconds. Only the last quote can close the title, so "[^\s"]*$ matches exactly what it did, in linear time. test_header_redos now also times a quote run (10k and 40k), and retries up to 20 rounds rather than 5, so a heavily loaded machine does not fail it; the 15 s retry cap still bounds a quadratic parse. --- mdit_py_plugins/admon/index.py | 5 ++++- tests/test_admon.py | 30 ++++++++++++++++++++---------- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/mdit_py_plugins/admon/index.py b/mdit_py_plugins/admon/index.py index e54c1ae..8d58c22 100644 --- a/mdit_py_plugins/admon/index.py +++ b/mdit_py_plugins/admon/index.py @@ -20,8 +20,11 @@ def _get_multiple_tags(params: str) -> tuple[list[str], str]: """Check for multiple tags when the title is double quoted.""" + # This matches what r'^\s*(?P<tokens>[^"]+)\s+"(?P<title>.*)"\S*$' matches, + # but the tokens end on a non-space and only the last quote can close the title, + # so a long run of spaces or quotes cannot make it backtrack quadratically (#143). re_tags = re.compile( - r'^\s*(?P<tokens>[^"\s](?:[^"]*[^"\s])?)\s+"(?P<title>.*)"\S*$' + r'^\s*(?P<tokens>[^"\s](?:[^"]*[^"\s])?)\s+"(?P<title>.*)"[^\s"]*$' ) match = re_tags.match(params) if match: diff --git a/tests/test_admon.py b/tests/test_admon.py index 20a7127..1235fa8 100644 --- a/tests/test_admon.py +++ b/tests/test_admon.py @@ -52,32 +52,42 @@ def _render_seconds(md: MarkdownIt, source: str) -> float: @pytest.mark.timeout(60) -def test_header_redos(): - """A multi-tag header with a long unquoted space run parses in linear time. +@pytest.mark.parametrize( + ("header", "filler", "small_run"), + [ + # tokens + space run: ``[^"]+\s+"`` tried every split of the run + pytest.param("note a{run}b", " ", 25_000, id="space-run"), + # quote run: ``"(?P<title>.*)"\S*$`` tried every quote as the closing one + pytest.param('note "{run} y', '"', 10_000, id="quote-run"), + ], +) +def test_header_redos(header, filler, small_run): + """A multi-tag header with a long run of spaces or quotes parses in linear time. - The title pattern ``[^"]+\\s+"`` backtracked quadratically on this input (#143), - taking seconds at 100k characters. + The title pattern backtracked quadratically on these inputs (#143), + taking seconds at tens of thousands of characters. The guard compares two run lengths instead of asserting a time budget, so it does not depend on how fast the machine is: a run four times longer takes about 4x as long when parsing is linear, and about 16x when it is quadratic. The two lengths are timed in alternation and the fastest of each kept, stopping once the ratio holds, - so a single slow sample (a busy machine) cannot fail the test, + so a single slow sample (a busy machine) does not fail the test, while a quadratic parse is slow on every sample. Retries also stop once the renders have taken 15 seconds in all, which a linear parse (milliseconds per round) never reaches, so a regression fails on the assertion rather than the timeout. + The quote run is shorter, because each quote costs a quadratic parse more. """ md = MarkdownIt().use(admon_plugin) def source(run: int) -> str: - return "!!! note a" + " " * run + "b\n content" + return "!!! " + header.format(run=filler * run) + "\n content" - small_source, large_source = source(25_000), source(100_000) + small_source, large_source = source(small_run), source(4 * small_run) small = large = float("inf") spent = 0.0 - for _ in range(5): + for _ in range(20): small_sample = _render_seconds(md, small_source) large_sample = _render_seconds(md, large_source) small, large = min(small, small_sample), min(large, large_sample) @@ -85,6 +95,6 @@ def source(run: int) -> str: if large < 8 * small or spent > 15: break assert large < 8 * small, ( - f"25k run: {small:.4f}s, 100k run: {large:.4f}s ({large / small:.1f}x); " - "the admon header parse is super-linear (ReDoS)" + f"{small_run} run: {small:.4f}s, {4 * small_run} run: {large:.4f}s " + f"({large / small:.1f}x); the admon header parse is super-linear (ReDoS)" ) From 96e0b75e7010bf478dd7c329e56c5a59d328e58e Mon Sep 17 00:00:00 2001 From: Chris Sewell <chrisj_sewell@hotmail.com> Date: Mon, 9 Nov 2026 21:46:20 +0100 Subject: [PATCH 11/11] =?UTF-8?q?=F0=9F=90=9B=20FIX:=20texmath:=20keep=20a?= =?UTF-8?q?=20whitespace-only=20```math=20body=20a=20math=20block?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The linear ```math patterns needed a non-whitespace character in the body, so a body that is only whitespace fell through to the code-fence rule. With an equation number the closer ``` (1) cannot close a code fence, so the fence then ran to the end of the document. The |\s branch captures the single whitespace character the previous pattern captured there, so the output is unchanged and matching stays linear. --- mdit_py_plugins/texmath/index.py | 13 +++++++++---- tests/test_texmath.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/mdit_py_plugins/texmath/index.py b/mdit_py_plugins/texmath/index.py index 3d4a1a4..8b47df7 100644 --- a/mdit_py_plugins/texmath/index.py +++ b/mdit_py_plugins/texmath/index.py @@ -188,6 +188,11 @@ def render(tex: str, displayMode: bool, macros: Any) -> str: # https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/RegExp/sticky +# The ```math block patterns capture what markdown-it-texmath's r"\s+?([^`]+?)\s+?`{3}" +# captures, but the body must start after one whitespace character and end on a +# non-whitespace one, so a long whitespace run with no closing fence is matched in +# linear time (#143); the |\s branch is the single character that pattern captures +# from a body that is only whitespace. rules: dict[str, dict[str, list[RuleDictType]]] = { "brackets": { "inline": [ @@ -228,7 +233,7 @@ def render(tex: str, displayMode: bool, macros: Any) -> str: { "name": "math_block_eqno", "rex": re.compile( - r"^`{3}math\s(\s*[^`\s](?:[^`]*[^`\s])?)\s+`{3}\s*?\(([^)$\r\n]+?)\)", + r"^`{3}math\s(\s*[^`\s](?:[^`]*[^`\s])?|\s)\s+`{3}\s*?\(([^)$\r\n]+?)\)", re.M, ), "tmpl": '<section class="eqno">\n<eqn>{0}</eqn><span>({1})</span>\n</section>\n', @@ -237,7 +242,7 @@ def render(tex: str, displayMode: bool, macros: Any) -> str: { "name": "math_block", "rex": re.compile( - r"^`{3}math\s(\s*[^`\s](?:[^`]*[^`\s])?)\s+`{3}", re.M + r"^`{3}math\s(\s*[^`\s](?:[^`]*[^`\s])?|\s)\s+`{3}", re.M ), "tmpl": "<section>\n<eqn>{0}</eqn>\n</section>\n", "tag": "```math", @@ -273,7 +278,7 @@ def render(tex: str, displayMode: bool, macros: Any) -> str: { "name": "math_block_eqno", "rex": re.compile( - r"^`{3}math\s(\s*[^`\s](?:[^`]*[^`\s])?)\s+`{3}\s*?\(([^)$\r\n]+?)\)", + r"^`{3}math\s(\s*[^`\s](?:[^`]*[^`\s])?|\s)\s+`{3}\s*?\(([^)$\r\n]+?)\)", re.M, ), "tmpl": '<section class="eqno"><eqn>{0}</eqn><span>({1})</span></section>', @@ -282,7 +287,7 @@ def render(tex: str, displayMode: bool, macros: Any) -> str: { "name": "math_block", "rex": re.compile( - r"^`{3}math\s(\s*[^`\s](?:[^`]*[^`\s])?)\s+`{3}", re.M + r"^`{3}math\s(\s*[^`\s](?:[^`]*[^`\s])?|\s)\s+`{3}", re.M ), "tmpl": "<section><eqn>{0}</eqn></section>", "tag": "```math", diff --git a/tests/test_texmath.py b/tests/test_texmath.py index d22ad5b..4125a4a 100644 --- a/tests/test_texmath.py +++ b/tests/test_texmath.py @@ -138,6 +138,35 @@ def test_fenced_math_block_keeps_indent(delimiters, source, expected): assert (token.type, token.content, token.info)[: len(expected)] == expected +@pytest.mark.parametrize("delimiters", ["gitlab", "julia"]) +@pytest.mark.parametrize( + "source,expected", + [ + ("```math\n \n```\n\nafter\n", ("math_block", " ")), + ("```math\n \n``` (1)\n\nafter\n", ("math_block_eqno", " ", "1")), + ], + ids=["math_block", "math_block_eqno"], +) +def test_fenced_math_block_whitespace_only(delimiters, source, expected): + """A body that is only whitespace is still a math block, as it was before #143. + + As a code fence instead, the eqno form would run to the end of the document, + because ``` ``` (1) ``` cannot close a code fence, + so the paragraph after it would become code. + """ + md = MarkdownIt("commonmark").use(texmath_plugin, delimiters=delimiters) + tokens = md.parse(source) + assert [t.type for t in tokens] == [ + expected[0], + "paragraph_open", + "inline", + "paragraph_close", + ] + assert (tokens[0].type, tokens[0].content, tokens[0].info)[: len(expected)] == ( + expected + ) + + @pytest.mark.parametrize("delimiters", ["gitlab", "julia"]) @pytest.mark.timeout(2) def test_fenced_math_block_redos(delimiters):