diff --git a/mdit_py_plugins/admon/index.py b/mdit_py_plugins/admon/index.py index 6e28940..8d58c22 100644 --- a/mdit_py_plugins/admon/index.py +++ b/mdit_py_plugins/admon/index.py @@ -20,7 +20,12 @@ def _get_multiple_tags(params: str) -> tuple[list[str], str]: """Check for multiple tags when the title is double quoted.""" - re_tags = re.compile(r'^\s*(?P[^"]+)\s+"(?P.*)"\S*$') + # This matches what r'^\s*(?P<tokens>[^"]+)\s+"(?P<title>.*)"\S*$' matches, + # but the tokens end on a non-space and only the last quote can close the title, + # so a long run of spaces or quotes cannot make it backtrack quadratically (#143). + re_tags = re.compile( + r'^\s*(?P<tokens>[^"\s](?:[^"]*[^"\s])?)\s+"(?P<title>.*)"[^\s"]*$' + ) match = re_tags.match(params) if match: tags = match["tokens"].strip().split(" ") diff --git a/mdit_py_plugins/texmath/index.py b/mdit_py_plugins/texmath/index.py index 5f54f91..58a1bb7 100644 --- a/mdit_py_plugins/texmath/index.py +++ b/mdit_py_plugins/texmath/index.py @@ -191,6 +191,11 @@ def render(tex: str, displayMode: bool, macros: Any) -> str: # https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/RegExp/sticky +# The ```math block patterns capture what markdown-it-texmath's r"\s+?([^`]+?)\s+?`{3}" +# captures, but the body must start after one whitespace character and end on a +# non-whitespace one, so a long whitespace run with no closing fence is matched in +# linear time (#143); the |\s branch is the single character that pattern captures +# from a body that is only whitespace. rules: dict[str, dict[str, list[RuleDictType]]] = { "brackets": { "inline": [ @@ -231,14 +236,17 @@ def render(tex: str, displayMode: bool, macros: Any) -> str: { "name": "math_block_eqno", "rex": re.compile( - r"^`{3}math\s+?([^`]+?)\s+?`{3}\s*?\(([^)$\r\n]+?)\)", re.M + r"^`{3}math\s(\s*[^`\s](?:[^`]*[^`\s])?|\s)\s+`{3}\s*?\(([^)$\r\n]+?)\)", + re.M, ), "tmpl": '<section class="eqno">\n<eqn>{0}</eqn><span>({1})</span>\n</section>\n', "tag": "```math", }, { "name": "math_block", - "rex": re.compile(r"^`{3}math\s+?([^`]+?)\s+?`{3}", re.M), + "rex": re.compile( + r"^`{3}math\s(\s*[^`\s](?:[^`]*[^`\s])?|\s)\s+`{3}", re.M + ), "tmpl": "<section>\n<eqn>{0}</eqn>\n</section>\n", "tag": "```math", }, @@ -273,14 +281,17 @@ def render(tex: str, displayMode: bool, macros: Any) -> str: { "name": "math_block_eqno", "rex": re.compile( - r"^`{3}math\s+?([^`]+?)\s+?`{3}\s*?\(([^)$\r\n]+?)\)", re.M + r"^`{3}math\s(\s*[^`\s](?:[^`]*[^`\s])?|\s)\s+`{3}\s*?\(([^)$\r\n]+?)\)", + re.M, ), "tmpl": '<section class="eqno"><eqn>{0}</eqn><span>({1})</span></section>', "tag": "```math", }, { "name": "math_block", - "rex": re.compile(r"^`{3}math\s+?([^`]+?)\s+?`{3}", re.M), + "rex": re.compile( + r"^`{3}math\s(\s*[^`\s](?:[^`]*[^`\s])?|\s)\s+`{3}", re.M + ), "tmpl": "<section><eqn>{0}</eqn></section>", "tag": "```math", }, diff --git a/tests/test_admon.py b/tests/test_admon.py index f0ef479..1235fa8 100644 --- a/tests/test_admon.py +++ b/tests/test_admon.py @@ -1,5 +1,7 @@ +import gc from pathlib import Path from textwrap import dedent +import time from markdown_it import MarkdownIt from markdown_it.utils import read_fixture_file @@ -34,3 +36,65 @@ def test_plugin_parse(data_regression, text_idx): md = MarkdownIt().use(admon_plugin) tokens = md.parse(dedent(texts[text_idx])) data_regression.check([t.as_dict() for t in tokens]) + + +def _render_seconds(md: MarkdownIt, source: str) -> float: + """Time one render with the garbage collector paused.""" + gc_was_enabled = gc.isenabled() + gc.disable() + try: + start = time.perf_counter() + md.render(source) + return time.perf_counter() - start + finally: + if gc_was_enabled: + gc.enable() + + +@pytest.mark.timeout(60) +@pytest.mark.parametrize( + ("header", "filler", "small_run"), + [ + # tokens + space run: ``[^"]+\s+"`` tried every split of the run + pytest.param("note a{run}b", " ", 25_000, id="space-run"), + # quote run: ``"(?P<title>.*)"\S*$`` tried every quote as the closing one + pytest.param('note "{run} y', '"', 10_000, id="quote-run"), + ], +) +def test_header_redos(header, filler, small_run): + """A multi-tag header with a long run of spaces or quotes parses in linear time. + + The title pattern backtracked quadratically on these inputs (#143), + taking seconds at tens of thousands of characters. + The guard compares two run lengths instead of asserting a time budget, + so it does not depend on how fast the machine is: + a run four times longer takes about 4x as long when parsing is linear, + and about 16x when it is quadratic. + The two lengths are timed in alternation and the fastest of each kept, + stopping once the ratio holds, + so a single slow sample (a busy machine) does not fail the test, + while a quadratic parse is slow on every sample. + Retries also stop once the renders have taken 15 seconds in all, + which a linear parse (milliseconds per round) never reaches, + so a regression fails on the assertion rather than the timeout. + The quote run is shorter, because each quote costs a quadratic parse more. + """ + md = MarkdownIt().use(admon_plugin) + + def source(run: int) -> str: + return "!!! " + header.format(run=filler * run) + "\n content" + + small_source, large_source = source(small_run), source(4 * small_run) + small = large = float("inf") + spent = 0.0 + for _ in range(20): + small_sample = _render_seconds(md, small_source) + large_sample = _render_seconds(md, large_source) + small, large = min(small, small_sample), min(large, large_sample) + spent += small_sample + large_sample + if large < 8 * small or spent > 15: + break + assert large < 8 * small, ( + f"{small_run} run: {small:.4f}s, {4 * small_run} run: {large:.4f}s " + f"({large / small:.1f}x); the admon header parse is super-linear (ReDoS)" + ) diff --git a/tests/test_redos.py b/tests/test_redos.py new file mode 100644 index 0000000..cfc1320 --- /dev/null +++ b/tests/test_redos.py @@ -0,0 +1,164 @@ +"""Behavioral ReDoS guard: render adversarial markdown through each plugin. + +New plugins add regexes over time. Rather than trust review to catch catastrophic +backtracking, this drives each plugin end-to-end (``MarkdownIt().use(plugin)`` then +``md.render(payload)``) against adversarial input and asserts it stays fast. + +Two properties make this faithful to the real attack surface without introspecting +any pattern: + +- The payload embeds a long *homogeneous run* of a single character class, which is + what pushes overlapping or nested quantifiers (e.g. ``\\s+ [^x]+ \\s+``) into + super-linear backtracking. We fuzz every run character, so we never need to know + which regex is vulnerable or how it is structured. +- Rendering exercises the *whole* rule chain (the parser re-invoking a block rule + across offsets), so it also catches the O(n^2) amplification an isolated-regex + check would miss, and never false-positives on a pattern adversarial input can't + actually reach. + +The only per-plugin cost is one hand-written activation payload: the plugin's public +markdown syntax with a ``{run}`` placeholder for the homogeneous run. +""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass +import time + +from markdown_it import MarkdownIt +import pytest + +from mdit_py_plugins.admon import admon_plugin +from mdit_py_plugins.amsmath import amsmath_plugin +from mdit_py_plugins.anchors import anchors_plugin +from mdit_py_plugins.colon_fence import colon_fence_plugin +from mdit_py_plugins.deflist import deflist_plugin +from mdit_py_plugins.dollarmath import dollarmath_plugin +from mdit_py_plugins.field_list import fieldlist_plugin +from mdit_py_plugins.footnote import footnote_plugin +from mdit_py_plugins.myst_blocks import myst_block_plugin +from mdit_py_plugins.myst_role import myst_role_plugin +from mdit_py_plugins.tasklists import tasklists_plugin +from mdit_py_plugins.texmath import texmath_plugin + +# Homogeneous runs of one character class are what drive overlapping/nested +# quantifiers into catastrophic backtracking; fuzz every plugin with each +_FILLERS = [" ", "\t", "$", "`", "\\", '"', "-"] +_RUN = 15_000 +_BUDGET_SECONDS = 1.0 # Safe threshold relaxed for CI + + +@dataclass(frozen=True) +class Case: + id: str + configure: Callable[[MarkdownIt], None] + payload: Callable[[str], str] + + +_CASES = [ + Case( + # An interior run with no closing quote: ``[^"]+\s+"`` backtracks the run + # exhaustively looking for a ``"`` that never comes. A trailing run would be + # stripped off before the regex sees it, so the run must sit between tokens. + "admon-header", + lambda md: md.use(admon_plugin), + lambda run: f"!!! note a{run}b\n content", + ), + Case( + "amsmath-open", + lambda md: md.use(amsmath_plugin), + lambda run: f"\\begin{{{run}", + ), + Case( + "amsmath-unclosed", + lambda md: md.use(amsmath_plugin), + lambda run: f"\\begin{{equation}}\n{run}", + ), + Case( + "anchors-heading", + lambda md: md.use(anchors_plugin), + lambda run: f"# {run}", + ), + Case( + "colon-fence", + lambda md: md.use(colon_fence_plugin), + lambda run: f"::: {run}\ncontent\n:::", + ), + Case( + "deflist", + lambda md: md.use(deflist_plugin), + lambda run: f"term\n:{run}", + ), + Case( + "dollarmath-inline", + lambda md: md.use(dollarmath_plugin), + lambda run: f"${run}$", + ), + Case( + "dollarmath-block", + lambda md: md.use(dollarmath_plugin, allow_labels=True), + lambda run: f"$$\n{run}", + ), + Case( + "dollarmath-label", + lambda md: md.use(dollarmath_plugin, allow_labels=True), + lambda run: f"$$a$$ ({run})", + ), + Case( + "fieldlist", + lambda md: md.use(fieldlist_plugin), + lambda run: f":name:{run}", + ), + Case( + "footnote-ref", + lambda md: md.use(footnote_plugin), + lambda run: f"[^{run}]: body", + ), + Case( + "myst-block-target", + lambda md: md.use(myst_block_plugin), + lambda run: f"({run})=", + ), + Case( + "myst-role", + lambda md: md.use(myst_role_plugin), + lambda run: f"{{role}}`{run}`", + ), + Case( + "tasklist", + lambda md: md.use(tasklists_plugin), + lambda run: f"- [ ]{run}x", + ), + Case( + "texmath-fence", + lambda md: md.use(texmath_plugin, delimiters="gitlab"), + lambda run: f"```math\n{run}", + ), + Case( + "texmath-fence-eqno", + lambda md: md.use(texmath_plugin, delimiters="gitlab"), + lambda run: f"```math\n{run}``` (1)", + ), + Case( + "texmath-dollar", + lambda md: md.use(texmath_plugin, delimiters="dollars"), + lambda run: f"${run}$", + ), +] + + +@pytest.mark.timeout(10) +@pytest.mark.parametrize("case", _CASES, ids=lambda case: case.id) +def test_plugin_is_redos_safe(case: Case) -> None: + for filler in _FILLERS: + md = MarkdownIt("commonmark") + case.configure(md) + source = case.payload(filler * _RUN) + start = time.perf_counter() + md.render(source) + elapsed = time.perf_counter() - start + assert elapsed < _BUDGET_SECONDS, ( + f"{case.id} took {elapsed:.3f}s on a {_RUN}x{filler!r} run; " + "likely catastrophic backtracking (ReDoS)" + ) diff --git a/tests/test_texmath.py b/tests/test_texmath.py index 3739150..a625b6a 100644 --- a/tests/test_texmath.py +++ b/tests/test_texmath.py @@ -123,3 +123,70 @@ def test_bracket_fixtures(line, title, input, expected): text = md.render(input) print(text) assert text.rstrip() == expected.rstrip() + + +@pytest.mark.parametrize("delimiters", ["gitlab", "julia"]) +def test_fenced_math_block(delimiters): + md = MarkdownIt("commonmark").use(texmath_plugin, delimiters=delimiters) + text = md.render("```math\n\\alpha = 1\n```\n") + assert "<eqn>\\alpha = 1</eqn>" in text + + +@pytest.mark.parametrize("delimiters", ["gitlab", "julia"]) +@pytest.mark.parametrize( + "source,expected", + [ + ("```math\n a = 1\n b = 2\n```\n", ("math_block", " a = 1\n b = 2")), + ("```math\n a = 1\n``` (1)\n", ("math_block_eqno", " a = 1", "1")), + ], + ids=["math_block", "math_block_eqno"], +) +def test_fenced_math_block_keeps_indent(delimiters, source, expected): + """The ReDoS fix must not change output: the first line keeps its indentation. + + One whitespace character (the newline after the opening fence) separates + the fence from the body; a pattern that skips all leading whitespace + would give the content ``a = 1`` instead. + The tokens are checked rather than the HTML, + because every math block renders with the same template. + """ + md = MarkdownIt("commonmark").use(texmath_plugin, delimiters=delimiters) + [token] = md.parse(source) + assert (token.type, token.content, token.info)[: len(expected)] == expected + + +@pytest.mark.parametrize("delimiters", ["gitlab", "julia"]) +@pytest.mark.parametrize( + "source,expected", + [ + ("```math\n \n```\n\nafter\n", ("math_block", " ")), + ("```math\n \n``` (1)\n\nafter\n", ("math_block_eqno", " ", "1")), + ], + ids=["math_block", "math_block_eqno"], +) +def test_fenced_math_block_whitespace_only(delimiters, source, expected): + """A body that is only whitespace is still a math block, as it was before #143. + + As a code fence instead, the eqno form would run to the end of the document, + because ``` ``` (1) ``` cannot close a code fence, + so the paragraph after it would become code. + """ + md = MarkdownIt("commonmark").use(texmath_plugin, delimiters=delimiters) + tokens = md.parse(source) + assert [t.type for t in tokens] == [ + expected[0], + "paragraph_open", + "inline", + "paragraph_close", + ] + assert (tokens[0].type, tokens[0].content, tokens[0].info)[: len(expected)] == ( + expected + ) + + +@pytest.mark.parametrize("delimiters", ["gitlab", "julia"]) +@pytest.mark.timeout(2) +def test_fenced_math_block_redos(delimiters): + """An unclosed ``` ```math ``` fence must not trigger catastrophic backtracking.""" + md = MarkdownIt("commonmark").use(texmath_plugin, delimiters=delimiters) + md.render("```math\n" + " " * 20000)