📚 DOCS: freeze the section_ref token shape and make its grammar normative

mdit_py_plugins/section_ref/index.py

@@ -41,29 +41,50 @@
 def section_ref_plugin(md: MarkdownIt) -> None:
     """Parse section-sign references such as ``§1``, ``§1.1`` and ``§2.3.4``.
 
-    A reference is a ``§`` immediately followed by ASCII digits, optionally
-    grouped into further ``.``-separated levels (no spaces are allowed).
-    A trailing dot is not consumed, so ``see §1.`` captures ``§1``.  A ``§``
-    directly followed by an ASCII letter (e.g. ``§1a``) is *not* a reference,
-    but a following non-ASCII letter does not block a match, so ``见§3章``
-    still captures ``§3``.  A reference with any level longer than nine
-    digits (e.g. ``§1.1234567890``) is not a reference either: the whole
-    text stays literal, rather than a shorter prefix being captured.
-    Leading zeros count towards the nine digits, so ``§0000000001`` stays
-    literal too.
-    Following CommonMark backslash-escape behaviour, ``\\§1`` stays literal
-    text, while ``\\\\§1`` renders a literal backslash followed by a live
-    reference.
-
-    Each reference becomes a ``section_ref`` token, with the matched text
-    (e.g. ``"§1.1"``) as its ``content``, ``markup`` set to ``§`` and
-    ``meta["numbers"]`` holding the parsed section number as a list of ints
-    (e.g. ``[1, 1]`` for ``§1.1``), so that downstream renderers can resolve
-    it to a heading target without re-parsing.  Leading zeros are normalized
-    (``§01.02`` gives ``[1, 2]``); the original text remains in ``content``.
+    The grammar is normative:
+
+    - A reference is ``§`` (U+00A7) immediately followed by one or more
+      ``.``-separated levels, with no spaces.
+      Each level is 1 to 9 ASCII digits, ``§([0-9]{1,9}(\\.[0-9]{1,9})*)``,
+      and the match is maximal.
+      Only ASCII ``0``-``9`` are digits: other Unicode digits, such as
+      Arabic-Indic (U+0660 to U+0669) or fullwidth (U+FF10 to U+FF19) digits,
+      do not form a reference, so a ``§`` followed by one stays literal text.
+    - A trailing ``.`` is not part of the reference,
+      so in ``see §1.`` the reference is ``§1`` and the ``.`` stays text.
+    - There is no reference, and the whole run stays literal text,
+      if the character after the match is an ASCII letter (``§1a``, ``§1.2b``),
+      or if any level has more than nine digits, counting leading zeros
+      (``§1234567890``, ``§0000000001``).
+      No shorter prefix is taken, so ``§1.1234567890.2`` stays literal too.
+      A non-ASCII letter after the match does not block it,
+      so ``见§3章`` captures ``§3``.
+    - References are recognised wherever inline text is parsed,
+      and not inside code spans, autolinks, inline math or raw HTML tags
+      (an attribute value is not inline text, but the text between two tags is,
+      so ``<span title="§1">§2</span>`` captures ``§2`` only).
+    - ``\\§`` is not a CommonMark backslash escape,
+      because ``§`` is not ASCII punctuation:
+      in ``\\§1`` the backslash stays visible and no reference is formed.
+      ``\\\\§1`` is a literal backslash followed by the reference ``§1``.
+      To write a literal sign, use ``&sect;`` or ``&#167;``,
+      which never form a reference (``&sect;1`` renders as the text ``§1``).
+
+    Each reference becomes a single ``section_ref`` leaf token
+    (``nesting=0``, ``children=None``, ``tag=""``, ``attrs={}``),
+    with ``markup`` set to ``"§"``, the text as written (e.g. ``"§01.02"``)
+    as its ``content``, and ``meta["numbers"]`` holding the integer value of
+    each level as a list of ints (``[1, 2]`` for ``§01.02``; every value is at
+    most 999,999,999), so that downstream renderers can resolve it to a
+    heading target without re-parsing.
+    The token shape is part of the public API from 0.7.0.
 
     The default rendering is ``<span class="section-ref">§1.1</span>``.
     Override it with ``md.add_render_rule("section_ref", ...)``.
+    markdown-it-py's ``renderInlineAsText``, which renders image ``alt`` text,
+    keeps only ``text`` tokens, so ``![a §7 b](x.png)`` renders ``alt="a  b"``
+    (as it does for inline math). A renderer that wants the reference in the
+    alt text should use the ``section_ref`` token's ``content``.
 
     .. versionadded:: 0.7.0
 

tests/fixtures/section_ref.md

@@ -83,6 +83,20 @@ Inside inline code (not captured):
 <p><code>§1</code></p>
 .
 
+Inside an autolink (not captured):
+.
+<https://example.com/§1>
+.
+<p><a href="https://example.com/%C2%A71">https://example.com/§1</a></p>
+.
+
+Inside raw HTML (not captured), but captured in its content:
+.
+<span title="§1">§2</span>
+.
+<p><span title="§1"><span class="section-ref">§2</span></span></p>
+.
+
 Inside fenced code block (not captured):
 .
 ```
@@ -135,13 +149,27 @@ Escaped backslash before ref (captured):
 <p>\<span class="section-ref">§1</span></p>
 .
 
+Entity section signs give a literal sign (not captured):
+.
+&sect;1 and &#167;2
+.
+<p>§1 and §2</p>
+.
+
 Followed by non-digit, non-space char (not captured):
 .
 §x and §.5
 .
 <p>§x and §.5</p>
 .
 
+Non-ASCII digits (not captured):
+.
+§١ and §３
+.
+<p>§١ and §３</p>
+.
+
 Leading zeros preserved in content (meta numbers are normalized):
 .
 §01.02

tests/test_section_ref.py

@@ -112,6 +112,27 @@ def test_token():
     ]
 
 
+def test_token_shape_is_public_api():
+    """Every field of the ``section_ref`` token, which is public API from 0.7.0.
+
+    Downstream renderers (MyST-Parser) and ports of the grammar (ubCode) read
+    these fields directly, so changing any of them is a breaking change.
+    """
+    md = MarkdownIt().use(section_ref_plugin)
+    children = md.parseInline("§01.02")[0].children
+    assert children is not None
+    assert len(children) == 1
+    token = children[0]
+    assert token.type == "section_ref"
+    assert token.nesting == 0
+    assert token.children is None
+    assert token.tag == ""
+    assert token.attrs == {}
+    assert token.markup == "§"
+    assert token.content == "§01.02"
+    assert token.meta == {"numbers": [1, 2]}
+
+
 def test_runtime_error():
     """The plugin requires the ``add_terminator_char`` API (markdown-it-py >= 4.1.0)."""
     md = MarkdownIt()