Skip to content

Commit 86a4408

Browse files
committed
Fix footnote numbering for repeated references
Substack numbers footnote anchors by document position and pairs them 1:1 with footnote blocks in order (ignoring any explicit number and not supporting one block for multiple anchors). Previously a repeated reference emitted two anchors but only one block, so every footnote after the repeat was misnumbered and its backlink pointed at the wrong (or a missing) block. Now each reference emits its own sequentially-numbered anchor with a matching block; a definition referenced more than once is duplicated, mirroring how Substack's editor behaves. Verified in-browser that anchors 1..N each resolve to their block. Updates the repeated-reference test and the e2e golden.
1 parent 57e95d6 commit 86a4408

3 files changed

Lines changed: 106 additions & 45 deletions

File tree

‎substack/mdrender.py‎

Lines changed: 53 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,18 @@
66
77
Node construction goes through ``substack.nodes`` so the (undocumented) schema
88
lives in exactly one place.
9+
10+
Footnotes: Substack numbers footnote anchors by their position in the document
11+
and pairs them one-to-one, in order, with the footnote blocks at the end (it
12+
ignores any explicit number and does not support one block serving several
13+
anchors). So each reference is emitted as its own sequentially-numbered anchor,
14+
and a matching footnote block is appended for each -- a definition referenced
15+
more than once is duplicated, which mirrors how Substack's own editor behaves.
916
"""
1017

1118
from __future__ import annotations
1219

20+
import copy
1321
from typing import Dict, List, Optional
1422

1523
from markdown_it import MarkdownIt
@@ -46,7 +54,7 @@ def _coalesce(out_nodes: List[Dict]) -> List[Dict]:
4654
return merged
4755

4856

49-
def _render_inline(node: SyntaxTreeNode, marks: List[Dict]) -> List[Dict]:
57+
def _render_inline(node: SyntaxTreeNode, marks: List[Dict], ctx: Dict) -> List[Dict]:
5058
"""Render an inline subtree into a flat list of text / anchor nodes."""
5159
out: List[Dict] = []
5260
for child in node.children:
@@ -57,14 +65,17 @@ def _render_inline(node: SyntaxTreeNode, marks: List[Dict]) -> List[Dict]:
5765
elif t == "code_inline":
5866
out.append(nodes.text(child.content, marks + [nodes.code_mark()]))
5967
elif t in _MARK_FOR:
60-
out.extend(_render_inline(child, marks + [_MARK_FOR[t]]))
68+
out.extend(_render_inline(child, marks + [_MARK_FOR[t]], ctx))
6169
elif t == "link":
6270
href = child.attrs.get("href", "")
63-
out.extend(_render_inline(child, marks + [nodes.link_mark(href)]))
71+
out.extend(_render_inline(child, marks + [nodes.link_mark(href)], ctx))
6472
elif t in ("softbreak", "hardbreak"):
6573
out.append(nodes.text(" ", marks))
6674
elif t == "footnote_ref":
67-
out.append(nodes.footnote_anchor(child.meta["id"] + 1))
75+
# Number anchors by document position and record which definition each
76+
# one points to, so matching blocks can be emitted 1:1 afterwards.
77+
ctx["order"].append(child.meta["id"])
78+
out.append(nodes.footnote_anchor(len(ctx["order"])))
6879
elif t == "image":
6980
# Inline images are rare in this schema; fall back to alt text.
7081
alt = child.attrs.get("alt") or "".join(
@@ -107,7 +118,7 @@ def _captioned_image(img: SyntaxTreeNode, api) -> Dict:
107118
)
108119

109120

110-
def _render_block(node: SyntaxTreeNode, api) -> List[Dict]:
121+
def _render_block(node: SyntaxTreeNode, api, ctx: Dict) -> List[Dict]:
111122
"""Render a block-level node into zero or more Substack nodes."""
112123
t = node.type
113124

@@ -116,11 +127,11 @@ def _render_block(node: SyntaxTreeNode, api) -> List[Dict]:
116127
img = _only_image(inline)
117128
if img is not None:
118129
return [_captioned_image(img, api)]
119-
return [nodes.paragraph(_render_inline(inline, []))]
130+
return [nodes.paragraph(_render_inline(inline, [], ctx))]
120131

121132
if t == "heading":
122133
level = int(node.tag[1])
123-
return [nodes.heading(_render_inline(node.children[0], []), level=level)]
134+
return [nodes.heading(_render_inline(node.children[0], [], ctx), level=level)]
124135

125136
if t == "hr":
126137
return [nodes.horizontal_rule()]
@@ -135,47 +146,61 @@ def _render_block(node: SyntaxTreeNode, api) -> List[Dict]:
135146
if t == "blockquote":
136147
paras: List[Dict] = []
137148
for child in node.children:
138-
paras.extend(_render_block(child, api))
149+
paras.extend(_render_block(child, api, ctx))
139150
return [nodes.blockquote(paras)]
140151

141152
if t == "bullet_list":
142-
return [nodes.bullet_list(_render_list_items(node, api))]
153+
return [nodes.bullet_list(_render_list_items(node, api, ctx))]
143154

144155
if t == "ordered_list":
145-
return [nodes.ordered_list(_render_list_items(node, api))]
146-
147-
if t == "footnote_block":
148-
out = []
149-
for fn in node.children:
150-
number = fn.meta["id"] + 1
151-
# Render every child block (paragraphs, lists, code, ...) so footnote
152-
# content is preserved. The trailing footnote_anchor backref renders to
153-
# nothing and is dropped.
154-
children_nodes = []
155-
for child in fn.children:
156-
children_nodes.extend(_render_block(child, api))
157-
out.append(nodes.footnote(number, children_nodes))
158-
return out
156+
return [nodes.ordered_list(_render_list_items(node, api, ctx))]
159157

158+
# footnote_block is handled separately in markdown_to_doc; ignore it here.
160159
return []
161160

162161

163-
def _render_list_items(list_node: SyntaxTreeNode, api) -> List[Dict]:
162+
def _render_list_items(list_node: SyntaxTreeNode, api, ctx: Dict) -> List[Dict]:
164163
items = []
165164
for li in list_node.children:
166-
# A list_item built by nodes.list_item wraps inline content in a single
167-
# paragraph; here items may already contain block nodes, so build directly.
168165
content: List[Dict] = []
169166
for child in li.children:
170-
content.extend(_render_block(child, api))
167+
content.extend(_render_block(child, api, ctx))
171168
items.append({"type": NodeType.LIST_ITEM, "content": content})
172169
return items
173170

174171

172+
def _footnote_definitions(tree: SyntaxTreeNode, api) -> Dict[int, List[Dict]]:
173+
"""Map each footnote id to its rendered block content."""
174+
definitions: Dict[int, List[Dict]] = {}
175+
for node in tree.children:
176+
if node.type != "footnote_block":
177+
continue
178+
for fn in node.children:
179+
# A footnote's own content should not register anchors of its own.
180+
local_ctx = {"order": []}
181+
content: List[Dict] = []
182+
for child in fn.children:
183+
content.extend(_render_block(child, api, local_ctx))
184+
definitions[fn.meta["id"]] = content
185+
return definitions
186+
187+
175188
def markdown_to_doc(markdown_content: str, api=None) -> List[Dict]:
176189
"""Convert Markdown into a list of Substack ProseMirror block nodes."""
177190
tree = SyntaxTreeNode(_make_parser().parse(markdown_content))
191+
192+
definitions = _footnote_definitions(tree, api)
193+
194+
ctx: Dict = {"order": []}
178195
out: List[Dict] = []
179196
for node in tree.children:
180-
out.extend(_render_block(node, api))
197+
if node.type == "footnote_block":
198+
continue
199+
out.extend(_render_block(node, api, ctx))
200+
201+
# Emit one footnote block per reference, in anchor order, numbered to match.
202+
for number, footnote_id in enumerate(ctx["order"], start=1):
203+
content = copy.deepcopy(definitions.get(footnote_id, []))
204+
out.append(nodes.footnote(number, content))
205+
181206
return out

‎tests/substack/fixtures/full_features.expected.json‎

Lines changed: 35 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -194,7 +194,7 @@
194194
{
195195
"type": "footnoteAnchor",
196196
"attrs": {
197-
"number": 1
197+
"number": 3
198198
}
199199
}
200200
]
@@ -209,7 +209,7 @@
209209
{
210210
"type": "footnoteAnchor",
211211
"attrs": {
212-
"number": 3
212+
"number": 4
213213
}
214214
}
215215
]
@@ -770,6 +770,39 @@
770770
"attrs": {
771771
"number": 3
772772
},
773+
"content": [
774+
{
775+
"type": "paragraph",
776+
"content": [
777+
{
778+
"type": "text",
779+
"text": "The first footnote, with a "
780+
},
781+
{
782+
"type": "text",
783+
"text": "link",
784+
"marks": [
785+
{
786+
"type": "link",
787+
"attrs": {
788+
"href": "https://example.com"
789+
}
790+
}
791+
]
792+
},
793+
{
794+
"type": "text",
795+
"text": "."
796+
}
797+
]
798+
}
799+
]
800+
},
801+
{
802+
"type": "footnote",
803+
"attrs": {
804+
"number": 4
805+
},
773806
"content": [
774807
{
775808
"type": "bullet_list",

‎tests/substack/test_footnotes.py‎

Lines changed: 18 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
# Helpers
88
# ---------------------------------------------------------------------------
99

10+
1011
def make_post():
1112
"""Create a fresh Post instance for testing."""
1213
return Post(title="Test", subtitle="Sub", user_id=1)
@@ -44,6 +45,7 @@ def footnotes(post):
4445
# TestFootnoteHelpers
4546
# ---------------------------------------------------------------------------
4647

48+
4749
class TestFootnoteHelpers:
4850
def test_footnote_anchor_added_inline(self):
4951
post = make_post()
@@ -68,13 +70,16 @@ def test_footnote_block_parses_inline_markdown(self):
6870
text_nodes = block["content"][0]["content"]
6971
link_node = next(n for n in text_nodes if n.get("marks"))
7072
assert link_node["text"] == "the source"
71-
assert link_node["marks"] == [{"type": "link", "attrs": {"href": "https://example.com"}}]
73+
assert link_node["marks"] == [
74+
{"type": "link", "attrs": {"href": "https://example.com"}}
75+
]
7276

7377

7478
# ---------------------------------------------------------------------------
7579
# TestFromMarkdownFootnotes
7680
# ---------------------------------------------------------------------------
7781

82+
7883
class TestFromMarkdownFootnotes:
7984
def test_basic_reference_and_definition(self):
8085
post = make_post()
@@ -92,9 +97,7 @@ def test_definition_removed_from_body(self):
9297
# The definition line must not leak into a paragraph.
9398
paragraphs = find_nodes(post.draft_body, "paragraph")
9499
body_text = " ".join(
95-
n.get("text", "")
96-
for p in paragraphs
97-
for n in p.get("content", [])
100+
n.get("text", "") for p in paragraphs for n in p.get("content", [])
98101
)
99102
assert "[^1]:" not in body_text
100103

@@ -121,20 +124,24 @@ def test_named_labels_numbered_by_first_appearance(self):
121124
assert blocks[0]["content"][0]["content"][0]["text"] == "First definition."
122125
assert blocks[1]["content"][0]["content"][0]["text"] == "Second definition."
123126

124-
def test_repeated_reference_reuses_number(self):
127+
def test_repeated_reference_duplicates_footnote(self):
128+
# Substack numbers anchors by position and pairs them 1:1 with footnote
129+
# blocks, so a definition referenced twice yields two sequentially-numbered
130+
# anchors and two footnote blocks with identical content.
125131
post = make_post()
126132
post.from_markdown("One[^a] two[^a].\n\n[^a]: Note.")
127133
nums = [a["attrs"]["number"] for a in anchors(post)]
128-
assert nums == [1, 1]
129-
assert len(footnotes(post)) == 1
134+
assert nums == [1, 2]
135+
blocks = footnotes(post)
136+
assert [b["attrs"]["number"] for b in blocks] == [1, 2]
137+
assert blocks[0]["content"] == blocks[1]["content"]
138+
assert blocks[0]["content"][0]["content"][0]["text"] == "Note."
130139

131140
def test_link_inside_definition_preserved(self):
132141
post = make_post()
133142
post.from_markdown("Claim.[^1]\n\n[^1]: See [docs](https://example.com).")
134143
block = footnotes(post)[0]
135-
link_node = next(
136-
n for n in block["content"][0]["content"] if n.get("marks")
137-
)
144+
link_node = next(n for n in block["content"][0]["content"] if n.get("marks"))
138145
assert link_node["marks"][0]["attrs"]["href"] == "https://example.com"
139146

140147
def test_multiline_definition(self):
@@ -167,11 +174,7 @@ def test_reference_without_definition_left_as_text(self):
167174

168175
def test_definition_in_middle_moves_to_end(self):
169176
post = make_post()
170-
md = (
171-
"First paragraph.[^1]\n\n"
172-
"[^1]: First footnote.\n\n"
173-
"Second paragraph."
174-
)
177+
md = "First paragraph.[^1]\n\n[^1]: First footnote.\n\nSecond paragraph."
175178
post.from_markdown(md)
176179

177180
types = [node["type"] for node in body_content(post)]

0 commit comments

Comments
 (0)