Preserve footnote block content; make e2e opt-in

- Render footnote definition children through the block renderer so block
  content (e.g. a list) is preserved instead of dropped; verified in the
  browser that Substack renders it as a real nested list. Adds a regression
  test and a footnote-with-list case to the e2e fixture.
- The live e2e test now requires RUN_SUBSTACK_E2E=1 in addition to
  credentials, so a normal pytest run with a .env does not create/delete
  real drafts by surprise.

Author: MattFisher

substack/mdrender.py

@@ -148,12 +148,13 @@ def _render_block(node: SyntaxTreeNode, api) -> List[Dict]:
         out = []
         for fn in node.children:
             number = fn.meta["id"] + 1
-            paras = [
-                nodes.paragraph(_render_inline(child.children[0], []))
-                for child in fn.children
-                if child.type == "paragraph"
-            ]
-            out.append(nodes.footnote(number, paras))
+            # Render every child block (paragraphs, lists, code, ...) so footnote
+            # content is preserved. The trailing footnote_anchor backref renders to
+            # nothing and is dropped.
+            children_nodes = []
+            for child in fn.children:
+                children_nodes.extend(_render_block(child, api))
+            out.append(nodes.footnote(number, children_nodes))
         return out
 
     return []

tests/substack/fixtures/full_features.expected.json

@@ -199,6 +199,21 @@
       }
     ]
   },
+  {
+    "type": "paragraph",
+    "content": [
+      {
+        "type": "text",
+        "text": "A footnote whose definition is a list."
+      },
+      {
+        "type": "footnoteAnchor",
+        "attrs": {
+          "number": 3
+        }
+      }
+    ]
+  },
   {
     "type": "bullet_list",
     "content": [
@@ -749,5 +764,46 @@
         ]
       }
     ]
+  },
+  {
+    "type": "footnote",
+    "attrs": {
+      "number": 3
+    },
+    "content": [
+      {
+        "type": "bullet_list",
+        "content": [
+          {
+            "type": "list_item",
+            "content": [
+              {
+                "type": "paragraph",
+                "content": [
+                  {
+                    "type": "text",
+                    "text": "list item one"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "type": "list_item",
+            "content": [
+              {
+                "type": "paragraph",
+                "content": [
+                  {
+                    "type": "text",
+                    "text": "list item two"
+                  }
+                ]
+              }
+            ]
+          }
+        ]
+      }
+    ]
   }
 ]

tests/substack/fixtures/full_features.md

@@ -14,6 +14,8 @@ A paragraph with **bold**, *italic*, ***bold italic***, `inline code`, ~~striket
 
 A sentence with a numeric footnote[^1] and a named footnote[^note], plus a repeat of the first.[^1]
 
+A footnote whose definition is a list.[^listnote]
+
 - Bullet one
 - Bullet two with **bold**
 
@@ -71,3 +73,5 @@ An autolink to <https://example.com> becomes a link.
 
     And a second paragraph after a blank line.
 [^unused]: This definition is never referenced and should be dropped.
+[^listnote]: - list item one
+    - list item two

tests/substack/test_end_to_end.py

@@ -4,14 +4,15 @@
 retrieves it, and compares the (normalised) stored document against a saved
 golden file (``full_features.expected.json``).
 
-Requires live credentials in the environment (a ``.env`` file is loaded):
+This test creates and deletes a real draft, so it is opt-in: it only runs when
+``RUN_SUBSTACK_E2E`` is set **and** credentials are configured (a ``.env`` file
+is loaded). A normal ``pytest`` run with a ``.env`` present will skip it rather
+than touching the account.
 
+  - ``RUN_SUBSTACK_E2E=1`` to enable
   - ``COOKIES_STRING`` (or ``COOKIES_PATH``), **or** ``EMAIL`` + ``PASSWORD``
   - ``PUBLICATION_URL`` (optional but recommended)
 
-The test is skipped when no credentials are configured, so it is safe to run in
-CI without secrets.
-
 To regenerate the golden file after an intentional change::
 
     python -m tests.substack.test_end_to_end --generate
@@ -60,14 +61,23 @@ def walk(node):
 
 
 def _has_credentials() -> bool:
-    """Whether auth is configured, checked without any network calls."""
     return bool(
         os.getenv("COOKIES_STRING")
         or os.getenv("COOKIES_PATH")
         or (os.getenv("EMAIL") and os.getenv("PASSWORD"))
     )
 
 
+def _e2e_enabled() -> bool:
+    """Only run when explicitly opted in AND credentials are present.
+
+    Creating/deleting real drafts should never happen by surprise on a local
+    ``pytest`` run that merely has a ``.env`` file, so an explicit
+    ``RUN_SUBSTACK_E2E`` opt-in is required.
+    """
+    return bool(os.getenv("RUN_SUBSTACK_E2E")) and _has_credentials()
+
+
 def _api_from_env() -> Api:
     cookies_string = os.getenv("COOKIES_STRING")
     cookies_path = os.getenv("COOKIES_PATH")
@@ -141,7 +151,10 @@ def _roundtrip(api: Api):
             pass
 
 
-@pytest.mark.skipif(not _has_credentials(), reason="no Substack credentials configured")
+@pytest.mark.skipif(
+    not _e2e_enabled(),
+    reason="set RUN_SUBSTACK_E2E=1 and configure credentials to run the live e2e test",
+)
 def test_full_features_roundtrip():
     api = _api_from_env()
     content = _roundtrip(api)

tests/substack/test_footnotes.py

@@ -243,6 +243,22 @@ def test_footnote_helper_splits_paragraphs(self):
         assert len(block["content"]) == 2
         assert block["content"][1]["content"][0]["text"] == "Para two."
 
+    def test_definition_with_block_content_is_preserved(self):
+        # A footnote whose definition is a list (or other block content) must keep
+        # its text rather than dropping it.
+        post = make_post()
+        post.from_markdown("Claim[^1]\n\n[^1]: - item one\n    - item two")
+        note = footnotes(post)[0]
+        text = " ".join(
+            n.get("text", "")
+            for para in find_nodes(note, "paragraph")
+            for n in para.get("content", [])
+        )
+        assert "item one" in text
+        assert "item two" in text
+        # rendered as a real nested list inside the footnote
+        assert find_nodes(note, "bullet_list")
+
     def test_no_footnotes_is_unchanged(self):
         post = make_post()
         post.from_markdown("Just a plain paragraph.")