Streamline character variants to one canonical form for reproducible JSON

The scrub decomposed some things (superscripts to digits) but left
typographic and symbol variants untouched, so the character set the
downstream LLM converts to JSON was inconsistent -- e.g. "datatype
folder(s) -> modality folder(s)" kept a unicode arrow while everything
around it was ASCII. Inconsistent input makes the LLM's JSON output
non-reproducible.

_normalize_chars collapses each variant family to a single form:
  - curly quotes -> straight, en/em/minus dashes -> "-", ellipsis -> "...",
    unicode spaces -> space, arrows -> "->"/"<-";
  - box-drawing and block characters are dropped -- they are the model
    rendering a folder tree as |-- lines, structure not content;
  - NFKC folds remaining compatibility duplicates (micro sign U+00B5 vs
    greek mu U+03BC, any unicode super/subscript digits) to one encoding.

Real content is preserved: verified 124 Hebrew characters survive on
8228476, and Greek, accented Latin (Argudin, Munz-Manor) and units
(degree, euro, micro, multiplication, >=) are left intact. The step is
metric-neutral by construction -- the scorer already strips non-
alphanumerics, so rField/rGlobal are unchanged at 0.784/0.816 -- its
value is a clean, uniform character set for the LLM->JSON stage.

Author: jimnoneill

calibration/vlm/survey_chars.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python3
+"""Enumerate every non-ASCII character in the scrubbed VLM outputs, so the
+normalization map is built from what actually appears, not guesses."""
+import glob
+import os
+import sys
+import unicodedata
+from collections import Counter
+
+REPO = "/home/joneill/Nextcloud/vaults/jmind/calmi2/poster_science/poster2json"
+sys.path.insert(0, os.path.join(REPO, "calibration/vlm"))
+from vlm_scrub import scrub
+
+V = os.path.join(REPO, "calibration/vlm/out")
+counts = Counter()
+for f in glob.glob(os.path.join(V, "*.md")):
+    t = scrub(open(f, encoding="utf-8").read())
+    for ch in t:
+        if ord(ch) > 127:
+            counts[ch] += 1
+
+print(f"{'char':>5} {'codepoint':>10} {'count':>6}  name")
+for ch, n in counts.most_common(60):
+    try:
+        name = unicodedata.name(ch)
+    except ValueError:
+        name = "?"
+    print(f"{ch!r:>5} {'U+%04X' % ord(ch):>10} {n:>6}  {name}")

calibration/vlm/vlm_scrub.py

@@ -18,6 +18,7 @@
 structure. Only the tabular data payload is removed; every caption survives.
 """
 import re
+import unicodedata
 
 _FENCE_MARKER = re.compile(r"^[ \t]*```.*$", re.MULTILINE)
 
@@ -70,6 +71,32 @@ def _math(m):
 _BOLD_ONLY = re.compile(r"\*\*[^*]+\*\*$")
 
 
+# Reproducibility: the model leaves typographic variants (curly quotes, en/em
+# dashes, arrows) untouched while other things get decomposed, so the character
+# set the downstream LLM converts to JSON is inconsistent. Collapse each variant
+# family to ONE ASCII form, drop box-drawing art (the model rendering a folder
+# tree as |-- lines), and NFKC-normalize the rest -- which unifies duplicate
+# encodings (micro sign U+00B5 vs greek mu U+03BC, super/subscript digits) while
+# LEAVING real content: Hebrew/Arabic/Greek letters, accented names, units.
+_BOXDRAW = re.compile(r"[─-▟]+")   # box-drawing + block elements
+_PUNCT = str.maketrans({
+    "‘": "'", "’": "'", "‚": "'", "‛": "'",
+    "“": '"', "”": '"', "„": '"', "‟": '"',
+    "–": "-", "—": "-", "‒": "-", "―": "-", "−": "-",
+    "…": "...", " ": " ", " ": " ", " ": " ", " ": " ",
+    "→": "->", "⟶": "->", "➔": "->", "⇒": "->", "➙": "->",
+    "←": "<-", "⟵": "<-", "⇐": "<-",
+})
+
+
+def _normalize_chars(text: str) -> str:
+    text = _BOXDRAW.sub(" ", text)
+    text = text.translate(_PUNCT)
+    # NFKC collapses compatibility variants (micro->mu, super/subscripts) to a
+    # single representation; combining marks on Hebrew/accents are preserved.
+    return unicodedata.normalize("NFKC", text)
+
+
 def _strip_footer_logos(text: str) -> str:
     lines = text.splitlines()
     while lines:
@@ -112,6 +139,7 @@ def scrub(text: str) -> str:
     text = _IMAGE.sub("", text)            # drop image placeholders
     text = _delatex(text)                  # LaTeX -> plain, keep affil markers
     text = _TAG.sub("", text)              # remaining tags, keep inner text
+    text = _normalize_chars(text)          # one canonical char per variant family
     text = _TRAIL_WS.sub("\n", text)
     text = _BLANKS.sub("\n\n", text)
     text = _strip_footer_logos(text.strip())   # peel sponsor/logo lines off foot