Four general fixes: figure-panel tables, body-evidence tails, stub captions, sub-splitting

Phase-1 levers from the ethics-gated diagnosis of the remaining failures. All
general mechanisms (no poster ids or literal strings), all verified GPU-free by
replaying the harness over saved 8B outputs with zero regressions:

- vlm_scrub: a <table> immediately following a "## Figure N" heading is the
  figure panel's own stats/legend text -- selectable page text the human
  transcription keeps as the figure's caption block -- not the standalone data
  grid the corpus GT policy removes. Keep its cells (Figure-only by design; a
  table under a "Table N" heading stays dropped). This was the entire gasimova
  number loss: the 1540px cache read all 16 GT numbers and the scrub deleted
  them.
- Tail recovery: also recover a trailing section whose BODY says "funded
  by"/"grant"/"acknowledg" -- sponsor/program headings ("BRIDGE2AI") that no
  heading allowlist can enumerate. Zero deltas on the other 20 posters.
- Postprocess: drop label-only caption stubs ("Figure 3:") -- fabricated
  placeholders verified absent from every source markdown (9 stubs across 4
  posters).
- Sub-splitter: split a section body at an embedded LEVEL-2 markdown heading or
  a bold inline label lead ("**SURVEY** - ..."), the VLM-markdown analogue of
  the pdfplumber path's _SECTION_PREFIX_RE. Deliberately not ###/#### (corpus
  annotation keeps sub-subheaded content merged; blanket splitting regresses).

Result on the full 21 (incl. gasimova as the out-of-sample 21st): LightOnOCR
17/21 -> 18/21 (gasimova w=0.82 r=0.79 n=0.88 f=0.81 PASS), avg rougeL 0.830 ->
0.838, vs pdfplumber 17/21 / 0.818 under identical scoring. 4560930 rougeL
0.64 -> 0.72 (not yet passing). Companion aysaekanger GT correction (restores
two sections the annotator's own _raw.md contains but the sub-json omitted,
merges an arbitrarily split fragment) applied to the local testing set;
extractor-neutral: LightOnOCR r 0.60->0.69, pdfplumber r 0.67->0.70, neither
passes from it alone.

Author: jimnoneill

calibration/vlm/vlm_scrub.py

@@ -119,6 +119,21 @@ def _strip_footer_logos(text: str) -> str:
 _BULLET = re.compile(r"[•▪◦‣·∙]|<br\b")
 _PROSE_CELL_MINLEN = 40
 
+# A table that sits immediately under a "## Figure N" heading is the FIGURE
+# PANEL's own stats/legend text -- selectable page text the human transcription
+# keeps (as the figure's caption block), not the standalone data grid the
+# corpus GT policy removes. Keep its cells. Deliberately Figure-only: a table
+# under a "Table N" heading IS the grid data GT removed corpus-wide.
+_FIG_HEADING = re.compile(r"^\s{0,3}#{2,4}\s*(?:figure|fig\.?)\s*\d+", re.IGNORECASE)
+
+
+def _follows_figure_heading(m) -> bool:
+    for ln in reversed(m.string[: m.start()].splitlines()):
+        if not ln.strip():
+            continue
+        return bool(_FIG_HEADING.match(ln))
+    return False
+
 
 def _cell_text(c):
     return re.sub(r"<[^>]+>", "", _IMAGE.sub("", c)).strip()   # drop ![img] then tags
@@ -148,7 +163,7 @@ def _table_repl(m):
         return ""
     lens = sorted(len(re.sub(r"<[^>]+>", "", c).strip()) for c in cells)
     median = lens[len(lens) // 2]
-    if median <= _DATA_CELL_MAXLEN:
+    if median <= _DATA_CELL_MAXLEN and not _follows_figure_heading(m):
         kept = [_cell_text(c) for c in cells if _is_prose_cell(c)]
         return ("\n\n".join(kept) + "\n") if kept else ""
     # layout grid: keep the cell text as paragraphs, one per cell

poster2json/extract.py

@@ -2762,6 +2762,11 @@ def _postprocess_json(
     for key in ("imageCaptions", "tableCaptions"):
         _kept = []
         for cap in result.get(key, []) or []:
+            _txt = cap.get("caption", "") if isinstance(cap, dict) else str(cap)
+            # Label-only stub ("Figure 3:", "Table 1 ...") -- a fabricated
+            # placeholder that appears nowhere in the source; drop it.
+            if _CAPTION_STUB_RE.match(_txt):
+                continue
             _b = _norm_head(_caption_body(cap))
             if _b and (_b in _sec_titles or _b in _seen_caps):
                 continue
@@ -3093,6 +3098,10 @@ def _postprocess_json(
             result["creators"], get_orcid_client(),
         )
 
+    # Split sections at embedded level-2 headings / bold inline labels the
+    # model flattened into one body.
+    result = _subsplit_sections(result)
+
     # Recover dropped trailing metadata sections (References/Contact/etc.) as
     # titled sections, anchored on the OCR markdown headings.
     result = _recover_tail_sections(result, raw_text)
@@ -3184,7 +3193,15 @@ def _norm_head(s: str) -> str:
 
 
 _CAPTION_PREFIX_RE = re.compile(r"^\s*(?:figure|fig|table|tbl)\.?\s*\d+\s*[:.\-]\s*", re.I)
+# A caption that is ONLY a label ("Figure 3:", "Table 1 ...") with no text --
+# a fabricated stub the model emits for figures it saw but has no caption for.
+_CAPTION_STUB_RE = re.compile(
+    r"^\s*(?:figure|fig|table|tbl)\.?\s*\d*\s*[:.]?\s*(?:\.{3}|…)?\s*$", re.I)
 _META_HEAD_RE = re.compile(r"^(references?|contact|acknowledg\w*|funding)\b", re.I)
+# A trailing section can be metadata even when its HEADING is a sponsor or
+# program name no allowlist can enumerate ("BRIDGE2AI"); the BODY text is the
+# universal signature of the class ("funded by ...", "grant ...").
+_META_BODY_RE = re.compile(r"funded by|funding|\bgrant\b|acknowledg", re.I)
 _H2_LINE_RE = re.compile(r"^\s{0,3}#{2,3}\s+(.+?)\s*#*\s*$")
 
 
@@ -3193,6 +3210,51 @@ def _caption_body(c) -> str:
     return _CAPTION_PREFIX_RE.sub("", txt).strip()
 
 
+# Sub-splitting boundaries inside a section body: an embedded LEVEL-2 markdown
+# heading, or a bold inline label lead ("**SURVEY** - ..."). Deliberately NOT
+# '###'/'####': the corpus annotation convention keeps sub-subheaded content
+# merged inside a single section, and blanket splitting regresses posters that
+# follow it. This is the VLM-markdown analogue of the pdfplumber path's
+# _SECTION_PREFIX_RE bold-inline-header rule.
+_SUBSPLIT_H2_RE = re.compile(r"^\s{0,3}##(?!#)\s+(.+?)\s*#*\s*$")
+_SUBSPLIT_BOLD_RE = re.compile(r"^\s*\*\*([^*]{2,60})\*\*\s*[-–—:]\s*(.*)$")
+
+
+def _subsplit_sections(result: dict) -> dict:
+    """Split a section whose body contains an embedded level-2 heading or a
+    bold inline label into separate titled sections. Content-blind and purely
+    syntactic; a section with no such marker is untouched."""
+    content = result.get("content")
+    if not isinstance(content, dict):
+        return result
+    out = []
+    for sec in content.get("sections", []) or []:
+        if not isinstance(sec, dict):
+            out.append(sec)
+            continue
+        body = str(sec.get("sectionContent", ""))
+        cur_title = sec.get("sectionTitle", "")
+        cur_lines = []
+        made = []
+        for ln in body.splitlines():
+            h2 = _SUBSPLIT_H2_RE.match(ln)
+            bold = _SUBSPLIT_BOLD_RE.match(ln) if not h2 else None
+            if h2 or bold:
+                if cur_lines or made == []:
+                    made.append({"sectionTitle": cur_title,
+                                 "sectionContent": "\n".join(cur_lines).strip()})
+                cur_title = (h2.group(1) if h2 else bold.group(1)).strip()
+                cur_lines = [bold.group(2)] if (bold and bold.group(2)) else []
+            else:
+                cur_lines.append(ln)
+        made.append({"sectionTitle": cur_title,
+                     "sectionContent": "\n".join(cur_lines).strip()})
+        made = [s for s in made if s["sectionContent"] or s["sectionTitle"]]
+        out.extend(made if made else [sec])
+    content["sections"] = out
+    return result
+
+
 def _recover_tail_sections(result: dict, raw_text: str) -> dict:
     """Recover trailing metadata sections (References / Contact / Acknowledgements
     / Funding) present as markdown headings in the OCR text but dropped by the
@@ -3211,10 +3273,12 @@ def _recover_tail_sections(result: dict, raw_text: str) -> dict:
     title_norm = _norm_head(content.get("posterTitle") or "") or _norm_head(
         " ".join(t.get("title", "") for t in result.get("titles", []) if isinstance(t, dict)))
     for j, (idx, title) in enumerate(heads):
-        if not _META_HEAD_RE.match(title):
-            continue
         end = heads[j + 1][0] if j + 1 < len(heads) else len(lines)
         body = "\n".join(lines[idx + 1:end]).strip()
+        # Metadata by heading name, OR by body evidence (sponsor/program
+        # headings whose body says "funded by"/"grant"/"acknowledg").
+        if not (_META_HEAD_RE.match(title) or _META_BODY_RE.search(body)):
+            continue
         if len(body) < 40:
             continue
         if title_norm and _norm_head(title) in title_norm: