Rescue prose cells from data tables; strip figure panel-label scaffolding

Two scrub refinements from an adversarially-verified analysis pass.

Data tables: _table_repl dropped a short-median table whole, which lost
8228568's interview-quote tables (Reason/Participant columns are short so
the median is short, but the Example column holds the poster's actual
Results as long quotes). That starved the downstream 8B and produced no
JSON. The data-table branch now RESCUES cells that are genuine free text
(>=40 chars, <=2 bullets, >=50% letters) and still drops short/numeric/
bulleted-grid cells. Table classification (median length) is unchanged,
so layout tables -- and 4448680, whose earlier blanket keep-all crashed
word capture -- are byte-identical. 8228568 input mass restored
3035->4402 chars with its quotes back.

Scaffold: drop lone figure panel labels ("A"/"B"/"C") and an orphaned
code-fence language token ("plaintext") the model emits on their own
lines; they match no caption and dilute per-field precision. The
bare-number-run variant was rejected in review because it deletes real
F1 result values on 8228476.

Extraction-stage rField (calibration harness): out 0.784 -> 0.788,
w 0.929 -> 0.946, rGlobal 0.816 -> 0.827; out_rope2464 0.746 -> 0.751.
No per-poster regression. Scrubbed LightOnOCR now leads pdfplumber
0.788 vs 0.762 on the extraction stage.

Author: jimnoneill

calibration/vlm/vlm_scrub.py

@@ -116,21 +116,59 @@ def _strip_footer_logos(text: str) -> str:
 # -- which the model uses to reproduce a poster's column grid -- holds whole
 # prose paragraphs. Median cell length separates them.
 _DATA_CELL_MAXLEN = 45
+_BULLET = re.compile(r"[•▪◦‣·∙]|<br\b")
+_PROSE_CELL_MINLEN = 40
+
+
+def _cell_text(c):
+    return re.sub(r"<[^>]+>", "", _IMAGE.sub("", c)).strip()   # drop ![img] then tags
+
+
+def _is_prose_cell(raw):
+    """A table cell that is genuine free text (a quote, a finding), not a short
+    label, a number, or a bulleted grid column."""
+    breaks = len(_BULLET.findall(raw))            # count on RAW (bullets/<br> intact)
+    txt = _cell_text(raw)
+    if len(txt) < _PROSE_CELL_MINLEN:             # short label / numeric cell
+        return False
+    if breaks > 2:                                # bulleted grid column (the 4448680 noise)
+        return False
+    letters = sum(ch.isalpha() for ch in txt)
+    return bool(txt) and letters >= 0.5 * len(txt)  # not a pure-numeric grid
 
 
 def _table_repl(m):
-    """Drop a data table (short cells); keep a layout table's prose."""
+    """Layout table -> keep every cell's prose. Data table -> drop the grid but
+    RESCUE any long free-text cells (interview quotes, findings) that would
+    otherwise be lost with the table; a poster's Results sometimes live in an
+    Example column. Table CLASSIFICATION (median cell length) is unchanged, so
+    layout tables are handled byte-identically to before."""
     cells = _CELL.findall(m.group(1))
     if not cells:
         return ""
     lens = sorted(len(re.sub(r"<[^>]+>", "", c).strip()) for c in cells)
     median = lens[len(lens) // 2]
     if median <= _DATA_CELL_MAXLEN:
-        return ""                                   # data within an image
+        kept = [_cell_text(c) for c in cells if _is_prose_cell(c)]
+        return ("\n\n".join(kept) + "\n") if kept else ""
     # layout grid: keep the cell text as paragraphs, one per cell
     return "\n\n".join(re.sub(r"<[^>]+>", "", c).strip() for c in cells) + "\n"
 
 
+# Figure/chart scaffolding the VLM reads out of image regions: lone panel labels
+# ("A"/"B"/"C") and an orphaned code-fence language token emitted without its
+# backticks ("plaintext"). These survive as their own lines, match no caption,
+# and drag per-field ROUGE-L precision down. (The bare-number-run variant was
+# rejected in review — it deletes real F1 result values on 8228476.)
+_SCAFFOLD_LABEL = re.compile(
+    r"^(?:[A-Za-z][.)]?|plaintext|html|python|json|text|markdown|yaml|css|bash|sql)$")
+
+
+def _strip_scaffold(text: str) -> str:
+    return "\n".join(ln for ln in text.splitlines()
+                     if not (ln.strip() and _SCAFFOLD_LABEL.match(ln.strip())))
+
+
 def scrub(text: str) -> str:
     if not text:
         return text
@@ -142,6 +180,7 @@ def scrub(text: str) -> str:
     text = _normalize_chars(text)          # one canonical char per variant family
     text = _TRAIL_WS.sub("\n", text)
     text = _BLANKS.sub("\n\n", text)
+    text = _strip_scaffold(text)           # drop lone panel labels / orphan fence tokens
     text = _strip_footer_logos(text.strip())   # peel sponsor/logo lines off foot
     return text.strip()