From 78dee353a90da15e16ab7e0ba4de241864f8da87 Mon Sep 17 00:00:00 2001 From: "Austin L." <86896075+rvnminers-A-and-N@users.noreply.github.com> Date: Wed, 29 Jul 2026 16:18:17 +0000 Subject: [PATCH] feat: publish each head's calibration in the catalog and on its bar MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #261 gave every head its own decision threshold and a measured precision, but only a molecule read exposed them. The head catalog — the thing you consult to ask "how good is this head?" — still showed AUROC alone, which is precisely the number that hid the problem: AUROC is computed on ranking and is blind to class imbalance, so `ginger` reads 0.979 there while being right 10% of the time when it fires. /api/heads now carries threshold, precision, recall, n_pos and confident_capable per head, for aroma and mouthfeel alike. A head that fires at 0.16 and is right 6% of the time says so on its own row, not only buried inside one molecule's read. Also fixes headBar, which still hard-coded 0.5 to decide whether a bar was lit. That silently disagreed with the calibrated mouthfeel heads: `cooling` fires at 0.64 and `astringent` at 0.71, so a score of 0.55 was being drawn as firing when the head says it is not. Bars now read the head's own threshold, show where that bar sits and how precise it is there, and hatch the fill for indicative heads. Tox heads keep the flat 0.5 — they are not calibrated and, with hundreds of positives each, were never shy. Taste heads are published with an explicit threshold of 0.5 rather than a null, so a consumer never has to guess whether the field is missing or genuinely flat. Signed-off-by: Austin L. <86896075+rvnminers-A-and-N@users.noreply.github.com> --- training/predict.py | 20 +++++++++++++++++--- training/workbench.html | 23 +++++++++++++++++------ 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/training/predict.py b/training/predict.py index df6e014..05b96fa 100644 --- a/training/predict.py +++ b/training/predict.py @@ -1290,15 +1290,29 @@ def _taste_auroc(t): meta = _TASTE_META.get(t) if isinstance(_TASTE_META, dict) else None return meta.get("auroc") if isinstance(meta, dict) else None + def _cal(meta, h): + """The head's published calibration: where its bar sits, how precise it is there, and + whether it may be shown as confident. Surfaced so the catalog is auditable — a head that + fires at 0.16 and is right 6% of the time should say so on its own row, not only inside a + molecule read.""" + m = meta.get(h, {}) + return {"threshold": m.get("threshold"), "precision": m.get("cv_precision"), + "recall": m.get("cv_recall"), "n_pos": m.get("n_pos"), + "confident_capable": m.get("confident_capable", True)} + def _aroma(a): - return {"head": a, "auroc": _AROMA_META.get(a, {}).get("auroc"), "desc": AROMA_DESC.get(a)} + return {"head": a, "auroc": _AROMA_META.get(a, {}).get("auroc"), + "desc": AROMA_DESC.get(a), **_cal(_AROMA_META, a)} # mouthfeel = the aroma heads tagged mouthfeel (cooling/pungent) + the dedicated mouthfeel heads mouthfeel = [_aroma(a) for a in aroma_heads if a in _MOUTHFEEL_HEADS] - mouthfeel += [{"head": h, "auroc": _MOUTHFEEL_META.get(h, {}).get("auroc"), "desc": AROMA_DESC.get(h)} + mouthfeel += [{"head": h, "auroc": _MOUTHFEEL_META.get(h, {}).get("auroc"), + "desc": AROMA_DESC.get(h), **_cal(_MOUTHFEEL_META, h)} for h in mouthfeel_heads] return { - "taste": [{"head": t, "auroc": _taste_auroc(t)} for t in taste_heads], + # taste heads keep a flat 0.5: hundreds of positives each, so they were never shy + "taste": [{"head": t, "auroc": _taste_auroc(t), "threshold": 0.5, + "confident_capable": True} for t in taste_heads], "aroma": [_aroma(a) for a in aroma_heads], "mouthfeel": mouthfeel, "safety": [{"head": t, "auroc": _TOX_META.get(t, {}).get("auroc"), diff --git a/training/workbench.html b/training/workbench.html index 5914242..082986a 100644 --- a/training/workbench.html +++ b/training/workbench.html @@ -284,6 +284,8 @@ /* a head that fires but is right <50% of the time out-of-fold: shown, never sold as confident */ .ameter.indicative .afill{background-image:repeating-linear-gradient(45deg,transparent,transparent 3px,rgba(0,0,0,.28) 3px,rgba(0,0,0,.28) 6px)} .ameter.indicative .aval b{font-weight:600;opacity:.75;font-style:italic} + .hbar.indicative .hb-fill{background-image:repeating-linear-gradient(45deg,transparent,transparent 3px,rgba(0,0,0,.28) 3px,rgba(0,0,0,.28) 6px)} + .hb-ind{font-family:var(--mono);font-size:9px;letter-spacing:.06em;text-transform:uppercase;opacity:.6;margin-left:6px;border:1px solid currentColor;border-radius:3px;padding:0 3px} .arow{display:flex;justify-content:space-between;align-items:baseline;margin-bottom:4px} .aname{font-weight:600;font-size:13px;text-transform:capitalize} .aval{font-family:var(--mono);font-size:12px;color:var(--muted)} @@ -1867,20 +1869,29 @@

Software & type

} // one compact head bar (name · track · value · AUROC) — shared by the mouthfeel + safety groups -function headBar(name, score, auroc, color, title){ - const pct=Math.round((score||0)*100), on=score>=0.5; +// `cal` carries the head's published calibration when it has one: {threshold, precision, +// indicative}. Tox heads aren't calibrated and fall back to a flat 0.5, which is correct for them — +// they have hundreds of positives each and were never shy. +function headBar(name, score, auroc, color, title, cal){ + const thr = (cal && cal.threshold!=null) ? cal.threshold : 0.5; + const pct=Math.round((score||0)*100), on=(score||0)>=thr; + const ind = !!(cal && cal.indicative); const au = auroc!=null ? ` · AUROC ${auroc}` : ''; - return `
`+ - `${name}`+ + // show WHERE the bar sits and how precise the head is there — a head firing at 16% should say so + const cl = (cal && cal.threshold!=null) + ? ` · fires ≥${Math.round(thr*100)}%${cal.precision!=null?` · ${Math.round(cal.precision*100)}% precise`:''}` : ''; + return `
`+ + `${name}${ind?'indicative':''}`+ ``+ - `${pct}%${au}
`; + `${pct}%${au}${cl}
`; } // Mouthfeel (chemesthesis) — the 5 trained trigeminal heads, ranked, each with its CV-AUROC function renderMouthfeel(mf){ const grp=$('mouthfeelGroup'), box=$('mouthfeelBars'); const ds=(mf&&mf.descriptors)||[]; if(!ds.length){ if(grp) grp.style.display='none'; return; } - box.innerHTML = ds.map(d=> headBar(d.sensation, d.score, d.auroc, aromaColor(d.sensation), d.desc)).join(''); + box.innerHTML = ds.map(d=> headBar(d.sensation, d.score, d.auroc, aromaColor(d.sensation), d.desc, + {threshold:d.threshold, precision:d.precision, indicative:d.indicative})).join(''); grp.style.display='block'; } // Safety — the 12 Tox21 caution-only assay heads, most-active first, each with its CV-AUROC