Skip to content

Commit adb0ddd

Browse files
feat: publish each head's calibration in the catalog and on its bar (#263)
#261 gave every head its own decision threshold and a measured precision, but only a molecule read exposed them. The head catalog — the thing you consult to ask "how good is this head?" — still showed AUROC alone, which is precisely the number that hid the problem: AUROC is computed on ranking and is blind to class imbalance, so `ginger` reads 0.979 there while being right 10% of the time when it fires. /api/heads now carries threshold, precision, recall, n_pos and confident_capable per head, for aroma and mouthfeel alike. A head that fires at 0.16 and is right 6% of the time says so on its own row, not only buried inside one molecule's read. Also fixes headBar, which still hard-coded 0.5 to decide whether a bar was lit. That silently disagreed with the calibrated mouthfeel heads: `cooling` fires at 0.64 and `astringent` at 0.71, so a score of 0.55 was being drawn as firing when the head says it is not. Bars now read the head's own threshold, show where that bar sits and how precise it is there, and hatch the fill for indicative heads. Tox heads keep the flat 0.5 — they are not calibrated and, with hundreds of positives each, were never shy. Taste heads are published with an explicit threshold of 0.5 rather than a null, so a consumer never has to guess whether the field is missing or genuinely flat. Signed-off-by: Austin L. <86896075+rvnminers-A-and-N@users.noreply.github.com>
1 parent 40caeb7 commit adb0ddd

2 files changed

Lines changed: 34 additions & 9 deletions

File tree

training/predict.py

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1290,15 +1290,29 @@ def _taste_auroc(t):
12901290
meta = _TASTE_META.get(t) if isinstance(_TASTE_META, dict) else None
12911291
return meta.get("auroc") if isinstance(meta, dict) else None
12921292

1293+
def _cal(meta, h):
1294+
"""The head's published calibration: where its bar sits, how precise it is there, and
1295+
whether it may be shown as confident. Surfaced so the catalog is auditable — a head that
1296+
fires at 0.16 and is right 6% of the time should say so on its own row, not only inside a
1297+
molecule read."""
1298+
m = meta.get(h, {})
1299+
return {"threshold": m.get("threshold"), "precision": m.get("cv_precision"),
1300+
"recall": m.get("cv_recall"), "n_pos": m.get("n_pos"),
1301+
"confident_capable": m.get("confident_capable", True)}
1302+
12931303
def _aroma(a):
1294-
return {"head": a, "auroc": _AROMA_META.get(a, {}).get("auroc"), "desc": AROMA_DESC.get(a)}
1304+
return {"head": a, "auroc": _AROMA_META.get(a, {}).get("auroc"),
1305+
"desc": AROMA_DESC.get(a), **_cal(_AROMA_META, a)}
12951306

12961307
# mouthfeel = the aroma heads tagged mouthfeel (cooling/pungent) + the dedicated mouthfeel heads
12971308
mouthfeel = [_aroma(a) for a in aroma_heads if a in _MOUTHFEEL_HEADS]
1298-
mouthfeel += [{"head": h, "auroc": _MOUTHFEEL_META.get(h, {}).get("auroc"), "desc": AROMA_DESC.get(h)}
1309+
mouthfeel += [{"head": h, "auroc": _MOUTHFEEL_META.get(h, {}).get("auroc"),
1310+
"desc": AROMA_DESC.get(h), **_cal(_MOUTHFEEL_META, h)}
12991311
for h in mouthfeel_heads]
13001312
return {
1301-
"taste": [{"head": t, "auroc": _taste_auroc(t)} for t in taste_heads],
1313+
# taste heads keep a flat 0.5: hundreds of positives each, so they were never shy
1314+
"taste": [{"head": t, "auroc": _taste_auroc(t), "threshold": 0.5,
1315+
"confident_capable": True} for t in taste_heads],
13021316
"aroma": [_aroma(a) for a in aroma_heads],
13031317
"mouthfeel": mouthfeel,
13041318
"safety": [{"head": t, "auroc": _TOX_META.get(t, {}).get("auroc"),

training/workbench.html

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -284,6 +284,8 @@
284284
/* a head that fires but is right <50% of the time out-of-fold: shown, never sold as confident */
285285
.ameter.indicative .afill{background-image:repeating-linear-gradient(45deg,transparent,transparent 3px,rgba(0,0,0,.28) 3px,rgba(0,0,0,.28) 6px)}
286286
.ameter.indicative .aval b{font-weight:600;opacity:.75;font-style:italic}
287+
.hbar.indicative .hb-fill{background-image:repeating-linear-gradient(45deg,transparent,transparent 3px,rgba(0,0,0,.28) 3px,rgba(0,0,0,.28) 6px)}
288+
.hb-ind{font-family:var(--mono);font-size:9px;letter-spacing:.06em;text-transform:uppercase;opacity:.6;margin-left:6px;border:1px solid currentColor;border-radius:3px;padding:0 3px}
287289
.arow{display:flex;justify-content:space-between;align-items:baseline;margin-bottom:4px}
288290
.aname{font-weight:600;font-size:13px;text-transform:capitalize}
289291
.aval{font-family:var(--mono);font-size:12px;color:var(--muted)}
@@ -1867,20 +1869,29 @@ <h4>Software &amp; type</h4>
18671869
}
18681870

18691871
// one compact head bar (name · track · value · AUROC) — shared by the mouthfeel + safety groups
1870-
function headBar(name, score, auroc, color, title){
1871-
const pct=Math.round((score||0)*100), on=score>=0.5;
1872+
// `cal` carries the head's published calibration when it has one: {threshold, precision,
1873+
// indicative}. Tox heads aren't calibrated and fall back to a flat 0.5, which is correct for them —
1874+
// they have hundreds of positives each and were never shy.
1875+
function headBar(name, score, auroc, color, title, cal){
1876+
const thr = (cal && cal.threshold!=null) ? cal.threshold : 0.5;
1877+
const pct=Math.round((score||0)*100), on=(score||0)>=thr;
1878+
const ind = !!(cal && cal.indicative);
18721879
const au = auroc!=null ? ` <span class="hb-au">· AUROC ${auroc}</span>` : '';
1873-
return `<div class="hbar${on?'':' dim'}"${title?` title="${title.replace(/"/g,'&quot;')}"`:''}>`+
1874-
`<span class="hb-name">${name}</span>`+
1880+
// show WHERE the bar sits and how precise the head is there — a head firing at 16% should say so
1881+
const cl = (cal && cal.threshold!=null)
1882+
? ` <span class="hb-au">· fires ≥${Math.round(thr*100)}%${cal.precision!=null?` · ${Math.round(cal.precision*100)}% precise`:''}</span>` : '';
1883+
return `<div class="hbar${on?'':' dim'}${ind?' indicative':''}"${title?` title="${title.replace(/"/g,'&quot;')}"`:''}>`+
1884+
`<span class="hb-name">${name}${ind?'<span class="hb-ind">indicative</span>':''}</span>`+
18751885
`<span class="hb-track"><span class="hb-fill" style="width:${pct}%;background:${color}"></span></span>`+
1876-
`<span class="hb-val">${pct}%${au}</span></div>`;
1886+
`<span class="hb-val">${pct}%${au}${cl}</span></div>`;
18771887
}
18781888
// Mouthfeel (chemesthesis) — the 5 trained trigeminal heads, ranked, each with its CV-AUROC
18791889
function renderMouthfeel(mf){
18801890
const grp=$('mouthfeelGroup'), box=$('mouthfeelBars');
18811891
const ds=(mf&&mf.descriptors)||[];
18821892
if(!ds.length){ if(grp) grp.style.display='none'; return; }
1883-
box.innerHTML = ds.map(d=> headBar(d.sensation, d.score, d.auroc, aromaColor(d.sensation), d.desc)).join('');
1893+
box.innerHTML = ds.map(d=> headBar(d.sensation, d.score, d.auroc, aromaColor(d.sensation), d.desc,
1894+
{threshold:d.threshold, precision:d.precision, indicative:d.indicative})).join('');
18841895
grp.style.display='block';
18851896
}
18861897
// Safety — the 12 Tox21 caution-only assay heads, most-active first, each with its CV-AUROC

0 commit comments

Comments
 (0)