Skip to content

Commit 0888b09

Browse files
committed
noul: commit P(yes) outside the 0.2..0.8 abstention band (VON_NOUL_DECISION=band, default); raw restores the calibrated posterior
JevBench v1.5 scores Noul as No <=0.2, Yes >=0.8, in-between = abstain = wrong. Von's calibrated posterior sat in 0.4..0.75 on 97% of hard-tier Noul items (temperature map flattens long states), giving Noul CC -79 open / -98 sealed and Intelligence 0. Piecewise-affine map p' = 0.8 + 0.1(p-0.5) keeps argmax and ordering; slope chosen on jabr v2 Noul (ECE 0.14 -> 0.17), never JevBench. Public Noul CC -76 -> +19 (n=74), jabr v2 CC -42 -> +32 (n=312).
1 parent e1378c9 commit 0888b09

4 files changed

Lines changed: 104 additions & 4 deletions

File tree

‎README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,7 @@ A CPU and a CUDA image are in [PR #12](https://github.com/wfzyx/von/pull/12); on
125125
| `--device` / `VON_DEVICE` | `auto` | `cuda`, `mps`, `openvino:gpu`, `openvino:cpu`, `cpu`. Auto prefers OpenVINO CPU over plain torch CPU. |
126126
| `--max-state-tokens N` / `VON_MAX_STATE_TOKENS` | 8192 | States longer than N tokens are middle-truncated (60 % head, 40 % tail) so question and options always fit the 8192 window. Truncated responses carry a `truncation` field and `X-Von-Truncated` / `Warning` headers. |
127127
| `--on-overflow truncate\|refuse` / `VON_ON_OVERFLOW` | truncate | `refuse` answers an oversize state with HTTP 422 ("exceeds the … context window") instead of truncating. Required under no-truncation rules such as the Decision Index. |
128+
| `--noul-decision band\|raw` / `VON_NOUL_DECISION` | band | Noul `P(yes)` decision rule. `band` maps the calibrated posterior to `0.8 + 0.1·(p−0.5)` (mirrored below 0.5) so every answer commits outside the 0.2–0.8 abstention band (JevBench v1.5 Noul rule); argmax and ordering are unchanged. `raw` returns the calibrated posterior as before 1.3.2. Edge/slope: `VON_NOUL_BAND_EDGE`, `VON_NOUL_BAND_SLOPE`. |
128129
| `--chains DIR` / `VON_CHAINS_DIR` | bundled library | Chain-of-options library (below). `--no-chains` / `VON_CHAINS_DIR=off` disables it. |
129130
| `VON_CHAINS_MAX_CALLS` | 16 | Encoder sub-decisions a chained item may spend. |
130131
| `VON_CHAINS_MAX_STATE_TOKENS` | 4096 | Chains stand down on longer states (each sub-decision re-encodes the state). |

‎src/von/backends/option_marker_backend.py‎

Lines changed: 44 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,31 @@ def _validate_noul_prior(raw: object) -> Optional[Dict[str, float]]:
121121
return None
122122

123123

124+
def _noul_decide(p: float, edge: float, slope: float) -> float:
125+
"""Map a calibrated P(yes) to a committed decision probability.
126+
127+
The calibrated posterior sits near 0.5 on long or ambiguous states (the
128+
input-conditioned temperature map flattens it), so a consumer that reads
129+
P(yes) <= 0.2 as No, >= 0.8 as Yes and everything between as an
130+
abstention (JevBench v1.5 Noul rule) records almost every answer as an
131+
abstain. This piecewise-affine map keeps the argmax and the ordering, and
132+
guarantees |p' - 0.5| >= edge - 0.5:
133+
134+
p >= 0.5: p' = edge + slope * (p - 0.5)
135+
p < 0.5: p' = (1 - edge) - slope * (0.5 - p)
136+
137+
edge=0.5, slope=1 is the identity. Slope was chosen on jabr v2 Noul tasks
138+
(never JevBench): 0 has the lowest ECE, 0.1 costs ~0.02 ECE and keeps the
139+
map strictly monotone.
140+
"""
141+
p = max(0.0, min(1.0, p))
142+
if p >= 0.5:
143+
out = edge + slope * (p - 0.5)
144+
else:
145+
out = (1.0 - edge) - slope * (0.5 - p)
146+
return max(0.0, min(1.0, out))
147+
148+
124149
class _IndependentOptionsEncoder(torch.nn.Module):
125150
"""Flat-tensor view of the encoder for tracing in independent_options mode.
126151
@@ -340,6 +365,16 @@ def __init__(
340365
self.on_overflow = os.environ.get("VON_ON_OVERFLOW", "truncate").strip().lower()
341366
if self.on_overflow not in ("truncate", "refuse"):
342367
raise ValueError(f"VON_ON_OVERFLOW must be 'truncate' or 'refuse', got {self.on_overflow!r}")
368+
# Noul decision rule (see _noul_decide). "band" (default) commits every
369+
# answer outside the 0.2..0.8 abstention band; "raw" returns the
370+
# calibrated posterior unchanged (pre-1.3.2 behaviour).
371+
self.noul_decision = os.environ.get("VON_NOUL_DECISION", "band").strip().lower()
372+
if self.noul_decision not in ("band", "raw"):
373+
raise ValueError(f"VON_NOUL_DECISION must be 'band' or 'raw', got {self.noul_decision!r}")
374+
self.noul_band_edge = float(os.environ.get("VON_NOUL_BAND_EDGE", "0.8"))
375+
self.noul_band_slope = float(os.environ.get("VON_NOUL_BAND_SLOPE", "0.1"))
376+
if not (0.5 <= self.noul_band_edge <= 1.0) or not (0.0 <= self.noul_band_slope <= 1.0):
377+
raise ValueError("VON_NOUL_BAND_EDGE must be in [0.5, 1] and VON_NOUL_BAND_SLOPE in [0, 1]")
343378
self._trunc_local = threading.local()
344379
# Chain-of-options: on by default with the bundled library in bindall
345380
# mode (Von 1.3). VON_CHAINS_DIR=<dir> swaps the library, VON_CHAINS_DIR=off
@@ -613,7 +648,8 @@ def evaluate_noul(
613648
) -> NoulAnswer:
614649
chained = self._maybe_chain(state_text, q)
615650
if chained is not None:
616-
return chained
651+
return self._commit_noul(chained)
652+
617653

618654
model = self._get_model()
619655
tok = model.tokenizer
@@ -681,7 +717,13 @@ def evaluate_noul(
681717
probs = torch.softmax(scaled, dim=-1).cpu().tolist()
682718

683719
prob_true = round(max(0.0, min(1.0, probs[0])), 4)
684-
return NoulAnswer(noul=prob_true)
720+
return self._commit_noul(NoulAnswer(noul=prob_true))
721+
722+
def _commit_noul(self, ans: NoulAnswer) -> NoulAnswer:
723+
if self.noul_decision == "raw":
724+
return ans
725+
ans.noul = round(_noul_decide(float(ans.noul), self.noul_band_edge, self.noul_band_slope), 4)
726+
return ans
685727

686728
def evaluate_score(
687729
self,

‎src/von/cli.py‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -58,11 +58,20 @@ def main():
5858
help="State longer than the window: middle-truncate with a warning (default) or refuse with HTTP 422 "
5959
"(env VON_ON_OVERFLOW). Use refuse under no-truncation benchmark rules.",
6060
)
61+
@click.option(
62+
"--noul-decision",
63+
type=click.Choice(["band", "raw"]),
64+
default=None,
65+
help="Noul P(yes): 'band' (default) commits every answer outside the 0.2..0.8 abstention band, "
66+
"'raw' returns the calibrated posterior unchanged (env VON_NOUL_DECISION).",
67+
)
6168
def serve(host: str, port: int, backend: str, device: str, reload: bool, max_state_tokens: Optional[int],
62-
chains: Optional[str], no_chains: bool, on_overflow: Optional[str]):
69+
chains: Optional[str], no_chains: bool, on_overflow: Optional[str], noul_decision: Optional[str]):
70+
"""Start the Von System One HTTP server."""
6371
if on_overflow:
6472
os.environ["VON_ON_OVERFLOW"] = on_overflow
65-
"""Start the Von System One HTTP server."""
73+
if noul_decision:
74+
os.environ["VON_NOUL_DECISION"] = noul_decision
6675
os.environ["VON_BACKEND"] = backend
6776
if no_chains:
6877
os.environ["VON_CHAINS_DIR"] = "off"

‎tests/test_noul_band.py‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
"""Noul decision rule: committed P(yes) outside the 0.2..0.8 abstention band."""
2+
import os
3+
4+
import pytest
5+
6+
from von.backends.option_marker_backend import _noul_decide
7+
8+
9+
@pytest.mark.parametrize("p", [0.0, 0.1, 0.3, 0.49, 0.5, 0.51, 0.7, 0.9, 1.0])
10+
def test_band_never_lands_in_abstention_band(p):
11+
out = _noul_decide(p, 0.8, 0.1)
12+
assert out <= 0.2 or out >= 0.8
13+
assert (out >= 0.5) == (p >= 0.5), "argmax must be preserved"
14+
15+
16+
def test_band_is_monotone_and_bounded():
17+
ps = [i / 100 for i in range(101)]
18+
outs = [_noul_decide(p, 0.8, 0.1) for p in ps]
19+
assert all(a <= b for a, b in zip(outs, outs[1:]))
20+
assert 0.0 <= min(outs) and max(outs) <= 1.0
21+
assert _noul_decide(0.5, 0.8, 0.1) == pytest.approx(0.8)
22+
assert _noul_decide(1.0, 0.8, 0.1) == pytest.approx(0.85)
23+
assert _noul_decide(0.0, 0.8, 0.1) == pytest.approx(0.15)
24+
25+
26+
def test_identity_parameters_are_a_noop():
27+
for p in (0.0, 0.25, 0.5, 0.75, 1.0):
28+
assert _noul_decide(p, 0.5, 1.0) == pytest.approx(p)
29+
30+
31+
def test_backend_applies_band_by_default_and_raw_disables_it(monkeypatch):
32+
from von.backends.option_marker_backend import OptionMarkerBackend
33+
from von.types import NoulAnswer
34+
35+
monkeypatch.delenv("VON_NOUL_DECISION", raising=False)
36+
b = OptionMarkerBackend.__new__(OptionMarkerBackend)
37+
b.noul_decision, b.noul_band_edge, b.noul_band_slope = "band", 0.8, 0.1
38+
assert b._commit_noul(NoulAnswer(noul=0.62)).noul == pytest.approx(0.812)
39+
b.noul_decision = "raw"
40+
assert b._commit_noul(NoulAnswer(noul=0.62)).noul == pytest.approx(0.62)
41+
42+
43+
def test_env_validation(monkeypatch):
44+
from von.backends.option_marker_backend import OptionMarkerBackend
45+
46+
monkeypatch.setenv("VON_NOUL_DECISION", "sometimes")
47+
with pytest.raises(ValueError, match="VON_NOUL_DECISION"):
48+
OptionMarkerBackend(checkpoint_dir=os.path.join("checkpoints", "von-1.2"), device="cpu")

0 commit comments

Comments
 (0)