Skip to content

Commit df1cfe9

Browse files
barretodavidclaude
andcommitted
Move Context back to the third column
Right after SCI, where it was. The audit takes the budget cells from the end of the row rather than at a fixed offset, so a reference column added in front cannot shift which cells it reads. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
1 parent 8f6c1f3 commit df1cfe9

3 files changed

Lines changed: 8 additions & 12 deletions

File tree

bench/audit.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -407,10 +407,10 @@ def chart_labels(svg):
407407
bad_cell.append(f"{label}: not a leaderboard row")
408408
continue
409409
files = (_mm.get(spec.partition("@")[0], {}).get("gguf") or {}).get("files") or {}
410-
# SCI leads and Context trails, so the budget columns are the cells between,
411-
# in VRAM_BUDGETS order. Sliced from both ends rather than counted from one,
412-
# so adding a trailing column cannot silently shift which cells are read.
413-
tds = re.findall(r'<td class="r[^"]*"[^>]*>(.*?)</td>', rest, re.S)[1:-1]
410+
# SCI and Context lead, so the budget columns are the last len(VRAM_BUDGETS)
411+
# cells, in that order. Taken from the END rather than by a fixed offset, so
412+
# another reference column added in front cannot shift which cells are read.
413+
tds = re.findall(r'<td class="r[^"]*"[^>]*>(.*?)</td>', rest, re.S)[-len(VRAM_BUDGETS):]
414414
if len(tds) != len(VRAM_BUDGETS):
415415
bad_cell.append(f"{label}: {len(tds)} budget cells, expected {len(VRAM_BUDGETS)}")
416416
continue

bench/html_report.py

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1172,12 +1172,8 @@ def h2h_stats(spec):
11721172
# the index rides along so this table can be read in score order too,
11731173
# which is the order it ships in; the models table above has the rest
11741174
+ num_td(r["sci"], f'{r["sci"]:.1f}')
1175-
# Context sits after the budget columns rather than before them: it is
1176-
# reference, not the answer, and in front it pushed every budget column
1177-
# off a 390px screen.
1178-
+ cells
11791175
+ num_td(mm["context_length"], fmt_ctx(mm["context_length"]))
1180-
+ "</tr>"
1176+
+ cells + "</tr>"
11811177
)
11821178

11831179
# every sortable table on the page, not one by id: the open-weight table
@@ -1235,7 +1231,7 @@ def h2h_stats(spec):
12351231
<p class="takeaway" style="margin:0 0 10px">The weights do not get the whole machine, so each column reserves room for the OS, the KV cache and activations: {budget_reserve_note}. A quant in <b>full black</b> is <b>Q6 or better</b>, where quantization is not the thing holding the model back; a greyed one is a 4-to-5-bit compromise.</p>
12361232
<p class="takeaway" style="margin:0 0 10px">A blank means <b>nothing 4-bit or better fits</b>, which is a real answer rather than a missing one: below 4-bit degradation stops being minor and coding feels it first, so no cell recommends going lower. {too_big_note}</p>
12371233
<div class="tablewrap"><table id="opentable" class="sortable">
1238-
<tr><th>Model</th><th class="r desc" data-num aria-sort="descending">SCI</th>{"".join(f'<th class="r" data-num>{b} GB</th>' for b in VRAM_BUDGETS)}<th class="r" data-num>Context</th></tr>
1234+
<tr><th>Model</th><th class="r desc" data-num aria-sort="descending">SCI</th><th class="r" data-num>Context</th>{"".join(f'<th class="r" data-num>{b} GB</th>' for b in VRAM_BUDGETS)}</tr>
12391235
{"".join(open_rows_html)}
12401236
</table></div>
12411237
</section>"""

results/report.html

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -372,8 +372,8 @@ <h2 id="which-quantization-for-my-machine"><a class="anchor" href="#which-quanti
372372
<p class="takeaway" style="margin:0 0 10px">The weights do not get the whole machine, so each column reserves room for the OS, the KV cache and activations: 16 GB at 128, 8 GB at 64, 4 GB at 32, 4 GB at 24. A quant in <b>full black</b> is <b>Q6 or better</b>, where quantization is not the thing holding the model back; a greyed one is a 4-to-5-bit compromise.</p>
373373
<p class="takeaway" style="margin:0 0 10px">A blank means <b>nothing 4-bit or better fits</b>, which is a real answer rather than a missing one: below 4-bit degradation stops being minor and coding feels it first, so no cell recommends going lower. The nine open models not listed need more than 128 GB even at 4-bit: Kimi K3, MiMo-V2.5-Pro, MiniMax M3, DeepSeek V4 Flash, Qwen3.8 Max, DeepSeek V4-Pro, Hy3, Inkling and GLM 5.2.</p>
374374
<div class="tablewrap"><table id="opentable" class="sortable">
375-
<tr><th>Model</th><th class="r desc" data-num aria-sort="descending">SCI</th><th class="r" data-num>128 GB</th><th class="r" data-num>64 GB</th><th class="r" data-num>32 GB</th><th class="r" data-num>24 GB</th><th class="r" data-num>Context</th></tr>
376-
<tr><td>Muse Glimmer 30B</td><td class="r" data-s="53.54335822195509">53.5</td><td class="r fits" data-s="29.6"><span class="q">Q8_0</span> <span class="gb">30 GB</span></td><td class="r fits" data-s="29.6"><span class="q">Q8_0</span> <span class="gb">30 GB</span></td><td class="r fits" data-s="24.1"><span class="q">Q6_K_L</span> <span class="gb">24 GB</span></td><td class="r nofit" data-s="19.4"><span class="q">Q5_K_S</span> <span class="gb">19 GB</span></td><td class="r" data-s="131072">131k</td></tr><tr><td>gpt-oss-120b</td><td class="r" data-s="17.741231416968965">17.7</td><td class="r nofit" data-s="62.8"><span class="q">Q4_K_M</span> <span class="gb">63 GB</span></td><td class="r"></td><td class="r"></td><td class="r"></td><td class="r" data-s="131072">131k</td></tr><tr><td>Qwen3.6-35B-A3B</td><td class="r" data-s="16.19082483091965">16.2</td><td class="r fits" data-s="36.9"><span class="q">Q8_0</span> <span class="gb">37 GB</span></td><td class="r fits" data-s="36.9"><span class="q">Q8_0</span> <span class="gb">37 GB</span></td><td class="r fits" data-s="26.6"><span class="q">UD-Q5_K_XL</span> <span class="gb">27 GB</span></td><td class="r nofit" data-s="19.5"><span class="q">UD-IQ4_NL_XL</span> <span class="gb">20 GB</span></td><td class="r" data-s="262144">262k</td></tr><tr><td>Gemma 4 31B</td><td class="r" data-s="15.806948103252044">15.8</td><td class="r fits" data-s="32.6"><span class="q">Q8_0</span> <span class="gb">33 GB</span></td><td class="r fits" data-s="32.6"><span class="q">Q8_0</span> <span class="gb">33 GB</span></td><td class="r fits" data-s="27.5"><span class="q">UD-Q6_K_XL</span> <span class="gb">28 GB</span></td><td class="r nofit" data-s="18.3"><span class="q">Q4_K_M</span> <span class="gb">18 GB</span></td><td class="r" data-s="262144">262k</td></tr><tr><td>Qwen3 Coder Next</td><td class="r" data-s="14.283251268776736">14.3</td><td class="r fits" data-s="84.8"><span class="q">Q8_0</span> <span class="gb">85 GB</span></td><td class="r nofit" data-s="55.8"><span class="q">UD-Q5_K_S</span> <span class="gb">56 GB</span></td><td class="r"></td><td class="r"></td><td class="r" data-s="262144">262k</td></tr><tr><td>Qwen3.6-27B</td><td class="r" data-s="13.317904173386546">13.3</td><td class="r fits" data-s="28.6"><span class="q">Q8_0</span> <span class="gb">29 GB</span></td><td class="r fits" data-s="28.6"><span class="q">Q8_0</span> <span class="gb">29 GB</span></td><td class="r fits" data-s="25.6"><span class="q">UD-Q6_K_XL</span> <span class="gb">26 GB</span></td><td class="r fits" data-s="20.0"><span class="q">UD-Q5_K_XL</span> <span class="gb">20 GB</span></td><td class="r" data-s="262144">262k</td></tr>
375+
<tr><th>Model</th><th class="r desc" data-num aria-sort="descending">SCI</th><th class="r" data-num>Context</th><th class="r" data-num>128 GB</th><th class="r" data-num>64 GB</th><th class="r" data-num>32 GB</th><th class="r" data-num>24 GB</th></tr>
376+
<tr><td>Muse Glimmer 30B</td><td class="r" data-s="53.54335822195509">53.5</td><td class="r" data-s="131072">131k</td><td class="r fits" data-s="29.6"><span class="q">Q8_0</span> <span class="gb">30 GB</span></td><td class="r fits" data-s="29.6"><span class="q">Q8_0</span> <span class="gb">30 GB</span></td><td class="r fits" data-s="24.1"><span class="q">Q6_K_L</span> <span class="gb">24 GB</span></td><td class="r nofit" data-s="19.4"><span class="q">Q5_K_S</span> <span class="gb">19 GB</span></td></tr><tr><td>gpt-oss-120b</td><td class="r" data-s="17.741231416968965">17.7</td><td class="r" data-s="131072">131k</td><td class="r nofit" data-s="62.8"><span class="q">Q4_K_M</span> <span class="gb">63 GB</span></td><td class="r"></td><td class="r"></td><td class="r"></td></tr><tr><td>Qwen3.6-35B-A3B</td><td class="r" data-s="16.19082483091965">16.2</td><td class="r" data-s="262144">262k</td><td class="r fits" data-s="36.9"><span class="q">Q8_0</span> <span class="gb">37 GB</span></td><td class="r fits" data-s="36.9"><span class="q">Q8_0</span> <span class="gb">37 GB</span></td><td class="r fits" data-s="26.6"><span class="q">UD-Q5_K_XL</span> <span class="gb">27 GB</span></td><td class="r nofit" data-s="19.5"><span class="q">UD-IQ4_NL_XL</span> <span class="gb">20 GB</span></td></tr><tr><td>Gemma 4 31B</td><td class="r" data-s="15.806948103252044">15.8</td><td class="r" data-s="262144">262k</td><td class="r fits" data-s="32.6"><span class="q">Q8_0</span> <span class="gb">33 GB</span></td><td class="r fits" data-s="32.6"><span class="q">Q8_0</span> <span class="gb">33 GB</span></td><td class="r fits" data-s="27.5"><span class="q">UD-Q6_K_XL</span> <span class="gb">28 GB</span></td><td class="r nofit" data-s="18.3"><span class="q">Q4_K_M</span> <span class="gb">18 GB</span></td></tr><tr><td>Qwen3 Coder Next</td><td class="r" data-s="14.283251268776736">14.3</td><td class="r" data-s="262144">262k</td><td class="r fits" data-s="84.8"><span class="q">Q8_0</span> <span class="gb">85 GB</span></td><td class="r nofit" data-s="55.8"><span class="q">UD-Q5_K_S</span> <span class="gb">56 GB</span></td><td class="r"></td><td class="r"></td></tr><tr><td>Qwen3.6-27B</td><td class="r" data-s="13.317904173386546">13.3</td><td class="r" data-s="262144">262k</td><td class="r fits" data-s="28.6"><span class="q">Q8_0</span> <span class="gb">29 GB</span></td><td class="r fits" data-s="28.6"><span class="q">Q8_0</span> <span class="gb">29 GB</span></td><td class="r fits" data-s="25.6"><span class="q">UD-Q6_K_XL</span> <span class="gb">26 GB</span></td><td class="r fits" data-s="20.0"><span class="q">UD-Q5_K_XL</span> <span class="gb">20 GB</span></td></tr>
377377
</table></div>
378378
</section>
379379

0 commit comments

Comments
 (0)