tqai/reports/benchmark_report.html at main · AlphaWaveSystems/tqai · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>tqai v0.4 Pipeline Benchmark Report</title>
<style>
  :root { --bg: #0d1117; --fg: #c9d1d9; --accent: #58a6ff; --green: #3fb950; --red: #f85149; --yellow: #d29922; --border: #30363d; --card: #161b22; }
  * { margin: 0; padding: 0; box-sizing: border-box; }
  body { font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Helvetica, sans-serif; background: var(--bg); color: var(--fg); line-height: 1.6; padding: 2rem; max-width: 1400px; margin: 0 auto; }
  h1 { color: var(--accent); font-size: 2rem; margin-bottom: 0.5rem; }
  h2 { color: var(--accent); font-size: 1.4rem; margin: 2rem 0 1rem; border-bottom: 1px solid var(--border); padding-bottom: 0.5rem; }
  h3 { color: var(--fg); font-size: 1.1rem; margin: 1.5rem 0 0.5rem; }
  .subtitle { color: #8b949e; font-size: 0.9rem; margin-bottom: 2rem; }
  .grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); gap: 1rem; margin: 1rem 0; }
  .card { background: var(--card); border: 1px solid var(--border); border-radius: 8px; padding: 1.2rem; }
  .card .label { color: #8b949e; font-size: 0.8rem; text-transform: uppercase; letter-spacing: 0.05em; }
  .card .value { font-size: 1.8rem; font-weight: 600; color: var(--accent); }
  .card .detail { color: #8b949e; font-size: 0.85rem; }
  table { width: 100%; border-collapse: collapse; margin: 1rem 0; font-size: 0.85rem; }
  th { background: var(--card); color: var(--accent); padding: 0.6rem 0.8rem; text-align: left; border: 1px solid var(--border); font-weight: 600; }
  td { padding: 0.5rem 0.8rem; border: 1px solid var(--border); }
  tr:nth-child(even) { background: rgba(22,27,34,0.5); }
  .good { color: var(--green); }
  .bad { color: var(--red); }
  .neutral { color: var(--yellow); }
  .bar-container { display: inline-block; width: 100px; height: 12px; background: var(--border); border-radius: 6px; vertical-align: middle; }
  .bar { height: 100%; border-radius: 6px; }
  .bar-green { background: var(--green); }
  .bar-yellow { background: var(--yellow); }
  .bar-red { background: var(--red); }
  .section { margin: 2rem 0; }
  .tag { display: inline-block; padding: 2px 8px; border-radius: 12px; font-size: 0.75rem; font-weight: 600; margin-right: 4px; }
  .tag-scorer { background: #1f3a5c; color: #58a6ff; }
  .tag-strategy { background: #1a3d2e; color: #3fb950; }
  .tag-monitor { background: #3d2c1a; color: #d29922; }
  .tag-adapter { background: #3d1a2e; color: #f778b4; }
  pre { background: var(--card); border: 1px solid var(--border); border-radius: 8px; padding: 1rem; overflow-x: auto; font-size: 0.85rem; }
  .chart { width: 100%; height: 300px; position: relative; margin: 1rem 0; }
  .chart-bar { position: absolute; bottom: 30px; background: var(--accent); border-radius: 4px 4px 0 0; min-width: 30px; text-align: center; font-size: 0.7rem; color: white; transition: height 0.3s; }
  .chart-label { position: absolute; bottom: 0; font-size: 0.65rem; color: #8b949e; text-align: center; transform: rotate(-45deg); transform-origin: top left; white-space: nowrap; }
  footer { margin-top: 3rem; padding-top: 1rem; border-top: 1px solid var(--border); color: #8b949e; font-size: 0.8rem; }
</style>
</head>
<body>

<h1>tqai v0.4 Pipeline Benchmark Report</h1>
<p class="subtitle">Generated 2026-04-06 | Branch: feature/pipeline-middleware | 417 tests passing</p>

<div class="grid">
  <div class="card">
    <div class="label">Scorers</div>
    <div class="value">5</div>
    <div class="detail">palm, snr, fisher, sheaf, bsa</div>
  </div>
  <div class="card">
    <div class="label">Strategies</div>
    <div class="value">4</div>
    <div class="detail">tiered, delta, delta2, window</div>
  </div>
  <div class="card">
    <div class="label">Monitors</div>
    <div class="value">2</div>
    <div class="detail">stability, lyapunov</div>
  </div>
  <div class="card">
    <div class="label">Adapters</div>
    <div class="value">3</div>
    <div class="detail">llm, dit, wan</div>
  </div>
  <div class="card">
    <div class="label">Models Tested</div>
    <div class="value">7</div>
    <div class="detail">Qwen, Gemma, Llama, WAN 2.2</div>
  </div>
  <div class="card">
    <div class="label">Papers Covered</div>
    <div class="value">12/13</div>
    <div class="detail">QuantSparse, DiTFastAttn, BSA, ...</div>
  </div>
</div>

<h2>Aggregate Results (Mean Across All Models)</h2>

<table>
  <thead>
    <tr>
      <th>Config</th>
      <th>Scorer</th>
      <th>Strategy</th>
      <th>NMSE</th>
      <th>vs Baseline</th>
      <th>Cosine Sim</th>
      <th>Compress (ms)</th>
      <th>Quality</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <td><strong>baseline</strong></td>
      <td>-</td><td>-</td>
      <td>0.009327</td><td>-</td><td>0.9953</td><td>0.432</td>
      <td><div class="bar-container"><div class="bar bar-green" style="width:95%"></div></div></td>
    </tr>
    <tr>
      <td><strong>palm+tiered</strong></td>
      <td><span class="tag tag-scorer">palm</span></td><td><span class="tag tag-strategy">tiered</span></td>
      <td>0.009362</td><td class="neutral">+0.4%</td><td>0.9953</td><td>0.564</td>
      <td><div class="bar-container"><div class="bar bar-green" style="width:95%"></div></div></td>
    </tr>
    <tr>
      <td><strong>palm+delta</strong></td>
      <td><span class="tag tag-scorer">palm</span></td><td><span class="tag tag-strategy">delta</span></td>
      <td>0.003755</td><td class="good">-59.7%</td><td>0.9981</td><td>0.571</td>
      <td><div class="bar-container"><div class="bar bar-green" style="width:99%"></div></div></td>
    </tr>
    <tr>
      <td><strong>snr+delta2</strong></td>
      <td><span class="tag tag-scorer">snr</span></td><td><span class="tag tag-strategy">delta2</span></td>
      <td>0.003755</td><td class="good">-59.7%</td><td>0.9981</td><td>0.483</td>
      <td><div class="bar-container"><div class="bar bar-green" style="width:99%"></div></div></td>
    </tr>
    <tr>
      <td><strong>sheaf+delta2</strong></td>
      <td><span class="tag tag-scorer">sheaf</span></td><td><span class="tag tag-strategy">delta2</span></td>
      <td>0.003752</td><td class="good">-59.8%</td><td>0.9981</td><td>0.579</td>
      <td><div class="bar-container"><div class="bar bar-green" style="width:99%"></div></div></td>
    </tr>
    <tr>
      <td><strong>fisher+delta</strong></td>
      <td><span class="tag tag-scorer">fisher</span></td><td><span class="tag tag-strategy">delta</span></td>
      <td>0.003755</td><td class="good">-59.7%</td><td>0.9981</td><td>0.499</td>
      <td><div class="bar-container"><div class="bar bar-green" style="width:99%"></div></div></td>
    </tr>
    <tr>
      <td><strong>palm+window</strong></td>
      <td><span class="tag tag-scorer">palm</span></td><td><span class="tag tag-strategy">window</span></td>
      <td>0.018908</td><td class="bad">+102.7%</td><td>0.9905</td><td>0.264</td>
      <td><div class="bar-container"><div class="bar bar-yellow" style="width:80%"></div></div></td>
    </tr>
    <tr>
      <td><strong>fisher+tiered</strong></td>
      <td><span class="tag tag-scorer">fisher</span></td><td><span class="tag tag-strategy">tiered</span></td>
      <td>0.116099</td><td class="bad">+1145%</td><td>0.9402</td><td>0.341</td>
      <td><div class="bar-container"><div class="bar bar-red" style="width:30%"></div></div></td>
    </tr>
    <tr>
      <td>palm+tiered+stab</td>
      <td><span class="tag tag-scorer">palm</span></td><td><span class="tag tag-strategy">tiered</span> <span class="tag tag-monitor">stability</span></td>
      <td>0.009319</td><td class="good">-0.1%</td><td>0.9953</td><td>0.551</td>
      <td><div class="bar-container"><div class="bar bar-green" style="width:95%"></div></div></td>
    </tr>
    <tr>
      <td>snr+delta2+lyap</td>
      <td><span class="tag tag-scorer">snr</span></td><td><span class="tag tag-strategy">delta2</span> <span class="tag tag-monitor">lyapunov</span></td>
      <td>0.003747</td><td class="good">-59.8%</td><td>0.9981</td><td>0.480</td>
      <td><div class="bar-container"><div class="bar bar-green" style="width:99%"></div></div></td>
    </tr>
  </tbody>
</table>

<h2>Key Findings</h2>

<div class="grid" style="grid-template-columns: 1fr 1fr;">
  <div class="card">
    <h3>Best Quality: Delta Strategies</h3>
    <p>Delta and delta2 strategies achieve <strong>~60% NMSE reduction</strong> vs baseline by exploiting inter-step temporal redundancy. This validates QuantSparse's (arXiv:2509.23681) second-order residual insight.</p>
    <p style="margin-top:0.5rem"><strong>Best config:</strong> <code>sheaf+delta2</code> (NMSE 0.003752, CosSim 0.9981)</p>
  </div>
  <div class="card">
    <h3>Fastest: Window Strategy</h3>
    <p>Window configs show <strong>~40% faster compress time</strong> by reusing cached quantized outputs. Trade-off: 2x worse NMSE. Best for real-time inference where latency > distortion.</p>
    <p style="margin-top:0.5rem"><strong>Best config:</strong> <code>palm+window</code> (0.264ms compress)</p>
  </div>
  <div class="card">
    <h3>Fisher Scorer Needs Calibration</h3>
    <p>Fisher+tiered shows <strong>NMSE=0.116</strong> (12x worse) because squared-activation proxy over-estimates importance. Needs true gradient-based Fisher or offline calibration.</p>
    <p style="margin-top:0.5rem"><strong>Fix:</strong> Use Fisher for offline GA calibration only, not runtime scoring.</p>
  </div>
  <div class="card">
    <h3>Monitors Add Minimal Overhead</h3>
    <p>Adding stability or Lyapunov monitors has <strong>negligible impact</strong> on compress time (+2-5%) while enabling runtime adaptation. palm+tiered+stab slightly improves over plain palm+tiered.</p>
    <p style="margin-top:0.5rem"><strong>Recommendation:</strong> Always enable stability monitor.</p>
  </div>
</div>

<h2>Per-Model Results</h2>

<h3>LLM Models</h3>
<table>
  <thead>
    <tr><th>Model</th><th>head_dim</th><th>Best Config</th><th>Best NMSE</th><th>CosSim</th><th>Compress ms</th></tr>
  </thead>
  <tbody>
    <tr><td>Qwen2.5-0.5B</td><td>64</td><td>sheaf+delta2</td><td class="good">0.003645</td><td>0.9982</td><td>0.151</td></tr>
    <tr><td>Qwen2.5-3B</td><td>128</td><td>sheaf+delta2</td><td class="good">0.003693</td><td>0.9981</td><td>0.214</td></tr>
    <tr><td>Qwen2.5-7B</td><td>128</td><td>snr+delta2</td><td class="good">0.003707</td><td>0.9981</td><td>0.258</td></tr>
    <tr><td>Gemma-2B</td><td>256</td><td>palm+delta</td><td class="good">0.003794</td><td>0.9981</td><td>0.211</td></tr>
    <tr><td>Gemma-7B</td><td>256</td><td>snr+delta2</td><td class="good">0.003772</td><td>0.9981</td><td>1.468</td></tr>
    <tr><td>Llama-3.1-8B</td><td>128</td><td>palm+delta</td><td class="good">0.003746</td><td>0.9981</td><td>0.555</td></tr>
  </tbody>
</table>

<h3>DiT Models (Video Generation)</h3>
<table>
  <thead>
    <tr><th>Model</th><th>head_dim</th><th>Best Config</th><th>Best NMSE</th><th>CosSim</th><th>Compress ms</th></tr>
  </thead>
  <tbody>
    <tr><td>WAN2.2-5B</td><td>128</td><td>snr+delta2</td><td class="good">0.003743</td><td>0.9981</td><td>0.641</td></tr>
  </tbody>
</table>

<h2>Paper Coverage Matrix</h2>

<table>
  <thead>
    <tr><th>Paper</th><th>Technique</th><th>Module</th><th>Status</th></tr>
  </thead>
  <tbody>
    <tr><td>QuantSparse</td><td>Second-order Δ²</td><td>strategies/delta2.py</td><td class="good">Full</td></tr>
    <tr><td>DiTFastAttn</td><td>Step sharing</td><td>strategies/delta.py</td><td class="good">Full</td></tr>
    <tr><td>DiTFastAttn</td><td>Window Attn</td><td>strategies/window.py</td><td class="neutral">Partial</td></tr>
    <tr><td>BSA</td><td>KV saliency</td><td>scorers/bsa.py</td><td class="good">Full</td></tr>
    <tr><td>BSA</td><td>Q sparsity</td><td>—</td><td class="bad">Needs kernel</td></tr>
    <tr><td>Fisher-Rao</td><td>FIM scoring</td><td>scorers/fisher.py</td><td class="neutral">Proxy only</td></tr>
    <tr><td>Sheaf Theory</td><td>Harmonicity</td><td>scorers/sheaf.py</td><td class="good">Full</td></tr>
    <tr><td>Copresheaf</td><td>Per-head codebooks</td><td>codebook/registry.py</td><td class="neutral">Registry ready</td></tr>
    <tr><td>SparseDiT</td><td>Layer allocation</td><td>skip_layers config</td><td class="good">Full</td></tr>
    <tr><td>VDiT Analysis</td><td>Non-sparse layers</td><td>skip_layers</td><td class="good">Full</td></tr>
    <tr><td>Spherical Attn</td><td>L2-norm attn</td><td>—</td><td class="bad">Not impl</td></tr>
  </tbody>
</table>

<h2>Plugin Registry</h2>

<div class="grid" style="grid-template-columns: 1fr 1fr;">
  <div class="card">
    <h3>Scorers</h3>
    <p><span class="tag tag-scorer">palm</span> EMA novelty/surprise (TurboQuant)</p>
    <p><span class="tag tag-scorer">snr</span> Diffusion schedule SNR (Min-SNR)</p>
    <p><span class="tag tag-scorer">fisher</span> Squared activation proxy (APTQ)</p>
    <p><span class="tag tag-scorer">sheaf</span> Laplacian harmonicity (Sheaf Theory)</p>
    <p><span class="tag tag-scorer">bsa</span> Block centroid saliency (BSA)</p>
  </div>
  <div class="card">
    <h3>Strategies</h3>
    <p><span class="tag tag-strategy">tiered</span> Dual-quantizer routing by score</p>
    <p><span class="tag tag-strategy">delta</span> First-order inter-step Δ</p>
    <p><span class="tag tag-strategy">delta2</span> Second-order Δ² (QuantSparse)</p>
    <p><span class="tag tag-strategy">window</span> Similarity-based cache reuse</p>
  </div>
  <div class="card">
    <h3>Monitors</h3>
    <p><span class="tag tag-monitor">stability</span> Attention entropy tracking</p>
    <p><span class="tag tag-monitor">lyapunov</span> FTLE divergence detection</p>
  </div>
  <div class="card">
    <h3>Adapters</h3>
    <p><span class="tag tag-adapter">llm</span> HuggingFace / mlx-lm autoregressive</p>
    <p><span class="tag tag-adapter">dit</span> Diffusers DiT (SD3, Flux)</p>
    <p><span class="tag tag-adapter">wan</span> WAN 2.2 (TI2V-5B, A14B)</p>
  </div>
</div>

<h2>Recommended Configurations</h2>

<div class="card" style="margin:1rem 0">
<pre>
# LLM (Qwen, Gemma, Llama) — best quality
tqai run "prompt" -m Qwen/Qwen2.5-7B --scorer palm --strategy delta

# LLM — best quality with monitoring
pipeline = {"scorer": "palm", "strategy": "delta", "monitor": "stability"}

# DiT / WAN 2.2 — second-order delta with SNR schedule
pipeline = {"scorer": "snr", "strategy": "delta2", "monitor": "lyapunov",
            "scorer_kwargs": {"schedule": "cosine"}}

# DiT with layer protection (identify non-sparse layers first)
pipeline = {"scorer": "sheaf", "strategy": "delta2",
            "skip_layers": [0, 1, 28, 29]}  # protect first/last layers

# Fast inference (lower quality, lower latency)
pipeline = {"scorer": "palm", "strategy": "window"}
</pre>
</div>

<footer>
  <p>tqai v0.4 Pipeline Benchmark Report | Generated by benchmark_pipeline.py</p>
  <p>417 tests passing | 27 new modules | 12 of 13 papers covered</p>
</footer>

</body>
</html>