|
| 1 | +import assert from 'node:assert/strict'; |
| 2 | +import { spawnSync } from 'node:child_process'; |
| 3 | +import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; |
| 4 | +import { tmpdir } from 'node:os'; |
| 5 | +import { join } from 'node:path'; |
| 6 | +import { test } from 'node:test'; |
| 7 | +import { fileURLToPath } from 'node:url'; |
| 8 | +import { runInNewContext } from 'node:vm'; |
| 9 | +import { ModuleKind, ScriptTarget, transpileModule } from 'typescript'; |
| 10 | + |
| 11 | +test('partial device runs replace matching variants and retain other published results', () => { |
| 12 | + const exports = {}; |
| 13 | + const source = readFileSync( |
| 14 | + new URL('../../../docs/src/data/benchmarks.ts', import.meta.url), |
| 15 | + 'utf8' |
| 16 | + ); |
| 17 | + const code = transpileModule(source, { |
| 18 | + compilerOptions: { module: ModuleKind.CommonJS, target: ScriptTarget.ES2022 }, |
| 19 | + }).outputText; |
| 20 | + runInNewContext(code, { exports }); |
| 21 | + const old = { |
| 22 | + device: 's20plus', |
| 23 | + task: 'llm', |
| 24 | + model: 'test', |
| 25 | + backend: 'xnnpack', |
| 26 | + precision: '8da4w', |
| 27 | + medianMs: 10, |
| 28 | + }; |
| 29 | + const fresh = { ...old, medianMs: 5 }; |
| 30 | + const otherDevice = { ...old, device: 's26ultra' }; |
| 31 | + const otherTask = { ...old, task: 'classification' }; |
| 32 | + const otherBackend = { ...old, backend: 'vulkan' }; |
| 33 | + const otherPrecision = { ...old, precision: '8da8w' }; |
| 34 | + const rows = exports.mergeBenchmarkRows( |
| 35 | + [fresh], |
| 36 | + [old, otherDevice, otherTask, otherBackend, otherPrecision] |
| 37 | + ); |
| 38 | + assert.equal(rows.length, 5); |
| 39 | + assert.equal(rows[0], fresh); |
| 40 | + assert.equal(rows[1], otherDevice); |
| 41 | + assert.equal(rows[2], otherTask); |
| 42 | + assert.equal(rows[3], otherBackend); |
| 43 | + assert.equal(rows[4], otherPrecision); |
| 44 | +}); |
| 45 | + |
| 46 | +test('docs generation publishes valid LLM throughput and rejects missing token stats', () => { |
| 47 | + const page = mkdtempSync(join(tmpdir(), 'rne-docs-test-')); |
| 48 | + try { |
| 49 | + const dir = join(page, 'runs', 'iphone17'); |
| 50 | + mkdirSync(dir, { recursive: true }); |
| 51 | + const file = join(dir, 'test.jsonl'); |
| 52 | + const header = { type: 'run', label: 'test', device: { model: 'test' } }; |
| 53 | + const entry = { |
| 54 | + id: 'llm/test-xnnpack-8-da4-w', |
| 55 | + status: 'ok', |
| 56 | + backend: 'xnnpack', |
| 57 | + precision: '8da4w', |
| 58 | + pipeline: { mean: 100, median: 100 }, |
| 59 | + units: 64, |
| 60 | + detail: { timeToFirstTokenMs: 5 }, |
| 61 | + taskLoadMs: 10, |
| 62 | + bytes: 1024 ** 2, |
| 63 | + memory: { peakMb: 20 }, |
| 64 | + }; |
| 65 | + const convert = () => |
| 66 | + spawnSync( |
| 67 | + process.execPath, |
| 68 | + [fileURLToPath(new URL('./docs-data.mjs', import.meta.url)), page], |
| 69 | + { encoding: 'utf8' } |
| 70 | + ); |
| 71 | + const write = (measurement) => |
| 72 | + writeFileSync(file, `${JSON.stringify(header)}\n${JSON.stringify(measurement)}\n`); |
| 73 | + write(entry); |
| 74 | + const valid = convert(); |
| 75 | + assert.equal(valid.status, 0, valid.stderr); |
| 76 | + const [row] = JSON.parse(readFileSync(join(page, 'measured.json'), 'utf8')); |
| 77 | + assert.equal(row.tokensPerSecond, 640); |
| 78 | + assert.equal(row.ttftMs, 5); |
| 79 | + write({ ...entry, units: undefined }); |
| 80 | + const invalid = convert(); |
| 81 | + assert.equal(invalid.status, 2); |
| 82 | + assert.match(invalid.stderr, /missing valid token or TTFT stats/); |
| 83 | + } finally { |
| 84 | + rmSync(page, { recursive: true, force: true }); |
| 85 | + } |
| 86 | +}); |
0 commit comments