|
| 1 | +/** |
| 2 | + * UNIT TESTS — degraded-concurrency tracker + backoff (src/zero-tvm/map-limited.ts). |
| 3 | + * |
| 4 | + * The weight loader drops shard concurrency 8 → 3 on the first mid-stream |
| 5 | + * failure and recovers after ten consecutive clean fetches. That policy used |
| 6 | + * to live in weight-loader.ts module state — next to GPUBufferUsage reads |
| 7 | + * that make the module unimportable in Node — so no test could touch it. |
| 8 | + * createDegradedTracker carries the identical policy in the import-safe |
| 9 | + * module; these tests pin it, including the streak-reset edge the old code |
| 10 | + * got wrong twice (first as a never-resetting flag, then as a streak the |
| 11 | + * failure path never zeroed). |
| 12 | + */ |
| 13 | + |
| 14 | +import { describe, test, expect } from 'vitest' |
| 15 | +import { backoffMs, createDegradedTracker } from '../../src/zero-tvm/map-limited.js' |
| 16 | + |
| 17 | +describe('backoffMs', () => { |
| 18 | + test('stays inside [0.5, 1.5) × base·2^attempt', () => { |
| 19 | + for (let attempt = 0; attempt < 6; attempt++) { |
| 20 | + for (let i = 0; i < 200; i++) { |
| 21 | + const ms = backoffMs(attempt) |
| 22 | + expect(ms).toBeGreaterThanOrEqual(0.5 * 500 * 2 ** attempt) |
| 23 | + expect(ms).toBeLessThan(1.5 * 500 * 2 ** attempt) |
| 24 | + } |
| 25 | + } |
| 26 | + }) |
| 27 | + |
| 28 | + test('honors a custom base', () => { |
| 29 | + for (let i = 0; i < 100; i++) { |
| 30 | + const ms = backoffMs(2, 100) |
| 31 | + expect(ms).toBeGreaterThanOrEqual(200) |
| 32 | + expect(ms).toBeLessThan(600) |
| 33 | + } |
| 34 | + }) |
| 35 | + |
| 36 | + test('means double per attempt: E[jitter] is 1.0', () => { |
| 37 | + // Bands overlap ([0.5,1.5) spans 3x against 2x growth), so per-sample |
| 38 | + // ordering is NOT guaranteed — but the mean of attempt a+1 is 2x the |
| 39 | + // mean of attempt a. 2000 samples make ±10% watertight (SE ≈ 0.6%). |
| 40 | + for (let attempt = 0; attempt < 4; attempt++) { |
| 41 | + let sum = 0 |
| 42 | + const N = 2000 |
| 43 | + for (let i = 0; i < N; i++) sum += backoffMs(attempt) |
| 44 | + const mean = sum / N |
| 45 | + const expected = 500 * 2 ** attempt |
| 46 | + expect(mean).toBeGreaterThan(expected * 0.9) |
| 47 | + expect(mean).toBeLessThan(expected * 1.1) |
| 48 | + } |
| 49 | + }) |
| 50 | +}) |
| 51 | + |
| 52 | +describe('createDegradedTracker', () => { |
| 53 | + test('starts healthy at full concurrency, successes are no-ops', () => { |
| 54 | + const t = createDegradedTracker({ full: 8, degraded: 3 }) |
| 55 | + expect(t.degraded).toBe(false) |
| 56 | + expect(t.limitOf()).toBe(8) |
| 57 | + const notes: string[] = [] |
| 58 | + t.markSuccess((m) => notes.push(m)) |
| 59 | + expect(t.degraded).toBe(false) |
| 60 | + expect(notes).toEqual([]) |
| 61 | + }) |
| 62 | + |
| 63 | + test('first transient degrades once and announces', () => { |
| 64 | + const t = createDegradedTracker({ full: 8, degraded: 3 }) |
| 65 | + const notes: string[] = [] |
| 66 | + t.markTransient((m) => notes.push(m)) |
| 67 | + expect(t.degraded).toBe(true) |
| 68 | + expect(t.limitOf()).toBe(3) |
| 69 | + expect(notes).toHaveLength(1) |
| 70 | + expect(notes[0]).toContain('8 → 3') |
| 71 | + // A second failure while degraded announces nothing new. |
| 72 | + t.markTransient((m) => notes.push(m)) |
| 73 | + expect(notes).toHaveLength(1) |
| 74 | + }) |
| 75 | + |
| 76 | + test('nine clean fetches do not recover; the tenth does', () => { |
| 77 | + const t = createDegradedTracker({ full: 8, degraded: 3 }) |
| 78 | + const notes: string[] = [] |
| 79 | + t.markTransient() |
| 80 | + for (let i = 0; i < 9; i++) t.markSuccess((m) => notes.push(m)) |
| 81 | + expect(t.degraded).toBe(true) |
| 82 | + expect(t.limitOf()).toBe(3) |
| 83 | + expect(notes).toEqual([]) |
| 84 | + t.markSuccess((m) => notes.push(m)) |
| 85 | + expect(t.degraded).toBe(false) |
| 86 | + expect(t.limitOf()).toBe(8) |
| 87 | + expect(notes).toHaveLength(1) |
| 88 | + expect(notes[0]).toContain('3 → 8') |
| 89 | + }) |
| 90 | + |
| 91 | + test('a failure zeroes the streak: nine + failure + nine stays degraded', () => { |
| 92 | + const t = createDegradedTracker({ full: 8, degraded: 3 }) |
| 93 | + t.markTransient() |
| 94 | + for (let i = 0; i < 9; i++) t.markSuccess() |
| 95 | + t.markTransient() |
| 96 | + for (let i = 0; i < 9; i++) t.markSuccess() |
| 97 | + expect(t.degraded).toBe(true) |
| 98 | + t.markSuccess() |
| 99 | + expect(t.degraded).toBe(false) |
| 100 | + }) |
| 101 | + |
| 102 | + test('recovery resets the streak: the next degradation needs ten again', () => { |
| 103 | + // Strictly, the reset is hygiene, not a behavior change: every degraded |
| 104 | + // episode begins with markTransient (the only path that sets degraded), |
| 105 | + // which zeroes the streak first — so the counter is 0 at each episode |
| 106 | + // start with or without the reset. It is kept because the invariant |
| 107 | + // ("streak counts clean fetches in the CURRENT episode") should hold |
| 108 | + // structurally, not incidentally via call order. This test pins the |
| 109 | + // ten-again behavior against a future refactor that breaks that order. |
| 110 | + const t = createDegradedTracker({ full: 8, degraded: 3 }) |
| 111 | + t.markTransient() |
| 112 | + for (let i = 0; i < 10; i++) t.markSuccess() |
| 113 | + expect(t.degraded).toBe(false) |
| 114 | + t.markTransient() |
| 115 | + t.markSuccess() |
| 116 | + expect(t.degraded).toBe(true) |
| 117 | + for (let i = 0; i < 9; i++) t.markSuccess() |
| 118 | + expect(t.degraded).toBe(false) |
| 119 | + }) |
| 120 | + |
| 121 | + test('recoverAfter is configurable', () => { |
| 122 | + const t = createDegradedTracker({ full: 4, degraded: 1, recoverAfter: 2 }) |
| 123 | + t.markTransient() |
| 124 | + t.markSuccess() |
| 125 | + expect(t.degraded).toBe(true) |
| 126 | + t.markSuccess() |
| 127 | + expect(t.degraded).toBe(false) |
| 128 | + expect(t.limitOf()).toBe(4) |
| 129 | + }) |
| 130 | +}) |
0 commit comments