|
| 1 | +import assert from 'node:assert/strict'; |
| 2 | +import { execFile } from 'node:child_process'; |
| 3 | +import { mkdtemp, mkdir, rm, writeFile } from 'node:fs/promises'; |
| 4 | +import { tmpdir } from 'node:os'; |
| 5 | +import path from 'node:path'; |
| 6 | +import { test } from 'node:test'; |
| 7 | +import { fileURLToPath } from 'node:url'; |
| 8 | +import { promisify } from 'node:util'; |
| 9 | + |
| 10 | +import { comparePerformance } from '../../scripts/comparators/performance.mjs'; |
| 11 | + |
| 12 | +const benchmark = { |
| 13 | + elapsedSeconds: 2, |
| 14 | + userCpuSeconds: 1, |
| 15 | + systemCpuSeconds: 0.5, |
| 16 | + maxRssKiB: 1024, |
| 17 | +}; |
| 18 | + |
| 19 | +const executeFile = promisify(execFile); |
| 20 | +const comparatorDirectory = fileURLToPath( |
| 21 | + new URL('../../scripts/comparators/', import.meta.url) |
| 22 | +); |
| 23 | + |
| 24 | +const createDirectories = async t => { |
| 25 | + const root = await mkdtemp(path.join(tmpdir(), 'doc-kit-')); |
| 26 | + const base = path.join(root, 'base'); |
| 27 | + const head = path.join(root, 'head'); |
| 28 | + |
| 29 | + await Promise.all([ |
| 30 | + mkdir(base, { recursive: true }), |
| 31 | + mkdir(head, { recursive: true }), |
| 32 | + ]); |
| 33 | + t.after(() => rm(root, { recursive: true, force: true })); |
| 34 | + |
| 35 | + return { base, head }; |
| 36 | +}; |
| 37 | + |
| 38 | +const writeBenchmark = (directory, value = benchmark) => |
| 39 | + writeFile( |
| 40 | + path.join(directory, 'benchmark.json'), |
| 41 | + JSON.stringify(value), |
| 42 | + 'utf8' |
| 43 | + ); |
| 44 | + |
| 45 | +const runComparator = async (name, base, head) => { |
| 46 | + const { stdout } = await executeFile( |
| 47 | + process.execPath, |
| 48 | + [path.join(comparatorDirectory, `${name}.mjs`)], |
| 49 | + { |
| 50 | + env: { ...process.env, BASE: base, HEAD: head, GENERATOR: 'test' }, |
| 51 | + } |
| 52 | + ); |
| 53 | + |
| 54 | + return stdout; |
| 55 | +}; |
| 56 | + |
| 57 | +test('comparePerformance formats benchmark differences', async t => { |
| 58 | + const { base, head } = await createDirectories(t); |
| 59 | + |
| 60 | + await Promise.all([ |
| 61 | + writeBenchmark(base), |
| 62 | + writeBenchmark(head, { |
| 63 | + elapsedSeconds: 3, |
| 64 | + userCpuSeconds: 0.75, |
| 65 | + systemCpuSeconds: 0.5, |
| 66 | + maxRssKiB: 1536, |
| 67 | + }), |
| 68 | + ]); |
| 69 | + |
| 70 | + const result = await comparePerformance(base, head); |
| 71 | + |
| 72 | + assert.match( |
| 73 | + result, |
| 74 | + /Elapsed time \| 2\.00 s \| 3\.00 s \| \+1\.00 s \(\+50\.00%\)/ |
| 75 | + ); |
| 76 | + assert.match( |
| 77 | + result, |
| 78 | + /User CPU time \| 1\.00 s \| 750\.00 ms \| -250\.00 ms \(-25\.00%\)/ |
| 79 | + ); |
| 80 | + assert.match( |
| 81 | + result, |
| 82 | + /Peak resident memory \| 1\.00 MB \| 1\.50 MB \| \+512\.00 KB \(\+50\.00%\)/ |
| 83 | + ); |
| 84 | +}); |
| 85 | + |
| 86 | +test('comparePerformance omits results when an artifact has no benchmark', async t => { |
| 87 | + const { base, head } = await createDirectories(t); |
| 88 | + |
| 89 | + await writeBenchmark(head); |
| 90 | + |
| 91 | + assert.equal(await comparePerformance(base, head), ''); |
| 92 | +}); |
| 93 | + |
| 94 | +test('comparePerformance rejects invalid benchmark values', async t => { |
| 95 | + const { base, head } = await createDirectories(t); |
| 96 | + |
| 97 | + await Promise.all([ |
| 98 | + writeBenchmark(base), |
| 99 | + writeBenchmark(head, { ...benchmark, maxRssKiB: undefined }), |
| 100 | + ]); |
| 101 | + |
| 102 | + await assert.rejects( |
| 103 | + comparePerformance(base, head), |
| 104 | + /Invalid peak resident memory benchmark value/ |
| 105 | + ); |
| 106 | +}); |
| 107 | + |
| 108 | +test('file-size comparator combines output and performance results', async t => { |
| 109 | + const { base, head } = await createDirectories(t); |
| 110 | + |
| 111 | + await Promise.all([ |
| 112 | + writeFile(path.join(base, 'result.txt'), 'base', 'utf8'), |
| 113 | + writeFile(path.join(head, 'result.txt'), 'a larger result', 'utf8'), |
| 114 | + writeBenchmark(base), |
| 115 | + writeBenchmark(head, { ...benchmark, elapsedSeconds: 3 }), |
| 116 | + ]); |
| 117 | + |
| 118 | + const result = await runComparator('file-size', base, head); |
| 119 | + |
| 120 | + assert.equal(result.match(/## `test` Generator/g)?.length, 1); |
| 121 | + assert.match(result, /### Output size/); |
| 122 | + assert.match(result, /`result\.txt`/); |
| 123 | + assert.match(result, /### Performance/); |
| 124 | + assert.doesNotMatch(result, /benchmark\.json/); |
| 125 | +}); |
| 126 | + |
| 127 | +test('object comparator treats benchmark data as metadata', async t => { |
| 128 | + const { base, head } = await createDirectories(t); |
| 129 | + |
| 130 | + await Promise.all([ |
| 131 | + writeFile(path.join(base, 'result.json'), '{"value":true}', 'utf8'), |
| 132 | + writeFile(path.join(head, 'result.json'), '{"value":true}', 'utf8'), |
| 133 | + writeBenchmark(base), |
| 134 | + writeBenchmark(head, { ...benchmark, maxRssKiB: 2048 }), |
| 135 | + ]); |
| 136 | + |
| 137 | + const result = await runComparator('object-assertion', base, head); |
| 138 | + |
| 139 | + assert.equal(result.match(/## `test` Generator/g)?.length, 1); |
| 140 | + assert.doesNotMatch(result, /### Output\n/); |
| 141 | + assert.match(result, /### Performance/); |
| 142 | + assert.doesNotMatch(result, /benchmark\.json/); |
| 143 | +}); |
0 commit comments