Skip to content

Commit 6bdfe2f

Browse files
authored
add memory benchmarking (#906)
1 parent b9750dc commit 6bdfe2f

8 files changed

Lines changed: 309 additions & 14 deletions

File tree

‎.github/workflows/generate.yml‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -146,7 +146,10 @@ jobs:
146146

147147
- name: Generate ${{ matrix.target }}
148148
run: |
149-
node bin/cli.mjs generate \
149+
/usr/bin/time \
150+
--output out/benchmark.json \
151+
--format '{"elapsedSeconds": %e, "userCpuSeconds": %U, "systemCpuSeconds": %S, "maxRssKiB": %M}' \
152+
node bin/cli.mjs generate \
150153
-t ${{ matrix.target }} \
151154
-i "${{ matrix.input }}" \
152155
-o out \

‎docs/comparators.md‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,14 +7,15 @@ This guide explains how to create build comparison scripts for `@node-core/doc-k
77
Comparators are scripts that:
88

99
1. **Compare** generated documentation between two builds (base vs. head)
10-
2. **Identify differences** in content, structure, or file size
10+
2. **Identify differences** in content, structure, file size, or performance
1111
3. **Report results** in a format suitable for CI/CD systems
1212
4. **Help catch regressions** before merging changes
1313

1414
### When to Use Comparators
1515

1616
- **Verify backward compatibility** - Ensure new code produces same output
1717
- **Track file size changes** - Monitor bundle size growth
18+
- **Catch performance regressions** - Compare elapsed time, CPU time, and peak memory
1819
- **Validate transformations** - Check that refactors don't alter output
1920
- **Debug generation issues** - Understand what changed between versions
2021

@@ -38,6 +39,12 @@ Comparators can be reused across multiple generators. You specify which comparat
3839
- `object-assertion.mjs` can compare JSON output from `legacy-json`, `json-simple`, etc.
3940
- `my-comparator.mjs` would be a custom comparator for specific needs
4041

42+
The generation workflow also stores timing, CPU, and peak resident memory in
43+
`benchmark.json`. The built-in comparators include these measurements in their
44+
Markdown report and exclude the metadata file from output comparisons. If a
45+
base artifact predates performance measurement, the output comparison still
46+
runs and the performance section is omitted.
47+
4148
## Creating a Comparator
4249

4350
### Step 1: Create the Comparator File

‎scripts/comparators/file-size.mjs‎

Lines changed: 20 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
import { stat, readdir } from 'node:fs/promises';
22
import path from 'node:path';
33

4-
import { BASE, HEAD, TITLE } from '../constants.mjs';
4+
import { BASE, BENCHMARK_FILE, HEAD, TITLE } from '../constants.mjs';
5+
import { comparePerformance } from './performance.mjs';
56

67
const UNITS = ['B', 'KB', 'MB', 'GB'];
78

@@ -25,7 +26,7 @@ const formatBytes = bytes => {
2526
* @returns {Promise<Map<string, number>>} Map of filename to size in bytes
2627
*/
2728
const getStats = async dir => {
28-
const files = await readdir(dir);
29+
const files = (await readdir(dir)).filter(file => file !== BENCHMARK_FILE);
2930
return new Map(
3031
await Promise.all(
3132
files.map(async f => [f, (await stat(path.join(dir, f))).size])
@@ -53,6 +54,8 @@ const changed = [...new Set([...baseStats.keys(), ...headStats.keys()])]
5354
.map(toDiffObject)
5455
.sort((a, b) => Math.abs(b.diff) - Math.abs(a.diff));
5556

57+
const sections = [];
58+
5659
// Output markdown table if there are changes
5760
if (changed.length) {
5861
const rows = changed.map(({ file, base, head, diff }) => {
@@ -63,8 +66,19 @@ if (changed.length) {
6366
return `| \`${file}\` | ${formatBytes(base)} | ${formatBytes(head)} | ${diffFormatted} |`;
6467
});
6568

66-
console.log(TITLE);
67-
console.log('| File | Base | Head | Diff |');
68-
console.log('|-|-|-|-|');
69-
console.log(rows.join('\n') + '\n');
69+
sections.push(
70+
'### Output size',
71+
'| File | Base | Head | Diff |',
72+
'|-|-|-|-|',
73+
rows.join('\n')
74+
);
75+
}
76+
77+
const performance = await comparePerformance();
78+
if (performance) {
79+
sections.push(performance);
80+
}
81+
82+
if (sections.length) {
83+
console.log(`${TITLE}\n\n${sections.join('\n\n')}\n`);
7084
}

‎scripts/comparators/object-assertion.mjs‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,10 @@ import assert from 'node:assert';
22
import { readdir, readFile } from 'node:fs/promises';
33
import { join } from 'node:path';
44

5-
import { BASE, HEAD, TITLE } from '../constants.mjs';
5+
import { BASE, BENCHMARK_FILE, HEAD, TITLE } from '../constants.mjs';
6+
import { comparePerformance } from './performance.mjs';
67

7-
const files = await readdir(BASE);
8+
const files = (await readdir(BASE)).filter(file => file !== BENCHMARK_FILE);
89

910
export const details = (summary, diff) =>
1011
`<details>\n<summary>${summary}</summary>\n\n\`\`\`diff\n${diff}\n\`\`\`\n\n</details>`;
@@ -28,7 +29,16 @@ const results = await Promise.all(files.map(getFileDiff));
2829

2930
const filteredResults = results.filter(Boolean);
3031

32+
const sections = [];
3133
if (filteredResults.length) {
32-
console.log(TITLE);
33-
console.log(filteredResults.join('\n') + '\n');
34+
sections.push('### Output', filteredResults.join('\n'));
35+
}
36+
37+
const performance = await comparePerformance();
38+
if (performance) {
39+
sections.push(performance);
40+
}
41+
42+
if (sections.length) {
43+
console.log(`${TITLE}\n\n${sections.join('\n\n')}\n`);
3444
}
Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
import { readFile } from 'node:fs/promises';
2+
import path from 'node:path';
3+
4+
import { BASE, BENCHMARK_FILE, HEAD } from '../constants.mjs';
5+
6+
const UNITS = ['B', 'KB', 'MB', 'GB'];
7+
8+
const formatBytes = bytes => {
9+
if (!bytes) {
10+
return '0 B';
11+
}
12+
13+
const unit = Math.min(
14+
Math.floor(Math.log(Math.abs(bytes)) / Math.log(1024)),
15+
UNITS.length - 1
16+
);
17+
return `${(bytes / Math.pow(1024, unit)).toFixed(2)} ${UNITS[unit]}`;
18+
};
19+
20+
const formatSeconds = seconds =>
21+
Math.abs(seconds) < 1
22+
? `${(seconds * 1_000).toFixed(2)} ms`
23+
: `${seconds.toFixed(2)} s`;
24+
25+
const formatPercent = (base, diff) => {
26+
if (base === 0) {
27+
return 'n/a';
28+
}
29+
30+
const percent = (diff / base) * 100;
31+
return `${percent > 0 ? '+' : ''}${percent.toFixed(2)}%`;
32+
};
33+
34+
const formatDiff = (base, head, formatter) => {
35+
const diff = head - base;
36+
const value = `${diff > 0 ? '+' : ''}${formatter(diff)}`;
37+
return `${value} (${formatPercent(base, diff)})`;
38+
};
39+
40+
const readBenchmark = async directory => {
41+
try {
42+
return JSON.parse(
43+
await readFile(path.join(directory, BENCHMARK_FILE), 'utf8')
44+
);
45+
} catch (error) {
46+
if (error.code === 'ENOENT') {
47+
return null;
48+
}
49+
50+
throw error;
51+
}
52+
};
53+
54+
const METRICS = [
55+
{
56+
label: 'Elapsed time',
57+
value: benchmark => benchmark.elapsedSeconds,
58+
format: formatSeconds,
59+
},
60+
{
61+
label: 'User CPU time',
62+
value: benchmark => benchmark.userCpuSeconds,
63+
format: formatSeconds,
64+
},
65+
{
66+
label: 'System CPU time',
67+
value: benchmark => benchmark.systemCpuSeconds,
68+
format: formatSeconds,
69+
},
70+
{
71+
label: 'Peak resident memory',
72+
value: benchmark => benchmark.maxRssKiB * 1024,
73+
format: formatBytes,
74+
},
75+
];
76+
77+
/**
78+
* Builds a Markdown comparison of generation performance.
79+
*
80+
* Missing benchmark files are expected while artifacts created before
81+
* performance measurement are still being used as the comparison base.
82+
*
83+
* @param {string} baseDirectory - Base artifact directory
84+
* @param {string} headDirectory - Head artifact directory
85+
* @returns {Promise<string>} Markdown table, or an empty string
86+
*/
87+
export const comparePerformance = async (
88+
baseDirectory = BASE,
89+
headDirectory = HEAD
90+
) => {
91+
const [base, head] = await Promise.all(
92+
[baseDirectory, headDirectory].map(readBenchmark)
93+
);
94+
95+
if (!base || !head) {
96+
return '';
97+
}
98+
99+
const rows = METRICS.map(({ label, value, format }) => {
100+
const baseValue = value(base);
101+
const headValue = value(head);
102+
103+
if (!Number.isFinite(baseValue) || !Number.isFinite(headValue)) {
104+
throw new TypeError(`Invalid ${label.toLowerCase()} benchmark value`);
105+
}
106+
107+
return `| ${label} | ${format(baseValue)} | ${format(headValue)} | ${formatDiff(baseValue, headValue, format)} |`;
108+
});
109+
110+
return [
111+
'### Performance',
112+
'| Metric | Base | Head | Diff |',
113+
'|-|-|-|-|',
114+
...rows,
115+
].join('\n');
116+
};

‎scripts/constants.mjs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@ export const HEAD =
1010
export const TITLE =
1111
process.env.TITLE || `## \`${process.env.GENERATOR ?? '...'}\` Generator`;
1212

13+
export const BENCHMARK_FILE = 'benchmark.json';
14+
1315
// MDN Constants
1416
export const MDN_COMPAT_URL =
1517
'https://github.com/mdn/browser-compat-data/releases/latest/download/data.json';

‎src/__tests__/comparators.test.mjs‎

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
import assert from 'node:assert/strict';
2+
import { execFile } from 'node:child_process';
3+
import { mkdtemp, mkdir, rm, writeFile } from 'node:fs/promises';
4+
import { tmpdir } from 'node:os';
5+
import path from 'node:path';
6+
import { test } from 'node:test';
7+
import { fileURLToPath } from 'node:url';
8+
import { promisify } from 'node:util';
9+
10+
import { comparePerformance } from '../../scripts/comparators/performance.mjs';
11+
12+
const benchmark = {
13+
elapsedSeconds: 2,
14+
userCpuSeconds: 1,
15+
systemCpuSeconds: 0.5,
16+
maxRssKiB: 1024,
17+
};
18+
19+
const executeFile = promisify(execFile);
20+
const comparatorDirectory = fileURLToPath(
21+
new URL('../../scripts/comparators/', import.meta.url)
22+
);
23+
24+
const createDirectories = async t => {
25+
const root = await mkdtemp(path.join(tmpdir(), 'doc-kit-'));
26+
const base = path.join(root, 'base');
27+
const head = path.join(root, 'head');
28+
29+
await Promise.all([
30+
mkdir(base, { recursive: true }),
31+
mkdir(head, { recursive: true }),
32+
]);
33+
t.after(() => rm(root, { recursive: true, force: true }));
34+
35+
return { base, head };
36+
};
37+
38+
const writeBenchmark = (directory, value = benchmark) =>
39+
writeFile(
40+
path.join(directory, 'benchmark.json'),
41+
JSON.stringify(value),
42+
'utf8'
43+
);
44+
45+
const runComparator = async (name, base, head) => {
46+
const { stdout } = await executeFile(
47+
process.execPath,
48+
[path.join(comparatorDirectory, `${name}.mjs`)],
49+
{
50+
env: { ...process.env, BASE: base, HEAD: head, GENERATOR: 'test' },
51+
}
52+
);
53+
54+
return stdout;
55+
};
56+
57+
test('comparePerformance formats benchmark differences', async t => {
58+
const { base, head } = await createDirectories(t);
59+
60+
await Promise.all([
61+
writeBenchmark(base),
62+
writeBenchmark(head, {
63+
elapsedSeconds: 3,
64+
userCpuSeconds: 0.75,
65+
systemCpuSeconds: 0.5,
66+
maxRssKiB: 1536,
67+
}),
68+
]);
69+
70+
const result = await comparePerformance(base, head);
71+
72+
assert.match(
73+
result,
74+
/Elapsed time \| 2\.00 s \| 3\.00 s \| \+1\.00 s \(\+50\.00%\)/
75+
);
76+
assert.match(
77+
result,
78+
/User CPU time \| 1\.00 s \| 750\.00 ms \| -250\.00 ms \(-25\.00%\)/
79+
);
80+
assert.match(
81+
result,
82+
/Peak resident memory \| 1\.00 MB \| 1\.50 MB \| \+512\.00 KB \(\+50\.00%\)/
83+
);
84+
});
85+
86+
test('comparePerformance omits results when an artifact has no benchmark', async t => {
87+
const { base, head } = await createDirectories(t);
88+
89+
await writeBenchmark(head);
90+
91+
assert.equal(await comparePerformance(base, head), '');
92+
});
93+
94+
test('comparePerformance rejects invalid benchmark values', async t => {
95+
const { base, head } = await createDirectories(t);
96+
97+
await Promise.all([
98+
writeBenchmark(base),
99+
writeBenchmark(head, { ...benchmark, maxRssKiB: undefined }),
100+
]);
101+
102+
await assert.rejects(
103+
comparePerformance(base, head),
104+
/Invalid peak resident memory benchmark value/
105+
);
106+
});
107+
108+
test('file-size comparator combines output and performance results', async t => {
109+
const { base, head } = await createDirectories(t);
110+
111+
await Promise.all([
112+
writeFile(path.join(base, 'result.txt'), 'base', 'utf8'),
113+
writeFile(path.join(head, 'result.txt'), 'a larger result', 'utf8'),
114+
writeBenchmark(base),
115+
writeBenchmark(head, { ...benchmark, elapsedSeconds: 3 }),
116+
]);
117+
118+
const result = await runComparator('file-size', base, head);
119+
120+
assert.equal(result.match(/## `test` Generator/g)?.length, 1);
121+
assert.match(result, /### Output size/);
122+
assert.match(result, /`result\.txt`/);
123+
assert.match(result, /### Performance/);
124+
assert.doesNotMatch(result, /benchmark\.json/);
125+
});
126+
127+
test('object comparator treats benchmark data as metadata', async t => {
128+
const { base, head } = await createDirectories(t);
129+
130+
await Promise.all([
131+
writeFile(path.join(base, 'result.json'), '{"value":true}', 'utf8'),
132+
writeFile(path.join(head, 'result.json'), '{"value":true}', 'utf8'),
133+
writeBenchmark(base),
134+
writeBenchmark(head, { ...benchmark, maxRssKiB: 2048 }),
135+
]);
136+
137+
const result = await runComparator('object-assertion', base, head);
138+
139+
assert.equal(result.match(/## `test` Generator/g)?.length, 1);
140+
assert.doesNotMatch(result, /### Output\n/);
141+
assert.match(result, /### Performance/);
142+
assert.doesNotMatch(result, /benchmark\.json/);
143+
});

0 commit comments

Comments
 (0)