Skip to content

Commit 26d8490

Browse files
TMHSDigitalclaude
andcommitted
build: dist writer in lib/dist-writer.mjs, tested in a temp dir (#265)
writeOut and pruneDist move out of build.mjs with the dist directory as a parameter. Tests cover: unchanged bytes are not rewritten, a same-size change is, a stale file and its empty directory are pruned, and a directory holding a produced file stays. Closes #265 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LpkeXQPaUhxjgqdqnk6BrJ
1 parent 1aed1c0 commit 26d8490

3 files changed

Lines changed: 86 additions & 45 deletions

File tree

‎site/build.mjs‎

Lines changed: 4 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ import { checkStacks } from "./checks.mjs";
3535
import { stackPageHtml } from "./lib/stack-page.mjs";
3636
import { freshnessLineHtml, freshnessPageHtml, freshnessStats, newEntriesLineHtml } from "./lib/freshness.mjs";
3737
import { licenceTerms, listStackFiles, owes, parseStack, pickPath } from "./lib/stacks.mjs";
38+
import { createDistWriter } from "./lib/dist-writer.mjs";
3839

3940
const __dirname = path.dirname(fileURLToPath(import.meta.url));
4041
const ROOT = path.resolve(__dirname, "..");
@@ -281,36 +282,8 @@ function notFoundHtml(site) {
281282
`;
282283
}
283284

284-
/*
285-
* Output goes through one writer that creates a directory once, writes a
286-
* file only when its bytes changed and remembers every path it produced, so
287-
* the build can prune what an earlier build left behind instead of wiping
288-
* dist/ and recreating 420 directories (94% of the build time on a slow
289-
* disk). `--clean` still wipes first; the result is byte-identical either way.
290-
*/
291-
const produced = new Set();
292-
const madeDirs = new Set();
293-
let writes = 0;
294-
295-
function ensureDir(dir) {
296-
if (madeDirs.has(dir)) return;
297-
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
298-
madeDirs.add(dir);
299-
}
300-
301-
/** Writes `content` (string or Buffer) to dist-relative `rel` if it differs. */
302-
function writeOut(rel, content) {
303-
const full = path.join(DIST, rel);
304-
produced.add(rel.split(path.sep).join("/"));
305-
ensureDir(path.dirname(full));
306-
const next = Buffer.isBuffer(content) ? content : Buffer.from(content, "utf8");
307-
if (fs.existsSync(full)) {
308-
const stat = fs.statSync(full);
309-
if (stat.isFile() && stat.size === next.length && fs.readFileSync(full).equals(next)) return;
310-
}
311-
fs.writeFileSync(full, next);
312-
writes += 1;
313-
}
285+
// Output writer: writes only changed bytes and prunes what it did not produce (lib/dist-writer.mjs).
286+
const { ensureDir, writeOut, pruneDist, produced, writes } = createDistWriter(DIST);
314287

315288
/** Copies a file into dist (through writeOut, so unchanged files are left alone). */
316289
function copyOut(src, rel) {
@@ -333,20 +306,6 @@ function copyPublic(src, skip, prefix = "") {
333306
}
334307
}
335308

336-
/** Removes every file and directory in dist that this build did not produce. */
337-
function pruneDist(dir = DIST, prefix = "") {
338-
let kept = 0;
339-
for (const name of fs.readdirSync(dir)) {
340-
const full = path.join(dir, name);
341-
const rel = prefix ? `${prefix}/${name}` : name;
342-
if (fs.statSync(full).isDirectory()) {
343-
if (pruneDist(full, rel) === 0) fs.rmSync(full, { recursive: true, force: true });
344-
else kept += 1;
345-
} else if (produced.has(rel)) kept += 1;
346-
else fs.rmSync(full, { force: true });
347-
}
348-
return kept;
349-
}
350309

351310
/** Returns file => link resolver for links in that file's body (`file.path` is repo-relative). */
352311
function linkResolvers(entries, config, entryBase = "../") {
@@ -769,7 +728,7 @@ function main() {
769728
console.log(
770729
`Prerendered ${visible.length} entry rows into index.html; wrote ${pages.written.length} entry pages, ${stackPages.written.length} stack pages, ${categoryPages.written.length} category pages, ${guidePages.written.length} guide pages, sitemap.xml, robots.txt, 404.html, freshness/index.html`
771730
);
772-
console.log(`${writes} of ${produced.size} output files changed${clean ? " (clean build)" : ""}; ${((Date.now() - started) / 1000).toFixed(1)}s`);
731+
console.log(`${writes()} of ${produced.size} output files changed${clean ? " (clean build)" : ""}; ${((Date.now() - started) / 1000).toFixed(1)}s`);
773732
}
774733

775734
main();

‎site/lib/dist-writer.mjs‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
/**
2+
* The build's output writer. It creates a directory once, writes a file only
3+
* when its bytes changed and remembers every path it produced, so the build
4+
* can prune what an earlier build left behind instead of wiping dist/ and
5+
* recreating 420 directories (94% of the build time on a slow disk).
6+
* `--clean` still wipes first; the result is byte-identical either way.
7+
* The directory is a parameter so the tests can run it in a temp dir (#265).
8+
*/
9+
import fs from "node:fs";
10+
import path from "node:path";
11+
12+
export function createDistWriter(dist) {
13+
const produced = new Set();
14+
const madeDirs = new Set();
15+
let writes = 0;
16+
17+
function ensureDir(dir) {
18+
if (madeDirs.has(dir)) return;
19+
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
20+
madeDirs.add(dir);
21+
}
22+
23+
/** Writes `content` (string or Buffer) to dist-relative `rel` if it differs. */
24+
function writeOut(rel, content) {
25+
const full = path.join(dist, rel);
26+
produced.add(rel.split(path.sep).join("/"));
27+
ensureDir(path.dirname(full));
28+
const next = Buffer.isBuffer(content) ? content : Buffer.from(content, "utf8");
29+
if (fs.existsSync(full)) {
30+
const stat = fs.statSync(full);
31+
if (stat.isFile() && stat.size === next.length && fs.readFileSync(full).equals(next)) return;
32+
}
33+
fs.writeFileSync(full, next);
34+
writes += 1;
35+
}
36+
37+
/** Removes every file and directory in dist that this build did not produce. */
38+
function pruneDist(dir = dist, prefix = "") {
39+
let kept = 0;
40+
for (const name of fs.readdirSync(dir)) {
41+
const full = path.join(dir, name);
42+
const rel = prefix ? `${prefix}/${name}` : name;
43+
if (fs.statSync(full).isDirectory()) {
44+
if (pruneDist(full, rel) === 0) fs.rmSync(full, { recursive: true, force: true });
45+
else kept += 1;
46+
} else if (produced.has(rel)) kept += 1;
47+
else fs.rmSync(full, { force: true });
48+
}
49+
return kept;
50+
}
51+
52+
return { ensureDir, writeOut, pruneDist, produced, writes: () => writes };
53+
}

‎site/lib/lib.test.mjs‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2120,6 +2120,35 @@ eq("search: q from the URL is capped", readState(`?q=${"a".repeat(5000)}`, { q:
21202120
eq("plainText: links and emphasis stripped", plainText("See **[the guide](a.md)** and `code`\n now"), "See the guide and code now");
21212121
}
21222122

2123+
/* incremental dist writer (#265) ------------------------------------------ */
2124+
{
2125+
const { createDistWriter } = await import("./dist-writer.mjs");
2126+
const fsm = (await import("node:fs")).default;
2127+
const osm = (await import("node:os")).default;
2128+
const pathm = (await import("node:path")).default;
2129+
const dir = fsm.mkdtempSync(pathm.join(osm.tmpdir(), "fgda-dist-"));
2130+
try {
2131+
const first = createDistWriter(dir);
2132+
first.writeOut("a/index.html", "hello");
2133+
first.writeOut("b.txt", "same");
2134+
eq("dist: first build writes both", first.writes(), 2);
2135+
fsm.mkdirSync(pathm.join(dir, "old"), { recursive: true });
2136+
fsm.writeFileSync(pathm.join(dir, "old", "x.html"), "stale");
2137+
2138+
const second = createDistWriter(dir);
2139+
second.writeOut("a/index.html", "hello");
2140+
second.writeOut("b.txt", "diff"); // same size, other bytes
2141+
eq("dist: unchanged bytes are not rewritten, changed ones are", second.writes(), 1);
2142+
eq("dist: same-size change lands", fsm.readFileSync(pathm.join(dir, "b.txt"), "utf8"), "diff");
2143+
second.pruneDist();
2144+
eq("dist: a stale file is removed", fsm.existsSync(pathm.join(dir, "old", "x.html")), false);
2145+
eq("dist: its empty directory is pruned", fsm.existsSync(pathm.join(dir, "old")), false);
2146+
eq("dist: a directory holding a produced file stays", fsm.existsSync(pathm.join(dir, "a", "index.html")), true);
2147+
} finally {
2148+
fsm.rmSync(dir, { recursive: true, force: true });
2149+
}
2150+
}
2151+
21232152
/* report (keep last) ------------------------------------------------------ */
21242153
if (failures.length) {
21252154
console.error(`lib.test failed (${failures.length}):`);

0 commit comments

Comments
 (0)