Skip to content

Commit dc615f3

Browse files
TMHSDigitalclaude
andcommitted
thumbs: main() split into planRun, recordFailure, settleRun and orphanThumbs, all tested (#264)
The state logic the monthly workflow commits is now pure and covered: licence change and removed entries drop records before any fetch, --limit skips with kind limit, a failure bumps a fresh record and drops a stale one, three same-hash records go to shared and a repeat run reports no change, orphan .webp files go and other files stay, and the report's empty, failed and no-converter cases. A dry run is unchanged. Closes #264 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LpkeXQPaUhxjgqdqnk6BrJ
1 parent 26d8490 commit dc615f3

2 files changed

Lines changed: 132 additions & 45 deletions

File tree

‎site/lib/lib.test.mjs‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2149,6 +2149,64 @@ eq("search: q from the URL is capped", readState(`?q=${"a".repeat(5000)}`, { q:
21492149
}
21502150
}
21512151

2152+
/* thumbs run planner and settler (#264) ----------------------------------- */
2153+
{
2154+
const { planRun, recordFailure, settleRun, orphanThumbs, reportMarkdown } = await import("../thumbs.mjs");
2155+
const today = "2026-11-06";
2156+
const opts = { force: false, dryRun: false, only: null, limit: Infinity, report: null };
2157+
const entry = (id, over = {}) => ({ id, url: `https://pub.itch.io/${id}`, license: "CC0", status: "active", preview: undefined, ...over });
2158+
const rec = (id, over = {}) => ({ source: `https://pub.itch.io/${id}`, image: `https://img/${id}.png`, fetched: "2026-10-01", hash: `h-${id}`, width: 1, height: 1, license: "CC0", ...over });
2159+
const all = () => true;
2160+
2161+
{
2162+
const manifest = { thumbs: { a: rec("a"), gone: rec("gone") } };
2163+
const plan = planRun({ entries: [entry("a", { license: "custom" })], manifest, fileExists: all, opts, today });
2164+
eq("thumbs plan: a licence changed to custom loses its record before any fetch", "a" in plan.thumbs, false);
2165+
eq("thumbs plan: an entry no longer in the catalog is removed", plan.summary.removed.map((r) => r.id).sort().join(), "a,gone");
2166+
const { changed } = settleRun({ manifest, thumbs: plan.thumbs, shared: plan.shared, pending: new Map(), summary: plan.summary });
2167+
eq("thumbs plan: the removals count as changes", changed, 2);
2168+
eq("thumbs plan: the custom entry is skipped by the gate", plan.summary.skipped[0].kind, "gate");
2169+
}
2170+
{
2171+
const plan = planRun({ entries: [entry("a"), entry("b"), entry("c")], manifest: { thumbs: {} }, fileExists: all, opts: { ...opts, limit: 1 }, today });
2172+
eq("thumbs plan: --limit 1 gives one todo", plan.todo.length, 1);
2173+
eq("thumbs plan: the rest are skipped with kind limit", plan.summary.skipped.filter((s) => s.kind === "limit").length, 2);
2174+
}
2175+
{
2176+
const plan = planRun({ entries: [entry("a")], manifest: { thumbs: { a: rec("a") } }, fileExists: () => false, opts, today });
2177+
eq("thumbs plan: a fresh record with no file on disk is fetched again", plan.todo.length, 1);
2178+
}
2179+
{
2180+
const summary = { failed: [], removed: [], bumped: [] };
2181+
const thumbs = { a: rec("a"), old: rec("old", { fetched: "2025-01-01" }) };
2182+
recordFailure({ thumbs, summary, entry: entry("a"), reason: "HTTP 500", today, countFailure: true });
2183+
eq("thumbs failure: a fresh record has its failure count raised", thumbs.a.failures, 1);
2184+
eq("thumbs failure: the bump is reported", summary.bumped.map((b) => b.id).join(), "a");
2185+
recordFailure({ thumbs, summary, entry: entry("old"), reason: "HTTP 500", today, countFailure: true });
2186+
eq("thumbs failure: a stale record is removed", "old" in thumbs, false);
2187+
eq("thumbs failure: and reported as removed", summary.removed[0].id, "old");
2188+
}
2189+
{
2190+
const fresh = { a: rec("a", { hash: "logo" }), b: rec("b", { hash: "logo" }), c: rec("c", { hash: "logo" }) };
2191+
const manifest = { thumbs: {} };
2192+
const pending = new Map([["a", Buffer.from("x")], ["b", Buffer.from("x")], ["c", Buffer.from("x")]]);
2193+
const summary = { dropped: [] };
2194+
const thumbs = { ...fresh };
2195+
const shared = {};
2196+
const first = settleRun({ manifest, thumbs, shared, pending, summary });
2197+
eq("thumbs settle: three same-hash records go to shared", Object.keys(first.sortedShared).join(), "a,b,c");
2198+
eq("thumbs settle: and leave pending", pending.size, 0);
2199+
const again = settleRun({ manifest: { thumbs: first.sorted, shared: first.sortedShared }, thumbs: { ...fresh }, shared: { ...first.sortedShared }, pending: new Map(), summary: { dropped: [] } });
2200+
eq("thumbs settle: re-dropping the same logos changes nothing", again.changed, 0);
2201+
}
2202+
eq("thumbs orphans: a .webp without a record goes, other files stay", orphanThumbs(["x.webp", "a.webp", "notes.txt"], { a: {} }).join(), "x.webp");
2203+
2204+
const empty = { today, converter: null, fetched: [], kept: 0, skipped: [], failed: [], bumped: [], removed: [], dropped: [] };
2205+
eq("thumbs report: an empty run has no sections", reportMarkdown(empty).includes("## "), false);
2206+
has("thumbs report: no converter prints none", reportMarkdown(empty), "Converter: none.");
2207+
has("thumbs report: one failure gives a Failed section", reportMarkdown({ ...empty, failed: [{ id: "a", reason: "HTTP 500" }] }), "## Failed\n\n- a: HTTP 500");
2208+
}
2209+
21522210
/* report (keep last) ------------------------------------------------------ */
21532211
if (failures.length) {
21542212
console.error(`lib.test failed (${failures.length}):`);

‎site/thumbs.mjs‎

Lines changed: 74 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -220,20 +220,20 @@ export function reportMarkdown(summary) {
220220
return `${lines.join("\n")}\n`;
221221
}
222222

223-
async function main() {
224-
const opts = parseArgs(process.argv.slice(2));
225-
const today = new Date().toISOString().slice(0, 10);
226-
const converter = detectConverter();
227-
const entries = loadEntries();
223+
/**
224+
* The run's plan, before any fetch (pure; #264): drops records whose entry
225+
* is gone or no longer eligible, then sorts every entry into kept, skipped
226+
* (gate, shared, limit) or todo. `fileExists(id)` says whether the entry's
227+
* .webp is on disk. Returns `{ thumbs, shared, todo, summary }`; the maps
228+
* are copies of the manifest's.
229+
*/
230+
export function planRun({ entries, manifest, fileExists, opts, today, converter = null }) {
228231
const byId = new Map(entries.map((e) => [e.id, e]));
229-
const manifest = loadManifest();
230232
const thumbs = { ...manifest.thumbs };
231233
// Entries whose image turned out to be shared (a logo): remembered so they
232234
// are not fetched again until the record would be refreshed anyway.
233235
const shared = { ...manifest.shared };
234-
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "fgda-thumbs-"));
235236
const summary = { today, converter, fetched: [], kept: 0, skipped: [], failed: [], bumped: [], removed: [], dropped: [] };
236-
const pending = new Map(); // id → bytes to write
237237

238238
// Records whose entry is gone or no longer eligible go first, so a licence
239239
// that changed to custom never survives a run.
@@ -264,8 +264,7 @@ async function main() {
264264
summary.skipped.push({ id: entry.id, reason: "image shared with other entries", kind: "shared" });
265265
continue;
266266
}
267-
const fileExists = fs.existsSync(path.join(THUMBS_DIR, `${entry.id}.webp`));
268-
if (!needsFetch(rec, fetchOpts) && fileExists) {
267+
if (!needsFetch(rec, fetchOpts) && fileExists(entry.id)) {
269268
summary.kept += 1;
270269
continue;
271270
}
@@ -275,39 +274,36 @@ async function main() {
275274
}
276275
todo.push(entry);
277276
}
277+
return { thumbs, shared, todo, summary };
278+
}
278279

279-
await pool(
280-
todo,
281-
async (entry) => {
282-
try {
283-
const { record, bytes } = await fetchOne(entry, { converter, tmpDir, today });
284-
thumbs[entry.id] = record;
285-
pending.set(entry.id, bytes);
286-
summary.fetched.push({ id: entry.id, width: record.width, height: record.height, bytes: bytes.length });
287-
process.stdout.write(` + ${entry.id} ${record.width}x${record.height} ${bytes.length} B\n`);
288-
} catch (err) {
289-
summary.failed.push({ id: entry.id, reason: err.message });
290-
// A source that keeps failing does not keep its old image forever,
291-
// and a record for a url or pin the entry no longer has goes at once.
292-
const old = thumbs[entry.id];
293-
if (old) {
294-
const kept = afterFailure(old, { today, url: entry.url, preview: entry.preview || "", countFailure: Boolean(converter) });
295-
if (!kept) {
296-
summary.removed.push({ id: entry.id, reason: `refresh failed and the stored image is stale (${err.message})`, hash: old.hash });
297-
delete thumbs[entry.id];
298-
} else if (kept !== old) {
299-
thumbs[entry.id] = kept;
300-
// A raised failure count is a change: without it the count never
301-
// reaches the repository and the MAX_FAILURES rule cannot fire.
302-
summary.bumped.push({ id: entry.id, failures: kept.failures });
303-
}
304-
}
305-
process.stdout.write(` ! ${entry.id}: ${err.message}\n`);
306-
}
307-
},
308-
CONCURRENCY
309-
);
280+
/**
281+
* A failed fetch (pure; mutates `thumbs` and `summary`). A source that keeps
282+
* failing does not keep its old image forever, and a record for a url or pin
283+
* the entry no longer has goes at once.
284+
*/
285+
export function recordFailure({ thumbs, summary, entry, reason, today, countFailure }) {
286+
summary.failed.push({ id: entry.id, reason });
287+
const old = thumbs[entry.id];
288+
if (!old) return;
289+
const kept = afterFailure(old, { today, url: entry.url, preview: entry.preview || "", countFailure });
290+
if (!kept) {
291+
summary.removed.push({ id: entry.id, reason: `refresh failed and the stored image is stale (${reason})`, hash: old.hash });
292+
delete thumbs[entry.id];
293+
} else if (kept !== old) {
294+
thumbs[entry.id] = kept;
295+
// A raised failure count is a change: without it the count never
296+
// reaches the repository and the MAX_FAILURES rule cannot fire.
297+
summary.bumped.push({ id: entry.id, failures: kept.failures });
298+
}
299+
}
310300

301+
/**
302+
* After the fetches (pure; mutates `thumbs`, `shared`, `pending`, `summary`):
303+
* moves logos to `shared`, sorts both maps and counts the net change against
304+
* the manifest. Returns `{ sorted, sortedShared, changed }`.
305+
*/
306+
export function settleRun({ manifest, thumbs, shared, pending, summary }) {
311307
// A hash shared by three or more entries is a logo, not a preview. Logos
312308
// dropped in earlier runs (in `shared`) count toward the three.
313309
for (const id of sharedImageIds(thumbs, shared)) {
@@ -324,14 +320,47 @@ async function main() {
324320
// Only a net change to the manifest or the files counts, so a run that
325321
// refetches and re-drops the same shared images changes nothing.
326322
const changed = manifestChanges(manifest, { thumbs: sorted, shared: sortedShared }) + pending.size;
323+
return { sorted, sortedShared, changed };
324+
}
325+
326+
/** Files in the thumbs folder to delete: a .webp with no manifest record. Other files stay. */
327+
export function orphanThumbs(names, sorted) {
328+
return names.filter((name) => name.endsWith(".webp") && !sorted[name.replace(/\.webp$/, "")]);
329+
}
330+
331+
async function main() {
332+
const opts = parseArgs(process.argv.slice(2));
333+
const today = new Date().toISOString().slice(0, 10);
334+
const converter = detectConverter();
335+
const manifest = loadManifest();
336+
const fileExists = (id) => fs.existsSync(path.join(THUMBS_DIR, `${id}.webp`));
337+
const { thumbs, shared, todo, summary } = planRun({ entries: loadEntries(), manifest, fileExists, opts, today, converter });
338+
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "fgda-thumbs-"));
339+
const pending = new Map(); // id → bytes to write
340+
341+
await pool(
342+
todo,
343+
async (entry) => {
344+
try {
345+
const { record, bytes } = await fetchOne(entry, { converter, tmpDir, today });
346+
thumbs[entry.id] = record;
347+
pending.set(entry.id, bytes);
348+
summary.fetched.push({ id: entry.id, width: record.width, height: record.height, bytes: bytes.length });
349+
process.stdout.write(` + ${entry.id} ${record.width}x${record.height} ${bytes.length} B\n`);
350+
} catch (err) {
351+
recordFailure({ thumbs, summary, entry, reason: err.message, today, countFailure: Boolean(converter) });
352+
process.stdout.write(` ! ${entry.id}: ${err.message}\n`);
353+
}
354+
},
355+
CONCURRENCY
356+
);
357+
358+
const { sorted, sortedShared, changed } = settleRun({ manifest, thumbs, shared, pending, summary });
327359

328360
if (!opts.dryRun) {
329361
fs.mkdirSync(THUMBS_DIR, { recursive: true });
330362
for (const [id, bytes] of pending) fs.writeFileSync(path.join(THUMBS_DIR, `${id}.webp`), bytes);
331-
for (const name of fs.readdirSync(THUMBS_DIR)) {
332-
const id = name.replace(/\.webp$/, "");
333-
if (name.endsWith(".webp") && !sorted[id]) fs.rmSync(path.join(THUMBS_DIR, name), { force: true });
334-
}
363+
for (const name of orphanThumbs(fs.readdirSync(THUMBS_DIR), sorted)) fs.rmSync(path.join(THUMBS_DIR, name), { force: true });
335364
if (changed) {
336365
const out = { generated: today, thumbs: sorted, ...(Object.keys(sortedShared).length ? { shared: sortedShared } : {}) };
337366
fs.writeFileSync(MANIFEST, `${JSON.stringify(out, null, 2)}\n`);

0 commit comments

Comments
 (0)