Skip to content

Commit 6a5b0bb

Browse files
TMHSDigitalclaude
andcommitted
Thumbnails: refresh when the source or pin changes, harden the fetch
- needsFetch also compares the record's source with the entry url and the stored pin and image with preview:; a source that fails three runs in a row, or whose image is older than twice the refresh age, is dropped (afterFailure); validate flags a record whose source no longer matches the entry url (Closes #133) - Image URLs must be public https hosts: http is upgraded, localhost, IP literals and credentials are refused, both for og:image and after redirects, and preview: is checked the same way. The body is streamed with a byte cap, the first bytes must be PNG, JPEG, GIF or WebP before the converter runs, and ffmpeg or magick is killed after 30 s. Job timeouts landed with #136 (Closes #134) Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent dbb0634 commit 6a5b0bb

5 files changed

Lines changed: 128 additions & 18 deletions

File tree

‎site/checks.mjs‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import path from "node:path";
1010
import { ID_RE } from "./lib/entry-files.mjs";
1111
import { evidenceSection } from "./lib/frontmatter.mjs";
1212
import { isRealDate } from "./lib/shared.mjs";
13-
import { previewEligibility } from "./lib/thumbs.mjs";
13+
import { isPrivateHost, previewEligibility, safeImageUrl } from "./lib/thumbs.mjs";
1414
import { parseStack, pickPath, StackError } from "./lib/stacks.mjs";
1515

1616
export { evidenceSection };
@@ -649,11 +649,6 @@ export function checkCategoryReadmeRows(categoryName, readmeText, entries) {
649649
* escaping HTML does nothing about a scheme.
650650
*/
651651
/** localhost, *.local / *.internal, and IP literals: never a catalog source. */
652-
function isPrivateHost(hostname) {
653-
const h = hostname.toLowerCase().replace(/^\[|\]$/g, "");
654-
if (h === "localhost" || /\.(localhost|local|internal|lan)$/.test(h)) return true;
655-
return /^\d{1,3}(\.\d{1,3}){3}$/.test(h) || h.includes(":");
656-
}
657652

658653
export function checkEntryUrl(rel, meta) {
659654
const errors = [];
@@ -788,7 +783,9 @@ export function checkCountTables(measured, docs) {
788783
export function checkPreviewField(rel, meta) {
789784
if (meta.preview === undefined || meta.preview === null || meta.preview === "") return [];
790785
const v = String(meta.preview);
791-
if (v === "none" || /^https:\/\/\S+$/.test(v)) return [];
786+
if (v === "none") return [];
787+
if (/^https:\/\/\S+$/.test(v) && safeImageUrl(v) === new URL(v).href) return [];
788+
if (/^https:\/\/\S+$/.test(v)) return [`${rel} preview must be an https:// image URL on a public host`];
792789
return [`${rel} preview must be "none" or an https:// image URL`];
793790
}
794791

@@ -824,6 +821,9 @@ export function checkThumbManifest(manifest, entriesById, files) {
824821
// entry's live licence, and the next script run refreshes the record.
825822
const gate = previewEligibility(entry);
826823
if (!gate.ok) errors.push(`thumbnail for "${id}" must be removed: ${gate.reason} (run node site/thumbs.mjs)`);
824+
else if (rec.source && entry.url && rec.source !== String(entry.url)) {
825+
errors.push(`thumbnail for "${id}" was fetched from ${rec.source} but the entry url is now ${entry.url} (run node site/thumbs.mjs --only ${id})`);
826+
}
827827
}
828828
return errors;
829829
}

‎site/checks.test.mjs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -793,6 +793,7 @@ accepts("preview field: absent", checkPreviewField("ok.md", {}));
793793
accepts("preview field: none", checkPreviewField("ok.md", { preview: "none" }));
794794
accepts("preview field: https url", checkPreviewField("ok.md", { preview: "https://a/b.png" }));
795795
rejects("preview field: http refused", checkPreviewField("bad.md", { preview: "http://a/b.png" }), 'bad.md preview must be "none" or an https:// image URL');
796+
rejects("preview field: loopback refused", checkPreviewField("bad.md", { preview: "https://127.0.0.1/a.png" }), "bad.md preview must be an https:// image URL on a public host");
796797
rejects("preview field: word refused", checkPreviewField("bad.md", { preview: "yes" }), 'bad.md preview must be "none" or an https:// image URL');
797798

798799
/* V30: line endings ------------------------------------------------------ */

‎site/lib/lib.test.mjs‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -898,7 +898,7 @@ eq("new-entry sets added to today", newEntryText("added: 2026-07-19\nverified: 2
898898
}
899899

900900
// Site refresh: thumbnail rules.
901-
import { filmstripPick, imageUrlFromHtml, needsFetch, previewCredit, previewEligibility, publisherKey, sharedImageIds, webpSize } from "./thumbs.mjs";
901+
import { afterFailure, filmstripPick, imageUrlFromHtml, isPrivateHost, needsFetch, previewCredit, previewEligibility, publisherKey, safeImageUrl, sharedImageIds, sniffImage, webpSize } from "./thumbs.mjs";
902902
eq("thumbs: CC0 active itch page is eligible", previewEligibility({ license: "CC0", status: "active", url: "https://kaylousberg.itch.io/block-bits" }).ok, true);
903903
eq("thumbs: custom licence refused", previewEligibility({ license: "custom", status: "active", url: "https://x.itch.io/a" }).reason, "licence custom is not in the preview allowlist");
904904
eq("thumbs: needs-review refused before licence", previewEligibility({ license: "custom", status: "needs-review", url: "https://x.itch.io/a" }).reason, "status is needs-review");
@@ -917,6 +917,24 @@ eq("thumbs: needsFetch fresh same licence", needsFetch({ fetched: "2026-09-01",
917917
eq("thumbs: needsFetch licence changed", needsFetch({ fetched: "2026-09-01", license: "CC0" }, { license: "CC-BY-4.0", today: "2026-10-23" }), true);
918918
eq("thumbs: needsFetch stale", needsFetch({ fetched: "2026-01-01", license: "CC0" }, { license: "CC0", today: "2026-10-23" }), true);
919919
eq("thumbs: needsFetch force", needsFetch({ fetched: "2026-10-20", license: "CC0" }, { license: "CC0", today: "2026-10-23", force: true }), true);
920+
const recA = { source: "https://a.example/", image: "https://a.example/og.png", fetched: "2026-10-20", license: "CC0" };
921+
eq("thumbs: needsFetch url moved", needsFetch(recA, { license: "CC0", today: "2026-10-23", url: "https://b.example/", preview: "" }), true);
922+
eq("thumbs: needsFetch unchanged entry", needsFetch(recA, { license: "CC0", today: "2026-10-23", url: "https://a.example/", preview: "" }), false);
923+
eq("thumbs: needsFetch pin added", needsFetch(recA, { license: "CC0", today: "2026-10-23", url: "https://a.example/", preview: "https://a.example/shot.png" }), true);
924+
eq("thumbs: needsFetch pin changed", needsFetch({ ...recA, pinned: true, image: "https://a.example/old.png" }, { license: "CC0", today: "2026-10-23", url: "https://a.example/", preview: "https://a.example/new.png" }), true);
925+
eq("thumbs: needsFetch pin removed", needsFetch({ ...recA, pinned: true }, { license: "CC0", today: "2026-10-23", url: "https://a.example/", preview: "" }), true);
926+
eq("thumbs: afterFailure counts", afterFailure(recA, { today: "2026-10-23" }).failures, 1);
927+
eq("thumbs: afterFailure drops at the limit", afterFailure({ ...recA, failures: 2 }, { today: "2026-10-23" }), null);
928+
eq("thumbs: afterFailure drops an ancient image", afterFailure({ ...recA, fetched: "2025-01-01" }, { today: "2026-10-23" }), null);
929+
eq("thumbs: private host localhost", isPrivateHost("localhost"), true);
930+
eq("thumbs: private host ip literal", isPrivateHost("127.0.0.1"), true);
931+
eq("thumbs: public host", isPrivateHost("img.itch.zone"), false);
932+
eq("thumbs: http image upgraded", safeImageUrl("http://a.example/x.png"), "https://a.example/x.png");
933+
eq("thumbs: loopback image refused", safeImageUrl("https://127.0.0.1/x.png"), null);
934+
eq("thumbs: file scheme refused", safeImageUrl("file:///etc/passwd"), null);
935+
eq("thumbs: og:image on loopback refused", imageUrlFromHtml('<meta property="og:image" content="http://localhost/a.png">', "https://a.example/"), null);
936+
eq("thumbs: sniff png", sniffImage(Uint8Array.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0, 0, 0, 0])), "png");
937+
eq("thumbs: sniff html refused", sniffImage(new TextEncoder().encode("<!doctype html><html>")), null);
920938
{
921939
const hex = (s) => Buffer.from(s.replaceAll(" ", ""), "hex");
922940
const vp8x = hex("52494646 24000000 57454250 56503858 0a000000 10000000 7f0200 8f0100");

‎site/lib/thumbs.mjs‎

Lines changed: 66 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,40 @@ export function isAttributionLicence(license) {
5353
return /^CC-BY/.test(String(license));
5454
}
5555

56+
/**
57+
* True for hosts a fetch from CI must never reach: localhost, local-only
58+
* suffixes and IP literals (a public catalog source is named by DNS).
59+
*/
60+
export function isPrivateHost(hostname) {
61+
const h = String(hostname).toLowerCase().replace(/^\[|\]$/g, "");
62+
if (h === "" || h === "localhost" || /\.(localhost|local|internal|lan)$/.test(h)) return true;
63+
return /^\d{1,3}(\.\d{1,3}){3}$/.test(h) || h.includes(":");
64+
}
65+
66+
/** An https URL on a public host, or null. http is upgraded to https; anything else is refused. */
67+
export function safeImageUrl(raw) {
68+
let u;
69+
try {
70+
u = new URL(String(raw));
71+
} catch {
72+
return null;
73+
}
74+
if (u.protocol === "http:") u.protocol = "https:";
75+
if (u.protocol !== "https:" || u.username || u.password || isPrivateHost(u.hostname)) return null;
76+
return u.href;
77+
}
78+
79+
/** The image format from its first bytes (png, jpeg, gif, webp), or null. Checked before a converter sees the file. */
80+
export function sniffImage(bytes) {
81+
const b = bytes;
82+
if (!b || b.length < 12) return null;
83+
if (b[0] === 0x89 && b[1] === 0x50 && b[2] === 0x4e && b[3] === 0x47) return "png";
84+
if (b[0] === 0xff && b[1] === 0xd8 && b[2] === 0xff) return "jpeg";
85+
if (b[0] === 0x47 && b[1] === 0x49 && b[2] === 0x46 && b[3] === 0x38) return "gif";
86+
if (b[0] === 0x52 && b[1] === 0x49 && b[2] === 0x46 && b[3] === 0x46 && b[8] === 0x57 && b[9] === 0x45 && b[10] === 0x42 && b[11] === 0x50) return "webp";
87+
return null;
88+
}
89+
5690
function hostOf(url) {
5791
try {
5892
return new URL(String(url)).hostname.toLowerCase();
@@ -115,7 +149,7 @@ export function imageUrlFromHtml(html, pageUrl) {
115149
if (!raw) return null;
116150
try {
117151
const abs = new URL(decodeEntities(raw).trim(), pageUrl);
118-
return /^https?:$/.test(abs.protocol) ? abs.href : null;
152+
return /^https?:$/.test(abs.protocol) ? safeImageUrl(abs.href) : null;
119153
} catch {
120154
return null;
121155
}
@@ -136,16 +170,45 @@ export function sharedImageIds(records) {
136170

137171
const DAY = 86400000;
138172

139-
/** Whether a stored record is missing, forced, under a different licence, or older than maxAgeDays. */
140-
export function needsFetch(record, { license, today, force = false, maxAgeDays = 180 }) {
173+
/**
174+
* Whether a stored record is missing, forced, under a different licence,
175+
* older than maxAgeDays, or no longer describes the entry: its `url` moved,
176+
* a `preview:` pin was added, removed or changed. Pass `url` and
177+
* `preview` to get the last three checks.
178+
*/
179+
export function needsFetch(record, { license, today, force = false, maxAgeDays = 180, url, preview }) {
141180
if (!record || force) return true;
142181
if (record.license !== license) return true;
182+
if (url !== undefined && record.source !== url) return true;
183+
if (preview !== undefined) {
184+
const pinned = /^https:\/\//.test(preview || "");
185+
if (Boolean(record.pinned) !== pinned) return true;
186+
if (pinned && record.image !== preview) return true;
187+
}
143188
const fetched = Date.parse(`${record.fetched}T00:00:00Z`);
144189
const now = Date.parse(`${today}T00:00:00Z`);
145190
if (Number.isNaN(fetched) || Number.isNaN(now)) return true;
146191
return now - fetched > maxAgeDays * DAY;
147192
}
148193

194+
/** Consecutive failed refreshes after which a stored image is dropped. */
195+
export const MAX_FAILURES = 3;
196+
197+
/**
198+
* After a failed refresh: the record to keep (with its failure count raised),
199+
* or null when it should be dropped because the source has failed
200+
* MAX_FAILURES times in a row or the image is older than twice maxAgeDays.
201+
*/
202+
export function afterFailure(record, { today, maxAgeDays = 180 }) {
203+
if (!record) return null;
204+
const failures = (Number(record.failures) || 0) + 1;
205+
const fetched = Date.parse(`${record.fetched}T00:00:00Z`);
206+
const now = Date.parse(`${today}T00:00:00Z`);
207+
const ancient = Number.isNaN(fetched) || Number.isNaN(now) || now - fetched > 2 * maxAgeDays * DAY;
208+
if (failures >= MAX_FAILURES || ancient) return null;
209+
return { ...record, failures };
210+
}
211+
149212
/** Width and height of a WebP file from its header (VP8, VP8L or VP8X), or null. */
150213
export function webpSize(buf) {
151214
if (!buf || buf.length < 25) return null;

‎site/thumbs.mjs‎

Lines changed: 35 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ import path from "node:path";
1919
import { fileURLToPath } from "node:url";
2020
import { listEntryFiles, idMatchesFile } from "./lib/entry-files.mjs";
2121
import { parseFrontmatter } from "./lib/frontmatter.mjs";
22-
import { imageUrlFromHtml, needsFetch, previewEligibility, sharedImageIds, webpSize } from "./lib/thumbs.mjs";
22+
import { afterFailure, imageUrlFromHtml, needsFetch, previewEligibility, safeImageUrl, sharedImageIds, sniffImage, webpSize } from "./lib/thumbs.mjs";
2323

2424
const __dirname = path.dirname(fileURLToPath(import.meta.url));
2525
const ROOT = path.resolve(__dirname, "..");
@@ -30,6 +30,8 @@ const USER_AGENT = "Mozilla/5.0 (compatible; FreeGameDevAssets-thumbs; +https://
3030
const MAX_IMAGE_BYTES = 15 * 1024 * 1024;
3131
const TIMEOUT_MS = 20000;
3232
const CONCURRENCY = 4;
33+
// A converter that hangs on a hostile file is killed rather than stalling the run.
34+
const CONVERT_TIMEOUT_MS = 30000;
3335

3436
function parseArgs(argv) {
3537
const opts = { force: false, only: null, limit: Infinity, dryRun: false, report: null };
@@ -91,20 +93,37 @@ function convert(converter, input, output) {
9193
execFileSync(
9294
"ffmpeg",
9395
["-y", "-loglevel", "error", "-i", input, "-frames:v", "1", "-vf", "scale='min(640,iw)':-2", "-c:v", "libwebp", "-quality", "78", output],
94-
{ stdio: ["ignore", "ignore", "pipe"] }
96+
{ stdio: ["ignore", "ignore", "pipe"], timeout: CONVERT_TIMEOUT_MS, killSignal: "SIGKILL", maxBuffer: 1024 * 1024 }
9597
);
9698
} else {
97-
execFileSync("magick", [`${input}[0]`, "-resize", "640x>", "-quality", "78", output], { stdio: ["ignore", "ignore", "pipe"] });
99+
execFileSync("magick", [`${input}[0]`, "-resize", "640x>", "-quality", "78", output], { stdio: ["ignore", "ignore", "pipe"], timeout: CONVERT_TIMEOUT_MS, killSignal: "SIGKILL", maxBuffer: 1024 * 1024 });
98100
}
99101
}
100102

101103
async function fetchBytes(url, { maxBytes = MAX_IMAGE_BYTES } = {}) {
102-
const res = await fetch(url, { headers: { "user-agent": USER_AGENT, accept: "*/*" }, signal: AbortSignal.timeout(TIMEOUT_MS), redirect: "follow" });
104+
const safe = safeImageUrl(url);
105+
if (!safe) throw new Error(`image URL refused (needs a public https host): ${url}`);
106+
const res = await fetch(safe, { headers: { "user-agent": USER_AGENT, accept: "image/*" }, signal: AbortSignal.timeout(TIMEOUT_MS), redirect: "follow" });
103107
if (!res.ok) throw new Error(`HTTP ${res.status}`);
108+
if (res.url && !safeImageUrl(res.url)) throw new Error(`image redirected to a refused URL: ${res.url}`);
104109
const declared = Number(res.headers.get("content-length") || 0);
105110
if (declared > maxBytes) throw new Error(`image is ${declared} bytes, over the ${maxBytes} limit`);
106-
const buf = Buffer.from(await res.arrayBuffer());
107-
if (buf.length > maxBytes) throw new Error(`image is ${buf.length} bytes, over the ${maxBytes} limit`);
111+
// Stream with a cap: content-length is optional and may lie.
112+
const chunks = [];
113+
let total = 0;
114+
const reader = res.body.getReader();
115+
for (;;) {
116+
const { done, value } = await reader.read();
117+
if (done) break;
118+
total += value.length;
119+
if (total > maxBytes) {
120+
await reader.cancel();
121+
throw new Error(`image is over the ${maxBytes} byte limit`);
122+
}
123+
chunks.push(value);
124+
}
125+
const buf = Buffer.concat(chunks);
126+
if (!sniffImage(buf)) throw new Error("response is not a PNG, JPEG, GIF or WebP image");
108127
return buf;
109128
}
110129

@@ -203,7 +222,7 @@ async function main() {
203222
}
204223
const rec = thumbs[entry.id];
205224
const fileExists = fs.existsSync(path.join(THUMBS_DIR, `${entry.id}.webp`));
206-
if (!needsFetch(rec, { license: entry.license, today, force: opts.force }) && fileExists) {
225+
if (!needsFetch(rec, { license: entry.license, today, force: opts.force, url: entry.url, preview: entry.preview || "" }) && fileExists) {
207226
summary.kept += 1;
208227
continue;
209228
}
@@ -225,6 +244,15 @@ async function main() {
225244
process.stdout.write(` + ${entry.id} ${record.width}x${record.height} ${bytes.length} B\n`);
226245
} catch (err) {
227246
summary.failed.push({ id: entry.id, reason: err.message });
247+
// A source that keeps failing does not keep its old image forever.
248+
if (thumbs[entry.id]) {
249+
const kept = afterFailure(thumbs[entry.id], { today });
250+
if (kept) thumbs[entry.id] = kept;
251+
else {
252+
summary.removed.push({ id: entry.id, reason: `refresh failed repeatedly or image too old (${err.message})`, hash: thumbs[entry.id].hash });
253+
delete thumbs[entry.id];
254+
}
255+
}
228256
process.stdout.write(` ! ${entry.id}: ${err.message}\n`);
229257
}
230258
},

0 commit comments

Comments
 (0)