@@ -220,20 +220,20 @@ export function reportMarkdown(summary) {
220220 return `${ lines . join ( "\n" ) } \n` ;
221221}
222222
223- async function main ( ) {
224- const opts = parseArgs ( process . argv . slice ( 2 ) ) ;
225- const today = new Date ( ) . toISOString ( ) . slice ( 0 , 10 ) ;
226- const converter = detectConverter ( ) ;
227- const entries = loadEntries ( ) ;
223+ /**
224+ * The run's plan, before any fetch (pure; #264): drops records whose entry
225+ * is gone or no longer eligible, then sorts every entry into kept, skipped
226+ * (gate, shared, limit) or todo. `fileExists(id)` says whether the entry's
227+ * .webp is on disk. Returns `{ thumbs, shared, todo, summary }`; the maps
228+ * are copies of the manifest's.
229+ */
230+ export function planRun ( { entries, manifest, fileExists, opts, today, converter = null } ) {
228231 const byId = new Map ( entries . map ( ( e ) => [ e . id , e ] ) ) ;
229- const manifest = loadManifest ( ) ;
230232 const thumbs = { ...manifest . thumbs } ;
231233 // Entries whose image turned out to be shared (a logo): remembered so they
232234 // are not fetched again until the record would be refreshed anyway.
233235 const shared = { ...manifest . shared } ;
234- const tmpDir = fs . mkdtempSync ( path . join ( os . tmpdir ( ) , "fgda-thumbs-" ) ) ;
235236 const summary = { today, converter, fetched : [ ] , kept : 0 , skipped : [ ] , failed : [ ] , bumped : [ ] , removed : [ ] , dropped : [ ] } ;
236- const pending = new Map ( ) ; // id → bytes to write
237237
238238 // Records whose entry is gone or no longer eligible go first, so a licence
239239 // that changed to custom never survives a run.
@@ -264,8 +264,7 @@ async function main() {
264264 summary . skipped . push ( { id : entry . id , reason : "image shared with other entries" , kind : "shared" } ) ;
265265 continue ;
266266 }
267- const fileExists = fs . existsSync ( path . join ( THUMBS_DIR , `${ entry . id } .webp` ) ) ;
268- if ( ! needsFetch ( rec , fetchOpts ) && fileExists ) {
267+ if ( ! needsFetch ( rec , fetchOpts ) && fileExists ( entry . id ) ) {
269268 summary . kept += 1 ;
270269 continue ;
271270 }
@@ -275,39 +274,36 @@ async function main() {
275274 }
276275 todo . push ( entry ) ;
277276 }
277+ return { thumbs, shared, todo, summary } ;
278+ }
278279
279- await pool (
280- todo ,
281- async ( entry ) => {
282- try {
283- const { record, bytes } = await fetchOne ( entry , { converter, tmpDir, today } ) ;
284- thumbs [ entry . id ] = record ;
285- pending . set ( entry . id , bytes ) ;
286- summary . fetched . push ( { id : entry . id , width : record . width , height : record . height , bytes : bytes . length } ) ;
287- process . stdout . write ( ` + ${ entry . id } ${ record . width } x${ record . height } ${ bytes . length } B\n` ) ;
288- } catch ( err ) {
289- summary . failed . push ( { id : entry . id , reason : err . message } ) ;
290- // A source that keeps failing does not keep its old image forever,
291- // and a record for a url or pin the entry no longer has goes at once.
292- const old = thumbs [ entry . id ] ;
293- if ( old ) {
294- const kept = afterFailure ( old , { today, url : entry . url , preview : entry . preview || "" , countFailure : Boolean ( converter ) } ) ;
295- if ( ! kept ) {
296- summary . removed . push ( { id : entry . id , reason : `refresh failed and the stored image is stale (${ err . message } )` , hash : old . hash } ) ;
297- delete thumbs [ entry . id ] ;
298- } else if ( kept !== old ) {
299- thumbs [ entry . id ] = kept ;
300- // A raised failure count is a change: without it the count never
301- // reaches the repository and the MAX_FAILURES rule cannot fire.
302- summary . bumped . push ( { id : entry . id , failures : kept . failures } ) ;
303- }
304- }
305- process . stdout . write ( ` ! ${ entry . id } : ${ err . message } \n` ) ;
306- }
307- } ,
308- CONCURRENCY
309- ) ;
280+ /**
281+ * A failed fetch (pure; mutates `thumbs` and `summary`). A source that keeps
282+ * failing does not keep its old image forever, and a record for a url or pin
283+ * the entry no longer has goes at once.
284+ */
285+ export function recordFailure ( { thumbs, summary, entry, reason, today, countFailure } ) {
286+ summary . failed . push ( { id : entry . id , reason } ) ;
287+ const old = thumbs [ entry . id ] ;
288+ if ( ! old ) return ;
289+ const kept = afterFailure ( old , { today, url : entry . url , preview : entry . preview || "" , countFailure } ) ;
290+ if ( ! kept ) {
291+ summary . removed . push ( { id : entry . id , reason : `refresh failed and the stored image is stale (${ reason } )` , hash : old . hash } ) ;
292+ delete thumbs [ entry . id ] ;
293+ } else if ( kept !== old ) {
294+ thumbs [ entry . id ] = kept ;
295+ // A raised failure count is a change: without it the count never
296+ // reaches the repository and the MAX_FAILURES rule cannot fire.
297+ summary . bumped . push ( { id : entry . id , failures : kept . failures } ) ;
298+ }
299+ }
310300
301+ /**
302+ * After the fetches (pure; mutates `thumbs`, `shared`, `pending`, `summary`):
303+ * moves logos to `shared`, sorts both maps and counts the net change against
304+ * the manifest. Returns `{ sorted, sortedShared, changed }`.
305+ */
306+ export function settleRun ( { manifest, thumbs, shared, pending, summary } ) {
311307 // A hash shared by three or more entries is a logo, not a preview. Logos
312308 // dropped in earlier runs (in `shared`) count toward the three.
313309 for ( const id of sharedImageIds ( thumbs , shared ) ) {
@@ -324,14 +320,47 @@ async function main() {
324320 // Only a net change to the manifest or the files counts, so a run that
325321 // refetches and re-drops the same shared images changes nothing.
326322 const changed = manifestChanges ( manifest , { thumbs : sorted , shared : sortedShared } ) + pending . size ;
323+ return { sorted, sortedShared, changed } ;
324+ }
325+
326+ /** Files in the thumbs folder to delete: a .webp with no manifest record. Other files stay. */
327+ export function orphanThumbs ( names , sorted ) {
328+ return names . filter ( ( name ) => name . endsWith ( ".webp" ) && ! sorted [ name . replace ( / \. w e b p $ / , "" ) ] ) ;
329+ }
330+
331+ async function main ( ) {
332+ const opts = parseArgs ( process . argv . slice ( 2 ) ) ;
333+ const today = new Date ( ) . toISOString ( ) . slice ( 0 , 10 ) ;
334+ const converter = detectConverter ( ) ;
335+ const manifest = loadManifest ( ) ;
336+ const fileExists = ( id ) => fs . existsSync ( path . join ( THUMBS_DIR , `${ id } .webp` ) ) ;
337+ const { thumbs, shared, todo, summary } = planRun ( { entries : loadEntries ( ) , manifest, fileExists, opts, today, converter } ) ;
338+ const tmpDir = fs . mkdtempSync ( path . join ( os . tmpdir ( ) , "fgda-thumbs-" ) ) ;
339+ const pending = new Map ( ) ; // id → bytes to write
340+
341+ await pool (
342+ todo ,
343+ async ( entry ) => {
344+ try {
345+ const { record, bytes } = await fetchOne ( entry , { converter, tmpDir, today } ) ;
346+ thumbs [ entry . id ] = record ;
347+ pending . set ( entry . id , bytes ) ;
348+ summary . fetched . push ( { id : entry . id , width : record . width , height : record . height , bytes : bytes . length } ) ;
349+ process . stdout . write ( ` + ${ entry . id } ${ record . width } x${ record . height } ${ bytes . length } B\n` ) ;
350+ } catch ( err ) {
351+ recordFailure ( { thumbs, summary, entry, reason : err . message , today, countFailure : Boolean ( converter ) } ) ;
352+ process . stdout . write ( ` ! ${ entry . id } : ${ err . message } \n` ) ;
353+ }
354+ } ,
355+ CONCURRENCY
356+ ) ;
357+
358+ const { sorted, sortedShared, changed } = settleRun ( { manifest, thumbs, shared, pending, summary } ) ;
327359
328360 if ( ! opts . dryRun ) {
329361 fs . mkdirSync ( THUMBS_DIR , { recursive : true } ) ;
330362 for ( const [ id , bytes ] of pending ) fs . writeFileSync ( path . join ( THUMBS_DIR , `${ id } .webp` ) , bytes ) ;
331- for ( const name of fs . readdirSync ( THUMBS_DIR ) ) {
332- const id = name . replace ( / \. w e b p $ / , "" ) ;
333- if ( name . endsWith ( ".webp" ) && ! sorted [ id ] ) fs . rmSync ( path . join ( THUMBS_DIR , name ) , { force : true } ) ;
334- }
363+ for ( const name of orphanThumbs ( fs . readdirSync ( THUMBS_DIR ) , sorted ) ) fs . rmSync ( path . join ( THUMBS_DIR , name ) , { force : true } ) ;
335364 if ( changed ) {
336365 const out = { generated : today , thumbs : sorted , ...( Object . keys ( sortedShared ) . length ? { shared : sortedShared } : { } ) } ;
337366 fs . writeFileSync ( MANIFEST , `${ JSON . stringify ( out , null , 2 ) } \n` ) ;
0 commit comments