Windows dependency matching, including missing-file recovery.

One shared content-root lookup per build, with retry after failure.
Configurable image-cache expiry: 30 days unused by default, with atomic publication for concurrent builds. Set cacheMaxAgeDays: 0 to disable expiry.

Author: koculu

packages/ts-images/README.md

@@ -64,6 +64,7 @@ can override. Settings are also available under typed `extensions.images`:
 | `metadata` | `strip`; auto-orient first, convert to sRGB, then omit metadata; `preserve` retains metadata with orientation normalized |
 | `include` / `exclude` | `['**/*']` / `[]`; glob patterns on content-root-relative paths using `/` |
 | `cacheDir` | OS temporary cache, namespaced by content root; custom paths resolve from `rootDir` and must be outside content and output directories |
+| `cacheMaxAgeDays` | `30`; entries unused for this many days expire; `0` disables automatic cleanup |
 | `concurrency` | `2`; per-site limit from 1–16, including encoding and output emission |
 | `maxInputPixels` | 40 million |
 | `maxSourceBytes` | 50 MiB |
@@ -128,6 +129,15 @@ loading, and `sizes` reuse encoded files. Cache contents are verified by hash,
 and incomplete or corrupted entries are rebuilt. Concurrent identical work is
 shared within a site; writes use atomic replacement.
 
+Cache reads renew an entry's last-use timestamp. During image processing, cleanup
+runs at most once an hour per processor and removes entries unused for
+`cacheMaxAgeDays`, including those in a custom `cacheDir`. Completed entries are
+published atomically so independent builds can safely share the cache. Cleanup
+also removes abandoned staging entries after the retention period and leaves
+unrelated files and symlinked entries alone. Expiry affects only the encoding
+cache, never source images or published assets. No cleanup runs while the plugin
+is idle; `cacheMaxAgeDays: 0` keeps entries until you remove them manually.
+
 Optimization emits diagnostics with the referring page, source, cache result,
 input bytes, total generated bytes, variant count, and duration. Generated bytes
 are the total of all variants, not the bytes a particular browser downloads.

packages/ts-images/src/cache.test.ts

@@ -0,0 +1,102 @@
+import fs from 'node:fs/promises'
+import os from 'node:os'
+import path from 'node:path'
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import { pruneImageCache } from './cache'
+
+describe('image cache retention', () => {
+  let root: string
+  const old = new Date(Date.now() - 60 * 86_400_000)
+  beforeEach(async () => {
+    root = await fs.mkdtemp(
+      path.join(os.tmpdir(), 'purestack-cache-retention-'),
+    )
+  })
+  afterEach(async () => {
+    vi.restoreAllMocks()
+    await fs.rm(root, { recursive: true, force: true })
+  })
+  async function entry(name: string, date = old) {
+    const directory = path.join(root, name)
+    await fs.mkdir(directory)
+    await fs.writeFile(
+      path.join(directory, 'manifest.json'),
+      JSON.stringify({ version: 1, key: name, variants: [] }),
+    )
+    await fs.utimes(path.join(directory, 'manifest.json'), date, date)
+    return directory
+  }
+
+  it('expires unused owned entries while retaining fresh, active, foreign, and symlinked paths', async () => {
+    const expired = await entry('a'.repeat(64))
+    await entry('b'.repeat(64), new Date())
+    await entry('c'.repeat(64))
+    await entry('notes')
+    const foreign = await entry('d'.repeat(64))
+    await fs.writeFile(
+      path.join(foreign, 'manifest.json'),
+      '{"key":"unrelated"}',
+    )
+    await fs.utimes(path.join(foreign, 'manifest.json'), old, old)
+    await fs.writeFile(path.join(root, 'keep.txt'), 'Keep')
+    const alias = path.join(root, 'e'.repeat(64))
+    await fs.symlink(
+      path.join(root, 'notes'),
+      alias,
+      process.platform === 'win32' ? 'junction' : 'dir',
+    )
+    // Two sites/processors can sweep the same shared cache concurrently.
+    await Promise.all(
+      Array.from({ length: 2 }, () =>
+        pruneImageCache(root, 30, new Set(['c'.repeat(64)])),
+      ),
+    )
+    await expect(fs.stat(expired)).rejects.toMatchObject({ code: 'ENOENT' })
+    expect((await fs.readdir(root)).sort()).toEqual(
+      [
+        'b'.repeat(64),
+        'c'.repeat(64),
+        'd'.repeat(64),
+        'e'.repeat(64),
+        'keep.txt',
+        'notes',
+      ].sort(),
+    )
+    expect(await fs.readFile(path.join(root, 'keep.txt'), 'utf8')).toBe('Keep')
+  })
+
+  it('allows automatic expiry to be disabled', async () => {
+    await entry('a'.repeat(64))
+    await pruneImageCache(root, 0, new Set())
+    expect(await fs.readdir(root)).toEqual(['a'.repeat(64)])
+  })
+
+  it('rechecks expiry after reading a manifest to respect concurrent cache hits', async () => {
+    const directory = await entry('a'.repeat(64))
+    const read = fs.readFile.bind(fs)
+    vi.spyOn(fs, 'readFile').mockImplementation(
+      async (...args: Parameters<typeof fs.readFile>) => {
+        const value = await read(...args)
+        const now = new Date()
+        await fs.utimes(path.join(directory, 'manifest.json'), now, now)
+        return value
+      },
+    )
+    await pruneImageCache(root, 30, new Set())
+    expect(await fs.readdir(root)).toEqual(['a'.repeat(64)])
+  })
+
+  it('cleans abandoned staging directories without touching fresh staging work', async () => {
+    const stale = '.purestack-images-12345678-1234-1234-1234-123456789abc.tmp'
+    const fresh = '.purestack-images-12345678-1234-1234-1234-123456789abd.tmp'
+    const retired =
+      '.purestack-images-12345678-1234-1234-1234-123456789abe.trash'
+    for (const name of [stale, fresh, retired]) {
+      await fs.mkdir(path.join(root, name))
+      await fs.writeFile(path.join(root, name, 'partial.png'), 'Incomplete')
+      if (name !== fresh) await fs.utimes(path.join(root, name), old, old)
+    }
+    await pruneImageCache(root, 30, new Set())
+    expect(await fs.readdir(root)).toEqual([fresh])
+  })
+})

packages/ts-images/src/cache.ts

@@ -0,0 +1,87 @@
+import { randomUUID } from 'node:crypto'
+import fs from 'node:fs/promises'
+import path from 'node:path'
+import { isPlainObject } from '@purestack/ts-util'
+
+const entryName = /^[a-f0-9]{64}$/
+const temporaryName =
+  /^\.purestack-images-[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}\.(?:tmp|trash)$/
+
+/** Retire by rename: readers either finish reading or treat a vanished entry as a miss. */
+export async function retireCacheEntry(root: string, key: string) {
+  if (!entryName.test(key)) throw new Error('Invalid image cache key.')
+  const directory = path.join(root, key)
+  const stat = await fs.lstat(directory).catch(missing)
+  if (!stat) return
+  if (!stat.isDirectory() || stat.isSymbolicLink())
+    throw new Error('Image cache entries must be directories, not symlinks.')
+  const retired = path.join(root, `.purestack-images-${randomUUID()}.trash`)
+  try {
+    await fs.rename(directory, retired)
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === 'ENOENT') return
+    // Windows can report EPERM when another process has already moved the directory.
+    if (
+      (error as NodeJS.ErrnoException).code === 'EPERM' &&
+      !(await fs.lstat(directory).catch(missing))
+    )
+      return
+    throw error
+  }
+  await fs.rm(retired, { recursive: true, force: true }).catch(cleanupError)
+}
+
+/** Only our expired entries are collected; a custom cache directory may contain other files. */
+export async function pruneImageCache(
+  root: string,
+  maxAgeDays: number,
+  activeKeys: ReadonlySet<string>,
+) {
+  if (maxAgeDays === 0) return
+  const cutoff = Date.now() - maxAgeDays * 86_400_000
+  for (const entry of await fs.readdir(root, { withFileTypes: true })) {
+    if (!entry.isDirectory() || entry.isSymbolicLink()) continue
+    const directory = path.join(root, entry.name)
+    try {
+      if (temporaryName.test(entry.name)) {
+        const stat = await fs.lstat(directory)
+        if (stat.mtimeMs < cutoff)
+          await fs.rm(directory, { recursive: true, force: true })
+        continue
+      }
+      if (!entryName.test(entry.name) || activeKeys.has(entry.name)) continue
+      const manifest = path.join(directory, 'manifest.json')
+      const stat = await fs.lstat(manifest)
+      if (!stat.isFile() || stat.size > 64 * 1024 || stat.mtimeMs >= cutoff)
+        continue
+      const value: unknown = JSON.parse(await fs.readFile(manifest, 'utf8'))
+      if (
+        !isPlainObject(value) ||
+        value.version !== 1 ||
+        value.key !== entry.name ||
+        !Array.isArray(value.variants)
+      )
+        continue
+      // A concurrent cache hit may have renewed this entry while we read it.
+      if ((await fs.stat(manifest)).mtimeMs < cutoff)
+        await retireCacheEntry(root, entry.name)
+    } catch (error) {
+      if (error instanceof SyntaxError) continue
+      cleanupError(error)
+    }
+  }
+}
+
+function missing(error: NodeJS.ErrnoException): undefined {
+  if (error.code !== 'ENOENT') throw error
+}
+
+function cleanupError(error: unknown) {
+  // Another process can retire the entry or still hold a Windows file handle.
+  if (
+    !['ENOENT', 'EACCES', 'EPERM', 'EBUSY', 'ENOTEMPTY'].includes(
+      (error as NodeJS.ErrnoException).code ?? '',
+    )
+  )
+    throw error
+}

packages/ts-images/src/config.ts

@@ -30,6 +30,8 @@ export interface ImagesOptions extends ImagePreset {
   include?: string[]
   exclude?: string[]
   cacheDir?: string
+  /** Remove entries unused for this many days; zero disables automatic cleanup. */
+  cacheMaxAgeDays?: number
   concurrency?: number
   maxInputPixels?: number
   maxSourceBytes?: number
@@ -41,6 +43,7 @@ export interface ImagesConfig extends ResolvedImagePreset {
   include: string[]
   exclude: string[]
   cacheDir: string
+  cacheMaxAgeDays: number
   concurrency: number
   maxInputPixels: number
   maxSourceBytes: number
@@ -99,6 +102,7 @@ export function resolveImagesConfig(
       'include',
       'exclude',
       'cacheDir',
+      'cacheMaxAgeDays',
       'concurrency',
       'maxInputPixels',
       'maxSourceBytes',
@@ -149,6 +153,13 @@ export function resolveImagesConfig(
     include: patterns(input.include, ['**/*'], 'images.include'),
     exclude: patterns(input.exclude, [], 'images.exclude'),
     cacheDir,
+    cacheMaxAgeDays: integer(
+      input.cacheMaxAgeDays,
+      30,
+      0,
+      36500,
+      'images.cacheMaxAgeDays',
+    ),
     concurrency: integer(input.concurrency, 2, 1, 16, 'images.concurrency'),
     maxInputPixels: integer(
       input.maxInputPixels,

packages/ts-images/src/processor.test.ts

@@ -2,7 +2,7 @@ import fs from 'node:fs/promises'
 import os from 'node:os'
 import path from 'node:path'
 import sharp from 'sharp'
-import { afterEach, beforeEach, describe, expect, it } from 'vitest'
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
 import { imagePreset, resolveImagesConfig } from './config'
 import { ImageProcessor } from './processor'
 
@@ -12,6 +12,7 @@ describe('image processor and persistent cache', () => {
     root = await fs.mkdtemp(path.join(os.tmpdir(), 'purestack-image-test-'))
   })
   afterEach(async () => {
+    vi.restoreAllMocks()
     await fs.rm(root, { recursive: true, force: true })
   })
   const config = (options = {}) =>
@@ -71,7 +72,8 @@ describe('image processor and persistent cache', () => {
       first?.variants.map((variant) => variant.hash),
     )
     const file = path.join(settings.cacheDir, first?.key ?? '', fallback.file)
-    await fs.writeFile(file, 'Corrupt cache')
+    // Same-size corruption must still be detected by content verification.
+    await fs.writeFile(file, Buffer.alloc(fallback.data.length))
     expect(
       (await new ImageProcessor(settings).process(source(), settings))?.cache,
     ).toBe('miss')
@@ -234,4 +236,63 @@ describe('image processor and persistent cache', () => {
     ).rejects.toThrow('including symlink targets')
     expect(await fs.readdir(content)).toEqual([])
   })
+
+  it('renews cache hits and prunes expired variants as source images change', async () => {
+    const settings = config({ widths: [16], formats: [], cacheMaxAgeDays: 1 })
+    const write = (background: string) =>
+      sharp({ create: { width: 32, height: 16, channels: 3, background } })
+        .png()
+        .toFile(source().absPath)
+    await write('red')
+    const first = await new ImageProcessor(settings).process(source(), settings)
+    if (!first) throw new Error('Expected image')
+    const manifest = path.join(settings.cacheDir, first.key, 'manifest.json')
+    const old = new Date(Date.now() - 3 * 86_400_000)
+    await fs.utimes(manifest, old, old)
+    // A requested entry is reused and renewed even if its prior age exceeded the limit.
+    expect(
+      (await new ImageProcessor(settings).process(source(), settings))?.cache,
+    ).toBe('hit')
+    expect((await fs.stat(manifest)).mtimeMs).toBeGreaterThan(old.getTime())
+    await fs.utimes(manifest, old, old)
+    await write('blue')
+    const next = await new ImageProcessor(settings).process(source(), settings)
+    expect(next?.key).not.toBe(first.key)
+    await expect(fs.stat(manifest)).rejects.toMatchObject({ code: 'ENOENT' })
+    expect(
+      (await new ImageProcessor(settings).process(source(), settings))?.cache,
+    ).toBe('hit')
+  })
+
+  it('publishes complete entries when independent processors share a cache, including corruption repair', async () => {
+    await sharp({
+      create: { width: 32, height: 16, channels: 3, background: 'green' },
+    })
+      .png()
+      .toFile(source().absPath)
+    const settings = config({ widths: [16], formats: ['webp'] })
+    const processTogether = () =>
+      Promise.all(
+        Array.from({ length: 4 }, () =>
+          new ImageProcessor(settings).process(source(), settings),
+        ),
+      )
+    const first = await processTogether()
+    expect(new Set(first.map((value) => value?.key)).size).toBe(1)
+    const value = first[0]
+    if (!value) throw new Error('Expected image')
+    await fs.writeFile(
+      path.join(settings.cacheDir, value.key, value.variants[0].file),
+      Buffer.alloc(value.variants[0].data.length),
+    )
+    const repaired = await processTogether()
+    for (const result of repaired)
+      expect(result?.variants.map((variant) => variant.hash)).toEqual(
+        value.variants.map((variant) => variant.hash),
+      )
+    expect(
+      (await new ImageProcessor(settings).process(source(), settings))?.cache,
+    ).toBe('hit')
+    expect(await fs.readdir(settings.cacheDir)).toEqual([value.key])
+  })
 })