ci(index): read-only audit of RAR4 reprocessing population

Author: ShulhaOleh

.github/workflows/audit-rar-reprocessing.yml

@@ -0,0 +1,38 @@
+name: Audit RAR reprocessing population
+
+on:
+    workflow_dispatch:
+    push:
+        branches:
+            - diagnose-rar-reprocessing
+
+permissions:
+    contents: read
+
+concurrency:
+    group: audit-rar-reprocessing
+    cancel-in-progress: false
+
+jobs:
+    audit:
+        runs-on: ubuntu-latest
+        timeout-minutes: 15
+        env:
+            INDEX_DATABASE_URL: ${{ secrets.INDEX_DATABASE_URL }}
+        steps:
+            - uses: actions/checkout@v7
+
+            - uses: pnpm/action-setup@v6
+              with:
+                  version: 10.32.1
+
+            - uses: actions/setup-node@v7
+              with:
+                  node-version: 22
+                  cache: pnpm
+
+            - name: Install dependencies
+              run: pnpm install --frozen-lockfile
+
+            - name: Run read-only population audit
+              run: pnpm --dir apps/index exec tsx audit-rar-reprocessing.mjs

apps/index/audit-rar-reprocessing.mjs

@@ -0,0 +1,120 @@
+// Disposable, read-only population audit for the RAR4 reprocessing investigation. Never
+// merged into main. Queries only; the session is put into a hard read-only transaction
+// before any SELECT runs, so an edited query cannot mutate production even by accident.
+
+import { Pool } from '@neondatabase/serverless'
+
+const databaseUrl = process.env.INDEX_DATABASE_URL
+if (!databaseUrl) throw new Error('INDEX_DATABASE_URL is required')
+
+// Unique ModWorkshop PD2 mod ids resolved from the 269 historical RAR4-failure URLs in
+// production run 31929156956: 167 matched against live ModWorkshop listing metadata by
+// name, plus 53 more recovered from our own retained mod_listings rows (sync-listings
+// only ever upserts, so a row can outlive its mod's removal from ModWorkshop). No archive
+// bytes were re-downloaded to produce this list. 3 further candidates were ambiguous
+// (multiple listings share the exact same name) and 46 remain unresolved; both are
+// excluded here — see the audit report for the full breakdown.
+const resolvedRemoteIds = [
+    13804, 12676, 13788, 13792, 13789, 13775, 13769, 13767, 13766, 13751, 13757, 13756, 13752,
+    13712, 13750, 13612, 13739, 13738, 13728, 13727, 13726, 228, 13711, 13719, 13713, 13707, 13700,
+    13706, 13705, 13692, 13681, 13640, 13605, 13593, 13621, 13576, 13579, 13607, 12575, 13604,
+    13602, 13565, 13570, 13560, 13553, 12987, 13547, 13522, 13514, 13503, 13499, 13475, 13397,
+    13462, 13461, 13459, 13162, 13430, 13411, 13402, 13362, 13123, 13350, 13351, 13326, 13285,
+    13268, 12991, 207, 224, 13239, 13238, 13237, 13234, 13230, 13205, 13201, 13195, 13190, 13174,
+    13137, 13121, 13111, 13117, 13115, 13114, 13107, 12978, 13095, 13090, 13084, 13074, 13068,
+    13054, 13053, 13051, 13046, 13033, 13020, 13014, 13011, 12998, 12989, 12990, 12985, 12955,
+    12963, 12939, 12957, 12936, 12915, 12918, 12719, 12898, 12895, 12889, 12885, 12877, 12873,
+    12872, 12863, 12862, 12838, 12835, 12818, 12828, 12820, 12815, 12814, 12760, 12782, 12745,
+    12700, 12695, 1287, 12686, 12653, 12645, 12599, 12570, 12637, 12636, 12634, 12598, 12571, 12583,
+    12553, 12552, 12549, 12544, 364, 12536, 12535, 12534, 12532, 12528, 12523, 12505, 12456, 12421,
+    12432, 12430, 12429, 1518, 12373, 12390, 12379, 266, 13540, 13157, 1410, 1409, 264, 13215,
+    13213, 411, 37, 406, 420, 422, 423, 182, 12933, 442, 12812, 12809, 12657, 12648, 12425, 12591,
+    12526, 574, 12527, 576, 575, 12495, 12354, 329, 331, 12424, 12397, 12395, 678, 666, 665, 659,
+    655, 654, 653, 652, 651, 649, 646, 645, 644, 640, 639, 638, 637, 636,
+]
+
+const pool = new Pool({ connectionString: databaseUrl })
+const client = await pool.connect()
+
+try {
+    await client.query('SET default_transaction_read_only = on')
+    await client.query('BEGIN TRANSACTION READ ONLY')
+
+    const source = await client.query(
+        `SELECT sources.id
+         FROM sources
+         JOIN games ON games.id = sources.game_id
+         WHERE games.slug = 'pd2' AND sources.name = 'modworkshop'`
+    )
+    if (source.rows.length !== 1) throw new Error('expected exactly one pd2/modworkshop source')
+    const sourceId = source.rows[0].id
+    console.log(`pd2 modworkshop source_id = ${sourceId}`)
+
+    const rows = await client.query(
+        `WITH target(remote_id) AS (
+             SELECT UNNEST($1::BIGINT[])
+         )
+         SELECT
+             target.remote_id,
+             mod_listings.remote_id IS NOT NULL AS listing_exists,
+             mod_listings.has_download,
+             mod_listings.updated_at AS listing_updated_at,
+             mod_listings.name AS listing_name,
+             mod_checks.remote_id IS NOT NULL AS check_exists,
+             mod_checks.updated_at AS check_updated_at,
+             mod_checks.checked_at,
+             mod_checks.file_ids,
+             (mod_checks.file_ids = '[]'::jsonb) AS check_empty,
+             (
+                 mod_listings.has_download
+                 AND (mod_checks.remote_id IS NULL OR mod_checks.updated_at <> mod_listings.updated_at)
+             ) AS currently_pending
+         FROM target
+         LEFT JOIN mod_listings
+             ON mod_listings.source_id = $2 AND mod_listings.remote_id = target.remote_id
+         LEFT JOIN mod_checks
+             ON mod_checks.source_id = $2 AND mod_checks.remote_id = target.remote_id
+         ORDER BY target.remote_id`,
+        [resolvedRemoteIds, sourceId]
+    )
+
+    let missing = 0
+    let noCheck = 0
+    let settledEmpty = 0
+    let indexed = 0
+    let pending = 0
+    let hasDownloadFalse = 0
+    let other = 0
+
+    for (const row of rows.rows) {
+        if (!row.listing_exists) {
+            missing++
+            continue
+        }
+        if (!row.has_download) hasDownloadFalse++
+        if (row.currently_pending) pending++
+        if (!row.check_exists) noCheck++
+        else if (row.check_empty) settledEmpty++
+        else indexed++
+        if (row.check_exists && !row.check_empty && row.currently_pending) other++
+    }
+
+    console.log(`total targeted remote_ids: ${resolvedRemoteIds.length}`)
+    console.log(`missing (no mod_listings row): ${missing}`)
+    console.log(`has_download = false: ${hasDownloadFalse}`)
+    console.log(`no mod_checks row at all: ${noCheck}`)
+    console.log(`settled-empty (file_ids = '[]'): ${settledEmpty}`)
+    console.log(`already indexed (file_ids non-empty): ${indexed}`)
+    console.log(`currently pending under production predicate: ${pending}`)
+    console.log(`unexpected (indexed but also pending): ${other}`)
+
+    console.log('--- full row dump ---')
+    for (const row of rows.rows) {
+        console.log(JSON.stringify(row))
+    }
+
+    await client.query('ROLLBACK')
+} finally {
+    client.release()
+    await pool.end()
+}