11/**
22 * prune-spam.ts — Re-runnable spam-prune operator script.
33 *
4- * Reads spam verdicts from the `spam-detection` branch of the data repo,
5- * aggregates them per the spec rule, and removes confident-spam people from
6- * the `published` branch with cascaded deletes of their associated records.
4+ * Reads machine spam verdicts from the private spam-detection repo and staff
5+ * `human-*` votes from the data repo's served branch, aggregates them per the
6+ * spec rule (a human vote is final), and removes spam people from the
7+ * `published` branch with cascaded deletes of their associated records.
78 *
89 * Spec: specs/behaviors/spam-exclusion.md
910 *
1011 * Usage:
1112 * npm run -w apps/api script:prune-spam -- \
12- * --data-repo=/path/to/codeforphilly-data \
13- * [--evaluations-ref=spam-detection] \
13+ * --data-repo=/path/to/codeforphilly-data.git \
14+ * --evaluations-repo=/path/to/codeforphilly-spam-detection \
15+ * [--evaluations-ref=HEAD] \
16+ * [--human-votes-ref=published] \
1417 * [--branch=published] \
1518 * [--threshold=0.8] \
1619 * [--dry-run] [--verbose]
1720 *
1821 * --data-repo Path to a local bare clone of the data repo.
1922 * Falls back to $CFP_DATA_REPO_PATH.
20- * --evaluations-ref Ref to read person-evaluations from (default: spam-detection).
23+ * --evaluations-repo Repo (bare or not) holding machine person-evaluations —
24+ * the private codeforphilly-spam-detection clone. Defaults
25+ * to --data-repo for single-repo setups.
26+ * --evaluations-ref Ref in --evaluations-repo to read (default: HEAD).
27+ * --human-votes-ref Ref in --data-repo carrying staff votes (default: --branch).
2128 * --branch Branch to prune (default: published).
2229 * --threshold Spam confidence threshold (default: 0.8).
2330 * --dry-run Report without writing.
@@ -43,7 +50,11 @@ import type {
4350
4451interface CliArgs {
4552 readonly dataRepo : string ;
53+ /** Repo holding the machine evaluations (the private spam-detection clone). Defaults to dataRepo. */
54+ readonly evaluationsRepo : string ;
4655 readonly evaluationsRef : string ;
56+ /** Ref in dataRepo carrying staff `human-*` votes. Defaults to --branch. */
57+ readonly humanVotesRef : string ;
4758 readonly branch : string ;
4859 readonly threshold : number ;
4960 readonly dryRun : boolean ;
@@ -73,16 +84,23 @@ function parseArgs(argv: readonly string[]): CliArgs {
7384 const threshold =
7485 typeof thresholdRaw === 'string' ? Number . parseFloat ( thresholdRaw ) : 0.8 ;
7586
87+ const branch =
88+ typeof opts [ 'branch' ] === 'string' && opts [ 'branch' ] !== '' ? opts [ 'branch' ] : 'published' ;
7689 return {
7790 dataRepo : resolve ( dataRepoRaw ) ,
91+ evaluationsRepo :
92+ typeof opts [ 'evaluations-repo' ] === 'string' && opts [ 'evaluations-repo' ] !== ''
93+ ? resolve ( opts [ 'evaluations-repo' ] )
94+ : resolve ( dataRepoRaw ) ,
7895 evaluationsRef :
7996 typeof opts [ 'evaluations-ref' ] === 'string' && opts [ 'evaluations-ref' ] !== ''
8097 ? opts [ 'evaluations-ref' ]
81- : 'spam-detection' ,
82- branch :
83- typeof opts [ 'branch' ] === 'string' && opts [ 'branch' ] !== ''
84- ? opts [ 'branch' ]
85- : 'published' ,
98+ : 'HEAD' ,
99+ humanVotesRef :
100+ typeof opts [ 'human-votes-ref' ] === 'string' && opts [ 'human-votes-ref' ] !== ''
101+ ? opts [ 'human-votes-ref' ]
102+ : branch ,
103+ branch,
86104 threshold : Number . isFinite ( threshold ) ? threshold : 0.8 ,
87105 dryRun : opts [ 'dry-run' ] === true ,
88106 verbose : opts [ 'verbose' ] === true ,
@@ -94,22 +112,31 @@ function parseArgs(argv: readonly string[]): CliArgs {
94112// ---------------------------------------------------------------------------
95113
96114interface PersonVerdict {
97- /** Whether any evaluator gave spam confidence >= threshold. */
115+ /** Whether any machine evaluator gave spam confidence >= threshold. */
98116 hasConfidentSpam : boolean ;
99- /** Whether any evaluator gave a legit verdict at any confidence. */
117+ /** Whether any machine evaluator gave a legit verdict at any confidence. */
100118 hasAnyLegit : boolean ;
119+ /**
120+ * The latest `human-*` verdict, if any. A human vote is final: `spam`
121+ * prunes regardless of machine verdicts or project membership; `legit`
122+ * keeps. Per specs/behaviors/spam-exclusion.md.
123+ */
124+ humanVerdict : string | null ;
125+ humanEvaluatedAt : string | null ;
101126}
102127
103128/**
104- * Parse verdict and confidence from TOML content using line-regex
105- * (tolerant, avoids pulling in a full TOML parser just for two fields).
129+ * Parse verdict, confidence, and evaluatedAt from TOML content using
130+ * line-regex (tolerant, avoids pulling in a full TOML parser for three fields).
106131 */
107132function parseEvaluationRecord ( tomlContent : string ) : {
108133 verdict : string | null ;
109134 confidence : number | null ;
135+ evaluatedAt : string | null ;
110136} {
111137 let verdict : string | null = null ;
112138 let confidence : number | null = null ;
139+ let evaluatedAt : string | null = null ;
113140
114141 for ( const line of tomlContent . split ( '\n' ) ) {
115142 const trimmed = line . trim ( ) ;
@@ -122,12 +149,17 @@ function parseEvaluationRecord(tomlContent: string): {
122149 if ( confidenceMatch ) {
123150 const parsed = Number . parseFloat ( confidenceMatch [ 1 ] ?? '' ) ;
124151 if ( Number . isFinite ( parsed ) ) confidence = parsed ;
152+ continue ;
125153 }
154+ const atMatch = trimmed . match ( / ^ e v a l u a t e d A t \s * = \s * " ( [ ^ " ] + ) " / ) ;
155+ if ( atMatch ) evaluatedAt = atMatch [ 1 ] ?? null ;
126156 }
127157
128- return { verdict, confidence } ;
158+ return { verdict, confidence, evaluatedAt } ;
129159}
130160
161+ const HUMAN_EVALUATOR_PREFIX = 'human-' ;
162+
131163/**
132164 * Read all person-evaluations from the given ref via `git cat-file` bulk read.
133165 * Does NOT go through gitsheets — there are ~54k records and we want a
@@ -138,8 +170,9 @@ async function aggregateVerdicts(
138170 evaluationsRef : string ,
139171 threshold : number ,
140172 log : ( msg : string ) => void ,
173+ verdictMap : Map < string , PersonVerdict > = new Map ( ) ,
141174) : Promise < Map < string , PersonVerdict > > {
142- log ( `[prune-spam] listing person-evaluations under ref=${ evaluationsRef } ` ) ;
175+ log ( `[prune-spam] listing person-evaluations in ${ repo } under ref=${ evaluationsRef } ` ) ;
143176
144177 // List all blobs under person-evaluations/ in the evaluations ref.
145178 const lsOutput = await exec (
@@ -152,7 +185,7 @@ async function aggregateVerdicts(
152185 log ( `[prune-spam] found ${ lines . length } evaluation records` ) ;
153186
154187 if ( lines . length === 0 ) {
155- return new Map ( ) ;
188+ return verdictMap ;
156189 }
157190
158191 // Build a batch-check-mailbox input: one object hash per line.
@@ -174,8 +207,6 @@ async function aggregateVerdicts(
174207 // Use child_process.spawn for streaming instead of execFile (fits in memory for this size).
175208 const { spawn } = await import ( 'node:child_process' ) ;
176209
177- const verdictMap = new Map < string , PersonVerdict > ( ) ;
178-
179210 await new Promise < void > ( ( resolvePromise , reject ) => {
180211 const catFile = spawn ( 'git' , [ 'cat-file' , '--batch' ] , { cwd : repo } ) ;
181212
@@ -248,20 +279,25 @@ async function aggregateVerdicts(
248279 const pathParts = currentExpected . path . split ( '/' ) ;
249280 // path is like: person-evaluations/<personSlug>/<evaluator>.toml
250281 const personSlug = pathParts [ 1 ] ;
282+ const evaluator = ( pathParts [ 2 ] ?? '' ) . replace ( / \. t o m l $ / , '' ) ;
251283 if ( personSlug ) {
252- const { verdict, confidence } = parseEvaluationRecord ( tomlContent ) ;
253- if ( verdict !== null && confidence !== null ) {
254- let entry = verdictMap . get ( personSlug ) ;
255- if ( ! entry ) {
256- entry = { hasConfidentSpam : false , hasAnyLegit : false } ;
257- verdictMap . set ( personSlug , entry ) ;
258- }
259- if ( verdict === 'spam' && confidence >= threshold ) {
260- ( entry as { hasConfidentSpam : boolean } ) . hasConfidentSpam = true ;
261- }
262- if ( verdict === 'legit' ) {
263- ( entry as { hasAnyLegit : boolean } ) . hasAnyLegit = true ;
284+ const { verdict, confidence, evaluatedAt } = parseEvaluationRecord ( tomlContent ) ;
285+ let entry = verdictMap . get ( personSlug ) ;
286+ if ( ! entry ) {
287+ entry = { hasConfidentSpam : false , hasAnyLegit : false , humanVerdict : null , humanEvaluatedAt : null } ;
288+ verdictMap . set ( personSlug , entry ) ;
289+ }
290+ if ( evaluator . startsWith ( HUMAN_EVALUATOR_PREFIX ) ) {
291+ // Latest human vote wins; ties resolve to whichever is read last.
292+ if ( verdict !== null && ( entry . humanEvaluatedAt === null || ( evaluatedAt ?? '' ) >= entry . humanEvaluatedAt ) ) {
293+ entry . humanVerdict = verdict ;
294+ entry . humanEvaluatedAt = evaluatedAt ?? '' ;
264295 }
296+ } else if ( verdict !== null && confidence !== null ) {
297+ // Heuristic records carry `score`, not `confidence`, and never
298+ // reach here — they must be LLM-confirmed before they can prune.
299+ if ( verdict === 'spam' && confidence >= threshold ) entry . hasConfidentSpam = true ;
300+ if ( verdict === 'legit' ) entry . hasAnyLegit = true ;
265301 }
266302 }
267303
@@ -292,17 +328,30 @@ async function aggregateVerdicts(
292328}
293329
294330/**
295- * Compute the set of person slugs to prune:
296- * prune iff hasConfidentSpam AND NOT hasAnyLegit.
331+ * Compute the set of person slugs to prune. A human vote is final: `spam`
332+ * prunes, `legit` keeps. Otherwise prune iff hasConfidentSpam AND NOT
333+ * hasAnyLegit. Returns the human-spam subset too, since those bypass the
334+ * project-membership protection.
297335 */
298- function computePruneSet ( verdictMap : Map < string , PersonVerdict > ) : Set < string > {
336+ function computePruneSet ( verdictMap : Map < string , PersonVerdict > ) : {
337+ pruneSet : Set < string > ;
338+ humanSpam : Set < string > ;
339+ } {
299340 const pruneSet = new Set < string > ( ) ;
341+ const humanSpam = new Set < string > ( ) ;
300342 for ( const [ slug , v ] of verdictMap ) {
343+ if ( v . humanVerdict !== null ) {
344+ if ( v . humanVerdict === 'spam' ) {
345+ pruneSet . add ( slug ) ;
346+ humanSpam . add ( slug ) ;
347+ }
348+ continue ;
349+ }
301350 if ( v . hasConfidentSpam && ! v . hasAnyLegit ) {
302351 pruneSet . add ( slug ) ;
303352 }
304353 }
305- return pruneSet ;
354+ return { pruneSet, humanSpam } ;
306355}
307356
308357/** Minimal read surface shared by the live store and an open transaction. */
@@ -326,6 +375,7 @@ interface CandidatePerson {
326375async function partitionCandidates (
327376 q : Queryable ,
328377 candidateSlugs : Set < string > ,
378+ humanSpam : Set < string > = new Set ( ) ,
329379) : Promise < { prune : CandidatePerson [ ] ; protectedByMembership : number } > {
330380 const candidates : CandidatePerson [ ] = [ ] ;
331381 for await ( const person of q . people . query ( ) ) {
@@ -343,7 +393,9 @@ async function partitionCandidates(
343393 if ( typeof pid === 'string' ) memberPersonIds . add ( pid ) ;
344394 }
345395
346- const prune = candidates . filter ( ( c ) => ! memberPersonIds . has ( c . id ) ) ;
396+ // Membership protects against machine verdicts only — a human looked at the
397+ // profile, so a human `spam` vote is not overridden by real-looking content.
398+ const prune = candidates . filter ( ( c ) => humanSpam . has ( c . slug ) || ! memberPersonIds . has ( c . id ) ) ;
347399 return { prune, protectedByMembership : candidates . length - prune . length } ;
348400}
349401
@@ -406,17 +458,22 @@ async function pruneSpam(args: CliArgs): Promise<PruneSummary> {
406458 // -------------------------------------------------------------------------
407459 // 1. Read verdicts from evaluations ref (efficient git read)
408460 // -------------------------------------------------------------------------
409- log ( `[prune-spam] reading verdicts from ref=${ args . evaluationsRef } , threshold=${ args . threshold } ` ) ;
461+ log (
462+ `[prune-spam] reading machine verdicts from ${ args . evaluationsRepo } @${ args . evaluationsRef } , ` +
463+ `human votes from ${ args . dataRepo } @${ args . humanVotesRef } , threshold=${ args . threshold } ` ,
464+ ) ;
410465 const verdictMap = await aggregateVerdicts (
411- args . dataRepo ,
466+ args . evaluationsRepo ,
412467 args . evaluationsRef ,
413468 args . threshold ,
414469 log ,
415470 ) ;
471+ await aggregateVerdicts ( args . dataRepo , args . humanVotesRef , args . threshold , log , verdictMap ) ;
416472
417- const pruneSet = computePruneSet ( verdictMap ) ;
473+ const { pruneSet, humanSpam } = computePruneSet ( verdictMap ) ;
418474 log (
419- `[prune-spam] evaluated=${ verdictMap . size } persons, pruneSet=${ pruneSet . size } (confident spam with no legit)` ,
475+ `[prune-spam] evaluated=${ verdictMap . size } persons, pruneSet=${ pruneSet . size } ` +
476+ `(${ humanSpam . size } by human vote; the rest confident machine spam with no legit)` ,
420477 ) ;
421478
422479 // -------------------------------------------------------------------------
@@ -443,6 +500,7 @@ async function pruneSpam(args: CliArgs): Promise<PruneSummary> {
443500 const { prune, protectedByMembership } = await partitionCandidates (
444501 store as unknown as Queryable ,
445502 pruneSet ,
503+ humanSpam ,
446504 ) ;
447505 console . log (
448506 `[prune-spam] dry-run: would prune ${ prune . length } (of ${ pruneSet . size } verdict-flagged slugs); ${ protectedByMembership } protected by project membership` ,
@@ -492,6 +550,7 @@ async function pruneSpam(args: CliArgs): Promise<PruneSummary> {
492550 const { prune, protectedByMembership : protectedCount } = await partitionCandidates (
493551 tx as unknown as Queryable ,
494552 pruneSet ,
553+ humanSpam ,
495554 ) ;
496555 protectedByMembership = protectedCount ;
497556
0 commit comments