Merge pull request #203 from CodeForPhilly/feat/person-evaluations-contract

feat: person-evaluations as a versioned gitsheets schema contract

Author: themightychris

apps/api/gitsheets/contracts/codeforphilly.org/person-evaluations/v1.toml

@@ -0,0 +1,46 @@
+"$id" = "https://codeforphilly.org/person-evaluations/v1"
+"$schema" = "http://json-schema.org/draft-07/schema#"
+description = 'One spam verdict about one person from one evaluator. Machine evaluators (heuristic, LLM, github-probe) write these in the private codeforphilly-spam-detection repo; staff votes are written to codeforphilly-data by the site with evaluator = "human-<voterSlug>". Records are interchangeable at prune time.'
+required = ["evaluatedAt", "evaluator", "flags", "personSlug", "verdict"]
+title = "Person evaluation"
+type = "object"
+
+[properties.confidence]
+description = "0-1. LLM and probe evaluators; 1 on human votes; absent on heuristic records."
+maximum = 1
+minimum = 0
+type = "number"
+
+[properties.evaluatedAt]
+description = "ISO 8601 UTC."
+format = "date-time"
+type = "string"
+
+[properties.evaluator]
+description = "Model/run id (e.g. haiku-2026-05) or human-<voterSlug>."
+minLength = 1
+type = "string"
+
+[properties.flags]
+description = "Short reason tags."
+type = "array"
+
+[properties.flags.items]
+type = "string"
+
+[properties.personSlug]
+description = "Slug of the evaluated person."
+minLength = 1
+type = "string"
+
+[properties.reasoning]
+description = "Free-text justification. Optional on human votes."
+type = "string"
+
+[properties.score]
+description = "Heuristic evaluators only: rule points, unbounded."
+type = "integer"
+
+[properties.verdict]
+enum = ["spam", "legit", "uncertain"]
+type = "string"

apps/api/gitsheets/person-evaluations.toml

@@ -5,44 +5,8 @@ root = "person-evaluations"
 # One spam verdict per (person, evaluator). On this repo's branches the only
 # writer is the site: staff votes with evaluator = "human-<voterSlug>" (see
 # codeforphilly-ng specs/api/moderation.md). Machine evaluators (heuristic,
-# LLM) live in the private codeforphilly-spam-detection repo and share this
-# schema so records are interchangeable at prune time.
-[gitsheet.schema]
-required = ["evaluatedAt", "evaluator", "flags", "personSlug", "verdict"]
-type = "object"
-
-[gitsheet.schema.properties.personSlug]
-type = "string"
-minLength = 1
-
-[gitsheet.schema.properties.evaluator]
-type = "string"
-minLength = 1
-
-[gitsheet.schema.properties.verdict]
-type = "string"
-enum = ["spam", "legit", "uncertain"]
-
-# LLM evaluators only (scripts/evaluate-llm.ts ResultSchema: 0..1)
-[gitsheet.schema.properties.confidence]
-type = "number"
-minimum = 0
-maximum = 1
-
-# Heuristic evaluators only (scripts/lib/heuristic.ts rule points, unbounded)
-[gitsheet.schema.properties.score]
-type = "integer"
-
-[gitsheet.schema.properties.flags]
-type = "array"
-
-[gitsheet.schema.properties.flags.items]
-type = "string"
-
-# LLM evaluators only
-[gitsheet.schema.properties.reasoning]
-type = "string"
-
-[gitsheet.schema.properties.evaluatedAt]
-type = "string"
-format = "date-time"
+# LLM) live in the private codeforphilly-spam-detection repo. The record
+# shape is the versioned contract below, which both repos and the site open
+# the sheet against; change it by adopting a new version (…/v2), never by
+# editing v1.
+implements = ["codeforphilly.org/person-evaluations/v1"]

apps/api/scripts/check-sheet-configs.ts

@@ -1,6 +1,7 @@
 /**
- * Verify the vendored sheet configs in `apps/api/gitsheets/` match the data
- * repo's `empty` branch, the source of truth for `.gitsheets/*.toml`
+ * Verify the vendored sheet configs and schema contracts in
+ * `apps/api/gitsheets/` match the data repo's `empty` branch, the source of
+ * truth for everything under `.gitsheets/`
  * (specs/behaviors/storage.md → "Why these path shapes").
  *
  * Tests seed every test data repo from the vendored copy, so drift between
@@ -11,12 +12,13 @@
  * Usage:
  *   npm run -w apps/api check:sheet-configs [-- --remote=<url>] [-- --ref=<branch>]
  *
- * Exits 1 and prints each differing, missing, or extra config on drift.
+ * Exits 1 when a vendored config differs from upstream or is missing there.
+ * A config only upstream (a sheet the app doesn't open yet) is a warning.
  */
 import { execFile } from 'node:child_process';
 import { mkdtemp, readFile, readdir, rm } from 'node:fs/promises';
 import { tmpdir } from 'node:os';
-import { dirname, join } from 'node:path';
+import { dirname, join, relative } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { promisify } from 'node:util';
 
@@ -27,6 +29,17 @@ const DEFAULT_REMOTE = 'git@github.com:CodeForPhilly/codeforphilly-data.git';
 const DEFAULT_REF = 'empty';
 const VENDORED_DIR = join(dirname(fileURLToPath(import.meta.url)), '../gitsheets');
 
+/** Every `.toml` file under `dir`, keyed by its path relative to `dir`. */
+async function readTomlTree(dir: string): Promise<Map<string, string>> {
+  const out = new Map<string, string>();
+  for (const entry of await readdir(dir, { recursive: true, withFileTypes: true })) {
+    if (!entry.isFile() || !entry.name.endsWith('.toml')) continue;
+    const abs = join(entry.parentPath, entry.name);
+    out.set(relative(dir, abs), await readFile(abs, 'utf8'));
+  }
+  return out;
+}
+
 function arg(name: string, fallback: string): string {
   const prefix = `--${name}=`;
   return process.argv.find((a) => a.startsWith(prefix))?.slice(prefix.length) ?? fallback;
@@ -42,34 +55,40 @@ async function main(): Promise<number> {
     await exec('git', ['init', '--bare', '-q', gitDir]);
     await git('fetch', '-q', '--depth=1', remote, ref);
 
-    const { stdout: listing } = await git('ls-tree', '--name-only', 'FETCH_HEAD', '.gitsheets/');
+    const { stdout: listing } = await git('ls-tree', '-r', '--name-only', 'FETCH_HEAD', '.gitsheets/');
     const upstream = new Map<string, string>();
     for (const path of listing.split('\n').filter((p) => p.endsWith('.toml'))) {
       const { stdout } = await git('show', `FETCH_HEAD:${path}`);
       upstream.set(path.slice('.gitsheets/'.length), stdout);
     }
 
-    const vendored = new Map<string, string>();
-    for (const name of (await readdir(VENDORED_DIR)).filter((f) => f.endsWith('.toml'))) {
-      vendored.set(name, await readFile(join(VENDORED_DIR, name), 'utf8'));
-    }
+    const vendored = await readTomlTree(VENDORED_DIR);
 
+    // A sheet lands in the data repo before the app opens it, so a config
+    // only upstream is expected mid-change: warn. Anything the app vendors
+    // must match upstream exactly.
     const problems: string[] = [];
+    const upstreamOnly: string[] = [];
     for (const [name, contents] of upstream) {
       const ours = vendored.get(name);
-      if (ours === undefined) problems.push(`missing locally: ${name}`);
+      if (ours === undefined) upstreamOnly.push(name);
       else if (ours !== contents) problems.push(`differs: ${name}`);
     }
     for (const name of vendored.keys()) {
       if (!upstream.has(name)) problems.push(`not in ${ref}: ${name}`);
     }
 
+    for (const name of upstreamOnly) {
+      const msg = `${name} is in ${ref} but not vendored in apps/api/gitsheets/ (expected while a new sheet is landing)`;
+      console.warn(process.env.GITHUB_ACTIONS ? `::warning title=Sheet config not vendored::${msg}` : `warning: ${msg}`);
+    }
+
     if (problems.length > 0) {
       console.error(`Sheet configs in apps/api/gitsheets/ drift from ${remote}#${ref}:`);
       for (const p of problems) console.error(`  - ${p}`);
       return 1;
     }
-    console.log(`${vendored.size} sheet configs match ${remote}#${ref}.`);
+    console.log(`${vendored.size} files under .gitsheets/ match ${remote}#${ref}.`);
     return 0;
   } finally {
     await rm(gitDir, { recursive: true, force: true });

apps/api/scripts/scrub-data.ts

@@ -33,8 +33,9 @@
 
 import { createHash } from 'node:crypto';
 import { execFile } from 'node:child_process';
-import { mkdir, readdir, readFile, rm } from 'node:fs/promises';
-import { join, resolve } from 'node:path';
+import { mkdir, mkdtemp, readdir, readFile, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { dirname, join, resolve } from 'node:path';
 import { promisify } from 'node:util';
 
 import { faker } from '@faker-js/faker';
@@ -398,7 +399,10 @@ async function writeConfigRootCommit(
   targetPath: string,
   configs: ReadonlyArray<{ name: string; content: Buffer }>,
 ): Promise<string> {
-  const entries: string[] = [];
+  // A throwaway index builds the nested `.gitsheets/` tree (configs plus
+  // `contracts/<host>/<path>.toml`) without touching any real index.
+  const indexFile = join(await mkdtemp(join(tmpdir(), 'cfp-scrub-index-')), 'index');
+  const indexEnv = { ...process.env, GIT_INDEX_FILE: indexFile };
   for (const { name, content } of configs) {
     const hash = await new Promise<string>((resolvePromise, reject) => {
       const child = execFile(
@@ -409,24 +413,20 @@ async function writeConfigRootCommit(
       );
       child.stdin?.end(content);
     });
-    entries.push(`100644 blob ${hash}\t${name}`);
+    await exec('git', ['update-index', '--add', '--cacheinfo', `100644,${hash},.gitsheets/${name}`], {
+      cwd: targetPath,
+      env: indexEnv,
+    });
   }
-  const configTree = (await gitWithInput(targetPath, ['mktree'], entries.join('\n') + '\n')).trim();
-  const rootTree = (await gitWithInput(targetPath, ['mktree'], `040000 tree ${configTree}\t.gitsheets\n`)).trim();
+  const rootTree = (await exec('git', ['write-tree'], { cwd: targetPath, env: indexEnv })).stdout.trim();
+  await rm(dirname(indexFile), { recursive: true, force: true });
   const { stdout } = await exec('git', ['commit-tree', rootTree, '-m', 'scrub-data: sheet configs'], {
     cwd: targetPath,
     env: { ...process.env, ...gitIdentityEnv(new Date().toISOString()) },
   });
   return stdout.trim();
 }
 
-function gitWithInput(cwd: string, args: string[], input: string): Promise<string> {
-  return new Promise((resolvePromise, reject) => {
-    const child = execFile('git', args, { cwd }, (err, stdout) => (err ? reject(err) : resolvePromise(stdout)));
-    child.stdin?.end(input);
-  });
-}
-
 function gitIdentityEnv(timestamp: string): Record<string, string> {
   return {
     GIT_AUTHOR_NAME: SNAPSHOT_AUTHOR.name,
@@ -536,10 +536,18 @@ export async function scrubRepo(opts: ScrubOptions): Promise<ScrubResult> {
   const targetGitDir = (await git('rev-parse', '--absolute-git-dir')).stdout.trim();
   const targetRepo = await openRepo({ gitDir: targetGitDir });
 
+  // Everything under the source's `.gitsheets/`: sheet configs plus the
+  // vendored schema contracts (`contracts/…`) that configs may declare.
+  const sourceRef = sourceHeadHash ?? 'HEAD';
+  const configPaths = (
+    await exec('git', ['ls-tree', '-r', '--name-only', sourceRef, '.gitsheets/'], { cwd: source })
+  ).stdout
+    .split('\n')
+    .filter((p) => p.length > 0);
   const configs = await Promise.all(
-    Object.values(sheets).map(async (sheet) => ({
-      name: `${sheet.name}.toml`,
-      content: await streamToBuffer(await sourceRepo.readBlobStream(sourceHeadHash ?? 'HEAD', sheet.configPath)),
+    configPaths.map(async (path) => ({
+      name: path.slice('.gitsheets/'.length),
+      content: await streamToBuffer(await sourceRepo.readBlobStream(sourceRef, path)),
     })),
   );
   const configRoot = await writeConfigRootCommit(target, configs);

apps/api/src/store/public.ts

@@ -3,6 +3,7 @@ import { join } from 'node:path';
 
 import { openRepo, openStore } from 'gitsheets';
 import type { Repository, Store, StoreTx, ValidatorMap } from 'gitsheets';
+import { PERSON_EVALUATIONS_CONTRACT_V1 } from '@cfp/shared';
 import {
   BlogPostSchema,
   HelpWantedInterestExpressionSchema,
@@ -83,5 +84,15 @@ export async function openPublicStore(
   repo.requireExplicitTransactions();
 
   const store = await openStore(repo, { validators: PUBLIC_SHEET_SCHEMAS });
+
+  // person-evaluations is shared with the private spam pipeline through a
+  // versioned schema contract (specs/behaviors/spam-exclusion.md). Refuse to
+  // boot against a data repo whose sheet doesn't satisfy it: gitsheets
+  // checks the declared contract's identity first, falling back to
+  // validating every record.
+  await repo.openSheet('person-evaluations', {
+    contract: { schema: PERSON_EVALUATIONS_CONTRACT_V1 },
+  });
+
   return { store, repo };
 }

apps/api/tests/helpers/sheet-configs.ts

@@ -1,11 +1,11 @@
 /**
- * The canonical `.gitsheets/*.toml` sheet configs, vendored from the data
- * repo's `empty` branch into `apps/api/gitsheets/`. Every test data repo is
+ * The canonical `.gitsheets/` sheet configs and schema contracts, vendored
+ * from the data repo's `empty` branch into `apps/api/gitsheets/`. Every test data repo is
  * seeded from these so tests run against the same path templates production
  * does. `npm run -w apps/api check:sheet-configs` verifies the vendored copy
  * still matches the data repo.
  */
-import { mkdir, readFile, readdir, writeFile } from 'node:fs/promises';
+import { cp, mkdir, readFile, readdir, writeFile } from 'node:fs/promises';
 import { dirname, join } from 'node:path';
 import { fileURLToPath } from 'node:url';
 
@@ -25,10 +25,16 @@ export async function loadSheetConfigs(
   return Object.fromEntries(entries);
 }
 
-/** Write the canonical configs into `<dir>/.gitsheets/`. */
+/**
+ * Write the canonical configs into `<dir>/.gitsheets/`, plus the vendored
+ * schema contracts (`contracts/`) that sheet configs may declare.
+ */
 export async function writeSheetConfigs(dir: string, only?: readonly string[]): Promise<void> {
   await mkdir(join(dir, '.gitsheets'), { recursive: true });
   for (const [name, contents] of Object.entries(await loadSheetConfigs(only))) {
     await writeFile(join(dir, '.gitsheets', `${name}.toml`), contents);
   }
+  await cp(join(SHEET_CONFIG_DIR, 'contracts'), join(dir, '.gitsheets', 'contracts'), {
+    recursive: true,
+  });
 }

apps/api/tests/sheet-configs.test.ts

@@ -11,9 +11,12 @@
 import { readFile, readdir } from 'node:fs/promises';
 import { join } from 'node:path';
 
+import { canonicalContractHash } from 'gitsheets';
 import { describe, expect, it } from 'vitest';
 import type { z } from 'zod';
 
+import { PERSON_EVALUATIONS_CONTRACT_NAME, PERSON_EVALUATIONS_CONTRACT_V1 } from '@cfp/shared';
+
 import { PUBLIC_SHEET_SCHEMAS } from '../src/store/public.js';
 import { SHEET_CONFIG_DIR } from './helpers/sheet-configs.js';
 
@@ -64,4 +67,17 @@ describe('sheet configs', () => {
       expect(missing, `template fields ${sheet}'s records may lack`).toEqual([]);
     });
   }
+
+  it('vendors the person-evaluations contract byte-identical to the shared document', async () => {
+    const vendored = await readFile(
+      join(SHEET_CONFIG_DIR, 'contracts', `${PERSON_EVALUATIONS_CONTRACT_NAME}.toml`),
+      'utf8',
+    );
+    expect(canonicalContractHash(vendored, { format: 'toml' })).toBe(
+      canonicalContractHash(PERSON_EVALUATIONS_CONTRACT_V1),
+    );
+    const config = await readFile(join(SHEET_CONFIG_DIR, 'person-evaluations.toml'), 'utf8');
+    expect(config).toContain(`implements = ["${PERSON_EVALUATIONS_CONTRACT_NAME}"]`);
+  });
 });
+