feat(qa/scenarios): refuse to export an app whose record has a not-measured turn

The scorecard counts a turn that rests on a refused seed row in neither
passed nor total, so its half can still read `pass`. The exporter now
reads every Run half's questions and operations and refuses the app
when any of them is marked not measured, naming each such turn. A
previous export of that app is removed like any other no-longer-passing
app.

Pinned by tests: a refused seed row (crm contact ct4) keeps every half
verdict at `pass` and still fails judgeRecord; an operation marked not
measured is its own reason; exportGallery refuses the app and removes
its earlier export.

Co-Authored-By: Claude <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017ihL2ebbZkr2QAEG1x7UC8

Author: claude

packages/qa/scenarios/src/export/pass.ts

@@ -19,6 +19,12 @@
  *     first Build and Run rows are `not-run` by construction (the mode installs
  *     a verified build and builds nothing first), so they are not asked. A
  *     template mode that was skipped holds `not-run` in every row and fails.
+ *   - No turn may be NOT MEASURED, in any Run half the record holds (the
+ *     first Run and the iterate re-run, both modes), questions and operations
+ *     alike. The scorecard leaves a not-measured turn (one that reads a seed
+ *     row the app refused) out of both passed and total, so its half can still
+ *     read `pass`; the gallery is trust-first, and a turn nobody measured is a
+ *     claim nobody proved. Each one is a reason of its own.
  */
 
 import type { ScenarioRunRecord, Verdict } from './record.js';
@@ -40,7 +46,7 @@ export interface PassJudgement {
   passed: boolean;
   /** Every owed half with its verdict, in order. */
   halves: HalfVerdict[];
-  /** Why it did not pass, one line per reason; empty when it passed. */
+  /** Why it did not pass, one line per reason (a failing half, or a not-measured turn); empty when it passed. */
   reasons: string[];
 }
 
@@ -59,5 +65,19 @@ export function judgeRecord(record: ScenarioRunRecord): PassJudgement {
       if (verdict !== 'pass') reasons.push(`${id}: ${verdict}`);
     }
   }
+  for (const m of record.cardRun.modes) {
+    const runs = [
+      ['run', m.run],
+      ['iterate-run', m.iterate?.run],
+    ] as const;
+    for (const [phase, half] of runs) {
+      for (const q of half?.questions ?? []) {
+        if (q.notMeasured) reasons.push(`${card}.${m.mode}.${phase} question ${q.question}${q.persona ? ` as ${q.persona}` : ''}: not measured — ${q.notMeasured}`);
+      }
+      for (const o of half?.operations ?? []) {
+        if (o.notMeasured) reasons.push(`${card}.${m.mode}.${phase} operation ${o.operation} as ${o.persona}: not measured — ${o.notMeasured}`);
+      }
+    }
+  }
   return { passed: reasons.length === 0, halves, reasons };
 }

packages/qa/scenarios/src/export/record.ts

@@ -31,7 +31,13 @@ const QuestionResultSchema = z.looseObject({
   error: z.string().optional(),
 });
 
-const RunHalfSchema = z.looseObject({ questions: z.array(QuestionResultSchema) });
+const OperationResultSchema = z.looseObject({
+  operation: z.string().min(1),
+  persona: z.string().min(1),
+  notMeasured: z.string().optional(),
+});
+
+const RunHalfSchema = z.looseObject({ questions: z.array(QuestionResultSchema), operations: z.array(OperationResultSchema) });
 
 const ModeRunSchema = z.looseObject({
   mode: z.enum(['scratch', 'template']),

packages/qa/scenarios/test/export/exporter.test.ts

@@ -103,6 +103,35 @@ describe('judgeRecord — "passed every half", read from the record', () => {
   });
 });
 
+describe('judgeRecord — ⛔ a not-measured turn is not a pass, even under a `pass` verdict', () => {
+  it('a refused seed row leaves every half `pass` in the record, and the record still does not pass', async () => {
+    // ct4 is a contact: every question on contacts, and every operation that names it, is not measured.
+    const { record } = await runCrm({ refuse: ['democrm_contact/ct4'] });
+    const verdicts = (record.results as Array<{ id: string; verdict: string }>).filter((r) => /scratch\.(run-ask|run-operations)$/.test(r.id)).map((r) => r.verdict);
+    expect(verdicts).toEqual(['pass', 'pass']); // the scorecard counts not-measured turns in neither passed nor total
+    const j = judgeRecord(asRecord(record));
+    expect(j.passed).toBe(false);
+    expect(j.halves.every((h) => h.verdict === 'pass')).toBe(true);
+    expect(j.reasons.length).toBeGreaterThan(0);
+    for (const r of j.reasons) expect(r).toMatch(/^crm_account_with_contacts\.(scratch|template)\.(run|iterate-run) (question|operation) \S+.*: not measured — seed: /);
+    // Every Run half the record holds is read: scratch first run, scratch re-run, template re-run.
+    expect(new Set(j.reasons.map((r) => r.split(' ')[0]))).toEqual(
+      new Set([`${CARD_ID}.scratch.run`, `${CARD_ID}.scratch.iterate-run`, `${CARD_ID}.template.iterate-run`]),
+    );
+  });
+
+  it('an operation marked not measured is a reason of its own', async () => {
+    const { record } = await runCrm();
+    const cardRun = record.cardRun as { modes: Array<{ mode: string; run: { operations: Array<Record<string, unknown>> } }> };
+    const scratch = cardRun.modes.find((m) => m.mode === 'scratch')!;
+    const [first, ...rest] = scratch.run.operations;
+    const edited = { ...record, cardRun: { ...cardRun, modes: cardRun.modes.map((m) => (m === scratch ? { ...m, run: { ...m.run, operations: [{ ...first, notMeasured: 'seed: needs democrm_account ac1, which the app refused' }, ...rest] } } : m)) } };
+    expect(judgeRecord(asRecord(edited)).reasons).toEqual([
+      `${CARD_ID}.scratch.run operation ${String(first!.operation)} as ${String(first!.persona)}: not measured — seed: needs democrm_account ac1, which the app refused`,
+    ]);
+  });
+});
+
 describe('exportGallery — one passing, generic app', () => {
   it('writes the apps/<id>/ layout: source, run record, scorecard, transcript, README, and the index', async () => {
     const { out, fx, opts } = await setup();
@@ -211,6 +240,20 @@ describe('exportGallery — one passing, generic app', () => {
     expect(existsSync(join(out, 'apps', 'hand_made'))).toBe(true);
   });
 
+  it('an app whose newest record has a not-measured turn is refused, and a previous export of it removed', async () => {
+    const { out, runs, opts } = await setup();
+    await exportGallery(opts());
+    const later = await runCrm({ date: '2026-12-01', refuse: ['democrm_contact/ct4'] });
+    writeRecord(runs, later.record);
+    const s = await exportGallery(opts({ recordings: [later.recording] }));
+    expect(s.exported).toEqual([]);
+    expect(s.refused[0]!.reasons[0]).toMatch(/did not pass every half/);
+    expect(s.refused[0]!.reasons.slice(1).every((r) => r.includes(': not measured — seed: '))).toBe(true);
+    expect(s.removed).toEqual([{ id: CARD_ID, why: expect.stringContaining('not measured') }]);
+    expect(existsSync(join(out, 'apps', CARD_ID))).toBe(false);
+    expect(parseIndex(readFileSync(join(out, 'index.json'), 'utf8')).apps).toEqual([]);
+  });
+
   it('the newest record decides, even when an older one passed', async () => {
     const { runs } = await setup();
     const later = await runCrm({ date: '2026-12-01', iterateRun: false });

packages/qa/scenarios/test/export/fixture.ts

@@ -236,15 +236,22 @@ export interface FixtureRun {
   recording: Recording;
 }
 
-/** One run of the crm card, both modes; `iterateRun: false` leaves the iterate Run halves not run. */
-export async function runCrm(opts: { work?: string; date?: string; iterateRun?: boolean; modes?: Array<'scratch' | 'template'> } = {}): Promise<FixtureRun> {
+/** One run of the crm card, both modes; `iterateRun: false` leaves the iterate Run halves not run; `refuse` makes every booted app refuse those seed rows. */
+export async function runCrm(
+  opts: { work?: string; date?: string; iterateRun?: boolean; modes?: Array<'scratch' | 'template'>; /** Seed rows every booted app refuses, `object/id`. */ refuse?: string[] } = {},
+): Promise<FixtureRun> {
   const scenario = loadCrm();
   const work = opts.work ?? tempDir('work');
-  let current = new FakeApp();
+  const fresh = () => {
+    const app = new FakeApp();
+    for (const r of opts.refuse ?? []) app.refuse.add(r);
+    return app;
+  };
+  let current = fresh();
   const ops = fakeOps(work, () => current);
   const boot = ops.boot.bind(ops);
   ops.boot = async (dir) => {
-    current = new FakeApp();
+    current = fresh();
     return boot(dir);
   };
   const driver = new RecordingDriver(new ScriptedDriver(scenario, ops, () => current));