Skip to content

Commit 15da5ac

Browse files
committed
wip(scenarios): gallery exporter tests with a real crm run and doubles
Co-Authored-By: Claude <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_017ihL2ebbZkr2QAEG1x7UC8
1 parent 639677d commit 15da5ac

3 files changed

Lines changed: 800 additions & 0 deletions

File tree

Lines changed: 358 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,358 @@
1+
// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license.
2+
3+
/**
4+
* The gallery exporter on a real run of the real crm card (`fixture.ts`): the
5+
* layout it writes, the pass rule it reads from the record, the generic-only
6+
* guard, re-export replacing what was there, an app that stops passing
7+
* leaving the gallery, the transcript joined with (and proven against) the
8+
* recording, and `index.json` validated and stable.
9+
*/
10+
11+
import { createHash } from 'node:crypto';
12+
import { existsSync, mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';
13+
import { join, relative } from 'node:path';
14+
15+
import { describe, expect, it } from 'vitest';
16+
17+
import { exportGallery, scratchProjectDir, type ExportOptions } from '../../src/export/exporter.js';
18+
import { NO_GENERIC_MARKER, type GenericGuard } from '../../src/export/generic.js';
19+
import { GalleryIndexSchema, parseIndex } from '../../src/export/manifest.js';
20+
import { judgeRecord } from '../../src/export/pass.js';
21+
import { readRunRecords, ScenarioRunRecordSchema, type ScenarioRunRecord } from '../../src/export/record.js';
22+
import type { CaptureScreenshots } from '../../src/export/screenshots.js';
23+
import { main, parseExportArgs } from '../../src/export/cli.js';
24+
import type { Recording } from '../../src/runner/driver.js';
25+
import { CARD_ID, loadAllCards, loadCrm, runCrm, tempDir, writeRecord, type FixtureRun } from './fixture.js';
26+
27+
/** The test's stand-in for the generic marker the card schema does not have yet. */
28+
const GENERIC_DOUBLE: GenericGuard = () => ({ generic: true, reason: 'test double' });
29+
30+
const asRecord = (r: Record<string, unknown>) => ScenarioRunRecordSchema.parse(r) as ScenarioRunRecord;
31+
32+
/** Every file under `dir`, relative path → sha256. */
33+
function tree(dir: string): Record<string, string> {
34+
const out: Record<string, string> = {};
35+
const walk = (d: string) => {
36+
for (const e of readdirSync(d, { withFileTypes: true })) {
37+
const p = join(d, e.name);
38+
if (e.isDirectory()) walk(p);
39+
else out[relative(dir, p)] = createHash('sha256').update(readFileSync(p)).digest('hex');
40+
}
41+
};
42+
walk(dir);
43+
return out;
44+
}
45+
46+
async function setup(run?: FixtureRun) {
47+
const fx = run ?? (await runCrm());
48+
const runs = tempDir('runs');
49+
const out = tempDir('out');
50+
writeRecord(runs, fx.record);
51+
const opts = (over: Partial<ExportOptions> = {}): ExportOptions => ({
52+
runsDir: runs,
53+
outDir: out,
54+
cards: [fx.scenario],
55+
recordings: [fx.recording],
56+
isGeneric: GENERIC_DOUBLE,
57+
screenshots: false,
58+
...over,
59+
});
60+
return { fx, runs, out, opts };
61+
}
62+
63+
describe('judgeRecord — "passed every half", read from the record', () => {
64+
it('a clean run passes: all six scratch halves and the three template iterate halves', async () => {
65+
const { record } = await runCrm();
66+
const j = judgeRecord(asRecord(record));
67+
expect(j.reasons).toEqual([]);
68+
expect(j.passed).toBe(true);
69+
expect(j.halves.map((h) => h.id)).toEqual([
70+
`${CARD_ID}.scratch.build`,
71+
`${CARD_ID}.scratch.run-ask`,
72+
`${CARD_ID}.scratch.run-operations`,
73+
`${CARD_ID}.scratch.iterate-build`,
74+
`${CARD_ID}.scratch.iterate-run-ask`,
75+
`${CARD_ID}.scratch.iterate-run-operations`,
76+
`${CARD_ID}.template.iterate-build`,
77+
`${CARD_ID}.template.iterate-run-ask`,
78+
`${CARD_ID}.template.iterate-run-operations`,
79+
]);
80+
// Template mode's first rows are not-run by construction and are not asked.
81+
const rows = (record.results as Array<{ id: string; verdict: string }>).filter((r) => r.id === `${CARD_ID}.template.build`);
82+
expect(rows[0]!.verdict).toBe('not-run');
83+
});
84+
85+
it('a not-run half is not a pass: a run without the iterate re-run fails', async () => {
86+
const { record } = await runCrm({ iterateRun: false, modes: ['scratch'] });
87+
const j = judgeRecord(asRecord(record));
88+
expect(j.passed).toBe(false);
89+
expect(j.reasons).toEqual([`${CARD_ID}.scratch.iterate-run-ask: not-run`, `${CARD_ID}.scratch.iterate-run-operations: not-run`]);
90+
});
91+
92+
it('partial, fail and a missing row are not a pass; a record without the scratch mode is refused', async () => {
93+
const { record } = await runCrm();
94+
const results = (record.results as Array<{ id: string; verdict: string }>).map((r) =>
95+
r.id.endsWith('scratch.run-ask') ? { ...r, verdict: 'partial' } : r.id.endsWith('template.iterate-build') ? { ...r, verdict: 'fail' } : r,
96+
);
97+
const j = judgeRecord(asRecord({ ...record, results: results.filter((r) => !r.id.endsWith('scratch.run-operations')) }));
98+
expect(j.reasons).toEqual([`${CARD_ID}.scratch.run-ask: partial`, `${CARD_ID}.scratch.run-operations: missing`, `${CARD_ID}.template.iterate-build: fail`]);
99+
100+
const cardRun = record.cardRun as { modes: Array<{ mode: string }> };
101+
const templateOnly = { ...record, cardRun: { ...cardRun, modes: cardRun.modes.filter((m) => m.mode === 'template') } };
102+
expect(judgeRecord(asRecord(templateOnly)).reasons[0]).toMatch(/no scratch mode/);
103+
});
104+
});
105+
106+
describe('exportGallery — one passing, generic app', () => {
107+
it('writes the apps/<id>/ layout: source, run record, scorecard, transcript, README, and the index', async () => {
108+
const { out, fx, opts } = await setup();
109+
const s = await exportGallery(opts());
110+
expect(s.refused).toEqual([]);
111+
expect(s.exported.map((e) => e.id)).toEqual([CARD_ID]);
112+
expect(s.indexChanged).toBe(true);
113+
114+
const files = Object.keys(tree(join(out, 'apps', CARD_ID))).sort();
115+
expect(files).toEqual([
116+
'README.md',
117+
'run-record.json',
118+
'scorecard.json',
119+
'scorecard.md',
120+
'source/objectstack.config.ts',
121+
'source/objectstack.manifest.json',
122+
'source/package.json',
123+
'source/src/objects/account.object.ts',
124+
'source/src/objects/contact.object.ts',
125+
'source/tsconfig.json',
126+
'transcript.json',
127+
'transcript.md',
128+
]);
129+
130+
// The run record, verbatim but for the local work directory.
131+
const recordText = readFileSync(join(out, 'apps', CARD_ID, 'run-record.json'), 'utf8');
132+
expect(recordText).not.toContain(fx.work);
133+
expect(recordText).toContain('$WORK');
134+
expect(JSON.parse(recordText).results).toEqual(fx.record.results);
135+
136+
const index = parseIndex(readFileSync(join(out, 'index.json'), 'utf8'));
137+
expect(index.apps).toEqual([
138+
{
139+
id: CARD_ID,
140+
code: 'crm',
141+
namespace: 'democrm',
142+
path: `apps/${CARD_ID}`,
143+
function: 'crm',
144+
title: { en: fx.scenario.card.title, zh: null },
145+
summary: { ...(fx.scenario.card.prompt.short.en ? { en: fx.scenario.card.prompt.short.en } : {}), ...(fx.scenario.card.prompt.short.zh ? { zh: fx.scenario.card.prompt.short.zh } : {}) },
146+
capabilities: [...fx.scenario.card.capabilities].sort(),
147+
screenshots: [],
148+
qa: { questions: 8, turns: 9, personas: 2 },
149+
operations: 5,
150+
framework: { version: '18.3.0', runner: 'abc1234' },
151+
runDate: '2026-11-06',
152+
run: `2026-11-06-scenario-${CARD_ID}`,
153+
},
154+
]);
155+
});
156+
157+
it('the transcript carries every persona answer with its grade, as JSON and as Markdown', async () => {
158+
const { out, opts } = await setup();
159+
await exportGallery(opts());
160+
const t = JSON.parse(readFileSync(join(out, 'apps', CARD_ID, 'transcript.json'), 'utf8'));
161+
expect(t.halves.map((h: { mode: string; phase: string }) => `${h.mode}/${h.phase}`)).toEqual(['scratch/run', 'scratch/iterate-run', 'template/iterate-run']);
162+
const first = t.halves[0].turns;
163+
expect(first).toHaveLength(9); // 8 questions, one of them asked as both personas
164+
for (const turn of first) {
165+
expect(turn.answer.replyText.length).toBeGreaterThan(0);
166+
expect(turn.grade).toMatchObject({ correct: true, cited: true, leaks: [] });
167+
}
168+
// Asked as the admin where a question names no persona, and as each persona it names.
169+
const askers = new Set(first.map((x: { persona: string | null }) => x.persona ?? 'admin'));
170+
expect([...askers].sort()).toEqual(['admin', 'partner_user', 'sdr']);
171+
const md = readFileSync(join(out, 'apps', CARD_ID, 'transcript.md'), 'utf8');
172+
expect(md).toContain('#### As sdr');
173+
expect(md).toContain('#### As partner_user');
174+
expect(md).toContain('✅ pass');
175+
});
176+
177+
it('a re-export with no change is a no-op: the same bytes, file for file, and the index is not rewritten', async () => {
178+
const { out, opts } = await setup();
179+
await exportGallery(opts());
180+
const first = tree(out);
181+
const again = await exportGallery(opts());
182+
expect(again.indexChanged).toBe(false);
183+
expect(tree(out)).toEqual(first);
184+
});
185+
186+
it('a re-export deletes the old directory first: nothing stale survives', async () => {
187+
const { out, opts } = await setup();
188+
await exportGallery(opts());
189+
writeFileSync(join(out, 'apps', CARD_ID, 'stale.txt'), 'left from an old release');
190+
mkdirSync(join(out, 'apps', CARD_ID, 'screenshots'), { recursive: true });
191+
writeFileSync(join(out, 'apps', CARD_ID, 'screenshots', 'old.png'), 'x');
192+
await exportGallery(opts());
193+
expect(existsSync(join(out, 'apps', CARD_ID, 'stale.txt'))).toBe(false);
194+
expect(existsSync(join(out, 'apps', CARD_ID, 'screenshots'))).toBe(false);
195+
expect(readdirSync(join(out, 'apps')).filter((n) => n.startsWith('.staging'))).toEqual([]);
196+
});
197+
198+
it('an app whose newest record no longer passes leaves apps/ and the index; a directory the index never listed is left alone', async () => {
199+
const { out, runs, opts } = await setup();
200+
await exportGallery(opts());
201+
mkdirSync(join(out, 'apps', 'hand_made'));
202+
// The next release's run: the iterate re-run was skipped, so two halves are not-run.
203+
const later = await runCrm({ date: '2026-12-01', iterateRun: false });
204+
writeRecord(runs, later.record);
205+
const s = await exportGallery(opts({ recordings: [later.recording] }));
206+
expect(s.exported).toEqual([]);
207+
expect(s.removed).toEqual([{ id: CARD_ID, why: expect.stringContaining(`${CARD_ID}.scratch.iterate-run-ask: not-run`) }]);
208+
expect(existsSync(join(out, 'apps', CARD_ID))).toBe(false);
209+
expect(parseIndex(readFileSync(join(out, 'index.json'), 'utf8')).apps).toEqual([]);
210+
expect(s.unowned).toEqual(['hand_made']);
211+
expect(existsSync(join(out, 'apps', 'hand_made'))).toBe(true);
212+
});
213+
214+
it('the newest record decides, even when an older one passed', async () => {
215+
const { runs } = await setup();
216+
const later = await runCrm({ date: '2026-12-01', iterateRun: false });
217+
writeRecord(runs, later.record);
218+
const { newest } = readRunRecords(runs);
219+
expect(newest.get(CARD_ID)!.record.date).toBe('2026-12-01');
220+
});
221+
222+
it('refuses, and touches nothing, when the previous index.json is not one it wrote', async () => {
223+
const { out, opts } = await setup();
224+
await exportGallery(opts());
225+
writeFileSync(join(out, 'index.json'), JSON.stringify({ apps: 'hand edited' }));
226+
const before = tree(out);
227+
await expect(exportGallery(opts())).rejects.toThrow(/not a gallery index this exporter wrote/);
228+
expect(tree(out)).toEqual(before);
229+
});
230+
231+
it('refuses the whole export on an unreadable scenario-runner record, and ignores other JSON', async () => {
232+
const { runs, opts } = await setup();
233+
writeFileSync(join(runs, 'checklist-run.json'), JSON.stringify({ runner: 'platform-checklist', results: [] }));
234+
const s = await exportGallery(opts());
235+
expect(s.ignoredFiles.map((f) => f.slice(f.lastIndexOf('/') + 1))).toEqual(['checklist-run.json']);
236+
writeFileSync(join(runs, '2026-11-07-scenario-x.json'), JSON.stringify({ runner: 'scenario-runner', app: 'x' }));
237+
await expect(exportGallery(opts())).rejects.toThrow(/not a readable scenario run record/);
238+
});
239+
});
240+
241+
describe('exportGallery — what it refuses', () => {
242+
it('⛔ generic only: with the default guard every card is refused and nothing is written but an empty index', async () => {
243+
const { out, opts } = await setup();
244+
const s = await exportGallery({ ...opts(), isGeneric: undefined });
245+
expect(s.exported).toEqual([]);
246+
expect(s.refused).toEqual([{ id: CARD_ID, reasons: [expect.stringMatching(/^not marked generic: the card schema carries no explicit generic marker/)] }]);
247+
expect(readdirSync(join(out, 'apps'))).toEqual([]);
248+
expect(GalleryIndexSchema.parse(JSON.parse(readFileSync(join(out, 'index.json'), 'utf8'))).apps).toEqual([]);
249+
});
250+
251+
it('the default guard refuses every card of the open library: none carries a generic marker', () => {
252+
const cards = loadAllCards();
253+
expect(cards.length).toBeGreaterThanOrEqual(9);
254+
for (const c of cards) expect(NO_GENERIC_MARKER(c)).toMatchObject({ generic: false });
255+
});
256+
257+
it('an asked turn with no recorded reply is refused: a grade is never shown without its answer', async () => {
258+
const { out, opts } = await setup();
259+
const s = await exportGallery(opts({ recordings: [] }));
260+
expect(s.refused[0]!.reasons[0]).toMatch(/^no recorded reply for `crm_account_with_contacts\/scratch\/run\//);
261+
expect(existsSync(join(out, 'apps', CARD_ID))).toBe(false);
262+
});
263+
264+
it("a recording that is not this run's is refused: its replies grade differently from the record", async () => {
265+
const { fx, opts } = await setup();
266+
const turns = { ...fx.recording.turns };
267+
const key = Object.keys(turns).find((k) => k.startsWith(`${CARD_ID}/scratch/run/`) && !k.includes('/author/'))!;
268+
turns[key] = { ...turns[key]!, replyText: 'No idea.' };
269+
const other: Recording = { ...fx.recording, turns };
270+
const s = await exportGallery(opts({ recordings: [other] }));
271+
expect(s.refused[0]!.reasons[0]).toBe(`the recorded reply for \`${key}\` grades differently from the run record: the recording is not this run's`);
272+
});
273+
274+
it("the run's project gone, or a card no longer loaded, is a refusal", async () => {
275+
const { fx, opts } = await setup();
276+
expect((await exportGallery(opts({ cards: [] }))).refused[0]!.reasons[0]).toMatch(/^no loaded card `crm_account_with_contacts`/);
277+
const elsewhere = tempDir('moved');
278+
expect((await exportGallery(opts({ workDir: elsewhere }))).refused[0]!.reasons[0]).toBe(`the run's scratch project is gone: ${scratchProjectDir(elsewhere, fx.scenario.card)}`);
279+
});
280+
});
281+
282+
describe('exportGallery — screenshots', () => {
283+
it('runs the screenshot step against the scratch project and lists every shot in the index', async () => {
284+
const { out, fx, opts } = await setup();
285+
const seen: string[] = [];
286+
const shoot: CaptureScreenshots = async (input) => {
287+
seen.push(input.projectDir);
288+
mkdirSync(join(input.dir, 'screenshots'), { recursive: true });
289+
writeFileSync(join(input.dir, 'screenshots', 'list.png'), 'png');
290+
return { shots: [{ kind: 'list', file: 'screenshots/list.png', path: '/_console/apps/democrm_crm/democrm_account', object: 'democrm_account' }], skipped: [{ kind: 'dashboard', reason: 'the build declares no dashboard' }] };
291+
};
292+
const s = await exportGallery(opts({ screenshots: shoot }));
293+
expect(seen).toEqual([scratchProjectDir(fx.work, fx.scenario.card)]);
294+
expect(s.exported[0]!.skippedShots).toEqual([{ kind: 'dashboard', reason: 'the build declares no dashboard' }]);
295+
expect(statSync(join(out, 'apps', CARD_ID, 'screenshots', 'list.png')).isFile()).toBe(true);
296+
const index = parseIndex(readFileSync(join(out, 'index.json'), 'utf8'));
297+
expect(index.apps[0]!.screenshots).toEqual([{ kind: 'list', path: `apps/${CARD_ID}/screenshots/list.png`, object: 'democrm_account' }]);
298+
expect(readFileSync(join(out, 'apps', CARD_ID, 'README.md'), 'utf8')).toContain('no dashboard screenshot: the build declares no dashboard');
299+
});
300+
301+
it('a screenshot step that fails refuses the app instead of exporting it without its proof', async () => {
302+
const { out, opts } = await setup();
303+
const s = await exportGallery(opts({ screenshots: async () => Promise.reject(new Error('objectstack dev exited before it was ready')) }));
304+
expect(s.refused).toEqual([{ id: CARD_ID, reasons: ['objectstack dev exited before it was ready'] }]);
305+
expect(existsSync(join(out, 'apps', CARD_ID))).toBe(false);
306+
});
307+
});
308+
309+
describe('index.json — the schema', () => {
310+
const app = (id: string) => ({
311+
id,
312+
code: 'crm',
313+
namespace: 'democrm',
314+
path: `apps/${id}`,
315+
function: 'crm',
316+
title: { en: 'CRM', zh: null },
317+
summary: { en: 'Build a CRM.' },
318+
capabilities: [],
319+
screenshots: [],
320+
qa: { questions: 8, turns: 9, personas: 2 },
321+
operations: 5,
322+
framework: { version: '18.3.0', runner: 'abc1234' },
323+
runDate: '2026-11-06',
324+
run: 'r',
325+
});
326+
it('accepts a sorted index and refuses an unsorted one, a duplicate, a wrong path and an unknown key', () => {
327+
expect(GalleryIndexSchema.safeParse({ schemaVersion: 1, apps: [app('a_card'), app('b_card')] }).success).toBe(true);
328+
expect(GalleryIndexSchema.safeParse({ schemaVersion: 1, apps: [app('b_card'), app('a_card')] }).success).toBe(false);
329+
expect(GalleryIndexSchema.safeParse({ schemaVersion: 1, apps: [app('a_card'), app('a_card')] }).success).toBe(false);
330+
expect(GalleryIndexSchema.safeParse({ schemaVersion: 1, apps: [{ ...app('a_card'), path: 'apps/other' }] }).success).toBe(false);
331+
expect(GalleryIndexSchema.safeParse({ schemaVersion: 1, apps: [{ ...app('a_card'), generatedAt: 'now' }] }).success).toBe(false);
332+
});
333+
});
334+
335+
describe('export-gallery CLI', () => {
336+
it('requires --out and refuses an unknown flag', () => {
337+
expect(() => parseExportArgs(['--runs', 'r'])).toThrow(/--out DIR is required/);
338+
expect(() => parseExportArgs(['--out', 'o', '--generic'])).toThrow(/unknown flag `--generic`/);
339+
expect(parseExportArgs(['--out', 'o', '--recording', 'a', '--recording', 'b', '--no-screenshots'])).toMatchObject({ out: 'o', recordings: ['a', 'b'], screenshots: false });
340+
});
341+
342+
it('end to end over the real card directory: today it refuses every card, so the gallery stays empty', async () => {
343+
const { fx, runs, out } = await setup();
344+
const recording = join(tempDir('rec'), 'recording.json');
345+
writeFileSync(recording, JSON.stringify(fx.recording));
346+
const errors: string[] = [];
347+
const log = console.error;
348+
console.error = (...a: unknown[]) => void errors.push(a.join(' '));
349+
try {
350+
expect(await main(['--runs', runs, '--out', out, '--recording', recording, '--no-screenshots'])).toBe(0);
351+
} finally {
352+
console.error = log;
353+
}
354+
expect(errors.some((l) => l.startsWith(`refused ${CARD_ID} · not marked generic`))).toBe(true);
355+
expect(parseIndex(readFileSync(join(out, 'index.json'), 'utf8')).apps).toEqual([]);
356+
expect(loadCrm().card.id).toBe(CARD_ID);
357+
});
358+
});

0 commit comments

Comments
 (0)