Skip to content

Commit 6a703ca

Browse files
committed
wip(objectql): stored-expression backfill tests
Claude-Session: https://claude.ai/code/session_01Utp6Cb2FL7H75wB1wA6PpZ Co-authored-by: Claude <noreply@anthropic.com>
1 parent f76afd3 commit 6a703ca

1 file changed

Lines changed: 293 additions & 0 deletions

File tree

Lines changed: 293 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,293 @@
1+
// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license.
2+
//
3+
// #29961 — `os migrate stored-expressions`: a stored computed field declared
4+
// on an object that already holds rows is NULL on every one of them, because
5+
// the engine computes it on writes only.
6+
//
7+
// Every fixture builds the state the run exists for: rows written STRAIGHT
8+
// INTO THE DRIVER STORE, as an upgraded database holds them — stored before
9+
// the field was declared, never written since. Inserting them through the
10+
// engine would compute the field and leave nothing to fill.
11+
//
12+
// The engine is the real one, over an in-file driver; the values compared are
13+
// the ones a write of the same row stores.
14+
15+
import { describe, it, expect, beforeEach } from 'vitest';
16+
import { ObjectQL } from './engine.js';
17+
import {
18+
announceStoredExpressionHoles,
19+
backfillStoredExpressions,
20+
formatStoredExpressionBackfillReport,
21+
sameStoredValue,
22+
storedExpressionBackfillComplete,
23+
} from './stored-expression-backfill.js';
24+
25+
function makeDriver() {
26+
const stores = new Map<string, Map<string, any>>();
27+
const writes: Array<{ object: string; id: string; data: Record<string, unknown> }> = [];
28+
const storeFor = (o: string) => {
29+
let s = stores.get(o);
30+
if (!s) { s = new Map(); stores.set(o, s); }
31+
return s;
32+
};
33+
// Implicit equality, the comparison operators the keyset walk emits (`$gt` on `id`), and `$in`.
34+
const checkOp = (value: any, cond: any): boolean => {
35+
if (cond === null || typeof cond !== 'object' || Array.isArray(cond) || cond instanceof Date) return value === cond;
36+
return Object.entries(cond).every(([op, target]: [string, any]) => {
37+
switch (op) {
38+
case '$eq': return value === target;
39+
case '$ne': return value !== target;
40+
case '$gt': return value > target;
41+
case '$in': return Array.isArray(target) && target.includes(value);
42+
default: return true;
43+
}
44+
});
45+
};
46+
const matches = (row: any, where: any): boolean => {
47+
if (!where || typeof where !== 'object') return true;
48+
return Object.entries(where).every(([k, v]: [string, any]) => {
49+
if (k === '$and') return (v as any[]).every((w) => matches(row, w));
50+
if (k === '$or') return (v as any[]).some((w) => matches(row, w));
51+
return checkOp(row?.[k], v);
52+
});
53+
};
54+
let failFind: string | null = null;
55+
let failUpdateFor: string | null = null;
56+
const driver: any = {
57+
name: 'memory', version: '0.0.0', supports: {},
58+
async connect() {}, async disconnect() {}, async checkHealth() { return true; }, async execute() { return null; },
59+
async find(object: string, ast: any) {
60+
if (failFind === object) throw new Error('driver cannot read this table');
61+
const rows = Array.from(storeFor(object).values()).filter((r) => matches(r, ast?.where));
62+
rows.sort((a, b) => String(a.id).localeCompare(String(b.id)));
63+
return typeof ast?.limit === 'number' ? rows.slice(0, ast.limit) : rows;
64+
},
65+
async findOne(object: string, ast: any) {
66+
for (const r of storeFor(object).values()) if (matches(r, ast?.where)) return r;
67+
return null;
68+
},
69+
async create(object: string, data: Record<string, unknown>) {
70+
const row = { ...data, id: data.id as string };
71+
storeFor(object).set(row.id, row);
72+
return row;
73+
},
74+
async update(object: string, id: string, data: Record<string, unknown>) {
75+
if (failUpdateFor === id) throw new Error('driver refused this row');
76+
writes.push({ object, id, data });
77+
const s = storeFor(object);
78+
const row = { ...s.get(id), ...data, id };
79+
s.set(id, row);
80+
return row;
81+
},
82+
async delete(object: string, id: string) { return storeFor(object).delete(id); },
83+
async count() { return 0; },
84+
async bulkCreate() { return []; }, async bulkUpdate() { return []; }, async bulkDelete() {},
85+
async beginTransaction() { return { __trx: true, commit: async () => {}, rollback: async () => {} }; },
86+
async commit() {}, async rollback() {},
87+
};
88+
return {
89+
driver,
90+
storeFor,
91+
writes,
92+
failFind: (object: string | null) => { failFind = object; },
93+
failUpdateFor: (id: string | null) => { failUpdateFor = id; },
94+
};
95+
}
96+
97+
const quietLogger = { info: () => {}, warn: () => {} };
98+
const cel = (source: string) => ({ dialect: 'cel', source });
99+
100+
describe('backfillStoredExpressions — stored computed fields on rows stored before the field (#29961)', () => {
101+
let engine: ObjectQL;
102+
let d: ReturnType<typeof makeDriver>;
103+
const contract = (id: string) => d.storeFor('contract').get(id);
104+
105+
beforeEach(async () => {
106+
engine = new ObjectQL();
107+
d = makeDriver();
108+
engine.registerDriver(d.driver, true);
109+
await engine.init();
110+
engine.registry.registerObject({
111+
name: 'contract_type',
112+
fields: { name: { type: 'text' }, review_sla_days: { type: 'number' } },
113+
} as any);
114+
engine.registry.registerObject({
115+
name: 'contract',
116+
fields: {
117+
name: { type: 'text' },
118+
qty: { type: 'number' },
119+
contract_type: { type: 'lookup', reference: 'contract_type' },
120+
updated_at: { type: 'datetime' },
121+
// The hotclm shape: an offset read through a lookup.
122+
review_sla_days_offset: { type: 'number', expression: cel('0 - record.contract_type.review_sla_days') },
123+
qty_double: { type: 'number', expression: cel('record.qty * 2') },
124+
// Read on every read; it has no column and is never a backfill's.
125+
qty_label: { type: 'formula', expression: cel("string(record.qty) + ' pcs'") },
126+
},
127+
} as any);
128+
engine.registry.registerObject({ name: 'note', fields: { body: { type: 'text' } } } as any);
129+
d.storeFor('contract_type').set('t1', { id: 't1', name: 'NDA', review_sla_days: 5 });
130+
// Stored before the two fields were declared: neither column holds a value.
131+
d.storeFor('contract').set('c1', { id: 'c1', name: 'c1', qty: 1, contract_type: 't1', updated_at: '2026-01-01T00:00:00.000Z' });
132+
d.storeFor('contract').set('c2', { id: 'c2', name: 'c2', qty: 2, contract_type: 't1', updated_at: '2026-01-02T00:00:00.000Z' });
133+
// No contract type: the offset's own value is NULL here (the hop has nothing to read).
134+
d.storeFor('contract').set('c3', { id: 'c3', name: 'c3', qty: 3, contract_type: null, updated_at: '2026-01-03T00:00:00.000Z' });
135+
});
136+
137+
it('a dry run counts each NULL the expression fills and writes nothing', async () => {
138+
const report = await backfillStoredExpressions(engine, quietLogger);
139+
140+
expect(report.applied).toBe(false);
141+
expect(report.scannedObjects).toEqual(['contract']);
142+
expect(report.scannedRecords).toBe(3);
143+
const byField = Object.fromEntries(report.fields.map((f) => [f.field, f]));
144+
expect(byField.qty_double).toMatchObject({ nullRows: 3, written: 0 });
145+
// c3's NULL is the expression's own value (an empty reference), not a hole.
146+
expect(byField.review_sla_days_offset).toMatchObject({ nullRows: 2, faultedRows: 1, written: 0 });
147+
expect(byField.review_sla_days_offset.sampleRecordIds).toEqual(['c1', 'c2']);
148+
expect(report.fields.map((f) => f.field)).not.toContain('qty_label');
149+
expect(report.nullRows).toBe(5);
150+
expect(d.writes).toEqual([]);
151+
expect(contract('c1').qty_double).toBeUndefined();
152+
expect(formatStoredExpressionBackfillReport(report).at(-1)).toMatch(/^Dry run — no row or table was written\./);
153+
expect(storedExpressionBackfillComplete(report)).toBe(false);
154+
});
155+
156+
it('--apply fills each hole with what a write stores, keeps updated_at, and a re-run finds nothing', async () => {
157+
const report = await backfillStoredExpressions(engine, quietLogger, { apply: true });
158+
159+
expect(report.writtenRows).toBe(3);
160+
expect(report.failures).toEqual([]);
161+
expect(contract('c1')).toMatchObject({ qty_double: 2, review_sla_days_offset: -5, updated_at: '2026-01-01T00:00:00.000Z' });
162+
expect(contract('c2')).toMatchObject({ qty_double: 4, review_sla_days_offset: -5, updated_at: '2026-01-02T00:00:00.000Z' });
163+
expect(contract('c3').qty_double).toBe(6);
164+
expect(contract('c3').review_sla_days_offset ?? null).toBeNull();
165+
expect(storedExpressionBackfillComplete(report)).toBe(true);
166+
167+
const again = await backfillStoredExpressions(engine, quietLogger);
168+
expect(again.nullRows).toBe(0);
169+
expect(again.staleRows).toBe(0);
170+
expect(storedExpressionBackfillComplete(again)).toBe(true);
171+
});
172+
173+
it('a value the expression no longer gives is counted on every run and rewritten only under changed', async () => {
174+
await backfillStoredExpressions(engine, quietLogger, { apply: true });
175+
engine.registry.registerObject({
176+
name: 'contract',
177+
fields: { ...(engine.getObject('contract') as any).fields, qty_double: { type: 'number', expression: cel('record.qty * 3') } },
178+
} as any);
179+
d.writes.length = 0;
180+
181+
const plain = await backfillStoredExpressions(engine, quietLogger, { apply: true });
182+
expect(plain.staleRows).toBe(3);
183+
expect(plain.writtenRows).toBe(0);
184+
expect(contract('c1').qty_double).toBe(2);
185+
expect(formatStoredExpressionBackfillReport(plain).join('\n')).toContain('re-run with --changed');
186+
187+
const changed = await backfillStoredExpressions(engine, quietLogger, { apply: true, changed: true });
188+
expect(changed.writtenRows).toBe(3);
189+
expect([contract('c1').qty_double, contract('c2').qty_double, contract('c3').qty_double]).toEqual([3, 6, 9]);
190+
expect((await backfillStoredExpressions(engine, quietLogger, { changed: true })).staleRows).toBe(0);
191+
});
192+
193+
it('a row holding a value an expression can no longer compute is listed and never written over', async () => {
194+
// c3 holds an offset, but has no contract type: the write would store NULL over it.
195+
d.storeFor('contract').set('c3', { ...contract('c3'), review_sla_days_offset: -9 });
196+
197+
const report = await backfillStoredExpressions(engine, quietLogger, { apply: true });
198+
199+
expect(report.skippedRows).toBe(1);
200+
expect(report.faults).toEqual([
201+
expect.objectContaining({ object: 'contract', field: 'review_sla_days_offset', recordId: 'c3', blocksWrite: true }),
202+
]);
203+
expect(contract('c3')).toMatchObject({ review_sla_days_offset: -9 });
204+
expect(contract('c3').qty_double).toBeUndefined();
205+
expect(contract('c1').qty_double).toBe(2);
206+
expect(storedExpressionBackfillComplete(report)).toBe(false);
207+
});
208+
209+
it('one row failing to write does not stop the run, and is listed', async () => {
210+
d.failUpdateFor('c2');
211+
const report = await backfillStoredExpressions(engine, quietLogger, { apply: true, retry: { maxRetries: 1 } });
212+
213+
expect(report.failures).toEqual([expect.objectContaining({ object: 'contract', recordId: 'c2' })]);
214+
expect(report.writtenRows).toBe(2);
215+
expect(contract('c1').qty_double).toBe(2);
216+
expect(contract('c3').qty_double).toBe(6);
217+
expect(storedExpressionBackfillComplete(report)).toBe(false);
218+
});
219+
220+
it('field narrows the walk to one stored computed field', async () => {
221+
const report = await backfillStoredExpressions(engine, quietLogger, { objects: ['contract'], field: 'qty_double' });
222+
expect(report.fields.map((f) => f.field)).toEqual(['qty_double']);
223+
expect(report.nullRows).toBe(3);
224+
});
225+
226+
it('refuses, before any row is read, a scope it cannot act on', async () => {
227+
const refused = (options: Parameters<typeof backfillStoredExpressions>[2]) =>
228+
backfillStoredExpressions(engine, quietLogger, options).then(() => null, (e) => e);
229+
230+
expect(await refused({ objects: ['nope'] })).toMatchObject({ code: 'OBJECT_NOT_FOUND' });
231+
expect(await refused({ objects: ['note'] })).toMatchObject({ code: 'INVALID_FIELD', status: 400, httpStatus: 400 });
232+
expect(await refused({ objects: ['contract'], field: 'qty_label' }))
233+
.toMatchObject({ code: 'INVALID_FIELD', status: 400, field: 'qty_label' });
234+
expect(await refused({ field: 'qty_double' })).toMatchObject({ code: 'INVALID_FIELD', status: 400 });
235+
expect(d.writes).toEqual([]);
236+
});
237+
238+
it('an object it cannot read is reported unreadable and truncates the run', async () => {
239+
d.failFind('contract');
240+
const report = await backfillStoredExpressions(engine, quietLogger);
241+
expect(report.unreadableObjects).toEqual(['contract']);
242+
expect(report.truncated).toBe(true);
243+
expect(storedExpressionBackfillComplete(report)).toBe(false);
244+
});
245+
246+
describe('the boot advisory', () => {
247+
const capture = () => {
248+
const warned: string[] = [];
249+
return { warned, logger: { info: () => {}, warn: (m: string) => warned.push(m), debug: () => {} } };
250+
};
251+
252+
it('names each field holding a hole and the command, once', async () => {
253+
const { warned, logger } = capture();
254+
const samples = await announceStoredExpressionHoles(engine, logger);
255+
256+
expect(samples).toEqual([{ object: 'contract', fields: { review_sla_days_offset: 2, qty_double: 3 }, sampled: 3 }]);
257+
expect(warned).toHaveLength(1);
258+
expect(warned[0]).toContain('contract.qty_double (3 of the first 3 rows)');
259+
expect(warned[0]).toContain('contract.review_sla_days_offset (2 of the first 3 rows)');
260+
expect(warned[0]).toContain('os migrate stored-expressions');
261+
expect(d.writes).toEqual([]);
262+
});
263+
264+
it('is silent once the holes are filled', async () => {
265+
await backfillStoredExpressions(engine, quietLogger, { apply: true });
266+
const { warned, logger } = capture();
267+
expect(await announceStoredExpressionHoles(engine, logger)).toEqual([]);
268+
expect(warned).toEqual([]);
269+
});
270+
271+
it('never fails the boot over a table it cannot read', async () => {
272+
d.failFind('contract');
273+
const { warned, logger } = capture();
274+
await expect(announceStoredExpressionHoles(engine, logger)).resolves.toEqual([]);
275+
expect(warned).toEqual([]);
276+
});
277+
});
278+
});
279+
280+
describe('sameStoredValue — a driver\'s spelling of the value a write stored', () => {
281+
it.each([
282+
[4, 4, true], ['4', 4, true], ['4.50', 4.5, true], [5, 4, false],
283+
[1, true, true], [0, false, true], ['1', true, true], [1, false, false],
284+
['c1-x', 'c1-x', true], ['c1', 'c1-x', false],
285+
['2026-01-02T00:00:00.000Z', new Date('2026-01-02T00:00:00.000Z'), true],
286+
['2026-01-02', new Date('2026-01-02T00:00:00.000Z'), true],
287+
['2026-01-03', new Date('2026-01-02T00:00:00.000Z'), false],
288+
['["a","b"]', ['a', 'b'], true],
289+
[null, null, true], [undefined, null, true], [null, 0, false], [0, null, false],
290+
])('%j vs %j → %s', (stored, computed, same) => {
291+
expect(sameStoredValue(stored, computed)).toBe(same);
292+
});
293+
});

0 commit comments

Comments
 (0)