|
| 1 | +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. |
| 2 | +// |
| 3 | +// #29961 — `os migrate stored-expressions`: a stored computed field declared |
| 4 | +// on an object that already holds rows is NULL on every one of them, because |
| 5 | +// the engine computes it on writes only. |
| 6 | +// |
| 7 | +// Every fixture builds the state the run exists for: rows written STRAIGHT |
| 8 | +// INTO THE DRIVER STORE, as an upgraded database holds them — stored before |
| 9 | +// the field was declared, never written since. Inserting them through the |
| 10 | +// engine would compute the field and leave nothing to fill. |
| 11 | +// |
| 12 | +// The engine is the real one, over an in-file driver; the values compared are |
| 13 | +// the ones a write of the same row stores. |
| 14 | + |
| 15 | +import { describe, it, expect, beforeEach } from 'vitest'; |
| 16 | +import { ObjectQL } from './engine.js'; |
| 17 | +import { |
| 18 | + announceStoredExpressionHoles, |
| 19 | + backfillStoredExpressions, |
| 20 | + formatStoredExpressionBackfillReport, |
| 21 | + sameStoredValue, |
| 22 | + storedExpressionBackfillComplete, |
| 23 | +} from './stored-expression-backfill.js'; |
| 24 | + |
| 25 | +function makeDriver() { |
| 26 | + const stores = new Map<string, Map<string, any>>(); |
| 27 | + const writes: Array<{ object: string; id: string; data: Record<string, unknown> }> = []; |
| 28 | + const storeFor = (o: string) => { |
| 29 | + let s = stores.get(o); |
| 30 | + if (!s) { s = new Map(); stores.set(o, s); } |
| 31 | + return s; |
| 32 | + }; |
| 33 | + // Implicit equality, the comparison operators the keyset walk emits (`$gt` on `id`), and `$in`. |
| 34 | + const checkOp = (value: any, cond: any): boolean => { |
| 35 | + if (cond === null || typeof cond !== 'object' || Array.isArray(cond) || cond instanceof Date) return value === cond; |
| 36 | + return Object.entries(cond).every(([op, target]: [string, any]) => { |
| 37 | + switch (op) { |
| 38 | + case '$eq': return value === target; |
| 39 | + case '$ne': return value !== target; |
| 40 | + case '$gt': return value > target; |
| 41 | + case '$in': return Array.isArray(target) && target.includes(value); |
| 42 | + default: return true; |
| 43 | + } |
| 44 | + }); |
| 45 | + }; |
| 46 | + const matches = (row: any, where: any): boolean => { |
| 47 | + if (!where || typeof where !== 'object') return true; |
| 48 | + return Object.entries(where).every(([k, v]: [string, any]) => { |
| 49 | + if (k === '$and') return (v as any[]).every((w) => matches(row, w)); |
| 50 | + if (k === '$or') return (v as any[]).some((w) => matches(row, w)); |
| 51 | + return checkOp(row?.[k], v); |
| 52 | + }); |
| 53 | + }; |
| 54 | + let failFind: string | null = null; |
| 55 | + let failUpdateFor: string | null = null; |
| 56 | + const driver: any = { |
| 57 | + name: 'memory', version: '0.0.0', supports: {}, |
| 58 | + async connect() {}, async disconnect() {}, async checkHealth() { return true; }, async execute() { return null; }, |
| 59 | + async find(object: string, ast: any) { |
| 60 | + if (failFind === object) throw new Error('driver cannot read this table'); |
| 61 | + const rows = Array.from(storeFor(object).values()).filter((r) => matches(r, ast?.where)); |
| 62 | + rows.sort((a, b) => String(a.id).localeCompare(String(b.id))); |
| 63 | + return typeof ast?.limit === 'number' ? rows.slice(0, ast.limit) : rows; |
| 64 | + }, |
| 65 | + async findOne(object: string, ast: any) { |
| 66 | + for (const r of storeFor(object).values()) if (matches(r, ast?.where)) return r; |
| 67 | + return null; |
| 68 | + }, |
| 69 | + async create(object: string, data: Record<string, unknown>) { |
| 70 | + const row = { ...data, id: data.id as string }; |
| 71 | + storeFor(object).set(row.id, row); |
| 72 | + return row; |
| 73 | + }, |
| 74 | + async update(object: string, id: string, data: Record<string, unknown>) { |
| 75 | + if (failUpdateFor === id) throw new Error('driver refused this row'); |
| 76 | + writes.push({ object, id, data }); |
| 77 | + const s = storeFor(object); |
| 78 | + const row = { ...s.get(id), ...data, id }; |
| 79 | + s.set(id, row); |
| 80 | + return row; |
| 81 | + }, |
| 82 | + async delete(object: string, id: string) { return storeFor(object).delete(id); }, |
| 83 | + async count() { return 0; }, |
| 84 | + async bulkCreate() { return []; }, async bulkUpdate() { return []; }, async bulkDelete() {}, |
| 85 | + async beginTransaction() { return { __trx: true, commit: async () => {}, rollback: async () => {} }; }, |
| 86 | + async commit() {}, async rollback() {}, |
| 87 | + }; |
| 88 | + return { |
| 89 | + driver, |
| 90 | + storeFor, |
| 91 | + writes, |
| 92 | + failFind: (object: string | null) => { failFind = object; }, |
| 93 | + failUpdateFor: (id: string | null) => { failUpdateFor = id; }, |
| 94 | + }; |
| 95 | +} |
| 96 | + |
| 97 | +const quietLogger = { info: () => {}, warn: () => {} }; |
| 98 | +const cel = (source: string) => ({ dialect: 'cel', source }); |
| 99 | + |
| 100 | +describe('backfillStoredExpressions — stored computed fields on rows stored before the field (#29961)', () => { |
| 101 | + let engine: ObjectQL; |
| 102 | + let d: ReturnType<typeof makeDriver>; |
| 103 | + const contract = (id: string) => d.storeFor('contract').get(id); |
| 104 | + |
| 105 | + beforeEach(async () => { |
| 106 | + engine = new ObjectQL(); |
| 107 | + d = makeDriver(); |
| 108 | + engine.registerDriver(d.driver, true); |
| 109 | + await engine.init(); |
| 110 | + engine.registry.registerObject({ |
| 111 | + name: 'contract_type', |
| 112 | + fields: { name: { type: 'text' }, review_sla_days: { type: 'number' } }, |
| 113 | + } as any); |
| 114 | + engine.registry.registerObject({ |
| 115 | + name: 'contract', |
| 116 | + fields: { |
| 117 | + name: { type: 'text' }, |
| 118 | + qty: { type: 'number' }, |
| 119 | + contract_type: { type: 'lookup', reference: 'contract_type' }, |
| 120 | + updated_at: { type: 'datetime' }, |
| 121 | + // The hotclm shape: an offset read through a lookup. |
| 122 | + review_sla_days_offset: { type: 'number', expression: cel('0 - record.contract_type.review_sla_days') }, |
| 123 | + qty_double: { type: 'number', expression: cel('record.qty * 2') }, |
| 124 | + // Read on every read; it has no column and is never a backfill's. |
| 125 | + qty_label: { type: 'formula', expression: cel("string(record.qty) + ' pcs'") }, |
| 126 | + }, |
| 127 | + } as any); |
| 128 | + engine.registry.registerObject({ name: 'note', fields: { body: { type: 'text' } } } as any); |
| 129 | + d.storeFor('contract_type').set('t1', { id: 't1', name: 'NDA', review_sla_days: 5 }); |
| 130 | + // Stored before the two fields were declared: neither column holds a value. |
| 131 | + d.storeFor('contract').set('c1', { id: 'c1', name: 'c1', qty: 1, contract_type: 't1', updated_at: '2026-01-01T00:00:00.000Z' }); |
| 132 | + d.storeFor('contract').set('c2', { id: 'c2', name: 'c2', qty: 2, contract_type: 't1', updated_at: '2026-01-02T00:00:00.000Z' }); |
| 133 | + // No contract type: the offset's own value is NULL here (the hop has nothing to read). |
| 134 | + d.storeFor('contract').set('c3', { id: 'c3', name: 'c3', qty: 3, contract_type: null, updated_at: '2026-01-03T00:00:00.000Z' }); |
| 135 | + }); |
| 136 | + |
| 137 | + it('a dry run counts each NULL the expression fills and writes nothing', async () => { |
| 138 | + const report = await backfillStoredExpressions(engine, quietLogger); |
| 139 | + |
| 140 | + expect(report.applied).toBe(false); |
| 141 | + expect(report.scannedObjects).toEqual(['contract']); |
| 142 | + expect(report.scannedRecords).toBe(3); |
| 143 | + const byField = Object.fromEntries(report.fields.map((f) => [f.field, f])); |
| 144 | + expect(byField.qty_double).toMatchObject({ nullRows: 3, written: 0 }); |
| 145 | + // c3's NULL is the expression's own value (an empty reference), not a hole. |
| 146 | + expect(byField.review_sla_days_offset).toMatchObject({ nullRows: 2, faultedRows: 1, written: 0 }); |
| 147 | + expect(byField.review_sla_days_offset.sampleRecordIds).toEqual(['c1', 'c2']); |
| 148 | + expect(report.fields.map((f) => f.field)).not.toContain('qty_label'); |
| 149 | + expect(report.nullRows).toBe(5); |
| 150 | + expect(d.writes).toEqual([]); |
| 151 | + expect(contract('c1').qty_double).toBeUndefined(); |
| 152 | + expect(formatStoredExpressionBackfillReport(report).at(-1)).toMatch(/^Dry run — no row or table was written\./); |
| 153 | + expect(storedExpressionBackfillComplete(report)).toBe(false); |
| 154 | + }); |
| 155 | + |
| 156 | + it('--apply fills each hole with what a write stores, keeps updated_at, and a re-run finds nothing', async () => { |
| 157 | + const report = await backfillStoredExpressions(engine, quietLogger, { apply: true }); |
| 158 | + |
| 159 | + expect(report.writtenRows).toBe(3); |
| 160 | + expect(report.failures).toEqual([]); |
| 161 | + expect(contract('c1')).toMatchObject({ qty_double: 2, review_sla_days_offset: -5, updated_at: '2026-01-01T00:00:00.000Z' }); |
| 162 | + expect(contract('c2')).toMatchObject({ qty_double: 4, review_sla_days_offset: -5, updated_at: '2026-01-02T00:00:00.000Z' }); |
| 163 | + expect(contract('c3').qty_double).toBe(6); |
| 164 | + expect(contract('c3').review_sla_days_offset ?? null).toBeNull(); |
| 165 | + expect(storedExpressionBackfillComplete(report)).toBe(true); |
| 166 | + |
| 167 | + const again = await backfillStoredExpressions(engine, quietLogger); |
| 168 | + expect(again.nullRows).toBe(0); |
| 169 | + expect(again.staleRows).toBe(0); |
| 170 | + expect(storedExpressionBackfillComplete(again)).toBe(true); |
| 171 | + }); |
| 172 | + |
| 173 | + it('a value the expression no longer gives is counted on every run and rewritten only under changed', async () => { |
| 174 | + await backfillStoredExpressions(engine, quietLogger, { apply: true }); |
| 175 | + engine.registry.registerObject({ |
| 176 | + name: 'contract', |
| 177 | + fields: { ...(engine.getObject('contract') as any).fields, qty_double: { type: 'number', expression: cel('record.qty * 3') } }, |
| 178 | + } as any); |
| 179 | + d.writes.length = 0; |
| 180 | + |
| 181 | + const plain = await backfillStoredExpressions(engine, quietLogger, { apply: true }); |
| 182 | + expect(plain.staleRows).toBe(3); |
| 183 | + expect(plain.writtenRows).toBe(0); |
| 184 | + expect(contract('c1').qty_double).toBe(2); |
| 185 | + expect(formatStoredExpressionBackfillReport(plain).join('\n')).toContain('re-run with --changed'); |
| 186 | + |
| 187 | + const changed = await backfillStoredExpressions(engine, quietLogger, { apply: true, changed: true }); |
| 188 | + expect(changed.writtenRows).toBe(3); |
| 189 | + expect([contract('c1').qty_double, contract('c2').qty_double, contract('c3').qty_double]).toEqual([3, 6, 9]); |
| 190 | + expect((await backfillStoredExpressions(engine, quietLogger, { changed: true })).staleRows).toBe(0); |
| 191 | + }); |
| 192 | + |
| 193 | + it('a row holding a value an expression can no longer compute is listed and never written over', async () => { |
| 194 | + // c3 holds an offset, but has no contract type: the write would store NULL over it. |
| 195 | + d.storeFor('contract').set('c3', { ...contract('c3'), review_sla_days_offset: -9 }); |
| 196 | + |
| 197 | + const report = await backfillStoredExpressions(engine, quietLogger, { apply: true }); |
| 198 | + |
| 199 | + expect(report.skippedRows).toBe(1); |
| 200 | + expect(report.faults).toEqual([ |
| 201 | + expect.objectContaining({ object: 'contract', field: 'review_sla_days_offset', recordId: 'c3', blocksWrite: true }), |
| 202 | + ]); |
| 203 | + expect(contract('c3')).toMatchObject({ review_sla_days_offset: -9 }); |
| 204 | + expect(contract('c3').qty_double).toBeUndefined(); |
| 205 | + expect(contract('c1').qty_double).toBe(2); |
| 206 | + expect(storedExpressionBackfillComplete(report)).toBe(false); |
| 207 | + }); |
| 208 | + |
| 209 | + it('one row failing to write does not stop the run, and is listed', async () => { |
| 210 | + d.failUpdateFor('c2'); |
| 211 | + const report = await backfillStoredExpressions(engine, quietLogger, { apply: true, retry: { maxRetries: 1 } }); |
| 212 | + |
| 213 | + expect(report.failures).toEqual([expect.objectContaining({ object: 'contract', recordId: 'c2' })]); |
| 214 | + expect(report.writtenRows).toBe(2); |
| 215 | + expect(contract('c1').qty_double).toBe(2); |
| 216 | + expect(contract('c3').qty_double).toBe(6); |
| 217 | + expect(storedExpressionBackfillComplete(report)).toBe(false); |
| 218 | + }); |
| 219 | + |
| 220 | + it('field narrows the walk to one stored computed field', async () => { |
| 221 | + const report = await backfillStoredExpressions(engine, quietLogger, { objects: ['contract'], field: 'qty_double' }); |
| 222 | + expect(report.fields.map((f) => f.field)).toEqual(['qty_double']); |
| 223 | + expect(report.nullRows).toBe(3); |
| 224 | + }); |
| 225 | + |
| 226 | + it('refuses, before any row is read, a scope it cannot act on', async () => { |
| 227 | + const refused = (options: Parameters<typeof backfillStoredExpressions>[2]) => |
| 228 | + backfillStoredExpressions(engine, quietLogger, options).then(() => null, (e) => e); |
| 229 | + |
| 230 | + expect(await refused({ objects: ['nope'] })).toMatchObject({ code: 'OBJECT_NOT_FOUND' }); |
| 231 | + expect(await refused({ objects: ['note'] })).toMatchObject({ code: 'INVALID_FIELD', status: 400, httpStatus: 400 }); |
| 232 | + expect(await refused({ objects: ['contract'], field: 'qty_label' })) |
| 233 | + .toMatchObject({ code: 'INVALID_FIELD', status: 400, field: 'qty_label' }); |
| 234 | + expect(await refused({ field: 'qty_double' })).toMatchObject({ code: 'INVALID_FIELD', status: 400 }); |
| 235 | + expect(d.writes).toEqual([]); |
| 236 | + }); |
| 237 | + |
| 238 | + it('an object it cannot read is reported unreadable and truncates the run', async () => { |
| 239 | + d.failFind('contract'); |
| 240 | + const report = await backfillStoredExpressions(engine, quietLogger); |
| 241 | + expect(report.unreadableObjects).toEqual(['contract']); |
| 242 | + expect(report.truncated).toBe(true); |
| 243 | + expect(storedExpressionBackfillComplete(report)).toBe(false); |
| 244 | + }); |
| 245 | + |
| 246 | + describe('the boot advisory', () => { |
| 247 | + const capture = () => { |
| 248 | + const warned: string[] = []; |
| 249 | + return { warned, logger: { info: () => {}, warn: (m: string) => warned.push(m), debug: () => {} } }; |
| 250 | + }; |
| 251 | + |
| 252 | + it('names each field holding a hole and the command, once', async () => { |
| 253 | + const { warned, logger } = capture(); |
| 254 | + const samples = await announceStoredExpressionHoles(engine, logger); |
| 255 | + |
| 256 | + expect(samples).toEqual([{ object: 'contract', fields: { review_sla_days_offset: 2, qty_double: 3 }, sampled: 3 }]); |
| 257 | + expect(warned).toHaveLength(1); |
| 258 | + expect(warned[0]).toContain('contract.qty_double (3 of the first 3 rows)'); |
| 259 | + expect(warned[0]).toContain('contract.review_sla_days_offset (2 of the first 3 rows)'); |
| 260 | + expect(warned[0]).toContain('os migrate stored-expressions'); |
| 261 | + expect(d.writes).toEqual([]); |
| 262 | + }); |
| 263 | + |
| 264 | + it('is silent once the holes are filled', async () => { |
| 265 | + await backfillStoredExpressions(engine, quietLogger, { apply: true }); |
| 266 | + const { warned, logger } = capture(); |
| 267 | + expect(await announceStoredExpressionHoles(engine, logger)).toEqual([]); |
| 268 | + expect(warned).toEqual([]); |
| 269 | + }); |
| 270 | + |
| 271 | + it('never fails the boot over a table it cannot read', async () => { |
| 272 | + d.failFind('contract'); |
| 273 | + const { warned, logger } = capture(); |
| 274 | + await expect(announceStoredExpressionHoles(engine, logger)).resolves.toEqual([]); |
| 275 | + expect(warned).toEqual([]); |
| 276 | + }); |
| 277 | + }); |
| 278 | +}); |
| 279 | + |
| 280 | +describe('sameStoredValue — a driver\'s spelling of the value a write stored', () => { |
| 281 | + it.each([ |
| 282 | + [4, 4, true], ['4', 4, true], ['4.50', 4.5, true], [5, 4, false], |
| 283 | + [1, true, true], [0, false, true], ['1', true, true], [1, false, false], |
| 284 | + ['c1-x', 'c1-x', true], ['c1', 'c1-x', false], |
| 285 | + ['2026-01-02T00:00:00.000Z', new Date('2026-01-02T00:00:00.000Z'), true], |
| 286 | + ['2026-01-02', new Date('2026-01-02T00:00:00.000Z'), true], |
| 287 | + ['2026-01-03', new Date('2026-01-02T00:00:00.000Z'), false], |
| 288 | + ['["a","b"]', ['a', 'b'], true], |
| 289 | + [null, null, true], [undefined, null, true], [null, 0, false], [0, null, false], |
| 290 | + ])('%j vs %j → %s', (stored, computed, same) => { |
| 291 | + expect(sameStoredValue(stored, computed)).toBe(same); |
| 292 | + }); |
| 293 | +}); |
0 commit comments