66 newClient ,
77} from '@cipherstash/protect-ffi'
88import { Client } from 'pg'
9- import { beforeAll , describe , expect , test } from 'vitest'
9+ import { afterAll , beforeAll , describe , expect , test } from 'vitest'
1010import {
1111 CHECK_VIOLATION ,
1212 decryptV4 ,
@@ -20,9 +20,12 @@ import {
2020// EQL v2 -> v4 migration, end to end, against live ZeroKMS and Postgres.
2121//
2222// The v2 data is written by the real Protect.js, `@cipherstash/protect@10.2.1`
23- // (protect-ffi 0.18.1, cipherstash-client 0.28), the last client customers
24- // used before Stack. Not by today's protect-ffi in v2 mode: whether old
25- // ciphertext still reads is the question, so the old client writes it.
23+ // (protect-ffi 0.18.1, cipherstash-client 0.28), the release #1436 pins for
24+ // DynamoDB. Not by today's protect-ffi in v2 mode: whether old ciphertext
25+ // still reads is the question, so an old client writes it. It is not the last
26+ // Protect.js: 10.3 to 10.5, 11.x and 12.x (protect-ffi 0.19 to 0.23, and
27+ // `@cipherstash/schema` 3 from 12.0) also write EQL v2, and a pass here says
28+ // nothing about the payloads those releases write.
2629//
2730// v2 stores every column as one type, `eql_v2_encrypted`, and decides what a
2831// column can answer from the payload's index terms. v4 has a domain per
@@ -38,6 +41,13 @@ import {
3841//
3942// Requires EQL v2 and v4 both installed (`mise run setup` installs v2, v3 and
4043// v4) and the v4 codec built (`mise run eql:v4:codec`); see ./v4-codec.ts.
44+ //
45+ // Scope: the Protect.js index each column used, one column each:
46+ // `equality()` -> TextEq (`email`), `equality([{ kind: 'downcase' }])` ->
47+ // TextEq (`handle`), `orderAndRange()` on a number -> DoubleOrdOre (`age`) and
48+ // on text -> TextOrdOre (`surname`), `freeTextSearch()` -> TextMatch (`bio`),
49+ // and no index -> Boolean (`active`). Queried with `=`, `>`, `ORDER BY` and
50+ // `@@`; no `<`, `<=`, `>=` or `BETWEEN`, no JSON, and no `searchableJson()`.
4151
4252const TABLE = 'accounts'
4353
@@ -47,7 +57,12 @@ const encryptConfig: EncryptConfig = {
4757 tables : {
4858 [ TABLE ] : {
4959 email : { cast_as : 'text' , indexes : { unique : { } } } ,
60+ handle : {
61+ cast_as : 'text' ,
62+ indexes : { unique : { token_filters : [ { kind : 'downcase' } ] } } ,
63+ } ,
5064 age : { cast_as : 'number' , indexes : { ore : { } } } ,
65+ surname : { cast_as : 'text' , indexes : { ore : { } } } ,
5166 bio : { cast_as : 'text' , indexes : { match : { } } } ,
5267 active : { cast_as : 'boolean' , indexes : { } } ,
5368 } ,
@@ -71,7 +86,9 @@ async function legacyProtect() {
7186 const { csColumn, csTable, protect } = await import ( '@cipherstash/protect' )
7287 const table = csTable ( TABLE , {
7388 email : csColumn ( 'email' ) . equality ( ) ,
89+ handle : csColumn ( 'handle' ) . equality ( [ { kind : 'downcase' } ] ) ,
7490 age : csColumn ( 'age' ) . dataType ( 'number' ) . orderAndRange ( ) ,
91+ surname : csColumn ( 'surname' ) . orderAndRange ( ) ,
7592 bio : csColumn ( 'bio' ) . freeTextSearch ( ) ,
7693 active : csColumn ( 'active' ) . dataType ( 'boolean' ) ,
7794 } )
@@ -124,39 +141,51 @@ const asLegacy = (value: Plaintext) => value as string | number
124141
125142type Account = {
126143 email : string
144+ handle : string
127145 age : number
146+ surname : string
128147 bio : string
129148 active : boolean
130149}
131150
132151const accounts : Account [ ] = [
133152 {
134153 email : 'ada@example.com' ,
154+ handle : 'AdaL' ,
135155 age : 36 ,
156+ surname : 'lovelace' ,
136157 bio : 'analytical engine notes' ,
137158 active : true ,
138159 } ,
139160 {
140161 email : 'grace@example.com' ,
162+ handle : 'GraceH' ,
141163 age : 85 ,
164+ surname : 'hopper' ,
142165 bio : 'compiler pioneer and admiral' ,
143166 active : true ,
144167 } ,
145168 {
146169 email : 'alan@example.com' ,
170+ handle : 'AlanT' ,
147171 age : 41 ,
172+ surname : 'turing' ,
148173 bio : 'computable numbers and machines' ,
149174 active : false ,
150175 } ,
151176 {
152177 email : 'edsger@example.com' ,
178+ handle : 'EdsgerD' ,
153179 age : 72 ,
180+ surname : 'dijkstra' ,
154181 bio : 'structured programming advocate' ,
155182 active : false ,
156183 } ,
157184 {
158185 email : 'barbara@example.com' ,
186+ handle : 'BarbaraL' ,
159187 age : 29 ,
188+ surname : 'liskov' ,
160189 bio : 'abstraction and programming methodology' ,
161190 active : true ,
162191 } ,
@@ -171,7 +200,10 @@ type Column = keyof Account
171200 */
172201const columns : Record < Column , { v4 : V4Type ; v4Domain : string } > = {
173202 email : { v4 : 'TextEq' , v4Domain : 'public.eql_v4_text_eq' } ,
203+ // v4 has no case-folding equality: see the `handle` test below.
204+ handle : { v4 : 'TextEq' , v4Domain : 'public.eql_v4_text_eq' } ,
174205 age : { v4 : 'DoubleOrdOre' , v4Domain : 'public.eql_v4_double_ord_ore' } ,
206+ surname : { v4 : 'TextOrdOre' , v4Domain : 'public.eql_v4_text_ord_ore' } ,
175207 bio : { v4 : 'TextMatch' , v4Domain : 'public.eql_v4_text_match' } ,
176208 active : { v4 : 'Boolean' , v4Domain : 'public.eql_v4_boolean' } ,
177209}
@@ -180,6 +212,10 @@ const columnNames = Object.keys(columns) as Column[]
180212
181213const emails = ( rows : Account [ ] ) => rows . map ( ( row ) => row . email )
182214
215+ const bySurname = [ ...accounts ] . sort ( ( a , b ) =>
216+ a . surname < b . surname ? - 1 : a . surname > b . surname ? 1 : 0 ,
217+ )
218+
183219describe ( 'EQL v2 -> v4 migration' , ( ) => {
184220 const pg = new Client ( )
185221 let legacy : Awaited < ReturnType < typeof legacyProtect > >
@@ -215,6 +251,8 @@ describe('EQL v2 -> v4 migration', () => {
215251 return operand
216252 }
217253
254+ // A live `bulkEncrypt` per column and an insert per row can outrun Vitest's
255+ // 10 s hook default, and a hook that times out skips every test in the file.
218256 beforeAll ( async ( ) => {
219257 legacy = await legacyProtect ( )
220258 stackClient = await newClient ( { encryptConfig } )
@@ -257,11 +295,14 @@ describe('EQL v2 -> v4 migration', () => {
257295 [ index + 1 , ...columnNames . map ( ( c ) => byColumn . get ( c ) ?. [ index ] ) ] ,
258296 )
259297 }
298+ } , 60_000 )
260299
261- return async ( ) => {
262- await pg . query ( `DROP TABLE IF EXISTS ${ TABLE } ` )
263- await pg . end ( )
264- }
300+ // In `afterAll`, not returned from `beforeAll`: a returned teardown runs only
301+ // once the hook succeeds, so a setup failure would leave the table and the
302+ // connection behind.
303+ afterAll ( async ( ) => {
304+ await pg . query ( `DROP TABLE IF EXISTS ${ TABLE } ` )
305+ await pg . end ( )
265306 } )
266307
267308 test ( 'the v2 table answers the queries the migration must preserve' , async ( ) => {
@@ -291,6 +332,22 @@ describe('EQL v2 -> v4 migration', () => {
291332 await v2Term ( 'bio' , 'programming' ) ,
292333 ] ) ,
293334 ) . toEqual ( emails ( accounts . filter ( ( a ) => a . bio . includes ( 'programming' ) ) ) )
335+
336+ // The downcase filter folds both sides, so any case finds the row.
337+ expect (
338+ await emailsWhere ( 'WHERE handle = $1::jsonb' , [
339+ await v2Term ( 'handle' , 'alant' ) ,
340+ ] ) ,
341+ ) . toEqual ( [ 'alan@example.com' ] )
342+
343+ expect (
344+ await emailsWhere ( 'WHERE surname > $1::jsonb ORDER BY id' , [
345+ await v2Term ( 'surname' , 'hopper' ) ,
346+ ] ) ,
347+ ) . toEqual ( emails ( accounts . filter ( ( a ) => a . surname > 'hopper' ) ) )
348+ expect ( await emailsWhere ( 'ORDER BY eql_v2.order_by(surname)' ) ) . toEqual (
349+ emails ( bySurname ) ,
350+ )
294351 } )
295352
296353 test ( 'a v2 payload cannot be cast into a v4 column' , async ( ) => {
@@ -309,19 +366,44 @@ describe('EQL v2 -> v4 migration', () => {
309366 } )
310367
311368 test ( 'relabelling a v2 payload as v4 does not make it a v4 value' , async ( ) => {
312- const res = await pg . query < { email : Record < string , unknown > } > (
313- `SELECT email::jsonb AS email FROM ${ TABLE } WHERE id = 3` ,
314- )
369+ const res = await pg . query < {
370+ email : { i : unknown ; c : unknown ; hm : unknown }
371+ } > ( `SELECT email::jsonb AS email FROM ${ TABLE } WHERE id = 3` )
372+ const { i, c, hm } = res . rows [ 0 ] . email
373+ expect ( typeof c ) . toBe ( 'string' )
374+ expect ( typeof hm ) . toBe ( 'string' )
375+ // Exactly TextEq's fields, so the refusal below cannot come from a v2-only
376+ // key that v4's `deny_unknown_fields` rejects: only the ciphertext's
377+ // producer is left to refuse.
378+ const relabelled = { v : 4 , i, c, hm }
379+
380+ // The codec opens a genuine v4 payload, so the refusal below is about the
381+ // payload and not a missing binary or bad credentials.
382+ const [ genuine ] = encryptV4 ( [
383+ {
384+ type : 'TextEq' ,
385+ table : TABLE ,
386+ column : 'email' ,
387+ value : 'alan@example.com' ,
388+ } ,
389+ ] )
390+ expect (
391+ decryptV4 ( [
392+ { type : 'TextEq' , table : TABLE , column : 'email' , payload : genuine } ,
393+ ] ) ,
394+ ) . toEqual ( [ 'alan@example.com' ] )
395+
396+ // Stack Encrypt refuses to open a cipherstash-client ciphertext.
315397 expect ( ( ) =>
316398 decryptV4 ( [
317399 {
318400 type : 'TextEq' ,
319401 table : TABLE ,
320402 column : 'email' ,
321- payload : { ... res . rows [ 0 ] . email , v : 4 } ,
403+ payload : relabelled ,
322404 } ,
323405 ] ) ,
324- ) . toThrowError ( )
406+ ) . toThrowError ( 'unsupported EQL ciphertext producer or version' )
325407 } )
326408
327409 test ( 'backfill: decrypt every v2 value and re-encrypt it as v4 beside it' , async ( ) => {
@@ -445,6 +527,53 @@ describe('EQL v2 -> v4 migration', () => {
445527 ) . toEqual ( expected )
446528 } )
447529
530+ test ( 'text range and ordering: ORE on text agrees across versions' , async ( ) => {
531+ const after = emails ( accounts . filter ( ( a ) => a . surname > 'hopper' ) )
532+ expect (
533+ await emailsWhere ( 'WHERE surname > $1::jsonb ORDER BY id' , [
534+ await v2Term ( 'surname' , 'hopper' ) ,
535+ ] ) ,
536+ ) . toEqual ( after )
537+ expect (
538+ await emailsWhere (
539+ 'WHERE surname_v4 > $1::jsonb::eql_v4.query_text_ord_ore ORDER BY id' ,
540+ [ v4Operand ( 'surname' , 'hopper' ) ] ,
541+ ) ,
542+ ) . toEqual ( after )
543+
544+ expect ( await emailsWhere ( 'ORDER BY eql_v2.order_by(surname)' ) ) . toEqual (
545+ emails ( bySurname ) ,
546+ )
547+ expect (
548+ await emailsWhere ( 'ORDER BY eql_v4.ord_term_ore(surname_v4)' ) ,
549+ ) . toEqual ( emails ( bySurname ) )
550+ } )
551+
552+ // v4 has no equality type that folds case: TextEq's HMAC is over the exact
553+ // plaintext. A column Protect.js indexed with `equality([{ kind: 'downcase'
554+ // }])` therefore stops answering queries in a different case once it moves
555+ // to TextEq, and nothing in the backfill reports it. This pins that loss so
556+ // it cannot go unnoticed: an application that needs case-insensitive
557+ // equality has to fold case itself, before encrypting and querying, which
558+ // also changes the plaintext it stores.
559+ test ( 'downcase equality: the v2 column ignores case and the v4 TextEq column does not' , async ( ) => {
560+ expect (
561+ await emailsWhere ( 'WHERE handle = $1::jsonb' , [
562+ await v2Term ( 'handle' , 'alant' ) ,
563+ ] ) ,
564+ ) . toEqual ( [ 'alan@example.com' ] )
565+ expect (
566+ await emailsWhere ( 'WHERE handle_v4 = $1::jsonb::eql_v4.query_text_eq' , [
567+ v4Operand ( 'handle' , 'AlanT' ) ,
568+ ] ) ,
569+ ) . toEqual ( [ 'alan@example.com' ] )
570+ expect (
571+ await emailsWhere ( 'WHERE handle_v4 = $1::jsonb::eql_v4.query_text_eq' , [
572+ v4Operand ( 'handle' , 'alant' ) ,
573+ ] ) ,
574+ ) . toEqual ( [ ] )
575+ } )
576+
448577 test ( 'cut-over: drop the v2 columns and the v4 values answer under the original names' , async ( ) => {
449578 for ( const column of columnNames ) {
450579 await pg . query ( `ALTER TABLE ${ TABLE } DROP COLUMN ${ column } ` )
@@ -453,24 +582,59 @@ describe('EQL v2 -> v4 migration', () => {
453582 )
454583 }
455584
585+ // Every query the v2 columns answered, now under the original names.
456586 expect (
457587 await emailsWhere ( 'WHERE email = $1::jsonb::eql_v4.query_text_eq' , [
458588 v4Operand ( 'email' , 'grace@example.com' ) ,
459589 ] ) ,
460590 ) . toEqual ( [ 'grace@example.com' ] )
591+ expect (
592+ await emailsWhere ( 'WHERE handle = $1::jsonb::eql_v4.query_text_eq' , [
593+ v4Operand ( 'handle' , 'GraceH' ) ,
594+ ] ) ,
595+ ) . toEqual ( [ 'grace@example.com' ] )
596+ expect (
597+ await emailsWhere (
598+ 'WHERE age > $1::jsonb::eql_v4.query_double_ord_ore ORDER BY id' ,
599+ [ v4Operand ( 'age' , 40 ) ] ,
600+ ) ,
601+ ) . toEqual ( emails ( accounts . filter ( ( a ) => a . age > 40 ) ) )
602+ expect ( await emailsWhere ( 'ORDER BY eql_v4.ord_term_ore(age)' ) ) . toEqual (
603+ emails ( [ ...accounts ] . sort ( ( a , b ) => a . age - b . age ) ) ,
604+ )
605+ expect (
606+ await emailsWhere (
607+ 'WHERE surname > $1::jsonb::eql_v4.query_text_ord_ore ORDER BY id' ,
608+ [ v4Operand ( 'surname' , 'hopper' ) ] ,
609+ ) ,
610+ ) . toEqual ( emails ( accounts . filter ( ( a ) => a . surname > 'hopper' ) ) )
611+ expect ( await emailsWhere ( 'ORDER BY eql_v4.ord_term_ore(surname)' ) ) . toEqual (
612+ emails ( bySurname ) ,
613+ )
614+ expect (
615+ await emailsWhere (
616+ 'WHERE bio @@ $1::jsonb::eql_v4.query_text_match ORDER BY id' ,
617+ [ v4Operand ( 'bio' , 'programming' ) ] ,
618+ ) ,
619+ ) . toEqual ( emails ( accounts . filter ( ( a ) => a . bio . includes ( 'programming' ) ) ) )
461620
462- const res = await pg . query < { active : unknown } > (
463- `SELECT active::jsonb AS active FROM ${ TABLE } ORDER BY id` ,
621+ // And every renamed column decrypts, under the context it was written
622+ // with: the logical column name, so the rename needs no re-encryption.
623+ const res = await pg . query < Record < string , unknown > & { id : number } > (
624+ `SELECT id, ${ columnNames . map ( ( c ) => `${ c } ::jsonb AS ${ c } ` ) . join ( ', ' ) }
625+ FROM ${ TABLE } ORDER BY id` ,
464626 )
465627 expect (
466628 decryptV4 (
467- res . rows . map ( ( row ) => ( {
468- type : 'Boolean' as const ,
469- table : TABLE ,
470- column : 'active' ,
471- payload : row . active ,
472- } ) ) ,
629+ res . rows . flatMap ( ( row ) =>
630+ columnNames . map ( ( column ) => ( {
631+ type : columns [ column ] . v4 ,
632+ table : TABLE ,
633+ column,
634+ payload : row [ column ] ,
635+ } ) ) ,
636+ ) ,
473637 ) ,
474- ) . toEqual ( accounts . map ( ( a ) => a . active ) )
638+ ) . toEqual ( accounts . flatMap ( ( account ) => columnNames . map ( ( c ) => account [ c ] ) ) )
475639 } )
476640} )
0 commit comments