Skip to content

Commit 94db05b

Browse files
committed
test(eql): run compiled predicates over Stack Encrypt documents in PostgreSQL
Documents sealed by Stack Encrypt's JSON index, predicates compiled by `match` through a data plan over eql-bindings' target table, and `eql_v4.json_tree_match` and `@@` over the column, row by row against PostgreSQL's `jsonb_path_match` over the plaintexts, NULL included, and against the WHERE written from the operands each leaf's own operator derives. Refs #1441
1 parent ce38dcd commit 94db05b

3 files changed

Lines changed: 296 additions & 7 deletions

File tree

‎packages/eql/tests/encryption/tests/json_tree.rs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -515,6 +515,11 @@ mod generated;
515515
#[path = "json_tree/elements.rs"]
516516
mod elements;
517517

518+
/// The compiled predicate, through a data plan, against the column and
519+
/// against `jsonb_path_match` over the plaintexts.
520+
#[path = "json_tree/predicate.rs"]
521+
mod predicate;
522+
518523
/// Needs an EQL v4 installation. Extraction by selector opens to the leaf;
519524
/// the subtree walk by a client-derived token returns exactly the nodes
520525
/// under it and reassembles; the array functions count and list elements;

‎packages/eql/tests/encryption/tests/json_tree/elements.rs‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,10 @@ use stack_encrypt::dynamic::{
1111
};
1212
use stack_encrypt::{KeysetCipher, Label, Pending, StackCipher};
1313

14-
/// eql-bindings' table as the engine's resolver, naming each type's leaf
15-
/// and element predicate domains, as a host that answers `element_match`
16-
/// describes it.
17-
struct Eql;
14+
/// eql-bindings' table as the engine's resolver, naming each type's leaf,
15+
/// element predicate and compiled predicate domains, as a host that
16+
/// answers `element_match` and `match` describes it.
17+
pub(super) struct Eql;
1818

1919
fn describe(target: &Target) -> TargetDescriptor {
2020
let mut descriptor = TargetDescriptor::new(
@@ -35,6 +35,9 @@ fn describe(target: &Target) -> TargetDescriptor {
3535
if let Some(domain) = target.elements_query_sql_domain() {
3636
descriptor = descriptor.with_elements_query_sql_domain(domain);
3737
}
38+
if let Some(domain) = target.predicate_query_sql_domain() {
39+
descriptor = descriptor.with_predicate_query_sql_domain(domain);
40+
}
3841
descriptor
3942
}
4043

@@ -101,13 +104,13 @@ fn contact_plan() -> record::Plan {
101104
record::plan_with(&dynamic::frame::serde::to_vec(&wire).unwrap(), &Eql).unwrap()
102105
}
103106

104-
fn frame_of(value: &Value) -> ValueFrame {
107+
pub(super) fn frame_of(value: &Value) -> ValueFrame {
105108
ValueFrame::from_bytes(&dynamic::frame::serde::to_vec(value).unwrap()).unwrap()
106109
}
107110

108111
/// The operand of `operator` on `profile` for `input`, as the SQL binds
109112
/// it: an EQL operand's JSON text, or a list of tokens.
110-
fn derive(
113+
pub(super) fn derive(
111114
runtime: &tokio::runtime::Runtime,
112115
keyset: &KeysetCipher<'_, common::Observed>,
113116
plan: &record::Plan,
@@ -130,7 +133,7 @@ fn derive(
130133
term.operand().clone()
131134
}
132135

133-
fn eql_text(operand: &Operand) -> String {
136+
pub(super) fn eql_text(operand: &Operand) -> String {
134137
match operand {
135138
Operand::One(bytes) => String::from_utf8(bytes.clone()).unwrap(),
136139
other => panic!("one EQL operand, got {other:?}"),
Lines changed: 281 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,281 @@
1+
//! The compiled predicate end to end: documents sealed by Stack Encrypt's
2+
//! JSON index, jsonpath predicates compiled by the `match` operator through
3+
//! a data plan over eql-bindings' target table (the path the Go guest
4+
//! takes), and `eql_v4.json_tree_match` and `@@` over the stored column,
5+
//! against PostgreSQL's own `jsonb_path_match` over the plaintexts and
6+
//! against the WHERE a hand writes from the operators the leaves stand for.
7+
8+
use super::elements::{derive, eql_text, frame_of, Eql};
9+
use super::*;
10+
11+
use stack_encrypt::dynamic::{self, record, OpSpec, Operand};
12+
use stack_encrypt::KeysetCipher;
13+
14+
/// The paths the documents order: what the predicates' ranges ask at.
15+
const ORDERED: [&str; 4] = [
16+
"$.age",
17+
"$.name",
18+
"$.contacts[*].calls",
19+
"$.contacts[*].relationship",
20+
];
21+
22+
fn people_options() -> JsonOptions {
23+
JsonOptions::default().ordering(ORDERED).unwrap()
24+
}
25+
26+
/// `profile` under `users/profile`, the column the documents are sealed
27+
/// under, naming `JsonSearch` with [`people_options`].
28+
fn people_plan() -> record::Plan {
29+
let wire = json!({
30+
"profile": {
31+
"context": ["users", "profile"],
32+
"type": "json",
33+
"null": "sql",
34+
"target": {"name": "JsonSearch", "options": {"ordered": ORDERED}}
35+
}
36+
});
37+
record::plan_with(&dynamic::frame::serde::to_vec(&wire).unwrap(), &Eql).unwrap()
38+
}
39+
40+
/// Documents with every field absent, `null` or a value of the one type its
41+
/// path holds, and arrays only where the predicates name `[*]`.
42+
fn people() -> Vec<Value> {
43+
vec![
44+
json!({"age": 34, "name": "ann", "tags": ["admin", "ops"],
45+
"contacts": [{"relationship": "sibling", "calls": 3},
46+
{"relationship": "friend", "calls": 7}]}),
47+
json!({"age": 17, "name": "bob",
48+
"contacts": [{"relationship": "friend", "calls": 9},
49+
{"relationship": "sibling", "calls": 7}]}),
50+
json!({"age": null, "name": "cat", "tags": []}),
51+
json!({"name": null, "contacts": []}),
52+
json!({}),
53+
json!({"age": 60, "tags": ["ops"], "contacts": [{"relationship": "sibling"}]}),
54+
json!({"age": 18, "name": "dan", "ok": true}),
55+
]
56+
}
57+
58+
/// Each row's answer under `sql`, which takes `params`, NULL as `None`.
59+
fn answers(
60+
db: &mut postgres::Client,
61+
sql: &str,
62+
params: &[&(dyn postgres::types::ToSql + Sync)],
63+
) -> Vec<(i32, Option<bool>)> {
64+
db.query(sql, params)
65+
.unwrap_or_else(|error| panic!("{sql}: {error}"))
66+
.iter()
67+
.map(|row| (row.get(0), row.get(1)))
68+
.collect()
69+
}
70+
71+
/// `match`'s operand for `predicate` under `vars`.
72+
fn compiled(
73+
runtime: &tokio::runtime::Runtime,
74+
keyset: &KeysetCipher<'_, common::Observed>,
75+
plan: &record::Plan,
76+
predicate: &str,
77+
vars: &Value,
78+
) -> String {
79+
eql_text(&derive(
80+
runtime,
81+
keyset,
82+
plan,
83+
"match",
84+
json!({"predicate": predicate, "vars": vars.to_string(), "wildcard_bound": 4}),
85+
))
86+
}
87+
88+
/// Needs an EQL v4 installation. For each predicate, `json_tree_match` and
89+
/// `@@` over the encrypted column answer what `jsonb_path_match` answers of
90+
/// the same predicate and variables over the plaintexts, row by row, NULL
91+
/// included: equality, `!=` and the orderings with absent paths and JSON
92+
/// `null`s, wildcards, filters, and `&&`, `||` and `!` over them.
93+
#[test]
94+
#[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"]
95+
fn postgres_a_compiled_predicate_answers_as_jsonb_path_match() {
96+
let runtime = tokio::runtime::Builder::new_current_thread()
97+
.enable_all()
98+
.build()
99+
.unwrap();
100+
let (cipher, _) = wait(&runtime, common::cipher());
101+
let keyset = cipher.default_keyset();
102+
let plan = people_plan();
103+
let mut db = connect();
104+
store(
105+
&mut db,
106+
&runtime,
107+
&cipher,
108+
"json_tree_people",
109+
&people_options(),
110+
&people(),
111+
);
112+
let vars = json!({"min": 18, "who": "ann", "r": "sibling", "n": 5, "tag": "ops"});
113+
for predicate in [
114+
"$.age >= $min",
115+
"!($.age >= $min)",
116+
"$.age < 18",
117+
"$.age == null",
118+
"$.age != null",
119+
"$.age != 34",
120+
"$.name == $who",
121+
"$.name != $who",
122+
"$.name > \"bob\"",
123+
"!($.name <= \"bob\")",
124+
"$.ok == true",
125+
"exists($.tags)",
126+
"exists($.tags[*])",
127+
"$.tags[*] == $tag",
128+
"exists($.tags[*] ? (@ == $tag))",
129+
"$.contacts[*].calls > $n",
130+
"$.contacts[*].relationship == $r",
131+
"exists($.contacts[*] ? (@.relationship == $r))",
132+
"exists($.contacts[*] ? (@.relationship == $r && @.calls > $n))",
133+
"!exists($.contacts[*] ? (@.relationship == $r && @.calls > $n))",
134+
"$.age >= $min && exists($.contacts[*] ? (@.relationship == $r && @.calls > $n))",
135+
"$.name == $who || $.tags[*] == $tag",
136+
"!($.age < $min || $.name == null) && exists($.name)",
137+
"$.age > 20 && $.age < 50",
138+
] {
139+
let operand = compiled(&runtime, &keyset, &plan, predicate, &vars);
140+
let matched = answers(
141+
&mut db,
142+
"SELECT id, eql_v4.json_tree_match(doc, $1::text::jsonb::eql_v4.query_json_predicate) \
143+
FROM json_tree_people ORDER BY id",
144+
&[&operand],
145+
);
146+
let by_operator = answers(
147+
&mut db,
148+
"SELECT id, doc @@ $1::text::jsonb::eql_v4.query_json_predicate \
149+
FROM json_tree_people ORDER BY id",
150+
&[&operand],
151+
);
152+
let plaintext = answers(
153+
&mut db,
154+
"SELECT id, jsonb_path_match(plain, $1::text::jsonpath, $2::text::jsonb) \
155+
FROM json_tree_people ORDER BY id",
156+
&[&predicate, &vars.to_string()],
157+
);
158+
assert_eq!(matched, plaintext, "{predicate}: against jsonb_path_match");
159+
assert_eq!(by_operator, matched, "{predicate}: @@ is json_tree_match");
160+
}
161+
}
162+
163+
/// Needs an EQL v4 installation. A compiled predicate answers what the
164+
/// hand-written WHERE answers, NULL included, each leaf by the operand its
165+
/// own operator derives: `json_tree_path_exists` for the path, `@>` for
166+
/// the `null` needle and the equalities, `->` and `>=` for the ordering,
167+
/// and `json_tree_element_match` for the filter. And what `match` refuses
168+
/// it refuses before anything is derived, at the offset of the construct.
169+
#[test]
170+
#[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"]
171+
fn postgres_a_compiled_predicate_answers_as_its_operators() {
172+
let runtime = tokio::runtime::Builder::new_current_thread()
173+
.enable_all()
174+
.build()
175+
.unwrap();
176+
let (cipher, _) = wait(&runtime, common::cipher());
177+
let keyset = cipher.default_keyset();
178+
let plan = people_plan();
179+
let mut db = connect();
180+
store(
181+
&mut db,
182+
&runtime,
183+
&cipher,
184+
"json_tree_operators",
185+
&people_options(),
186+
&people(),
187+
);
188+
let one =
189+
|operator: &str, input: Value| eql_text(&derive(&runtime, &keyset, &plan, operator, input));
190+
let Operand::Text(age) = derive(
191+
&runtime,
192+
&keyset,
193+
&plan,
194+
"selector",
195+
json!({"path": "$.age"}),
196+
) else {
197+
panic!("a selector is text")
198+
};
199+
let null_age = one("eq_at", json!({"path": "$.age", "json": "null"}));
200+
let at_least = one("gte_at", json!({"path": "$.age", "json": "18"}));
201+
let sibling_over_five = one(
202+
"element_match",
203+
json!({"path": "$.contacts[*]", "wildcard_bound": 4, "predicates": [
204+
{"member": "relationship", "op": "eq", "json": "\"sibling\""},
205+
{"member": "calls", "op": "gt", "json": "5"},
206+
]}),
207+
);
208+
let ann = one("eq_at", json!({"path": "$.name", "json": "\"ann\""}));
209+
210+
let predicate =
211+
"!($.age >= 18 && exists($.contacts[*] ? (@.relationship == \"sibling\" && @.calls > 5))) \
212+
|| $.name == \"ann\"";
213+
let operand = compiled(&runtime, &keyset, &plan, predicate, &json!({}));
214+
let matched = answers(
215+
&mut db,
216+
"SELECT id, eql_v4.json_tree_match(doc, $1::text::jsonb::eql_v4.query_json_predicate) \
217+
FROM json_tree_operators ORDER BY id",
218+
&[&operand],
219+
);
220+
let written = answers(
221+
&mut db,
222+
"SELECT id, (NOT (eql_v4.json_tree_path_exists(doc, $1::text) \
223+
AND NOT (doc @> $2::text::jsonb::eql_v4.query_json) \
224+
AND (doc -> $1::text) >= $3::text::jsonb::eql_v4.query_json_leaf \
225+
AND eql_v4.json_tree_element_match(doc, $4::text::jsonb::eql_v4.query_json_elements))) \
226+
OR doc @> $5::text::jsonb::eql_v4.query_json \
227+
FROM json_tree_operators ORDER BY id",
228+
&[&age, &null_age, &at_least, &sibling_over_five, &ann],
229+
);
230+
assert_eq!(
231+
matched, written,
232+
"the tree against the operators written out"
233+
);
234+
let plaintext = answers(
235+
&mut db,
236+
"SELECT id, jsonb_path_match(plain, $1::text::jsonpath) FROM json_tree_operators ORDER BY id",
237+
&[&predicate],
238+
);
239+
assert_eq!(matched, plaintext, "and against jsonb_path_match");
240+
241+
for (text, reason, at) in [
242+
("$.age + 1 > 18", "json_predicate_unsupported", 6),
243+
("$.name like_regex \"^a\"", "json_predicate_unsupported", 7),
244+
("$.age.size() > 1", "json_predicate_unsupported", 5),
245+
(
246+
"$.tags[*] != \"ops\"",
247+
"json_predicate_not_equal_at_wildcard",
248+
10,
249+
),
250+
("$.age > $missing", "json_predicate_unknown_variable", 8),
251+
] {
252+
let error = dynamic::check_query(
253+
&plan,
254+
"profile",
255+
OpSpec::Match,
256+
frame_of(&json!({"predicate": text})),
257+
&Eql,
258+
)
259+
.unwrap_err();
260+
let payload = stack_encrypt::ErrorPayload::payload(&error);
261+
assert_eq!(payload["reason"], json!(reason), "{text}: {payload:?}");
262+
assert_eq!(payload["at"], json!(at), "{text}: {payload:?}");
263+
}
264+
let error = dynamic::check_query(
265+
&plan,
266+
"profile",
267+
OpSpec::Match,
268+
frame_of(&json!({"predicate": "$.height > 1"})),
269+
&Eql,
270+
)
271+
.unwrap_err();
272+
assert!(
273+
matches!(
274+
error,
275+
dynamic::Error::Cipher(stack_encrypt::Error::Term(
276+
stack_encrypt::sem::TermError::JsonPathNotOrdered
277+
))
278+
),
279+
"a range at a path the column does not order: {error:?}"
280+
);
281+
}

0 commit comments

Comments
 (0)