Skip to content

Commit 6ea438c

Browse files
committed
feat(go): test several members of one array element of a JsonSearch field
A filter such as "a contact who is a sibling and was called more than five times" had no Go method. An EqAt and an ordering at the contacts' paths can each be answered by a different contact, so the application had to read the array back and re-apply the predicate itself. The engine and EQL already answer it: element_match derives one eql_v4.query_json_elements operand, and json_tree_element_match is true of a row where one element passes every predicate. This wires it into the Go SDK. The EQL guest's resolver now names the elements domain, so the engine lists element_match (input path_predicates) on a field that orders a path, and the checker and stashgen read the operator. The compiled predicate domain stays unnamed: match is #1443's. stashgen groups the tag's ordered paths by the array their last wildcard names (record.TargetOptions.ElementArrays) and writes one accessor per array under Elements, named as At names a path, with Match. An array with no ordered member beneath it gets none, and a root array ($[*]) is Root. Two arrays whose keys give one name, or an array whose keys give none, get no accessor and a notice naming their ordered paths, so a tag that generated before still generates, with its At unchanged. Match takes encrypt.ElementPredicates made with MemberEq, MemberLt, MemberLte, MemberGt, MemberGte and MemberBetween, writes each value with gensupport.JSONText as the orderings do, and returns an eql.JSONSearchElementsQuery. The member check runs in the SDK, against the members the declaration orders beneath the array, before the guest is called: the engine's json_path_not_ordered names no member, so the first error a caller sees names the member, the array and the members it orders. It wraps ErrTerm, as the engine's refusal does; no predicates, a zero predicate and a member with a wildcard wrap ErrEncoding. Every other refusal is the engine's Diagnostic. Tests: the hermetic suite pins the operand against the subtree reads and the orderings of the same members, every pre-engine refusal, and each engine refusal's sentinel and code through the guest. The PostgreSQL suite (go:test:postgres) finds the row where one contact passes both predicates and not the row where two contacts each pass one, which the EqAt and ordering superset returns, and raises SQLSTATE 22023 past the bound. The guest test now pins the elements domain named and match not listed. Closes #1442 Claude-Session: https://claude.ai/code/session_01MrxG73fnX1p7oiizh3GRG5
1 parent 6109e38 commit 6ea438c

20 files changed

Lines changed: 1530 additions & 40 deletions

File tree

‎languages/golang/cmd/stashgen/main_test.go‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -442,6 +442,90 @@ func TestRunGeneratesJSONSearchFields(t *testing.T) {
442442
if !strings.Contains(written, "func (p ProfileFieldPath) Selector(ctx context.Context, c *encrypt.Cipher) (encrypt.JSONSelection[eql.JSONEntry], error)") {
443443
t.Error("an ordered path has no Selector of its own")
444444
}
445+
// The array with an ordered member gets an accessor under Elements,
446+
// named as At names its path, whose Match derives the element match.
447+
for _, want := range []string{
448+
"Elements ProfileFieldElements",
449+
"Visits ProfileFieldArray // $.visits[*]: at",
450+
`Visits: ProfileFieldArray{field: f, path: "$.visits[*]"},`,
451+
"func (a ProfileFieldArray) Match(ctx context.Context, c *encrypt.Cipher, predicates encrypt.ElementPredicates, opts encrypt.JSONReadOptions) (eql.JSONSearchElementsQuery, error)",
452+
"gensupport.JSONElementMatch[eql.JSONSearchElementsQuery](ctx, a.field, c, a.path, predicates, opts)",
453+
} {
454+
if !strings.Contains(written, want) {
455+
t.Errorf("the generated file lacks %q", want)
456+
}
457+
}
458+
if strings.Contains(written, "PrefsFieldElements") {
459+
t.Error("Prefs orders no member, and got Elements")
460+
}
461+
// The members of one array, named as At names their paths; an ordered
462+
// path that ends at its wildcard, or has none, is no array's member.
463+
written, stderr, code = generate("\tProfile json.RawMessage `stash:\"profile,encrypt_into=JsonSearch(ordered=$.extra.emergency_contacts[*].relationship;$.extra.emergency_contacts[*].previous_call_count;$.tags[*];$.age)\"`")
464+
if code != 0 {
465+
t.Fatalf("exit %d\n%s", code, stderr)
466+
}
467+
// gofmt aligns the struct fields: read each line's words.
468+
words := strings.Join(strings.Fields(written), " ")
469+
for _, want := range []string{
470+
"ExtraEmergencyContactsRelationship ProfileFieldPath // $.extra.emergency_contacts[*].relationship",
471+
"ExtraEmergencyContacts ProfileFieldArray // $.extra.emergency_contacts[*]: relationship, previous_call_count",
472+
} {
473+
if !strings.Contains(words, want) {
474+
t.Errorf("the generated file lacks %q", want)
475+
}
476+
}
477+
if strings.Contains(written, "Tags ProfileFieldArray") {
478+
t.Error("$.tags[*] orders the elements themselves, no member, and got an accessor under Elements")
479+
}
480+
// A field that orders no member beneath an array has no Elements.
481+
written, stderr, code = generate("\tProfile json.RawMessage `stash:\"profile,encrypt_into=JsonSearch(ordered=$.age;$.tags[*])\"`")
482+
if code != 0 {
483+
t.Fatalf("exit %d\n%s", code, stderr)
484+
}
485+
if strings.Contains(written, "Elements") || strings.Contains(written, "JSONElementMatch") {
486+
t.Error("a field with no ordered member beneath an array got Elements")
487+
}
488+
// A root array's accessor is Root, beside its paths' At accessors.
489+
written, stderr, code = generate("\tProfile json.RawMessage `stash:\"profile,encrypt_into=JsonSearch(ordered=$[*].name;$[*].age)\"`")
490+
if code != 0 {
491+
t.Fatalf("a root array: exit %d\n%s", code, stderr)
492+
}
493+
words = strings.Join(strings.Fields(written), " ")
494+
for _, want := range []string{
495+
"Name ProfileFieldPath // $[*].name",
496+
"Root ProfileFieldArray // $[*]: name, age",
497+
`Root: ProfileFieldArray{field: f, path: "$[*]"},`,
498+
} {
499+
if !strings.Contains(words, want) {
500+
t.Errorf("a root array: the generated file lacks %q", want)
501+
}
502+
}
503+
// Arrays whose keys give one accessor name, or none, get no accessor:
504+
// the tag still generates, At is as it is without them, and a notice
505+
// in the file and on stderr names the paths as the tag writes them.
506+
for _, c := range []struct{ name, ordered, at, notice string }{
507+
{"two arrays, one name", "$.a_b[*].x;$.aB[*].y;$.c[*].z", "ABX ProfileFieldPath // $.a_b[*].x", "no Elements accessor for the elements of $.a_b[*] (ordered $.a_b[*].x): another array's keys give the same accessor name, Elements.AB"},
508+
{"an array with no name", "$.__[*].x;$.c[*].z", "X ProfileFieldPath // $.__[*].x", "no Elements accessor for the elements of $.__[*] (ordered $.__[*].x): its keys give no Go accessor name"},
509+
} {
510+
written, stderr, code := generate("\tProfile json.RawMessage `stash:\"profile,encrypt_into=JsonSearch(ordered=" + c.ordered + ")\"`")
511+
if code != 0 {
512+
t.Fatalf("%s: exit %d\n%s", c.name, code, stderr)
513+
}
514+
// A wrapped comment reads as one line.
515+
words := strings.Join(strings.Fields(strings.ReplaceAll(written, "\n// ", " ")), " ")
516+
for _, want := range []string{c.at, "C ProfileFieldArray // $.c[*]: z", "Profile: " + c.notice} {
517+
if !strings.Contains(words, want) {
518+
t.Errorf("%s: the generated file lacks %q", c.name, want)
519+
}
520+
}
521+
if !strings.Contains(stderr, "stashgen: Customer.Profile: "+c.notice) {
522+
t.Errorf("%s: stderr = %q", c.name, stderr)
523+
}
524+
}
525+
written, _, code = generate("\tProfile json.RawMessage `stash:\"profile,encrypt_into=JsonSearch(ordered=$.a_b[*].x;$.aB[*].y)\"`")
526+
if code != 0 || strings.Contains(written, "ProfileFieldElements") || !strings.Contains(written, "ABY ProfileFieldPath // $.aB[*].y") {
527+
t.Errorf("two colliding arrays and no other: exit %d, Elements written or At changed", code)
528+
}
445529
// A json.RawMessage sealed any other way is bytes, as it was before
446530
// documents; an object key of digits names its accessor.
447531
written, stderr, code = generate("\tData json.RawMessage `stash:\"data,encrypt,index=equality\"`\n\tProfile json.RawMessage `stash:\"profile,encrypt_into=JsonSearch(ordered=$.a.1)\"`")
@@ -473,6 +557,19 @@ func TestRunGeneratesJSONSearchFields(t *testing.T) {
473557
if _, err := os.Stat(filepath.Join(dir, "customer_stash.go")); !os.IsNotExist(err) {
474558
t.Fatal("a file was written after the refusal")
475559
}
560+
// The same for an array whose elements have an ordered member, and an
561+
// engine that answers no element_match.
562+
dir = writeModule(t, map[string]string{"model.go": source("\tProfile json.RawMessage `stash:\"profile,encrypt_into=JsonSearch(ordered=$.visits[*].at)\"`")})
563+
stderrBuf.Reset()
564+
if code := run([]string{"-type", "Customer"}, dir, &bytes.Buffer{}, &stderrBuf, withoutOperator("element_match")); code != 1 {
565+
t.Fatalf("exit %d, want 1\n%s", code, stderrBuf.String())
566+
}
567+
if got := stderrBuf.String(); !strings.Contains(got, "Customer.Profile") || !strings.Contains(got, "the engine answers no element_match") {
568+
t.Fatalf("stderr = %q", got)
569+
}
570+
if _, err := os.Stat(filepath.Join(dir, "customer_stash.go")); !os.IsNotExist(err) {
571+
t.Fatal("a file was written after the refusal")
572+
}
476573
for _, c := range []struct{ name, fields, want string }{
477574
{"a sealed null", "\tPrefs *json.RawMessage `stash:\"prefs,encrypt_into=JsonSearch,null=sealed\"`", "Customer.Prefs: a nil has an equality term and no other, so another index's column would show which values are absent; write null=sql"},
478575
{"a nil document that does not say null=sql", "\tPrefs *json.RawMessage `stash:\"prefs,encrypt_into=JsonSearch\"`", "null=sql"},

‎languages/golang/encrypt/checker.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -258,7 +258,7 @@ func parseOperators(value any) ([]record.Operator, error) {
258258
op.Index, err = parseOperatorIndex(s)
259259
case "input":
260260
switch s {
261-
case "value", "pair", "json", "path_json", "path_pair", "path", "path_bound":
261+
case "value", "pair", "json", "path_json", "path_pair", "path", "path_bound", "path_predicates":
262262
op.Input = s
263263
default:
264264
return nil, fmt.Errorf("operator %d: unknown input %q", i, s)
@@ -282,14 +282,14 @@ func parseOperators(value any) ([]record.Operator, error) {
282282
// of the names the engine's OpSpec::parse accepts, the scalar operators
283283
// (eq, lt, lte, gt, gte, between, matches) and a document's (contains,
284284
// contained_by, eq_at, the orderings at a path, between_at, selector,
285-
// subtree, children, child_count). The catalog (se_targets) and se_query_term are
285+
// subtree, children, child_count, element_match). The catalog (se_targets) and se_query_term are
286286
// read through this, parseFidelity and parseOperatorIndex, so the two
287287
// readers refuse the same values.
288288
func parseOperatorName(s string) (string, error) {
289289
switch s {
290290
case "eq", "lt", "lte", "gt", "gte", "between", "matches",
291291
"contains", "contained_by", "eq_at", "lt_at", "lte_at", "gt_at", "gte_at", "between_at",
292-
"selector", "subtree", "children", "child_count":
292+
"selector", "subtree", "children", "child_count", "element_match":
293293
return s, nil
294294
}
295295
return "", fmt.Errorf("unknown operator %q", s)

‎languages/golang/encrypt/gensupport/json.go‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ import (
88
"errors"
99
"fmt"
1010
"reflect"
11+
"slices"
12+
"strings"
1113

1214
"github.com/cipherstash/stack/languages/golang/encrypt"
1315
"github.com/cipherstash/stack/languages/golang/internal/record"
@@ -241,6 +243,94 @@ func JSONBetweenAt[Q ~[]byte, T any](ctx context.Context, f Field[T], c *encrypt
241243
return encrypt.JSONRange[Q]{Low: Q(term.Terms[0]), High: Q(term.Terms[1])}, nil
242244
}
243245

246+
// JSONElementMatch derives element_match on a JSON field: the one operand
247+
// eql_v4.json_tree_element_match takes, which is true of a row where some
248+
// element of the array at path passes every predicate. path is an array
249+
// through its last wildcard ($.contacts[*]) that the field orders members
250+
// beneath ([record.TargetOptions.ElementArrays]).
251+
//
252+
// Refused before the engine is asked, each wrapping the sentinel the
253+
// engine's own refusal would: no predicates, or a zero ElementPredicate
254+
// ([encrypt.ErrEncoding]); a member with a wildcard ([encrypt.ErrEncoding]);
255+
// a member whose full path the field does not order ([encrypt.ErrTerm]),
256+
// naming the member, the array and the members it orders. A value is
257+
// written as [JSONText] writes it. What the engine refuses past these, a
258+
// value that is not a number or a string say, is its [encrypt.Diagnostic].
259+
// opts.WildcardBound 0 sends no bound, and the engine's default, 64,
260+
// applies.
261+
func JSONElementMatch[Q ~[]byte, T any](ctx context.Context, f Field[T], c *encrypt.Cipher, path string, predicates encrypt.ElementPredicates, opts encrypt.JSONReadOptions) (Q, error) {
262+
if f.err != nil {
263+
return nil, f.err
264+
}
265+
if len(predicates) == 0 {
266+
return nil, fmt.Errorf("%w: gensupport: %s: Match at %s needs at least one predicate: with none it is true of every row with an element", encrypt.ErrEncoding, f.name, path)
267+
}
268+
members := f.elementMembers(path)
269+
list := make([]any, len(predicates))
270+
for i, p := range predicates {
271+
member, op, values := p.Member(), p.Op(), p.Values()
272+
if op == "" {
273+
return nil, fmt.Errorf("%w: gensupport: %s: Match at %s: predicate %d is a zero ElementPredicate; make one with encrypt.MemberEq, MemberLt, MemberLte, MemberGt, MemberGte or MemberBetween", encrypt.ErrEncoding, f.name, path, i)
274+
}
275+
if strings.Contains(member, "[*]") {
276+
return nil, fmt.Errorf("%w: gensupport: %s: Match at %s: the member %q has a wildcard; a member names one field of each element (%s)", encrypt.ErrEncoding, f.name, path, member, orderedMembers(members))
277+
}
278+
if !slices.Contains(members, member) {
279+
return nil, fmt.Errorf("%w: gensupport: %s: Match at %s: the member %q is not ordered, so no element carries a term to compare (%s); order its full path in the field's tag, ordered=", encrypt.ErrTerm, f.name, path, member, orderedMembers(members))
280+
}
281+
entry := vcvalue.Object{{Key: "member", Value: member}, {Key: "op", Value: op}}
282+
// Each constructor gives its op its values: between the low and
283+
// the high, every other op one.
284+
keys := []string{"json"}
285+
if op == "between" {
286+
keys = []string{"low", "high"}
287+
}
288+
for j, key := range keys {
289+
text, err := f.jsonOperand(values[j])
290+
if err != nil {
291+
return nil, err
292+
}
293+
entry = append(entry, vcvalue.Field{Key: key, Value: text})
294+
}
295+
list[i] = entry
296+
}
297+
input := vcvalue.Object{{Key: "path", Value: path}}
298+
if opts.WildcardBound != 0 {
299+
input = append(input, vcvalue.Field{Key: "wildcard_bound", Value: opts.WildcardBound})
300+
}
301+
input = append(input, vcvalue.Field{Key: "predicates", Value: list})
302+
term, err := f.jsonTerm(ctx, c, "element_match", "eql", 1, input)
303+
if err != nil {
304+
return nil, err
305+
}
306+
return Q(term.Terms[0]), nil
307+
}
308+
309+
// orderedMembers says which members of an array's elements the field
310+
// orders.
311+
func orderedMembers(members []string) string {
312+
if len(members) == 0 {
313+
return "the field orders no member beneath this array"
314+
}
315+
return "the field orders " + strings.Join(members, ", ") + " beneath this array"
316+
}
317+
318+
// elementMembers are the members the field orders beneath the array at
319+
// path, as its declaration's ordered paths give them.
320+
func (f Field[T]) elementMembers(path string) []string {
321+
if f.plan == nil {
322+
return nil
323+
}
324+
if pf := f.plan.Field(f.name); pf != nil {
325+
for _, a := range pf.TargetOptions.ElementArrays() {
326+
if a.Path == path {
327+
return a.Members
328+
}
329+
}
330+
}
331+
return nil
332+
}
333+
244334
// JSONSelect prepares `->` at a path on a JSON field: the selector to
245335
// bind, and what opens the entries it returns. E is the field's entry type.
246336
func JSONSelect[E ~[]byte, T any](ctx context.Context, f Field[T], c *encrypt.Cipher, path string) (encrypt.JSONSelection[E], error) {

‎languages/golang/encrypt/guest/src/targets.rs‎

Lines changed: 44 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -114,10 +114,17 @@ fn describe(target: &eql_bindings::encryption::targets::Target) -> TargetDescrip
114114
Some(options) => descriptor.with_match_options(options),
115115
None => descriptor,
116116
};
117-
// A document's orderings at a path bind as its leaf operand.
118-
match target.leaf_query_sql_domain() {
117+
// A document's orderings at a path bind as its leaf operand, and its
118+
// element predicate (`element_match`) as its elements operand. Its
119+
// compiled predicate (`match`) is not named: the Go SDK does not take
120+
// that operator yet.
121+
let descriptor = match target.leaf_query_sql_domain() {
119122
Some(domain) => descriptor.with_leaf_query_sql_domain(domain),
120123
None => descriptor,
124+
};
125+
match target.elements_query_sql_domain() {
126+
Some(domain) => descriptor.with_elements_query_sql_domain(domain),
127+
None => descriptor,
121128
}
122129
}
123130

@@ -270,13 +277,14 @@ mod tests {
270277

271278
/// The JSON document's query domains this guest names, and so the JSON
272279
/// operators the Go SDK is offered: the leaf operand, for the orderings
273-
/// at a path, and not the element-scoped or compiled predicate, whose
274-
/// operators (`element_match`, `match`) the Go side does not parse yet.
275-
/// Naming one is a change to the Go SDK's operator list: update this
276-
/// test with the Go side that takes the operator.
280+
/// at a path, and the element predicate, for `element_match`; not the
281+
/// compiled predicate, whose operator (`match`) the Go side does not
282+
/// parse yet. Naming it is a change to the Go SDK's operator list:
283+
/// update this test with the Go side that takes the operator.
277284
#[cfg(feature = "eql")]
278285
#[test]
279-
fn a_document_names_the_leaf_domain_and_not_the_predicate_domains() {
286+
fn a_document_names_the_leaf_and_element_domains_and_not_the_predicate_domain() {
287+
use stack_encrypt::dynamic::{self, record, OpSpec};
280288
let targets = resolver().targets();
281289
let document = targets
282290
.iter()
@@ -286,14 +294,36 @@ mod tests {
286294
document.leaf_query_sql_domain.as_deref(),
287295
Some("eql_v4.query_json_leaf")
288296
);
289-
assert_eq!(document.elements_query_sql_domain, None);
297+
assert_eq!(
298+
document.elements_query_sql_domain.as_deref(),
299+
Some("eql_v4.query_json_elements")
300+
);
290301
assert_eq!(document.predicate_query_sql_domain, None);
291-
let operators = document.operators();
292-
for absent in [
293-
stack_encrypt::dynamic::OpSpec::ElementMatch,
294-
stack_encrypt::dynamic::OpSpec::Match,
295-
] {
296-
assert!(!operators.contains(&absent), "{absent:?}: {operators:?}");
302+
// The operators a field answers: the orderings and `element_match`
303+
// only where it orders a path, and `match` nowhere.
304+
let field_operators = |ordered: &[&str]| {
305+
let wire = serde_json::json!({
306+
"profile": {
307+
"context": ["users", "profile"],
308+
"type": "json",
309+
"null": "sql",
310+
"target": {"name": "JsonSearch", "options": {"ordered": ordered}}
311+
}
312+
});
313+
let frame = dynamic::frame::serde::to_vec(&wire).expect("plan frame");
314+
let plan = record::plan_with(&frame, &resolver()).expect("plan");
315+
dynamic::field_operators(&plan, "profile", &resolver())
316+
.expect("operators")
317+
.into_iter()
318+
.map(|(op, _)| op)
319+
.collect::<Vec<_>>()
320+
};
321+
let ordered = field_operators(&["$.contacts[*].relationship"]);
322+
assert!(ordered.contains(&OpSpec::ElementMatch), "{ordered:?}");
323+
assert!(!ordered.contains(&OpSpec::Match), "{ordered:?}");
324+
let unordered = field_operators(&[]);
325+
for absent in [OpSpec::ElementMatch, OpSpec::Match] {
326+
assert!(!unordered.contains(&absent), "{absent:?}: {unordered:?}");
297327
}
298328
}
299329

‎languages/golang/encrypt/internal/testusers/customer_stash.go‎

Lines changed: 50 additions & 4 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)