Skip to content

Commit 58ef9f1

Browse files
committed
fix(cypher): expansion materializes every matched row — the cap falsified aggregates
Second mechanism of DeusData#1196, exposed by the reporter's v0.10.6 retest: every relationship-expansion site capped its output buffer at bind_cap*10, so edges past the cap were silently dropped BEFORE WHERE and aggregation ever ran. count() then reported the scanned prefix as if it were a fact — 9,360 of 13,691 DEFINES field-measured at --max-rows 1000 — and a LABEL on the source did not protect you (the label workaround only ever fixed source enumeration, which DeusData#1323 already made exact). max_rows is an output-row limit per the public header; projection already enforces it. All five capped sites now share one growable append (geometric growth, size_t sizing): the per-hop expansion, its fixed/variable-length helpers and process_edges, the bound-terminal driver, and the cross-join outer buffer. Only allocation failure stops materialisation; match_count stays truthful either way, so the DeusData#627 OPTIONAL contracts hold (a saturated buffer can no longer exist, and the fallback rows share the same append). The DeusData#601 deadline still bounds pathological time, and hop caps keep bounding depth — this removes only the silent row-dropping. Regression test: 2 labeled sources x 30 edges with max_rows=2 — the old cap returned count=20; ground truth 60 now holds, and the list form returns exactly max_rows rows. Proven RED before and RED again on revert; cypher 181/181 and mcp suites green, including every OPTIONAL/DeusData#627 semantics test. Fixes DeusData#1196 (together with DeusData#1323, which fixed the unlabeled source scan). Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
1 parent 07cac7d commit 58ef9f1

2 files changed

Lines changed: 164 additions & 85 deletions

File tree

‎src/cypher/cypher.c‎

Lines changed: 77 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ enum {
2525
CYP_MAX_VARS = 16, /* max Cypher variables in a query */
2626
CYP_MAX_EDGE_VARS = 8, /* max edge variables */
2727
CYP_GROWTH_10 = 10, /* binding growth factor */
28+
CYP_GROWTH_2 = 2, /* geometric buffer growth (#1196) */
2829
CYP_CHAR_IDX1 = 1, /* second character index (e.g. op[1]) */
2930
CYP_EBUF_MASK = 7,
3031
CYP_NODE_COLS = 4, /* columns per node var: name, qn, label, file */
@@ -3101,10 +3102,33 @@ static void scan_pattern_nodes(cbm_store_t *store, const char *project, cbm_node
31013102

31023103
/* Process edges: look up target node, filter by label/props, add binding.
31033104
* `inbound` controls which end of the edge is the target id. */
3105+
/* #1196 (second mechanism): a hop's output buffer must hold EVERY matched
3106+
* row. max_rows is an OUTPUT-row limit — projection already enforces it — and
3107+
* WHERE/aggregation run after expansion, so any cap here silently falsifies
3108+
* results: field-measured, count() reported 9,360 of 13,691 DEFINES because
3109+
* the old bind_cap*10 ceiling dropped edges before aggregation ever saw them,
3110+
* and a labeled source did not save you. The buffer now grows geometrically;
3111+
* only allocation failure stops materialisation (the #601 deadline still
3112+
* bounds time), and match_count stays truthful either way (#627 contract). */
3113+
static bool binding_out_append(binding_t **rows, int *count, int *cap, binding_t *nb) {
3114+
if (*count == *cap) {
3115+
int next = *cap > 0 ? *cap * CYP_GROWTH_2 : CYP_GROWTH_10;
3116+
binding_t *grown = realloc(*rows, (size_t)next * sizeof(binding_t));
3117+
if (!grown) {
3118+
binding_free(nb);
3119+
return false;
3120+
}
3121+
*rows = grown;
3122+
*cap = next;
3123+
}
3124+
(*rows)[(*count)++] = *nb;
3125+
return true;
3126+
}
3127+
31043128
static void process_edges(cbm_store_t *store, cbm_edge_t *edges, int edge_count, bool inbound,
31053129
const cbm_node_pattern_t *target_node, binding_t *b, const char *to_var,
3106-
const char *rel_var, binding_t *new_bindings, int *new_count, int max_new,
3107-
int *match_count) {
3130+
const char *rel_var, binding_t **new_bindings, int *new_count,
3131+
int *new_cap, int *match_count) {
31083132
/* When the terminal node variable is ALREADY bound (e.g. the second pattern
31093133
* `(c)-[:CALLS]->(f)` where `f` came from an earlier MATCH), we must FILTER
31103134
* to edges that actually reach the bound node — not overwrite the caller's
@@ -3137,16 +3161,14 @@ static void process_edges(cbm_store_t *store, cbm_edge_t *edges, int edge_count,
31373161
node_fields_free(&found);
31383162
continue;
31393163
}
3140-
(*match_count)++; /* a real neighbour exists, budget or not */
3141-
if (*new_count < max_new) {
3142-
binding_t nb = {0};
3143-
binding_copy(&nb, b);
3144-
binding_set(&nb, to_var, &found);
3145-
if (rel_var) {
3146-
binding_set_edge(&nb, rel_var, &edges[ei]);
3147-
}
3148-
new_bindings[(*new_count)++] = nb;
3164+
(*match_count)++; /* a real neighbour exists, OOM or not */
3165+
binding_t nb = {0};
3166+
binding_copy(&nb, b);
3167+
binding_set(&nb, to_var, &found);
3168+
if (rel_var) {
3169+
binding_set_edge(&nb, rel_var, &edges[ei]);
31493170
}
3171+
(void)binding_out_append(new_bindings, new_count, new_cap, &nb);
31503172
node_fields_free(&found);
31513173
}
31523174
}
@@ -3161,8 +3183,8 @@ static _Thread_local int g_cypher_depth_clamped = 0;
31613183

31623184
static void expand_var_length(cbm_store_t *store, cbm_rel_pattern_t *rel,
31633185
cbm_node_pattern_t *target_node, binding_t *b, cbm_node_t *src,
3164-
const char *to_var, binding_t *new_bindings, int *new_count,
3165-
int max_new, int *match_count) {
3186+
const char *to_var, binding_t **new_bindings, int *new_count,
3187+
int *new_cap, int *match_count) {
31663188
/* Clamp BOTH the explicit (`*1..N`) and unbounded (`*`, `*..m`) forms to the
31673189
* engine ceiling: an explicit N above the cap was previously honoured
31683190
* verbatim, driving cbm_store_bfs to an unbounded hop count (#887). WARN on
@@ -3201,21 +3223,19 @@ static void expand_var_length(cbm_store_t *store, cbm_rel_pattern_t *rel,
32013223
continue;
32023224
}
32033225
(*match_count)++;
3204-
if (*new_count < max_new) {
3205-
binding_t nb = {0};
3206-
binding_copy(&nb, b);
3207-
binding_set(&nb, to_var, &hop->node);
3208-
new_bindings[(*new_count)++] = nb;
3209-
}
3226+
binding_t nb = {0};
3227+
binding_copy(&nb, b);
3228+
binding_set(&nb, to_var, &hop->node);
3229+
(void)binding_out_append(new_bindings, new_count, new_cap, &nb);
32103230
}
32113231
cbm_store_traverse_free(&tr);
32123232
}
32133233

32143234
/* Expand fixed-length (1-hop) relationship edges */
32153235
static void expand_fixed_length(cbm_store_t *store, cbm_rel_pattern_t *rel,
32163236
cbm_node_pattern_t *target_node, binding_t *b, cbm_node_t *src,
3217-
const char *to_var, binding_t *new_bindings, int *new_count,
3218-
int max_new, int *match_count) {
3237+
const char *to_var, binding_t **new_bindings, int *new_count,
3238+
int *new_cap, int *match_count) {
32193239
bool is_inbound = rel->direction && strcmp(rel->direction, "inbound") == 0;
32203240
bool is_any = rel->direction && strcmp(rel->direction, "any") == 0;
32213241
const char *rel_var = rel->variable;
@@ -3232,7 +3252,7 @@ static void expand_fixed_length(cbm_store_t *store, cbm_rel_pattern_t *rel,
32323252
&edge_count);
32333253
}
32343254
process_edges(store, edges, edge_count, is_inbound, target_node, b, to_var, rel_var,
3235-
new_bindings, new_count, max_new, match_count);
3255+
new_bindings, new_count, new_cap, match_count);
32363256
cbm_store_free_edges(edges, edge_count);
32373257
}
32383258
if (is_any) {
@@ -3242,7 +3262,7 @@ static void expand_fixed_length(cbm_store_t *store, cbm_rel_pattern_t *rel,
32423262
cbm_store_find_edges_by_target_type(store, src->id, rel->types[ti], &edges,
32433263
&edge_count);
32443264
process_edges(store, edges, edge_count, true, target_node, b, to_var, rel_var,
3245-
new_bindings, new_count, max_new, match_count);
3265+
new_bindings, new_count, new_cap, match_count);
32463266
cbm_store_free_edges(edges, edge_count);
32473267
}
32483268
}
@@ -3255,21 +3275,21 @@ static void expand_fixed_length(cbm_store_t *store, cbm_rel_pattern_t *rel,
32553275
cbm_store_find_edges_by_source(store, src->id, &edges, &edge_count);
32563276
}
32573277
process_edges(store, edges, edge_count, is_inbound, target_node, b, to_var, rel_var,
3258-
new_bindings, new_count, max_new, match_count);
3278+
new_bindings, new_count, new_cap, match_count);
32593279
cbm_store_free_edges(edges, edge_count);
32603280
if (is_any) {
32613281
edges = NULL;
32623282
edge_count = 0;
32633283
cbm_store_find_edges_by_target(store, src->id, &edges, &edge_count);
32643284
process_edges(store, edges, edge_count, true, target_node, b, to_var, rel_var,
3265-
new_bindings, new_count, max_new, match_count);
3285+
new_bindings, new_count, new_cap, match_count);
32663286
cbm_store_free_edges(edges, edge_count);
32673287
}
32683288
}
32693289
}
32703290

32713291
static void expand_pattern_rels(cbm_store_t *store, cbm_pattern_t *pat, binding_t **bindings,
3272-
int *bind_count, const int *bind_cap, const char **var_name,
3292+
int *bind_count, int *bind_cap, const char **var_name,
32733293
bool is_optional) {
32743294
for (int ri = 0; ri < pat->rel_count; ri++) {
32753295
/* #601: stop expanding further hops once the wall-clock budget is spent
@@ -3283,16 +3303,14 @@ static void expand_pattern_rels(cbm_store_t *store, cbm_pattern_t *pat, binding_
32833303

32843304
bool is_variable_length = (rel->min_hops != SKIP_ONE || rel->max_hops != SKIP_ONE);
32853305

3286-
/* Size this hop's output for BOTH writers without dropping any row: the
3287-
* expansion helpers emit at most max_new = bind_cap*10 rows (they stop at
3288-
* max_new), and the OPTIONAL fallback emits at most one row per source
3289-
* (<= *bind_count). A source either matches (feeds the expansion) or takes
3290-
* the fallback, never both, so the two counts are additive and bounded by
3291-
* max_new + *bind_count. Computed in size_t so the product cannot overflow.
3292-
* The previous "+ 1" sizing fit only a SINGLE fallback row after a
3293-
* saturated expansion; a second one ran off the end (heap OOB, CWE-787). */
3294-
size_t alloc_n = (size_t)*bind_cap * (size_t)CYP_GROWTH_10 + (size_t)*bind_count;
3295-
binding_t *new_bindings = malloc(alloc_n * sizeof(binding_t));
3306+
/* #1196: the hop's output buffer GROWS to hold every matched row —
3307+
* the old bind_cap*10 ceiling silently dropped edges before WHERE and
3308+
* aggregation, falsifying counts. binding_out_append handles growth
3309+
* and the OPTIONAL fallback shares it, so no writer can run off the
3310+
* end and no row class is dropped (the old fixed sizing had exactly
3311+
* those two failure modes, CWE-787 and the OPTIONAL data loss). */
3312+
int new_cap = *bind_count > 0 ? *bind_count : CYP_GROWTH_10;
3313+
binding_t *new_bindings = malloc((size_t)new_cap * sizeof(binding_t));
32963314
if (!new_bindings) {
32973315
return; /* OOM: leave existing bindings untouched rather than corrupt */
32983316
}
@@ -3310,24 +3328,22 @@ static void expand_pattern_rels(cbm_store_t *store, cbm_pattern_t *pat, binding_
33103328

33113329
int match_count = 0;
33123330

3313-
int max_new = *bind_cap * CYP_GROWTH_10;
33143331
if (is_variable_length) {
3315-
expand_var_length(store, rel, target_node, b, src, to_var, new_bindings, &new_count,
3316-
max_new, &match_count);
3332+
expand_var_length(store, rel, target_node, b, src, to_var, &new_bindings,
3333+
&new_count, &new_cap, &match_count);
33173334
} else {
3318-
expand_fixed_length(store, rel, target_node, b, src, to_var, new_bindings,
3319-
&new_count, max_new, &match_count);
3335+
expand_fixed_length(store, rel, target_node, b, src, to_var, &new_bindings,
3336+
&new_count, &new_cap, &match_count);
33203337
}
33213338

33223339
/* OPTIONAL MATCH: no expansion for this source, so keep the binding
3323-
* with the target unbound (projection renders it ""). The buffer is
3324-
* sized max_new + *bind_count precisely so every such fallback row has
3325-
* a slot — no guard needed, and no OPTIONAL no-match row is dropped. */
3340+
* with the target unbound (projection renders it ""). The shared
3341+
* growable append gives every fallback row a slot. */
33263342
if (is_optional && match_count == 0) {
33273343
binding_t nb = {0};
33283344
binding_copy(&nb, b);
33293345
/* Don't set to_var — it remains unbound; projection returns "" */
3330-
new_bindings[new_count++] = nb;
3346+
(void)binding_out_append(&new_bindings, &new_count, &new_cap, &nb);
33313347
}
33323348
}
33333349

@@ -3337,6 +3353,7 @@ static void expand_pattern_rels(cbm_store_t *store, cbm_pattern_t *pat, binding_
33373353
free(*bindings);
33383354
*bindings = new_bindings;
33393355
*bind_count = new_count;
3356+
*bind_cap = new_cap;
33403357
*var_name = to_var;
33413358
}
33423359
}
@@ -4577,8 +4594,8 @@ static void cross_join_with_rels(cbm_store_t *store, cbm_pattern_t *patn, bindin
45774594
* graphs (e.g. an unbound `c` scanned against ~29 K `f` bindings), wrapping
45784595
* the int product negative and yielding a tiny/garbage malloc → heap OOB
45794596
* write → SIGSEGV/SIGABRT (#627). */
4580-
size_t alloc_n = (size_t)*bind_count * (size_t)extra_count * (size_t)CYP_GROWTH_10 + SKIP_ONE;
4581-
binding_t *new_bindings = malloc(alloc_n * sizeof(binding_t));
4597+
int new_cap = *bind_count > 0 && extra_count > 0 ? *bind_count : SKIP_ONE;
4598+
binding_t *new_bindings = malloc((size_t)new_cap * sizeof(binding_t));
45824599
if (!new_bindings) {
45834600
return; /* OOM: leave existing bindings untouched rather than corrupt */
45844601
}
@@ -4595,7 +4612,7 @@ static void cross_join_with_rels(cbm_store_t *store, cbm_pattern_t *patn, bindin
45954612
const char *tv = nvar;
45964613
expand_pattern_rels(store, patn, &tmp, &tc, &tcap, &tv, opt);
45974614
for (int ti = 0; ti < tc; ti++) {
4598-
new_bindings[new_count++] = tmp[ti];
4615+
(void)binding_out_append(&new_bindings, &new_count, &new_cap, &tmp[ti]);
45994616
}
46004617
free(tmp);
46014618
}
@@ -4647,13 +4664,12 @@ static void expand_from_bound_terminal(cbm_store_t *store, cbm_pattern_t *patn,
46474664
* OPTIONAL no-match row (a data-loss bug, not an OOB — the fallback stayed
46484665
* in-bounds behind that guard) — exactly the rows
46494666
* `OPTIONAL MATCH ... WHERE <start> IS NULL` is meant to surface. */
4650-
size_t alloc_n = (size_t)*bind_count * (size_t)CYP_GROWTH_10 + (size_t)*bind_count;
4651-
binding_t *new_bindings = malloc(alloc_n * sizeof(binding_t));
4667+
int new_cap = *bind_count > 0 ? *bind_count : CYP_GROWTH_10;
4668+
binding_t *new_bindings = malloc((size_t)new_cap * sizeof(binding_t));
46524669
if (!new_bindings) {
46534670
return;
46544671
}
46554672
int new_count = 0;
4656-
int max_new = *bind_count * CYP_GROWTH_10;
46574673

46584674
for (int bi = 0; bi < *bind_count; bi++) {
46594675
binding_t *b = &(*bindings)[bi];
@@ -4694,15 +4710,13 @@ static void expand_from_bound_terminal(cbm_store_t *store, cbm_pattern_t *patn,
46944710
continue;
46954711
}
46964712
match_count++;
4697-
if (new_count < max_new) {
4698-
binding_t nb = {0};
4699-
binding_copy(&nb, b);
4700-
binding_set(&nb, start_var, &found);
4701-
if (rel->variable) {
4702-
binding_set_edge(&nb, rel->variable, &edges[ei]);
4703-
}
4704-
new_bindings[new_count++] = nb;
4713+
binding_t nb = {0};
4714+
binding_copy(&nb, b);
4715+
binding_set(&nb, start_var, &found);
4716+
if (rel->variable) {
4717+
binding_set_edge(&nb, rel->variable, &edges[ei]);
47054718
}
4719+
(void)binding_out_append(&new_bindings, &new_count, &new_cap, &nb);
47064720
node_fields_free(&found);
47074721
}
47084722
cbm_store_free_edges(edges, edge_count);
@@ -4711,15 +4725,12 @@ static void expand_from_bound_terminal(cbm_store_t *store, cbm_pattern_t *patn,
47114725
if (opt && match_count == 0) {
47124726
/* GENUINELY no matching neighbour: the scan above runs unconditionally,
47134727
* so match_count is the true neighbour count and match_count == 0 here
4714-
* means the terminal really has none — not merely that the buffer filled
4715-
* first. Keep the row with start_var left UNBOUND
4716-
* so `WHERE <start> IS NULL` correctly identifies the no-edge case. The
4717-
* buffer is sized max_new + *bind_count precisely so every fallback row
4718-
* has a slot — no guard needed, and no OPTIONAL no-match row is dropped
4719-
* even after the expansion has saturated max_new. */
4728+
* means the terminal really has none. Keep the row with start_var left
4729+
* UNBOUND so `WHERE <start> IS NULL` correctly identifies the no-edge
4730+
* case; the shared growable append gives every fallback row a slot. */
47204731
binding_t nb = {0};
47214732
binding_copy(&nb, b);
4722-
new_bindings[new_count++] = nb;
4733+
(void)binding_out_append(&new_bindings, &new_count, &new_cap, &nb);
47234734
}
47244735
}
47254736

0 commit comments

Comments
 (0)