Skip to content

Commit 1b5ba09

Browse files
committed
fix(cypher): count DISTINCT rows against the row cap (DeusData#1364)
RETURN DISTINCT projected up to the engine row cap (max_rows, 100k by default) and only then removed duplicates in rb_apply_distinct. A value that first appeared past the cap was never seen. On a 150k-edge index `MATCH ()-[r]->() RETURN DISTINCT type(r)` returned 21 of 22 types: HAS_BRANCH, a single edge, was missing, and the result was flagged truncated:true / engine_limit. The projection loops (plain RETURN, RETURN *, RETURN * after WITH) now de-duplicate while projecting, through a hash set of length-prefixed row keys. The cap therefore counts distinct rows. Memory stays bounded by the cap: only admitted rows are remembered. Once the cap is full, the rest of the bindings are only probed (no insert), to decide whether a new distinct row exists, which is when `truncated` is set. An explicit LIMIT on DISTINCT now also counts distinct rows and can stop the projection early. ORDER BY and SKIP/LIMIT keep running after DISTINCT on the already-distinct rows. The O(n^2) post-pass remains only for aggregate rows, which are distinct by group key. Proof on django (55,377 nodes / 294,528 edges, 23 edge types in get_graph_schema), same index, before vs after: before: 22 types, HAS_BRANCH missing, truncated:true engine_limit after: 23 types, truncated:false (same 5.3 s wall time) Test: cypher_return_distinct_counts_distinct_rows_against_cap_issue1364 (rare type past a max_rows=5 cap; ORDER BY, SKIP/LIMIT and LIMIT variants; more distinct rows than the cap still reports truncated). Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
1 parent 64c23fa commit 1b5ba09

2 files changed

Lines changed: 226 additions & 18 deletions

File tree

‎src/cypher/cypher.c‎

Lines changed: 132 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@
1212
#include "foundation/platform.h"
1313
#include "foundation/limits.h"
1414
#include "foundation/log.h"
15+
#include "foundation/hash_table.h"
16+
#include "foundation/mem_core.h"
1517

1618
enum {
1719
CYP_BUF_16 = 16,
@@ -2874,6 +2876,104 @@ static void rb_add_row(result_builder_t *rb, const char **values) {
28742876
rb->rows[rb->row_count++] = row;
28752877
}
28762878

2879+
/* ── DISTINCT while projecting (#1364) ─────────────────────────────
2880+
*
2881+
* RETURN DISTINCT used to project up to the engine row cap and only then drop
2882+
* duplicates, so a value first seen past the cap never reached the result: on
2883+
* a 150k-edge index `RETURN DISTINCT type(r)` lost its one HAS_BRANCH edge.
2884+
* The projection loops now de-duplicate as they go, so the cap counts DISTINCT
2885+
* rows. Memory stays bounded by the cap: only admitted rows are remembered;
2886+
* rows past the cap are probed (no insert) solely to decide `truncated`. */
2887+
typedef struct {
2888+
CBMHashTable *seen; /* row key -> the same heap key; NULL = not DISTINCT */
2889+
} rb_distinct_t;
2890+
2891+
static void rb_distinct_init(rb_distinct_t *d, bool distinct) {
2892+
d->seen = distinct ? cbm_ht_create(CBM_SZ_64) : NULL;
2893+
}
2894+
2895+
static void rb_distinct_free_key(const char *key, void *value, void *userdata) {
2896+
(void)key;
2897+
(void)userdata;
2898+
cbm_free(CBM_MEM_CLASS_OTHER, value);
2899+
}
2900+
2901+
static void rb_distinct_free(rb_distinct_t *d) {
2902+
if (d->seen) {
2903+
cbm_ht_foreach(d->seen, rb_distinct_free_key, NULL);
2904+
cbm_ht_free(d->seen);
2905+
d->seen = NULL;
2906+
}
2907+
}
2908+
2909+
/* Length-prefixed row key ("<len>:<value>" per column), so no value content
2910+
* can make two different rows collide. NULL projects as "" (as rb_add_row). */
2911+
static char *rb_distinct_row_key(const char **vals, int n) {
2912+
size_t total = SKIP_ONE;
2913+
for (int i = 0; i < n; i++) {
2914+
total += (vals[i] ? strlen(vals[i]) : 0) + CBM_SZ_32;
2915+
}
2916+
char *key = cbm_alloc(CBM_MEM_CLASS_OTHER, total);
2917+
if (!key) {
2918+
return NULL;
2919+
}
2920+
size_t pos = 0;
2921+
key[0] = '\0';
2922+
for (int i = 0; i < n; i++) {
2923+
const char *v = vals[i] ? vals[i] : "";
2924+
int w = snprintf(key + pos, total - pos, "%zu:%s", strlen(v), v);
2925+
if (w > 0) {
2926+
pos += (size_t)w;
2927+
}
2928+
}
2929+
return key;
2930+
}
2931+
2932+
/* Admission for one projected row. Returns false for a row DISTINCT already
2933+
* holds. On true, *key_out is the key to hand to rb_distinct_keep (NULL when
2934+
* not DISTINCT, or when the key allocation failed — the row is then admitted
2935+
* without de-duplication rather than dropped). */
2936+
static bool rb_distinct_is_new(const rb_distinct_t *d, const char **vals, int n, char **key_out) {
2937+
*key_out = NULL;
2938+
if (!d->seen) {
2939+
return true;
2940+
}
2941+
char *key = rb_distinct_row_key(vals, n);
2942+
if (!key) {
2943+
return true;
2944+
}
2945+
if (cbm_ht_has(d->seen, key)) {
2946+
cbm_free(CBM_MEM_CLASS_OTHER, key);
2947+
return false;
2948+
}
2949+
*key_out = key;
2950+
return true;
2951+
}
2952+
2953+
static void rb_distinct_keep(rb_distinct_t *d, char *key) {
2954+
if (key) {
2955+
cbm_ht_set(d->seen, key, key);
2956+
}
2957+
}
2958+
2959+
/* Add one projected row under a row cap, de-duplicating when DISTINCT.
2960+
* Returns false once a NEW row arrives with the cap already full: the caller
2961+
* stops there, and flags truncation when the cap is the engine budget. A
2962+
* duplicate past the cap is neither kept nor evidence of truncation. */
2963+
static bool rb_add_row_capped(result_builder_t *rb, rb_distinct_t *d, const char **vals, int cap) {
2964+
char *key = NULL;
2965+
if (!rb_distinct_is_new(d, vals, rb->col_count, &key)) {
2966+
return true;
2967+
}
2968+
if (rb->row_count >= cap) {
2969+
cbm_free(CBM_MEM_CLASS_OTHER, key);
2970+
return false;
2971+
}
2972+
rb_distinct_keep(d, key);
2973+
rb_add_row(rb, vals);
2974+
return true;
2975+
}
2976+
28772977
/* ── Main execution ─────────────────────────────────────────────── */
28782978

28792979
/* Hard ceiling: queries returning more than this trigger an error instead of data.
@@ -4348,14 +4448,20 @@ static void execute_return_star_after_with(cbm_query_t *q, binding_t *bindings,
43484448
cols[i] = resolve_item_alias(&wc->items[i], name_bufs[i], sizeof(name_bufs[i]));
43494449
}
43504450
rb_set_columns(rb, cols, col_n);
4351-
for (int bi = 0; bi < bind_count && rb->row_count < max_rows; bi++) {
4451+
rb_distinct_t seen;
4452+
rb_distinct_init(&seen, q->ret && q->ret->distinct);
4453+
for (int bi = 0; bi < bind_count; bi++) {
43524454
const char *vals[CYP_MAX_VARS];
43534455
for (int i = 0; i < col_n; i++) {
43544456
cbm_node_t *vn = binding_get(&bindings[bi], cols[i]);
43554457
vals[i] = vn && vn->name ? vn->name : "";
43564458
}
4357-
rb_add_row(rb, vals);
4459+
if (!rb_add_row_capped(rb, &seen, vals, max_rows)) {
4460+
g_cypher_truncated = true;
4461+
break;
4462+
}
43584463
}
4464+
rb_distinct_free(&seen);
43594465
}
43604466

43614467
static void execute_return_star(cbm_query_t *q, binding_t *bindings, int bind_count, int max_rows,
@@ -4370,19 +4476,22 @@ static void execute_return_star(cbm_query_t *q, binding_t *bindings, int bind_co
43704476
int projection_cap = max_rows;
43714477
bool cap_is_engine_budget = true;
43724478
cbm_return_clause_t *ret = q->ret;
4373-
if (ret && ret->limit >= 0 && ret->limit <= max_rows && !ret->distinct &&
4374-
ret->order_key_count == 0 && ret->skip <= 0) {
4375-
projection_cap = ret->limit;
4479+
if (ret && ret->limit >= 0 && ret->limit <= max_rows && ret->order_key_count == 0 &&
4480+
ret->skip <= 0) {
4481+
projection_cap = ret->limit; /* DISTINCT: the cap counts distinct rows (#1364) */
43764482
cap_is_engine_budget = false;
43774483
}
4378-
for (int bi = 0; bi < bind_count && rb->row_count < projection_cap; bi++) {
4484+
rb_distinct_t seen;
4485+
rb_distinct_init(&seen, ret && ret->distinct);
4486+
for (int bi = 0; bi < bind_count; bi++) {
43794487
const char *vals[CBM_SZ_128];
43804488
project_star_row(&bindings[bi], vars, vc, vals);
4381-
rb_add_row(rb, vals);
4382-
}
4383-
if (cap_is_engine_budget && bind_count > projection_cap) {
4384-
g_cypher_truncated = true;
4489+
if (!rb_add_row_capped(rb, &seen, vals, projection_cap)) {
4490+
g_cypher_truncated = g_cypher_truncated || cap_is_engine_budget;
4491+
break;
4492+
}
43854493
}
4494+
rb_distinct_free(&seen);
43864495
}
43874496

43884497
/* Format an aggregate value into buf based on function name */
@@ -4651,22 +4760,25 @@ static void execute_return_simple(cbm_return_clause_t *ret, binding_t *bindings,
46514760
int max_rows, result_builder_t *rb) {
46524761
int proj_cap = max_rows;
46534762
bool cap_is_engine_budget = true;
4654-
if (ret->limit >= 0 && !ret->distinct && ret->order_key_count == 0 && ret->skip <= 0) {
4655-
proj_cap = ret->limit;
4763+
if (ret->limit >= 0 && ret->order_key_count == 0 && ret->skip <= 0) {
4764+
proj_cap = ret->limit; /* DISTINCT: the cap counts distinct rows (#1364) */
46564765
cap_is_engine_budget = false;
46574766
}
4658-
for (int bi = 0; bi < bind_count && rb->row_count < proj_cap; bi++) {
4767+
rb_distinct_t seen;
4768+
rb_distinct_init(&seen, ret->distinct);
4769+
for (int bi = 0; bi < bind_count; bi++) {
46594770
const char *vals[CBM_SZ_32];
46604771
char func_bufs[CBM_SZ_32][CBM_SZ_512];
46614772
for (int ci = 0; ci < ret->count; ci++) {
46624773
vals[ci] =
46634774
project_item(&bindings[bi], &ret->items[ci], func_bufs[ci], sizeof(func_bufs[ci]));
46644775
}
4665-
rb_add_row(rb, vals);
4666-
}
4667-
if (cap_is_engine_budget && bind_count > proj_cap) {
4668-
g_cypher_truncated = true;
4776+
if (!rb_add_row_capped(rb, &seen, vals, proj_cap)) {
4777+
g_cypher_truncated = g_cypher_truncated || cap_is_engine_budget;
4778+
break;
4779+
}
46694780
}
4781+
rb_distinct_free(&seen);
46704782
}
46714783

46724784
/* Build default 3-column headers (name, qualified_name, label) per variable */
@@ -5005,7 +5117,9 @@ static void execute_return_clause(cbm_query_t *q, cbm_return_clause_t *ret, bind
50055117
}
50065118
}
50075119

5008-
if (ret->distinct) {
5120+
/* The simple and star projections de-duplicate while projecting (#1364);
5121+
* only aggregate rows still need the post-pass. */
5122+
if (ret->distinct && has_agg && !ret->star) {
50095123
rb_apply_distinct(rb);
50105124
}
50115125
rb_apply_order_by(rb, ret);

‎tests/test_cypher.c‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1230,6 +1230,99 @@ TEST(cypher_exec_aggregate_sees_all_edges_beyond_expansion_cap_issue1196) {
12301230
PASS();
12311231
}
12321232

1233+
/* #1364: RETURN DISTINCT de-duplicated only AFTER the engine row cap, so a
1234+
* value that first appears past the cap was never seen: on a 150k-edge index
1235+
* `RETURN DISTINCT type(r)` returned 21 of 22 types (HAS_BRANCH, one edge,
1236+
* missing). The cap must count DISTINCT rows. Fixture: 20 CALLS edges, then
1237+
* one HAS_BRANCH edge from the last-inserted node; max_rows=5 puts the rare
1238+
* type far past the raw-row cap. */
1239+
TEST(cypher_return_distinct_counts_distinct_rows_against_cap_issue1364) {
1240+
cbm_store_t *s = cbm_store_open_memory();
1241+
ASSERT_NOT_NULL(s);
1242+
ASSERT_EQ(cbm_store_upsert_project(s, "test", "/tmp/test"), CBM_STORE_OK);
1243+
1244+
int64_t ids[22];
1245+
for (int i = 0; i < 22; i++) {
1246+
char name[32];
1247+
char qn[64];
1248+
snprintf(name, sizeof(name), "fn_%02d", i);
1249+
snprintf(qn, sizeof(qn), "test.%s", name);
1250+
cbm_node_t n = {.project = "test",
1251+
.label = "Function",
1252+
.name = name,
1253+
.qualified_name = qn,
1254+
.file_path = "a.py"};
1255+
ids[i] = cbm_store_upsert_node(s, &n);
1256+
ASSERT_GT(ids[i], 0);
1257+
}
1258+
for (int i = 0; i < 20; i++) {
1259+
cbm_edge_t e = {
1260+
.project = "test", .source_id = ids[i], .target_id = ids[i + 1], .type = "CALLS"};
1261+
cbm_store_insert_edge(s, &e);
1262+
}
1263+
cbm_edge_t rare = {
1264+
.project = "test", .source_id = ids[21], .target_id = ids[0], .type = "HAS_BRANCH"};
1265+
cbm_store_insert_edge(s, &rare);
1266+
1267+
/* Plain DISTINCT: both types, complete (2 distinct rows << cap of 5). */
1268+
cbm_cypher_result_t r = {0};
1269+
int rc = cbm_cypher_execute(s, "MATCH ()-[r]->() RETURN DISTINCT type(r) AS t", "test", 5, &r);
1270+
ASSERT_EQ(rc, 0);
1271+
ASSERT_EQ(r.row_count, 2);
1272+
bool saw_calls = false;
1273+
bool saw_branch = false;
1274+
for (int i = 0; i < r.row_count; i++) {
1275+
saw_calls = saw_calls || strcmp(r.rows[i][0], "CALLS") == 0;
1276+
saw_branch = saw_branch || strcmp(r.rows[i][0], "HAS_BRANCH") == 0;
1277+
}
1278+
ASSERT_TRUE(saw_calls);
1279+
ASSERT_TRUE(saw_branch);
1280+
ASSERT_FALSE(r.truncated);
1281+
cbm_cypher_result_free(&r);
1282+
1283+
/* DISTINCT + ORDER BY shares the cap. */
1284+
memset(&r, 0, sizeof(r));
1285+
rc = cbm_cypher_execute(s, "MATCH ()-[r]->() RETURN DISTINCT type(r) AS t ORDER BY t DESC",
1286+
"test", 5, &r);
1287+
ASSERT_EQ(rc, 0);
1288+
ASSERT_EQ(r.row_count, 2);
1289+
ASSERT_STR_EQ(r.rows[0][0], "HAS_BRANCH");
1290+
ASSERT_STR_EQ(r.rows[1][0], "CALLS");
1291+
ASSERT_FALSE(r.truncated);
1292+
cbm_cypher_result_free(&r);
1293+
1294+
/* DISTINCT + ORDER BY + SKIP/LIMIT: SKIP counts distinct rows. */
1295+
memset(&r, 0, sizeof(r));
1296+
rc = cbm_cypher_execute(
1297+
s, "MATCH ()-[r]->() RETURN DISTINCT type(r) AS t ORDER BY t SKIP 1 LIMIT 1", "test", 5,
1298+
&r);
1299+
ASSERT_EQ(rc, 0);
1300+
ASSERT_EQ(r.row_count, 1);
1301+
ASSERT_STR_EQ(r.rows[0][0], "HAS_BRANCH");
1302+
ASSERT_FALSE(r.truncated);
1303+
cbm_cypher_result_free(&r);
1304+
1305+
/* An explicit LIMIT on DISTINCT counts distinct rows too. */
1306+
memset(&r, 0, sizeof(r));
1307+
rc = cbm_cypher_execute(s, "MATCH ()-[r]->() RETURN DISTINCT type(r) AS t LIMIT 2", "test", 5,
1308+
&r);
1309+
ASSERT_EQ(rc, 0);
1310+
ASSERT_EQ(r.row_count, 2);
1311+
ASSERT_FALSE(r.truncated);
1312+
cbm_cypher_result_free(&r);
1313+
1314+
/* More distinct rows than the cap: still bounded and reported truncated. */
1315+
memset(&r, 0, sizeof(r));
1316+
rc = cbm_cypher_execute(s, "MATCH (a)-[r]->(b) RETURN DISTINCT b.name", "test", 5, &r);
1317+
ASSERT_EQ(rc, 0);
1318+
ASSERT_EQ(r.row_count, 5);
1319+
ASSERT_TRUE(r.truncated);
1320+
cbm_cypher_result_free(&r);
1321+
1322+
cbm_store_close(s);
1323+
PASS();
1324+
}
1325+
12331326
/* #874: coalesce(var.prop, literal) in WHERE — null-safe numeric filters
12341327
* for audit queries over OPTIONAL graph properties. The parser rejected the
12351328
* call outright ("unexpected operator"); RETURN-side coalesce already
@@ -4689,6 +4782,7 @@ SUITE(cypher) {
46894782
RUN_TEST(cypher_exec_where_eq);
46904783
RUN_TEST(cypher_exec_unlabeled_where_beyond_result_limit_issue1196);
46914784
RUN_TEST(cypher_exec_aggregate_sees_all_edges_beyond_expansion_cap_issue1196);
4785+
RUN_TEST(cypher_return_distinct_counts_distinct_rows_against_cap_issue1364);
46924786
RUN_TEST(cypher_exec_varlength_path_semantics_issue797);
46934787
RUN_TEST(cypher_exec_where_coalesce_issue874);
46944788
RUN_TEST(cypher_exec_where_regex);

0 commit comments

Comments
 (0)