Skip to content

Commit ea5c1f7

Browse files
authored
Merge pull request DeusData#2689 from DeusData/fix/data-flows-every-caller
fix(pipeline): DATA_FLOWS link every caller and handler of a route
2 parents 1a7839d + 7c51129 commit ea5c1f7

2 files changed

Lines changed: 157 additions & 72 deletions

File tree

‎src/pipeline/pass_route_nodes.c‎

Lines changed: 62 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -677,12 +677,6 @@ static void finish_data_flow_props(char *props, size_t propsz, size_t pos,
677677
}
678678
}
679679

680-
typedef struct {
681-
int64_t source_id;
682-
const char *props;
683-
const char *edge_type;
684-
} caller_edge_ref_t;
685-
686680
static bool http_call_edge_has_valid_route(const cbm_gbuf_edge_t *edge) {
687681
char url_buf[CBM_SZ_512];
688682
const char *url = json_extract(edge->properties_json, "url_path", url_buf, sizeof(url_buf));
@@ -747,84 +741,82 @@ static int try_create_data_flow(cbm_gbuf_t *gb, int64_t caller_id, int64_t handl
747741
return SKIP_ONE;
748742
}
749743

750-
/* Collect extra handler IDs reachable via INFRA_MAPS chain from a route. */
751-
static int collect_infra_handlers(cbm_gbuf_t *gb, int64_t route_id, int64_t *out, int max_out) {
752-
const cbm_gbuf_edge_t **infra_edges = NULL;
753-
int infra_count = 0;
754-
cbm_gbuf_find_edges_by_source_type(gb, route_id, "INFRA_MAPS", &infra_edges, &infra_count);
744+
/* One route's DATA_FLOWS context: its direct handlers (handler -[HANDLES]-> route) and its
745+
* INFRA_MAPS hops (route -[INFRA_MAPS]-> endpoint, whose handlers are reached via the endpoint).
746+
* The edge arrays are the graph buffer's own index arrays. Inserting DATA_FLOWS never appends to
747+
* a HANDLES, INFRA_MAPS, HTTP_CALLS or ASYNC_CALLS array, so they stay valid while flows are added
748+
* (only edge deletion rebuilds the indexes). */
749+
typedef struct {
750+
cbm_gbuf_t *gb;
751+
const cbm_gbuf_node_t *route;
752+
const cbm_gbuf_edge_t **handles;
753+
int handles_count;
754+
const cbm_gbuf_edge_t **infra;
755+
int infra_count;
756+
int flows;
757+
int skipped;
758+
} route_flows_t;
759+
760+
static void route_flow_add(route_flows_t *rf, const cbm_gbuf_edge_t *caller, int64_t handler_id,
761+
const char *edge_type, bool via_infra) {
762+
int rc = try_create_data_flow(rf->gb, caller->source_id, handler_id, rf->route, edge_type,
763+
caller->properties_json, via_infra);
764+
if (rc > 0) {
765+
rf->flows++;
766+
} else if (rc < 0) {
767+
rf->skipped++;
768+
}
769+
}
755770

756-
int n = 0;
757-
for (int ie = 0; ie < infra_count; ie++) {
771+
/* DATA_FLOWS from one caller edge to every handler of the route, direct and via infra. */
772+
static void caller_data_flows(route_flows_t *rf, const cbm_gbuf_edge_t *caller,
773+
const char *edge_type) {
774+
for (int hi = 0; hi < rf->handles_count; hi++) {
775+
route_flow_add(rf, caller, rf->handles[hi]->source_id, edge_type, false);
776+
}
777+
for (int ie = 0; ie < rf->infra_count; ie++) {
758778
const cbm_gbuf_edge_t **ep_handles = NULL;
759779
int ep_hcount = 0;
760-
cbm_gbuf_find_edges_by_target_type(gb, infra_edges[ie]->target_id, "HANDLES", &ep_handles,
780+
cbm_gbuf_find_edges_by_target_type(rf->gb, rf->infra[ie]->target_id, "HANDLES", &ep_handles,
761781
&ep_hcount);
762-
for (int eh = 0; eh < ep_hcount && n < max_out; eh++) {
763-
out[n++] = ep_handles[eh]->source_id;
782+
for (int eh = 0; eh < ep_hcount; eh++) {
783+
route_flow_add(rf, caller, ep_handles[eh]->source_id, edge_type, true);
764784
}
765785
}
766-
return n;
767786
}
768787

769-
/* Collect caller edges (HTTP_CALLS + ASYNC_CALLS) targeting a route. */
770-
static int collect_caller_edges(cbm_gbuf_t *gb, int64_t route_id, caller_edge_ref_t *out,
771-
int max_out) {
772-
int n = 0;
788+
/* Create DATA_FLOWS between EVERY caller (HTTP_CALLS + ASYNC_CALLS) and EVERY handler of one
789+
* route node.
790+
*
791+
* Callers were copied into a 64-slot buffer and infra handlers into a 32-slot one, each keeping the
792+
* first entries in edge-insertion order -- i.e. in parallel worker interleaving. Which DATA_FLOWS
793+
* existed therefore changed between two indexes of the same tree with the same binary
794+
* (thingsboard, 2026-10-17), and every caller past the 64th lost its flow outright. The flow set
795+
* is now the full caller x handler product. A (caller, handler) pair reached twice -- through two
796+
* routes, or directly and via infra -- keeps the order-free property winner of
797+
* cbm_gbuf_insert_edge. */
798+
static void create_route_data_flows(cbm_gbuf_t *gb, const cbm_gbuf_node_t *route, int *flows,
799+
int *skipped) {
800+
route_flows_t rf = {.gb = gb, .route = route};
801+
cbm_gbuf_find_edges_by_target_type(gb, route->id, "HANDLES", &rf.handles, &rf.handles_count);
802+
cbm_gbuf_find_edges_by_source_type(gb, route->id, "INFRA_MAPS", &rf.infra, &rf.infra_count);
803+
773804
const cbm_gbuf_edge_t **http_edges = NULL;
774805
int http_count = 0;
775-
cbm_gbuf_find_edges_by_target_type(gb, route_id, "HTTP_CALLS", &http_edges, &http_count);
776-
for (int i = 0; i < http_count && n < max_out; i++) {
777-
if (!http_call_edge_has_valid_route(http_edges[i])) {
778-
continue;
806+
cbm_gbuf_find_edges_by_target_type(gb, route->id, "HTTP_CALLS", &http_edges, &http_count);
807+
for (int i = 0; i < http_count; i++) {
808+
if (http_call_edge_has_valid_route(http_edges[i])) {
809+
caller_data_flows(&rf, http_edges[i], "HTTP_CALLS");
779810
}
780-
out[n].source_id = http_edges[i]->source_id;
781-
out[n].props = http_edges[i]->properties_json;
782-
out[n].edge_type = "HTTP_CALLS";
783-
n++;
784811
}
785812
const cbm_gbuf_edge_t **async_edges = NULL;
786813
int async_count = 0;
787-
cbm_gbuf_find_edges_by_target_type(gb, route_id, "ASYNC_CALLS", &async_edges, &async_count);
788-
for (int i = 0; i < async_count && n < max_out; i++) {
789-
out[n].source_id = async_edges[i]->source_id;
790-
out[n].props = async_edges[i]->properties_json;
791-
out[n].edge_type = "ASYNC_CALLS";
792-
n++;
793-
}
794-
return n;
795-
}
796-
797-
/* Create DATA_FLOWS between callers and handlers for one route node. */
798-
static void create_route_data_flows(cbm_gbuf_t *gb, const cbm_gbuf_node_t *route,
799-
const caller_edge_ref_t *callers, int n_callers, int *flows,
800-
int *skipped) {
801-
const cbm_gbuf_edge_t **handles_edges = NULL;
802-
int handles_count = 0;
803-
cbm_gbuf_find_edges_by_target_type(gb, route->id, "HANDLES", &handles_edges, &handles_count);
804-
805-
int64_t extra_handlers[CBM_SZ_32];
806-
int n_extra = collect_infra_handlers(gb, route->id, extra_handlers, CBM_SZ_32);
807-
808-
for (int ci = 0; ci < n_callers; ci++) {
809-
for (int hi = 0; hi < handles_count; hi++) {
810-
int rc = try_create_data_flow(gb, callers[ci].source_id, handles_edges[hi]->source_id,
811-
route, callers[ci].edge_type, callers[ci].props, false);
812-
if (rc > 0) {
813-
(*flows)++;
814-
} else if (rc < 0) {
815-
(*skipped)++;
816-
}
817-
}
818-
for (int xi = 0; xi < n_extra; xi++) {
819-
int rc = try_create_data_flow(gb, callers[ci].source_id, extra_handlers[xi], route,
820-
callers[ci].edge_type, callers[ci].props, true);
821-
if (rc > 0) {
822-
(*flows)++;
823-
} else if (rc < 0) {
824-
(*skipped)++;
825-
}
826-
}
814+
cbm_gbuf_find_edges_by_target_type(gb, route->id, "ASYNC_CALLS", &async_edges, &async_count);
815+
for (int i = 0; i < async_count; i++) {
816+
caller_data_flows(&rf, async_edges[i], "ASYNC_CALLS");
827817
}
818+
*flows += rf.flows;
819+
*skipped += rf.skipped;
828820
}
829821

830822
/* Find the enclosing proto service for a function by file + line range. */
@@ -931,9 +923,7 @@ static void create_data_flows(cbm_gbuf_t *gb) {
931923
int skipped = 0;
932924

933925
for (int ri = 0; ri < route_count; ri++) {
934-
caller_edge_ref_t caller_edges[CBM_SZ_64];
935-
int n_callers = collect_caller_edges(gb, routes[ri]->id, caller_edges, CBM_SZ_64);
936-
create_route_data_flows(gb, routes[ri], caller_edges, n_callers, &flows, &skipped);
926+
create_route_data_flows(gb, routes[ri], &flows, &skipped);
937927
}
938928

939929
if (flows > 0 || skipped > 0) {

‎tests/test_infrascan.c‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -385,6 +385,98 @@ TEST(infrascan_http_calls_join_matching_handler_route) {
385385
PASS();
386386
}
387387

388+
/* FD16: a route's DATA_FLOWS were built from the first 64 callers and the first 32 infra handlers
389+
* in edge-insertion order (parallel worker interleaving), so the flow set changed between two
390+
* indexes of the same tree and every caller past the 64th lost its flow. */
391+
enum { FD16_CALLERS = 70, FD16_INFRA_HANDLERS = 40 };
392+
393+
static int64_t fd16_route_with_callers(cbm_gbuf_t *gb, int64_t *callers, bool reverse) {
394+
int64_t route = cbm_gbuf_upsert_node(gb, "Route", "/api/orders", "__route__GET__/api/orders",
395+
"", 0, 0, "{\"method\":\"GET\"}");
396+
int64_t handler = cbm_gbuf_upsert_node(gb, "Function", "get_orders", "test.get_orders",
397+
"server.py", 1, 3, "{}");
398+
cbm_gbuf_insert_edge(gb, handler, route, "HANDLES", "{\"handler\":\"test.get_orders\"}");
399+
for (int i = 0; i < FD16_CALLERS; i++) {
400+
char name[32];
401+
char qn[48];
402+
snprintf(name, sizeof(name), "client_%02d", i);
403+
snprintf(qn, sizeof(qn), "test.client_%02d", i);
404+
callers[i] =
405+
cbm_gbuf_upsert_node(gb, "Function", name, qn, "client.py", i + 1, i + 1, "{}");
406+
}
407+
for (int k = 0; k < FD16_CALLERS; k++) {
408+
int i = reverse ? FD16_CALLERS - 1 - k : k;
409+
cbm_gbuf_insert_edge(gb, callers[i], route, "HTTP_CALLS",
410+
"{\"callee\":\"requests.get\",\"url_path\":\"/api/orders\","
411+
"\"method\":\"GET\"}");
412+
}
413+
return handler;
414+
}
415+
416+
TEST(infrascan_data_flows_link_every_caller_of_a_route) {
417+
cbm_gbuf_t *gb = cbm_gbuf_new("test", "/tmp/cbm_infrascan_fd16_callers");
418+
ASSERT_NOT_NULL(gb);
419+
int64_t callers[FD16_CALLERS];
420+
int64_t handler = fd16_route_with_callers(gb, callers, false);
421+
cbm_pipeline_create_route_nodes(gb);
422+
for (int i = 0; i < FD16_CALLERS; i++) {
423+
ASSERT_TRUE(has_data_flow(gb, callers[i], handler));
424+
}
425+
cbm_gbuf_free(gb);
426+
PASS();
427+
}
428+
429+
TEST(infrascan_data_flows_do_not_depend_on_caller_insertion_order) {
430+
cbm_gbuf_t *fwd = cbm_gbuf_new("test", "/tmp/cbm_infrascan_fd16_fwd");
431+
cbm_gbuf_t *rev = cbm_gbuf_new("test", "/tmp/cbm_infrascan_fd16_rev");
432+
ASSERT_NOT_NULL(fwd);
433+
ASSERT_NOT_NULL(rev);
434+
int64_t fwd_callers[FD16_CALLERS];
435+
int64_t rev_callers[FD16_CALLERS];
436+
int64_t fwd_handler = fd16_route_with_callers(fwd, fwd_callers, false);
437+
int64_t rev_handler = fd16_route_with_callers(rev, rev_callers, true);
438+
cbm_pipeline_create_route_nodes(fwd);
439+
cbm_pipeline_create_route_nodes(rev);
440+
for (int i = 0; i < FD16_CALLERS; i++) {
441+
ASSERT_EQ(has_data_flow(fwd, fwd_callers[i], fwd_handler),
442+
has_data_flow(rev, rev_callers[i], rev_handler));
443+
}
444+
cbm_gbuf_free(fwd);
445+
cbm_gbuf_free(rev);
446+
PASS();
447+
}
448+
449+
TEST(infrascan_data_flows_reach_every_infra_handler) {
450+
cbm_gbuf_t *gb = cbm_gbuf_new("test", "/tmp/cbm_infrascan_fd16_infra");
451+
ASSERT_NOT_NULL(gb);
452+
int64_t route = cbm_gbuf_upsert_node(gb, "Route", "/api/orders", "__route__GET__/api/orders",
453+
"", 0, 0, "{\"method\":\"GET\"}");
454+
int64_t endpoint = cbm_gbuf_upsert_node(gb, "Route", "/orders", "__route__GET__/orders", "", 0,
455+
0, "{\"method\":\"GET\"}");
456+
int64_t client =
457+
cbm_gbuf_upsert_node(gb, "Function", "client", "test.client", "client.py", 1, 3, "{}");
458+
cbm_gbuf_insert_edge(gb, route, endpoint, "INFRA_MAPS", "{}");
459+
cbm_gbuf_insert_edge(gb, client, route, "HTTP_CALLS",
460+
"{\"callee\":\"requests.get\",\"url_path\":\"/api/orders\","
461+
"\"method\":\"GET\"}");
462+
int64_t handlers[FD16_INFRA_HANDLERS];
463+
for (int i = 0; i < FD16_INFRA_HANDLERS; i++) {
464+
char name[32];
465+
char qn[48];
466+
snprintf(name, sizeof(name), "orders_%02d", i);
467+
snprintf(qn, sizeof(qn), "test.orders_%02d", i);
468+
handlers[i] =
469+
cbm_gbuf_upsert_node(gb, "Function", name, qn, "server.py", i + 1, i + 1, "{}");
470+
cbm_gbuf_insert_edge(gb, handlers[i], endpoint, "HANDLES", "{}");
471+
}
472+
cbm_pipeline_create_route_nodes(gb);
473+
for (int i = 0; i < FD16_INFRA_HANDLERS; i++) {
474+
ASSERT_TRUE(has_data_flow(gb, client, handlers[i]));
475+
}
476+
cbm_gbuf_free(gb);
477+
PASS();
478+
}
479+
388480
SUITE(infrascan) {
389481
RUN_TEST(infrascan_http_route_literal_guard_rejects_filesystem_paths);
390482
RUN_TEST(infrascan_http_route_literal_guard_rejects_document_extensions);
@@ -393,4 +485,7 @@ SUITE(infrascan) {
393485
RUN_TEST(infrascan_service_pattern_match_keeps_real_library_qns);
394486
RUN_TEST(infrascan_route_nodes_skip_bad_http_url_paths);
395487
RUN_TEST(infrascan_http_calls_join_matching_handler_route);
488+
RUN_TEST(infrascan_data_flows_link_every_caller_of_a_route);
489+
RUN_TEST(infrascan_data_flows_do_not_depend_on_caller_insertion_order);
490+
RUN_TEST(infrascan_data_flows_reach_every_infra_handler);
396491
}

0 commit comments

Comments
 (0)