Skip to content

Commit 4b843c2

Browse files
committed
fix(extract): YAML walkers grow their stacks instead of capping content
walk_yaml_mapping (dotted key paths, 256 frames) and scan_yaml_for_infra_bindings (512 nodes) kept their pending nodes in fixed-size stacks and stopped pushing once full. A mapping with more than 256 nested sibling mappings lost the full key_path of every leaf past the cap: `k299.url` was only recorded as the shorter path that the walk of a nested block emits on its own. The infra Route pass decides on those paths. A `data:` payload block with 300 entries lost `data.svcN.webhook_url` for N >= 256, the payload deny never saw those values, and 44 of them were minted as infra Route nodes. Both walkers now start on the same-sized on-stack chunk and grow through cbm_walk_stack_reserve / cbm_walk_stack_release (DeusData#2375). If the stack cannot grow, the failure is logged as extract.walk_stack_alloc_failed and the walk stops, instead of truncating silently. Part of DeusData#2388. Signed-off-by: Gaurav Dubey <gauravdubey0107@gmail.com>
1 parent ff28841 commit 4b843c2

3 files changed

Lines changed: 153 additions & 13 deletions

File tree

‎internal/cbm/extract_unified.c‎

Lines changed: 38 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1663,27 +1663,39 @@ typedef struct {
16631663
TSNode node;
16641664
const char *prefix;
16651665
} yaml_walk_frame_t;
1666-
#define YAML_WALK_STACK_CAP CBM_SZ_256
1667-
1668-
/* Push block_mapping children of a block_node/block_mapping value onto the walk stack. */
1669-
static void push_yaml_block_children(TSNode val, const char *path, yaml_walk_frame_t *stack,
1670-
int *top) {
1666+
/* On-stack first chunk of the key-path walk; wider mappings spill to the heap
1667+
* through cbm_walk_stack_reserve instead of dropping frames (#2388). */
1668+
#define YAML_WALK_STACK_INLINE CBM_SZ_256
1669+
1670+
/* Push block_mapping children of a block_node/block_mapping value onto the walk
1671+
* stack, growing it as needed. False when the stack could not grow (logged by
1672+
* cbm_walk_stack_reserve as extract.walk_stack_alloc_failed). */
1673+
static bool push_yaml_block_children(TSNode val, const char *path, yaml_walk_frame_t **stack,
1674+
int *cap, int *top, const yaml_walk_frame_t *inline_buf) {
16711675
uint32_t vnc = ts_node_named_child_count(val);
1672-
for (int vi = (int)vnc - SKIP_ONE; vi >= 0 && *top < YAML_WALK_STACK_CAP; vi--) {
1676+
if (!cbm_walk_stack_reserve((void **)stack, cap, *top + (int)vnc, sizeof(yaml_walk_frame_t),
1677+
inline_buf, "yaml_key_paths")) {
1678+
return false;
1679+
}
1680+
for (int vi = (int)vnc - SKIP_ONE; vi >= 0; vi--) {
16731681
TSNode vc = ts_node_named_child(val, (uint32_t)vi);
16741682
const char *vctype = ts_node_type(vc);
16751683
if (strcmp(vctype, "block_mapping") == 0 || strcmp(vctype, "block_mapping_pair") == 0) {
1676-
stack[(*top)++] = (yaml_walk_frame_t){vc, path};
1684+
(*stack)[(*top)++] = (yaml_walk_frame_t){vc, path};
16771685
}
16781686
}
1687+
return true;
16791688
}
16801689

16811690
static void walk_yaml_mapping(CBMExtractCtx *ctx, TSNode root, const char *root_prefix) {
1682-
yaml_walk_frame_t stack[YAML_WALK_STACK_CAP];
1691+
yaml_walk_frame_t inline_stack[YAML_WALK_STACK_INLINE];
1692+
yaml_walk_frame_t *stack = inline_stack;
1693+
int cap = YAML_WALK_STACK_INLINE;
16831694
int top = 0;
16841695
stack[top++] = (yaml_walk_frame_t){root, root_prefix};
1696+
bool grown = true;
16851697

1686-
while (top > 0) {
1698+
while (grown && top > 0) {
16871699
yaml_walk_frame_t frame = stack[--top];
16881700
TSNode node = frame.node;
16891701
const char *prefix = frame.prefix;
@@ -1709,12 +1721,16 @@ static void walk_yaml_mapping(CBMExtractCtx *ctx, TSNode root, const char *root_
17091721
}
17101722
const char *vk = ts_node_type(val);
17111723
if (strcmp(vk, "block_node") == 0 || strcmp(vk, "block_mapping") == 0) {
1712-
push_yaml_block_children(val, path, stack, &top);
1724+
grown = push_yaml_block_children(val, path, &stack, &cap, &top, inline_stack);
1725+
if (!grown) {
1726+
break;
1727+
}
17131728
continue;
17141729
}
17151730
emit_yaml_leaf_value(ctx, val, path);
17161731
}
17171732
}
1733+
cbm_walk_stack_release(stack, inline_stack);
17181734
}
17191735

17201736
/* ── Infrastructure binding extraction ─────────────────────────────
@@ -1873,9 +1889,13 @@ static void scan_mapping_for_bindings(CBMExtractCtx *ctx, TSNode mapping) {
18731889
emit_infra_bindings(ctx, sources, source_keys, n_sources, targets, n_targets);
18741890
}
18751891

1876-
#define INFRA_SCAN_STACK_CAP CBM_SZ_512
1892+
/* On-stack first chunk of the binding scan; wider subtrees spill to the heap
1893+
* through cbm_walk_stack_reserve instead of dropping nodes (#2388). */
1894+
#define INFRA_SCAN_STACK_INLINE CBM_SZ_512
18771895
static void scan_yaml_for_infra_bindings(CBMExtractCtx *ctx, TSNode root) {
1878-
TSNode stack[INFRA_SCAN_STACK_CAP];
1896+
TSNode inline_stack[INFRA_SCAN_STACK_INLINE];
1897+
TSNode *stack = inline_stack;
1898+
int cap = INFRA_SCAN_STACK_INLINE;
18791899
int top = 0;
18801900
stack[top++] = root;
18811901
while (top > 0) {
@@ -1884,10 +1904,15 @@ static void scan_yaml_for_infra_bindings(CBMExtractCtx *ctx, TSNode root) {
18841904
scan_mapping_for_bindings(ctx, node);
18851905
}
18861906
uint32_t nc = ts_node_named_child_count(node);
1887-
for (int i = (int)nc - SKIP_ONE; i >= 0 && top < INFRA_SCAN_STACK_CAP; i--) {
1907+
if (!cbm_walk_stack_reserve((void **)&stack, &cap, top + (int)nc, sizeof(TSNode),
1908+
inline_stack, "yaml_infra_bindings")) {
1909+
break;
1910+
}
1911+
for (int i = (int)nc - SKIP_ONE; i >= 0; i--) {
18881912
stack[top++] = ts_node_named_child(node, (uint32_t)i);
18891913
}
18901914
}
1915+
cbm_walk_stack_release(stack, inline_stack);
18911916
}
18921917

18931918
/* ── HCL infrastructure binding extraction ───────────────────────────

‎tests/test_extraction.c‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7229,6 +7229,85 @@ TEST(body_tokens_wide_body_samples_from_the_start) {
72297229
PASS();
72307230
}
72317231

7232+
/* The YAML walkers kept their pending mappings in fixed-size stacks (256 for
7233+
* the dotted key-path walk, 512 for the infrastructure-binding scan) and
7234+
* silently stopped pushing once full (#2388). A root mapping with more nested
7235+
* sibling mappings than the old cap lost the full dotted key_path of every
7236+
* leaf past it: `k299.url` was only ever recorded as the bare `url` that the
7237+
* per-mapping walk of the nested block emits. */
7238+
enum { WIDE_YAML_KEYS = 300, WIDE_YAML_ITEMS = 600 };
7239+
7240+
static int has_string_ref_path(CBMFileResult *r, const char *key_path, const char *value) {
7241+
for (int i = 0; i < r->string_refs.count; i++) {
7242+
const CBMStringRef *s = &r->string_refs.items[i];
7243+
if (s->key_path && s->value && strcmp(s->key_path, key_path) == 0 &&
7244+
strcmp(s->value, value) == 0) {
7245+
return 1;
7246+
}
7247+
}
7248+
return 0;
7249+
}
7250+
7251+
TEST(yaml_key_paths_wide_mapping_exceed_old_stack_cap) {
7252+
size_t cap = (size_t)WIDE_YAML_KEYS * 48 + 64;
7253+
char *src = malloc(cap);
7254+
ASSERT_NOT_NULL(src);
7255+
size_t pos = 0;
7256+
for (int i = 0; i < WIDE_YAML_KEYS; i++) {
7257+
pos += (size_t)snprintf(src + pos, cap - pos,
7258+
"k%d:\n url: \"https://h%d.example.com/x\"\n", i, i);
7259+
}
7260+
CBMFileResult *r = extract(src, CBM_LANG_YAML, "t", "wide.yaml");
7261+
ASSERT_NOT_NULL(r);
7262+
ASSERT_FALSE(r->has_error);
7263+
for (int i = 0; i < WIDE_YAML_KEYS; i++) {
7264+
char path[32];
7265+
char value[48];
7266+
snprintf(path, sizeof(path), "k%d.url", i);
7267+
snprintf(value, sizeof(value), "https://h%d.example.com/x", i);
7268+
ASSERT(has_string_ref_path(r, path, value));
7269+
}
7270+
cbm_free_result(r);
7271+
free(src);
7272+
PASS();
7273+
}
7274+
7275+
static int has_infra_source(CBMFileResult *r, const char *source, const char *url) {
7276+
for (int i = 0; i < r->infra_bindings.count; i++) {
7277+
const CBMInfraBinding *b = &r->infra_bindings.items[i];
7278+
if (b->source_name && b->target_url && strcmp(b->source_name, source) == 0 &&
7279+
strcmp(b->target_url, url) == 0) {
7280+
return 1;
7281+
}
7282+
}
7283+
return 0;
7284+
}
7285+
7286+
TEST(yaml_infra_bindings_wide_sequence_exceed_old_stack_cap) {
7287+
size_t cap = (size_t)WIDE_YAML_ITEMS * 64 + 64;
7288+
char *src = malloc(cap);
7289+
ASSERT_NOT_NULL(src);
7290+
size_t pos = 0;
7291+
for (int i = 0; i < WIDE_YAML_ITEMS; i++) {
7292+
pos += (size_t)snprintf(src + pos, cap - pos,
7293+
"- topic: t%d\n push_endpoint: \"https://h%d.example.com/p\"\n", i,
7294+
i);
7295+
}
7296+
CBMFileResult *r = extract(src, CBM_LANG_YAML, "t", "pubsub.yaml");
7297+
ASSERT_NOT_NULL(r);
7298+
ASSERT_FALSE(r->has_error);
7299+
for (int i = 0; i < WIDE_YAML_ITEMS; i++) {
7300+
char source[16];
7301+
char url[48];
7302+
snprintf(source, sizeof(source), "t%d", i);
7303+
snprintf(url, sizeof(url), "https://h%d.example.com/p", i);
7304+
ASSERT(has_infra_source(r, source, url));
7305+
}
7306+
cbm_free_result(r);
7307+
free(src);
7308+
PASS();
7309+
}
7310+
72327311
/* ═══════════════════════════════════════════════════════════════════
72337312
* Perl call-graph noise (#459 follow-up)
72347313
* ═══════════════════════════════════════════════════════════════════ */
@@ -13483,6 +13562,8 @@ SUITE(extraction) {
1348313562
RUN_TEST(complexity_wide_body_exceeds_old_stack_cap);
1348413563
RUN_TEST(count_branching_wide_body_exceeds_old_stack_cap);
1348513564
RUN_TEST(body_tokens_wide_body_samples_from_the_start);
13565+
RUN_TEST(yaml_key_paths_wide_mapping_exceed_old_stack_cap);
13566+
RUN_TEST(yaml_infra_bindings_wide_sequence_exceed_old_stack_cap);
1348613567
RUN_TEST(extract_c_ifdef_split_brace_fn_recovered_issue961);
1348713568
RUN_TEST(extract_cpp_preproc_signature_gap_issue946);
1348813569
RUN_TEST(extract_cpp_preproc_macro_generated_callable_skipped_issue949);

‎tests/test_lang_contract.c‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1553,6 +1553,39 @@ TEST(contract_edge_infra_routes_from_deploy_configs_still_minted) {
15531553
PASS();
15541554
}
15551555

1556+
/* #2388: the YAML key-path walk kept its pending mappings in a 256-frame stack
1557+
* and stopped pushing once full. A `data:` payload block with more nested
1558+
* entries than that lost the full `data.<entry>.webhook_url` path of every entry
1559+
* past the cap; only the shorter per-mapping refs survived, the payload deny
1560+
* never saw them, and each of those values minted an infra Route (44 of 300
1561+
* here). Every payload value must stay unminted however wide the block is. */
1562+
enum { WIDE_PAYLOAD_ENTRIES = 300 };
1563+
1564+
TEST(contract_edge_no_infra_routes_from_wide_payload_block_issue2388) {
1565+
size_t cap = (size_t)WIDE_PAYLOAD_ENTRIES * 64 + 16;
1566+
char *src = malloc(cap);
1567+
ASSERT_NOT_NULL(src);
1568+
size_t pos = (size_t)snprintf(src, cap, "data:\n");
1569+
for (int i = 0; i < WIDE_PAYLOAD_ENTRIES; i++) {
1570+
pos +=
1571+
(size_t)snprintf(src + pos, cap - pos,
1572+
" svc%d:\n webhook_url: https://h%d.payload.example/hook\n", i, i);
1573+
}
1574+
LangProj lp;
1575+
LangFile f[] = {{"service.yaml", src}};
1576+
cbm_store_t *store = lang_index_files(&lp, f, 1);
1577+
ASSERT_NOT_NULL(store);
1578+
int routes = count_infra_routes_matching(store, lp.project, "payload.example");
1579+
if (routes != 0) {
1580+
fprintf(stderr, " [2388] FAIL wide payload block minted %d infra routes (expected 0)\n",
1581+
routes);
1582+
}
1583+
ASSERT_EQ(routes, 0);
1584+
lang_cleanup(&lp, store);
1585+
free(src);
1586+
PASS();
1587+
}
1588+
15561589
/* A config URL becomes an infra Route only under a key that names an endpoint
15571590
* this service exposes. Reference links are not: across 17 infra repositories
15581591
* the infra Routes were changelog `pull_request` links (deckhouse 6,240),
@@ -1945,6 +1978,7 @@ SUITE(lang_contract) {
19451978
RUN_TEST(contract_edge_python_aliased_import_call_resolves_issue988);
19461979
RUN_TEST(contract_edge_no_infra_routes_from_ci_configs_issue999);
19471980
RUN_TEST(contract_edge_infra_routes_from_deploy_configs_still_minted);
1981+
RUN_TEST(contract_edge_no_infra_routes_from_wide_payload_block_issue2388);
19481982
RUN_TEST(contract_edge_no_infra_routes_from_reference_links);
19491983
RUN_TEST(contract_edge_infra_routes_from_endpoint_key_spellings);
19501984
RUN_TEST(contract_edge_commonjs_require_call_resolves_issue871);

0 commit comments

Comments
 (0)