Skip to content

Commit 5992ea6

Browse files
committed
fix(pipeline): size definition properties for every field
A definition's properties buffer was sized for its docstring, variants and variant_key only; every other field had to share the remaining 2 KB. Fields that do not fit are dropped whole, so a long signature, decorator list or parameter list silently squeezed out the MinHash fingerprint (fp), the structural profile (sp) and the body tokens (bt) appended behind them. Those definitions got no SIMILAR_TO edge and fewer SEMANTICALLY_RELATED edges, and any newly added property (route_path, route_method) pushed more of them out. pd_props_buf and pp_props_buf (twins, kept in lockstep) now count the fixed metrics prefix plus the whole serialized form of every field build_def_props appends, including the reserve the fp/sp/bt room checks use. Definitions that fit keep the stack buffer. Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
1 parent 354ccec commit 5992ea6

3 files changed

Lines changed: 210 additions & 30 deletions

File tree

‎src/pipeline/pass_definitions.c‎

Lines changed: 62 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -309,8 +309,8 @@ static void build_def_props(char *buf, size_t bufsize, const CBMDefinition *def)
309309
}
310310
size_t pos = (size_t)n;
311311
append_json_string(buf, bufsize, &pos, "docstring", def->docstring);
312-
/* Right after the docstring: the buffer is sized for exactly these three
313-
* uncapped fields (pd_props_buf), so none can be squeezed out. */
312+
/* The buffer is sized for every field below (pd_props_buf), so none can be
313+
* squeezed out. */
314314
append_json_raw(buf, bufsize, &pos, "variants", def->variants);
315315
append_json_string(buf, bufsize, &pos, "variant_key", def->variant_key);
316316
append_json_string(buf, bufsize, &pos, "signature", def->signature);
@@ -353,24 +353,71 @@ static void build_def_props(char *buf, size_t bufsize, const CBMDefinition *def)
353353
}
354354
}
355355

356-
/* A def's properties buffer: CBM_SZ_2K for every other field plus the whole
357-
* serialized docstring, variants and variant_key fields, which have no length
358-
* cap (a field that does not fit is dropped whole). Returns `stack` for a def
359-
* with none of them, or when the larger buffer cannot be allocated. Twin of
360-
* pass_parallel.c -- keep both in sync. */
361-
static char *pd_props_buf(const CBMDefinition *def, char *stack, size_t *size) {
362-
size_t need = *size;
363-
if (def->docstring && def->docstring[0]) {
364-
need += strlen("docstring") + def_json_escaped_len(def->docstring) + PD_JSON_FIELD_OVERHEAD;
356+
/* Upper bound of build_def_props' fixed prefix: the Function/Method metrics
357+
* literal (about 330 B) with nine ints and seven true/false/test_role strings at
358+
* their widest stays under 600 B. */
359+
enum { PD_PROPS_PREFIX_MAX = CBM_SZ_1K };
360+
361+
/* Serialized size of one optional field as append_json_string emits it
362+
* (0 when the field is absent). */
363+
static size_t pd_str_field_len(const char *key, const char *val) {
364+
return val && val[0] ? strlen(key) + def_json_escaped_len(val) + PD_JSON_FIELD_OVERHEAD : 0;
365+
}
366+
367+
/* Serialized size of a string array as append_json_str_array emits it. */
368+
static size_t pd_arr_field_len(const char *key, const char **arr) {
369+
if (!arr || !arr[0]) {
370+
return 0;
371+
}
372+
size_t n = strlen(key) + PD_JSON_FIELD_OVERHEAD;
373+
for (int i = 0; arr[i]; i++) {
374+
n += def_json_escaped_len(arr[i]) + PD_ESC_SPACE + (i > 0 ? SKIP_ONE : 0);
365375
}
376+
return n;
377+
}
378+
379+
/* A def's properties buffer, sized so EVERY field build_def_props appends fits:
380+
* the fixed metrics prefix (PD_PROPS_PREFIX_MAX) plus each optional field's
381+
* whole serialized form. A field that does not fit is dropped whole, so a buffer
382+
* sized for only some of them let a long signature, decorator list or parameter
383+
* list squeeze out the MinHash fingerprint ("fp"), structural profile ("sp") and
384+
* body tokens ("bt") behind them: no SIMILAR_TO edge and fewer
385+
* SEMANTICALLY_RELATED edges for exactly the widest definitions (jellyfin's
386+
* ImageController actions, 2026-10-18). fp/sp/bt are appended behind their own
387+
* room checks, so both the check's reserve and the field are counted. Returns
388+
* `stack` when everything fits it, or when the larger buffer cannot be
389+
* allocated. Twin of pass_parallel.c -- keep both in sync. */
390+
static char *pd_props_buf(const CBMDefinition *def, char *stack, size_t *size) {
391+
size_t need = PD_PROPS_PREFIX_MAX + pd_str_field_len("docstring", def->docstring);
366392
if (def->variants && def->variants[0]) {
367393
need += strlen("variants") + strlen(def->variants) + PD_JSON_FIELD_OVERHEAD;
368394
}
369-
if (def->variant_key && def->variant_key[0]) {
370-
need +=
371-
strlen("variant_key") + def_json_escaped_len(def->variant_key) + PD_JSON_FIELD_OVERHEAD;
395+
need += pd_str_field_len("variant_key", def->variant_key) +
396+
pd_str_field_len("signature", def->signature) +
397+
pd_str_field_len("return_type", def->return_type) +
398+
pd_str_field_len("parent_class", def->parent_class) +
399+
pd_arr_field_len("decorators", def->decorators) +
400+
pd_arr_field_len("base_classes", def->base_classes) +
401+
pd_arr_field_len("param_names", def->param_names) +
402+
pd_arr_field_len("param_types", def->param_types) +
403+
pd_str_field_len("route_path", def->route_path) +
404+
pd_str_field_len("route_method", def->route_method) +
405+
pd_str_field_len("http_client", def->http_client) +
406+
pd_str_field_len("http_base_url", def->http_base_url);
407+
for (const char **kv = def->extra_props; kv && kv[0] && kv[SKIP_ONE]; kv += PAIR_LEN) {
408+
need += pd_str_field_len(kv[0], kv[SKIP_ONE]);
409+
}
410+
if (def->fingerprint && def->fingerprint_k > 0) {
411+
need += CBM_MINHASH_HEX_LEN + CBM_MINHASH_JSON_OVERHEAD + SKIP_ONE;
412+
}
413+
if (def->structural_profile) {
414+
need += CBM_AST_PROFILE_BUF + pd_str_field_len("sp", def->structural_profile) + SKIP_ONE;
415+
}
416+
if (def->body_tokens) {
417+
need += CBM_SZ_512 + pd_str_field_len("bt", def->body_tokens) + SKIP_ONE;
372418
}
373-
if (need == *size) {
419+
need += PD_ESC_SPACE; /* the closing '}' + NUL */
420+
if (need <= *size) {
374421
return stack;
375422
}
376423
char *buf = cbm_alloc(CBM_MEM_CLASS_GBUF_STRING, need);

‎src/pipeline/pass_parallel.c‎

Lines changed: 62 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -539,8 +539,8 @@ static void build_def_props(char *buf, size_t bufsize, const CBMDefinition *def)
539539
}
540540
size_t pos = (size_t)n;
541541
append_json_string(buf, bufsize, &pos, "docstring", def->docstring);
542-
/* Right after the docstring: the buffer is sized for exactly these three
543-
* uncapped fields (pp_props_buf), so none can be squeezed out. */
542+
/* The buffer is sized for every field below (pp_props_buf), so none can be
543+
* squeezed out. */
544544
append_json_raw(buf, bufsize, &pos, "variants", def->variants);
545545
append_json_string(buf, bufsize, &pos, "variant_key", def->variant_key);
546546
append_json_string(buf, bufsize, &pos, "signature", def->signature);
@@ -731,24 +731,71 @@ typedef struct {
731731
* to the response/logfile — this only throttles the stderr noise). */
732732
enum { PP_OVERSIZED_WARN_MAX = 32 };
733733

734-
/* A def's properties buffer: CBM_SZ_2K for every other field plus the whole
735-
* serialized docstring, variants and variant_key fields, which have no length
736-
* cap (a field that does not fit is dropped whole). Returns `stack` for a def
737-
* with none of them, or when the larger buffer cannot be allocated. Twin of
738-
* pass_definitions.c -- keep both in sync. */
739-
static char *pp_props_buf(const CBMDefinition *def, char *stack, size_t *size) {
740-
size_t need = *size;
741-
if (def->docstring && def->docstring[0]) {
742-
need += strlen("docstring") + pp_json_escaped_len(def->docstring) + PP_JSON_FIELD_OVERHEAD;
734+
/* Upper bound of build_def_props' fixed prefix: the Function/Method metrics
735+
* literal (about 330 B) with nine ints and seven true/false/test_role strings at
736+
* their widest stays under 600 B. */
737+
enum { PP_PROPS_PREFIX_MAX = CBM_SZ_1K };
738+
739+
/* Serialized size of one optional field as append_json_string emits it
740+
* (0 when the field is absent). */
741+
static size_t pp_str_field_len(const char *key, const char *val) {
742+
return val && val[0] ? strlen(key) + pp_json_escaped_len(val) + PP_JSON_FIELD_OVERHEAD : 0;
743+
}
744+
745+
/* Serialized size of a string array as append_json_str_array emits it. */
746+
static size_t pp_arr_field_len(const char *key, const char **arr) {
747+
if (!arr || !arr[0]) {
748+
return 0;
749+
}
750+
size_t n = strlen(key) + PP_JSON_FIELD_OVERHEAD;
751+
for (int i = 0; arr[i]; i++) {
752+
n += pp_json_escaped_len(arr[i]) + PP_ESC_SPACE + (i > 0 ? SKIP_ONE : 0);
743753
}
754+
return n;
755+
}
756+
757+
/* A def's properties buffer, sized so EVERY field build_def_props appends fits:
758+
* the fixed metrics prefix (PP_PROPS_PREFIX_MAX) plus each optional field's
759+
* whole serialized form. A field that does not fit is dropped whole, so a buffer
760+
* sized for only some of them let a long signature, decorator list or parameter
761+
* list squeeze out the MinHash fingerprint ("fp"), structural profile ("sp") and
762+
* body tokens ("bt") behind them: no SIMILAR_TO edge and fewer
763+
* SEMANTICALLY_RELATED edges for exactly the widest definitions (jellyfin's
764+
* ImageController actions, 2026-10-18). fp/sp/bt are appended behind their own
765+
* room checks, so both the check's reserve and the field are counted. Returns
766+
* `stack` when everything fits it, or when the larger buffer cannot be
767+
* allocated. Twin of pass_definitions.c -- keep both in sync. */
768+
static char *pp_props_buf(const CBMDefinition *def, char *stack, size_t *size) {
769+
size_t need = PP_PROPS_PREFIX_MAX + pp_str_field_len("docstring", def->docstring);
744770
if (def->variants && def->variants[0]) {
745771
need += strlen("variants") + strlen(def->variants) + PP_JSON_FIELD_OVERHEAD;
746772
}
747-
if (def->variant_key && def->variant_key[0]) {
748-
need +=
749-
strlen("variant_key") + pp_json_escaped_len(def->variant_key) + PP_JSON_FIELD_OVERHEAD;
773+
need += pp_str_field_len("variant_key", def->variant_key) +
774+
pp_str_field_len("signature", def->signature) +
775+
pp_str_field_len("return_type", def->return_type) +
776+
pp_str_field_len("parent_class", def->parent_class) +
777+
pp_arr_field_len("decorators", def->decorators) +
778+
pp_arr_field_len("base_classes", def->base_classes) +
779+
pp_arr_field_len("param_names", def->param_names) +
780+
pp_arr_field_len("param_types", def->param_types) +
781+
pp_str_field_len("route_path", def->route_path) +
782+
pp_str_field_len("route_method", def->route_method) +
783+
pp_str_field_len("http_client", def->http_client) +
784+
pp_str_field_len("http_base_url", def->http_base_url);
785+
for (const char **kv = def->extra_props; kv && kv[0] && kv[SKIP_ONE]; kv += PAIR_LEN) {
786+
need += pp_str_field_len(kv[0], kv[SKIP_ONE]);
787+
}
788+
if (def->fingerprint && def->fingerprint_k > 0) {
789+
need += CBM_MINHASH_HEX_LEN + CBM_MINHASH_JSON_OVERHEAD + SKIP_ONE;
790+
}
791+
if (def->structural_profile) {
792+
need += CBM_AST_PROFILE_BUF + pp_str_field_len("sp", def->structural_profile) + SKIP_ONE;
793+
}
794+
if (def->body_tokens) {
795+
need += CBM_SZ_512 + pp_str_field_len("bt", def->body_tokens) + SKIP_ONE;
750796
}
751-
if (need == *size) {
797+
need += PP_ESC_SPACE; /* the closing '}' + NUL */
798+
if (need <= *size) {
752799
return stack;
753800
}
754801
char *buf = cbm_alloc(CBM_MEM_CLASS_GBUF_STRING, need);

‎tests/test_parallel.c‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -847,6 +847,91 @@ TEST(parallel_args_json_no_overflow) {
847847
PASS();
848848
}
849849

850+
/* F14: a definition's properties buffer was sized for its docstring and variants
851+
* only. A wide signature (50 annotated parameters) filled the rest of the 2 KB, and
852+
* the MinHash fingerprint "fp", structural profile "sp" and body tokens "bt"
853+
* appended behind it were dropped whole: no SIMILAR_TO edge and fewer
854+
* SEMANTICALLY_RELATED edges for exactly the widest definitions (jellyfin's
855+
* ImageController actions, 2026-10-18). `small` has the same body and is the
856+
* control that the fixture produces all three at all. Both resolvers. */
857+
static const char f14_body[] =
858+
" total = 0\n"
859+
" for item in range(10):\n"
860+
" if item % 3 == 0:\n"
861+
" total += item * 2\n"
862+
" elif item % 5 == 1:\n"
863+
" total -= len(str(item))\n"
864+
" else:\n"
865+
" total ^= hash((item, total))\n"
866+
" values = [x * x for x in range(total % 7 + 3)]\n"
867+
" mapping = {str(k): v for k, v in enumerate(values)}\n"
868+
" while total > 100:\n"
869+
" total //= 3\n"
870+
" try:\n"
871+
" result = sorted(mapping.items(), key=lambda kv: kv[1])\n"
872+
" except ValueError as err:\n"
873+
" result = [(\"error\", str(err))]\n"
874+
" return result, total, mapping\n";
875+
876+
static bool f14_props_have(const cbm_gbuf_t *gb, const char *project, const char *name,
877+
const char *venue) {
878+
char *qn = cbm_pipeline_fqn_compute(project, "pkg/wide.py", name);
879+
const cbm_gbuf_node_t *n = qn ? cbm_gbuf_find_by_qn(gb, qn) : NULL;
880+
free(qn);
881+
const char *p = n && n->properties_json ? n->properties_json : "";
882+
bool fp = strstr(p, "\"fp\":\"") != NULL;
883+
bool sp = strstr(p, "\"sp\":\"") != NULL;
884+
bool bt = strstr(p, "\"bt\":\"") != NULL;
885+
bool sig = strstr(p, "\"signature\":\"") != NULL;
886+
printf(" f14 %s %s: found=%d len=%zu fp=%d sp=%d bt=%d signature=%d\n", venue, name, n != NULL,
887+
strlen(p), fp, sp, bt, sig);
888+
return n && fp && sp && bt && sig;
889+
}
890+
891+
TEST(parallel_wide_definition_keeps_fp_sp_bt_f14) {
892+
char params[CBM_SZ_4K];
893+
size_t pos = 0;
894+
for (int i = 0; i < 50; i++) {
895+
pos += (size_t)snprintf(params + pos, sizeof(params) - pos, "%sp%02d: Dict[str, int] = None",
896+
i ? ", " : "", i);
897+
}
898+
char src[CBM_SZ_8K];
899+
snprintf(src, sizeof(src), "def small(a, b):\n%s\n\ndef wide(%s):\n%s", f14_body, params,
900+
f14_body);
901+
const char *project = "cbm_f14";
902+
char tmpdir[256];
903+
snprintf(tmpdir, sizeof(tmpdir), "/tmp/cbm_par_f14_XXXXXX");
904+
if (!cbm_mkdtemp(tmpdir)) {
905+
FAIL("mkdtemp failed");
906+
}
907+
char path[512];
908+
snprintf(path, sizeof(path), "%s/pkg/wide.py", tmpdir);
909+
if (th_write_file(path, src) != 0) {
910+
th_rmtree(tmpdir);
911+
FAIL("failed to write the F14 fixture");
912+
}
913+
cbm_file_info_t files[1] = {0};
914+
files[0].path = path;
915+
files[0].rel_path = (char *)"pkg/wide.py";
916+
files[0].language = CBM_LANG_PYTHON;
917+
cbm_gbuf_t *graphs[2] = {run_sequential(project, tmpdir, files, 1),
918+
run_parallel(project, tmpdir, files, 1, 2)};
919+
ASSERT_NOT_NULL(graphs[0]);
920+
ASSERT_NOT_NULL(graphs[1]);
921+
bool all = true;
922+
for (int g = 0; g < 2; g++) {
923+
const char *venue = g == 0 ? "sequential" : "parallel";
924+
bool control = f14_props_have(graphs[g], project, "small", venue);
925+
bool wide = f14_props_have(graphs[g], project, "wide", venue);
926+
all = all && control && wide;
927+
}
928+
cbm_gbuf_free(graphs[0]);
929+
cbm_gbuf_free(graphs[1]);
930+
th_rmtree(tmpdir);
931+
ASSERT_TRUE(all);
932+
PASS();
933+
}
934+
850935
/* ── Graph buffer merge tests ─────────────────────────────────────── */
851936

852937
TEST(gbuf_shared_ids_unique) {
@@ -5346,6 +5431,7 @@ SUITE(parallel) {
53465431
RUN_TEST(parallel_post_extract_projection_spills_before_resolve);
53475432
RUN_TEST(parallel_empty_files);
53485433
RUN_TEST(parallel_args_json_no_overflow);
5434+
RUN_TEST(parallel_wide_definition_keeps_fp_sp_bt_f14);
53495435

53505436
/* Cleanup shared state */
53515437
parity_teardown();

0 commit comments

Comments
 (0)