Skip to content

Commit 1a7839d

Browse files
authored
Merge pull request DeusData#2687 from DeusData/fix/def-props-buffer-every-field
fix(pipeline): size definition properties for every field
2 parents 1346ac4 + 9cc3b76 commit 1a7839d

3 files changed

Lines changed: 210 additions & 30 deletions

File tree

‎src/pipeline/pass_definitions.c‎

Lines changed: 62 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -309,8 +309,8 @@ static void build_def_props(char *buf, size_t bufsize, const CBMDefinition *def)
309309
}
310310
size_t pos = (size_t)n;
311311
append_json_string(buf, bufsize, &pos, "docstring", def->docstring);
312-
/* Right after the docstring: the buffer is sized for exactly these three
313-
* uncapped fields (pd_props_buf), so none can be squeezed out. */
312+
/* The buffer is sized for every field below (pd_props_buf), so none can be
313+
* squeezed out. */
314314
append_json_raw(buf, bufsize, &pos, "variants", def->variants);
315315
append_json_string(buf, bufsize, &pos, "variant_key", def->variant_key);
316316
append_json_string(buf, bufsize, &pos, "signature", def->signature);
@@ -353,24 +353,71 @@ static void build_def_props(char *buf, size_t bufsize, const CBMDefinition *def)
353353
}
354354
}
355355

356-
/* A def's properties buffer: CBM_SZ_2K for every other field plus the whole
357-
* serialized docstring, variants and variant_key fields, which have no length
358-
* cap (a field that does not fit is dropped whole). Returns `stack` for a def
359-
* with none of them, or when the larger buffer cannot be allocated. Twin of
360-
* pass_parallel.c -- keep both in sync. */
361-
static char *pd_props_buf(const CBMDefinition *def, char *stack, size_t *size) {
362-
size_t need = *size;
363-
if (def->docstring && def->docstring[0]) {
364-
need += strlen("docstring") + def_json_escaped_len(def->docstring) + PD_JSON_FIELD_OVERHEAD;
356+
/* Upper bound of build_def_props' fixed prefix: the Function/Method metrics
357+
* literal (about 330 B) with nine ints and seven true/false/test_role strings at
358+
* their widest stays under 600 B. */
359+
enum { PD_PROPS_PREFIX_MAX = CBM_SZ_1K };
360+
361+
/* Serialized size of one optional field as append_json_string emits it
362+
* (0 when the field is absent). */
363+
static size_t pd_str_field_len(const char *key, const char *val) {
364+
return val && val[0] ? strlen(key) + def_json_escaped_len(val) + PD_JSON_FIELD_OVERHEAD : 0;
365+
}
366+
367+
/* Serialized size of a string array as append_json_str_array emits it. */
368+
static size_t pd_arr_field_len(const char *key, const char **arr) {
369+
if (!arr || !arr[0]) {
370+
return 0;
371+
}
372+
size_t n = strlen(key) + PD_JSON_FIELD_OVERHEAD;
373+
for (int i = 0; arr[i]; i++) {
374+
n += def_json_escaped_len(arr[i]) + PD_ESC_SPACE + (i > 0 ? SKIP_ONE : 0);
365375
}
376+
return n;
377+
}
378+
379+
/* A def's properties buffer, sized so EVERY field build_def_props appends fits:
380+
* the fixed metrics prefix (PD_PROPS_PREFIX_MAX) plus each optional field's
381+
* whole serialized form. A field that does not fit is dropped whole, so a buffer
382+
* sized for only some of them let a long signature, decorator list or parameter
383+
* list squeeze out the MinHash fingerprint ("fp"), structural profile ("sp") and
384+
* body tokens ("bt") behind them: no SIMILAR_TO edge and fewer
385+
* SEMANTICALLY_RELATED edges for exactly the widest definitions (jellyfin's
386+
* ImageController actions, 2026-10-18). fp/sp/bt are appended behind their own
387+
* room checks, so both the check's reserve and the field are counted. Returns
388+
* `stack` when everything fits it, or when the larger buffer cannot be
389+
* allocated. Twin of pass_parallel.c -- keep both in sync. */
390+
static char *pd_props_buf(const CBMDefinition *def, char *stack, size_t *size) {
391+
size_t need = PD_PROPS_PREFIX_MAX + pd_str_field_len("docstring", def->docstring);
366392
if (def->variants && def->variants[0]) {
367393
need += strlen("variants") + strlen(def->variants) + PD_JSON_FIELD_OVERHEAD;
368394
}
369-
if (def->variant_key && def->variant_key[0]) {
370-
need +=
371-
strlen("variant_key") + def_json_escaped_len(def->variant_key) + PD_JSON_FIELD_OVERHEAD;
395+
need += pd_str_field_len("variant_key", def->variant_key) +
396+
pd_str_field_len("signature", def->signature) +
397+
pd_str_field_len("return_type", def->return_type) +
398+
pd_str_field_len("parent_class", def->parent_class) +
399+
pd_arr_field_len("decorators", def->decorators) +
400+
pd_arr_field_len("base_classes", def->base_classes) +
401+
pd_arr_field_len("param_names", def->param_names) +
402+
pd_arr_field_len("param_types", def->param_types) +
403+
pd_str_field_len("route_path", def->route_path) +
404+
pd_str_field_len("route_method", def->route_method) +
405+
pd_str_field_len("http_client", def->http_client) +
406+
pd_str_field_len("http_base_url", def->http_base_url);
407+
for (const char **kv = def->extra_props; kv && kv[0] && kv[SKIP_ONE]; kv += PAIR_LEN) {
408+
need += pd_str_field_len(kv[0], kv[SKIP_ONE]);
409+
}
410+
if (def->fingerprint && def->fingerprint_k > 0) {
411+
need += CBM_MINHASH_HEX_LEN + CBM_MINHASH_JSON_OVERHEAD + SKIP_ONE;
412+
}
413+
if (def->structural_profile) {
414+
need += CBM_AST_PROFILE_BUF + pd_str_field_len("sp", def->structural_profile) + SKIP_ONE;
415+
}
416+
if (def->body_tokens) {
417+
need += CBM_SZ_512 + pd_str_field_len("bt", def->body_tokens) + SKIP_ONE;
372418
}
373-
if (need == *size) {
419+
need += PD_ESC_SPACE; /* the closing '}' + NUL */
420+
if (need <= *size) {
374421
return stack;
375422
}
376423
char *buf = cbm_alloc(CBM_MEM_CLASS_GBUF_STRING, need);

‎src/pipeline/pass_parallel.c‎

Lines changed: 62 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -539,8 +539,8 @@ static void build_def_props(char *buf, size_t bufsize, const CBMDefinition *def)
539539
}
540540
size_t pos = (size_t)n;
541541
append_json_string(buf, bufsize, &pos, "docstring", def->docstring);
542-
/* Right after the docstring: the buffer is sized for exactly these three
543-
* uncapped fields (pp_props_buf), so none can be squeezed out. */
542+
/* The buffer is sized for every field below (pp_props_buf), so none can be
543+
* squeezed out. */
544544
append_json_raw(buf, bufsize, &pos, "variants", def->variants);
545545
append_json_string(buf, bufsize, &pos, "variant_key", def->variant_key);
546546
append_json_string(buf, bufsize, &pos, "signature", def->signature);
@@ -731,24 +731,71 @@ typedef struct {
731731
* to the response/logfile — this only throttles the stderr noise). */
732732
enum { PP_OVERSIZED_WARN_MAX = 32 };
733733

734-
/* A def's properties buffer: CBM_SZ_2K for every other field plus the whole
735-
* serialized docstring, variants and variant_key fields, which have no length
736-
* cap (a field that does not fit is dropped whole). Returns `stack` for a def
737-
* with none of them, or when the larger buffer cannot be allocated. Twin of
738-
* pass_definitions.c -- keep both in sync. */
739-
static char *pp_props_buf(const CBMDefinition *def, char *stack, size_t *size) {
740-
size_t need = *size;
741-
if (def->docstring && def->docstring[0]) {
742-
need += strlen("docstring") + pp_json_escaped_len(def->docstring) + PP_JSON_FIELD_OVERHEAD;
734+
/* Upper bound of build_def_props' fixed prefix: the Function/Method metrics
735+
* literal (about 330 B) with nine ints and seven true/false/test_role strings at
736+
* their widest stays under 600 B. */
737+
enum { PP_PROPS_PREFIX_MAX = CBM_SZ_1K };
738+
739+
/* Serialized size of one optional field as append_json_string emits it
740+
* (0 when the field is absent). */
741+
static size_t pp_str_field_len(const char *key, const char *val) {
742+
return val && val[0] ? strlen(key) + pp_json_escaped_len(val) + PP_JSON_FIELD_OVERHEAD : 0;
743+
}
744+
745+
/* Serialized size of a string array as append_json_str_array emits it. */
746+
static size_t pp_arr_field_len(const char *key, const char **arr) {
747+
if (!arr || !arr[0]) {
748+
return 0;
749+
}
750+
size_t n = strlen(key) + PP_JSON_FIELD_OVERHEAD;
751+
for (int i = 0; arr[i]; i++) {
752+
n += pp_json_escaped_len(arr[i]) + PP_ESC_SPACE + (i > 0 ? SKIP_ONE : 0);
743753
}
754+
return n;
755+
}
756+
757+
/* A def's properties buffer, sized so EVERY field build_def_props appends fits:
758+
* the fixed metrics prefix (PP_PROPS_PREFIX_MAX) plus each optional field's
759+
* whole serialized form. A field that does not fit is dropped whole, so a buffer
760+
* sized for only some of them let a long signature, decorator list or parameter
761+
* list squeeze out the MinHash fingerprint ("fp"), structural profile ("sp") and
762+
* body tokens ("bt") behind them: no SIMILAR_TO edge and fewer
763+
* SEMANTICALLY_RELATED edges for exactly the widest definitions (jellyfin's
764+
* ImageController actions, 2026-10-18). fp/sp/bt are appended behind their own
765+
* room checks, so both the check's reserve and the field are counted. Returns
766+
* `stack` when everything fits it, or when the larger buffer cannot be
767+
* allocated. Twin of pass_definitions.c -- keep both in sync. */
768+
static char *pp_props_buf(const CBMDefinition *def, char *stack, size_t *size) {
769+
size_t need = PP_PROPS_PREFIX_MAX + pp_str_field_len("docstring", def->docstring);
744770
if (def->variants && def->variants[0]) {
745771
need += strlen("variants") + strlen(def->variants) + PP_JSON_FIELD_OVERHEAD;
746772
}
747-
if (def->variant_key && def->variant_key[0]) {
748-
need +=
749-
strlen("variant_key") + pp_json_escaped_len(def->variant_key) + PP_JSON_FIELD_OVERHEAD;
773+
need += pp_str_field_len("variant_key", def->variant_key) +
774+
pp_str_field_len("signature", def->signature) +
775+
pp_str_field_len("return_type", def->return_type) +
776+
pp_str_field_len("parent_class", def->parent_class) +
777+
pp_arr_field_len("decorators", def->decorators) +
778+
pp_arr_field_len("base_classes", def->base_classes) +
779+
pp_arr_field_len("param_names", def->param_names) +
780+
pp_arr_field_len("param_types", def->param_types) +
781+
pp_str_field_len("route_path", def->route_path) +
782+
pp_str_field_len("route_method", def->route_method) +
783+
pp_str_field_len("http_client", def->http_client) +
784+
pp_str_field_len("http_base_url", def->http_base_url);
785+
for (const char **kv = def->extra_props; kv && kv[0] && kv[SKIP_ONE]; kv += PAIR_LEN) {
786+
need += pp_str_field_len(kv[0], kv[SKIP_ONE]);
787+
}
788+
if (def->fingerprint && def->fingerprint_k > 0) {
789+
need += CBM_MINHASH_HEX_LEN + CBM_MINHASH_JSON_OVERHEAD + SKIP_ONE;
790+
}
791+
if (def->structural_profile) {
792+
need += CBM_AST_PROFILE_BUF + pp_str_field_len("sp", def->structural_profile) + SKIP_ONE;
793+
}
794+
if (def->body_tokens) {
795+
need += CBM_SZ_512 + pp_str_field_len("bt", def->body_tokens) + SKIP_ONE;
750796
}
751-
if (need == *size) {
797+
need += PP_ESC_SPACE; /* the closing '}' + NUL */
798+
if (need <= *size) {
752799
return stack;
753800
}
754801
char *buf = cbm_alloc(CBM_MEM_CLASS_GBUF_STRING, need);

‎tests/test_parallel.c‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -847,6 +847,91 @@ TEST(parallel_args_json_no_overflow) {
847847
PASS();
848848
}
849849

850+
/* F14: a definition's properties buffer was sized for its docstring and variants
851+
* only. A wide signature (50 annotated parameters) filled the rest of the 2 KB, and
852+
* the MinHash fingerprint "fp", structural profile "sp" and body tokens "bt"
853+
* appended behind it were dropped whole: no SIMILAR_TO edge and fewer
854+
* SEMANTICALLY_RELATED edges for exactly the widest definitions (jellyfin's
855+
* ImageController actions, 2026-10-18). `small` has the same body and is the
856+
* control that the fixture produces all three at all. Both resolvers. */
857+
static const char f14_body[] =
858+
" total = 0\n"
859+
" for item in range(10):\n"
860+
" if item % 3 == 0:\n"
861+
" total += item * 2\n"
862+
" elif item % 5 == 1:\n"
863+
" total -= len(str(item))\n"
864+
" else:\n"
865+
" total ^= hash((item, total))\n"
866+
" values = [x * x for x in range(total % 7 + 3)]\n"
867+
" mapping = {str(k): v for k, v in enumerate(values)}\n"
868+
" while total > 100:\n"
869+
" total //= 3\n"
870+
" try:\n"
871+
" result = sorted(mapping.items(), key=lambda kv: kv[1])\n"
872+
" except ValueError as err:\n"
873+
" result = [(\"error\", str(err))]\n"
874+
" return result, total, mapping\n";
875+
876+
static bool f14_props_have(const cbm_gbuf_t *gb, const char *project, const char *name,
877+
const char *venue) {
878+
char *qn = cbm_pipeline_fqn_compute(project, "pkg/wide.py", name);
879+
const cbm_gbuf_node_t *n = qn ? cbm_gbuf_find_by_qn(gb, qn) : NULL;
880+
free(qn);
881+
const char *p = n && n->properties_json ? n->properties_json : "";
882+
bool fp = strstr(p, "\"fp\":\"") != NULL;
883+
bool sp = strstr(p, "\"sp\":\"") != NULL;
884+
bool bt = strstr(p, "\"bt\":\"") != NULL;
885+
bool sig = strstr(p, "\"signature\":\"") != NULL;
886+
printf(" f14 %s %s: found=%d len=%zu fp=%d sp=%d bt=%d signature=%d\n", venue, name, n != NULL,
887+
strlen(p), fp, sp, bt, sig);
888+
return n && fp && sp && bt && sig;
889+
}
890+
891+
TEST(parallel_wide_definition_keeps_fp_sp_bt_f14) {
892+
char params[CBM_SZ_4K];
893+
size_t pos = 0;
894+
for (int i = 0; i < 50; i++) {
895+
pos += (size_t)snprintf(params + pos, sizeof(params) - pos, "%sp%02d: Dict[str, int] = None",
896+
i ? ", " : "", i);
897+
}
898+
char src[CBM_SZ_8K];
899+
snprintf(src, sizeof(src), "def small(a, b):\n%s\n\ndef wide(%s):\n%s", f14_body, params,
900+
f14_body);
901+
const char *project = "cbm_f14";
902+
char tmpdir[256];
903+
snprintf(tmpdir, sizeof(tmpdir), "/tmp/cbm_par_f14_XXXXXX");
904+
if (!cbm_mkdtemp(tmpdir)) {
905+
FAIL("mkdtemp failed");
906+
}
907+
char path[512];
908+
snprintf(path, sizeof(path), "%s/pkg/wide.py", tmpdir);
909+
if (th_write_file(path, src) != 0) {
910+
th_rmtree(tmpdir);
911+
FAIL("failed to write the F14 fixture");
912+
}
913+
cbm_file_info_t files[1] = {0};
914+
files[0].path = path;
915+
files[0].rel_path = (char *)"pkg/wide.py";
916+
files[0].language = CBM_LANG_PYTHON;
917+
cbm_gbuf_t *graphs[2] = {run_sequential(project, tmpdir, files, 1),
918+
run_parallel(project, tmpdir, files, 1, 2)};
919+
ASSERT_NOT_NULL(graphs[0]);
920+
ASSERT_NOT_NULL(graphs[1]);
921+
bool all = true;
922+
for (int g = 0; g < 2; g++) {
923+
const char *venue = g == 0 ? "sequential" : "parallel";
924+
bool control = f14_props_have(graphs[g], project, "small", venue);
925+
bool wide = f14_props_have(graphs[g], project, "wide", venue);
926+
all = all && control && wide;
927+
}
928+
cbm_gbuf_free(graphs[0]);
929+
cbm_gbuf_free(graphs[1]);
930+
th_rmtree(tmpdir);
931+
ASSERT_TRUE(all);
932+
PASS();
933+
}
934+
850935
/* ── Graph buffer merge tests ─────────────────────────────────────── */
851936

852937
TEST(gbuf_shared_ids_unique) {
@@ -5346,6 +5431,7 @@ SUITE(parallel) {
53465431
RUN_TEST(parallel_post_extract_projection_spills_before_resolve);
53475432
RUN_TEST(parallel_empty_files);
53485433
RUN_TEST(parallel_args_json_no_overflow);
5434+
RUN_TEST(parallel_wide_definition_keeps_fp_sp_bt_f14);
53495435

53505436
/* Cleanup shared state */
53515437
parity_teardown();

0 commit comments

Comments
 (0)