Skip to content

Commit e369307

Browse files
perf(py-lsp): O(1) type membership in the shared Python cross-registry build
The shared tier-2 registry registered Python defs one at a time without a mid-build finalize, so every Method's receiver probe scanned all registered types linearly: O(methods x types), single-threaded in lsp_cross_prepare. Keep the per-def registration order and back only the existence probe with a cbm_ht set of type QNs, as the Go and Rust builders do; the registry is identical by construction. On the Python 3.14 stdlib + site-packages the builder drops from 67-75 s to 0.20-0.25 s, with a byte-identical graph. Signed-off-by: Nelson Spence <nelson@projectnavi.ai>
1 parent 2278498 commit e369307

3 files changed

Lines changed: 199 additions & 5 deletions

File tree

‎internal/cbm/lsp/py_lsp.c‎

Lines changed: 47 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
#include "py_lsp.h"
1414
#include "../cbm.h"
1515
#include "../helpers.h"
16+
#include "../../../src/foundation/hash_table.h"
1617
#include "tree_sitter/api.h"
1718
#include <ctype.h>
1819
#include <stdint.h>
@@ -4973,10 +4974,36 @@ static const char **py_split_pipe(CBMArena *arena, const char *text) {
49734974
return out;
49744975
}
49754976

4977+
/* Qualified names of reg->types[0..noted), for a caller that registers into a
4978+
* registry it does not finalize until the end (cbm_py_build_cross_registry):
4979+
* that registry's own lookup scans every type. qns == NULL probes the
4980+
* registry instead. */
4981+
typedef struct {
4982+
CBMHashTable *qns;
4983+
int noted;
4984+
} PyKnownTypes;
4985+
4986+
/* Note the types added since the last call. Run after every add_type, so an
4987+
* add that failed silently cannot desync set and registry. If the set cannot
4988+
* grow it is dropped and probes fall back to the registry. */
4989+
static void py_note_known_types(PyKnownTypes *known, const CBMTypeRegistry *reg) {
4990+
for (; known && known->qns && known->noted < reg->type_count; known->noted++) {
4991+
const char *qn = reg->types[known->noted].qualified_name;
4992+
if (!qn || cbm_ht_has(known->qns, qn))
4993+
continue;
4994+
cbm_ht_set(known->qns, qn, (void *)(uintptr_t)(known->noted + 1));
4995+
if (!cbm_ht_has(known->qns, qn)) {
4996+
cbm_ht_free(known->qns);
4997+
known->qns = NULL;
4998+
}
4999+
}
5000+
}
5001+
49765002
/* Build a registry from CBMLSPDef[] supplied by the caller — covers both
4977-
* the source file's own defs and cross-file referenced defs. */
5003+
* the source file's own defs and cross-file referenced defs. known is NULL
5004+
* except for the shared Tier-2 build. */
49785005
static void py_register_lsp_defs(CBMArena *arena, CBMArena *idx_arena, CBMTypeRegistry *reg,
4979-
CBMLSPDef *defs, int def_count) {
5006+
CBMLSPDef *defs, int def_count, PyKnownTypes *known) {
49805007
/* Pass 1: types only — the method pass probes the registry per Method def
49815008
* (receiver auto-registration), which is a LINEAR scan pre-finalize:
49825009
* O(methods x types) per file (same quadratic as php_register_lsp_defs;
@@ -4999,6 +5026,7 @@ static void py_register_lsp_defs(CBMArena *arena, CBMArena *idx_arena, CBMTypeRe
49995026
rt.method_names = py_split_pipe(arena, d->method_names_str);
50005027
}
50015028
cbm_registry_add_type(reg, rt);
5029+
py_note_known_types(known, reg);
50025030
}
50035031
}
50045032
/* idx_arena == NULL skips the mid-build finalize: the tier-2 builder
@@ -5046,13 +5074,17 @@ static void py_register_lsp_defs(CBMArena *arena, CBMArena *idx_arena, CBMTypeRe
50465074

50475075
if (strcmp(d->label, "Method") == 0 && d->receiver_type && d->receiver_type[0]) {
50485076
rf.receiver_type = d->receiver_type; /* borrowed */
5049-
if (!cbm_registry_lookup_type(reg, rf.receiver_type)) {
5077+
bool recv_known = known && known->qns
5078+
? cbm_ht_has(known->qns, rf.receiver_type)
5079+
: cbm_registry_lookup_type(reg, rf.receiver_type) != NULL;
5080+
if (!recv_known) {
50505081
CBMRegisteredType auto_t;
50515082
memset(&auto_t, 0, sizeof(auto_t));
50525083
auto_t.qualified_name = rf.receiver_type;
50535084
const char *dot = strrchr(d->receiver_type, '.');
50545085
auto_t.short_name = dot ? dot + 1 : rf.receiver_type; /* borrowed substring */
50555086
cbm_registry_add_type(reg, auto_t);
5087+
py_note_known_types(known, reg);
50565088
}
50575089
}
50585090
cbm_registry_add_func(reg, rf);
@@ -5092,7 +5124,7 @@ void cbm_run_py_lsp_cross(CBMArena *arena, const char *source, int source_len,
50925124
/* Index allocations go to a per-call scratch arena (see php_lsp_cross). */
50935125
CBMArena idx_arena;
50945126
cbm_arena_init(&idx_arena);
5095-
py_register_lsp_defs(arena, &idx_arena, &reg, defs, def_count);
5127+
py_register_lsp_defs(arena, &idx_arena, &reg, defs, def_count, NULL);
50965128
py_mark_ambiguous_callable_bindings(&reg);
50975129

50985130
/* Finalize registry — O(1) lookups. See go_lsp.c "3c. Finalize"
@@ -5126,14 +5158,24 @@ CBMTypeRegistry *cbm_py_build_cross_registry(CBMArena *arena, CBMLSPDef *defs, i
51265158
cbm_registry_init(reg, arena);
51275159
cbm_python_stdlib_register(reg, arena);
51285160

5161+
/* The registry is not finalized until the end, so probing it for each
5162+
* Method's receiver scanned every type: O(methods x types), single-threaded
5163+
* (~54 s of lsp_cross_prepare on a large Python tree). Probe a set of
5164+
* the registered type QNs instead; registration order is unchanged. Sized
5165+
* for the stdlib seed, not def_count (the mixed all-language count). NULL
5166+
* on allocation failure: the registry probe runs as before. */
5167+
PyKnownTypes known = {cbm_ht_create((uint32_t)reg->type_count), 0};
5168+
py_note_known_types(&known, reg);
5169+
51295170
/* Filter to Python defs only — defs[] is mixed-language all_defs. */
51305171
for (int i = 0; i < def_count; i++) {
51315172
CBMLSPDef *d = &defs[i];
51325173
if (d->lang != CBM_LANG_PYTHON)
51335174
continue;
51345175
/* Reuse the existing register fn on a single-def slice (n=1 inline). */
5135-
py_register_lsp_defs(arena, NULL, reg, d, 1);
5176+
py_register_lsp_defs(arena, NULL, reg, d, 1, &known);
51365177
}
5178+
cbm_ht_free(known.qns);
51375179

51385180
py_mark_ambiguous_callable_bindings(reg);
51395181
cbm_registry_finalize(reg);

‎tests/test_py_lsp.c‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -637,6 +637,91 @@ TEST(pylsp_fused_self_attr_chain_via_overlay) {
637637
PASS();
638638
}
639639

640+
/* Characterization of the shared Tier-2 build order. Defs register one at a
641+
* time in defs[] order, so a Method seen before its Class auto-registers a
642+
* placeholder receiver type that stays ahead of the Class entry, and a Method
643+
* whose receiver is already known adds no type. Any change to how the build
644+
* probes for known receivers must keep this exact sequence. */
645+
TEST(pylsp_cross_registry_interleaved_order) {
646+
CBMArena arena;
647+
cbm_arena_init(&arena);
648+
CBMTypeRegistry *base = cbm_py_build_cross_registry(&arena, NULL, 0);
649+
ASSERT_NOT_NULL(base);
650+
int tb = base->type_count, fb = base->func_count;
651+
652+
CBMLSPDef defs[] = {
653+
{.qualified_name = "t.m.A.m",
654+
.short_name = "m",
655+
.label = "Method",
656+
.receiver_type = "t.m.A",
657+
.def_module_qn = "t.m",
658+
.lang = CBM_LANG_PYTHON},
659+
{.qualified_name = "t.m.A",
660+
.short_name = "A",
661+
.label = "Class",
662+
.def_module_qn = "t.m",
663+
.lang = CBM_LANG_PYTHON},
664+
{.qualified_name = "t.m.B",
665+
.short_name = "B",
666+
.label = "Class",
667+
.def_module_qn = "t.m",
668+
.lang = CBM_LANG_PYTHON},
669+
{.qualified_name = "t.m.B.f",
670+
.short_name = "f",
671+
.label = "Method",
672+
.receiver_type = "t.m.B",
673+
.def_module_qn = "t.m",
674+
.lang = CBM_LANG_PYTHON},
675+
{.qualified_name = "t.m.C.g",
676+
.short_name = "g",
677+
.label = "Method",
678+
.receiver_type = "t.m.C",
679+
.def_module_qn = "t.m",
680+
.lang = CBM_LANG_PYTHON},
681+
/* Receiver auto-registered by C.g just above: no second placeholder. */
682+
{.qualified_name = "t.m.C.h",
683+
.short_name = "h",
684+
.label = "Method",
685+
.receiver_type = "t.m.C",
686+
.def_module_qn = "t.m",
687+
.lang = CBM_LANG_PYTHON},
688+
/* Non-Python defs are skipped by the builder. */
689+
{.qualified_name = "t.m.D",
690+
.short_name = "D",
691+
.label = "Class",
692+
.def_module_qn = "t.m",
693+
.lang = CBM_LANG_GO},
694+
};
695+
CBMTypeRegistry *reg =
696+
cbm_py_build_cross_registry(&arena, defs, (int)(sizeof(defs) / sizeof(defs[0])));
697+
ASSERT_NOT_NULL(reg);
698+
699+
/* Types: placeholder A (from A.m), Class A, Class B, placeholder C. */
700+
ASSERT_EQ(reg->type_count, tb + 4);
701+
ASSERT_TRUE(reg->types[tb].qualified_name == defs[0].receiver_type);
702+
ASSERT_STR_EQ(reg->types[tb].short_name, "A");
703+
ASSERT_TRUE(reg->types[tb + 1].qualified_name == defs[1].qualified_name);
704+
ASSERT_TRUE(reg->types[tb + 2].qualified_name == defs[2].qualified_name);
705+
ASSERT_TRUE(reg->types[tb + 3].qualified_name == defs[4].receiver_type);
706+
ASSERT_STR_EQ(reg->types[tb + 3].short_name, "C");
707+
708+
/* Funcs keep defs[] order. */
709+
ASSERT_EQ(reg->func_count, fb + 4);
710+
ASSERT_STR_EQ(reg->funcs[fb].qualified_name, "t.m.A.m");
711+
ASSERT_STR_EQ(reg->funcs[fb + 1].qualified_name, "t.m.B.f");
712+
ASSERT_STR_EQ(reg->funcs[fb + 2].qualified_name, "t.m.C.g");
713+
ASSERT_STR_EQ(reg->funcs[fb + 2].receiver_type, "t.m.C");
714+
ASSERT_STR_EQ(reg->funcs[fb + 3].qualified_name, "t.m.C.h");
715+
716+
const CBMRegisteredType *a = cbm_registry_lookup_type(reg, "t.m.A");
717+
ASSERT_NOT_NULL(a);
718+
ASSERT_STR_EQ(a->qualified_name, "t.m.A");
719+
ASSERT_NULL(cbm_registry_lookup_type(reg, "t.m.D"));
720+
721+
cbm_arena_destroy(&arena);
722+
PASS();
723+
}
724+
640725
/* Issue #228: a class/static method invoked directly on a CROSS-FILE imported
641726
* class name — ActionRecordX.build_from_text(...) — produced no CALLS edge, so
642727
* the method showed in/out degree 0 and was flagged as dead code. Distinct from
@@ -2194,6 +2279,7 @@ SUITE(py_lsp) {
21942279
/* Phase 9 — cross-file + batch */
21952280
RUN_TEST(pylsp_crossfile_method_dispatch);
21962281
RUN_TEST(pylsp_fused_self_attr_chain_via_overlay);
2282+
RUN_TEST(pylsp_cross_registry_interleaved_order);
21972283
RUN_TEST(pylsp_crossfile_classmethod_on_class_issue228);
21982284
RUN_TEST(pylsp_crossfile_inheritance);
21992285
RUN_TEST(pylsp_batch_two_files);

‎tests/test_py_lsp_scale.c‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,72 @@ TEST(pylsp_scale_linear_growth) {
111111
PASS();
112112
}
113113

114+
/* Time cbm_py_build_cross_registry over n classes, each followed by two
115+
* methods (extraction order): every method probes the registry for its
116+
* receiver while the registry is still being built. */
117+
static double measure_cross_registry(int n_classes, int *out_types) {
118+
CBMArena arena;
119+
cbm_arena_init(&arena);
120+
int n_defs = n_classes * 3;
121+
CBMLSPDef *defs = (CBMLSPDef *)cbm_arena_calloc(&arena, (size_t)n_defs * sizeof(*defs));
122+
if (!defs) {
123+
cbm_arena_destroy(&arena);
124+
return -1.0;
125+
}
126+
for (int i = 0; i < n_classes; i++) {
127+
CBMLSPDef *cls = &defs[i * 3];
128+
cls->qualified_name = cbm_arena_sprintf(&arena, "scale.mod.Cls%d", i);
129+
cls->short_name = cls->qualified_name + strlen("scale.mod.");
130+
cls->label = "Class";
131+
cls->def_module_qn = "scale.mod";
132+
cls->lang = CBM_LANG_PYTHON;
133+
for (int j = 1; j <= 2; j++) {
134+
CBMLSPDef *m = &defs[i * 3 + j];
135+
m->qualified_name = cbm_arena_sprintf(&arena, "%s.m%d", cls->qualified_name, j);
136+
m->short_name = j == 1 ? "m1" : "m2";
137+
m->label = "Method";
138+
m->receiver_type = cls->qualified_name;
139+
m->def_module_qn = "scale.mod";
140+
m->lang = CBM_LANG_PYTHON;
141+
}
142+
}
143+
struct timespec t0, t1;
144+
clock_gettime(CLOCK_MONOTONIC, &t0);
145+
CBMTypeRegistry *reg = cbm_py_build_cross_registry(&arena, defs, n_defs);
146+
clock_gettime(CLOCK_MONOTONIC, &t1);
147+
if (out_types)
148+
*out_types = reg ? reg->type_count : -1;
149+
cbm_arena_destroy(&arena);
150+
return elapsed_ms(t0, t1);
151+
}
152+
153+
/* The shared build registers defs one at a time into an unfinalized registry;
154+
* a receiver probe that scans every type registered so far makes the build
155+
* O(methods x types). 16x input: linear ~16x time, quadratic well over 100x. */
156+
TEST(pylsp_scale_cross_registry_build_linear) {
157+
int ty_small = 0, ty_large = 0;
158+
/* Best of three for the small size: it is short enough for host noise. */
159+
double t_small = measure_cross_registry(1000, &ty_small);
160+
for (int k = 0; k < 2; k++) {
161+
double t = measure_cross_registry(1000, NULL);
162+
if (t >= 0.0 && t < t_small)
163+
t_small = t;
164+
}
165+
double t_large = measure_cross_registry(16000, &ty_large);
166+
printf(" cross registry: 1000 classes=%.1fms (types=%d) 16000 classes=%.1fms "
167+
"(types=%d)\n",
168+
t_small, ty_small, t_large, ty_large);
169+
ASSERT(t_small >= 0.0 && t_large >= 0.0);
170+
ASSERT_EQ(ty_large - ty_small, 15000);
171+
if (t_small > 0.5) {
172+
double ratio = t_large / t_small;
173+
printf(" cross registry ratio 16000/1000: %.1fx (linear ~16x)\n", ratio);
174+
ASSERT(ratio < 48.0);
175+
}
176+
PASS();
177+
}
178+
114179
SUITE(py_lsp_scale) {
115180
RUN_TEST(pylsp_scale_linear_growth);
181+
RUN_TEST(pylsp_scale_cross_registry_build_linear);
116182
}

0 commit comments

Comments
 (0)