Break-glass merge PR #2532

Author: DeusData

internal/cbm/lsp/py_lsp.c

@@ -40,6 +40,11 @@
 #define PY_LSP_MAX_EVAL_DEPTH 256
 #include "lsp_work.h"
 
+#ifdef CBM_ENABLE_TEST_SEAMS
+_Atomic uint64_t cbm_py_submodule_probes = 0;
+_Atomic uint64_t cbm_py_submodule_probe_visits = 0;
+#endif
+
 #ifdef CBM_ENABLE_TEST_SEAMS
 static _Thread_local int py_test_memo_allocations_left = -1;
 static _Thread_local bool py_test_depth_failure;
@@ -167,51 +172,32 @@ static bool py_func_is_exact_callable_value(const CBMRegisteredFunc *func) {
            (!func->decorator_qns || !func->decorator_qns[0]);
 }
 
-static int py_func_qn_pointer_cmp(const void *left, const void *right) {
-    const CBMRegisteredFunc *a = *(CBMRegisteredFunc *const *)left;
-    const CBMRegisteredFunc *b = *(CBMRegisteredFunc *const *)right;
-    const char *aqn = a ? a->qualified_name : NULL;
-    const char *bqn = b ? b->qualified_name : NULL;
-    if (!aqn || !bqn)
-        return aqn ? 1 : bqn ? -1 : 0;
-    return strcmp(aqn, bqn);
-}
-
-/* The graph identifies functions by QN.  When Python source redefines the
- * same QN, the runtime's last binding is knowable but the graph no longer has
- * one occurrence-exact definition target.  Mark the whole duplicate group so
- * callable values fail closed without changing ordinary call overload/rebind
- * behavior. */
-static void py_mark_ambiguous_callable_bindings(CBMTypeRegistry *registry) {
-    if (!registry || registry->func_count <= 1)
+/* Reuse the QN snapshot for duplicate marks and later submodule probes.
+ * The graph cannot identify one exact callable value for a duplicate QN;
+ * ordinary exact lookups retain their original registration-order policy. */
+static void py_mark_ambiguous_callable_bindings(CBMTypeRegistry *registry, CBMArena *idx_arena) {
+    if (!registry)
+        return;
+    cbm_registry_build_func_qn_order(registry, idx_arena);
+    if (registry->func_count <= 1)
         return;
-    size_t count = (size_t)registry->func_count;
-    CBMRegisteredFunc **sorted = count <= SIZE_MAX / sizeof(*sorted)
-                                     ? (CBMRegisteredFunc **)malloc(count * sizeof(*sorted))
-                                     : NULL;
-    if (!sorted) {
-        /* Allocation failure must reduce precision, never fabricate it. */
+    const CBMRegistryQNEntry *sorted = registry->func_qn_sorted;
+    if (!sorted || registry->func_qn_sorted_upto != registry->func_count) {
         for (int i = 0; i < registry->func_count; i++)
             registry->funcs[i].flags |= CBM_FUNC_FLAG_AMBIGUOUS_BINDING;
         return;
     }
-    for (int i = 0; i < registry->func_count; i++)
-        sorted[i] = &registry->funcs[i];
-    qsort(sorted, count, sizeof(*sorted), py_func_qn_pointer_cmp);
-    for (size_t first = 0; first < count;) {
-        size_t end = first + 1;
-        const char *qn = sorted[first]->qualified_name;
-        while (qn && end < count && sorted[end]->qualified_name &&
-               strcmp(qn, sorted[end]->qualified_name) == 0) {
+    int count = registry->func_qn_sorted_count;
+    for (int first = 0; first < count;) {
+        int end = first + 1;
+        while (end < count && strcmp(sorted[first].qualified_name, sorted[end].qualified_name) == 0)
             end++;
-        }
-        if (qn && end - first > 1) {
-            for (size_t i = first; i < end; i++)
-                sorted[i]->flags |= CBM_FUNC_FLAG_AMBIGUOUS_BINDING;
+        if (end - first > 1) {
+            for (int i = first; i < end; i++)
+                registry->funcs[sorted[i].func_index].flags |= CBM_FUNC_FLAG_AMBIGUOUS_BINDING;
         }
         first = end;
     }
-    free(sorted);
 }
 
 /* Depth-guarded entry for the AST call-resolution walk. The walk recurses once
@@ -1588,6 +1574,22 @@ static const CBMType *py_iterable_element_type(PyLSPContext *ctx, const CBMType
 /* The real recursive-descent evaluator. Never call directly — go through
  * the memoizing, depth-guarded py_eval_expr_type wrapper below
  * (every recursive call inside this body already does). */
+/* Existing function-only submodule predicate, after exact symbol/type misses. */
+static bool py_names_submodule(PyLSPContext *ctx, const char *qn) {
+    const char *prefix = qn ? cbm_arena_sprintf(ctx->arena, "%s.", qn) : NULL;
+    if (!prefix)
+        return false;
+#ifdef CBM_ENABLE_TEST_SEAMS
+    uint64_t visits = 0;
+    bool found = cbm_registry_has_func_qn_prefix(ctx->registry, prefix, &visits);
+    atomic_fetch_add_explicit(&cbm_py_submodule_probes, 1, memory_order_relaxed);
+    atomic_fetch_add_explicit(&cbm_py_submodule_probe_visits, visits, memory_order_relaxed);
+    return found;
+#else
+    return cbm_registry_has_func_qn_prefix(ctx->registry, prefix, NULL);
+#endif
+}
+
 static const CBMType *py_eval_expr_type_uncached(PyLSPContext *ctx, TSNode node) {
     if (!ctx || ts_node_is_null(node))
         return cbm_type_unknown();
@@ -1708,25 +1710,7 @@ static const CBMType *py_eval_expr_type_uncached(PyLSPContext *ctx, TSNode node)
             const CBMRegisteredType *rt = cbm_registry_lookup_type(ctx->registry, qn);
             if (rt)
                 return cbm_type_named(ctx->arena, qn);
-            // Submodule: if any registered function/type has qn starting
-            // with "<mod>.<attr>." then mod.attr is itself a module.
-            // Linear scan over registry funcs is O(R) per access; we
-            // skip it for the common case where mod.attr is already
-            // matched as a function/type above. With ~900 stdlib funcs
-            // and many module-attr accesses per file, this can dominate
-            // — keeping the loop tight and bailing early on first match.
-            const char *prefix = cbm_arena_sprintf(ctx->arena, "%s.", qn);
-            size_t prefix_len = strlen(prefix);
-            bool is_submodule = false;
-            CBMFreeFuncIter all_funcs;
-            cbm_registry_all_funcs_chain(ctx->registry, &all_funcs);
-            for (int i = -1; !is_submodule && (i = cbm_free_func_iter_next(&all_funcs)) >= 0;) {
-                const char *fqn = all_funcs.reg->funcs[i].qualified_name;
-                if (fqn && strncmp(fqn, prefix, prefix_len) == 0) {
-                    is_submodule = true;
-                }
-            }
-            if (is_submodule)
+            if (py_names_submodule(ctx, qn))
                 return cbm_type_module(ctx->arena, qn);
             return cbm_type_unknown();
         }
@@ -5459,7 +5443,6 @@ void cbm_run_py_lsp(CBMArena *arena, CBMFileResult *result, const char *source,
     for (int i = 0; i < result->defs.count; i++) {
         py_register_def(arena, &reg, &result->defs.items[i], module_qn);
     }
-    py_mark_ambiguous_callable_bindings(&reg);
 
     /* Hash-index the registry before the walk (#1527). Unfinalized, every
      * lookup scanned the stdlib plus all of this file's definitions, which
@@ -5470,6 +5453,7 @@ void cbm_run_py_lsp(CBMArena *arena, CBMFileResult *result, const char *source,
      * scratch arena that dies with this call, not in the result arena. */
     CBMArena idx_arena;
     cbm_arena_init(&idx_arena);
+    py_mark_ambiguous_callable_bindings(&reg, &idx_arena);
     reg.index_first_registered = true;
     cbm_registry_finalize_into(&reg, &idx_arena);
 
@@ -5720,7 +5704,7 @@ CBMLSPStatus cbm_run_py_lsp_cross_status(CBMArena *arena, const char *source, in
         ctx.eval_failure = CBM_LSP_SCOPE_FAILED;
         goto cross_cleanup;
     }
-    py_mark_ambiguous_callable_bindings(&reg);
+    py_mark_ambiguous_callable_bindings(&reg, &idx_arena);
 
     /* Finalize registry — O(1) lookups. See go_lsp.c "3c. Finalize"
      * comment for the rationale. */
@@ -5764,7 +5748,7 @@ CBMTypeRegistry *cbm_py_build_cross_registry(CBMArena *arena, CBMLSPDef *defs, i
             return NULL; /* caller owns the shared arena and other registries */
     }
 
-    py_mark_ambiguous_callable_bindings(reg);
+    py_mark_ambiguous_callable_bindings(reg, arena);
     cbm_registry_finalize(reg);
     reg->read_only = true; /* seal: shared Tier-2 registry is read-only during resolve */
     return reg;

internal/cbm/lsp/py_lsp.h

@@ -244,4 +244,12 @@ typedef struct {
 bool cbm_batch_py_lsp_cross(CBMArena *arena, CBMBatchPyLSPFile *files, int file_count,
                             CBMResolvedCallArray *out);
 
+#ifdef CBM_ENABLE_TEST_SEAMS
+#include <stdatomic.h>
+/* Process-wide comparison counts for deterministic resolver operation tests.
+ * Only missing exact symbol/type attributes reach this submodule predicate. */
+extern _Atomic uint64_t cbm_py_submodule_probes;
+extern _Atomic uint64_t cbm_py_submodule_probe_visits;
+#endif
+
 #endif // CBM_LSP_PY_LSP_H

internal/cbm/lsp/type_registry.c

@@ -909,7 +909,8 @@ static const CBMRegisteredFunc *lookup_func_self(const CBMTypeRegistry *reg,
 
     for (int i = 0; i < reg->func_count; i++) {
         CBM_LSP_WORK(1);
-        if (strcmp(reg->funcs[i].qualified_name, qualified_name) == 0) {
+        if (reg->funcs[i].qualified_name &&
+            strcmp(reg->funcs[i].qualified_name, qualified_name) == 0) {
             return &reg->funcs[i];
         }
     }
@@ -985,6 +986,99 @@ const CBMRegisteredFunc *cbm_registry_lookup_symbol(const CBMTypeRegistry *reg,
     return cbm_registry_lookup_func(reg, buf);
 }
 
+/* A stable total order, independent of qsort's treatment of equal keys. */
+static int registry_qn_entry_cmp(const void *left, const void *right) {
+    const CBMRegistryQNEntry *a = left;
+    const CBMRegistryQNEntry *b = right;
+    int cmp = strcmp(a->qualified_name, b->qualified_name);
+    return cmp ? cmp : (a->func_index > b->func_index) - (a->func_index < b->func_index);
+}
+
+#ifdef CBM_ENABLE_TEST_SEAMS
+static _Thread_local int registry_qn_order_allocations_left = -1;
+void cbm_registry_test_qn_order_fail_after(int successful_allocations) {
+    registry_qn_order_allocations_left = successful_allocations;
+}
+#endif
+
+void cbm_registry_build_func_qn_order(CBMTypeRegistry *reg, CBMArena *arena) {
+    if (!reg || !arena || reg->read_only)
+        return;
+    /* A failed rebuild must not leave an old snapshot authoritative. */
+    reg->func_qn_sorted = NULL;
+    reg->func_qn_sorted_count = 0;
+    reg->func_qn_sorted_upto = 0;
+    if (reg->func_count <= 0 || (size_t)reg->func_count > SIZE_MAX / sizeof(CBMRegistryQNEntry))
+        return;
+#ifdef CBM_ENABLE_TEST_SEAMS
+    if (registry_qn_order_allocations_left == 0)
+        return;
+    if (registry_qn_order_allocations_left > 0)
+        registry_qn_order_allocations_left--;
+#endif
+    CBMRegistryQNEntry *sorted = cbm_arena_alloc(arena, (size_t)reg->func_count * sizeof(*sorted));
+    if (!sorted)
+        return;
+    int count = 0;
+    for (int i = 0; i < reg->func_count; i++) {
+        if (reg->funcs[i].qualified_name) {
+            sorted[count].qualified_name = reg->funcs[i].qualified_name;
+            sorted[count].func_index = i;
+            count++;
+        }
+    }
+    qsort(sorted, (size_t)count, sizeof(*sorted), registry_qn_entry_cmp);
+    reg->func_qn_sorted = sorted;
+    reg->func_qn_sorted_count = count;
+    reg->func_qn_sorted_upto = reg->func_count;
+}
+
+static bool registry_link_has_func_qn_prefix(const CBMTypeRegistry *reg, const char *prefix,
+                                             size_t length, uint64_t *visits) {
+    int scan_from = 0;
+    if (reg->func_qn_sorted && reg->func_qn_sorted_upto <= reg->func_count) {
+        int lo = 0, hi = reg->func_qn_sorted_count;
+        while (lo < hi) {
+            int mid = lo + (hi - lo) / 2;
+            if (visits)
+                (*visits)++;
+            if (strcmp(reg->func_qn_sorted[mid].qualified_name, prefix) < 0)
+                lo = mid + 1;
+            else
+                hi = mid;
+        }
+        if (lo < reg->func_qn_sorted_count) {
+            if (visits)
+                (*visits)++;
+            if (strncmp(reg->func_qn_sorted[lo].qualified_name, prefix, length) == 0)
+                return true;
+        }
+        scan_from = reg->func_qn_sorted_upto;
+    }
+    for (int i = scan_from; i < reg->func_count; i++) {
+        if (visits)
+            (*visits)++;
+        const char *qn = reg->funcs[i].qualified_name;
+        if (qn && strncmp(qn, prefix, length) == 0)
+            return true;
+    }
+    return false;
+}
+
+bool cbm_registry_has_func_qn_prefix(const CBMTypeRegistry *reg, const char *prefix,
+                                     uint64_t *visits) {
+    if (!prefix)
+        return false;
+    size_t length = strlen(prefix);
+    /* The old iterator omits shadowed fallback QNs, whose equal head QN has
+     * the same prefix answer. This boolean union needs no winner selection. */
+    for (const CBMTypeRegistry *link = reg; link; link = link->fallback) {
+        if (registry_link_has_func_qn_prefix(link, prefix, length, visits))
+            return true;
+    }
+    return false;
+}
+
 // Count parameters in a FUNC signature.
 static int count_func_params(const CBMRegisteredFunc *f) {
     if (!f || !f->signature || f->signature->kind != CBM_TYPE_FUNC)

internal/cbm/lsp/type_registry.h

@@ -83,6 +83,12 @@ typedef struct {
     int slot;          // bucket slot this entry sits in (for resize)
 } CBMRegistryHashEntry;
 
+/* A borrowed immutable QN and stable array index, never a movable func pointer. */
+typedef struct {
+    const char *qualified_name;
+    int func_index;
+} CBMRegistryQNEntry;
+
 // Cross-file type/function registry.
 typedef struct CBMTypeRegistry {
     CBMRegisteredFunc *funcs;
@@ -159,6 +165,13 @@ typedef struct CBMTypeRegistry {
     // hashed chains answer with the LAST registration (the Tier-2 behavior,
     // left unchanged).
     bool index_first_registered;
+
+    /* Additional prefix/duplicate index; exact lookup order is unchanged.
+     * Covers non-NULL QNs in funcs[0..func_qn_sorted_upto), ordered by (QN,index).
+     * Appended functions remain visible through a tail scan. */
+    CBMRegistryQNEntry *func_qn_sorted;
+    int func_qn_sorted_count;
+    int func_qn_sorted_upto;
 } CBMTypeRegistry;
 
 // Initialize a registry.
@@ -207,6 +220,25 @@ const CBMRegisteredFunc *cbm_registry_lookup_func(const CBMTypeRegistry *reg,
 const CBMRegisteredFunc *cbm_registry_lookup_symbol(const CBMTypeRegistry *reg,
                                                     const char *package_qn, const char *name);
 
+/* Build in an arena that outlives every use (owner arena for shared registries,
+ * walk scratch for per-file registries). QN strings and registered identities
+ * must remain immutable while a snapshot is used; end its use before changing
+ * identities, then rebuild before querying again. Appends do not invalidate it.
+ * Never frees caller arenas. Sealed registries are unchanged. A failed rebuild
+ * discards the old order and queries scan the complete registry instead. */
+void cbm_registry_build_func_qn_order(CBMTypeRegistry *reg, CBMArena *arena);
+
+/* Boolean starts-with over all function/method QNs in the fallback chain.
+ * Does not allocate or mutate. NULL prefix is false; empty prefix matches any
+ * non-NULL QN. Optional visits accumulates compared entries, including tail
+ * scans; fully indexed links use a lower bound plus at most one candidate. */
+bool cbm_registry_has_func_qn_prefix(const CBMTypeRegistry *reg, const char *prefix,
+                                     uint64_t *visits);
+#ifdef CBM_ENABLE_TEST_SEAMS
+/* -1 disables; zero fails this allocation until reset; positive counts down. */
+void cbm_registry_test_qn_order_fail_after(int successful_allocations);
+#endif
+
 // Resolve type alias chain: follow alias_of until concrete type found (max 16 levels).
 const CBMRegisteredType *cbm_registry_resolve_alias(const CBMTypeRegistry *reg,
                                                     const char *type_qn);

scripts/memory-core-baseline.txt

@@ -10,7 +10,6 @@ internal/cbm/extract_defs.c	4
 internal/cbm/helpers.c	3
 internal/cbm/lsp/c_lsp.c	4
 internal/cbm/lsp/perl_lsp.c	21
-internal/cbm/lsp/py_lsp.c	2
 internal/cbm/macro_table.c	1
 internal/cbm/service_patterns.c	2
 internal/cbm/sqlite_writer.c	74