Skip to content

Commit 3a46d69

Browse files
committed
fix(python): resolve annotations through TYPE_CHECKING imports (DeusData#1277)
The import an annotated field's type comes from usually sits under a module-level `if TYPE_CHECKING:` (to avoid an import cycle). parse_python_imports walked only the module's direct statements, so that import was never recorded and the reporter's chain trainer.strategies.process_batch() lost its type exactly where real code writes it. - Extraction records the imports of a module-level `if TYPE_CHECKING:` / `if <module>.TYPE_CHECKING:` consequence block, flagged CBMImport.type_checking_only. elif/else branches stay unrecorded, as every other compound statement's imports do. - One rule for every CALLS import-map builder: cbm_import_binds_runtime_name() (false for a flagged import). The sequential build_import_map applies it in its fast and slow paths; the shared cbm_pxc_build_import_map (= _ex(..., false)) applies it in both loops, so no CALLS import map ever binds an annotation-only name. - The annotation consumers take cbm_pxc_build_import_map_ex(..., true): the cross-file base/field-type resolution and the Python LSP dispatch. The sequential dispatch always takes it; the parallel one builds that map only for a file that records a flagged import and otherwise passes its CALLS map, which is identical there by construction. - The Python LSP binds those names into an annotation-only frame consulted by annotation resolution after the ordinary scope chain: a runtime call through such a name never resolves. Tests: python_type_checking_imports_flagged_issue1277 (extraction), python_type_checking_typed_field_issue1277(_parallel) (pipeline, both venues; controls: no import_map and no LSP CALLS edge through the name). Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
1 parent 257aea1 commit 3a46d69

9 files changed

Lines changed: 213 additions & 18 deletions

File tree

‎internal/cbm/cbm.c‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -402,6 +402,24 @@ void cbm_imports_push(CBMImportArray *arr, CBMArena *a, CBMImport imp) {
402402
arr->items[arr->count++] = imp;
403403
}
404404

405+
bool cbm_imports_local_binds_runtime_name(const CBMImportArray *imports, const char *local_name) {
406+
if (!imports || !local_name) {
407+
return true;
408+
}
409+
bool seen = false;
410+
for (int i = 0; i < imports->count; i++) {
411+
const CBMImport *imp = &imports->items[i];
412+
if (!imp->local_name || strcmp(imp->local_name, local_name) != 0) {
413+
continue;
414+
}
415+
if (cbm_import_binds_runtime_name(imp)) {
416+
return true;
417+
}
418+
seen = true;
419+
}
420+
return !seen;
421+
}
422+
405423
void cbm_usages_push(CBMUsageArray *arr, CBMArena *a, CBMUsage usage) {
406424
GROW_ARRAY(arr, a);
407425
arr->items[arr->count++] = usage;

‎internal/cbm/cbm.h‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -403,6 +403,15 @@ typedef struct {
403403
CBMImportKind kind; // CBM_IMPORT_KIND_DEFAULT unless the syntax names a member kind
404404
} CBMImport;
405405

406+
/* True when an import binds its local name at runtime. A Python import under
407+
* `if TYPE_CHECKING:` exists for annotations only: a call through that name
408+
* cannot happen, so no CALLS import map may bind it (an import_map edge
409+
* through it would be fabricated by construction). THE one rule every CALLS
410+
* import-map builder applies. */
411+
static inline bool cbm_import_binds_runtime_name(const CBMImport *imp) {
412+
return imp && !imp->type_checking_only;
413+
}
414+
406415
/* Content-only ordered Python namespace facts; no resolved target identities. */
407416
#define CBM_PY_NAMESPACE_FACTS_VERSION 1u
408417

@@ -1308,6 +1317,10 @@ void cbm_py_namespace_test_copy_fail_after(int successful_allocations);
13081317
void cbm_defs_push(CBMDefArray *arr, CBMArena *a, CBMDefinition def);
13091318
void cbm_calls_push(CBMCallArray *arr, CBMArena *a, CBMCall call);
13101319
void cbm_imports_push(CBMImportArray *arr, CBMArena *a, CBMImport imp);
1320+
/* True unless every import of `imports` that binds `local_name` is
1321+
* annotation-only (cbm_import_binds_runtime_name), so also true when no import
1322+
* names it: the rule for the builders that key IMPORTS edges by local name. */
1323+
bool cbm_imports_local_binds_runtime_name(const CBMImportArray *imports, const char *local_name);
13111324
void cbm_usages_push(CBMUsageArray *arr, CBMArena *a, CBMUsage usage);
13121325
void cbm_throws_push(CBMThrowArray *arr, CBMArena *a, CBMThrow thr);
13131326
void cbm_rw_push(CBMRWArray *arr, CBMArena *a, CBMReadWrite rw);

‎internal/cbm/extract_imports.c‎

Lines changed: 61 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -321,6 +321,63 @@ static void process_py_import_from(CBMExtractCtx *ctx, TSNode node) {
321321
}
322322
}
323323

324+
/* `TYPE_CHECKING` or `<module>.TYPE_CHECKING` (typing.TYPE_CHECKING, t.TYPE_CHECKING). */
325+
static bool py_is_type_checking_condition(CBMExtractCtx *ctx, TSNode cond) {
326+
if (ts_node_is_null(cond)) {
327+
return false;
328+
}
329+
TSNode name = cond;
330+
if (strcmp(ts_node_type(cond), "attribute") == 0) {
331+
name = ts_node_child_by_field_name(cond, TS_FIELD("attribute"));
332+
}
333+
if (ts_node_is_null(name) || strcmp(ts_node_type(name), "identifier") != 0) {
334+
return false;
335+
}
336+
const char *text = cbm_node_text(ctx->arena, name, ctx->source);
337+
return text && strcmp(text, "TYPE_CHECKING") == 0;
338+
}
339+
340+
/* One import statement; `type_only` flags every import it records. */
341+
static void process_py_import_node(CBMExtractCtx *ctx, TSNode node, bool type_only) {
342+
const char *kind = ts_node_type(node);
343+
int before = ctx->result->imports.count;
344+
if (strcmp(kind, "import_statement") == 0) {
345+
process_py_import_stmt(ctx, node);
346+
} else if (strcmp(kind, "import_from_statement") == 0 ||
347+
strcmp(kind, "future_import_statement") == 0) {
348+
// `from __future__ import annotations` is a distinct node type in
349+
// tree-sitter-python but has the same shape (module + name list).
350+
process_py_import_from(ctx, node);
351+
}
352+
for (int i = before; type_only && i < ctx->result->imports.count; i++) {
353+
ctx->result->imports.items[i].type_checking_only = true;
354+
}
355+
}
356+
357+
/* A module-level `if TYPE_CHECKING:` holds imports a type checker sees and
358+
* the interpreter never runs (#1277: an annotated field's type is usually
359+
* imported exactly there, to avoid an import cycle). Record the imports of
360+
* its consequence block, flagged type_checking_only; its elif/else branches
361+
* are runtime alternatives and stay unrecorded, as every other compound
362+
* statement's imports do. */
363+
static void process_py_type_checking_block(CBMExtractCtx *ctx, TSNode if_node) {
364+
if (!py_is_type_checking_condition(
365+
ctx, ts_node_child_by_field_name(if_node, TS_FIELD("condition")))) {
366+
return;
367+
}
368+
TSNode body = ts_node_child_by_field_name(if_node, TS_FIELD("consequence"));
369+
if (ts_node_is_null(body)) {
370+
return;
371+
}
372+
TSTreeCursor cursor = ts_tree_cursor_new(body);
373+
if (ts_tree_cursor_goto_first_child(&cursor)) {
374+
do {
375+
process_py_import_node(ctx, ts_tree_cursor_current_node(&cursor), true);
376+
} while (ts_tree_cursor_goto_next_sibling(&cursor));
377+
}
378+
ts_tree_cursor_delete(&cursor);
379+
}
380+
324381
static void parse_python_imports(CBMExtractCtx *ctx) {
325382
TSTreeCursor cursor = ts_tree_cursor_new(ctx->root);
326383
if (!ts_tree_cursor_goto_first_child(&cursor)) {
@@ -329,15 +386,10 @@ static void parse_python_imports(CBMExtractCtx *ctx) {
329386
}
330387
do {
331388
TSNode node = ts_tree_cursor_current_node(&cursor);
332-
const char *kind = ts_node_type(node);
333-
334-
if (strcmp(kind, "import_statement") == 0) {
335-
process_py_import_stmt(ctx, node);
336-
} else if (strcmp(kind, "import_from_statement") == 0 ||
337-
strcmp(kind, "future_import_statement") == 0) {
338-
// `from __future__ import annotations` is a distinct node type in
339-
// tree-sitter-python but has the same shape (module + name list).
340-
process_py_import_from(ctx, node);
389+
if (strcmp(ts_node_type(node), "if_statement") == 0) {
390+
process_py_type_checking_block(ctx, node);
391+
} else {
392+
process_py_import_node(ctx, node, false);
341393
}
342394
} while (ts_tree_cursor_goto_next_sibling(&cursor));
343395
ts_tree_cursor_delete(&cursor);

‎internal/cbm/lsp/py_lsp.c‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4278,6 +4278,11 @@ static const CBMType *py_resolve_annotation(PyLSPContext *ctx, const char *ann)
42784278
const CBMType *t = cbm_scope_lookup(ctx->current_scope, ann);
42794279
if (!cbm_type_is_unknown(t))
42804280
return t;
4281+
if (ctx->annotation_scope) {
4282+
t = cbm_scope_lookup(ctx->annotation_scope, ann);
4283+
if (!cbm_type_is_unknown(t))
4284+
return t;
4285+
}
42814286
if (ctx->module_qn) {
42824287
const char *qn = cbm_arena_sprintf(ctx->arena, "%s.%s", ctx->module_qn, ann);
42834288
const CBMRegisteredType *rt = cbm_registry_lookup_type(ctx->registry, qn);
@@ -5360,6 +5365,27 @@ static void py_replay_import_statement(PyLSPContext *ctx, TSNode stmt, unsigned
53605365
}
53615366
}
53625367

5368+
/* The consequence block of a module-level `if TYPE_CHECKING:` /
5369+
* `if <module>.TYPE_CHECKING:` statement, else a null node. Mirrors the
5370+
* extraction's py_is_type_checking_condition (extract_imports.c). */
5371+
static TSNode py_type_checking_body(PyLSPContext *ctx, TSNode stmt) {
5372+
TSNode none = {0};
5373+
if (strcmp(ts_node_type(stmt), "if_statement") != 0)
5374+
return none;
5375+
TSNode cond = ts_node_child_by_field_name(stmt, "condition", 9);
5376+
if (ts_node_is_null(cond))
5377+
return none;
5378+
TSNode name = cond;
5379+
if (strcmp(ts_node_type(cond), "attribute") == 0)
5380+
name = ts_node_child_by_field_name(cond, "attribute", 9);
5381+
if (ts_node_is_null(name) || strcmp(ts_node_type(name), "identifier") != 0)
5382+
return none;
5383+
char *text = py_node_text(ctx, name);
5384+
if (!text || strcmp(text, "TYPE_CHECKING") != 0)
5385+
return none;
5386+
return ts_node_child_by_field_name(stmt, "consequence", 11);
5387+
}
5388+
53635389
static bool py_expression_is_annotation_only_assignment(TSNode statement) {
53645390
if (ts_node_named_child_count(statement) != 1)
53655391
return false;
@@ -5444,6 +5470,29 @@ void py_lsp_process_file(PyLSPContext *ctx, TSNode root) {
54445470
* Invalidate only syntactic binding targets at the control-flow
54455471
* join; a later unconditional statement may restore exact proof. */
54465472
py_invalidate_possible_bindings(ctx, c, 0);
5473+
/* `if TYPE_CHECKING:` holds the imports annotations are written
5474+
* against (#1277). Bind them into the annotation-only frame: an
5475+
* annotation resolves through them, a runtime call never does (the
5476+
* interpreter never runs the block). Extraction recorded them
5477+
* flagged type_checking_only, so they reach this walk through the
5478+
* LSP import map only, never a CALLS import map. */
5479+
TSNode tc_body = py_type_checking_body(ctx, c);
5480+
if (!ts_node_is_null(tc_body)) {
5481+
if (!ctx->annotation_scope)
5482+
ctx->annotation_scope = cbm_scope_push(ctx->arena, NULL);
5483+
CBMScope *runtime_scope = ctx->current_scope;
5484+
PyKids tk = py_kids(ctx->arena, tc_body);
5485+
for (uint32_t t = 0; ctx->annotation_scope && t < tk.n && !ctx->eval_failure; t++) {
5486+
TSNode stmt = py_kid(&tk, t);
5487+
const char *sk = ts_node_type(stmt);
5488+
if (strcmp(sk, "import_statement") == 0 ||
5489+
strcmp(sk, "import_from_statement") == 0) {
5490+
ctx->current_scope = ctx->annotation_scope;
5491+
py_replay_import_statement(ctx, stmt, consumed_imports);
5492+
ctx->current_scope = runtime_scope;
5493+
}
5494+
}
5495+
}
54475496
}
54485497
}
54495498
// Pass 2: top-level calls (rare) and nested definitions.

‎internal/cbm/lsp/py_lsp.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,10 @@ typedef struct {
4949
* cbm_registry_*_for_update, never into anything behind ->fallback. */
5050
CBMTypeRegistry *registry_head;
5151
CBMScope *current_scope;
52+
// Names a module-level `if TYPE_CHECKING:` imports (#1277): consulted by
53+
// annotation resolution only, after the ordinary scope chain misses, so a
54+
// runtime call through such a name never resolves. NULL when none.
55+
CBMScope *annotation_scope;
5256

5357
// Import map: local_name -> module_qn (arena-allocated, NULL-terminated).
5458
// Built from CBMFileResult.imports.

‎src/pipeline/pass_calls.c‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -141,7 +141,8 @@ static int build_import_map(cbm_pipeline_ctx_t *ctx, const char *rel_path,
141141

142142
for (int i = 0; i < result->imports.count; i++) {
143143
const CBMImport *imp = &result->imports.items[i];
144-
if (!imp->local_name || !imp->local_name[0] || !imp->module_path) {
144+
if (!imp->local_name || !imp->local_name[0] || !imp->module_path ||
145+
!cbm_import_binds_runtime_name(imp)) {
145146
continue;
146147
}
147148
const cbm_gbuf_node_t *target =
@@ -193,6 +194,10 @@ static int build_import_map(cbm_pipeline_ctx_t *ctx, const char *rel_path,
193194
continue;
194195
}
195196
char *key = extract_local_name_from_json(e->properties_json);
197+
if (key && !cbm_imports_local_binds_runtime_name(result ? &result->imports : NULL, key)) {
198+
free(key); /* annotation-only import (#1277): no CALLS binding */
199+
key = NULL;
200+
}
196201
if (key) {
197202
keys[count] = key;
198203
vals[count] = target->qualified_name;

‎src/pipeline/pass_lsp_cross.c‎

Lines changed: 32 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -876,9 +876,11 @@ CBMLSPDef *cbm_pxc_collect_all_defs(const cbm_pipeline_ctx_t *ctx, CBMArena *are
876876
if (ctx && ctx->registry && pxc_lang_resolves_base_qns(files[fi].language)) {
877877
bs.reg = ctx->registry;
878878
bs.gbuf = ctx->gbuf;
879-
cbm_pxc_build_import_map(ctx->gbuf, project_name, files[fi].rel_path,
880-
files[fi].language, fr, &bs.imp_keys, &bs.imp_vals,
881-
&bs.imp_count);
879+
/* Bases and field annotations resolve through annotation-only
880+
* imports too: `if TYPE_CHECKING:` exists for them (#1277). */
881+
cbm_pxc_build_import_map_ex(ctx->gbuf, project_name, files[fi].rel_path,
882+
files[fi].language, fr, true, &bs.imp_keys, &bs.imp_vals,
883+
&bs.imp_count);
882884
}
883885
for (int di = 0; di < fr->defs.count; di++) {
884886
if (pxc_build_lsp_def(arena, &fr->defs.items[di], def_modules[fi], namespace_name,
@@ -1068,10 +1070,26 @@ static char *pxc_python_import_from_metadata(const cbm_gbuf_t *gbuf, const char
10681070

10691071
/* Build per-file import map (local_name -> semantic import QN) from gbuf
10701072
* IMPORTS edges. Both pipeline drivers call this implementation. Returns 0
1071-
* with *out_count = 0 when the file has no IMPORTS edges. */
1073+
* with *out_count = 0 when the file has no IMPORTS edges. The CALLS map. */
10721074
int cbm_pxc_build_import_map(const cbm_gbuf_t *gbuf, const char *project_name, const char *rel_path,
10731075
CBMLanguage lang, const CBMFileResult *result, const char ***out_keys,
10741076
const char ***out_vals, int *out_count) {
1077+
return cbm_pxc_build_import_map_ex(gbuf, project_name, rel_path, lang, result, false, out_keys,
1078+
out_vals, out_count);
1079+
}
1080+
1081+
bool cbm_pxc_result_has_type_only_import(const CBMFileResult *result) {
1082+
for (int i = 0; result && i < result->imports.count; i++) {
1083+
if (!cbm_import_binds_runtime_name(&result->imports.items[i]))
1084+
return true;
1085+
}
1086+
return false;
1087+
}
1088+
1089+
int cbm_pxc_build_import_map_ex(const cbm_gbuf_t *gbuf, const char *project_name,
1090+
const char *rel_path, CBMLanguage lang, const CBMFileResult *result,
1091+
bool include_type_only, const char ***out_keys,
1092+
const char ***out_vals, int *out_count) {
10751093
*out_keys = NULL;
10761094
*out_vals = NULL;
10771095
*out_count = 0;
@@ -1129,6 +1147,11 @@ int cbm_pxc_build_import_map(const cbm_gbuf_t *gbuf, const char *project_name, c
11291147
continue;
11301148
memcpy(local, start, n);
11311149
local[n] = '\0';
1150+
if (!include_type_only &&
1151+
!cbm_imports_local_binds_runtime_name(result ? &result->imports : NULL, local)) {
1152+
free(local); /* annotation-only import (#1277): no CALLS binding */
1153+
continue;
1154+
}
11321155
char *value = pxc_import_value_qn(lang, result, local, target);
11331156
if (!value) {
11341157
free(local);
@@ -1146,6 +1169,7 @@ int cbm_pxc_build_import_map(const cbm_gbuf_t *gbuf, const char *project_name, c
11461169
for (int i = 0; i < metadata_count; i++) {
11471170
const CBMImport *imp = &result->imports.items[i];
11481171
if (!imp->local_name || !imp->local_name[0] || !imp->module_path ||
1172+
(!include_type_only && !cbm_import_binds_runtime_name(imp)) ||
11491173
pxc_import_map_has_local(keys, count, imp->local_name)) {
11501174
continue;
11511175
}
@@ -2212,8 +2236,10 @@ int cbm_pipeline_pass_lsp_cross(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *
22122236
const char **imp_keys = NULL;
22132237
const char **imp_vals = NULL;
22142238
int imp_count = 0;
2215-
cbm_pxc_build_import_map(ctx->gbuf, ctx->project_name, files[i].rel_path, lang, cache[i],
2216-
&imp_keys, &imp_vals, &imp_count);
2239+
/* The LSP map: annotation-only imports bind for annotations (#1277).
2240+
* MUST match the LSP map of pass_parallel.c. */
2241+
cbm_pxc_build_import_map_ex(ctx->gbuf, ctx->project_name, files[i].rel_path, lang, cache[i],
2242+
true, &imp_keys, &imp_vals, &imp_count);
22172243

22182244
/* Journal around the resolve: a hang here must be attributed to THIS
22192245
* file, not to a stale extraction marker (the innocent-quarantine

‎src/pipeline/pass_lsp_cross.h‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,17 @@ void cbm_pxc_ts_modes(CBMLanguage lang, const char *rel_path, bool *out_js, bool
8383
int cbm_pxc_build_import_map(const cbm_gbuf_t *gbuf, const char *project_name, const char *rel_path,
8484
CBMLanguage lang, const CBMFileResult *result, const char ***out_keys,
8585
const char ***out_vals, int *out_count);
86+
/* As cbm_pxc_build_import_map (which is include_type_only = false, the CALLS
87+
* map), but with include_type_only also binding annotation-only imports
88+
* (Python `if TYPE_CHECKING:`), for the consumers that resolve annotations:
89+
* the Python LSP and the cross-file base/field-type resolution (#1277). */
90+
int cbm_pxc_build_import_map_ex(const cbm_gbuf_t *gbuf, const char *project_name,
91+
const char *rel_path, CBMLanguage lang, const CBMFileResult *result,
92+
bool include_type_only, const char ***out_keys,
93+
const char ***out_vals, int *out_count);
94+
/* True when the file records an annotation-only import (the LSP map then
95+
* differs from the CALLS map and must be built separately). */
96+
bool cbm_pxc_result_has_type_only_import(const CBMFileResult *result);
8697

8798
void cbm_pxc_free_import_map(const char **keys, const char **vals, int count);
8899

‎src/pipeline/pass_parallel.c‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4323,10 +4323,27 @@ static void resolve_worker(int worker_id, void *ctx_ptr) {
43234323
* this file, not to a stale extraction marker. */
43244324
cbm_index_mark_start(rel);
43254325
bool had_error = result->has_error;
4326+
/* The LSP map also binds annotation-only imports (#1277);
4327+
* only a file that records one needs its own map, every
4328+
* other file passes the CALLS map unchanged. MUST match the
4329+
* sequential dispatch in pass_lsp_cross.c. */
4330+
const char **lsp_keys = NULL;
4331+
const char **lsp_vals = NULL;
4332+
int lsp_count = 0;
4333+
bool own_lsp_map = cbm_pxc_result_has_type_only_import(result);
4334+
if (own_lsp_map) {
4335+
cbm_pxc_build_import_map_ex(rc->main_gbuf, rc->project_name, rel, lang, result,
4336+
true, &lsp_keys, &lsp_vals, &lsp_count);
4337+
}
43264338
cbm_pxc_dispatch_file_with_php_bindings(
43274339
lang, result, lsp_source, lsp_source_len, rel, def_module, rc->cross_registries,
4328-
rc->module_def_index, rc->all_defs, rc->def_count, imp_keys, imp_vals,
4329-
imp_count, pp_rust_shared_registry_get, rc, &php_vendor);
4340+
rc->module_def_index, rc->all_defs, rc->def_count,
4341+
own_lsp_map ? lsp_keys : imp_keys, own_lsp_map ? lsp_vals : imp_vals,
4342+
own_lsp_map ? lsp_count : imp_count, pp_rust_shared_registry_get, rc,
4343+
&php_vendor);
4344+
if (own_lsp_map) {
4345+
cbm_pxc_free_import_map(lsp_keys, lsp_vals, lsp_count);
4346+
}
43304347
cbm_index_mark_done(rel);
43314348
if (!had_error && result->has_error && result->error_msg &&
43324349
(strcmp(result->error_msg, CBM_C_LSP_MEMO_ERROR) == 0 ||

0 commit comments

Comments
 (0)