Skip to content

Commit 240b24a

Browse files
committed
fix(python): docstrings and __future__ imports are no namespace barriers
The namespace facts proved only plain one-line strings safe, so every triple-quoted string statement (a module, class or attribute docstring) was an unknown EFFECT, and from __future__ import x was UNSUPPORTED. A barrier withdraws every name bound before it: sqlalchemy's sql/expression.py lost its 149 re-exports to the docstring after nullsfirst = nulls_first, and a docstring made an otherwise plain class unproven. A string literal now evaluates nothing in any quoting or prefix (r, b, u, rb), implicit concatenations included; an f-string with a replacement field stays an effect (__format__/__str__ may run code). from __future__ import x [as y] binds like a from-import of module __future__; a star import from __future__ is a syntax error and stays unsupported. CBM_PY_NAMESPACE_FACTS_VERSION moves to 2. The version travels in the lsp_surface py payload and a payload of another version does not decode, so the first incremental run on an index of an older binary falls back to a full re-index (delta.rehydrate_failed -> CBM_PIPELINE_FORCE_FULL_REINDEX) instead of mixing old and new facts. The codec test now spells the current version and rejects a version-1 payload. Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
1 parent 35598e6 commit 240b24a

3 files changed

Lines changed: 87 additions & 16 deletions

File tree

‎internal/cbm/cbm.h‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -401,8 +401,12 @@ typedef struct {
401401
CBMImportKind kind; // CBM_IMPORT_KIND_DEFAULT unless the syntax names a member kind
402402
} CBMImport;
403403

404-
/* Content-only ordered Python namespace facts; no resolved target identities. */
405-
#define CBM_PY_NAMESPACE_FACTS_VERSION 1u
404+
/* Content-only ordered Python namespace facts; no resolved target identities.
405+
* The version travels in the lsp_surface "py" payload, and a stored payload of
406+
* another version does not decode: an incremental run then re-extracts in full,
407+
* so stored facts of an older extraction never mix with fresh ones.
408+
* 2: string-literal statements are no effect; `from __future__` imports bind. */
409+
#define CBM_PY_NAMESPACE_FACTS_VERSION 2u
406410

407411
typedef enum {
408412
CBM_PY_NS_NOT_CAPTURED = 0,

‎internal/cbm/extract_imports.c‎

Lines changed: 67 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -484,6 +484,37 @@ static bool py_ns_plain_string(PyNamespaceBuilder *b, TSNode node, const char **
484484
return true;
485485
}
486486

487+
/* One string of a literal: any quoting or prefix (r, b, u, rb), unless it is an
488+
* f-string with a replacement field, whose __format__ / __str__ may run code. */
489+
static bool py_ns_string_without_field(TSNode node) {
490+
if (strcmp(ts_node_type(node), "string") != 0)
491+
return false;
492+
uint32_t count = ts_node_named_child_count(node);
493+
for (uint32_t i = 0; i < count; i++) {
494+
if (strcmp(ts_node_type(ts_node_named_child(node, i)), "interpolation") == 0)
495+
return false;
496+
}
497+
return true;
498+
}
499+
500+
/* A string literal, implicit concatenations included, evaluates nothing: a
501+
* docstring (module, class or attribute) is no namespace effect. */
502+
static bool py_ns_literal_string(TSNode node) {
503+
if (strcmp(ts_node_type(node), "concatenated_string") != 0)
504+
return py_ns_string_without_field(node);
505+
uint32_t count = ts_node_named_child_count(node);
506+
bool parts = false;
507+
for (uint32_t i = 0; i < count; i++) {
508+
TSNode part = ts_node_named_child(node, i);
509+
if (strcmp(ts_node_type(part), "comment") == 0)
510+
continue;
511+
if (!py_ns_string_without_field(part))
512+
return false;
513+
parts = true;
514+
}
515+
return parts;
516+
}
517+
487518
/* Bounded syntactic proof only. Unsupported or deep expressions become an
488519
* unknown-effect barrier; no evaluator or source execution is involved. */
489520
static bool py_ns_safe_expr(PyNamespaceBuilder *b, TSNode node, int depth) {
@@ -496,11 +527,8 @@ static bool py_ns_safe_expr(PyNamespaceBuilder *b, TSNode node, int depth) {
496527
strcmp(kind, "float") == 0 || strcmp(kind, "true") == 0 || strcmp(kind, "false") == 0 ||
497528
strcmp(kind, "none") == 0 || strcmp(kind, "comment") == 0)
498529
return true;
499-
if (strcmp(kind, "string") == 0) {
500-
const char *text = NULL;
501-
size_t len = 0;
502-
return py_ns_plain_string(b, node, &text, &len);
503-
}
530+
if (strcmp(kind, "string") == 0 || strcmp(kind, "concatenated_string") == 0)
531+
return py_ns_literal_string(node);
504532
bool container =
505533
strcmp(kind, "tuple") == 0 || strcmp(kind, "list") == 0 ||
506534
strcmp(kind, "parenthesized_expression") == 0 || strcmp(kind, "parameters") == 0 ||
@@ -549,8 +577,7 @@ static bool py_ns_class_safe(PyNamespaceBuilder *b, TSNode node) {
549577
continue;
550578
if (strcmp(kind, "expression_statement") == 0 &&
551579
ts_node_named_child_count(child) == 1) {
552-
TSNode expr = ts_node_named_child(child, 0);
553-
if (strcmp(ts_node_type(expr), "string") == 0 && py_ns_safe_expr(b, expr, 0))
580+
if (py_ns_literal_string(ts_node_named_child(child, 0)))
554581
continue;
555582
/* Even an identifier assignment can install a descriptor whose
556583
* __set_name__ hook executes during class creation. */
@@ -573,6 +600,8 @@ static bool py_ns_import_fact(PyNamespaceBuilder *b, TSNode child, bool from,
573600
.module_name = module_name,
574601
.relative_level = level};
575602
if (from && strcmp(kind, "wildcard_import") == 0) {
603+
if (level == 0 && module_name && strcmp(module_name, "__future__") == 0)
604+
return false; /* a syntax error in Python: stays unsupported */
576605
f.kind = CBM_PY_NS_IMPORT_STAR;
577606
*out = f;
578607
return true;
@@ -840,6 +869,35 @@ static bool py_ns_type_checking_block(PyNamespaceBuilder *b, TSNode node) {
840869
return !b->failure;
841870
}
842871

872+
/* `from __future__ import x [as y]` is its own node type, without a module
873+
* field: a compiler directive that also binds x (or y) to the feature object,
874+
* like a from-import of module `__future__`. */
875+
static bool py_ns_future_import(PyNamespaceBuilder *b, TSNode node) {
876+
const char *module_name = py_ns_copy(b, "__future__", strlen("__future__"));
877+
if (!module_name)
878+
return false;
879+
TSTreeCursor cursor = ts_tree_cursor_new(node);
880+
bool ok = true;
881+
int emitted = 0;
882+
if (ts_tree_cursor_goto_first_child(&cursor)) {
883+
do {
884+
const char *field = ts_tree_cursor_current_field_name(&cursor);
885+
if (!field || strcmp(field, "name") != 0)
886+
continue;
887+
CBMPyNamespaceFact f;
888+
if (!py_ns_import_fact(b, ts_tree_cursor_current_node(&cursor), true, module_name, 0,
889+
&f) ||
890+
!py_ns_push(b, f)) {
891+
ok = false;
892+
break;
893+
}
894+
emitted++;
895+
} while (ts_tree_cursor_goto_next_sibling(&cursor));
896+
}
897+
ts_tree_cursor_delete(&cursor);
898+
return ok && emitted > 0;
899+
}
900+
843901
static bool py_ns_statement(PyNamespaceBuilder *b, TSNode node) {
844902
const char *kind = ts_node_type(node);
845903
if (strcmp(kind, "comment") == 0 || strcmp(kind, "pass_statement") == 0)
@@ -848,6 +906,8 @@ static bool py_ns_statement(PyNamespaceBuilder *b, TSNode node) {
848906
return py_ns_import(b, node, false);
849907
if (strcmp(kind, "import_from_statement") == 0)
850908
return py_ns_import(b, node, true);
909+
if (strcmp(kind, "future_import_statement") == 0)
910+
return py_ns_future_import(b, node);
851911
bool decorated = strcmp(kind, "decorated_definition") == 0;
852912
TSNode def = decorated ? ts_node_child_by_field_name(node, TS_FIELD("definition")) : node;
853913
if (!ts_node_is_null(def)) {

‎tests/test_pipeline.c‎

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -22988,26 +22988,32 @@ TEST(pipeline_python_namespace_codec_owns_ordered_facts) {
2298822988
}
2298922989

2299022990
TEST(pipeline_python_namespace_codec_rejects_non_authoritative_rows) {
22991-
/* Numeric values below are the frozen version-1 namespace wire values.
22992-
* Even an empty LSP array must validate its namespace envelope. */
22991+
/* Numeric values below are the frozen namespace wire values (version 2
22992+
* changed which statements extraction proves harmless, not the wire
22993+
* shape). Even an empty LSP array must validate its namespace envelope,
22994+
* and a payload of an older facts version is no authority: its facts
22995+
* were extracted under other rules, so the incremental run that reads
22996+
* it re-extracts in full. */
2299322997
const char *bad[] = {
2299422998
"{\"v\":1,\"lsp\":[]}",
2299522999
"{\"v\":2,\"lang\":1,\"lsp\":[]}",
2299623000
"{\"v\":2,\"lang\":1,\"lsp\":[],\"py\":null}",
2299723001
"{\"v\":2,\"lang\":1.0,\"lsp\":[],\"py\":null}",
2299823002
"{\"v\":2,\"v\":2,\"lang\":0,\"lsp\":[],\"py\":null}",
22999-
"{\"v\":2,\"lang\":1,\"lsp\":[],\"py\":{\"v\":1,\"lang\":1,"
23003+
"{\"v\":2,\"lang\":1,\"lsp\":[],\"py\":{\"v\":2,\"lang\":1,"
2300023004
"\"module\":\"p.m\",\"path\":\"m.py\",\"status\":3,\"failure\":2,\"events\":[]}}",
23005+
"{\"v\":2,\"lang\":1,\"lsp\":[],\"py\":{\"v\":1,\"lang\":1,"
23006+
"\"module\":\"p.m\",\"path\":\"m.py\",\"status\":2,\"failure\":0,\"events\":[]}}",
2300123007
"{\"v\":2,\"lang\":1,\"lsp\":[],\"py\":{\"v\":99,\"lang\":1,"
2300223008
"\"module\":\"p.m\",\"path\":\"m.py\",\"status\":2,\"failure\":0,\"events\":[]}}",
23003-
"{\"v\":2,\"lang\":1,\"lsp\":[],\"py\":{\"v\":1,\"lang\":1,"
23009+
"{\"v\":2,\"lang\":1,\"lsp\":[],\"py\":{\"v\":2,\"lang\":1,"
2300423010
"\"module\":\"p.m\",\"path\":\"m.py\\u0000suffix\",\"status\":2,"
2300523011
"\"failure\":0,\"events\":[]}}",
23006-
"{\"v\":2,\"lang\":1,\"lsp\":[],\"py\":{\"v\":1,\"lang\":1,"
23012+
"{\"v\":2,\"lang\":1,\"lsp\":[],\"py\":{\"v\":2,\"lang\":1,"
2300723013
"\"module\":\"p.m\",\"path\":\"m.py\",\"status\":2,\"failure\":0,\"events\":["
2300823014
"{\"k\":999,\"d\":0,\"s\":0,\"r\":0,\"f\":0,\"l\":0,"
2300923015
"\"n\":null,\"m\":null,\"i\":null,\"a\":[]}]}}",
23010-
"{\"v\":2,\"lang\":1,\"lsp\":[],\"py\":{\"v\":1,\"lang\":1,"
23016+
"{\"v\":2,\"lang\":1,\"lsp\":[],\"py\":{\"v\":2,\"lang\":1,"
2301123017
"\"module\":\"p.m\",\"path\":\"m.py\",\"status\":2,\"failure\":0,\"events\":["
2301223018
"{\"k\":8,\"d\":0,\"s\":1,\"r\":0,\"f\":0,\"l\":4294967296,"
2301323019
"\"n\":null,\"m\":null,\"i\":null,\"a\":[]}]}}",
@@ -23027,7 +23033,7 @@ TEST(pipeline_python_namespace_codec_rejects_non_authoritative_rows) {
2302723033
cbm_arena_destroy(&arena);
2302823034
}
2302923035
const char *unknown =
23030-
"{\"v\":2,\"lang\":1,\"lsp\":[],\"reg\":[],\"py\":{\"v\":1,\"lang\":1,"
23036+
"{\"v\":2,\"lang\":1,\"lsp\":[],\"reg\":[],\"py\":{\"v\":2,\"lang\":1,"
2303123037
"\"module\":\"p.m\",\"path\":\"m.py\",\"status\":2,\"failure\":0,\"events\":["
2303223038
"{\"k\":7,\"d\":0,\"s\":0,\"r\":7,\"f\":0,\"l\":0,"
2303323039
"\"n\":null,\"m\":null,\"i\":null,\"a\":[]}]}}";
@@ -23040,6 +23046,7 @@ TEST(pipeline_python_namespace_codec_rejects_non_authoritative_rows) {
2304023046
facts.facts.items[0].kind == CBM_PY_NS_UNKNOWN &&
2304123047
facts.facts.items[0].reason == CBM_PY_NS_REASON_PARSE;
2304223048
cbm_arena_destroy(&arena);
23049+
ASSERT_EQ(CBM_PY_NAMESPACE_FACTS_VERSION, 2u); /* the rows above spell it */
2304323050
ASSERT_TRUE(rejected);
2304423051
ASSERT_EQ(unknown_rc, 1);
2304523052
ASSERT_EQ(equal_rc, 0);

0 commit comments

Comments
 (0)