diff --git a/Makefile.cbm b/Makefile.cbm index de57fe785..4f1ab1371 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -98,9 +98,10 @@ endif KOTLIN_DEDUP_TEST_DEFINE = -DCBM_KOTLIN_DEDUP_TEST_API=1 CALL_REFERENCE_LOOKUP_TEST_DEFINE = -DCBM_CALL_REFERENCE_LOOKUP_TEST_API=1 INCREMENTAL_TEST_DEFINE = -DCBM_INCREMENTAL_TEST_API=1 +COVERAGE_MARKER_TEST_DEFINE = -DCBM_COVERAGE_MARKER_TEST_API=1 CFLAGS_TEST = $(CFLAGS_COMMON) $(EDITOR_TEST_DEFINES) $(SANITIZED_DEFINE) \ $(KOTLIN_DEDUP_TEST_DEFINE) $(CALL_REFERENCE_LOOKUP_TEST_DEFINE) \ - $(INCREMENTAL_TEST_DEFINE) -g -O1 $(SANITIZE) + $(INCREMENTAL_TEST_DEFINE) $(COVERAGE_MARKER_TEST_DEFINE) -g -O1 $(SANITIZE) CXXFLAGS_TEST = $(CXXFLAGS_COMMON) $(SANITIZED_DEFINE) -g -O1 $(SANITIZE) $(CXX_STDLIB_FLAGS) # TSan (can't combine with ASan) @@ -118,6 +119,7 @@ TSAN_SANITIZE = -fsanitize=thread -fno-omit-frame-pointer # macro of ours. CFLAGS_TSAN = $(CFLAGS_COMMON) $(EDITOR_TEST_DEFINES) $(KOTLIN_DEDUP_TEST_DEFINE) \ $(CALL_REFERENCE_LOOKUP_TEST_DEFINE) $(INCREMENTAL_TEST_DEFINE) \ + $(COVERAGE_MARKER_TEST_DEFINE) \ -DCBM_SANITIZED_BUILD=1 -g -O1 $(TSAN_SANITIZE) CXXFLAGS_TSAN = $(CXXFLAGS_COMMON) -DCBM_SANITIZED_BUILD=1 -g -O1 \ $(TSAN_SANITIZE) diff --git a/internal/cbm/cbm.c b/internal/cbm/cbm.c index 97f00ca2a..124d082f7 100644 --- a/internal/cbm/cbm.c +++ b/internal/cbm/cbm.c @@ -778,16 +778,24 @@ static bool cbm_source_nesting_exceeds(const char *source, int source_len, int c * nodes (does not descend into an error subtree — one range per failed region). * Bounded by CBM_MAX_ERROR_REGIONS so pathological input can't blow up the * output. The ranges mark where constructs were dropped; they are a detection - * aid, never a completeness proof. */ -#define CBM_MAX_ERROR_REGIONS 64 + * aid, never a completeness proof. + * + * `dropped` counts the ranges the cap threw away. It exists so a clipped list + * cannot read as a complete one: cbm_error_ranges_str turns a non-zero count + * into a trailing "+" marker. Phase 2 split one whole-file range into many + * small ones, which pushed real files straight into a cap that used to be + * unreachable, so the clip is live behaviour and not a theoretical limit. */ +#define CBM_MAX_ERROR_REGIONS 256 typedef struct { uint32_t starts[CBM_MAX_ERROR_REGIONS]; uint32_t ends[CBM_MAX_ERROR_REGIONS]; int count; + int dropped; } cbm_error_regions_t; static void cbm_error_regions_push(cbm_error_regions_t *acc, TSNode n) { if (acc->count >= CBM_MAX_ERROR_REGIONS) { + acc->dropped++; return; } acc->starts[acc->count] = ts_node_start_point(n).row + 1; @@ -851,13 +859,14 @@ static bool cbm_is_eof_terminator_miss(TSNode n, const char *source, int source_ return true; } +/* Walks to the end even after the cap is full, so `dropped` is the real number + * of ranges lost rather than a lower bound. This costs little: the walk never + * descends into an ERROR subtree — it records the top-most node and moves on — + * so it only visits the spine of nodes that contain an error, plus one level. */ static void cbm_collect_error_regions(TSNode n, cbm_error_regions_t *acc, const char *source, int source_len) { - if (acc->count >= CBM_MAX_ERROR_REGIONS) { - return; - } uint32_t k = ts_node_child_count(n); - for (uint32_t i = 0; i < k && acc->count < CBM_MAX_ERROR_REGIONS; i++) { + for (uint32_t i = 0; i < k; i++) { TSNode c = ts_node_child(n, i); if (ts_node_is_missing(c) || strcmp(ts_node_type(c), "ERROR") == 0) { if (cbm_is_eof_terminator_miss(c, source, source_len)) { @@ -870,6 +879,130 @@ static void cbm_collect_error_regions(TSNode n, cbm_error_regions_t *acc, const } } +/* ── Phase 2 line map: what the preprocessed parse already explained ─────── + * + * The raw parse is preprocessor-blind. When an #ifdef splits a brace it sees + * both branches at once, the braces do not balance, and the ERROR node + * swallows the whole construct — at file scope it swallows the whole FILE. + * The second parse, on preprocessed source, does not have that problem: the + * preprocessor already picked one branch, so that parse is clean. + * + * So we build one byte per ORIGINAL line and use it to cut the raw ranges + * down to the lines the second parse cannot vouch for. Lines in the branch + * the preprocessor threw away never appear in the second parse at all, so + * they stay flagged — which is right, because they really are missing from + * the graph. + * + * CBM_LINE_PP_PARSED — the preprocessed parse covered this original line and + * found no error on it. Nothing here was dropped. + * CBM_LINE_NO_CODE — the line is empty, is only a comment, or is a + * preprocessor directive. A reported range must never + * begin or end on one. + * + * Directives are in this set because the preprocessor + * CONSUMES them: no directive line ever survives into + * the expanded text, so the second parse can never + * vouch for one, and treating that silence as a miss + * would flag every #include block in the file. The + * known cost is a #define that the raw parse really did + * drop: it no longer shows up on its own. That trade is + * deliberate — it removes far more noise than signal. */ +enum { CBM_LINE_PP_PARSED = 1u, CBM_LINE_NO_CODE = 2u }; + +/* True when the line's first non-blank character starts a preprocessor + * directive. */ +static bool cbm_is_directive_line(const char *line, int len) { + int i = 0; + while (i < len && (line[i] == ' ' || line[i] == '\t')) { + i++; + } + return i < len && line[i] == '#'; +} + +/* True when the line ends with a backslash, so the directive carries on to + * the next line. */ +static bool cbm_line_continues(const char *line, int len) { + int end = len; + while (end > 0 && (line[end - 1] == ' ' || line[end - 1] == '\t' || line[end - 1] == '\r')) { + end--; + } + return end > 0 && line[end - 1] == '\\'; +} + +/* Set CBM_LINE_NO_CODE on every line of `src` that holds no construct. + * One pass over the file. Carries block-comment state across lines so a line + * in the middle of a comment counts as no-code too. */ +static void cbm_mark_no_code_lines(const char *src, int src_len, uint8_t *map, + uint32_t line_count) { + bool in_block = false; + bool in_directive = false; + uint32_t line = 1; + int i = 0; + while (i <= src_len && line <= line_count) { + int end = i; + while (end < src_len && src[end] != '\n') { + end++; + } + bool has_code = false; + bool line_starts_in_block = in_block; + for (int j = i; j < end; j++) { + if (in_block) { + if (src[j] == '*' && j + 1 < end && src[j + 1] == '/') { + in_block = false; + j++; + } + continue; + } + if (src[j] == '/' && j + 1 < end && src[j + 1] == '*') { + in_block = true; + j++; + continue; + } + if (src[j] == '/' && j + 1 < end && src[j + 1] == '/') { + break; /* rest of the line is a comment */ + } + if (src[j] != ' ' && src[j] != '\t' && src[j] != '\r') { + has_code = true; + } + } + bool directive = + !line_starts_in_block && (in_directive || cbm_is_directive_line(src + i, end - i)); + if (!has_code || directive) { + map[line] |= CBM_LINE_NO_CODE; + } + in_directive = directive && cbm_line_continues(src + i, end - i); + line++; + i = end + 1; + } +} + +/* Paint CBM_LINE_PP_PARSED for every original line the preprocessed parse + * covered without an error on it. + * + * Step 1 marks the EXPANDED rows that sit under an ERROR/MISSING node. + * Step 2 walks the expanded lines and, for each one that is unmarked, belongs + * to the file itself (not an included header) and maps back to a real + * original line, records that original line as parsed. */ +static void cbm_mark_pp_error_rows(TSNode n, uint8_t *rows, uint32_t row_count, const char *src, + int src_len) { + uint32_t k = ts_node_child_count(n); + for (uint32_t i = 0; i < k; i++) { + TSNode c = ts_node_child(n, i); + if (ts_node_is_missing(c) || strcmp(ts_node_type(c), "ERROR") == 0) { + if (cbm_is_eof_terminator_miss(c, src, src_len)) { + continue; /* absent final newline only — nothing was dropped */ + } + uint32_t s = ts_node_start_point(c).row + 1; + uint32_t e = ts_node_end_point(c).row + 1; + for (uint32_t r = s; r <= e && r <= row_count; r++) { + rows[r] = 1; + } + } else if (ts_node_has_error(c)) { + cbm_mark_pp_error_rows(c, rows, row_count, src, src_len); + } + } +} + /* Recovery subtraction (#963): tree-sitter error recovery plus the * ERROR-descending def walker often still extract constructs INSIDE a failed * region (verified: a function in an #ifdef-split ERROR region and even a @@ -1075,27 +1208,12 @@ static void cbm_subtract_recovered_regions(cbm_error_regions_t *regs, const CBMD * missing from the graph; it's a benign call the grammar can't parse without the * preprocessor. True if the [start_line, end_line] span contains a call `NAME(` to * a file-defined function-like macro (Macro label + a parameter signature). */ -static bool cbm_span_is_macro_invocation(const char *src, int src_len, uint32_t start_line, - uint32_t end_line, const CBMDefArray *defs) { - if (!src || src_len <= 0 || !defs || start_line == 0 || end_line < start_line) { +static bool cbm_byte_span_is_macro_invocation(const char *src, int src_len, int span_start, + int span_end, const CBMDefArray *defs) { + if (!src || src_len <= 0 || !defs || span_start < 0 || span_end > src_len || + span_start >= span_end) { return false; } - int span_start = 0; - uint32_t line = 1; - while (span_start < src_len && line < start_line) { - if (src[span_start++] == '\n') { - line++; - } - } - if (line != start_line) { - return false; - } - int span_end = span_start; - while (span_end < src_len && line <= end_line) { - if (src[span_end++] == '\n') { - line++; - } - } for (int di = 0; di < defs->count; di++) { const CBMDefinition *d = &defs->items[di]; /* Function-like macros only: an object-like macro (#define PI 3.14) has no @@ -1123,6 +1241,66 @@ static bool cbm_span_is_macro_invocation(const char *src, int src_len, uint32_t return false; } +/* Byte offset where every 1-based line starts, so finding a line's span costs + * one table read instead of a walk from the start of the file. + * + * The table holds line_count + 2 entries. Entry [L] is where line L starts, and + * the last entry is the end of the source, which gives the final line somewhere + * to stop. A line the file never reaches starts at the end of the source, so its + * span is empty and nothing can match inside it — the same answer the walk gives. + * + * Returns NULL when the allocation fails; a caller then falls back to the walk. */ +static int *cbm_build_line_offsets(const char *src, int src_len, uint32_t line_count) { + int *offsets = (int *)malloc(((size_t)line_count + 2) * sizeof(int)); + if (!offsets) { + return NULL; + } + for (uint32_t l = 0; l <= line_count + 1; l++) { + offsets[l] = src_len; + } + offsets[0] = 0; + offsets[1] = 0; + uint32_t line = 1; + for (int i = 0; i < src_len; i++) { + if (src[i] != '\n') { + continue; + } + line++; + if (line > line_count + 1) { + break; + } + offsets[line] = i + 1; + } + return offsets; +} + +/* Same question by line number, for the few callers that ask about one region + * rather than every line of a file. This form walks the source to find the span, + * which is why the per-line caller below uses a table instead. */ +static bool cbm_span_is_macro_invocation(const char *src, int src_len, uint32_t start_line, + uint32_t end_line, const CBMDefArray *defs) { + if (!src || src_len <= 0 || !defs || start_line == 0 || end_line < start_line) { + return false; + } + int span_start = 0; + uint32_t line = 1; + while (span_start < src_len && line < start_line) { + if (src[span_start++] == '\n') { + line++; + } + } + if (line != start_line) { + return false; + } + int span_end = span_start; + while (span_end < src_len && line <= end_line) { + if (src[span_end++] == '\n') { + line++; + } + } + return cbm_byte_span_is_macro_invocation(src, src_len, span_start, span_end, defs); +} + /* True if [rs, re] is fully enclosed by an extracted callable definition (a * Function/Method body). A macro invocation INSIDE a real function body is an * expression-level use where nothing is missing (#1071). A TOP-LEVEL invocation @@ -1163,13 +1341,123 @@ static void cbm_subtract_macro_invocation_regions(cbm_error_regions_t *regs, regs->count = kept; } +/* Push [start, end] after trimming no-code lines off both ends. A run made + * only of directives, comments or blank lines disappears entirely — there was + * never a construct on it to lose. */ +static void cbm_push_trimmed_run(cbm_error_regions_t *out, uint32_t start, uint32_t end, + const uint8_t *map, uint32_t line_count) { + while (start <= end && start <= line_count && (map[start] & CBM_LINE_NO_CODE)) { + start++; + } + while (end >= start && end <= line_count && (map[end] & CBM_LINE_NO_CODE)) { + end--; + } + if (start > end) { + return; /* nothing but blank, comment or directive lines — no construct lost */ + } + if (out->count >= CBM_MAX_ERROR_REGIONS) { + out->dropped++; + return; + } + out->starts[out->count] = start; + out->ends[out->count] = end; + out->count++; +} + +/* #949: a top-level macro invocation is the one place where a clean second + * parse proves nothing. The macro can expand to a whole definition, and the + * recovery walker deliberately refuses to adopt that definition because it is + * absent from the original span. So the expanded line parses fine while the + * construct really is missing from the graph, and the line must stay flagged. + * An invocation INSIDE a function body is the benign #1071 case and is left + * alone here — cbm_subtract_macro_invocation_regions handles it later. */ +static bool cbm_line_is_toplevel_macro_call(const char *src, int src_len, uint32_t line, + const int *line_offsets, const CBMDefArray *defs) { + bool is_call = line_offsets + ? cbm_byte_span_is_macro_invocation(src, src_len, line_offsets[line], + line_offsets[line + 1], defs) + : cbm_span_is_macro_invocation(src, src_len, line, line, defs); + return is_call && !cbm_region_inside_callable(line, line, defs); +} + +/* Cut every raw region down to the lines the preprocessed parse could not + * vouch for. Each region becomes zero or more smaller ranges: one per run of + * consecutive lines that the second parse did not cover cleanly. + * + * This is what collapses a whole-file range on a file whose only real problem + * is an #ifdef splitting a brace. It deliberately does NOT clear the region + * outright — the branch the preprocessor discarded is genuinely absent from + * the graph and must stay flagged. */ +static void cbm_refine_regions_with_pp_lines(cbm_error_regions_t *regs, const uint8_t *map, + uint32_t line_count, const char *src, int src_len, + const CBMDefArray *defs) { + cbm_error_regions_t out = {{0}, {0}, 0, regs->dropped}; + /* One offset table for the whole file. The macro check below runs once per + * line, and without the table each of those calls walks the source from byte + * 0 to find its line — bytes times lines, on exactly the whole-file-error + * shape this refinement exists to narrow. */ + int *line_offsets = cbm_build_line_offsets(src, src_len, line_count); + for (int i = 0; i < regs->count; i++) { + uint32_t run_start = 0; + uint32_t run_end = 0; + uint32_t end = regs->ends[i] < line_count ? regs->ends[i] : line_count; + for (uint32_t line = regs->starts[i]; line <= end; line++) { + if ((map[line] & CBM_LINE_PP_PARSED) && + !cbm_line_is_toplevel_macro_call(src, src_len, line, line_offsets, defs)) { + if (run_start != 0) { + cbm_push_trimmed_run(&out, run_start, run_end, map, line_count); + run_start = 0; + } + } else { + if (run_start == 0) { + run_start = line; + } + run_end = line; + } + } + if (run_start != 0) { + cbm_push_trimmed_run(&out, run_start, run_end, map, line_count); + } + } + free(line_offsets); + *regs = out; +} + /* Serialize collected regions as "start-end,start-end,..." into the arena. */ +/* Share of a file one range must cover before the range stops being advice and + * becomes noise. 80% is well clear of anything real: the widest single range in + * this repo covers 25.5% of its file, and the next widest 3.9%. */ +#define CBM_UNUSABLE_PCT 80 + +/* Number of 1-based lines in `src`. A file that does not end with a newline + * still has a last line, so the count is separators plus one. */ +static uint32_t cbm_count_lines(const char *src, int src_len) { + uint32_t n = 1; + for (int i = 0; i < src_len; i++) { + if (src[i] == '\n' && i + 1 < src_len) { + n++; + } + } + return n; +} + +/* Serialize collected regions as "start-end,start-end,...", with a trailing + * ",+" when the cap threw N ranges away. + * + * The marker must stay a SUFFIX and nothing else. Every reader stops at the + * first token that is not a range, so a marker in the middle of a string + * silently hides everything after it. objectscript_export_append_error_ranges + * strips markers before joining two parts for exactly that reason. + * + * N can be non-zero while the kept list is short, because the recovery and + * macro rules run after collection and remove ranges the cap never saw. That + * still reports honestly: the cap bound, so what was lost is unknown. */ static const char *cbm_error_ranges_str(CBMArena *a, const cbm_error_regions_t *regs) { - if (regs->count <= 0) { + if (regs->count <= 0 && regs->dropped <= 0) { return NULL; } enum { RANGE_MAX = 24 }; /* "4294967295-4294967295," */ - char *buf = (char *)cbm_arena_alloc(a, (size_t)regs->count * RANGE_MAX); + char *buf = (char *)cbm_arena_alloc(a, (size_t)(regs->count + 1) * RANGE_MAX); if (!buf) { return NULL; } @@ -1178,6 +1466,9 @@ static const char *cbm_error_ranges_str(CBMArena *a, const cbm_error_regions_t * off += (size_t)snprintf(buf + off, RANGE_MAX, "%s%u-%u", i ? "," : "", regs->starts[i], regs->ends[i]); } + if (regs->dropped > 0) { + snprintf(buf + off, RANGE_MAX, "%s+%d", off ? "," : "", regs->dropped); + } return buf; } @@ -1419,6 +1710,14 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C // metrics. Remember the boundary. int orig_calls_count = result->calls.count; + /* Phase 2 line map, built by the second (preprocessed) pass below and read + * by the parse-coverage block near the end of this function. Stays NULL + * for every language that has no second pass, which leaves the coverage + * signal exactly as it was. Arena-allocated so it outlives the + * preprocessed source and its tree. */ + uint8_t *pp_line_map = NULL; + uint32_t pp_line_map_lines = 0; + // Second pass: preprocess C/C++/CUDA and extract additional macro-hidden calls. // Defs keep original-source line numbers; only CALLS are extracted from expanded source. if (language == CBM_LANG_C || language == CBM_LANG_CPP || language == CBM_LANG_CUDA) { @@ -1505,7 +1804,7 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C * the raw source line, and whose QN the raw pass did not * already extract. */ if (ts_node_has_error(root)) { - cbm_error_regions_t raw_regs = {{0}, {0}, 0}; + cbm_error_regions_t raw_regs = {{0}, {0}, 0, 0}; cbm_collect_error_regions(root, &raw_regs, source, source_len); if (raw_regs.count > 0) { int defs_before = result->defs.count; @@ -1545,6 +1844,59 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C } } + /* Build the original-line map before the expanded tree + * goes away. Skipped when the expanded parse is itself a + * total loss (root is ERROR), because then it vouches for + * nothing and there is no refinement to make. */ + if (strcmp(ts_node_type(pp_root), "ERROR") != 0) { + uint32_t orig_lines = 1; + for (int ci = 0; ci < source_len; ci++) { + if (source[ci] == '\n') { + orig_lines++; + } + } + uint8_t *map = (uint8_t *)cbm_arena_alloc(a, (size_t)orig_lines + 2); + int exp_lines = preprocessed->expanded_line_count; + uint8_t *bad_rows = + exp_lines > 0 ? (uint8_t *)calloc((size_t)exp_lines + 2, 1) : NULL; + if (map && bad_rows) { + memset(map, 0, (size_t)orig_lines + 2); + cbm_mark_no_code_lines(source, source_len, map, orig_lines); + cbm_mark_pp_error_rows(pp_root, bad_rows, (uint32_t)exp_lines, expanded, + expanded_len); + /* Walk the expanded text once. An expanded line + * only vouches for its original line when it + * actually HAS content: the preprocessor emits a + * blank line where it dropped a branch, and a + * blank line proves nothing about the code that + * used to be there. */ + uint32_t eline = 1; + bool eline_has_text = false; + for (int ci = 0; ci <= expanded_len; ci++) { + if (ci < expanded_len && expanded[ci] != '\n') { + char ch = expanded[ci]; + if (ch != ' ' && ch != '\t' && ch != '\r') { + eline_has_text = true; + } + continue; + } + if (eline_has_text && (int)eline <= exp_lines && !bad_rows[eline] && + preprocessed->belongs_to_main_file[eline]) { + uint32_t orig = + preprocessed->original_line_by_expanded_line[eline]; + if (orig >= 1 && orig <= orig_lines) { + map[orig] |= CBM_LINE_PP_PARSED; + } + } + eline++; + eline_has_text = false; + } + pp_line_map = map; + pp_line_map_lines = orig_lines; + } + free(bad_rows); + } + ts_tree_delete(pp_tree); } } @@ -1660,20 +2012,50 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C * miss, and a fully recovered file is not flagged at all. Detection aid * only: the absence of this flag is NOT a completeness guarantee. */ if (ts_node_has_error(root)) { - cbm_error_regions_t regs = {{0}, {0}, 0}; + cbm_error_regions_t regs = {{0}, {0}, 0, 0}; if (strcmp(ts_node_type(root), "ERROR") == 0) { cbm_error_regions_push(®s, root); /* whole file unparseable */ } else { cbm_collect_error_regions(root, ®s, source, source_len); } + /* Recovery subtraction runs on the RAW ranges, before the Phase 2 + * refinement below. Its evidence is a whole definition that starts + * inside the range, so it has to be asked while the range still + * matches the construct. Ask it after the refinement and an #ifdef + * splitting a brace inside a recovered function looks unrecovered: + * the refinement keeps only the discarded branch, the function starts + * above it, and the evidence falls outside the range. */ cbm_subtract_recovered_regions(®s, &result->defs); + /* Phase 2: cut what is left down to the lines the preprocessed parse + * could not explain. */ + if (pp_line_map) { + cbm_refine_regions_with_pp_lines(®s, pp_line_map, pp_line_map_lines, source, + source_len, &result->defs); + } /* #1071: don't flag a benign function-like-macro call (defined in-file) - * that tree-sitter can't parse without the preprocessor. */ + * that tree-sitter can't parse without the preprocessor. Runs AFTER the + * refinement, because its evidence is per-line: a narrow range points at + * the call itself instead of the whole blob around it. */ cbm_subtract_macro_invocation_regions(®s, &result->defs, source, source_len); - if (regs.count > 0) { + /* A file whose kept list is empty but whose cap still bound is NOT clean: + * the ranges the cap threw away were never judged by the two rules + * above, so nothing proves they were recovered. Flag it. */ + if (regs.count > 0 || regs.dropped > 0) { result->parse_incomplete = true; result->error_region_count = regs.count; result->error_ranges = cbm_error_ranges_str(a, ®s); + /* One range covering nearly the whole file is not advice, it is + * noise: "look at lines 1 to 13047" of a 13046-line file tells a + * reader nothing they did not already know. Mark those separately + * so the report can say "read the source" instead. See + * parse_unusable in cbm.h for which files land here and why. */ + if (regs.count == 1 && regs.dropped == 0) { + uint32_t total = cbm_count_lines(source, source_len); + uint32_t span = regs.ends[0] - regs.starts[0] + 1; + if (total > 0 && span * 100 >= total * CBM_UNUSABLE_PCT) { + result->parse_unusable = true; + } + } } } diff --git a/internal/cbm/cbm.h b/internal/cbm/cbm.h index f5636738d..a5bdba999 100644 --- a/internal/cbm/cbm.h +++ b/internal/cbm/cbm.h @@ -522,6 +522,21 @@ typedef struct CBMFileResult { * completeness guarantee. Callers should treat a flagged file as "prefer * grep here", never treat an unflagged file as provably complete. */ bool parse_incomplete; + /* True when the ranges cover so much of the file that they are no longer + * useful advice — one range over 80% of the line count. The file WAS + * indexed, but pointing a reader at almost every line tells them nothing, + * so the report says "read the source" instead of listing the range. + * + * Its main customers are non-C languages. The refinement that narrows a + * whole-file range using the preprocessed parse only runs for C, C++ and + * CUDA, so a Python, Java or Ruby file whose root node is ERROR still + * reports 1-N. + * + * Note the naming: this field and the phase string it produces are both + * `parse_unusable`. The older `parse_incomplete` field emits the phase + * `parse_partial` instead. That mismatch is historical, not deliberate — + * do not copy it. */ + bool parse_unusable; const char *error_ranges; int error_region_count; bool is_test_file; diff --git a/internal/cbm/extract_imports.c b/internal/cbm/extract_imports.c index b3d705a9f..544020db3 100644 --- a/internal/cbm/extract_imports.c +++ b/internal/cbm/extract_imports.c @@ -1383,12 +1383,13 @@ static void parse_spec_imports(CBMExtractCtx *ctx) { ts_tree_cursor_delete(&cursor); } -// --- Embedded-language imports --- -// Generic walker for host grammars (Svelte, Vue, HTML, Astro, ...) whose AST -// keeps embedded sub-language source as raw_text (or similar) without parsing -// it. The host's CBMLangSpec.embedded_imports declares which content nodes -// hold which sub-language; we re-parse each match with the embedded grammar -// and run the standard ES import walker over the inner AST. +// --- Embedded-language structure and imports --- +// Generic walker for host grammars (Svelte, Vue, HTML, Astro, CFML, ...) whose +// AST keeps embedded sub-language source as raw_text (or similar) without +// parsing it. The host's CBMLangSpec.embedded_imports declares which content +// nodes hold which sub-language; we re-parse each match with the embedded +// grammar and run the definition, import and call extractors over the inner +// AST, so a component's script contributes symbols like a standalone file. // // No grammar symbols are referenced here — the embedded TSLanguage is // resolved through cbm_ts_language(spec->embedded_language), the same hook @@ -1458,66 +1459,94 @@ static bool ascii_trimmed_equals(const char *value, const char *expected) { return true; } -/* Vue is the only host that opts into structural embedded extraction. Restrict - * its inline programs to the language forms whose parsers we can select - * exactly. A src attribute always denotes an external program and therefore - * suppresses any inline extraction, even for malformed mixed markup. */ -static bool vue_embedded_language(CBMExtractCtx *ctx, TSNode script, CBMLanguage *language) { - enum { VUE_ATTR_STACK_CAP = 128 }; - TSNode stack[VUE_ATTR_STACK_CAP]; +/* The text of a \n" + "\n"}, + }; + ASSERT_EQ(es_exact_edge_by_name(f, 1, "CALLS", "caller", "callee"), 1); + PASS(); +} + /* Java: caller in Main.java calls static method from Util.java (same package). */ TEST(es_calls_crossfile_java) { static const ES_LangFile f[] = { @@ -927,6 +940,7 @@ SUITE(edge_structural) { RUN_TEST(es_calls_crossfile_kotlin); RUN_TEST(es_calls_crossfile_csharp); RUN_TEST(es_calls_vue_embedded_issue1410); + RUN_TEST(es_calls_svelte_embedded_issue1807); /* ── FAMILY 2: INHERITS cross-file ────────────────────────── */ /* GREEN: Java, C#, C++ (extraction confirmed correct). */ diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 5c63c98aa..cb01dcf14 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -3372,7 +3372,10 @@ TEST(vue_embedded_structure_negative_controls_issue1410) { PASS(); } -TEST(vue_embedded_structure_host_controls_issue1410) { +/* The sibling hosts ride Vue's embedded seam: the function and call a .ts file + * yields come out of a plain \n"}, - {CBM_LANG_HTML, "control.html", - "\n"}, - {CBM_LANG_ASTRO, "Control.astro", - "---\nimport value from './astro.js'; function hidden() { target(); }\n---\n"}, + {CBM_LANG_SVELTE, "Sibling.svelte", + "\n"}, + {CBM_LANG_HTML, "sibling.html", + "\n"}, + {CBM_LANG_ASTRO, "Sibling.astro", + "---\nimport value from './astro.js'; function visible() { target(); }\n---\n"}, }; for (int i = 0; i < 3; i++) { CBMFileResult *r = extract(hosts[i].source, hosts[i].language, "t", hosts[i].path); ASSERT_NOT_NULL(r); ASSERT_FALSE(r->has_error); ASSERT_EQ(count_defs_with_label(r, "Module"), 1); + ASSERT_EQ(count_defs_named(r, "Function", "visible"), 1); + ASSERT_EQ(count_calls_named(r, "target"), 1); + ASSERT_EQ(r->imports.count, 1); + cbm_free_result(r); + } + PASS(); +} + +/* Blocks that must never yield inline symbols, whatever the host: an external + * program (src=) and a non-JavaScript MIME type. The bodies are deliberately + * code, so a leak would surface as a definition and a call. HTML's own tag + * walker still records the src= reference as an import; that edge names the + * external file and is not the inline body leaking through. */ +TEST(embedded_structure_inert_blocks_issue1807) { + static const struct { + CBMLanguage language; + const char *path; + const char *source; + int imports; + } blocks[] = { + {CBM_LANG_VUE, "Inert.vue", + "\n", 0}, + {CBM_LANG_VUE, "Inert.vue", + "\n", 0}, + {CBM_LANG_SVELTE, "Inert.svelte", + "\n", 0}, + {CBM_LANG_SVELTE, "Inert.svelte", + "\n", 0}, + {CBM_LANG_HTML, "inert.html", + "\n", 0}, + {CBM_LANG_HTML, "inert.html", + "\n", 0}, + {CBM_LANG_HTML, "inert.html", + "\n", 0}, + {CBM_LANG_HTML, "inert.html", + "\n", 1}, + {CBM_LANG_ASTRO, "Inert.astro", + "\n", 0}, + {CBM_LANG_ASTRO, "Inert.astro", + "\n", 0}, + }; + for (int i = 0; i < 10; i++) { + CBMFileResult *r = extract(blocks[i].source, blocks[i].language, "t", blocks[i].path); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + ASSERT_EQ(count_defs_with_label(r, "Module"), 1); ASSERT_EQ(count_defs_with_label(r, "Function"), 0); ASSERT_EQ(r->calls.count, 0); + ASSERT_EQ(r->imports.count, blocks[i].imports); + cbm_free_result(r); + } + PASS(); +} + +/* Svelte's module-level block (\n" + "\n" + "\n", + "\n" + "\n" + "\n", + }; + for (int i = 0; i < 2; i++) { + CBMFileResult *r = extract(sources[i], CBM_LANG_SVELTE, "t", "Widget.svelte"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + ASSERT_EQ(count_defs_with_label(r, "Function"), 2); + ASSERT_EQ(count_defs_named(r, "Function", "fromModule"), 1); + ASSERT_EQ(count_defs_named(r, "Function", "fromInstance"), 1); + ASSERT_EQ(count_calls_named(r, "shared"), 2); + ASSERT_EQ(count_calls_named(r, "fromModule"), 1); ASSERT_EQ(r->imports.count, 1); + ASSERT(has_import(r, "shared")); + for (int d = 0; d < r->defs.count; d++) { + const CBMDefinition *def = &r->defs.items[d]; + if (strcmp(def->name, "fromModule") == 0) { + ASSERT_EQ(def->start_line, 2); + } else if (strcmp(def->name, "fromInstance") == 0) { + ASSERT_EQ(def->start_line, 6); + } + } cbm_free_result(r); } PASS(); } +/* Every type= form HTML itself runs as JavaScript contributes, in host-file + * coordinates; a data block on the same page does not. */ +TEST(html_embedded_structure_issue1807) { + CBMFileResult *r = + extract("\n" + "\n" + "\n" + "\n" + "\n" + "\n", + CBM_LANG_HTML, "t", "index.html"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + ASSERT_EQ(count_defs_with_label(r, "Function"), 3); + ASSERT_EQ(count_calls_named(r, "renderApp"), 1); + ASSERT_EQ(count_calls_named(r, "boot"), 1); + ASSERT_EQ(count_calls_named(r, "legacy"), 1); + ASSERT_EQ(r->imports.count, 1); + for (int i = 0; i < r->defs.count; i++) { + const CBMDefinition *d = &r->defs.items[i]; + if (strcmp(d->name, "boot") == 0) { + ASSERT_EQ(d->start_line, 4); + } else if (strcmp(d->name, "legacy") == 0) { + ASSERT_EQ(d->start_line, 7); + } else if (strcmp(d->name, "fallback") == 0) { + ASSERT_EQ(d->start_line, 10); + } + } + cbm_free_result(r); + PASS(); +} + +/* Astro's frontmatter fence and \n", + CBM_LANG_ASTRO, "t", "Page.astro"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + ASSERT(has_def_any(r, "Props")); + ASSERT_EQ(count_defs_named(r, "Function", "heading"), 1); + ASSERT_EQ(count_defs_named(r, "Function", "hydrate"), 1); + ASSERT_EQ(count_calls_named(r, "format"), 1); + ASSERT_EQ(count_calls_named(r, "heading"), 1); + ASSERT_EQ(r->imports.count, 1); + ASSERT(has_import(r, "Header.astro")); + for (int i = 0; i < r->defs.count; i++) { + const CBMDefinition *d = &r->defs.items[i]; + if (strcmp(d->name, "heading") == 0) { + ASSERT_EQ(d->start_line, 5); + } else if (strcmp(d->name, "hydrate") == 0) { + ASSERT_EQ(d->start_line, 9); + } + } + cbm_free_result(r); + PASS(); +} + TEST(html_imports_basic) { /* Plain HTML with inline ES module imports — same generic walker. */ CBMFileResult *r = extract("\n" @@ -7235,7 +7405,11 @@ SUITE(extraction) { RUN_TEST(vue_imports_basic); RUN_TEST(vue_embedded_structure_issue1410); RUN_TEST(vue_embedded_structure_negative_controls_issue1410); - RUN_TEST(vue_embedded_structure_host_controls_issue1410); + RUN_TEST(embedded_structure_sibling_hosts_issue1807); + RUN_TEST(embedded_structure_inert_blocks_issue1807); + RUN_TEST(svelte_embedded_structure_both_blocks_issue1807); + RUN_TEST(html_embedded_structure_issue1807); + RUN_TEST(astro_embedded_structure_issue1807); RUN_TEST(html_imports_basic); /* config_extraction_test.go ports */ diff --git a/tests/test_grammar_labels.c b/tests/test_grammar_labels.c index 4008b6d2c..7818c4c44 100644 --- a/tests/test_grammar_labels.c +++ b/tests/test_grammar_labels.c @@ -236,8 +236,10 @@ static const LabelGolden LABEL_GOLDENS[] = { {"liquid", "Module:1"}, {"blade", "Module:1"}, {"vue", "Module:1"}, - {"svelte", "Module:1"}, - {"astro", "Module:1"}, + /* Svelte's