Skip to content

Commit be7b578

Browse files
authored
Merge pull request DeusData#1882 from DeusData/feat/518-519-bm25-prose
feat(search): index prose content for BM25 full-text search
2 parents 51c48e2 + dc23628 commit be7b578

10 files changed

Lines changed: 1287 additions & 110 deletions

File tree

‎internal/cbm/extract_defs.c‎

Lines changed: 260 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -3861,7 +3861,132 @@ static const char *qn_safe_segment(CBMArena *a, const char *name) {
38613861
return out;
38623862
}
38633863

3864-
static void push_simple_class_def(CBMExtractCtx *ctx, TSNode node, char *name, const char *label) {
3864+
/* UTF-8 sequence classification: a lead byte's high bits name the sequence
3865+
* length, and every continuation byte matches 10xxxxxx. */
3866+
enum {
3867+
UTF8_CONT_MASK = 0xC0, /* isolate the two high bits ... */
3868+
UTF8_CONT_MARK = 0x80, /* ... which are 10 on a continuation byte */
3869+
UTF8_LEAD2_MASK = 0xE0, /* 110xxxxx → 2-byte sequence */
3870+
UTF8_LEAD2_MARK = 0xC0,
3871+
UTF8_LEAD3_MASK = 0xF0, /* 1110xxxx → 3-byte sequence */
3872+
UTF8_LEAD3_MARK = 0xE0,
3873+
UTF8_LEAD4_MASK = 0xF8, /* 11110xxx → 4-byte sequence */
3874+
UTF8_LEAD4_MARK = 0xF0,
3875+
UTF8_LEN_1 = 1,
3876+
UTF8_LEN_2 = 2,
3877+
UTF8_LEN_3 = 3,
3878+
UTF8_LEN_4 = 4,
3879+
};
3880+
3881+
/* Bytes the sequence starting with `lead` occupies (1 for ASCII or a byte that
3882+
* is not a valid lead). */
3883+
static size_t utf8_sequence_len(unsigned char lead) {
3884+
if ((lead & UTF8_LEAD2_MASK) == UTF8_LEAD2_MARK) {
3885+
return UTF8_LEN_2;
3886+
}
3887+
if ((lead & UTF8_LEAD3_MASK) == UTF8_LEAD3_MARK) {
3888+
return UTF8_LEN_3;
3889+
}
3890+
if ((lead & UTF8_LEAD4_MASK) == UTF8_LEAD4_MARK) {
3891+
return UTF8_LEN_4;
3892+
}
3893+
return UTF8_LEN_1;
3894+
}
3895+
3896+
/* Drop a trailing PARTIAL UTF-8 sequence left behind by a byte-length cut, so
3897+
* a capped prose value never ends mid-codepoint (#1017's rule, applied to the
3898+
* prose bodies below rather than to comments). */
3899+
static void utf8_trim_partial_tail(char *text) {
3900+
size_t n = strlen(text);
3901+
if (n == 0) {
3902+
return;
3903+
}
3904+
size_t i = n;
3905+
while (i > 0 && ((unsigned char)text[i - UTF8_LEN_1] & UTF8_CONT_MASK) == UTF8_CONT_MARK) {
3906+
i--;
3907+
}
3908+
if (i == 0) {
3909+
text[0] = '\0'; /* continuation bytes only — not decodable */
3910+
return;
3911+
}
3912+
size_t need = utf8_sequence_len((unsigned char)text[i - UTF8_LEN_1]);
3913+
if (i - UTF8_LEN_1 + need > n) {
3914+
text[i - UTF8_LEN_1] = '\0';
3915+
}
3916+
}
3917+
3918+
/* Collapse `len` bytes of raw prose into a single-spaced value capped at
3919+
* MAX_COMMENT_LEN (the same 500-byte ceiling docstrings already use, which
3920+
* leaves room inside the 2 KB properties buffer that carries it).
3921+
*
3922+
* The output buffer is FIXED at that cap, so a section body of any size costs a
3923+
* bounded copy rather than a copy of the whole section. Returns NULL when
3924+
* nothing but whitespace was there. */
3925+
static char *collapse_prose(CBMArena *a, const char *src, size_t len) {
3926+
if (!src || len == 0) {
3927+
return NULL;
3928+
}
3929+
char *out = (char *)cbm_arena_alloc(a, MAX_COMMENT_LEN + NULL_TERM);
3930+
if (!out) {
3931+
return NULL;
3932+
}
3933+
size_t w = 0;
3934+
bool in_ws = true; /* start true so leading whitespace is swallowed */
3935+
for (size_t i = 0; i < len && w < MAX_COMMENT_LEN; i++) {
3936+
unsigned char c = (unsigned char)src[i];
3937+
if (c == ' ' || c == '\t' || c == '\n' || c == '\r') {
3938+
in_ws = true;
3939+
continue;
3940+
}
3941+
if (in_ws && w > 0) {
3942+
out[w++] = ' ';
3943+
if (w >= MAX_COMMENT_LEN) {
3944+
break;
3945+
}
3946+
}
3947+
in_ws = false;
3948+
out[w++] = (char)c;
3949+
}
3950+
out[w] = '\0';
3951+
utf8_trim_partial_tail(out);
3952+
return out[0] ? out : NULL;
3953+
}
3954+
3955+
static bool is_markdown_heading_kind(const char *kind) {
3956+
return strcmp(kind, "atx_heading") == 0 || strcmp(kind, "setext_heading") == 0;
3957+
}
3958+
3959+
/* #518: a Markdown heading node is only the title line — the section's prose
3960+
* lives in the blocks that FOLLOW it. Collect that prose as the Section's
3961+
* docstring so the node carries the text a reader would search for; nodes_fts
3962+
* indexes it from there (`body`), which is the whole point of the issue.
3963+
*
3964+
* tree-sitter-markdown wraps a heading and its content in a `section` node,
3965+
* with nested subsections as further `section` children that own their own
3966+
* text. Stopping at either a `section` or another heading therefore gives each
3967+
* Section exactly its OWN body under both that shape and a flat one. The bytes
3968+
* between are contiguous in the source, so one slice beats concatenation. */
3969+
static const char *extract_markdown_section_body(CBMArena *a, TSNode heading, const char *source) {
3970+
uint32_t start = ts_node_end_byte(heading);
3971+
uint32_t end = start;
3972+
for (TSNode sib = ts_node_next_sibling(heading); !ts_node_is_null(sib);
3973+
sib = ts_node_next_sibling(sib)) {
3974+
const char *sk = ts_node_type(sib);
3975+
if (strcmp(sk, "section") == 0 || is_markdown_heading_kind(sk)) {
3976+
break;
3977+
}
3978+
end = ts_node_end_byte(sib);
3979+
}
3980+
if (end <= start) {
3981+
return NULL;
3982+
}
3983+
return collapse_prose(a, source + start, (size_t)(end - start));
3984+
}
3985+
3986+
/* Push a config-language definition that carries prose. push_simple_class_def
3987+
* delegates here with no docstring. */
3988+
static void push_simple_class_def_doc(CBMExtractCtx *ctx, TSNode node, char *name,
3989+
const char *label, const char *docstring) {
38653990
CBMArena *a = ctx->arena;
38663991
CBMDefinition def;
38673992
memset(&def, 0, sizeof(def));
@@ -3872,9 +3997,14 @@ static void push_simple_class_def(CBMExtractCtx *ctx, TSNode node, char *name, c
38723997
def.start_line = ts_node_start_point(node).row + TS_LINE_OFFSET;
38733998
def.end_line = ts_node_end_point(node).row + TS_LINE_OFFSET;
38743999
def.is_exported = true;
4000+
def.docstring = docstring;
38754001
cbm_defs_push(&ctx->result->defs, a, def);
38764002
}
38774003

4004+
static void push_simple_class_def(CBMExtractCtx *ctx, TSNode node, char *name, const char *label) {
4005+
push_simple_class_def_doc(ctx, node, name, label, NULL);
4006+
}
4007+
38784008
// Find TOML table key name from children.
38794009
static char *find_toml_key_name(CBMArena *a, TSNode node, const char *source) {
38804010
uint32_t nc = ts_node_child_count(node);
@@ -4033,6 +4163,7 @@ static bool extract_config_class_def(CBMExtractCtx *ctx, TSNode node, const char
40334163
CBMArena *a = ctx->arena;
40344164
char *name = NULL;
40354165
const char *label = "Class";
4166+
const char *docstring = NULL;
40364167

40374168
if (ctx->language == CBM_LANG_TOML &&
40384169
(strcmp(kind, "table") == 0 || strcmp(kind, "table_array_element") == 0)) {
@@ -4048,14 +4179,16 @@ static bool extract_config_class_def(CBMExtractCtx *ctx, TSNode node, const char
40484179
// label rather than degrade it to match a test. The markdown repro asserts
40494180
// "Class"; that assertion is the inaccurate side and is flagged for review.
40504181
label = "Section";
4182+
// #518: index what the section SAYS, not just its title.
4183+
docstring = extract_markdown_section_body(a, node, ctx->source);
40514184
} else if (ctx->language == CBM_LANG_HCL && strcmp(kind, "block") == 0) {
40524185
name = find_hcl_block_name(a, node, ctx->source);
40534186
} else {
40544187
return false;
40554188
}
40564189

40574190
if (name && name[0]) {
4058-
push_simple_class_def(ctx, node, name, label);
4191+
push_simple_class_def_doc(ctx, node, name, label, docstring);
40594192
}
40604193
return true;
40614194
}
@@ -6175,12 +6308,9 @@ static bool is_helm_values_file(const char *rel) {
61756308
return strcmp(b, "values.yaml") == 0 || strcmp(b, "values.yml") == 0;
61766309
}
61776310

6178-
// Extract ONLY top-level keys of a YAML document (no leaf explosion). Used for
6179-
// Helm values.yaml so each chart's tunables surface as a handful of structured
6180-
// Variables instead of one node per nested leaf (#338).
6181-
static void extract_yaml_toplevel_keys(CBMExtractCtx *ctx, TSNode root) {
6182-
CBMArena *a = ctx->arena;
6183-
// Descend stream -> document -> block_node down to the first block_mapping.
6311+
// Descend stream -> document -> block_node to a YAML document's top-level
6312+
// block_mapping. Returns a null node when the document has none.
6313+
static TSNode find_yaml_toplevel_mapping(TSNode root) {
61846314
TSNode bm = {0};
61856315
TSNode cur = root;
61866316
for (int depth = 0; depth < 6 && ts_node_is_null(bm); depth++) {
@@ -6204,6 +6334,126 @@ static void extract_yaml_toplevel_keys(CBMExtractCtx *ctx, TSNode root) {
62046334
}
62056335
cur = next;
62066336
}
6337+
return bm;
6338+
}
6339+
6340+
// Descend to a JSON document's top-level object. Returns a null node if absent.
6341+
static TSNode find_json_toplevel_object(TSNode root) {
6342+
if (strcmp(ts_node_type(root), "object") == 0) {
6343+
return root;
6344+
}
6345+
TSNode obj = cbm_find_child_by_kind(root, "object");
6346+
if (!ts_node_is_null(obj)) {
6347+
return obj;
6348+
}
6349+
uint32_t n = ts_node_named_child_count(root);
6350+
for (uint32_t i = 0; i < n; i++) {
6351+
TSNode inner = cbm_find_child_by_kind(ts_node_named_child(root, i), "object");
6352+
if (!ts_node_is_null(inner)) {
6353+
return inner;
6354+
}
6355+
}
6356+
TSNode null_node = {0};
6357+
return null_node;
6358+
}
6359+
6360+
/* #519: a config file's own prose sits in a top-level `description` (or one of
6361+
* its usual synonyms) — META.yaml, action.yml, an OpenAPI document,
6362+
* package.json. Nothing indexed it: the VALUE is not a definition, so no node
6363+
* carried it and BM25 could not see it at all. Checked in priority order; the
6364+
* first key present wins. */
6365+
static const char *const config_desc_keys[] = {"description", "summary", "purpose", NULL};
6366+
6367+
// Strip one layer of matching surrounding quotes. Operates on arena text.
6368+
static char *strip_surrounding_quotes(char *t) {
6369+
if (!t) {
6370+
return t;
6371+
}
6372+
size_t n = strlen(t);
6373+
if (n >= PAIR_CHARS && (t[0] == '"' || t[0] == '\'') && t[n - SKIP_CHAR] == t[0]) {
6374+
t[n - SKIP_CHAR] = '\0';
6375+
return t + SKIP_CHAR;
6376+
}
6377+
return t;
6378+
}
6379+
6380+
// Normalise a config scalar into an indexable value: drop a YAML block-scalar
6381+
// header (`|`/`>` plus its chomping and indent modifiers), collapse whitespace
6382+
// to the shared MAX_COMMENT_LEN cap, then unquote.
6383+
static const char *config_scalar_value(CBMArena *a, const char *raw) {
6384+
if (!raw) {
6385+
return NULL;
6386+
}
6387+
while (*raw == ' ' || *raw == '\t' || *raw == '\n' || *raw == '\r') {
6388+
raw++;
6389+
}
6390+
if (*raw == '|' || *raw == '>') {
6391+
while (*raw && *raw != '\n') {
6392+
raw++;
6393+
}
6394+
}
6395+
char *v = collapse_prose(a, raw, strlen(raw));
6396+
return v ? strip_surrounding_quotes(v) : NULL;
6397+
}
6398+
6399+
// Value of `pair` when its key is `want`, else NULL. A YAML block_mapping_pair
6400+
// and a JSON pair both expose key/value fields, so one reader serves both.
6401+
static const char *config_pair_value_if_key(CBMExtractCtx *ctx, TSNode pair, const char *want) {
6402+
TSNode key = ts_node_child_by_field_name(pair, TS_FIELD("key"));
6403+
TSNode val = ts_node_child_by_field_name(pair, TS_FIELD("value"));
6404+
if (ts_node_is_null(key) || ts_node_is_null(val)) {
6405+
return NULL;
6406+
}
6407+
char *kt = cbm_node_text(ctx->arena, key, ctx->source);
6408+
if (!kt || strcmp(strip_surrounding_quotes(kt), want) != 0) {
6409+
return NULL;
6410+
}
6411+
return config_scalar_value(ctx->arena, cbm_node_text(ctx->arena, val, ctx->source));
6412+
}
6413+
6414+
// First non-empty description value among `container`'s direct pairs, scanned
6415+
// in config_desc_keys priority order.
6416+
static const char *config_container_description(CBMExtractCtx *ctx, TSNode container,
6417+
const char *pair_kind) {
6418+
uint32_t n = ts_node_named_child_count(container);
6419+
for (const char *const *k = config_desc_keys; *k; k++) {
6420+
for (uint32_t i = 0; i < n; i++) {
6421+
TSNode pair = ts_node_named_child(container, i);
6422+
if (strcmp(ts_node_type(pair), pair_kind) != 0) {
6423+
continue;
6424+
}
6425+
const char *v = config_pair_value_if_key(ctx, pair, *k);
6426+
if (v && v[0]) {
6427+
return v;
6428+
}
6429+
}
6430+
}
6431+
return NULL;
6432+
}
6433+
6434+
/* #519 entry point: the file-level description a config document declares about
6435+
* itself, promoted onto the Module node by cbm_extract_definitions so
6436+
* nodes_fts.body indexes it and the file is findable by what it SAYS it does,
6437+
* not only by its path. NULL for every other language. */
6438+
static const char *extract_config_module_description(CBMExtractCtx *ctx) {
6439+
if (ctx->language == CBM_LANG_YAML) {
6440+
TSNode bm = find_yaml_toplevel_mapping(ctx->root);
6441+
return ts_node_is_null(bm) ? NULL
6442+
: config_container_description(ctx, bm, "block_mapping_pair");
6443+
}
6444+
if (ctx->language == CBM_LANG_JSON) {
6445+
TSNode obj = find_json_toplevel_object(ctx->root);
6446+
return ts_node_is_null(obj) ? NULL : config_container_description(ctx, obj, "pair");
6447+
}
6448+
return NULL;
6449+
}
6450+
6451+
// Extract ONLY top-level keys of a YAML document (no leaf explosion). Used for
6452+
// Helm values.yaml so each chart's tunables surface as a handful of structured
6453+
// Variables instead of one node per nested leaf (#338).
6454+
static void extract_yaml_toplevel_keys(CBMExtractCtx *ctx, TSNode root) {
6455+
CBMArena *a = ctx->arena;
6456+
TSNode bm = find_yaml_toplevel_mapping(root);
62076457
if (ts_node_is_null(bm)) {
62086458
return;
62096459
}
@@ -7587,6 +7837,8 @@ void cbm_extract_definitions(CBMExtractCtx *ctx) {
75877837
mod.end_line = ts_node_end_point(ctx->root).row + TS_LINE_OFFSET;
75887838
mod.is_exported = true;
75897839
mod.is_test = ctx->result->is_test_file;
7840+
// #519: index what a config file declares itself to be, not only its path.
7841+
mod.docstring = extract_config_module_description(ctx);
75907842
cbm_defs_push(&ctx->result->defs, a, mod);
75917843

75927844
cbm_extract_definitions_without_module(ctx);

0 commit comments

Comments
 (0)