@@ -3861,7 +3861,132 @@ static const char *qn_safe_segment(CBMArena *a, const char *name) {
38613861 return out ;
38623862}
38633863
3864- static void push_simple_class_def (CBMExtractCtx * ctx , TSNode node , char * name , const char * label ) {
3864+ /* UTF-8 sequence classification: a lead byte's high bits name the sequence
3865+ * length, and every continuation byte matches 10xxxxxx. */
3866+ enum {
3867+ UTF8_CONT_MASK = 0xC0 , /* isolate the two high bits ... */
3868+ UTF8_CONT_MARK = 0x80 , /* ... which are 10 on a continuation byte */
3869+ UTF8_LEAD2_MASK = 0xE0 , /* 110xxxxx → 2-byte sequence */
3870+ UTF8_LEAD2_MARK = 0xC0 ,
3871+ UTF8_LEAD3_MASK = 0xF0 , /* 1110xxxx → 3-byte sequence */
3872+ UTF8_LEAD3_MARK = 0xE0 ,
3873+ UTF8_LEAD4_MASK = 0xF8 , /* 11110xxx → 4-byte sequence */
3874+ UTF8_LEAD4_MARK = 0xF0 ,
3875+ UTF8_LEN_1 = 1 ,
3876+ UTF8_LEN_2 = 2 ,
3877+ UTF8_LEN_3 = 3 ,
3878+ UTF8_LEN_4 = 4 ,
3879+ };
3880+
3881+ /* Bytes the sequence starting with `lead` occupies (1 for ASCII or a byte that
3882+ * is not a valid lead). */
3883+ static size_t utf8_sequence_len (unsigned char lead ) {
3884+ if ((lead & UTF8_LEAD2_MASK ) == UTF8_LEAD2_MARK ) {
3885+ return UTF8_LEN_2 ;
3886+ }
3887+ if ((lead & UTF8_LEAD3_MASK ) == UTF8_LEAD3_MARK ) {
3888+ return UTF8_LEN_3 ;
3889+ }
3890+ if ((lead & UTF8_LEAD4_MASK ) == UTF8_LEAD4_MARK ) {
3891+ return UTF8_LEN_4 ;
3892+ }
3893+ return UTF8_LEN_1 ;
3894+ }
3895+
3896+ /* Drop a trailing PARTIAL UTF-8 sequence left behind by a byte-length cut, so
3897+ * a capped prose value never ends mid-codepoint (#1017's rule, applied to the
3898+ * prose bodies below rather than to comments). */
3899+ static void utf8_trim_partial_tail (char * text ) {
3900+ size_t n = strlen (text );
3901+ if (n == 0 ) {
3902+ return ;
3903+ }
3904+ size_t i = n ;
3905+ while (i > 0 && ((unsigned char )text [i - UTF8_LEN_1 ] & UTF8_CONT_MASK ) == UTF8_CONT_MARK ) {
3906+ i -- ;
3907+ }
3908+ if (i == 0 ) {
3909+ text [0 ] = '\0' ; /* continuation bytes only — not decodable */
3910+ return ;
3911+ }
3912+ size_t need = utf8_sequence_len ((unsigned char )text [i - UTF8_LEN_1 ]);
3913+ if (i - UTF8_LEN_1 + need > n ) {
3914+ text [i - UTF8_LEN_1 ] = '\0' ;
3915+ }
3916+ }
3917+
3918+ /* Collapse `len` bytes of raw prose into a single-spaced value capped at
3919+ * MAX_COMMENT_LEN (the same 500-byte ceiling docstrings already use, which
3920+ * leaves room inside the 2 KB properties buffer that carries it).
3921+ *
3922+ * The output buffer is FIXED at that cap, so a section body of any size costs a
3923+ * bounded copy rather than a copy of the whole section. Returns NULL when
3924+ * nothing but whitespace was there. */
3925+ static char * collapse_prose (CBMArena * a , const char * src , size_t len ) {
3926+ if (!src || len == 0 ) {
3927+ return NULL ;
3928+ }
3929+ char * out = (char * )cbm_arena_alloc (a , MAX_COMMENT_LEN + NULL_TERM );
3930+ if (!out ) {
3931+ return NULL ;
3932+ }
3933+ size_t w = 0 ;
3934+ bool in_ws = true; /* start true so leading whitespace is swallowed */
3935+ for (size_t i = 0 ; i < len && w < MAX_COMMENT_LEN ; i ++ ) {
3936+ unsigned char c = (unsigned char )src [i ];
3937+ if (c == ' ' || c == '\t' || c == '\n' || c == '\r' ) {
3938+ in_ws = true;
3939+ continue ;
3940+ }
3941+ if (in_ws && w > 0 ) {
3942+ out [w ++ ] = ' ' ;
3943+ if (w >= MAX_COMMENT_LEN ) {
3944+ break ;
3945+ }
3946+ }
3947+ in_ws = false;
3948+ out [w ++ ] = (char )c ;
3949+ }
3950+ out [w ] = '\0' ;
3951+ utf8_trim_partial_tail (out );
3952+ return out [0 ] ? out : NULL ;
3953+ }
3954+
3955+ static bool is_markdown_heading_kind (const char * kind ) {
3956+ return strcmp (kind , "atx_heading" ) == 0 || strcmp (kind , "setext_heading" ) == 0 ;
3957+ }
3958+
3959+ /* #518: a Markdown heading node is only the title line — the section's prose
3960+ * lives in the blocks that FOLLOW it. Collect that prose as the Section's
3961+ * docstring so the node carries the text a reader would search for; nodes_fts
3962+ * indexes it from there (`body`), which is the whole point of the issue.
3963+ *
3964+ * tree-sitter-markdown wraps a heading and its content in a `section` node,
3965+ * with nested subsections as further `section` children that own their own
3966+ * text. Stopping at either a `section` or another heading therefore gives each
3967+ * Section exactly its OWN body under both that shape and a flat one. The bytes
3968+ * between are contiguous in the source, so one slice beats concatenation. */
3969+ static const char * extract_markdown_section_body (CBMArena * a , TSNode heading , const char * source ) {
3970+ uint32_t start = ts_node_end_byte (heading );
3971+ uint32_t end = start ;
3972+ for (TSNode sib = ts_node_next_sibling (heading ); !ts_node_is_null (sib );
3973+ sib = ts_node_next_sibling (sib )) {
3974+ const char * sk = ts_node_type (sib );
3975+ if (strcmp (sk , "section" ) == 0 || is_markdown_heading_kind (sk )) {
3976+ break ;
3977+ }
3978+ end = ts_node_end_byte (sib );
3979+ }
3980+ if (end <= start ) {
3981+ return NULL ;
3982+ }
3983+ return collapse_prose (a , source + start , (size_t )(end - start ));
3984+ }
3985+
3986+ /* Push a config-language definition that carries prose. push_simple_class_def
3987+ * delegates here with no docstring. */
3988+ static void push_simple_class_def_doc (CBMExtractCtx * ctx , TSNode node , char * name ,
3989+ const char * label , const char * docstring ) {
38653990 CBMArena * a = ctx -> arena ;
38663991 CBMDefinition def ;
38673992 memset (& def , 0 , sizeof (def ));
@@ -3872,9 +3997,14 @@ static void push_simple_class_def(CBMExtractCtx *ctx, TSNode node, char *name, c
38723997 def .start_line = ts_node_start_point (node ).row + TS_LINE_OFFSET ;
38733998 def .end_line = ts_node_end_point (node ).row + TS_LINE_OFFSET ;
38743999 def .is_exported = true;
4000+ def .docstring = docstring ;
38754001 cbm_defs_push (& ctx -> result -> defs , a , def );
38764002}
38774003
4004+ static void push_simple_class_def (CBMExtractCtx * ctx , TSNode node , char * name , const char * label ) {
4005+ push_simple_class_def_doc (ctx , node , name , label , NULL );
4006+ }
4007+
38784008// Find TOML table key name from children.
38794009static char * find_toml_key_name (CBMArena * a , TSNode node , const char * source ) {
38804010 uint32_t nc = ts_node_child_count (node );
@@ -4033,6 +4163,7 @@ static bool extract_config_class_def(CBMExtractCtx *ctx, TSNode node, const char
40334163 CBMArena * a = ctx -> arena ;
40344164 char * name = NULL ;
40354165 const char * label = "Class" ;
4166+ const char * docstring = NULL ;
40364167
40374168 if (ctx -> language == CBM_LANG_TOML &&
40384169 (strcmp (kind , "table" ) == 0 || strcmp (kind , "table_array_element" ) == 0 )) {
@@ -4048,14 +4179,16 @@ static bool extract_config_class_def(CBMExtractCtx *ctx, TSNode node, const char
40484179 // label rather than degrade it to match a test. The markdown repro asserts
40494180 // "Class"; that assertion is the inaccurate side and is flagged for review.
40504181 label = "Section" ;
4182+ // #518: index what the section SAYS, not just its title.
4183+ docstring = extract_markdown_section_body (a , node , ctx -> source );
40514184 } else if (ctx -> language == CBM_LANG_HCL && strcmp (kind , "block" ) == 0 ) {
40524185 name = find_hcl_block_name (a , node , ctx -> source );
40534186 } else {
40544187 return false;
40554188 }
40564189
40574190 if (name && name [0 ]) {
4058- push_simple_class_def (ctx , node , name , label );
4191+ push_simple_class_def_doc (ctx , node , name , label , docstring );
40594192 }
40604193 return true;
40614194}
@@ -6175,12 +6308,9 @@ static bool is_helm_values_file(const char *rel) {
61756308 return strcmp (b , "values.yaml" ) == 0 || strcmp (b , "values.yml" ) == 0 ;
61766309}
61776310
6178- // Extract ONLY top-level keys of a YAML document (no leaf explosion). Used for
6179- // Helm values.yaml so each chart's tunables surface as a handful of structured
6180- // Variables instead of one node per nested leaf (#338).
6181- static void extract_yaml_toplevel_keys (CBMExtractCtx * ctx , TSNode root ) {
6182- CBMArena * a = ctx -> arena ;
6183- // Descend stream -> document -> block_node down to the first block_mapping.
6311+ // Descend stream -> document -> block_node to a YAML document's top-level
6312+ // block_mapping. Returns a null node when the document has none.
6313+ static TSNode find_yaml_toplevel_mapping (TSNode root ) {
61846314 TSNode bm = {0 };
61856315 TSNode cur = root ;
61866316 for (int depth = 0 ; depth < 6 && ts_node_is_null (bm ); depth ++ ) {
@@ -6204,6 +6334,126 @@ static void extract_yaml_toplevel_keys(CBMExtractCtx *ctx, TSNode root) {
62046334 }
62056335 cur = next ;
62066336 }
6337+ return bm ;
6338+ }
6339+
6340+ // Descend to a JSON document's top-level object. Returns a null node if absent.
6341+ static TSNode find_json_toplevel_object (TSNode root ) {
6342+ if (strcmp (ts_node_type (root ), "object" ) == 0 ) {
6343+ return root ;
6344+ }
6345+ TSNode obj = cbm_find_child_by_kind (root , "object" );
6346+ if (!ts_node_is_null (obj )) {
6347+ return obj ;
6348+ }
6349+ uint32_t n = ts_node_named_child_count (root );
6350+ for (uint32_t i = 0 ; i < n ; i ++ ) {
6351+ TSNode inner = cbm_find_child_by_kind (ts_node_named_child (root , i ), "object" );
6352+ if (!ts_node_is_null (inner )) {
6353+ return inner ;
6354+ }
6355+ }
6356+ TSNode null_node = {0 };
6357+ return null_node ;
6358+ }
6359+
6360+ /* #519: a config file's own prose sits in a top-level `description` (or one of
6361+ * its usual synonyms) — META.yaml, action.yml, an OpenAPI document,
6362+ * package.json. Nothing indexed it: the VALUE is not a definition, so no node
6363+ * carried it and BM25 could not see it at all. Checked in priority order; the
6364+ * first key present wins. */
6365+ static const char * const config_desc_keys [] = {"description" , "summary" , "purpose" , NULL };
6366+
6367+ // Strip one layer of matching surrounding quotes. Operates on arena text.
6368+ static char * strip_surrounding_quotes (char * t ) {
6369+ if (!t ) {
6370+ return t ;
6371+ }
6372+ size_t n = strlen (t );
6373+ if (n >= PAIR_CHARS && (t [0 ] == '"' || t [0 ] == '\'' ) && t [n - SKIP_CHAR ] == t [0 ]) {
6374+ t [n - SKIP_CHAR ] = '\0' ;
6375+ return t + SKIP_CHAR ;
6376+ }
6377+ return t ;
6378+ }
6379+
6380+ // Normalise a config scalar into an indexable value: drop a YAML block-scalar
6381+ // header (`|`/`>` plus its chomping and indent modifiers), collapse whitespace
6382+ // to the shared MAX_COMMENT_LEN cap, then unquote.
6383+ static const char * config_scalar_value (CBMArena * a , const char * raw ) {
6384+ if (!raw ) {
6385+ return NULL ;
6386+ }
6387+ while (* raw == ' ' || * raw == '\t' || * raw == '\n' || * raw == '\r' ) {
6388+ raw ++ ;
6389+ }
6390+ if (* raw == '|' || * raw == '>' ) {
6391+ while (* raw && * raw != '\n' ) {
6392+ raw ++ ;
6393+ }
6394+ }
6395+ char * v = collapse_prose (a , raw , strlen (raw ));
6396+ return v ? strip_surrounding_quotes (v ) : NULL ;
6397+ }
6398+
6399+ // Value of `pair` when its key is `want`, else NULL. A YAML block_mapping_pair
6400+ // and a JSON pair both expose key/value fields, so one reader serves both.
6401+ static const char * config_pair_value_if_key (CBMExtractCtx * ctx , TSNode pair , const char * want ) {
6402+ TSNode key = ts_node_child_by_field_name (pair , TS_FIELD ("key" ));
6403+ TSNode val = ts_node_child_by_field_name (pair , TS_FIELD ("value" ));
6404+ if (ts_node_is_null (key ) || ts_node_is_null (val )) {
6405+ return NULL ;
6406+ }
6407+ char * kt = cbm_node_text (ctx -> arena , key , ctx -> source );
6408+ if (!kt || strcmp (strip_surrounding_quotes (kt ), want ) != 0 ) {
6409+ return NULL ;
6410+ }
6411+ return config_scalar_value (ctx -> arena , cbm_node_text (ctx -> arena , val , ctx -> source ));
6412+ }
6413+
6414+ // First non-empty description value among `container`'s direct pairs, scanned
6415+ // in config_desc_keys priority order.
6416+ static const char * config_container_description (CBMExtractCtx * ctx , TSNode container ,
6417+ const char * pair_kind ) {
6418+ uint32_t n = ts_node_named_child_count (container );
6419+ for (const char * const * k = config_desc_keys ; * k ; k ++ ) {
6420+ for (uint32_t i = 0 ; i < n ; i ++ ) {
6421+ TSNode pair = ts_node_named_child (container , i );
6422+ if (strcmp (ts_node_type (pair ), pair_kind ) != 0 ) {
6423+ continue ;
6424+ }
6425+ const char * v = config_pair_value_if_key (ctx , pair , * k );
6426+ if (v && v [0 ]) {
6427+ return v ;
6428+ }
6429+ }
6430+ }
6431+ return NULL ;
6432+ }
6433+
6434+ /* #519 entry point: the file-level description a config document declares about
6435+ * itself, promoted onto the Module node by cbm_extract_definitions so
6436+ * nodes_fts.body indexes it and the file is findable by what it SAYS it does,
6437+ * not only by its path. NULL for every other language. */
6438+ static const char * extract_config_module_description (CBMExtractCtx * ctx ) {
6439+ if (ctx -> language == CBM_LANG_YAML ) {
6440+ TSNode bm = find_yaml_toplevel_mapping (ctx -> root );
6441+ return ts_node_is_null (bm ) ? NULL
6442+ : config_container_description (ctx , bm , "block_mapping_pair" );
6443+ }
6444+ if (ctx -> language == CBM_LANG_JSON ) {
6445+ TSNode obj = find_json_toplevel_object (ctx -> root );
6446+ return ts_node_is_null (obj ) ? NULL : config_container_description (ctx , obj , "pair" );
6447+ }
6448+ return NULL ;
6449+ }
6450+
6451+ // Extract ONLY top-level keys of a YAML document (no leaf explosion). Used for
6452+ // Helm values.yaml so each chart's tunables surface as a handful of structured
6453+ // Variables instead of one node per nested leaf (#338).
6454+ static void extract_yaml_toplevel_keys (CBMExtractCtx * ctx , TSNode root ) {
6455+ CBMArena * a = ctx -> arena ;
6456+ TSNode bm = find_yaml_toplevel_mapping (root );
62076457 if (ts_node_is_null (bm )) {
62086458 return ;
62096459 }
@@ -7587,6 +7837,8 @@ void cbm_extract_definitions(CBMExtractCtx *ctx) {
75877837 mod .end_line = ts_node_end_point (ctx -> root ).row + TS_LINE_OFFSET ;
75887838 mod .is_exported = true;
75897839 mod .is_test = ctx -> result -> is_test_file ;
7840+ // #519: index what a config file declares itself to be, not only its path.
7841+ mod .docstring = extract_config_module_description (ctx );
75907842 cbm_defs_push (& ctx -> result -> defs , a , mod );
75917843
75927844 cbm_extract_definitions_without_module (ctx );
0 commit comments