Merge pull request #2551 from DeusData/feat/doc-links-documents

feat(doc-links): Markdown, ADR, reST, AsciiDoc and PDF sections link to the code they name

Author: DeusData

Makefile.cbm

@@ -375,6 +375,17 @@ EXTRACTION_SRCS = \
     $(CBM_DIR)/extract_dbt.c \
     $(CBM_DIR)/doclink.c \
     $(CBM_DIR)/doclink_cs.c \
+    $(CBM_DIR)/doclink_md.c \
+    $(CBM_DIR)/doclink_rst.c \
+    $(CBM_DIR)/doclink_py.c \
+    $(CBM_DIR)/doc_adr.c \
+    $(CBM_DIR)/doc_pdf.c \
+    $(CBM_DIR)/doc_adoc.c \
+    $(CBM_DIR)/pdf/pdf_obj.c \
+    $(CBM_DIR)/pdf/pdf_font.c \
+    $(CBM_DIR)/pdf/pdf_text.c \
+    $(CBM_DIR)/pdf/pdf_tables.c \
+    $(CBM_DIR)/pdf/pdf_unicode.c \
     $(CBM_DIR)/sql_values.c \
     $(CBM_DIR)/helpers.c \
     $(CBM_DIR)/cpp_branch_views.c \
@@ -497,6 +508,10 @@ PIPELINE_SRCS = \
     src/pipeline/pass_doclinks.c \
     src/pipeline/doc_links.c \
     src/pipeline/doc_links_cs.c \
+    src/pipeline/doc_links_md.c \
+    src/pipeline/doc_links_pdf.c \
+    src/pipeline/doc_links_rst.c \
+    src/pipeline/doc_links_adoc.c \
     src/pipeline/doc_links_msbuild.c \
     src/pipeline/pass_route_nodes.c \
     src/pipeline/pass_enrichment.c \
@@ -759,7 +774,7 @@ TEST_DISCOVER_SRCS = \
 
 TEST_GRAPH_BUFFER_SRCS = tests/test_graph_buffer.c
 
-TEST_PIPELINE_SRCS = tests/test_registry.c tests/test_pipeline.c tests/test_pipeline_conventions.c tests/test_pipeline_frozen.c tests/test_importance.c tests/test_cross_repo.c tests/test_fqn.c tests/test_route_canon.c tests/test_path_alias.c tests/test_configlink.c tests/test_doclinks.c tests/test_doc_mentions.c tests/test_infrascan.c tests/test_worker_pool.c tests/test_parallel.c tests/test_index_resilience.c tests/test_index_format.c tests/test_call_reference_contract.c tests/repro/repro_call_scope_usages.c tests/repro/repro_call_argument_usages.c tests/repro/repro_reference_precision.c tests/repro/repro_lexical_binding_precision.c tests/repro/repro_call_argument_matrix_a.c tests/repro/repro_call_argument_matrix_b.c tests/repro/repro_call_node_behaviors.c tests/repro/repro_language_registry.c tests/repro/repro_call_node_manifest.c tests/repro/repro_lsp_ordered_signatures.c tests/repro/repro_lsp_ordered_local.c tests/repro/repro_ts_overload_return_chains.c tests/repro/repro_harness_cleanup.c tests/repro/repro_runner_filter.c
+TEST_PIPELINE_SRCS = tests/test_registry.c tests/test_pipeline.c tests/test_pipeline_conventions.c tests/test_pipeline_frozen.c tests/test_importance.c tests/test_cross_repo.c tests/test_fqn.c tests/test_route_canon.c tests/test_path_alias.c tests/test_configlink.c tests/test_doclinks.c tests/test_doc_mentions.c tests/test_doc_links_md.c tests/test_doc_links_pdf.c tests/test_doc_links_rst.c tests/test_doc_links_adoc.c tests/test_infrascan.c tests/test_worker_pool.c tests/test_parallel.c tests/test_index_resilience.c tests/test_index_format.c tests/test_call_reference_contract.c tests/repro/repro_call_scope_usages.c tests/repro/repro_call_argument_usages.c tests/repro/repro_reference_precision.c tests/repro/repro_lexical_binding_precision.c tests/repro/repro_call_argument_matrix_a.c tests/repro/repro_call_argument_matrix_b.c tests/repro/repro_call_node_behaviors.c tests/repro/repro_language_registry.c tests/repro/repro_call_node_manifest.c tests/repro/repro_lsp_ordered_signatures.c tests/repro/repro_lsp_ordered_local.c tests/repro/repro_ts_overload_return_chains.c tests/repro/repro_harness_cleanup.c tests/repro/repro_runner_filter.c
 
 TEST_WATCHER_SRCS = tests/test_watcher.c
 

THIRD_PARTY.md

@@ -138,6 +138,109 @@ Semantic vector search uses static token embeddings derived from the
 See `vendored/nomic/NOTICE` for the exact derivation procedure
 (per-token inference + int8 quantization via `scripts/extract_nomic_vectors.py`).
 
+## Embedded Character Data (PDF text layer)
+
+The PDF text-layer extractor (`internal/cbm/pdf/`) carries static tables
+derived from the following sources. No upstream source code is included.
+
+- **Adobe Glyph List** — the glyph-name subset in `internal/cbm/pdf/pdf_tables.c`
+  (the names of the standard Latin encodings and common symbols, mapped to
+  Unicode) follows [adobe-type-tools/agl-aglfn](https://github.com/adobe-type-tools/agl-aglfn)
+  `glyphlist.txt`.
+  - **License:** BSD-3-Clause
+  - **Copyright:** Copyright 2002-2019 Adobe (http://www.adobe.com/).
+- **Adobe core-14 font metrics** — the advance widths of the printable ASCII
+  range of Helvetica and Times-Roman in `pdf_tables.c` (used only for a
+  standard-14 font that omits its `/Widths`) are the values of Adobe's
+  core-14 AFM files. The AFM files themselves are not distributed.
+  - **Copyright:** Copyright (c) 1985, 1987, 1989, 1990, 1997 Adobe Systems
+    Incorporated. All Rights Reserved.
+- **Unicode Character Database** — `internal/cbm/pdf/pdf_unicode.c` (NFKC of
+  each code point, canonical compositions, character classes) is generated
+  by `scripts/gen-pdf-unicode.py` from CPython's `unicodedata` (Unicode
+  16.0.0).
+  - **License:** Unicode License v3 (text below)
+  - **Copyright:** Copyright © 1991-2026 Unicode, Inc.
+- The base encodings (Standard, WinAnsi, MacRoman, Symbol) are those of
+  ISO 32000-1, Annex D.
+
+Adobe Glyph List license (BSD-3-Clause):
+
+```
+Copyright 2002-2019 Adobe (http://www.adobe.com/).
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+Redistributions of source code must retain the above copyright notice,
+this list of conditions and the following disclaimer.
+
+Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+Neither the name of Adobe nor the names of its contributors may be
+used to endorse or promote products derived from this software without
+specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+```
+
+Unicode License v3:
+
+```
+UNICODE LICENSE V3
+
+COPYRIGHT AND PERMISSION NOTICE
+
+Copyright © 1991-2026 Unicode, Inc.
+
+NOTICE TO USER: Carefully read the following legal agreement. BY
+DOWNLOADING, INSTALLING, COPYING OR OTHERWISE USING DATA FILES, AND/OR
+SOFTWARE, YOU UNEQUIVOCALLY ACCEPT, AND AGREE TO BE BOUND BY, ALL OF THE
+TERMS AND CONDITIONS OF THIS AGREEMENT. IF YOU DO NOT AGREE, DO NOT
+DOWNLOAD, INSTALL, COPY, DISTRIBUTE OR USE THE DATA FILES OR SOFTWARE.
+
+Permission is hereby granted, free of charge, to any person obtaining a
+copy of data files and any associated documentation (the "Data Files") or
+software and any associated documentation (the "Software") to deal in the
+Data Files or Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, distribute, and/or sell
+copies of the Data Files or Software, and to permit persons to whom the
+Data Files or Software are furnished to do so, provided that either (a)
+this copyright and permission notice appear with all copies of the Data
+Files or Software, or (b) this copyright and permission notice appear in
+associated Documentation.
+
+THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF
+THIRD PARTY RIGHTS.
+
+IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE
+BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES,
+OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
+WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION,
+ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THE DATA
+FILES OR SOFTWARE.
+
+Except as contained in this notice, the name of a copyright holder shall
+not be used in advertising or otherwise to promote the sale, use or other
+dealings in these Data Files or Software without prior written
+authorization of the copyright holder.
+```
+
 ## Hybrid LSP — Reference Language Servers
 
 The Hybrid LSP layer (`internal/cbm/lsp/`) is an original C implementation

internal/cbm/cbm.c

@@ -2802,6 +2802,45 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C
     cbm_test_fault_inject(rel_path);
 #endif
 
+    /* A PDF has no grammar: its text layer is read directly, a Section per
+     * page plus the pages' code mentions (doc_pdf.c). */
+    if (language == CBM_LANG_PDF) {
+        result->module_qn = cbm_fqn_module_source_lang(a, project, rel_path, language);
+        CBMExtractCtx pdf_ctx = {
+            .arena = a,
+            .scratch = scratch,
+            .result = result,
+            .source = source,
+            .source_len = source_len,
+            .language = language,
+            .project = project,
+            .rel_path = rel_path,
+            .module_qn = result->module_qn,
+        };
+        cbm_pdf_extract_document(&pdf_ctx);
+        cbm_index_mark_done(rel_path);
+        return result;
+    }
+    /* AsciiDoc has no grammar either: its headings, includes, attribute
+     * references and monospace spans are read from the text (doc_adoc.c). */
+    if (language == CBM_LANG_ASCIIDOC) {
+        result->module_qn = cbm_fqn_module_source_lang(a, project, rel_path, language);
+        CBMExtractCtx adoc_ctx = {
+            .arena = a,
+            .scratch = scratch,
+            .result = result,
+            .source = source,
+            .source_len = source_len,
+            .language = language,
+            .project = project,
+            .rel_path = rel_path,
+            .module_qn = result->module_qn,
+        };
+        cbm_adoc_extract_document(&adoc_ctx);
+        cbm_index_mark_done(rel_path);
+        return result;
+    }
+
     // Get language spec
     const CBMLangSpec *spec = cbm_lang_spec(language);
     if (!spec) {

internal/cbm/cbm.h

@@ -209,6 +209,8 @@ typedef enum {
     CBM_LANG_ARKTS,    // ArkTS (HarmonyOS/OpenHarmony .ets — TypeScript superset + ArkUI)
     CBM_LANG_PLSQL,    // Oracle PL/SQL
     CBM_LANG_CHIALISP, // Chialisp (.clsp/.clib/.clinc — Chia smart-coin s-expression language)
+    CBM_LANG_PDF,      // PDF documents: the text layer, read without a grammar (doc_pdf.c)
+    CBM_LANG_ASCIIDOC, // AsciiDoc documents, read without a grammar (doc_adoc.c)
     CBM_LANG_COUNT
 } CBMLanguage;
 
@@ -292,6 +294,12 @@ typedef struct {
      * are bound to the owning result's source identity before cross-file use. */
     uint32_t test_name_start_byte, test_name_end_byte;
     uint32_t test_body_start_byte, test_body_end_byte;
+    /* Documents only: facts a document definition adds to its node's
+     * properties, as a NULL-terminated key, value, key, value, ... list (an
+     * ADR's id, status, date, deciders, title). Keys are fixed literals; the
+     * values are escaped where the properties are written. Tail field:
+     * zero-init stays valid. */
+    const char **extra_props;
 } CBMDefinition;
 
 /* Argument captured from a call expression */