Skip to content

Commit 2dca0a3

Browse files
authored
Merge pull request DeusData#461 from halindrome/perl-lsp-semantic-resolution
feat(perl-lsp): Perl LSP-tier semantic resolution (Closes DeusData#459)
2 parents 23fd0e8 + 0815d78 commit 2dca0a3

12 files changed

Lines changed: 2399 additions & 4 deletions

File tree

‎Makefile.cbm‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -369,6 +369,8 @@ TEST_CS_LSP_SRCS = tests/test_cs_lsp.c
369369

370370
TEST_CS_LSP_BENCH_SRCS = tests/test_cs_lsp_bench.c
371371

372+
TEST_PERL_LSP_SRCS = tests/test_perl_lsp.c
373+
372374
TEST_SCOPE_SRCS = tests/test_scope.c
373375

374376
TEST_TYPE_REP_SRCS = tests/test_type_rep.c
@@ -472,7 +474,7 @@ TEST_REPRO_SRCS = \
472474
tests/repro/repro_lsp_java_cs.c \
473475
tests/repro/repro_lsp_kt_php_rust.c
474476

475-
ALL_TEST_SRCS =$(TEST_FOUNDATION_SRCS) $(TEST_EXTRACTION_SRCS) $(TEST_STORE_SRCS) $(TEST_CYPHER_SRCS) $(TEST_MCP_SRCS) $(TEST_DISCOVER_SRCS) $(TEST_GRAPH_BUFFER_SRCS) $(TEST_PIPELINE_SRCS) $(TEST_WATCHER_SRCS) $(TEST_LZ4_SRCS) $(TEST_ZSTD_SRCS) $(TEST_ARTIFACT_SRCS) $(TEST_SQLITE_WRITER_SRCS) $(TEST_GO_LSP_SRCS) $(TEST_C_LSP_SRCS) $(TEST_PHP_LSP_SRCS) $(TEST_CS_LSP_SRCS) $(TEST_CS_LSP_BENCH_SRCS) $(TEST_SCOPE_SRCS) $(TEST_TYPE_REP_SRCS) $(TEST_PY_LSP_SRCS) $(TEST_PY_LSP_BENCH_SRCS) $(TEST_PY_LSP_STRESS_SRCS) $(TEST_PY_LSP_SCALE_SRCS) $(TEST_TS_LSP_SRCS) $(TEST_JAVA_LSP_SRCS) $(TEST_KOTLIN_LSP_SRCS) $(TEST_RUST_LSP_SRCS) $(TEST_TRACES_SRCS) $(TEST_CLI_SRCS) $(TEST_MEM_SRCS) $(TEST_UI_SRCS) $(TEST_HTTPD_SRCS) $(TEST_SECURITY_SRCS) $(TEST_YAML_SRCS) $(TEST_SEMANTIC_SRCS) $(TEST_AST_PROFILE_SRCS) $(TEST_SLAB_ALLOC_SRCS) $(TEST_SIMHASH_SRCS) $(TEST_STACK_OVERFLOW_SRCS) $(TEST_INTEGRATION_SRCS)
477+
ALL_TEST_SRCS =$(TEST_FOUNDATION_SRCS) $(TEST_EXTRACTION_SRCS) $(TEST_STORE_SRCS) $(TEST_CYPHER_SRCS) $(TEST_MCP_SRCS) $(TEST_DISCOVER_SRCS) $(TEST_GRAPH_BUFFER_SRCS) $(TEST_PIPELINE_SRCS) $(TEST_WATCHER_SRCS) $(TEST_LZ4_SRCS) $(TEST_ZSTD_SRCS) $(TEST_ARTIFACT_SRCS) $(TEST_SQLITE_WRITER_SRCS) $(TEST_GO_LSP_SRCS) $(TEST_C_LSP_SRCS) $(TEST_PHP_LSP_SRCS) $(TEST_CS_LSP_SRCS) $(TEST_CS_LSP_BENCH_SRCS) $(TEST_PERL_LSP_SRCS) $(TEST_SCOPE_SRCS) $(TEST_TYPE_REP_SRCS) $(TEST_PY_LSP_SRCS) $(TEST_PY_LSP_BENCH_SRCS) $(TEST_PY_LSP_STRESS_SRCS) $(TEST_PY_LSP_SCALE_SRCS) $(TEST_TS_LSP_SRCS) $(TEST_JAVA_LSP_SRCS) $(TEST_KOTLIN_LSP_SRCS) $(TEST_RUST_LSP_SRCS) $(TEST_TRACES_SRCS) $(TEST_CLI_SRCS) $(TEST_MEM_SRCS) $(TEST_UI_SRCS) $(TEST_HTTPD_SRCS) $(TEST_SECURITY_SRCS) $(TEST_YAML_SRCS) $(TEST_SEMANTIC_SRCS) $(TEST_AST_PROFILE_SRCS) $(TEST_SLAB_ALLOC_SRCS) $(TEST_SIMHASH_SRCS) $(TEST_STACK_OVERFLOW_SRCS) $(TEST_INTEGRATION_SRCS)
476478

477479

478480
# ── Build directories ────────────────────────────────────────────

‎README.md‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
[![CI](https://img.shields.io/github/actions/workflow/status/DeusData/codebase-memory-mcp/dry-run.yml?label=CI)](https://github.com/DeusData/codebase-memory-mcp/actions/workflows/dry-run.yml)
66
[![Tests](https://img.shields.io/badge/tests-5604_passing-brightgreen)](https://github.com/DeusData/codebase-memory-mcp)
77
[![Languages](https://img.shields.io/badge/languages-158-orange)](https://github.com/DeusData/codebase-memory-mcp)
8-
[![Hybrid LSP](https://img.shields.io/badge/Hybrid_LSP-9_languages-blue)](#hybrid-lsp)
8+
[![Hybrid LSP](https://img.shields.io/badge/Hybrid_LSP-10_languages-blue)](#hybrid-lsp)
99
[![Agents](https://img.shields.io/badge/agents-11-purple)](https://github.com/DeusData/codebase-memory-mcp)
1010
[![Pure C](https://img.shields.io/badge/pure_C-zero_dependencies-blue)](https://github.com/DeusData/codebase-memory-mcp)
1111
[![Platform](https://img.shields.io/badge/macOS_%7C_Linux_%7C_Windows-supported-lightgrey)](https://github.com/DeusData/codebase-memory-mcp/releases/latest)
@@ -16,7 +16,7 @@
1616

1717
**The fastest and most efficient code intelligence engine for AI coding agents.** Full-indexes an average repository in milliseconds, the Linux kernel (28M LOC, 75K files) in 3 minutes. Answers structural queries in under 1ms. Ships as a single static binary for macOS, Linux, and Windows — download, run `install`, done.
1818

19-
High-quality parsing through [tree-sitter](https://tree-sitter.github.io/tree-sitter/) AST analysis across all 158 languages, enhanced with [**Hybrid LSP** semantic type resolution](#hybrid-lsp) for Python, TypeScript / JavaScript / JSX / TSX, PHP, C#, Go, C, C++, Java, Kotlin, and Rust — producing a persistent knowledge graph of functions, classes, call chains, HTTP routes, and cross-service links. 14 MCP tools. Zero dependencies. Plug and play across 11 coding agents.
19+
High-quality parsing through [tree-sitter](https://tree-sitter.github.io/tree-sitter/) AST analysis across all 158 languages, enhanced with [**Hybrid LSP** semantic type resolution](#hybrid-lsp) for Python, TypeScript / JavaScript / JSX / TSX, PHP, C#, Go, C, C++, Java, Kotlin, Rust, and Perl — producing a persistent knowledge graph of functions, classes, call chains, HTTP routes, and cross-service links. 14 MCP tools. Zero dependencies. Plug and play across 11 coding agents.
2020

2121
> **Research** — The design and benchmarks behind this project are described in the preprint [*Codebase-Memory: Tree-Sitter-Based Knowledge Graphs for LLM Code Exploration via MCP*](https://arxiv.org/abs/2603.27277) (arXiv:2603.27277). Evaluated across 31 real-world repositories: 83% answer quality, 10× fewer tokens, 2.1× fewer tool calls vs. file-by-file exploration.
2222
@@ -186,7 +186,7 @@ Removes all agent configs, skills, hooks, and instructions. Does not remove the
186186
- **158 vendored tree-sitter grammars** compiled into the binary
187187
- **Generic package / module resolution** — bare specifiers like `@myorg/pkg`, `github.com/foo/bar`, `use my_crate::foo` resolved via manifest scanning (`package.json`, `go.mod`, `Cargo.toml`, `pyproject.toml`, `composer.json`, `pubspec.yaml`, `pom.xml`, `build.gradle`, `mix.exs`, `*.gemspec`)
188188
- **Infrastructure-as-code indexing** — Dockerfiles, Kubernetes manifests, Kustomize overlays as graph nodes
189-
- **[Hybrid LSP semantic type resolution](#hybrid-lsp)** for Python, TypeScript / JavaScript / JSX / TSX, PHP, C#, Go, C, C++, Java, Kotlin, and Rust — a lightweight C implementation of language type-resolution algorithms, structurally inspired by and compatible with major language servers including tsserver / typescript-go, pyright, gopls, Roslyn, Eclipse JDT, and rust-analyzer (parameter binding, return-type inference, generic substitution, JSX component dispatch, JSDoc inference for plain JS files, namespace + trait + late-static-binding resolution for PHP, file-scoped namespaces + records + LINQ method syntax for C#, class-hierarchy + overload + lambda resolution for Java, extension-function + scope-function resolution for Kotlin, trait-method + UFCS resolution for Rust)
189+
- **[Hybrid LSP semantic type resolution](#hybrid-lsp)** for Python, TypeScript / JavaScript / JSX / TSX, PHP, C#, Go, C, C++, Java, Kotlin, Rust, and Perl — a lightweight C implementation of language type-resolution algorithms, structurally inspired by and compatible with major language servers including tsserver / typescript-go, pyright, gopls, Roslyn, Eclipse JDT, and rust-analyzer (parameter binding, return-type inference, generic substitution, JSX component dispatch, JSDoc inference for plain JS files, namespace + trait + late-static-binding resolution for PHP, file-scoped namespaces + records + LINQ method syntax for C#, class-hierarchy + overload + lambda resolution for Java, extension-function + scope-function resolution for Kotlin, trait-method + UFCS resolution for Rust)
190190
- **RAM-first pipeline**: LZ4 compression, in-memory SQLite, single dump at end. Memory released after.
191191

192192
### Distribution & operation
@@ -559,6 +559,7 @@ codebase-memory-mcp ships a **lightweight C implementation of language type-reso
559559
| **Java** *(new in v0.8.0)* | imports (single-type, on-demand, static), class hierarchies with `this` / `super` dispatch, generics, annotations, overload matching by arity and parameter types, lambdas / method references bound to functional interfaces, field-type inference, common JDK stdlib |
560560
| **Kotlin** *(new in v0.8.0)* | imports + same-package resolution, classes / objects / companion objects, extension functions, data classes, nullable-type unwrapping, scope functions (`let` / `apply` / `run` / `also` / `with`), infix calls, common stdlib |
561561
| **Rust** *(new in v0.8.0)* | `use` declarations + module paths, `impl` blocks and trait methods, struct fields, generics with trait bounds, operator-trait desugaring, derive-macro method synthesis, UFCS static paths, common std prelude |
562+
| **Perl** | packages + `@ISA` / `use parent` / `use base` inheritance with method-resolution-order dispatch, `SUPER::` calls, Exporter (`use Foo qw(...)`) import maps, `bless` / `ref($class)\|\|$class` self-type inference, qualified `Pkg::sub` static calls, curated perlfunc + CPAN OOP stdlib; unresolved receivers emit no edge (zero-edge guarantee) |
562563

563564
**Two-layer architecture:**
564565

‎internal/cbm/cbm.c‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
#include "lsp/go_lsp.h"
77
#include "lsp/c_lsp.h"
88
#include "lsp/php_lsp.h"
9+
#include "lsp/perl_lsp.h"
910
#include "lsp/py_lsp.h"
1011
#include "lsp/ts_lsp.h"
1112
#include "lsp/cs_lsp.h"
@@ -683,6 +684,43 @@ static void cbm_test_fault_inject(const char *rel_path) {
683684
}
684685
}
685686

687+
/* Pre-parse nesting guard for pathologically nested input. tree-sitter's GLR
688+
* parser recurses once per nesting level inside stack_node_add_link
689+
* (vendored ts_runtime/src/stack.c) while merging ambiguous parse-stack heads.
690+
* The Perl grammar is genuinely ambiguous for `f(...)` (function call vs.
691+
* bareword), so a deeply nested call chain `f(f(f(...)))` drives that recursion
692+
* as deep as the nesting and overflows a small (1 MB Windows) stack *during the
693+
* parse* — before any of the LSP walk-depth guards can fire. Unambiguous
694+
* grammars (C/Java/Python) keep a single stack head and don't hit this, which is
695+
* why only Perl crashed on the Windows/ARM CI runners.
696+
*
697+
* This is a workaround: the proper fix is bounding the GLR stack-merge recursion
698+
* inside the vendored tree-sitter runtime, tracked upstream as #913. Remove this
699+
* guard once that lands.
700+
*
701+
* cbm_source_nesting_exceeds scans the raw bytes for the maximum bracket-nesting
702+
* depth and returns true as soon as it passes the cap (early-exit, O(n)). Real
703+
* source never nests brackets this deep, so a file that does is skipped as a
704+
* parse error (zero edges — graceful degradation, never a crash). Brackets in
705+
* strings/comments are counted too: the only consequence of a false positive is
706+
* skipping one absurd file, so string-awareness is not worth the cost. */
707+
#define CBM_PERL_MAX_PARSE_NESTING 128
708+
709+
static bool cbm_source_nesting_exceeds(const char *source, int source_len, int cap) {
710+
int depth = 0;
711+
for (int i = 0; i < source_len; i++) {
712+
char c = source[i];
713+
if (c == '(' || c == '[' || c == '{') {
714+
if (++depth > cap) {
715+
return true;
716+
}
717+
} else if ((c == ')' || c == ']' || c == '}') && depth > 0) {
718+
depth--;
719+
}
720+
}
721+
return false;
722+
}
723+
686724
static CBMFileResult *cbm_extract_file_impl(const char *source, int source_len,
687725
CBMLanguage language, const char *project,
688726
const char *rel_path, int64_t timeout_micros,
@@ -867,6 +905,17 @@ static CBMFileResult *cbm_extract_file_impl(const char *source, int source_len,
867905
return result;
868906
}
869907

908+
// Skip pathologically nested Perl before tree-sitter's recursive GLR stack
909+
// merge overflows a small stack during the parse (see
910+
// cbm_source_nesting_exceeds). Scoped to Perl: its ambiguous call grammar is
911+
// the only one that drives that recursion to the nesting depth.
912+
if (language == CBM_LANG_PERL &&
913+
cbm_source_nesting_exceeds(source, source_len, CBM_PERL_MAX_PARSE_NESTING)) {
914+
result->has_error = true;
915+
result->error_msg = cbm_arena_strdup(a, "perl source nesting too deep; skipped");
916+
return result;
917+
}
918+
870919
// Get thread-local parser (reused across files on same thread)
871920
TSParser *parser = get_thread_parser(ts_lang, language);
872921
if (!parser) {
@@ -957,6 +1006,9 @@ static CBMFileResult *cbm_extract_file_impl(const char *source, int source_len,
9571006
if (language == CBM_LANG_PHP) {
9581007
cbm_run_php_lsp(a, result, source, source_len, root);
9591008
}
1009+
if (language == CBM_LANG_PERL) {
1010+
cbm_run_perl_lsp(a, result, source, source_len, root);
1011+
}
9601012
if (language == CBM_LANG_PYTHON) {
9611013
cbm_run_py_lsp(a, result, source, source_len, root);
9621014
}

0 commit comments

Comments
 (0)