|
5 | 5 | * definition (file, class, function, method, interface, type, enum, and TS |
6 | 6 | * arrow-function consts) plus unresolved edge intents. Edge *targets* are |
7 | 7 | * resolved against the whole-repo node index later, in build.ts. |
| 8 | + * |
| 9 | + * Native tree-sitter addons load via `require` inside a try (issue #119). When |
| 10 | + * they are missing — linux/arm64 has no prebuild in tree-sitter@0.21.1 — |
| 11 | + * `extractFile` delegates to the WASM breadth extractor instead of throwing |
| 12 | + * at module load. Darwin / linux-x64 keep the hand-written walk unchanged. |
8 | 13 | */ |
9 | | -import Parser from "tree-sitter"; |
10 | | -import TypeScript from "tree-sitter-typescript"; |
11 | | -import Python from "tree-sitter-python"; |
12 | | -import Go from "tree-sitter-go"; |
13 | | -import R from "tree-sitter-r"; |
14 | | -import Java from "tree-sitter-java"; |
15 | | -import Kotlin from "tree-sitter-kotlin"; |
16 | | -import PHP from "tree-sitter-php"; |
| 14 | +import type Parser from "tree-sitter"; |
| 15 | +import { createRequire } from "node:module"; |
17 | 16 | import { basename } from "node:path"; |
18 | 17 | import { contentHash } from "../util/id.js"; |
19 | 18 | import { collectBindings, goReceiverVarOf, resolveRecvType, type FileBindings } from "./bindings.js"; |
| 19 | +import { extractGeneric } from "./generic.js"; |
20 | 20 | import type { Kind, NodeV1, Relation } from "./types.js"; |
21 | 21 |
|
| 22 | +const require = createRequire(import.meta.url); |
| 23 | + |
22 | 24 | export type Language = "typescript" | "tsx" | "python" | "go" | "java" | "kotlin" | "php" | "r"; |
23 | 25 |
|
24 | 26 | /** |
@@ -81,6 +83,83 @@ export function languageLabelOf(path: string): string | null { |
81 | 83 | return entryFor(path)?.label ?? null; |
82 | 84 | } |
83 | 85 |
|
| 86 | +export const NATIVE_FALLBACK_WARNING = |
| 87 | + "native parser unavailable — falling back to WASM (reduced binding fidelity)"; |
| 88 | + |
| 89 | +const DEPTH_LANGS: Language[] = ["typescript", "tsx", "python", "go", "java", "kotlin", "php", "r"]; |
| 90 | + |
| 91 | +function asParserCtor(mod: unknown): (new () => Parser) | null { |
| 92 | + if (typeof mod === "function") return mod as new () => Parser; |
| 93 | + if (mod && typeof mod === "object" && "default" in mod) { |
| 94 | + const d = (mod as { default: unknown }).default; |
| 95 | + if (typeof d === "function") return d as new () => Parser; |
| 96 | + } |
| 97 | + return null; |
| 98 | +} |
| 99 | + |
| 100 | +export interface NativeLoadResult { |
| 101 | + Parser: (new () => Parser) | null; |
| 102 | + grammars: Partial<Record<Language, unknown>>; |
| 103 | + error: string | null; |
| 104 | +} |
| 105 | + |
| 106 | +/** Load native grammars via `require`. Injectable so tests can simulate a missing |
| 107 | + * prebuild (`No native build was found for platform=linux arch=arm64`) without |
| 108 | + * unloading this process's real addons. Native available → same grammars as |
| 109 | + * the old static imports; a throw or a missing Parser ctor → empty grammars. */ |
| 110 | +export function loadNativeGrammars(req: (id: string) => unknown = require): NativeLoadResult { |
| 111 | + try { |
| 112 | + const ParserCtor = asParserCtor(req("tree-sitter")); |
| 113 | + if (!ParserCtor) { |
| 114 | + return { Parser: null, grammars: {}, error: "tree-sitter did not export a Parser constructor" }; |
| 115 | + } |
| 116 | + const grammars: Partial<Record<Language, unknown>> = {}; |
| 117 | + const one = (id: string): unknown | null => { |
| 118 | + try { |
| 119 | + return req(id); |
| 120 | + } catch { |
| 121 | + return null; |
| 122 | + } |
| 123 | + }; |
| 124 | + const ts = one("tree-sitter-typescript") as { typescript?: unknown; tsx?: unknown } | null; |
| 125 | + if (ts?.typescript) grammars.typescript = ts.typescript; |
| 126 | + if (ts?.tsx) grammars.tsx = ts.tsx; |
| 127 | + const py = one("tree-sitter-python"); |
| 128 | + if (py) grammars.python = py; |
| 129 | + const go = one("tree-sitter-go"); |
| 130 | + if (go) grammars.go = go; |
| 131 | + const r = one("tree-sitter-r"); |
| 132 | + if (r) grammars.r = r; |
| 133 | + const java = one("tree-sitter-java"); |
| 134 | + if (java) grammars.java = java; |
| 135 | + const kotlin = one("tree-sitter-kotlin"); |
| 136 | + if (kotlin) grammars.kotlin = kotlin; |
| 137 | + const php = one("tree-sitter-php") as { php?: unknown } | null; |
| 138 | + if (php?.php) grammars.php = php.php; |
| 139 | + return { Parser: ParserCtor, grammars, error: null }; |
| 140 | + } catch (err) { |
| 141 | + const message = err instanceof Error ? err.message : String(err); |
| 142 | + return { Parser: null, grammars: {}, error: message }; |
| 143 | + } |
| 144 | +} |
| 145 | + |
| 146 | +const native = loadNativeGrammars(); |
| 147 | +const parser: Parser | null = native.Parser ? new native.Parser() : null; |
| 148 | +const GRAMMARS: Partial<Record<Language, unknown>> = native.grammars; |
| 149 | + |
| 150 | +/** Depth-tier languages whose native grammar did not load — warm these as WASM. */ |
| 151 | +export function wasmFallbackLangs(): Language[] { |
| 152 | + if (!parser) return [...DEPTH_LANGS]; |
| 153 | + return DEPTH_LANGS.filter((l) => GRAMMARS[l] === undefined); |
| 154 | +} |
| 155 | + |
| 156 | +/** One-line banner when the native runtime itself failed to load (issue #119). |
| 157 | + * Missing *individual* grammars still WASM-fall back, but that is not the |
| 158 | + * linux/arm64 crash and must not warn on darwin where tree-sitter loaded. */ |
| 159 | +export function nativeFallbackWarning(): string | null { |
| 160 | + return parser ? null : NATIVE_FALLBACK_WARNING; |
| 161 | +} |
| 162 | + |
84 | 163 | /** |
85 | 164 | * An edge whose target isn't resolved yet. build.ts turns these into EdgeV1 by |
86 | 165 | * matching `name`/`specifier` against the repo-wide node index. |
@@ -274,18 +353,6 @@ const FUNCTION_VALUE_TYPES = new Set([ |
274 | 353 |
|
275 | 354 | const EMPTY_SET: ReadonlySet<string> = new Set(); |
276 | 355 |
|
277 | | -const parser = new Parser(); |
278 | | -const GRAMMARS: Record<Language, unknown> = { |
279 | | - typescript: TypeScript.typescript, |
280 | | - tsx: TypeScript.tsx, |
281 | | - python: Python, |
282 | | - go: Go, |
283 | | - r: R, |
284 | | - java: Java, |
285 | | - kotlin: Kotlin, |
286 | | - php: PHP.php, |
287 | | -}; |
288 | | - |
289 | 356 | export interface WalkCtx { |
290 | 357 | rel: string; |
291 | 358 | source: string; |
@@ -344,10 +411,13 @@ interface DefDescriptor { |
344 | 411 | * the source in <32 KB slices. Code-unit indexing matches `String.slice`. */ |
345 | 412 | const PARSE_CHUNK = 16384; |
346 | 413 | function parseSource(source: string): Parser.SyntaxNode { |
347 | | - return parser.parse((index: number) => source.slice(index, index + PARSE_CHUNK)).rootNode; |
| 414 | + return parser!.parse((index: number) => source.slice(index, index + PARSE_CHUNK)).rootNode; |
348 | 415 | } |
349 | 416 |
|
350 | | -export function extractFile(rel: string, source: string, lang: Language): ExtractResult { |
| 417 | +export function extractFile(rel: string, source: string, lang: Language, opts?: { wasm?: boolean }): ExtractResult { |
| 418 | + if (opts?.wasm || !parser || GRAMMARS[lang] === undefined) { |
| 419 | + return extractGeneric(rel, source, lang); |
| 420 | + } |
351 | 421 | parser.setLanguage(GRAMMARS[lang] as never); |
352 | 422 | const root = parseSource(source); |
353 | 423 | const bindings = collectBindings(root, lang); |
|
0 commit comments