Skip to content

Commit 597e2ae

Browse files
fix(build): fall back to WASM grammars when native tree-sitter is unavailable (#119)
linux/arm64 has no tree-sitter@0.21.1 prebuild, so require() crashed every CLI command. Catch the native load, extract depth languages via the existing WASM path, and warn about reduced binding fidelity. Darwin/linux-x64 native behavior is unchanged. Co-authored-by: Cursor <cursoragent@cursor.com>
1 parent ee1ef03 commit 597e2ae

13 files changed

Lines changed: 406 additions & 29 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,20 @@
11
# Changelog
22

3+
## Unreleased
4+
5+
### Fixed
6+
7+
- **`graft init` / `graft build` no longer crash on linux/arm64** when
8+
`tree-sitter@0.21.1` has no native prebuild (`No native build was found for
9+
platform=linux arch=arm64`). Native parsers still load when they exist
10+
(darwin, linux-x64). If `require("tree-sitter")` fails, depth languages
11+
(TypeScript/JavaScript, Python, Go, Java, Kotlin, PHP, R) fall back to the
12+
existing WASM breadth grammars. Fidelity is reduced: no bindings /
13+
receiver-type resolution; call edges come from `tags.scm` (symbols only if
14+
a query is missing). The build prints
15+
`native parser unavailable — falling back to WASM (reduced binding fidelity)`
16+
and `graft check` / the statusline do not treat it as an error. (#119)
17+
318
## 0.13.0
419

520
### Added

‎src/cli.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -470,6 +470,7 @@ program
470470
{ repo: buildRoot },
471471
);
472472
for (const e of g.errors) console.error(`✗ ${e}`);
473+
for (const w of g.warnings) console.error(`⚠ ${w}`);
473474

474475
const rel = relative(process.cwd(), g.contextDir) || "graft";
475476
console.log(` ${rel}/ is git-ignored (added automatically) — a local cache; teammates run \`graft build\` to get their own.`);

‎src/graph/build.ts‎

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ import { readFileSync } from "node:fs";
1717
import { basename, dirname, resolve } from "node:path";
1818
import { walkDir } from "../ingest/fs.js";
1919
import { contextDirFor, ensureGitignored, ensureSearchable } from "../context/node-file.js";
20-
import { extractFile, languageLabelOf, languageOf, type RawEdge } from "./extract.js";
20+
import { extractFile, languageLabelOf, languageOf, nativeFallbackWarning, wasmFallbackLangs, type RawEdge } from "./extract.js";
2121
import { extractGeneric, genericLangOf, warmGenericGrammars } from "./generic.js";
2222
import { containerLangOf, extractContainer, warmContainerGrammars } from "./container.js";
2323
import { contentHash } from "../util/id.js";
@@ -117,6 +117,8 @@ export interface GraphBuildResult {
117117
languages: string[];
118118
meaning: EnrichStats;
119119
errors: string[];
120+
/** Non-fatal notices (native parser WASM fallback, etc.). Not check-failures. */
121+
warnings: string[];
120122
}
121123

122124
/** Every Go module in the repo: each `go.mod`'s declared `module` path and the repo
@@ -185,9 +187,22 @@ export async function buildGraph(
185187

186188
// Breadth tier: WASM grammars load asynchronously, so warm the ones this repo
187189
// needs ONCE here (buildGraph is async) before the synchronous parse loop below
188-
// can call extractGeneric. Depth-tier (native) grammars need no warmup.
190+
// can call extractGeneric. Depth-tier (native) grammars need no warmup — unless
191+
// they failed to load, in which case the same WASM path covers them (#119).
192+
const fallback = new Set(wasmFallbackLangs());
189193
await warmGenericGrammars(
190-
new Set(files.map((f) => genericLangOf(f.abs)?.name).filter((n): n is string => !!n)),
194+
new Set(
195+
files.flatMap((f) => {
196+
const names: string[] = [];
197+
const generic = genericLangOf(f.abs)?.name;
198+
if (generic) names.push(generic);
199+
const lang = languageOf(f.abs);
200+
if (lang && fallback.has(lang)) names.push(lang);
201+
const container = containerLangOf(f.abs);
202+
if (container && fallback.has(container.inner)) names.push(container.inner);
203+
return names;
204+
}),
205+
),
191206
);
192207
// Container tier (.vue and friends) loads its wrapper grammars the same way,
193208
// for the same reason: extractContainer runs inside the sync loop below.
@@ -399,5 +414,6 @@ export async function buildGraph(
399414
languages: [...langs].sort(),
400415
meaning,
401416
errors,
417+
warnings: [nativeFallbackWarning()].filter((w): w is string => w !== null),
402418
};
403419
}

‎src/graph/check.ts‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
import { resolve } from "node:path";
2121
import { relPosix } from "../util/paths.js";
2222
import { contextDirFor } from "../context/node-file.js";
23-
import { extractFile, languageOf } from "./extract.js";
23+
import { extractFile, languageOf, wasmFallbackLangs } from "./extract.js";
2424
import { extractGeneric, genericLangOf, warmGenericGrammars } from "./generic.js";
2525
import { listSourceFiles } from "./build.js";
2626
import { readGraph, wiringPath } from "./write.js";
@@ -80,8 +80,18 @@ export async function checkGraph(
8080

8181
// Freshly extract Tier-1 nodes from the code on disk (same file set as build).
8282
const sourceFiles = listSourceFiles(root, outDir);
83+
const fallback = new Set(wasmFallbackLangs());
8384
await warmGenericGrammars(
84-
new Set(sourceFiles.map((f) => genericLangOf(f)?.name).filter((n): n is string => !!n)),
85+
new Set(
86+
sourceFiles.flatMap((f) => {
87+
const names: string[] = [];
88+
const generic = genericLangOf(f)?.name;
89+
if (generic) names.push(generic);
90+
const lang = languageOf(f);
91+
if (lang && fallback.has(lang)) names.push(lang);
92+
return names;
93+
}),
94+
),
8595
);
8696
const current = new Map<string, string>(); // id → body_hash
8797
for (const file of sourceFiles) {

‎src/graph/extract.ts‎

Lines changed: 92 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -5,20 +5,22 @@
55
* definition (file, class, function, method, interface, type, enum, and TS
66
* arrow-function consts) plus unresolved edge intents. Edge *targets* are
77
* resolved against the whole-repo node index later, in build.ts.
8+
*
9+
* Native tree-sitter addons load via `require` inside a try (issue #119). When
10+
* they are missing — linux/arm64 has no prebuild in tree-sitter@0.21.1 —
11+
* `extractFile` delegates to the WASM breadth extractor instead of throwing
12+
* at module load. Darwin / linux-x64 keep the hand-written walk unchanged.
813
*/
9-
import Parser from "tree-sitter";
10-
import TypeScript from "tree-sitter-typescript";
11-
import Python from "tree-sitter-python";
12-
import Go from "tree-sitter-go";
13-
import R from "tree-sitter-r";
14-
import Java from "tree-sitter-java";
15-
import Kotlin from "tree-sitter-kotlin";
16-
import PHP from "tree-sitter-php";
14+
import type Parser from "tree-sitter";
15+
import { createRequire } from "node:module";
1716
import { basename } from "node:path";
1817
import { contentHash } from "../util/id.js";
1918
import { collectBindings, goReceiverVarOf, resolveRecvType, type FileBindings } from "./bindings.js";
19+
import { extractGeneric } from "./generic.js";
2020
import type { Kind, NodeV1, Relation } from "./types.js";
2121

22+
const require = createRequire(import.meta.url);
23+
2224
export type Language = "typescript" | "tsx" | "python" | "go" | "java" | "kotlin" | "php" | "r";
2325

2426
/**
@@ -81,6 +83,83 @@ export function languageLabelOf(path: string): string | null {
8183
return entryFor(path)?.label ?? null;
8284
}
8385

86+
export const NATIVE_FALLBACK_WARNING =
87+
"native parser unavailable — falling back to WASM (reduced binding fidelity)";
88+
89+
const DEPTH_LANGS: Language[] = ["typescript", "tsx", "python", "go", "java", "kotlin", "php", "r"];
90+
91+
function asParserCtor(mod: unknown): (new () => Parser) | null {
92+
if (typeof mod === "function") return mod as new () => Parser;
93+
if (mod && typeof mod === "object" && "default" in mod) {
94+
const d = (mod as { default: unknown }).default;
95+
if (typeof d === "function") return d as new () => Parser;
96+
}
97+
return null;
98+
}
99+
100+
export interface NativeLoadResult {
101+
Parser: (new () => Parser) | null;
102+
grammars: Partial<Record<Language, unknown>>;
103+
error: string | null;
104+
}
105+
106+
/** Load native grammars via `require`. Injectable so tests can simulate a missing
107+
* prebuild (`No native build was found for platform=linux arch=arm64`) without
108+
* unloading this process's real addons. Native available → same grammars as
109+
* the old static imports; a throw or a missing Parser ctor → empty grammars. */
110+
export function loadNativeGrammars(req: (id: string) => unknown = require): NativeLoadResult {
111+
try {
112+
const ParserCtor = asParserCtor(req("tree-sitter"));
113+
if (!ParserCtor) {
114+
return { Parser: null, grammars: {}, error: "tree-sitter did not export a Parser constructor" };
115+
}
116+
const grammars: Partial<Record<Language, unknown>> = {};
117+
const one = (id: string): unknown | null => {
118+
try {
119+
return req(id);
120+
} catch {
121+
return null;
122+
}
123+
};
124+
const ts = one("tree-sitter-typescript") as { typescript?: unknown; tsx?: unknown } | null;
125+
if (ts?.typescript) grammars.typescript = ts.typescript;
126+
if (ts?.tsx) grammars.tsx = ts.tsx;
127+
const py = one("tree-sitter-python");
128+
if (py) grammars.python = py;
129+
const go = one("tree-sitter-go");
130+
if (go) grammars.go = go;
131+
const r = one("tree-sitter-r");
132+
if (r) grammars.r = r;
133+
const java = one("tree-sitter-java");
134+
if (java) grammars.java = java;
135+
const kotlin = one("tree-sitter-kotlin");
136+
if (kotlin) grammars.kotlin = kotlin;
137+
const php = one("tree-sitter-php") as { php?: unknown } | null;
138+
if (php?.php) grammars.php = php.php;
139+
return { Parser: ParserCtor, grammars, error: null };
140+
} catch (err) {
141+
const message = err instanceof Error ? err.message : String(err);
142+
return { Parser: null, grammars: {}, error: message };
143+
}
144+
}
145+
146+
const native = loadNativeGrammars();
147+
const parser: Parser | null = native.Parser ? new native.Parser() : null;
148+
const GRAMMARS: Partial<Record<Language, unknown>> = native.grammars;
149+
150+
/** Depth-tier languages whose native grammar did not load — warm these as WASM. */
151+
export function wasmFallbackLangs(): Language[] {
152+
if (!parser) return [...DEPTH_LANGS];
153+
return DEPTH_LANGS.filter((l) => GRAMMARS[l] === undefined);
154+
}
155+
156+
/** One-line banner when the native runtime itself failed to load (issue #119).
157+
* Missing *individual* grammars still WASM-fall back, but that is not the
158+
* linux/arm64 crash and must not warn on darwin where tree-sitter loaded. */
159+
export function nativeFallbackWarning(): string | null {
160+
return parser ? null : NATIVE_FALLBACK_WARNING;
161+
}
162+
84163
/**
85164
* An edge whose target isn't resolved yet. build.ts turns these into EdgeV1 by
86165
* matching `name`/`specifier` against the repo-wide node index.
@@ -274,18 +353,6 @@ const FUNCTION_VALUE_TYPES = new Set([
274353

275354
const EMPTY_SET: ReadonlySet<string> = new Set();
276355

277-
const parser = new Parser();
278-
const GRAMMARS: Record<Language, unknown> = {
279-
typescript: TypeScript.typescript,
280-
tsx: TypeScript.tsx,
281-
python: Python,
282-
go: Go,
283-
r: R,
284-
java: Java,
285-
kotlin: Kotlin,
286-
php: PHP.php,
287-
};
288-
289356
export interface WalkCtx {
290357
rel: string;
291358
source: string;
@@ -344,10 +411,13 @@ interface DefDescriptor {
344411
* the source in <32 KB slices. Code-unit indexing matches `String.slice`. */
345412
const PARSE_CHUNK = 16384;
346413
function parseSource(source: string): Parser.SyntaxNode {
347-
return parser.parse((index: number) => source.slice(index, index + PARSE_CHUNK)).rootNode;
414+
return parser!.parse((index: number) => source.slice(index, index + PARSE_CHUNK)).rootNode;
348415
}
349416

350-
export function extractFile(rel: string, source: string, lang: Language): ExtractResult {
417+
export function extractFile(rel: string, source: string, lang: Language, opts?: { wasm?: boolean }): ExtractResult {
418+
if (opts?.wasm || !parser || GRAMMARS[lang] === undefined) {
419+
return extractGeneric(rel, source, lang);
420+
}
351421
parser.setLanguage(GRAMMARS[lang] as never);
352422
const root = parseSource(source);
353423
const bindings = collectBindings(root, lang);

‎src/graph/generic.ts‎

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,26 @@ export const GENERIC_LANGS: readonly GenericLang[] = [
6161
{ name: "lua", exts: [".lua"], wasm: "lua" },
6262
];
6363

64+
/**
65+
* WASM rows for depth-tier languages. Used only when the native grammar failed
66+
* to load (`extract.ts` wasmFallbackLangs). `genericLangOf` does NOT consult
67+
* this table — depth extensions stay owned by `languageOf` so the native path
68+
* is unchanged when addons exist. java is already in GENERIC_LANGS.
69+
*/
70+
export const DEPTH_WASM_LANGS: readonly GenericLang[] = [
71+
{ name: "typescript", exts: [".ts", ".mts", ".cts", ".js", ".mjs", ".cjs"], wasm: "typescript" },
72+
{ name: "tsx", exts: [".tsx", ".jsx"], wasm: "tsx" },
73+
{ name: "python", exts: [".py", ".pyi"], wasm: "python" },
74+
{ name: "go", exts: [".go"], wasm: "go" },
75+
{ name: "kotlin", exts: [".kt", ".kts"], wasm: "kotlin" },
76+
{ name: "php", exts: [".php"], wasm: "php" },
77+
{ name: "r", exts: [".r"], wasm: "r" },
78+
];
79+
80+
function wasmRow(name: string): GenericLang | undefined {
81+
return GENERIC_LANGS.find((l) => l.name === name) ?? DEPTH_WASM_LANGS.find((l) => l.name === name);
82+
}
83+
6484
const byExt = new Map<string, GenericLang>();
6585
for (const l of GENERIC_LANGS) for (const e of l.exts) byExt.set(e, l);
6686

@@ -123,7 +143,7 @@ function loadQuery(name: string): string | null {
123143
* grammars are silently skipped (their files then extract as file-only). */
124144
export async function warmGenericGrammars(langNames: Iterable<string>): Promise<void> {
125145
const want = new Set(langNames);
126-
const need = [...want].filter((n) => !loaded.has(n) && GENERIC_LANGS.some((l) => l.name === n));
146+
const need = [...want].filter((n) => !loaded.has(n) && wasmRow(n));
127147
if (need.length === 0) return;
128148
if (!tsMod) {
129149
tsMod = await import("web-tree-sitter");
@@ -132,7 +152,7 @@ export async function warmGenericGrammars(langNames: Iterable<string>): Promise<
132152
await initPromise;
133153
const { Language, Query } = tsMod;
134154
for (const name of need) {
135-
const row = GENERIC_LANGS.find((l) => l.name === name)!;
155+
const row = wasmRow(name)!;
136156
const bytes = requireWasm(row.wasm);
137157
if (!bytes) continue;
138158
try {

‎src/graph/queries/go.scm‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
; Depth-tier WASM fallback (#119).
2+
3+
(function_declaration
4+
name: (identifier) @name) @definition.function
5+
6+
(method_declaration
7+
name: (field_identifier) @name) @definition.method
8+
9+
(type_spec
10+
name: (type_identifier) @name
11+
type: (struct_type)) @definition.struct
12+
13+
(type_spec
14+
name: (type_identifier) @name
15+
type: (interface_type)) @definition.interface
16+
17+
(type_spec
18+
name: (type_identifier) @name) @definition.type
19+
20+
(call_expression
21+
function: [
22+
(identifier) @name
23+
(selector_expression
24+
field: (field_identifier) @name)
25+
]) @reference.call

‎src/graph/queries/python.scm‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
; Depth-tier WASM fallback (#119).
2+
3+
(function_definition
4+
name: (identifier) @name) @definition.function
5+
6+
(class_definition
7+
name: (identifier) @name) @definition.class
8+
9+
(call
10+
function: [
11+
(identifier) @name
12+
(attribute
13+
attribute: (identifier) @name)
14+
]) @reference.call

‎src/graph/queries/r.scm‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
; Depth-tier WASM fallback (#119).
2+
3+
(binary_operator
4+
lhs: (identifier) @name
5+
operator: "<-"
6+
rhs: (function_definition)
7+
) @definition.function
8+
9+
(binary_operator
10+
lhs: (identifier) @name
11+
operator: "="
12+
rhs: (function_definition)
13+
) @definition.function
14+
15+
(call
16+
function: (identifier) @name
17+
) @reference.call

0 commit comments

Comments
 (0)