Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 36 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -77,3 +77,39 @@ jobs:

- name: Exercise every breadth grammar
run: node --import tsx --test test/generic-node24.test.ts

windows-node26-install:
name: Windows Node 26 install regression
runs-on: windows-latest
steps:
- name: Checkout code
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false

- name: Setup Node.js
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: '26'
cache: npm

- name: Install dependencies
run: npm ci

- name: Build
run: npm run build

- name: Test
run: npm test

- name: Pack and install the published artifact shape
shell: pwsh
run: |
$tarball = (npm pack --json | ConvertFrom-Json)[0].filename
$consumer = Join-Path $env:RUNNER_TEMP 'graft-packed-install'
New-Item -ItemType Directory -Force -Path $consumer | Out-Null
Push-Location $consumer
npm init --yes
npm install (Join-Path $env:GITHUB_WORKSPACE $tarball)
npx graft --version
Pop-Location
7 changes: 6 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -486,7 +486,12 @@ compiler-grade layer, all `$0` and deterministic (no model, no key):
`library()`/`source()` imports), and **Vue** single-file components (their
`<script>` blocks go through the TypeScript extractor).

- **Broad**: symbols (functions, classes, methods, types, …) plus name-resolved
Kotlin's native parser is an optional install so a platform without a compatible
native build can still install and use Graft. If it is unavailable, a build that
encounters `.kt` or `.kts` files prints one warning and skips only those files;
every other supported language remains indexed.

- **Broad** — symbols (functions, classes, methods, types, …) plus name-resolved
call edges via a generic tree-sitter extractor, one grammar per language:
**Rust, C, C++, C#, Ruby, Scala, Elixir, Solidity,
Dart, Clojure, Nix, Lua**, and **OCaml, Zig** (symbols only for now, no call edges).
Expand Down
8 changes: 6 additions & 2 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 3 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,6 @@
"tree-sitter": "^0.21.1",
"tree-sitter-go": "^0.23.4",
"tree-sitter-java": "^0.23.5",
"tree-sitter-kotlin": "^0.3.8",
"tree-sitter-php": "^0.23.4",
"tree-sitter-python": "^0.21.0",
"tree-sitter-r": "npm:@davisvaughan/tree-sitter-r@^1.3.0",
Expand All @@ -74,6 +73,9 @@
"vscode-languageserver-protocol": "^3.18.2",
"web-tree-sitter": "^0.26.13"
},
"optionalDependencies": {
"tree-sitter-kotlin": "^0.3.8"
},
"devDependencies": {
"@types/d3-force": "^3.0.10",
"@types/node": "^26.1.0",
Expand Down
29 changes: 24 additions & 5 deletions src/graph/build.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,15 @@ import { readFileSync } from "node:fs";
import { basename, dirname, resolve } from "node:path";
import { walkDir } from "../ingest/fs.js";
import { contextDirFor, ensureGitignored, ensureSearchable } from "../context/node-file.js";
import { extractFile, languageLabelOf, languageOf, type Language, type RawEdge } from "./extract.js";
import { extractGeneric, fallbackLangOf, genericLangOf, warmGenericGrammars, type GenericLang } from "./generic.js";
import { containerLangOf, extractContainer, warmContainerGrammars, type ContainerLang } from "./container.js";
import { missingNativePackages, nativeReady, type NativeFallback } from "./native.js";
import {
extractFile,
kotlinParserUnavailableMessage,
languageLabelOf,
languageOf,
type RawEdge,
} from "./extract.js";
import { extractGeneric, genericLangOf, warmGenericGrammars } from "./generic.js";
import { containerLangOf, extractContainer, warmContainerGrammars } from "./container.js";
import { contentHash } from "../util/id.js";
import { relPosix } from "../util/paths.js";
import { readSourceFile } from "../util/source.js";
Expand Down Expand Up @@ -226,6 +231,13 @@ export async function buildGraph(
* javascript, or the banner claims a repo's JavaScript went unindexed. */
const langs = new Set<string>();
const errors: string[] = [];
// Kotlin is the only optional depth-tier grammar. Probe it once, and only when
// the repository actually contains Kotlin, so a failed native install cannot
// affect a TypeScript/JavaScript-only build or produce startup noise.
const kotlinUnavailable = files.some((f) => languageOf(f.abs) === "kotlin")
? kotlinParserUnavailableMessage()
: null;
if (kotlinUnavailable) errors.push(kotlinUnavailable);

// In a git worktree there is nothing to reuse *yet* — `graft/` is gitignored, so
// git never checked it out — but the parent checkout's graph is one directory away.
Expand Down Expand Up @@ -303,7 +315,14 @@ export async function buildGraph(
}

const hash = contentHash(source);
if (cached && hash === cached.hash && cached.via === via) {
if (lang === "kotlin" && kotlinUnavailable) {
// Keep the source fingerprint current, but don't cache this as a parse
// failure: a later install may make the optional addon available.
entries[rel] = { size: f.size, mtimeMs: f.mtimeMs, hash, nodes: [], rawEdges: [], skipped: true };
return;
}

if (cached && hash === cached.hash && !cached.skipped) {
entries[rel] = { ...cached, size: f.size, mtimeMs: f.mtimeMs };
sources.set(rel, source);
reused++;
Expand Down
29 changes: 21 additions & 8 deletions src/graph/check.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,10 @@
import { resolve } from "node:path";
import { relPosix } from "../util/paths.js";
import { contextDirFor } from "../context/node-file.js";
import { extractFile } from "./extract.js";
import { extractGeneric } from "./generic.js";
import { extractContainer } from "./container.js";
import { listSourceFiles, tierOf, warmGrammarsFor } from "./build.js";
import { extractFile, kotlinParserUnavailableMessage, languageOf } from "./extract.js";
import { extractGeneric, genericLangOf, warmGenericGrammars } from "./generic.js";
import { containerLangOf, extractContainer, warmContainerGrammars } from "./container.js";
import { listSourceFiles } from "./build.js";
import { readGraph, wiringPath } from "./write.js";
import { readFingerprint } from "./fingerprint.js";
import { readSourceFile } from "../util/source.js";
Expand Down Expand Up @@ -88,6 +88,12 @@ export async function checkGraph(
const fpOnlyDirs = readFingerprint(outDir)?.onlyDirs;
const onlyDirs = fpOnlyDirs && fpOnlyDirs.length > 0 ? new Set(fpOnlyDirs) : undefined;
const sourceFiles = listSourceFiles(root, outDir, undefined, onlyDirs);
const kotlinUnavailable = sourceFiles.some((file) => languageOf(file) === "kotlin")
? kotlinParserUnavailableMessage()
: null;
await warmGenericGrammars(
new Set(sourceFiles.map((f) => genericLangOf(f)?.name).filter((n): n is string => !!n)),
);
// Container-tier grammars need the same warmup as the generic ones, for the same
// reason: extraction below is synchronous. Missing that is what made `graft
// check` report every `.vue` node as `removed` right after a clean build (#236)
Expand All @@ -97,10 +103,14 @@ export async function checkGraph(
await warmGrammarsFor(sourceFiles);
const current = new Map<string, string>(); // id → body_hash
for (const file of sourceFiles) {
// The same dispatch `buildGraph` uses (`tierOf`). The two must stay in step: a
// tier the build extracts and the check cannot see reports as `removed`
// forever, and the `graft build` the check tells you to run can never repair it.
const { lang, container, generic } = tierOf(file);
// The same three-way branch `buildGraph` uses, in the same order. The two must
// stay in step: a tier the build extracts and the check cannot see reports as
// `removed` forever, and the `graft build` the check tells you to run can never
// repair it.
const lang = languageOf(file);
if (lang === "kotlin" && kotlinUnavailable) continue;
const container = lang ? null : containerLangOf(file);
const generic = lang || container ? null : genericLangOf(file);
let source: string | null;
try {
source = readSourceFile(file);
Expand Down Expand Up @@ -133,6 +143,9 @@ export async function checkGraph(
const committedById = new Map(committed.nodes.map((n) => [n.id, n]));
result.nodes = committedById.size;
for (const [id, node] of committedById) {
// Do not call a graph stale merely because an optional parser isn't installed
// in this environment. The next build will give the actionable diagnostic.
if (kotlinUnavailable && languageOf(node.path) === "kotlin") continue;
const now = current.get(id);
if (now === undefined) result.removed.push(id);
else if (now !== node.body_hash) result.changed.push(id);
Expand Down
7 changes: 3 additions & 4 deletions src/graph/extract-cache.ts
Original file line number Diff line number Diff line change
Expand Up @@ -48,10 +48,9 @@ export interface ExtractEntry {
* re-reports the same error and contributes no nodes, exactly as a cold build
* would. */
error?: string;
/** The breadth-tier grammar that parsed this file standing in for its depth-tier
* one, whose native grammar didn't load. Absent for every other file. The entry is
* replayed only while the same stand-in would parse it again. */
via?: string;
/** The optional parser for this file's language was unavailable. Unlike `error`,
* this entry is re-parsed if that parser becomes available on a later run. */
skipped?: true;
}

export interface ExtractCache {
Expand Down
70 changes: 66 additions & 4 deletions src/graph/extract.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,15 @@
* arrow-function consts) plus unresolved edge intents. Edge *targets* are
* resolved against the whole-repo node index later, in build.ts.
*/
// Types only: the native grammars themselves load on first use, in native.ts, so
// a missing binary costs its own language instead of every command.
import type Parser from "tree-sitter";
import Parser from "tree-sitter";
import { createRequire } from "node:module";
import TypeScript from "tree-sitter-typescript";
import Python from "tree-sitter-python";
import Go from "tree-sitter-go";
import R from "tree-sitter-r";
import Java from "tree-sitter-java";
import Swift from "tree-sitter-swift";
import PHP from "tree-sitter-php";
import { basename } from "node:path";
import { contentHash } from "../util/id.js";
import { collectBindings, goReceiverVarOf, resolveRecvType, type FileBindings } from "./bindings.js";
Expand Down Expand Up @@ -305,6 +311,61 @@ const FUNCTION_VALUE_TYPES = new Set([

const EMPTY_SET: ReadonlySet<string> = new Set();

const parser = new Parser();
const require = createRequire(import.meta.url);

const GRAMMARS: Record<Exclude<Language, "kotlin">, unknown> = {
typescript: TypeScript.typescript,
tsx: TypeScript.tsx,
python: Python,
go: Go,
r: R,
java: Java,
swift: Swift,
php: PHP.php,
};

const KOTLIN_PARSER_UNAVAILABLE =
"Kotlin files were skipped because the optional tree-sitter-kotlin parser is unavailable. " +
"Reinstall @nanonets/graft in an environment where tree-sitter-kotlin can build to index .kt and .kts files.";

let kotlinGrammar: unknown | null | undefined;
let loadKotlinGrammar = (): unknown => require("tree-sitter-kotlin");

/**
* Resolve the optional Kotlin native addon only when a Kotlin file is encountered.
* A missing or unloadable addon must never prevent another language from building.
*/
export function kotlinParserUnavailableMessage(): string | null {
if (kotlinGrammar === undefined) {
try {
kotlinGrammar = loadKotlinGrammar();
} catch {
kotlinGrammar = null;
}
}
return kotlinGrammar === null ? KOTLIN_PARSER_UNAVAILABLE : null;
}

/** Test seam for the optional native addon. Not part of Graft's package API. */
export function swapKotlinGrammarLoaderForTest(loader: () => unknown): () => void {
const previousLoader = loadKotlinGrammar;
const previousGrammar = kotlinGrammar;
loadKotlinGrammar = loader;
kotlinGrammar = undefined;
return () => {
loadKotlinGrammar = previousLoader;
kotlinGrammar = previousGrammar;
};
}

function grammarFor(lang: Language): unknown {
if (lang !== "kotlin") return GRAMMARS[lang];
const unavailable = kotlinParserUnavailableMessage();
if (unavailable) throw new Error(unavailable);
return kotlinGrammar;
}

export interface WalkCtx {
rel: string;
source: string;
Expand Down Expand Up @@ -369,7 +430,8 @@ function parseSource(parser: Parser, source: string): Parser.SyntaxNode {
/** Throws when `lang`'s native grammar is unavailable: callers check `nativeReady`
* (native.ts) first and hand such a file to its breadth-tier stand-in instead. */
export function extractFile(rel: string, source: string, lang: Language): ExtractResult {
const root = parseSource(nativeParser(lang), source);
parser.setLanguage(grammarFor(lang) as never);
const root = parseSource(source);
const bindings = collectBindings(root, lang);
const importedSymbols = collectImportedSymbols(root, lang);
const rGenerics = lang === "r" ? collectRGenerics(root) : EMPTY_SET;
Expand Down
47 changes: 45 additions & 2 deletions test/graph-languages.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,12 @@ import { tmpdir } from "node:os";
import { join } from "node:path";
import { buildGraph } from "../src/graph/build.js";
import { buildRepoMap } from "../src/graph/map.js";
import { languageLabelOf, languageOf } from "../src/graph/extract.js";
import {
kotlinParserUnavailableMessage,
languageLabelOf,
languageOf,
swapKotlinGrammarLoaderForTest,
} from "../src/graph/extract.js";
import { readGraph, wiringPath } from "../src/graph/write.js";

/** Every extension graft claims to index, one per grammar/label pairing. */
Expand Down Expand Up @@ -92,7 +97,11 @@ test("the build banner and repo map report every language they indexed", async (
writeFileSync(join(d, "src", "a.swift"), "func swiftOnly() -> Int {\n return 1\n}\n");

const r = await buildGraph(d);
assert.deepEqual(r.languages, ["javascript", "jsx", "kotlin", "python", "swift", "tsx", "typescript"]);
const kotlinAvailable = !r.errors.some((error) => error.startsWith("Kotlin files were skipped"));
assert.deepEqual(
r.languages,
["javascript", "jsx", ...(kotlinAvailable ? ["kotlin"] : []), "python", "swift", "tsx", "typescript"],
);

// The reported symbol was queryable all along — that mismatch is what the issue was
// about, so pin both halves together.
Expand All @@ -105,3 +114,37 @@ test("the build banner and repo map report every language they indexed", async (
// `map` derives labels from paths independently, so it gets its own assertion.
assert.deepEqual(buildRepoMap(graph!).totals.languages, r.languages);
});

test("Kotlin is indexed when its optional parser is available", async (t) => {
if (kotlinParserUnavailableMessage()) {
t.skip("tree-sitter-kotlin is unavailable in this runtime");
return;
}
const dir = mkdtempSync(join(tmpdir(), "graft-kotlin-available-"));
writeFileSync(join(dir, "app.kt"), "fun indexedKotlin(): Int = 1\n");
const result = await buildGraph(dir, { reuse: false });
assert.deepEqual(result.errors, []);
assert.deepEqual(result.languages, ["kotlin"]);
const graph = readGraph(wiringPath(result.contextDir))!;
assert.ok(graph.nodes.some((node) => node.name === "indexedKotlin"), "Kotlin function is indexed");
});

test("a missing Kotlin parser skips Kotlin only and leaves TypeScript indexed", async () => {
const dir = mkdtempSync(join(tmpdir(), "graft-kotlin-optional-"));
writeFileSync(join(dir, "app.ts"), "export function stillIndexed() { return 1; }\n");
writeFileSync(join(dir, "app.kt"), "fun skippedKotlin(): Int = 1\n");
const restore = swapKotlinGrammarLoaderForTest(() => {
throw new Error("Cannot find module 'tree-sitter-kotlin'");
});
try {
const result = await buildGraph(dir, { reuse: false });
assert.equal(result.errors.length, 1, "one Kotlin availability warning");
assert.match(result.errors[0], /Kotlin files were skipped/);
assert.deepEqual(result.languages, ["typescript"]);
const graph = readGraph(wiringPath(result.contextDir))!;
assert.ok(graph.nodes.some((node) => node.name === "stillIndexed"), "TypeScript remains indexed");
assert.ok(!graph.nodes.some((node) => node.path === "app.kt"), "Kotlin contributes no partial nodes");
} finally {
restore();
}
});
Loading