Skip to content

Commit a8794e4

Browse files
Rusty Conoverclaude
andcommitted
codegen: generate Java wire records for vgi-java (java_types)
vgi-java derives every wire schema from a Java record by reflection (vgi-rpc-java's SchemaDerivation), so a hand-declared record is a second description of the protocol that drifts silently. vgi.codegen.java_types emits records from each protocol class's ARROW_SCHEMA instead, in vgi-java's existing idiom (record implements ArrowSerializableRecord, @nullable, @Arrowfield overrides, List<byte[]>, Map<String, String>), one file per record in farm.query.vgi.protocol so the public names are unchanged. Data-driven: JAVA_RECORDS lists the generated classes (CatalogAttachResult, SchemaContents, CatalogContentsResponse). Arrow shapes the derivation cannot reproduce exactly raise GeneratorError rather than emit a lookalike. scripts/regen_generated.py derives one target per record from the generator and accepts `module:record` targets. tests/test_generated_java_types.py checks determinism, drift against the vgi-java checkout (VGI_JAVA_ROOT overrides), and that the generated declarations derive back to ARROW_SCHEMA under SchemaDerivation's rules. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
1 parent 3db2d37 commit a8794e4

3 files changed

Lines changed: 503 additions & 2 deletions

File tree

‎scripts/regen_generated.py‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,15 @@
7070
),
7171
]
7272

73+
# vgi-java's generated wire records: one file per record (Java allows one public
74+
# top-level type per file), so one target each. ``module:record`` tells
75+
# `_render` to call ``emit(out, record=...)``. Derived from the generator's own
76+
# list so adding a record there is all it takes.
77+
_TARGETS += [
78+
(f"vgi.codegen.java_types:{record}", relative, "vgi-java")
79+
for record, relative in importlib.import_module("vgi.codegen.java_types").targets()
80+
]
81+
7382

7483
def _repo_root(name: str) -> Path | None:
7584
"""Locate a sibling checkout, or ``None`` when it is not present."""
@@ -81,10 +90,18 @@ def _repo_root(name: str) -> Path | None:
8190

8291

8392
def _render(module_name: str) -> str:
84-
"""Render one artifact into memory. Raises if the generator fails."""
93+
"""Render one artifact into memory. Raises if the generator fails.
94+
95+
``module:record`` renders one file of a generator that emits several
96+
(``emit(out, record=record)``).
97+
"""
98+
module_name, _, record = module_name.partition(":")
8599
module = importlib.import_module(module_name)
86100
buf = io.StringIO()
87-
module.emit(buf)
101+
if record:
102+
module.emit(buf, record=record)
103+
else:
104+
module.emit(buf)
88105
text = buf.getvalue()
89106
if not text.strip():
90107
raise RuntimeError(f"{module_name}.emit() produced no output")

‎tests/test_generated_java_types.py‎

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
# Copyright 2025, 2026 Query Farm LLC - https://query.farm
2+
3+
"""Drift + determinism tests for `vgi.codegen.java_types`.
4+
5+
vgi-java checks in one generated record per entry of
6+
``java_types.JAVA_RECORDS`` under ``vgi/src/main/java/farm/query/vgi/protocol/``;
7+
each must match what the generator emits right now. Like the C# types, the
8+
comparison is textual: the file is what vgi-java compiles, and vgi-rpc-java
9+
derives the wire schema from it.
10+
11+
A second test closes the loop the textual one cannot: it reads the generated
12+
component declarations back into an Arrow schema using vgi-rpc-java's
13+
``SchemaDerivation`` rules and compares that with the record's ``ARROW_SCHEMA``,
14+
so a mapping bug in the generator fails here rather than at the C++ client.
15+
16+
Skipped when the `vgi-java` checkout isn't present. It lives at ``~/vgi-java``
17+
for some checkouts and beside the other SDKs for others; ``VGI_JAVA_ROOT``
18+
overrides both.
19+
"""
20+
21+
from __future__ import annotations
22+
23+
import os
24+
import re
25+
from pathlib import Path
26+
from typing import Any
27+
28+
import pyarrow as pa
29+
import pytest
30+
31+
from vgi.codegen import java_types
32+
from vgi.codegen._common import GeneratorError
33+
34+
_REGEN_HINT = (
35+
"To regenerate, run:\n"
36+
" uv run --project ~/Development/vgi-python python scripts/regen_generated.py\n"
37+
"\n"
38+
"Do not redirect a generator into its destination with `>`: the shell\n"
39+
"truncates the file before the generator runs, so any failure destroys\n"
40+
"the checked-in artifact. The script renders to memory first."
41+
)
42+
43+
44+
def _vgi_java_root() -> Path:
45+
override = os.environ.get("VGI_JAVA_ROOT")
46+
if override:
47+
return Path(override)
48+
# Same order as scripts/regen_generated.py: beside vgi-python first, then ~.
49+
candidates = [Path(__file__).resolve().parents[2] / "vgi-java", Path.home() / "vgi-java"]
50+
for candidate in candidates:
51+
if (candidate / java_types.JAVA_PROTOCOL_DIR).is_dir():
52+
return candidate
53+
return candidates[0]
54+
55+
56+
_RECORDS = java_types.record_names()
57+
58+
59+
@pytest.mark.parametrize("record", _RECORDS)
60+
def test_generator_is_deterministic(record: str) -> None:
61+
"""Rendering a record twice produces byte-identical output."""
62+
assert java_types.render(record) == java_types.render(record)
63+
64+
65+
@pytest.mark.parametrize(("record", "relative"), java_types.targets())
66+
def test_checked_in_java_matches_generator(record: str, relative: str) -> None:
67+
"""Drift check: the checked-in Java record matches what the generator produces."""
68+
path = _vgi_java_root() / relative
69+
if not path.parent.is_dir():
70+
pytest.skip(f"{path.parent} not found; set VGI_JAVA_ROOT or check out vgi-java")
71+
assert path.exists(), f"{path} is missing.\n{_REGEN_HINT}"
72+
assert path.read_text() == java_types.render(record), f"{path} is stale.\n{_REGEN_HINT}"
73+
74+
75+
# ---------------------------------------------------------------------------
76+
# Generated declaration -> Arrow schema, by vgi-rpc-java's SchemaDerivation rules
77+
# ---------------------------------------------------------------------------
78+
79+
_INFERRED: dict[str, pa.DataType] = {
80+
"boolean": pa.bool_(),
81+
"Boolean": pa.bool_(),
82+
"long": pa.int64(),
83+
"Long": pa.int64(),
84+
"double": pa.float64(),
85+
"Double": pa.float64(),
86+
"String": pa.string(),
87+
"byte[]": pa.binary(),
88+
}
89+
90+
_OVERRIDES: dict[str, pa.DataType] = {
91+
"DICT_INT16_UTF8": pa.dictionary(pa.int16(), pa.string()),
92+
"LARGE_BINARY": pa.large_binary(),
93+
}
94+
95+
_COMPONENT_RE = re.compile(
96+
r"^\s+((?:@\w+(?:\([^)]*\))? )*)([\w\[\]<>, ]+?) (\w+)(?:,|\) implements ArrowSerializableRecord \{)$"
97+
)
98+
99+
100+
def _derive(java_type: str, annotations: str) -> pa.DataType:
101+
override = re.search(r"@ArrowField\(ArrowFieldType\.(\w+)\)", annotations)
102+
if override:
103+
return _OVERRIDES[override.group(1)]
104+
if java_type.startswith("List<"):
105+
return pa.list_(pa.field("item", _INFERRED[java_type[5:-1]], nullable=True))
106+
if java_type.startswith("Map<"):
107+
key, value = (t.strip() for t in java_type[4:-1].split(","))
108+
return pa.map_(_INFERRED[key], _INFERRED[value])
109+
return _INFERRED[java_type]
110+
111+
112+
def _derived_schema(text: str) -> pa.Schema:
113+
fields: list[pa.Field[Any]] = []
114+
in_record = False
115+
for line in text.splitlines():
116+
if line.startswith("public record "):
117+
in_record = True
118+
continue
119+
if not in_record:
120+
continue
121+
match = _COMPONENT_RE.match(line)
122+
if match is None:
123+
break
124+
annotations, java_type, name = match.groups()
125+
fields.append(pa.field(name, _derive(java_type, annotations), nullable="@Nullable" in annotations))
126+
return pa.schema(fields)
127+
128+
129+
@pytest.mark.parametrize("record", _RECORDS)
130+
def test_generated_declaration_derives_the_protocol_schema(record: str) -> None:
131+
"""vgi-rpc-java's derivation of the generated record is the protocol's ARROW_SCHEMA."""
132+
cls = next(c for c in java_types.JAVA_RECORDS if java_types.java_name(c) == record)
133+
derived = _derived_schema(java_types.render(record))
134+
expected: pa.Schema = cls.ARROW_SCHEMA # type: ignore[attr-defined]
135+
assert derived.equals(expected, check_metadata=False), f"{record}:\n derived {derived}\n expected {expected}"
136+
137+
138+
def test_unsupported_arrow_type_is_refused() -> None:
139+
"""A shape vgi-rpc-java cannot derive raises instead of emitting a lookalike."""
140+
with pytest.raises(GeneratorError):
141+
java_types._component(pa.field("t", pa.timestamp("us")), "X")
142+
with pytest.raises(GeneratorError):
143+
java_types._component(pa.field("l", pa.list_(pa.field("item", pa.binary(), nullable=False))), "X")

0 commit comments

Comments
 (0)