Skip to content

Commit 23d25ab

Browse files
rustyconoverclaude
andcommitted
codegen: emit the protocol's Arrow schemas as Java, for vgi-java to check against
vgi-java writes every wire schema by deriving it from its record components, so within that SDK there is exactly one description of each schema and nothing can contradict it -- including when it is wrong. PlanResponse marked two components nullable that the protocol declares non-null; every test in vgi-java passed and the C++ client rejected the whole response as an "out-of-date Apache Arrow schema". Its conformance test could classify codec-serialised records but had nothing to compare them to. This gives it something: a generated VgiProtocolSchemas.java, emitted from the protocol the same way the C++/Go/Rust/TypeScript schemas are. Unlike those it is a test artifact -- vgi-java has no runtime use for it -- so it lands in that repo's test source set. Two differences from the other emitters, both deliberate: - The type mapping covers large_binary / large_utf8 / large_list. InitRequest carries large_binary and Java described it as binary; an emitter without the case would have hidden that instead of naming it. - collect_schemas() reaches responses and info types, not the REQUEST records that ride inside a params envelope as an opaque blob. Java mirrors those as records and is the side that reads them, so JAVA_EXTRA_TYPES adds them here rather than in the shared list -- widening that would churn four checked-in generated files for schemas no other client references. Regen command is `python -m vgi.codegen.java_schemas`, matching how these generators are actually invoked; the vgi-gen-* console scripts the older docstrings name have never existed. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
1 parent 8ee9e09 commit 23d25ab

2 files changed

Lines changed: 607 additions & 0 deletions

File tree

Lines changed: 232 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,232 @@
1+
# Copyright 2025, 2026 Query Farm LLC - https://query.farm
2+
3+
"""Drift + determinism tests for `vgi.codegen.java_schemas`.
4+
5+
Enforces that
6+
``vgi-java/vgi/src/test/java/farm/query/vgi/generated/VgiProtocolSchemas.java``
7+
matches what the generator would emit right now. When they fail, the error
8+
message prints the regeneration command.
9+
10+
The Java file is a TEST artifact, unlike the C++ / Go / Rust / TypeScript ones —
11+
vgi-java derives its wire schemas from its record declarations and has no
12+
runtime use for a generated schema. Its job is to be the second description
13+
those declarations are checked against, which only works while it stays in step
14+
with the protocol; hence this test.
15+
16+
Skipped when the `vgi-java` checkout isn't present. It lives at
17+
``~/vgi-java`` rather than beside the other SDKs, so the default search covers
18+
both locations and ``VGI_JAVA_GENERATED_JAVA`` overrides either.
19+
"""
20+
21+
from __future__ import annotations
22+
23+
import io
24+
import os
25+
import re
26+
from pathlib import Path
27+
from typing import Any, cast
28+
29+
import pyarrow as pa
30+
import pytest
31+
32+
from vgi.codegen._common import EXTRA_RESPONSE_TYPES, collect_schemas
33+
from vgi.codegen.java_schemas import JAVA_EXTRA_TYPES, emit
34+
35+
_RELATIVE = Path("vgi") / "src" / "test" / "java" / "farm" / "query" / "vgi" / "generated" / "VgiProtocolSchemas.java"
36+
37+
38+
def _vgi_java_generated_path() -> Path:
39+
override = os.environ.get("VGI_JAVA_GENERATED_JAVA")
40+
if override:
41+
return Path(override)
42+
candidates = [
43+
Path.home() / "vgi-java" / _RELATIVE,
44+
Path(__file__).resolve().parents[2] / "vgi-java" / _RELATIVE,
45+
]
46+
for candidate in candidates:
47+
if candidate.exists():
48+
return candidate
49+
return candidates[0]
50+
51+
52+
_REGEN_HINT = (
53+
"To regenerate, run:\n"
54+
" uv run --project ~/Development/vgi-python python -m vgi.codegen.java_schemas \\\n"
55+
" > ~/vgi-java/vgi/src/test/java/farm/query/vgi/generated/VgiProtocolSchemas.java"
56+
)
57+
58+
59+
def _expected() -> dict[str, pa.Schema]:
60+
return {
61+
es.name: es.schema
62+
for es in collect_schemas(extra_response_types=(*EXTRA_RESPONSE_TYPES, *JAVA_EXTRA_TYPES))
63+
}
64+
65+
66+
def test_generator_is_deterministic() -> None:
67+
"""Calling emit() twice produces byte-identical output."""
68+
out1 = io.StringIO()
69+
emit(out1)
70+
out2 = io.StringIO()
71+
emit(out2)
72+
assert out1.getvalue() == out2.getvalue(), "java_schemas generator is non-deterministic"
73+
74+
75+
# One `private static Schema xxx() { return new Schema(List.of(...)); }` method.
76+
_SCHEMA_RE = re.compile(
77+
r"private static Schema (\w+)\(\) \{\s*return new Schema\(List\.of\((.*?)\)\);\s*\}",
78+
re.DOTALL,
79+
)
80+
81+
# The name -> method map at the bottom, which is what Java actually reads.
82+
_ENTRY_RE = re.compile(r'Map\.entry\("([^"]+)", (\w+)\(\)\)')
83+
84+
85+
def _split_top_level_comma(expr: str) -> list[str]:
86+
parts: list[str] = []
87+
depth = 0
88+
start = 0
89+
for i, ch in enumerate(expr):
90+
if ch in "([{":
91+
depth += 1
92+
elif ch in ")]}":
93+
depth -= 1
94+
elif ch == "," and depth == 0:
95+
parts.append(expr[start:i])
96+
start = i + 1
97+
parts.append(expr[start:])
98+
return [p.strip() for p in parts if p.strip()]
99+
100+
101+
_SIMPLE_TYPE: dict[str, pa.DataType] = {
102+
"new ArrowType.Bool()": pa.bool_(),
103+
"new ArrowType.Int(8, true)": pa.int8(),
104+
"new ArrowType.Int(16, true)": pa.int16(),
105+
"new ArrowType.Int(32, true)": pa.int32(),
106+
"new ArrowType.Int(64, true)": pa.int64(),
107+
"new ArrowType.Int(8, false)": pa.uint8(),
108+
"new ArrowType.Int(16, false)": pa.uint16(),
109+
"new ArrowType.Int(32, false)": pa.uint32(),
110+
"new ArrowType.Int(64, false)": pa.uint64(),
111+
"new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)": pa.float32(),
112+
"new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)": pa.float64(),
113+
"new ArrowType.Utf8()": pa.string(),
114+
"new ArrowType.LargeUtf8()": pa.large_string(),
115+
"new ArrowType.Binary()": pa.binary(),
116+
"new ArrowType.LargeBinary()": pa.large_binary(),
117+
}
118+
119+
_TIMESTAMP_RE = re.compile(r"new ArrowType\.Timestamp\(TimeUnit\.(\w+), (null|\"[^\"]*\")\)")
120+
_UNITS = {"SECOND": "s", "MILLISECOND": "ms", "MICROSECOND": "us", "NANOSECOND": "ns"}
121+
122+
123+
def _parse_type(expr: str, children: list[pa.Field[Any]]) -> pa.DataType:
124+
expr = expr.strip()
125+
if expr in _SIMPLE_TYPE:
126+
return _SIMPLE_TYPE[expr]
127+
ts = _TIMESTAMP_RE.fullmatch(expr)
128+
if ts:
129+
tz = None if ts.group(2) == "null" else ts.group(2)[1:-1]
130+
return cast(pa.DataType, pa.timestamp(_UNITS[ts.group(1)], tz=tz))
131+
if expr == "new ArrowType.List()":
132+
return pa.list_(children[0])
133+
if expr == "new ArrowType.LargeList()":
134+
return pa.large_list(children[0])
135+
if expr == "new ArrowType.Struct()":
136+
return pa.struct(children)
137+
if expr == "new ArrowType.Map(false)":
138+
# Arrow-Java models a map as one `entries` struct child; pyarrow as
139+
# key/item. Re-fold so the comparison is against the pyarrow shape.
140+
entries = children[0]
141+
key = entries.type.field(0)
142+
value = entries.type.field(1)
143+
return cast(pa.DataType, pa.map_(key.type, value.type))
144+
raise AssertionError(f"cannot parse generated Java type expression: {expr!r}")
145+
146+
147+
_DICT_RE = re.compile(
148+
r"new DictionaryEncoding\(0L, (true|false), (new ArrowType\.Int\(\d+, \w+\))\)"
149+
)
150+
151+
152+
def _parse_field(expr: str) -> pa.Field[Any]:
153+
expr = expr.strip()
154+
is_dict = expr.startswith("dict(")
155+
prefix = "dict(" if is_dict else "f("
156+
if not (expr.startswith(prefix) and expr.endswith(")")):
157+
raise AssertionError(f"cannot parse generated Java field expression: {expr!r}")
158+
parts = _split_top_level_comma(expr[len(prefix) : -1])
159+
160+
name = parts[0].strip()[1:-1]
161+
nullable = parts[1].strip() == "true"
162+
type_expr = parts[2]
163+
if is_dict:
164+
enc = _DICT_RE.fullmatch(parts[3].strip())
165+
assert enc, f"cannot parse dictionary encoding: {parts[3]!r}"
166+
value_type = _parse_type(type_expr, [])
167+
index_type = _SIMPLE_TYPE[enc.group(2)]
168+
dtype: pa.DataType = cast(
169+
pa.DataType,
170+
pa.dictionary(index_type, value_type, ordered=enc.group(1) == "true"),
171+
)
172+
return pa.field(name, dtype, nullable=nullable)
173+
174+
children = [_parse_field(p) for p in parts[3:]]
175+
return pa.field(name, _parse_type(type_expr, children), nullable=nullable)
176+
177+
178+
def _parse_generated_java(text: str) -> dict[str, pa.Schema]:
179+
by_method: dict[str, pa.Schema] = {}
180+
for match in _SCHEMA_RE.finditer(text):
181+
body = match.group(2).strip()
182+
fields = [_parse_field(e) for e in _split_top_level_comma(body)] if body else []
183+
by_method[match.group(1)] = pa.schema(fields)
184+
185+
# An empty schema is emitted as `new Schema(List.of());`, which the regex
186+
# above does not match — pick those up so a schema losing every field reads
187+
# as an empty schema rather than as a schema that vanished.
188+
for empty in re.finditer(r"private static Schema (\w+)\(\) \{\s*return new Schema\(List\.of\(\)\);", text):
189+
by_method.setdefault(empty.group(1), pa.schema([]))
190+
191+
result: dict[str, pa.Schema] = {}
192+
for name, method in _ENTRY_RE.findall(text):
193+
assert method in by_method, f"map entry '{name}' names missing method {method}()"
194+
result[name] = by_method[method]
195+
return result
196+
197+
198+
def test_checked_in_generated_java_matches_generator() -> None:
199+
"""Drift check: the checked-in Java matches what the generator produces."""
200+
path = _vgi_java_generated_path()
201+
if not path.exists():
202+
pytest.skip(f"{path} not found; set VGI_JAVA_GENERATED_JAVA or check out vgi-java")
203+
204+
actual = _parse_generated_java(path.read_text())
205+
expected = _expected()
206+
207+
missing = set(expected) - set(actual)
208+
extra = set(actual) - set(expected)
209+
assert not missing, f"checked-in .java is missing schemas: {sorted(missing)}\n{_REGEN_HINT}"
210+
assert not extra, f"checked-in .java has stale schemas no longer in the Protocol: {sorted(extra)}\n{_REGEN_HINT}"
211+
212+
for name, expected_schema in expected.items():
213+
if not expected_schema.equals(actual[name], check_metadata=False):
214+
raise AssertionError(
215+
f"schema '{name}' in checked-in .java differs from generator output.\n"
216+
f" expected: {expected_schema}\n"
217+
f" actual: {actual[name]}\n"
218+
f"{_REGEN_HINT}",
219+
)
220+
221+
222+
def test_parser_roundtrip_self_test() -> None:
223+
"""Self-test: the local parser round-trips the generator's own output."""
224+
buf = io.StringIO()
225+
emit(buf)
226+
parsed = _parse_generated_java(buf.getvalue())
227+
expected = _expected()
228+
assert set(parsed) == set(expected), "parser missed a schema the generator emitted"
229+
for name, schema in expected.items():
230+
assert schema.equals(parsed[name], check_metadata=False), (
231+
f"parser round-trip broke schema '{name}': expected {schema}, got {parsed[name]}"
232+
)

0 commit comments

Comments
 (0)