|
| 1 | +# Copyright 2025, 2026 Query Farm LLC - https://query.farm |
| 2 | + |
| 3 | +"""Drift + determinism tests for `vgi.codegen.java_schemas`. |
| 4 | +
|
| 5 | +Enforces that |
| 6 | +``vgi-java/vgi/src/test/java/farm/query/vgi/generated/VgiProtocolSchemas.java`` |
| 7 | +matches what the generator would emit right now. When they fail, the error |
| 8 | +message prints the regeneration command. |
| 9 | +
|
| 10 | +The Java file is a TEST artifact, unlike the C++ / Go / Rust / TypeScript ones — |
| 11 | +vgi-java derives its wire schemas from its record declarations and has no |
| 12 | +runtime use for a generated schema. Its job is to be the second description |
| 13 | +those declarations are checked against, which only works while it stays in step |
| 14 | +with the protocol; hence this test. |
| 15 | +
|
| 16 | +Skipped when the `vgi-java` checkout isn't present. It lives at |
| 17 | +``~/vgi-java`` rather than beside the other SDKs, so the default search covers |
| 18 | +both locations and ``VGI_JAVA_GENERATED_JAVA`` overrides either. |
| 19 | +""" |
| 20 | + |
| 21 | +from __future__ import annotations |
| 22 | + |
| 23 | +import io |
| 24 | +import os |
| 25 | +import re |
| 26 | +from pathlib import Path |
| 27 | +from typing import Any, cast |
| 28 | + |
| 29 | +import pyarrow as pa |
| 30 | +import pytest |
| 31 | + |
| 32 | +from vgi.codegen._common import EXTRA_RESPONSE_TYPES, collect_schemas |
| 33 | +from vgi.codegen.java_schemas import JAVA_EXTRA_TYPES, emit |
| 34 | + |
| 35 | +_RELATIVE = Path("vgi") / "src" / "test" / "java" / "farm" / "query" / "vgi" / "generated" / "VgiProtocolSchemas.java" |
| 36 | + |
| 37 | + |
| 38 | +def _vgi_java_generated_path() -> Path: |
| 39 | + override = os.environ.get("VGI_JAVA_GENERATED_JAVA") |
| 40 | + if override: |
| 41 | + return Path(override) |
| 42 | + candidates = [ |
| 43 | + Path.home() / "vgi-java" / _RELATIVE, |
| 44 | + Path(__file__).resolve().parents[2] / "vgi-java" / _RELATIVE, |
| 45 | + ] |
| 46 | + for candidate in candidates: |
| 47 | + if candidate.exists(): |
| 48 | + return candidate |
| 49 | + return candidates[0] |
| 50 | + |
| 51 | + |
| 52 | +_REGEN_HINT = ( |
| 53 | + "To regenerate, run:\n" |
| 54 | + " uv run --project ~/Development/vgi-python python -m vgi.codegen.java_schemas \\\n" |
| 55 | + " > ~/vgi-java/vgi/src/test/java/farm/query/vgi/generated/VgiProtocolSchemas.java" |
| 56 | +) |
| 57 | + |
| 58 | + |
| 59 | +def _expected() -> dict[str, pa.Schema]: |
| 60 | + return { |
| 61 | + es.name: es.schema |
| 62 | + for es in collect_schemas(extra_response_types=(*EXTRA_RESPONSE_TYPES, *JAVA_EXTRA_TYPES)) |
| 63 | + } |
| 64 | + |
| 65 | + |
| 66 | +def test_generator_is_deterministic() -> None: |
| 67 | + """Calling emit() twice produces byte-identical output.""" |
| 68 | + out1 = io.StringIO() |
| 69 | + emit(out1) |
| 70 | + out2 = io.StringIO() |
| 71 | + emit(out2) |
| 72 | + assert out1.getvalue() == out2.getvalue(), "java_schemas generator is non-deterministic" |
| 73 | + |
| 74 | + |
| 75 | +# One `private static Schema xxx() { return new Schema(List.of(...)); }` method. |
| 76 | +_SCHEMA_RE = re.compile( |
| 77 | + r"private static Schema (\w+)\(\) \{\s*return new Schema\(List\.of\((.*?)\)\);\s*\}", |
| 78 | + re.DOTALL, |
| 79 | +) |
| 80 | + |
| 81 | +# The name -> method map at the bottom, which is what Java actually reads. |
| 82 | +_ENTRY_RE = re.compile(r'Map\.entry\("([^"]+)", (\w+)\(\)\)') |
| 83 | + |
| 84 | + |
| 85 | +def _split_top_level_comma(expr: str) -> list[str]: |
| 86 | + parts: list[str] = [] |
| 87 | + depth = 0 |
| 88 | + start = 0 |
| 89 | + for i, ch in enumerate(expr): |
| 90 | + if ch in "([{": |
| 91 | + depth += 1 |
| 92 | + elif ch in ")]}": |
| 93 | + depth -= 1 |
| 94 | + elif ch == "," and depth == 0: |
| 95 | + parts.append(expr[start:i]) |
| 96 | + start = i + 1 |
| 97 | + parts.append(expr[start:]) |
| 98 | + return [p.strip() for p in parts if p.strip()] |
| 99 | + |
| 100 | + |
| 101 | +_SIMPLE_TYPE: dict[str, pa.DataType] = { |
| 102 | + "new ArrowType.Bool()": pa.bool_(), |
| 103 | + "new ArrowType.Int(8, true)": pa.int8(), |
| 104 | + "new ArrowType.Int(16, true)": pa.int16(), |
| 105 | + "new ArrowType.Int(32, true)": pa.int32(), |
| 106 | + "new ArrowType.Int(64, true)": pa.int64(), |
| 107 | + "new ArrowType.Int(8, false)": pa.uint8(), |
| 108 | + "new ArrowType.Int(16, false)": pa.uint16(), |
| 109 | + "new ArrowType.Int(32, false)": pa.uint32(), |
| 110 | + "new ArrowType.Int(64, false)": pa.uint64(), |
| 111 | + "new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)": pa.float32(), |
| 112 | + "new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)": pa.float64(), |
| 113 | + "new ArrowType.Utf8()": pa.string(), |
| 114 | + "new ArrowType.LargeUtf8()": pa.large_string(), |
| 115 | + "new ArrowType.Binary()": pa.binary(), |
| 116 | + "new ArrowType.LargeBinary()": pa.large_binary(), |
| 117 | +} |
| 118 | + |
| 119 | +_TIMESTAMP_RE = re.compile(r"new ArrowType\.Timestamp\(TimeUnit\.(\w+), (null|\"[^\"]*\")\)") |
| 120 | +_UNITS = {"SECOND": "s", "MILLISECOND": "ms", "MICROSECOND": "us", "NANOSECOND": "ns"} |
| 121 | + |
| 122 | + |
| 123 | +def _parse_type(expr: str, children: list[pa.Field[Any]]) -> pa.DataType: |
| 124 | + expr = expr.strip() |
| 125 | + if expr in _SIMPLE_TYPE: |
| 126 | + return _SIMPLE_TYPE[expr] |
| 127 | + ts = _TIMESTAMP_RE.fullmatch(expr) |
| 128 | + if ts: |
| 129 | + tz = None if ts.group(2) == "null" else ts.group(2)[1:-1] |
| 130 | + return cast(pa.DataType, pa.timestamp(_UNITS[ts.group(1)], tz=tz)) |
| 131 | + if expr == "new ArrowType.List()": |
| 132 | + return pa.list_(children[0]) |
| 133 | + if expr == "new ArrowType.LargeList()": |
| 134 | + return pa.large_list(children[0]) |
| 135 | + if expr == "new ArrowType.Struct()": |
| 136 | + return pa.struct(children) |
| 137 | + if expr == "new ArrowType.Map(false)": |
| 138 | + # Arrow-Java models a map as one `entries` struct child; pyarrow as |
| 139 | + # key/item. Re-fold so the comparison is against the pyarrow shape. |
| 140 | + entries = children[0] |
| 141 | + key = entries.type.field(0) |
| 142 | + value = entries.type.field(1) |
| 143 | + return cast(pa.DataType, pa.map_(key.type, value.type)) |
| 144 | + raise AssertionError(f"cannot parse generated Java type expression: {expr!r}") |
| 145 | + |
| 146 | + |
| 147 | +_DICT_RE = re.compile( |
| 148 | + r"new DictionaryEncoding\(0L, (true|false), (new ArrowType\.Int\(\d+, \w+\))\)" |
| 149 | +) |
| 150 | + |
| 151 | + |
| 152 | +def _parse_field(expr: str) -> pa.Field[Any]: |
| 153 | + expr = expr.strip() |
| 154 | + is_dict = expr.startswith("dict(") |
| 155 | + prefix = "dict(" if is_dict else "f(" |
| 156 | + if not (expr.startswith(prefix) and expr.endswith(")")): |
| 157 | + raise AssertionError(f"cannot parse generated Java field expression: {expr!r}") |
| 158 | + parts = _split_top_level_comma(expr[len(prefix) : -1]) |
| 159 | + |
| 160 | + name = parts[0].strip()[1:-1] |
| 161 | + nullable = parts[1].strip() == "true" |
| 162 | + type_expr = parts[2] |
| 163 | + if is_dict: |
| 164 | + enc = _DICT_RE.fullmatch(parts[3].strip()) |
| 165 | + assert enc, f"cannot parse dictionary encoding: {parts[3]!r}" |
| 166 | + value_type = _parse_type(type_expr, []) |
| 167 | + index_type = _SIMPLE_TYPE[enc.group(2)] |
| 168 | + dtype: pa.DataType = cast( |
| 169 | + pa.DataType, |
| 170 | + pa.dictionary(index_type, value_type, ordered=enc.group(1) == "true"), |
| 171 | + ) |
| 172 | + return pa.field(name, dtype, nullable=nullable) |
| 173 | + |
| 174 | + children = [_parse_field(p) for p in parts[3:]] |
| 175 | + return pa.field(name, _parse_type(type_expr, children), nullable=nullable) |
| 176 | + |
| 177 | + |
| 178 | +def _parse_generated_java(text: str) -> dict[str, pa.Schema]: |
| 179 | + by_method: dict[str, pa.Schema] = {} |
| 180 | + for match in _SCHEMA_RE.finditer(text): |
| 181 | + body = match.group(2).strip() |
| 182 | + fields = [_parse_field(e) for e in _split_top_level_comma(body)] if body else [] |
| 183 | + by_method[match.group(1)] = pa.schema(fields) |
| 184 | + |
| 185 | + # An empty schema is emitted as `new Schema(List.of());`, which the regex |
| 186 | + # above does not match — pick those up so a schema losing every field reads |
| 187 | + # as an empty schema rather than as a schema that vanished. |
| 188 | + for empty in re.finditer(r"private static Schema (\w+)\(\) \{\s*return new Schema\(List\.of\(\)\);", text): |
| 189 | + by_method.setdefault(empty.group(1), pa.schema([])) |
| 190 | + |
| 191 | + result: dict[str, pa.Schema] = {} |
| 192 | + for name, method in _ENTRY_RE.findall(text): |
| 193 | + assert method in by_method, f"map entry '{name}' names missing method {method}()" |
| 194 | + result[name] = by_method[method] |
| 195 | + return result |
| 196 | + |
| 197 | + |
| 198 | +def test_checked_in_generated_java_matches_generator() -> None: |
| 199 | + """Drift check: the checked-in Java matches what the generator produces.""" |
| 200 | + path = _vgi_java_generated_path() |
| 201 | + if not path.exists(): |
| 202 | + pytest.skip(f"{path} not found; set VGI_JAVA_GENERATED_JAVA or check out vgi-java") |
| 203 | + |
| 204 | + actual = _parse_generated_java(path.read_text()) |
| 205 | + expected = _expected() |
| 206 | + |
| 207 | + missing = set(expected) - set(actual) |
| 208 | + extra = set(actual) - set(expected) |
| 209 | + assert not missing, f"checked-in .java is missing schemas: {sorted(missing)}\n{_REGEN_HINT}" |
| 210 | + assert not extra, f"checked-in .java has stale schemas no longer in the Protocol: {sorted(extra)}\n{_REGEN_HINT}" |
| 211 | + |
| 212 | + for name, expected_schema in expected.items(): |
| 213 | + if not expected_schema.equals(actual[name], check_metadata=False): |
| 214 | + raise AssertionError( |
| 215 | + f"schema '{name}' in checked-in .java differs from generator output.\n" |
| 216 | + f" expected: {expected_schema}\n" |
| 217 | + f" actual: {actual[name]}\n" |
| 218 | + f"{_REGEN_HINT}", |
| 219 | + ) |
| 220 | + |
| 221 | + |
| 222 | +def test_parser_roundtrip_self_test() -> None: |
| 223 | + """Self-test: the local parser round-trips the generator's own output.""" |
| 224 | + buf = io.StringIO() |
| 225 | + emit(buf) |
| 226 | + parsed = _parse_generated_java(buf.getvalue()) |
| 227 | + expected = _expected() |
| 228 | + assert set(parsed) == set(expected), "parser missed a schema the generator emitted" |
| 229 | + for name, schema in expected.items(): |
| 230 | + assert schema.equals(parsed[name], check_metadata=False), ( |
| 231 | + f"parser round-trip broke schema '{name}': expected {schema}, got {parsed[name]}" |
| 232 | + ) |
0 commit comments