|
| 1 | +# Copyright 2025, 2026 Query Farm LLC - https://query.farm |
| 2 | + |
| 3 | +"""The generated ``vgi.v2`` registries reproduce the reference protocol hash. |
| 4 | +
|
| 5 | +Each SDK's vgi-rpc port derives the wire schemas from the generated interface |
| 6 | +by reflection, so the meaningful check is not "the generator read the model" |
| 7 | +but "what it *rendered* derives back to the reference". These tests parse the |
| 8 | +rendered Java and C#, derive every params / result / header field by that |
| 9 | +port's ``SchemaDerivation`` rules, and assert the resulting |
| 10 | +``vgi_rpc.protocol_hash.v1`` digest equals the live ``VgiProtocol`` hash -- |
| 11 | +computed here, never a pinned constant, so a protocol change moves both sides. |
| 12 | +
|
| 13 | +Drift: the checked-in ``VgiService.java`` / ``IVgiService.g.cs`` must equal |
| 14 | +what the generators emit now (skipped when the sibling checkout is absent; |
| 15 | +``VGI_JAVA_ROOT`` / ``VGI_CSHARP_ROOT`` override the location). |
| 16 | +""" |
| 17 | + |
| 18 | +from __future__ import annotations |
| 19 | + |
| 20 | +import dataclasses |
| 21 | +import os |
| 22 | +import re |
| 23 | +from pathlib import Path |
| 24 | +from typing import Any |
| 25 | + |
| 26 | +import pyarrow as pa |
| 27 | +import pytest |
| 28 | +from vgi_rpc.rpc._protocol_hash import compute_protocol_hash |
| 29 | +from vgi_rpc.rpc._types import MethodType, rpc_methods # type: ignore[attr-defined] |
| 30 | + |
| 31 | +from vgi.codegen import csharp_registry, csharp_types, java_registry |
| 32 | +from vgi.codegen._registry import RegistryMethod, preimage_hash, protocol_name, registry_methods |
| 33 | +from vgi.protocol import VgiProtocol |
| 34 | + |
| 35 | +_REGEN_HINT = ( |
| 36 | + "To regenerate, run:\n" |
| 37 | + " uv run --project ~/Development/vgi-python python scripts/regen_generated.py\n" |
| 38 | + "\n" |
| 39 | + "Do not redirect a generator into its destination with `>`: the shell\n" |
| 40 | + "truncates the file before the generator runs, so any failure destroys\n" |
| 41 | + "the checked-in artifact. The script renders to memory first." |
| 42 | +) |
| 43 | + |
| 44 | +_LIST_UTF8 = pa.list_(pa.field("item", pa.string(), nullable=True)) |
| 45 | +_MAP_UTF8 = pa.map_(pa.string(), pa.string()) |
| 46 | +_DICT_UTF8 = pa.dictionary(pa.int16(), pa.string()) |
| 47 | + |
| 48 | + |
| 49 | +def _live_hash() -> str: |
| 50 | + return str(compute_protocol_hash(protocol_name(), rpc_methods(VgiProtocol))) |
| 51 | + |
| 52 | + |
| 53 | +def _model() -> dict[str, RegistryMethod]: |
| 54 | + return {m.name: m for m in registry_methods()} |
| 55 | + |
| 56 | + |
| 57 | +def _header_class(name: str, names: dict[str, str]) -> type: |
| 58 | + """The Python header dataclass a rendered header type name stands for.""" |
| 59 | + for m in registry_methods(): |
| 60 | + if m.header_type is not None and names.get(m.header_type.__name__, m.header_type.__name__) == name: |
| 61 | + return m.header_type |
| 62 | + raise AssertionError(f"no vgi.v2 header type renders as {name}") |
| 63 | + |
| 64 | + |
| 65 | +def test_model_reproduces_the_live_hash() -> None: |
| 66 | + """The language-neutral model is lossless: its preimage is the reference's.""" |
| 67 | + assert preimage_hash(protocol_name(), registry_methods()) == _live_hash() |
| 68 | + |
| 69 | + |
| 70 | +# --------------------------------------------------------------------------- |
| 71 | +# Java: derive back by vgi-rpc-java's SchemaDerivation rules |
| 72 | +# --------------------------------------------------------------------------- |
| 73 | + |
| 74 | +_JAVA_METHOD = re.compile( |
| 75 | + r"((?: @\w+(?:\([^)]*\))?\n)*) default (.+?) (\w+)\(\n" |
| 76 | + r"((?: .+,\n)*) CallContext ctx\) \{\n" |
| 77 | + r' throw notImplemented\("(\w+)"\);\n \}' |
| 78 | +) |
| 79 | +_JAVA_PARAM = re.compile(r"^((?:@\w+(?:\([^)]*\))? )*)(.+) (\w+)$") |
| 80 | +_JAVA_RECORDS = set(java_registry.JAVA_TYPES.values()) | set(java_registry.JAVA_RAW_RESULTS.values()) |
| 81 | + |
| 82 | + |
| 83 | +def _java_type(java_type: str, annotations: str) -> pa.DataType: |
| 84 | + if "@ArrowField(ArrowFieldType.DICT_INT16_UTF8)" in annotations: |
| 85 | + return _DICT_UTF8 |
| 86 | + if java_type in _JAVA_RECORDS or java_type == "byte[]": |
| 87 | + return pa.binary() |
| 88 | + simple: dict[str, pa.DataType] = { |
| 89 | + "String": pa.string(), |
| 90 | + "boolean": pa.bool_(), |
| 91 | + "Boolean": pa.bool_(), |
| 92 | + "long": pa.int64(), |
| 93 | + "Long": pa.int64(), |
| 94 | + "List<String>": _LIST_UTF8, |
| 95 | + "Map<String, String>": _MAP_UTF8, |
| 96 | + } |
| 97 | + return simple[java_type] |
| 98 | + |
| 99 | + |
| 100 | +def _java_derived(text: str | None = None) -> list[RegistryMethod]: |
| 101 | + text = java_registry.render() if text is None else text |
| 102 | + model = _model() |
| 103 | + out: list[RegistryMethod] = [] |
| 104 | + for match in _JAVA_METHOD.finditer(text): |
| 105 | + method_ann, ret, name, params_text, stub_name = match.groups() |
| 106 | + assert stub_name == name, f"{name}'s default body reports {stub_name}" |
| 107 | + base = model[name] |
| 108 | + params = [] |
| 109 | + for line, p in zip(params_text.splitlines(), base.params, strict=True): |
| 110 | + pm = _JAVA_PARAM.match(line.strip().rstrip(",")) |
| 111 | + assert pm is not None, line |
| 112 | + annotations, java_type, pname = pm.groups() |
| 113 | + assert pname == p.name |
| 114 | + field = pa.field(pname, _java_type(java_type, annotations), nullable="@Nullable" in annotations) |
| 115 | + params.append(dataclasses.replace(p, field=field)) |
| 116 | + is_stream = ret.startswith("RpcStream<") |
| 117 | + result = None |
| 118 | + if not is_stream and ret != "void": |
| 119 | + result = pa.field("result", _java_type(ret, ""), nullable="@Nullable" in method_ann) |
| 120 | + header_match = re.search(r"@StreamHeader\((\w+)\.class\)", method_ann) |
| 121 | + header = _header_class(header_match.group(1), java_registry.JAVA_TYPES) if header_match else None |
| 122 | + out.append( |
| 123 | + dataclasses.replace( |
| 124 | + base, |
| 125 | + method_type=MethodType.STREAM if is_stream else MethodType.UNARY, |
| 126 | + params=tuple(params), |
| 127 | + result_field=result, |
| 128 | + header_type=header, |
| 129 | + ) |
| 130 | + ) |
| 131 | + return out |
| 132 | + |
| 133 | + |
| 134 | +def test_java_registry_derives_the_live_hash() -> None: |
| 135 | + """The rendered VgiService, read back as vgi-rpc-java reads it, is the reference vgi.v2.""" |
| 136 | + derived = _java_derived() |
| 137 | + assert sorted(m.name for m in derived) == sorted(_model()), "rendered method set differs from vgi.v2" |
| 138 | + assert preimage_hash(protocol_name(), derived) == _live_hash() |
| 139 | + |
| 140 | + |
| 141 | +# --------------------------------------------------------------------------- |
| 142 | +# C#: derive back by vgi-rpc-csharp's SchemaDerivation rules |
| 143 | +# --------------------------------------------------------------------------- |
| 144 | + |
| 145 | +_CS_METHOD = re.compile( |
| 146 | + r"((?: \[.+\]\n)*) (Task(?:<.+>)?) (\w+)Async\(\n((?: .+,\n)*) ICallContext\? ctx = null\) =>\n" |
| 147 | + r' throw UnimplementedMethod\.For\("(\w+)"\);' |
| 148 | +) |
| 149 | +_CS_PARAM = re.compile(r"^((?:\[[^\]]+\] )*)(.+?) (\w+)(?: = .+)?$") |
| 150 | + |
| 151 | + |
| 152 | +def _cs_type(clr: str, enums: set[str], records: set[str]) -> tuple[pa.DataType, bool]: |
| 153 | + nullable = clr.endswith("?") |
| 154 | + clr = clr.rstrip("?") |
| 155 | + if clr in enums: |
| 156 | + return _DICT_UTF8, nullable |
| 157 | + if clr in records or clr == "byte[]": |
| 158 | + return pa.binary(), nullable |
| 159 | + simple: dict[str, pa.DataType] = { |
| 160 | + "string": pa.string(), |
| 161 | + "bool": pa.bool_(), |
| 162 | + "long": pa.int64(), |
| 163 | + "List<string>": _LIST_UTF8, |
| 164 | + "Dictionary<string, string>": _MAP_UTF8, |
| 165 | + } |
| 166 | + return simple[clr], nullable |
| 167 | + |
| 168 | + |
| 169 | +def _cs_derived() -> list[RegistryMethod]: |
| 170 | + text = csharp_registry.render() |
| 171 | + cs_model = csharp_types.build_model() |
| 172 | + enums, records = set(cs_model.enums), set(cs_model.records) |
| 173 | + model = _model() |
| 174 | + by_pascal = {csharp_types._pascal(name): name for name in model} |
| 175 | + out: list[RegistryMethod] = [] |
| 176 | + for match in _CS_METHOD.finditer(text): |
| 177 | + attrs, ret, pascal, params_text, stub_name = match.groups() |
| 178 | + name = by_pascal[pascal] |
| 179 | + assert csharp_types._to_snake_case(pascal) == name, f"{pascal}Async does not map to {name}" |
| 180 | + assert stub_name == name, f"{name}'s default body reports {stub_name}" |
| 181 | + base = model[name] |
| 182 | + params = [] |
| 183 | + for line, p in zip(params_text.splitlines(), base.params, strict=True): |
| 184 | + pm = _CS_PARAM.match(line.strip().rstrip(",")) |
| 185 | + assert pm is not None, line |
| 186 | + _, clr, pname = pm.groups() |
| 187 | + assert csharp_types._to_snake_case(pname) == p.name |
| 188 | + dtype, nullable = _cs_type(clr, enums, records) |
| 189 | + params.append(dataclasses.replace(p, field=pa.field(p.name, dtype, nullable=nullable))) |
| 190 | + inner = ret[len("Task<") : -1] if ret.startswith("Task<") else None |
| 191 | + is_stream = inner is not None and inner.startswith("RpcStream<") |
| 192 | + result = None |
| 193 | + if inner is not None and not is_stream: |
| 194 | + dtype, nullable = _cs_type(inner, enums, records) |
| 195 | + result = pa.field("result", dtype, nullable=nullable) |
| 196 | + header_match = re.search(r"\[StreamHeader\(typeof\((\w+)\)\)\]", attrs) |
| 197 | + header = _header_class(header_match.group(1), csharp_types.CSHARP_NAMES) if header_match else None |
| 198 | + out.append( |
| 199 | + dataclasses.replace( |
| 200 | + base, |
| 201 | + method_type=MethodType.STREAM if is_stream else MethodType.UNARY, |
| 202 | + params=tuple(params), |
| 203 | + result_field=result, |
| 204 | + header_type=header, |
| 205 | + ) |
| 206 | + ) |
| 207 | + return out |
| 208 | + |
| 209 | + |
| 210 | +def test_csharp_registry_derives_the_live_hash() -> None: |
| 211 | + """The rendered IVgiService, read back as vgi-rpc-csharp reads it, is the reference vgi.v2.""" |
| 212 | + derived = _cs_derived() |
| 213 | + assert sorted(m.name for m in derived) == sorted(_model()), "rendered method set differs from vgi.v2" |
| 214 | + assert preimage_hash(protocol_name(), derived) == _live_hash() |
| 215 | + |
| 216 | + |
| 217 | +# --------------------------------------------------------------------------- |
| 218 | +# A changed protocol changes the rendering (the test above is not vacuous) |
| 219 | +# --------------------------------------------------------------------------- |
| 220 | + |
| 221 | + |
| 222 | +def test_a_flipped_nullability_changes_the_derived_hash() -> None: |
| 223 | + """Derivation reads the rendered text: one dropped ``@Nullable`` moves the digest.""" |
| 224 | + text = java_registry.render() |
| 225 | + start = text.index(" default ItemsResponse catalog_schemas(") |
| 226 | + end = text.index("CallContext ctx", start) |
| 227 | + target = "@Nullable byte[] transaction_opaque_data" |
| 228 | + assert target in text[start:end] |
| 229 | + tampered = text[:start] + text[start:end].replace(target, "byte[] transaction_opaque_data") + text[end:] |
| 230 | + assert preimage_hash(protocol_name(), _java_derived(tampered)) != _live_hash() |
| 231 | + |
| 232 | + |
| 233 | +# --------------------------------------------------------------------------- |
| 234 | +# Drift |
| 235 | +# --------------------------------------------------------------------------- |
| 236 | + |
| 237 | + |
| 238 | +def _root(env: str, repo: str, marker: str) -> Path: |
| 239 | + override = os.environ.get(env) |
| 240 | + if override: |
| 241 | + return Path(override) |
| 242 | + candidates = [Path(__file__).resolve().parents[2] / repo, Path.home() / repo] |
| 243 | + for candidate in candidates: |
| 244 | + if (candidate / marker).parent.is_dir(): |
| 245 | + return candidate |
| 246 | + return candidates[0] |
| 247 | + |
| 248 | + |
| 249 | +_TARGETS: list[tuple[Any, str, str, str]] = [ |
| 250 | + (java_registry, "VGI_JAVA_ROOT", "vgi-java", java_registry.TARGET), |
| 251 | + (csharp_registry, "VGI_CSHARP_ROOT", "vgi-csharp", csharp_registry.TARGET), |
| 252 | +] |
| 253 | + |
| 254 | + |
| 255 | +@pytest.mark.parametrize(("module", "env", "repo", "relative"), _TARGETS, ids=["java", "csharp"]) |
| 256 | +def test_generator_is_deterministic(module: Any, env: str, repo: str, relative: str) -> None: |
| 257 | + """Rendering twice produces byte-identical output.""" |
| 258 | + assert module.render() == module.render() |
| 259 | + |
| 260 | + |
| 261 | +@pytest.mark.parametrize(("module", "env", "repo", "relative"), _TARGETS, ids=["java", "csharp"]) |
| 262 | +def test_checked_in_registry_matches_generator(module: Any, env: str, repo: str, relative: str) -> None: |
| 263 | + """Drift check: the checked-in registry matches what the generator produces.""" |
| 264 | + path = _root(env, repo, relative) / relative |
| 265 | + if not path.parent.is_dir(): |
| 266 | + pytest.skip(f"{path.parent} not found; set {env} or check out {repo}") |
| 267 | + assert path.exists(), f"{path} is missing.\n{_REGEN_HINT}" |
| 268 | + assert path.read_text() == module.render(), f"{path} is stale.\n{_REGEN_HINT}" |
0 commit comments