|
| 1 | +# Copyright 2025, 2026 Query Farm LLC - https://query.farm |
| 2 | + |
| 3 | +"""The registry generator's backend contract: what one language supplies, and what it gets. |
| 4 | +
|
| 5 | +See :mod:`vgi.codegen._registry` for the design. A backend is a |
| 6 | +:class:`RegistryBackend` subclass with two methods -- ``render_body`` (model -> |
| 7 | +the language's file) and ``derive`` (that file -> :class:`DerivedMethod` |
| 8 | +values, by the port's own derivation rules) -- plus a few class attributes |
| 9 | +naming its target and conventions. Everything else is here, once: |
| 10 | +
|
| 11 | +- rendering: the provenance banner, ``emit`` / ``render`` / ``main``; |
| 12 | +- :meth:`RegistryBackend.rows` -- the registration rows an explicit-registration |
| 13 | + port walks (params / result / header fields), with the port's |
| 14 | + :class:`VoidResult` convention applied; |
| 15 | +- derive-back: :meth:`RegistryBackend.derived_methods` rebuilds what a backend |
| 16 | + parsed into :class:`~vgi.codegen._registry.RegistryMethod` values (checking |
| 17 | + the method set and parameter names against the model) so the caller can hash |
| 18 | + them; :class:`ArrowDialect` evaluates a rendered Arrow schema expression; |
| 19 | + :class:`Tamper` is the one-edit mutation that proves the derivation reads the |
| 20 | + rendered text; |
| 21 | +- :data:`REGISTRY_BACKENDS` / :func:`registry_backends` -- the list the regen |
| 22 | + script and the tests iterate. |
| 23 | +""" |
| 24 | + |
| 25 | +from __future__ import annotations |
| 26 | + |
| 27 | +import abc |
| 28 | +import dataclasses |
| 29 | +import enum |
| 30 | +import importlib |
| 31 | +import io |
| 32 | +import re |
| 33 | +import sys |
| 34 | +from collections.abc import Mapping, Sequence |
| 35 | +from dataclasses import dataclass, field |
| 36 | +from typing import TYPE_CHECKING, Any, ClassVar |
| 37 | + |
| 38 | +import pyarrow as pa |
| 39 | +from vgi_rpc.rpc._types import MethodType # type: ignore[attr-defined] |
| 40 | + |
| 41 | +from vgi.codegen._common import GeneratorError, provenance_comment |
| 42 | +from vgi.codegen._registry import MethodKind, RegistryMethod, registry_methods |
| 43 | + |
| 44 | +if TYPE_CHECKING: |
| 45 | + from typing import TextIO |
| 46 | + |
| 47 | +#: Every registry backend module, in regen order. Each binds ``BACKEND``. |
| 48 | +REGISTRY_BACKENDS: tuple[str, ...] = ( |
| 49 | + "vgi.codegen.csharp_registry", |
| 50 | + "vgi.codegen.java_registry", |
| 51 | + "vgi.codegen.ts_registry", |
| 52 | + "vgi.codegen.go_registry", |
| 53 | + "vgi.codegen.rust_registry", |
| 54 | + "vgi.codegen.cpp_registry", |
| 55 | +) |
| 56 | + |
| 57 | +REGEN_LINE = "uv run --project ~/Development/vgi-python python scripts/regen_generated.py" |
| 58 | + |
| 59 | + |
| 60 | +def registry_backends() -> list[RegistryBackend]: |
| 61 | + """Every registry backend, in :data:`REGISTRY_BACKENDS` order.""" |
| 62 | + out = [] |
| 63 | + for name in REGISTRY_BACKENDS: |
| 64 | + backend = getattr(importlib.import_module(name), "BACKEND", None) |
| 65 | + if not isinstance(backend, RegistryBackend): |
| 66 | + raise GeneratorError(f"{name} binds no RegistryBackend as BACKEND") |
| 67 | + out.append(backend) |
| 68 | + return out |
| 69 | + |
| 70 | + |
| 71 | +def expect(condition: object, message: str) -> None: |
| 72 | + """Fail a derive-back with *message* unless *condition* holds.""" |
| 73 | + if not condition: |
| 74 | + raise GeneratorError(message) |
| 75 | + |
| 76 | + |
| 77 | +class VoidResult(enum.Enum): |
| 78 | + """How a port's vgi-rpc spells a unary that returns nothing. |
| 79 | +
|
| 80 | + ``ABSENT``: there is no result slot at all (a ``void`` return type, Go's |
| 81 | + ``UnaryVoid``, a C++ ``Void`` row). ``EMPTY_SCHEMA``: the result schema is |
| 82 | + passed as a value and is empty; vgi-rpc reports ``has_return`` iff it has |
| 83 | + fields (TypeScript, Rust). |
| 84 | + """ |
| 85 | + |
| 86 | + ABSENT = "absent" |
| 87 | + EMPTY_SCHEMA = "empty-schema" |
| 88 | + |
| 89 | + |
| 90 | +@dataclass(frozen=True) |
| 91 | +class RegistrationRow: |
| 92 | + """One method as an explicit-registration table carries it. |
| 93 | +
|
| 94 | + Attributes: |
| 95 | + method: The model method. |
| 96 | + params: The params-schema fields. |
| 97 | + result: The result-schema fields: ``[result]``, ``[]`` for a void unary |
| 98 | + under :attr:`VoidResult.EMPTY_SCHEMA`, ``None`` when the port |
| 99 | + declares no result slot (a stream, or a void unary under ``ABSENT``). |
| 100 | + header: The stream header's fields, or ``None``. |
| 101 | + """ |
| 102 | + |
| 103 | + method: RegistryMethod |
| 104 | + params: list[pa.Field[Any]] |
| 105 | + result: list[pa.Field[Any]] | None |
| 106 | + header: list[pa.Field[Any]] | None |
| 107 | + |
| 108 | + |
| 109 | +@dataclass(frozen=True) |
| 110 | +class DerivedMethod: |
| 111 | + """One method as a port derives it from the rendered file: what the hash needs. |
| 112 | +
|
| 113 | + Attributes: |
| 114 | + name: The wire name the rendered file registers. |
| 115 | + kind: Unary, Void or Stream, as the rendered file declares it. |
| 116 | + params: The params-schema fields, as the port derives them. |
| 117 | + result: The ``result`` field (``Unary`` only). |
| 118 | + header: The stream header schema (``Stream`` only), or ``None``. |
| 119 | + """ |
| 120 | + |
| 121 | + name: str |
| 122 | + kind: MethodKind |
| 123 | + params: Sequence[pa.Field[Any]] |
| 124 | + result: pa.Field[Any] | None = None |
| 125 | + header: pa.Schema | None = None |
| 126 | + |
| 127 | + |
| 128 | +@dataclass(frozen=True) |
| 129 | +class Tamper: |
| 130 | + """A one-edit mutation of a rendered file that must move the derived hash. |
| 131 | +
|
| 132 | + Within the region from the first *start* to the next *end* after it, |
| 133 | + *old* (which must occur there) becomes *new*. |
| 134 | + """ |
| 135 | + |
| 136 | + start: str |
| 137 | + end: str |
| 138 | + old: str |
| 139 | + new: str |
| 140 | + |
| 141 | + def apply(self, text: str) -> str: |
| 142 | + """The tampered text.""" |
| 143 | + start = text.index(self.start) |
| 144 | + end = text.index(self.end, start + len(self.start)) |
| 145 | + region = text[start:end] |
| 146 | + expect(self.old in region, f"tamper target {self.old!r} not in its region") |
| 147 | + return text[:start] + region.replace(self.old, self.new) + text[end:] |
| 148 | + |
| 149 | + |
| 150 | +@dataclass(frozen=True) |
| 151 | +class ArrowDialect: |
| 152 | + """How one language spells Arrow schema expressions, for derive-back. |
| 153 | +
|
| 154 | + The shared evaluator turns a rendered expression into pyarrow values: apply |
| 155 | + *rewrites* (regex, replacement), in order, to reach Python call syntax, then |
| 156 | + evaluate with *names* (plus ``true`` / ``false`` / ``null``) and no |
| 157 | + builtins. A language supplies only those two tables; the rendered text is |
| 158 | + this generator's own output, never input from elsewhere. |
| 159 | + """ |
| 160 | + |
| 161 | + rewrites: tuple[tuple[str, str], ...] |
| 162 | + names: Mapping[str, object] = field(default_factory=dict) |
| 163 | + |
| 164 | + def eval(self, expr: str) -> Any: |
| 165 | + """Evaluate one rendered expression.""" |
| 166 | + py = expr |
| 167 | + for pattern, replacement in self.rewrites: |
| 168 | + py = re.sub(pattern, replacement, py) |
| 169 | + env: dict[str, object] = {"true": True, "false": False, "null": None, **self.names} |
| 170 | + try: |
| 171 | + return eval(py, {"__builtins__": {}}, env) # noqa: S307 - the generator's own output |
| 172 | + except Exception as exc: # noqa: BLE001 - report the expression that failed |
| 173 | + raise GeneratorError(f"cannot evaluate {expr!r} (as {py!r}): {exc}") from exc |
| 174 | + |
| 175 | + def schema(self, expr: str) -> pa.Schema: |
| 176 | + """Evaluate an expression that builds a schema, or a list of fields.""" |
| 177 | + built = self.eval(expr) |
| 178 | + if isinstance(built, list): |
| 179 | + built = pa.schema(built) |
| 180 | + expect(isinstance(built, pa.Schema), f"{expr!r} is not a schema") |
| 181 | + return built # type: ignore[no-any-return] |
| 182 | + |
| 183 | + |
| 184 | +def call_args(text: str, func: str) -> list[str]: |
| 185 | + """The argument text of every ``func(...)`` call in *text*, parentheses balanced. |
| 186 | +
|
| 187 | + *func* matches as a whole word (``schema`` does not match ``header_schema``). |
| 188 | + For the generator's own output, whose string literals hold no parentheses. |
| 189 | + """ |
| 190 | + out = [] |
| 191 | + for m in re.finditer(rf"(?<![\w.]){re.escape(func)}\(", text): |
| 192 | + depth, i = 1, m.end() |
| 193 | + while depth: |
| 194 | + expect(i < len(text), f"unbalanced {func}( call") |
| 195 | + depth += {"(": 1, ")": -1}.get(text[i], 0) |
| 196 | + i += 1 |
| 197 | + out.append(text[m.end() : i - 1]) |
| 198 | + return out |
| 199 | + |
| 200 | + |
| 201 | +def header_holder(name: str, schema: pa.Schema) -> type: |
| 202 | + """A stand-in header type carrying a derived ``ARROW_SCHEMA``, for :func:`preimage`.""" |
| 203 | + return type(f"{name}DerivedHeader", (), {"ARROW_SCHEMA": schema}) |
| 204 | + |
| 205 | + |
| 206 | +def rebuild(derived: Sequence[DerivedMethod]) -> list[RegistryMethod]: |
| 207 | + """Rebuild derived methods into model methods, every hashed field taken from the derivation. |
| 208 | +
|
| 209 | + Checks each against the model: a known, not repeated wire name, the |
| 210 | + model's parameter names in order, and a result / header only where the |
| 211 | + kind allows one. |
| 212 | + """ |
| 213 | + model = {m.name: m for m in registry_methods()} |
| 214 | + out: list[RegistryMethod] = [] |
| 215 | + for d in derived: |
| 216 | + base = model.get(d.name) |
| 217 | + expect(base is not None, f"the rendered file registers {d.name}, which is not a vgi.v2 method") |
| 218 | + assert base is not None |
| 219 | + expect(d.name not in {m.name for m in out}, f"{d.name} is registered twice") |
| 220 | + names = [f.name for f in d.params] |
| 221 | + expect(names == [p.name for p in base.params], f"{d.name}: params derive as {names}") |
| 222 | + if d.kind is MethodKind.UNARY: |
| 223 | + expect(d.result is not None and d.result.name == "result", f"{d.name}: a Unary needs a `result` column") |
| 224 | + else: |
| 225 | + expect(d.result is None, f"{d.name}: a {d.kind.value} derives no result") |
| 226 | + expect(d.header is None or d.kind is MethodKind.STREAM, f"{d.name}: only a stream has a header") |
| 227 | + out.append( |
| 228 | + dataclasses.replace( |
| 229 | + base, |
| 230 | + method_type=MethodType.STREAM if d.kind is MethodKind.STREAM else MethodType.UNARY, |
| 231 | + params=tuple(dataclasses.replace(p, field=f) for p, f in zip(base.params, d.params, strict=True)), |
| 232 | + result_field=d.result, |
| 233 | + header_type=None if d.header is None else header_holder(d.name, d.header), |
| 234 | + ) |
| 235 | + ) |
| 236 | + return out |
| 237 | + |
| 238 | + |
| 239 | +class RegistryBackend(abc.ABC): |
| 240 | + """One language's registry: how it renders the model, and how its port reads the rendering back. |
| 241 | +
|
| 242 | + Class attributes: |
| 243 | + key: Short id (``"java"``), used for test ids. |
| 244 | + language: Display name, used in messages (``"Java"``). |
| 245 | + module: The backend's module, named in the provenance banner. |
| 246 | + target: The generated file, relative to the SDK checkout. |
| 247 | + repo: The SDK checkout's directory name. |
| 248 | + root_env: Environment variable that overrides the checkout location. |
| 249 | + void_result: How the port spells a unary with no return. |
| 250 | + tamper: A one-edit mutation that must move the derived hash. |
| 251 | + banner_head / banner_tail: Text around the provenance comment. |
| 252 | + """ |
| 253 | + |
| 254 | + key: ClassVar[str] |
| 255 | + language: ClassVar[str] |
| 256 | + module: ClassVar[str] |
| 257 | + target: ClassVar[str] |
| 258 | + repo: ClassVar[str] |
| 259 | + root_env: ClassVar[str] |
| 260 | + void_result: ClassVar[VoidResult] = VoidResult.ABSENT |
| 261 | + tamper: ClassVar[Tamper] |
| 262 | + generator_version: ClassVar[str] = "1" |
| 263 | + banner_head: ClassVar[str] = "" |
| 264 | + banner_tail: ClassVar[str] = "" |
| 265 | + |
| 266 | + @abc.abstractmethod |
| 267 | + def render_body(self, methods: Sequence[RegistryMethod]) -> str: |
| 268 | + """The generated file's body (everything after the provenance banner).""" |
| 269 | + |
| 270 | + @abc.abstractmethod |
| 271 | + def derive(self, text: str) -> list[DerivedMethod]: |
| 272 | + """Read a rendered file back, as the language's vgi-rpc port would register it.""" |
| 273 | + |
| 274 | + # -- rows --------------------------------------------------------------- |
| 275 | + |
| 276 | + def row(self, m: RegistryMethod) -> RegistrationRow: |
| 277 | + """One method's registration row, under this port's :attr:`void_result`.""" |
| 278 | + result: list[pa.Field[Any]] | None = None |
| 279 | + if m.result_field is not None: |
| 280 | + result = [m.result_field] |
| 281 | + elif m.kind is MethodKind.VOID and self.void_result is VoidResult.EMPTY_SCHEMA: |
| 282 | + result = [] |
| 283 | + return RegistrationRow(m, m.params_fields, result, m.header_fields) |
| 284 | + |
| 285 | + def unary_from_result(self, name: str, fields: Sequence[pa.Field[Any]]) -> tuple[MethodKind, pa.Field[Any] | None]: |
| 286 | + """Interpret a derived result schema under :attr:`void_result`: ``[]`` is Void, ``[result]`` Unary.""" |
| 287 | + expect(self.void_result is VoidResult.EMPTY_SCHEMA or fields, f"{name}: an empty result schema") |
| 288 | + expect(len(fields) <= 1, f"{name}: a result schema with {len(fields)} fields") |
| 289 | + return (MethodKind.UNARY, fields[0]) if fields else (MethodKind.VOID, None) |
| 290 | + |
| 291 | + # -- rendering ---------------------------------------------------------- |
| 292 | + |
| 293 | + def emit(self, out: TextIO) -> None: |
| 294 | + """Write the generated file to *out*.""" |
| 295 | + body = self.render_body(registry_methods()) |
| 296 | + out.write(self.banner_head) |
| 297 | + out.write( |
| 298 | + provenance_comment( |
| 299 | + generator_module=self.module, |
| 300 | + generator_command=f"python -m {self.module}", |
| 301 | + generator_version=self.generator_version, |
| 302 | + regen_command_lines=[REGEN_LINE], |
| 303 | + body=body, |
| 304 | + ) |
| 305 | + ) |
| 306 | + out.write(self.banner_tail) |
| 307 | + out.write("\n") |
| 308 | + out.write(body) |
| 309 | + |
| 310 | + def render(self) -> str: |
| 311 | + """The full text of the generated file.""" |
| 312 | + buf = io.StringIO() |
| 313 | + self.emit(buf) |
| 314 | + return buf.getvalue() |
| 315 | + |
| 316 | + def main(self) -> None: |
| 317 | + """Console-script entrypoint: write the generated file to stdout.""" |
| 318 | + try: |
| 319 | + self.emit(sys.stdout) |
| 320 | + except GeneratorError as e: |
| 321 | + print(f"\nerror: {e}\n", file=sys.stderr) |
| 322 | + sys.exit(2) |
| 323 | + |
| 324 | + # -- derive-back -------------------------------------------------------- |
| 325 | + |
| 326 | + def derived_methods(self, text: str | None = None) -> list[RegistryMethod]: |
| 327 | + """*text* (default: a fresh rendering) derived back and rebuilt, ready to hash.""" |
| 328 | + return rebuild(self.derive(self.render() if text is None else text)) |
0 commit comments